From a02af039b52c1faadfc4bf9431cc2854bc4f3858 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Tue, 28 May 2013 19:03:53 -0500 Subject: [PATCH 001/163] Initial commit. --- .gitignore | 28 ++++++++++++++++++++++++++++ README.md | 4 ++++ 2 files changed, 32 insertions(+) create mode 100644 .gitignore create mode 100644 README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..5b97dbba --- /dev/null +++ b/.gitignore @@ -0,0 +1,28 @@ +# Temp files +*~ + +# Log files +*.log + +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe diff --git a/README.md b/README.md new file mode 100644 index 00000000..18692b1a --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +btcdb +===== + +Package btcdb provides a database interface for the bitcoin blockchain. From 752ca5dfbb81c13e4f836d65c46e956fe9582d5c Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Tue, 28 May 2013 19:07:21 -0500 Subject: [PATCH 002/163] Initial implementation. --- LICENSE | 13 + README.md | 58 ++- db.go | 176 ++++++++ doc.go | 56 +++ log.go | 56 +++ sqlite3/doc.go | 15 + sqlite3/internal_test.go | 48 +++ sqlite3/operational_test.go | 339 ++++++++++++++++ sqlite3/sqlite.go | 673 +++++++++++++++++++++++++++++++ sqlite3/sqliteblock.go | 312 ++++++++++++++ sqlite3/sqliteblock_test.go | 302 ++++++++++++++ sqlite3/sqlitedbcache.go | 261 ++++++++++++ sqlite3/sqlitetx.go | 253 ++++++++++++ sqlite3/testdata/blocks1-256.bz2 | Bin 0 -> 37555 bytes 14 files changed, 2561 insertions(+), 1 deletion(-) create mode 100644 LICENSE create mode 100644 db.go create mode 100644 doc.go create mode 100644 log.go create mode 100644 sqlite3/doc.go create mode 100644 sqlite3/internal_test.go create mode 100644 sqlite3/operational_test.go create mode 100644 sqlite3/sqlite.go create mode 100644 sqlite3/sqliteblock.go create mode 100644 sqlite3/sqliteblock_test.go create mode 100644 sqlite3/sqlitedbcache.go create mode 100644 sqlite3/sqlitetx.go create mode 100644 sqlite3/testdata/blocks1-256.bz2 diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..0d760cbb --- /dev/null +++ b/LICENSE @@ -0,0 +1,13 @@ +Copyright (c) 2013 Conformal Systems LLC. + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index 18692b1a..ac09b421 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,60 @@ btcdb ===== -Package btcdb provides a database interface for the bitcoin blockchain. +Package btcdb provides a database interface for the bitcoin block chain and +transactions. There is a test suite which is aiming to reach 100% code coverage +coverage. See `test_coverage.txt` for the current coverage (using gocov). On a +UNIX-like OS, the script `cov_report.sh` can be used to generate the report. +Package btcjson is licensed under the liberal ISC license. + +## Sample Use + +```Go + db, err := btcdb.CreateDB("sqlite", "dbexample") + newHeight, err := db.InsertBlock(block) + db.Sync() +``` + +## Documentation + +Full `go doc` style documentation for the project can be viewed online without +installing this package by using the GoDoc site +[here](http://godoc.org/github.com/conformal/btcdb). + +You can also view the documentation locally once the package is installed with +the `godoc` tool by running `godoc -http=":6060"` and pointing your browser to +http://localhost:6060/pkg/github.com/conformal/btcdb + +## Installation + +```bash +$ go get github.com/conformal/btcdb +``` + +## TODO +- Increase test coverage to 100% +- Allow other database backends + +## GPG Verification Key + +All official release tags are signed by Conformal so users can ensure the code +has not been tampered with and is coming from Conformal. To verify the +signature perform the following: + +- Download the public key from the Conformal website at + https://opensource.conformal.com/GIT-GPG-KEY-conformal.txt + +- Import the public key into your GPG keyring: + ```bash + gpg --import GIT-GPG-KEY-conformal.txt + ``` + +- Verify the release tag with the following command where `TAG_NAME` is a + placeholder for the specific tag: + ```bash + git tag -v TAG_NAME + ``` + +## License + +Package btcdb is licensed under the liberal ISC License. diff --git a/db.go b/db.go new file mode 100644 index 00000000..226f3488 --- /dev/null +++ b/db.go @@ -0,0 +1,176 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package btcdb + +import ( + "errors" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" +) + +var ( + PrevShaMissing = errors.New("Previous sha missing from database") + TxShaMissing = errors.New("Requested Tx does not exist") + DuplicateSha = errors.New("Duplicate insert attempted") + DbDoesNotExist = errors.New("Non-existant database") + DbUnknownType = errors.New("Non-existant database type") +) + +// AllShas is a special value that can be used as the final sha when requesting +// a range of shas by height to request them all. +const AllShas = int64(^uint64(0) >> 1) + +// InsertMode represents a hint to the database about how much data the +// application is expecting to send to the database in a short period of time. +// This in turn provides the database with the opportunity to work in optimized +// modes when it will be very busy such as during the initial block chain +// download. +type InsertMode int + +// Constants used to indicate the database insert mode hint. See InsertMode. +const ( + InsertNormal InsertMode = iota + InsertFast + InsertValidatedInput +) + +type Db interface { + // Close cleanly shuts down the database and syncs all data. + Close() + + // DropAfterBlockBySha will remove any blocks from the database after + // the given block. It terminates any existing transaction and performs + // its operations in an atomic transaction which is commited before + // the function returns. + DropAfterBlockBySha(btcwire.ShaHash) (err error) + + // ExistsSha returns whether or not the given block hash is present in + // the database. + ExistsSha(sha *btcwire.ShaHash) (exists bool) + + // FetchBlockBySha returns a btcutil Block. The implementation may + // cache the underlying object if desired. + FetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) + + // FetchBlockShaByIdx returns a block sha based on its height in the + // blockchain. + FetchBlockShaByIdx(blkid int64) (sha *btcwire.ShaHash, err error) + + // FetchIdxRange looks up a range of block by the start and ending ids. + // Fetch is inclusive of the start id and exclusive of the ending id. If + // the special id `AllShas' is provided as endid then FetchIdxRange will + // fetch all shas from startid until no more shas are present. + FetchIdxRange(startid, endid int64) (rshalist []btcwire.ShaHash, err error) + + // FetchTxAllBySha returns several pieces of data regarding the given sha. + FetchTxAllBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rtxbuf []byte, rpver uint32, rblksha *btcwire.ShaHash, err error) + + // FetchTxBufBySha returns the raw bytes and associated protocol version + // for the transaction with the requested sha. + FetchTxBufBySha(txsha *btcwire.ShaHash) (txbuf []byte, rpver uint32, err error) + + // FetchTxBySha returns some data for the given Tx Sha. + FetchTxBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rpver uint32, blksha *btcwire.ShaHash, err error) + + // FetchTxByShaList returns a TxListReply given an array of ShaHash, look up the transactions + // and return them in a TxListReply array. + FetchTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply + + // FetchTxUsedBySha returns the used/spent buffer for a given transaction. + FetchTxUsedBySha(txsha *btcwire.ShaHash) (spentbuf []byte, err error) + + // InsertBlock inserts the block data and transaction data from a block + // into the database. + InsertBlock(block *btcutil.Block) (blockid int64, err error) + + // InsertTx inserts a tx hash and its associated data into the database + InsertTx(txsha *btcwire.ShaHash, blockidx int64, txoff int, txlen int, usedbuf []byte) (err error) + + // InvalidateBlockCache releases all cached blocks. + InvalidateBlockCache() + + // InvalidateCache releases all cached blocks and transactions. + InvalidateCache() + + // InvalidateTxCache releases all cached transactions. + InvalidateTxCache() + + // NewIterateBlocks returns an iterator for all blocks in database. + NewIterateBlocks() (pbi BlockIterator, err error) + + // NewestSha provides an interface to quickly look up the sha of + // the most recent (end) of the block chain. + NewestSha() (sha *btcwire.ShaHash, blkid int64, err error) + + // RollbackClose discards the recent database changes to the previously + // saved data at last Sync and closes the database. + RollbackClose() + + // SetDBInsertMode provides hints to the database to how the application + // is running. This allows the database to work in optimized modes when + // the database may be very busy. + SetDBInsertMode(InsertMode) + + // Sync verifies that the database is coherent on disk and no + // outstanding transactions are in flight. + Sync() +} + +type BlockIterator interface { + // Close shuts down the iterator when done walking blocks in the database. + Close() + + // NextRow iterates thru all blocks in database. + NextRow() bool + // Row returns row data for block iterator. + Row() (key *btcwire.ShaHash, pver uint32, buf []byte, err error) +} + +type DriverDB struct { + DbType string + Create func(argstr string) (pbdb Db, err error) + Open func(filepath string) (pbdb Db, err error) +} + +type TxListReply struct { + Sha *btcwire.ShaHash + Tx *btcwire.MsgTx + Err error +} + +// driverList holds all of the registered database backends. +var driverList []DriverDB + +// AddDBDriver adds a back end database driver to available interfaces. +func AddDBDriver(instance DriverDB) { + // TODO(drahn) Does this really need to check for duplicate names ? + for _, drv := range driverList { + // TODO(drahn) should duplicates be an error? + if drv.DbType == instance.DbType { + return + } + } + driverList = append(driverList, instance) +} + +// CreateDB intializes and opens a database. +func CreateDB(dbtype string, argstr string) (pbdb Db, err error) { + for _, drv := range driverList { + if drv.DbType == dbtype { + return drv.Create(argstr) + } + } + return nil, DbUnknownType +} + +// OpenDB opens an existing database. +func OpenDB(dbtype string, argstr string) (pbdb Db, err error) { + for _, drv := range driverList { + if drv.DbType == dbtype { + return drv.Open(argstr) + } + } + return nil, DbUnknownType +} diff --git a/doc.go b/doc.go new file mode 100644 index 00000000..dc96fbf0 --- /dev/null +++ b/doc.go @@ -0,0 +1,56 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +/* +Package btcdb provides a database interface for the bitcoin block chain. + +As of May 2013, there are over 235,000 blocks in the bitcoin block chain and +and over 17 million transactions (which turns out to be over 11Gb of data). +btcdb provides a database layer to store and retrieve this data in a fairly +simple and efficient manner. The use of this should not require specific +knowledge of the database backend used although currently only db_sqlite is +provided. + +Basic Design + +The basic design of btcdb is to provide two classes of items in a +database; blocks and transactions (tx) where the block number +increases monotonically. Each transaction belongs to a single block +although a block can have a variable number of transactions. Along +with these two items, several convenience functions for dealing with +the database are provided as well as functions to query specific items +that may be present in a block or tx (although many of these are in +the db_sqlite subpackage). + +Usage + +At the highest level, the use of this packages just requires that you +import it, setup a database, insert some data into it, and optionally, +query the data back. In a more concrete example: + + // Import packages + import ( + "github.com/conformal/btcdb" + _ "github.com/conformal/btcdb/db_sqlite" + ) + + // Create a database + dbname := "dbexample" + db, err := btcdb.CreateDB("sqlite", dbname) + if err != nil { + fmt.Printf("Failed to open database %v", err) + return + } + + // Insert a block + newheight, err := db.InsertBlock(block) + if err != nil { + fmt.Printf("failed to insert block %v err %v", height, err) + } + + // Sync the database + db.Sync() + +*/ +package btcdb diff --git a/log.go b/log.go new file mode 100644 index 00000000..7701a92e --- /dev/null +++ b/log.go @@ -0,0 +1,56 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package btcdb + +import ( + "errors" + "github.com/conformal/seelog" + "io" +) + +// log is a logger that is initialized with no output filters. This +// means the package will not perform any logging by default until the caller +// requests it. +var log seelog.LoggerInterface + +// The default amount of logging is none. +func init() { + DisableLog() +} + +// DisableLog disables all library log output. Logging output is disabled +// by default until either UserLogger or SetLogWriter are called. +func DisableLog() { + log = seelog.Disabled +} + +// UseLogger uses a specified Logger to output package logging info. +// This should be used in preference to SetLogWriter if the caller is also +// using seelog. +func UseLogger(logger seelog.LoggerInterface) { + log = logger +} + +// SetLogWriter uses a specified io.Writer to output package logging info. +// This allows a caller to direct package logging output without needing a +// dependency on seelog. If the caller is also using seelog, UseLogger should +// be used instead. +func SetLogWriter(w io.Writer) error { + if w == nil { + return errors.New("nil writer") + } + + l, err := seelog.LoggerFromWriterWithMinLevel(w, seelog.TraceLvl) + if err != nil { + return err + } + + UseLogger(l) + return nil +} + +func GetLog() seelog.LoggerInterface { + return log +} diff --git a/sqlite3/doc.go b/sqlite3/doc.go new file mode 100644 index 00000000..f9ade3b6 --- /dev/null +++ b/sqlite3/doc.go @@ -0,0 +1,15 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +/* +Package sqlite3 implements a sqlite3 instance of btcdb. + +sqlite provides a zero setup, single file database. It requires cgo +and the presence of the sqlite library and headers, but nothing else. +The performance is generally high although it goes down with database +size. + +Many of the block or tx specific functions for btcdb are in this subpackage. +*/ +package sqlite3 diff --git a/sqlite3/internal_test.go b/sqlite3/internal_test.go new file mode 100644 index 00000000..90cb425d --- /dev/null +++ b/sqlite3/internal_test.go @@ -0,0 +1,48 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package sqlite3 + +import ( + "fmt" + "github.com/conformal/btcdb" + "github.com/conformal/btcwire" +) + +// FetchSha returns the datablock and pver for the given ShaHash. +// This is a testing only interface. +func FetchSha(db btcdb.Db, sha *btcwire.ShaHash) (buf []byte, pver uint32, + blkid int64, err error) { + sqldb, ok := db.(*SqliteDb) + if !ok { + err = fmt.Errorf("Invalid data type") + return + } + buf, pver, blkid, err = sqldb.fetchSha(*sha) + return +} + +// SetBlockCacheSize configures the maximum number of blocks in the cache to +// be the given size should be made before any fetching. +// This is a testing only interface. +func SetBlockCacheSize(db btcdb.Db, newsize int) { + sqldb, ok := db.(*SqliteDb) + if !ok { + return + } + bc := &sqldb.blockCache + bc.maxcount = newsize +} + +// SetTxCacheSize configures the maximum number of tx in the cache to +// be the given size should be made before any fetching. +// This is a testing only interface. +func SetTxCacheSize(db btcdb.Db, newsize int) { + sqldb, ok := db.(*SqliteDb) + if !ok { + return + } + tc := &sqldb.txCache + tc.maxcount = newsize +} diff --git a/sqlite3/operational_test.go b/sqlite3/operational_test.go new file mode 100644 index 00000000..81106cf7 --- /dev/null +++ b/sqlite3/operational_test.go @@ -0,0 +1,339 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package sqlite3_test + +import ( + "compress/bzip2" + "encoding/binary" + "github.com/confomral/btcdb" + "github.com/confomral/btcdb/db_sqlite" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +var network = btcwire.MainNet + +const ( + dbTmDefault = iota + dbTmNormal + dbTmFast + dbTmNoVerify +) + +func TestOperational(t *testing.T) { + testOperationalMode(t, dbTmDefault) + testOperationalMode(t, dbTmNormal) + testOperationalMode(t, dbTmFast) + testOperationalMode(t, dbTmNoVerify) +} + +func testOperationalMode(t *testing.T, mode int) { + // simplified basic operation is: + // 1) fetch block from remote server + // 2) look up all txin (except coinbase in db) + // 3) insert block + + // Ignore db remove errors since it means we didn't have an old one. + dbname := "tstdbop1" + _ = os.Remove(dbname) + db, err := btcdb.CreateDB("sqlite", dbname) + if err != nil { + t.Errorf("Failed to open test database %v", err) + return + } + defer os.Remove(dbname) + + switch mode { + case dbTmDefault: // default + // no setup + case dbTmNormal: // explicit normal + db.SetDBInsertMode(btcdb.InsertNormal) + case dbTmFast: // fast mode + db.SetDBInsertMode(btcdb.InsertFast) + if sqldb, ok := db.(*sqlite3.SqliteDb); ok { + sqldb.TempTblMax = 100 + } else { + t.Errorf("not right type") + } + case dbTmNoVerify: // validated block + db.SetDBInsertMode(btcdb.InsertValidatedInput) + } + + // Since we are dealing with small dataset, reduce cache size + sqlite3.SetBlockCacheSize(db, 2) + sqlite3.SetTxCacheSize(db, 3) + + testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + blocks, err := loadBlocks(t, testdatafile) + + var height = int64(1) + err = nil + for ; height < int64(len(blocks)); height++ { + + block := blocks[height] + + if mode != dbTmNoVerify { + // except for NoVerify which does not allow lookups check inputs + mblock := block.MsgBlock() + var txneededList []*btcwire.ShaHash + for _, tx := range mblock.Transactions { + for _, txin := range tx.TxIn { + if txin.PreviousOutpoint.Index == uint32(4294967295) { + continue + } + origintxsha := &txin.PreviousOutpoint.Hash + txneededList = append(txneededList, origintxsha) + _, _, _, _, err := db.FetchTxAllBySha(origintxsha) + if err != nil { + t.Errorf("referenced tx not found %v err %v ", origintxsha, err) + } + _, _, _, _, err = db.FetchTxAllBySha(origintxsha) + if err != nil { + t.Errorf("referenced tx not found %v err %v ", origintxsha, err) + } + _, _, _, err = db.FetchTxBySha(origintxsha) + if err != nil { + t.Errorf("referenced tx not found %v err %v ", origintxsha, err) + } + _, _, err = db.FetchTxBufBySha(origintxsha) + if err != nil { + t.Errorf("referenced tx not found %v err %v ", origintxsha, err) + } + _, err = db.FetchTxUsedBySha(origintxsha) + if err != nil { + t.Errorf("tx used fetch fail %v err %v ", origintxsha, err) + } + } + } + txlist := db.FetchTxByShaList(txneededList) + for _, txe := range txlist { + if txe.Err != nil { + t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) + } + } + + } + + t.Logf("Inserting Block %v", height) + newheight, err := db.InsertBlock(block) + if err != nil { + t.Errorf("failed to insert block %v err %v", height, err) + } + if newheight != height { + t.Errorf("height mismatch expect %v returned %v", height, newheight) + + } + } + + switch mode { + case dbTmDefault: // default + // no cleanup + case dbTmNormal: // explicit normal + // no cleanup + case dbTmFast: // fast mode + db.SetDBInsertMode(btcdb.InsertNormal) + case dbTmNoVerify: // validated block + db.SetDBInsertMode(btcdb.InsertNormal) + } +} + +func TestBackout(t *testing.T) { + testBackout(t, dbTmDefault) + testBackout(t, dbTmNormal) + testBackout(t, dbTmFast) +} + +func testBackout(t *testing.T, mode int) { + // simplified basic operation is: + // 1) fetch block from remote server + // 2) look up all txin (except coinbase in db) + // 3) insert block + + // Ignore db remove errors since it means we didn't have an old one. + dbname := "tstdbop2" + _ = os.Remove(dbname) + db, err := btcdb.CreateDB("sqlite", dbname) + if err != nil { + t.Errorf("Failed to open test database %v", err) + return + } + defer os.Remove(dbname) + + switch mode { + case dbTmDefault: // default + // no setup + case dbTmNormal: // explicit normal + db.SetDBInsertMode(btcdb.InsertNormal) + case dbTmFast: // fast mode + db.SetDBInsertMode(btcdb.InsertFast) + if sqldb, ok := db.(*sqlite3.SqliteDb); ok { + sqldb.TempTblMax = 100 + } else { + t.Errorf("not right type") + } + } + + // Since we are dealing with small dataset, reduce cache size + sqlite3.SetBlockCacheSize(db, 2) + sqlite3.SetTxCacheSize(db, 3) + + testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + blocks, err := loadBlocks(t, testdatafile) + + if len(blocks) < 120 { + t.Errorf("test data too small") + return + } + + var height = int64(1) + err = nil + for ; height < int64(len(blocks)); height++ { + + if height == 100 { + t.Logf("sync") + db.Sync() + } + if height == 120 { + t.Logf("wha?") + // Simulate unexpected application quit + db.RollbackClose() + break + } + + block := blocks[height] + + t.Logf("Inserting Block %v", height) + newheight, err := db.InsertBlock(block) + if err != nil { + t.Errorf("failed to insert block %v err %v", height, err) + } + if newheight != height { + t.Errorf("height mismatch expect %v returned %v", height, newheight) + + } + } + + // db was closed at height 120, so no cleanup is possible. + + // reopen db + db, err = btcdb.NewDB("sqlite", dbname) + if err != nil { + t.Errorf("Failed to open test database %v", err) + return + } + + sha, err := blocks[99].Sha() + if err != nil { + t.Errorf("failed to get block 99 sha err %v", err) + return + } + _ = db.ExistsSha(sha) + _, err = db.FetchBlockBySha(sha) + if err != nil { + t.Errorf("failed to load block 99 from db", err) + } + + sha, err = blocks[110].Sha() + if err != nil { + t.Errorf("failed to get block 110 sha err %v", err) + return + } + _ = db.ExistsSha(sha) + _, err = db.FetchBlockBySha(sha) + if err == nil { + t.Errorf("loaded block 110 from db, failure expected") + } + + block := blocks[110] + mblock := block.MsgBlock() + txsha, err := mblock.Transactions[0].TxSha(block.ProtocolVersion()) + t.Logf("txsha %v", txsha) + _, _, _, err = db.FetchTxBySha(&txsha) + _, err = db.FetchTxUsedBySha(&txsha) + + block = blocks[99] + mblock = block.MsgBlock() + txsha, err = mblock.Transactions[0].TxSha(block.ProtocolVersion()) + oldused, err := db.FetchTxUsedBySha(&txsha) + err = db.InsertTx(&txsha, 99, 1024, 1048, oldused) + if err == nil { + t.Errorf("dup insert of tx succeeded") + } +} + +func loadBlocks(t *testing.T, file string) (blocks []*btcutil.Block, err error) { + testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + var dr io.Reader + var fi io.ReadCloser + fi, err = os.Open(testdatafile) + if err != nil { + t.Errorf("failed to open file %v, err %v", testdatafile, err) + return + } + if strings.HasSuffix(testdatafile, ".bz2") { + z := bzip2.NewReader(fi) + dr = z + } else { + dr = fi + } + + defer func() { + if err := fi.Close(); err != nil { + t.Errorf("failed to close file %v %v", testdatafile, err) + } + }() + + var block *btcutil.Block + // block 0 isn't really there, put in nil + blocks = append(blocks, block) + + var height = int64(1) + err = nil + for ; err == nil; height++ { + var rintbuf uint32 + err = binary.Read(dr, binary.LittleEndian, &rintbuf) + if err == io.EOF { + // hit end of file at expected offset: no warning + height-- + break + } + if err != nil { + t.Errorf("failed to load network type, err %v", err) + break + } + if rintbuf != uint32(network) { + t.Errorf("Block doesn't match network: %v expects %v", + rintbuf, network) + break + } + err = binary.Read(dr, binary.LittleEndian, &rintbuf) + blocklen := rintbuf + + rbytes := make([]byte, blocklen) + + // read block + dr.Read(rbytes) + + var pver uint32 + switch { + case height < 200000: + pver = 1 + case height >= 200000: + pver = 2 + } + block, err = btcutil.NewBlockFromBytes(rbytes, pver) + if err != nil { + t.Errorf("failed to parse block %v", height) + return + } + blocks = append(blocks, block) + } + return +} diff --git a/sqlite3/sqlite.go b/sqlite3/sqlite.go new file mode 100644 index 00000000..2a152108 --- /dev/null +++ b/sqlite3/sqlite.go @@ -0,0 +1,673 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package sqlite3 + +import ( + "database/sql" + "fmt" + "github.com/conformal/btcdb" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" + "github.com/conformal/seelog" + _ "github.com/mattn/go-sqlite3" + "os" + "sync" +) + +const ( + dbVersion int = 2 + dbMaxTransCnt = 20000 + dbMaxTransMem = 64 * 1024 * 1024 // 64 MB +) + +const ( + blkInsertSha = iota + blkFetchSha + blkExistsSha + blkFetchIdx + blkFetchIdxList +) + +const ( + txInsertStmt = iota + txFetchUsedByShaStmt + txFetchLocationByShaStmt + txtmpInsertStmt + txtmpFetchUsedByShaStmt + txtmpFetchLocationByShaStmt + txMigrateCopy + txMigrateClear + txMigratePrep + txMigrateFinish + txMigrateCount + txPragmaVacuumOn + txPragmaVacuumOff + txVacuum +) + +var blkqueries []string = []string{ + blkInsertSha: "INSERT INTO block (key, pver, data) VALUES(?, ?, ?);", + blkFetchSha: "SELECT pver, data, blockid FROM block WHERE key = ?;", + blkExistsSha: "SELECT pver FROM block WHERE key = ?;", + blkFetchIdx: "SELECT key FROM block WHERE blockid = ?;", + blkFetchIdxList: "SELECT key FROM block WHERE blockid >= ? AND blockid < ? ORDER BY blockid ASC LIMIT 500;", +} + +var txqueries []string = []string{ + txInsertStmt: "INSERT INTO tx (key, blockid, txoff, txlen, data) VALUES(?, ?, ?, ?, ?);", + txFetchUsedByShaStmt: "SELECT data FROM tx WHERE key = ?;", + txFetchLocationByShaStmt: "SELECT blockid, txoff, txlen FROM tx WHERE key = ?;", + txtmpInsertStmt: "INSERT INTO txtmp (key, blockid, txoff, txlen, data) VALUES(?, ?, ?, ?, ?);", + txtmpFetchUsedByShaStmt: "SELECT data FROM txtmp WHERE key = ?;", + txtmpFetchLocationByShaStmt: "SELECT blockid, txoff, txlen FROM txtmp WHERE key = ?;", + txMigrateCopy: "INSERT INTO tx (key, blockid, txoff, txlen, data) SELECT key, blockid, txoff, txlen, data FROM txtmp;", + txMigrateClear: "DELETE from txtmp;", + txMigratePrep: "DROP index uniquetx;", + txMigrateFinish: "CREATE UNIQUE INDEX IF NOT EXISTS uniquetx ON tx (key);", + txMigrateCount: "SELECT COUNT(*) FROM txtmp;", + txPragmaVacuumOn: "PRAGMA auto_vacuum = FULL;", + txPragmaVacuumOff: "PRAGMA auto_vacuum = NONE;", + txVacuum: "VACUUM;", +} + +var log seelog.LoggerInterface = seelog.Disabled + +type tBlockInsertData struct { + sha btcwire.ShaHash + pver uint32 + buf []byte +} +type tTxInsertData struct { + txsha *btcwire.ShaHash + blockid int64 + txoff int + txlen int + usedbuf []byte +} + +type txState struct { + tx *sql.Tx + writeCount int + txDataSz int + txInsertList []interface{} +} +type SqliteDb struct { + sqldb *sql.DB + blkStmts []*sql.Stmt + blkBaseStmts []*sql.Stmt + txStmts []*sql.Stmt + txBaseStmts []*sql.Stmt + txState txState + dbLock sync.Mutex + + lastBlkShaCached bool + lastBlkSha btcwire.ShaHash + lastBlkIdx int64 + txCache txCache + blockCache blockCache + + UseTempTX bool + TempTblSz int + TempTblMax int + + dbInsertMode btcdb.InsertMode +} + +var self = btcdb.DriverDB{DbType: "sqlite", Create: CreateSqliteDB, Open: OpenSqliteDB} + +func init() { + btcdb.AddDBDriver(self) +} + +// createDB configure the database, setting up all tables to initial state. +func createDB(db *sql.DB) error { + log.Infof("Initializing new block database") + + // XXX check for old tables + buildTables := []string{ + "CREATE TABLE dbversion (version integer);", + "CREATE TABLE block ( blockid INTEGER PRIMARY KEY, key BLOB UNIQUE, " + + "pver INTEGER NOT NULL, data BLOB NOT NULL);", + "INSERT INTO dbversion (version) VALUES (" + fmt.Sprintf("%d", dbVersion) + + ");", + } + buildtxTables := []string{ + "CREATE TABLE tx (txidx INTEGER PRIMARY KEY, " + + "key TEXT, " + + "blockid INTEGER NOT NULL, " + + "txoff INTEGER NOT NULL, txlen INTEGER NOT NULL, " + + "data BLOB NOT NULL, " + + "FOREIGN KEY(blockid) REFERENCES block(blockid));", + "CREATE TABLE txtmp (key TEXT PRIMARY KEY, " + + "blockid INTEGER NOT NULL, " + + "txoff INTEGER NOT NULL, txlen INTEGER NOT NULL, " + + "data BLOB NOT NULL, " + + "FOREIGN KEY(blockid) REFERENCES block(blockid));", + "CREATE UNIQUE INDEX uniquetx ON tx (key);", + } + for _, sql := range buildTables { + _, err := db.Exec(sql) + if err != nil { + log.Warnf("sql table op failed %v [%v]", err, sql) + return err + } + } + for _, sql := range buildtxTables { + _, err := db.Exec(sql) + if err != nil { + log.Warnf("sql table op failed %v [%v]", err, sql) + return err + } + } + + // Insert the genesis block. + err := insertGenesisBlock(db) + if err != nil { + return err + } + + return nil +} + +// OpenSqliteDB opens an existing database for use. +func OpenSqliteDB(filepath string) (pbdb btcdb.Db, err error) { + log = btcdb.GetLog() + return newOrCreateSqliteDB(filepath, false) +} + +// CreateSqliteDB creates, initializes and opens a database for use. +func CreateSqliteDB(filepath string) (pbdb btcdb.Db, err error) { + log = btcdb.GetLog() + return newOrCreateSqliteDB(filepath, true) +} + +// newOrCreateSqliteDB opens a database, either creating it or opens +// existing database based on flag. +func newOrCreateSqliteDB(filepath string, create bool) (pbdb btcdb.Db, err error) { + var bdb SqliteDb + if create == false { + _, err = os.Stat(filepath) + if err != nil { + return nil, btcdb.DbDoesNotExist + } + } + + db, err := sql.Open("sqlite3", filepath) + if err != nil { + log.Warnf("db open failed %v\n", err) + return nil, err + } + + dbverstmt, err := db.Prepare("SELECT version FROM dbversion;") + if err != nil { + // about the only reason this would fail is that the database + // is not initialized + if create == false { + return nil, btcdb.DbDoesNotExist + } + err = createDB(db) + if err != nil { + // already warned in the called function + return nil, err + } + dbverstmt, err = db.Prepare("SELECT version FROM dbversion;") + if err != nil { + // if it failed this a second time, fail. + return nil, err + } + + } + row := dbverstmt.QueryRow() + var version int + err = row.Scan(&version) + if err != nil { + log.Warnf("unable to find db version: no row\n", err) + } + switch version { + case dbVersion: + // all good + default: + log.Warnf("mismatch db version: %v expected %v\n", version, dbVersion) + return nil, fmt.Errorf("Invalid version in database") + } + db.Exec("PRAGMA foreign_keys = ON;") + db.Exec("PRAGMA journal_mode=WAL;") + bdb.sqldb = db + + bdb.blkStmts = make([]*sql.Stmt, len(blkqueries)) + bdb.blkBaseStmts = make([]*sql.Stmt, len(blkqueries)) + for i := range blkqueries { + stmt, err := db.Prepare(blkqueries[i]) + if err != nil { + // XXX log/ + return nil, err + } + bdb.blkBaseStmts[i] = stmt + } + for i := range bdb.blkBaseStmts { + bdb.blkStmts[i] = bdb.blkBaseStmts[i] + } + + bdb.txBaseStmts = make([]*sql.Stmt, len(txqueries)) + for i := range txqueries { + stmt, err := db.Prepare(txqueries[i]) + if err != nil { + // XXX log/ + return nil, err + } + bdb.txBaseStmts[i] = stmt + } + // NOTE: all array entries in txStmts remain nil'ed + // tx statements are lazy bound + bdb.txStmts = make([]*sql.Stmt, len(txqueries)) + + bdb.blockCache.maxcount = 150 + bdb.blockCache.blockMap = map[btcwire.ShaHash]*blockCacheObj{} + bdb.txCache.maxcount = 2000 + bdb.txCache.txMap = map[btcwire.ShaHash]*txCacheObj{} + + bdb.UseTempTX = true + bdb.TempTblMax = 1000000 + + return &bdb, nil +} + +// Sync verifies that the database is coherent on disk, +// and no outstanding transactions are in flight. +func (db *SqliteDb) Sync() { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + db.endTx(true) +} + +// syncPoint notifies the db that this is a safe time to sync the database, +// if there are many outstanding transactions. +// Must be called with db lock held. +func (db *SqliteDb) syncPoint() { + + tx := &db.txState + + if db.TempTblSz > db.TempTblMax { + err := db.migrateTmpTable() + if err != nil { + return + } + } else { + if len(tx.txInsertList) > dbMaxTransCnt || tx.txDataSz > dbMaxTransMem { + db.endTx(true) + } + } +} + +// Close cleanly shuts down database, syncing all data. +func (db *SqliteDb) Close() { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + db.close() +} + +// RollbackClose discards the recent database changes to the previously +// saved data at last Sync. +func (db *SqliteDb) RollbackClose() { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + tx := &db.txState + if tx.tx != nil { + err := tx.tx.Rollback() + if err != nil { + log.Debugf("Rollback failed: %v", err) + } + } + db.close() +} + +// close performs the internal shutdown/close operation. +func (db *SqliteDb) close() { + db.endTx(true) + + db.InvalidateCache() + + for i := range db.blkBaseStmts { + db.blkBaseStmts[i].Close() + } + for i := range db.txBaseStmts { + if db.txBaseStmts[i] != nil { + db.txBaseStmts[i].Close() + db.txBaseStmts[i] = nil + } + } + db.sqldb.Close() +} + +// txop returns the appropriately prepared statement, based on +// transaction state of the database. +func (db *SqliteDb) txop(op int) *sql.Stmt { + if db.txStmts[op] != nil { + return db.txStmts[op] + } + if db.txState.tx == nil { + // we are not in a transaction, return the base statement + return db.txBaseStmts[op] + } + + if db.txStmts[op] == nil { + db.txStmts[op] = db.txState.tx.Stmt(db.txBaseStmts[op]) + } + + return db.txStmts[op] +} + +// startTx starts a transaction, preparing or scrubbing statements +// for proper operation inside a transaction. +func (db *SqliteDb) startTx() (err error) { + tx := &db.txState + if tx.tx != nil { + // this shouldn't happen... + log.Warnf("Db startTx called while in a transaction") + return + } + tx.tx, err = db.sqldb.Begin() + if err != nil { + log.Warnf("Db startTx: begin failed %v", err) + tx.tx = nil + return + } + for i := range db.blkBaseStmts { + db.blkStmts[i] = tx.tx.Stmt(db.blkBaseStmts[i]) + } + for i := range db.txBaseStmts { + db.txStmts[i] = nil // these are lazily prepared + } + return +} + +// endTx commits the current active transaction, it zaps all of the prepared +// statements associated with the transaction. +func (db *SqliteDb) endTx(recover bool) (err error) { + tx := &db.txState + + if tx.tx == nil { + return + } + + err = tx.tx.Commit() + if err != nil && recover { + // XXX - double check that the tx is dead after + // commit failure (rollback?) + + log.Warnf("Db endTx: commit failed %v", err) + err = db.rePlayTransaction() + if err != nil { + // We tried, return failure (after zeroing state) + // so the upper level can notice and restart + } + } + for i := range db.blkBaseStmts { + db.blkStmts[i].Close() + db.blkStmts[i] = db.blkBaseStmts[i] + } + for i := range db.txStmts { + if db.txStmts[i] != nil { + db.txStmts[i].Close() + db.txStmts[i] = nil + } + } + tx.tx = nil + var emptyTxList []interface{} + tx.txInsertList = emptyTxList + tx.txDataSz = 0 + return +} + +// rePlayTransaction will attempt to re-execute inserts performed +// sync the beginning of a transaction. This is to be used after +// a sql Commit operation fails to keep the database from losing data. +func (db *SqliteDb) rePlayTransaction() (err error) { + err = db.startTx() + if err != nil { + return + } + tx := &db.txState + for _, ins := range tx.txInsertList { + switch v := ins.(type) { + case tBlockInsertData: + block := v + _, err = db.blkStmts[blkInsertSha].Exec(block.sha.Bytes(), + block.pver, block.buf) + if err != nil { + break + } + case tTxInsertData: + txd := v + txnamebytes := txd.txsha.Bytes() + txop := db.txop(txInsertStmt) + _, err = txop.Exec(txd.blockid, txnamebytes, txd.txoff, + txd.txlen, txd.usedbuf) + if err != nil { + break + } + } + } + // This function is called even if we have failed. + // We need to clean up so the database can be used again. + // However we want the original error not any new error, + // unless there was no original error but the commit fails. + err2 := db.endTx(false) + if err == nil && err2 != nil { + err = err2 + } + + return +} + +// DropAfterBlockBySha will remove any blocks from the database after the given block. +// It terminates any existing transaction and performs its operations in an +// atomic transaction, it is terminated (committed) before exit. +func (db *SqliteDb) DropAfterBlockBySha(sha btcwire.ShaHash) (err error) { + var row *sql.Row + db.dbLock.Lock() + defer db.dbLock.Unlock() + + db.InvalidateCache() + + // This is a destructive operation and involves multiple requests + // so requires a transaction, terminate any transaction to date + // and start a new transaction + err = db.endTx(true) + if err != nil { + return err + } + err = db.startTx() + if err != nil { + return err + } + + // also drop any cached sha data + db.lastBlkShaCached = false + + querystr := "SELECT blockid FROM block WHERE key = ?;" + + tx := &db.txState + row = tx.tx.QueryRow(querystr, sha.Bytes()) + + var blockidx uint64 + err = row.Scan(&blockidx) + if err != nil { + // XXX + db.endTx(false) + return err + } + + _, err = tx.tx.Exec("DELETE FROM txtmp WHERE blockid > ?", blockidx) + if err != nil { + // XXX + db.endTx(false) + return err + } + + _, err = tx.tx.Exec("DELETE FROM tx WHERE blockid > ?", blockidx) + if err != nil { + // XXX + db.endTx(false) + return err + } + + // delete from block last in case of foreign keys + _, err = tx.tx.Exec("DELETE FROM block WHERE blockid > ?", blockidx) + if err != nil { + // XXX + db.endTx(false) + return err + } + + err = db.endTx(true) + if err != nil { + return err + } + return +} + +// InsertBlock inserts the block data and transaction data from a block +// into the database. +func (db *SqliteDb) InsertBlock(block *btcutil.Block) (height int64, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + blocksha, err := block.Sha() + if err != nil { + log.Warnf("Failed to compute block sha %v", blocksha) + return + } + mblock := block.MsgBlock() + rawMsg, pver, err := block.Bytes() + if err != nil { + log.Warnf("Failed to obtain raw block sha %v", blocksha) + return + } + txloc, err := block.TxLoc() + if err != nil { + log.Warnf("Failed to obtain raw block sha %v", blocksha) + return + } + + // Insert block into database + newheight, err := db.insertBlockData(blocksha, &mblock.Header.PrevBlock, + pver, rawMsg) + if err != nil { + log.Warnf("Failed to insert block %v %v %v", blocksha, + &mblock.Header.PrevBlock, err) + return + } + + // At least two blocks in the long past were generated by faulty + // miners, the sha of the transaction exists in a previous block, + // detect this condition and 'accept' the block. + for txidx, tx := range mblock.Transactions { + var txsha btcwire.ShaHash + txsha, err = tx.TxSha(pver) + if err != nil { + log.Warnf("failed to compute tx name block %v idx %v err %v", blocksha, txidx, err) + return + } + // Some old blocks contain duplicate transactions + // Attempt to cleanly bypass this problem + // http://blockexplorer.com/b/91842 + // http://blockexplorer.com/b/91880 + if newheight == 91842 { + dupsha, err := btcwire.NewShaHashFromStr("d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599") + if err != nil { + panic("invalid sha string in source") + } + if txsha == *dupsha { + log.Tracef("skipping sha %v %v", dupsha, newheight) + continue + } + } + if newheight == 91880 { + dupsha, err := btcwire.NewShaHashFromStr("e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468") + if err != nil { + panic("invalid sha string in source") + } + if txsha == *dupsha { + log.Tracef("skipping sha %v %v", dupsha, newheight) + continue + } + } + spentbuflen := (len(tx.TxOut) + 7) / 8 + spentbuf := make([]byte, spentbuflen, spentbuflen) + + err = db.insertTx(&txsha, newheight, txloc[txidx].TxStart, txloc[txidx].TxLen, spentbuf) + if err != nil { + log.Warnf("block %v idx %v failed to insert tx %v %v err %v", &blocksha, newheight, &txsha, txidx, err) + var oBlkIdx int64 + oBlkIdx, _, _, err = db.fetchLocationBySha(&txsha) + log.Warnf("oblkidx %v err %v", oBlkIdx, err) + + return + } + } + db.syncPoint() + return newheight, nil +} + +// SetDBInsertMode provides hints to the database to how the application +// is running this allows the database to work in optimized modes when the +// database may be very busy. +func (db *SqliteDb) SetDBInsertMode(newmode btcdb.InsertMode) { + + oldMode := db.dbInsertMode + switch newmode { + case btcdb.InsertNormal: + // Normal mode inserts tx directly into the tx table + db.UseTempTX = false + db.dbInsertMode = newmode + switch oldMode { + case btcdb.InsertFast: + if db.TempTblSz != 0 { + err := db.migrateTmpTable() + if err != nil { + return + } + } + case btcdb.InsertValidatedInput: + // generate tx indexes + txop := db.txop(txMigrateFinish) + _, err := txop.Exec() + if err != nil { + log.Warnf("Failed to create tx table index - %v", err) + } + } + case btcdb.InsertFast: + // Fast mode inserts tx into txtmp with validation, + // then dumps to tx then rebuilds indexes at thresholds + db.UseTempTX = true + if oldMode != btcdb.InsertNormal { + log.Warnf("switching between invalid DB modes") + break + } + db.dbInsertMode = newmode + case btcdb.InsertValidatedInput: + // ValidatedInput mode inserts into tx table with + // no duplicate checks, then builds index on exit from + // ValidatedInput mode + if oldMode != btcdb.InsertNormal { + log.Warnf("switching between invalid DB modes") + break + } + // remove tx table index + txop := db.txop(txMigratePrep) + _, err := txop.Exec() + if err != nil { + log.Warnf("Failed to clear tx table index - %v", err) + } + db.dbInsertMode = newmode + + // XXX + db.UseTempTX = false + } +} diff --git a/sqlite3/sqliteblock.go b/sqlite3/sqliteblock.go new file mode 100644 index 00000000..5416e160 --- /dev/null +++ b/sqlite3/sqliteblock.go @@ -0,0 +1,312 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package sqlite3 + +import ( + "bytes" + "database/sql" + "github.com/conformal/btcdb" + "github.com/conformal/btcwire" + _ "github.com/mattn/go-sqlite3" +) + +// insertGenesisBlock inserts the genesis block of the block chain into the +// database. +func insertGenesisBlock(db *sql.DB) error { + // Encode the genesis block to raw bytes. + pver := uint32(btcwire.ProtocolVersion) + var buf bytes.Buffer + err := btcwire.GenesisBlock.BtcEncode(&buf, pver) + if err != nil { + return err + } + + // Insert the genesis block along with its hash and protocol encoding + // version. + sql := blkqueries[blkInsertSha] + sha := btcwire.GenesisHash + _, err = db.Exec(sql, sha.Bytes(), pver, buf.Bytes()) + if err != nil { + return err + } + + return nil +} + +// InsertBlockData stores a block hash and its associated data block with a +// previous sha of `prevSha' and a version of `pver'. +func (db *SqliteDb) InsertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHash, pver uint32, buf []byte) (blockid int64, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + return db.insertBlockData(sha, prevSha, pver, buf) +} + +// insertSha stores a block hash and its associated data block with a +// previous sha of `prevSha' and a version of `pver'. +// insertSha shall be called with db lock held +func (db *SqliteDb) insertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHash, pver uint32, buf []byte) (blockid int64, err error) { + tx := &db.txState + if tx.tx == nil { + err = db.startTx() + if err != nil { + return + } + } + + var prevOk bool + var blkid int64 + + prevOk = db.blkExistsSha(prevSha) // exists -> ok + if !prevOk { + return 0, btcdb.PrevShaMissing + } + + result, err := db.blkStmts[blkInsertSha].Exec(sha.Bytes(), pver, buf) + if err != nil { + return + } + + blkid, err = result.LastInsertId() + if err != nil { + return 0, err + } + blkid -= 1 // skew between btc blockid and sql + + // Because we don't know know what the last idx is, we don't + // cache unless already cached + if db.lastBlkShaCached == true { + db.lastBlkSha = *sha + db.lastBlkIdx++ + } + + bid := tBlockInsertData{*sha, pver, buf} + tx.txInsertList = append(tx.txInsertList, bid) + tx.txDataSz += len(buf) + + blockid = blkid + return +} + +// fetchSha returns the datablock and pver for the given ShaHash. +func (db *SqliteDb) fetchSha(sha btcwire.ShaHash) (buf []byte, pver uint32, + blkid int64, err error) { + + db.dbLock.Lock() + defer db.dbLock.Unlock() + + row := db.blkStmts[blkFetchSha].QueryRow(sha.Bytes()) + + var blockidx int64 + var databytes []byte + err = row.Scan(&pver, &databytes, &blockidx) + if err == sql.ErrNoRows { + return // no warning + } + if err != nil { + log.Warnf("fail 2 %v", err) + return + } + buf = databytes + blkid = blockidx - 1 // skew between btc blockid and sql + return +} + +// ExistsSha looks up the given block hash +// returns true if it is present in the database. +func (db *SqliteDb) ExistsSha(sha *btcwire.ShaHash) (exists bool) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + _, exists = db.fetchBlockCache(sha) + if exists { + return + } + + // not in cache, try database + exists = db.blkExistsSha(sha) + return +} + +// blkExistsSha looks up the given block hash +// returns true if it is present in the database. +// CALLED WITH LOCK HELD +func (db *SqliteDb) blkExistsSha(sha *btcwire.ShaHash) bool { + var pver uint32 + + row := db.blkStmts[blkExistsSha].QueryRow(sha.Bytes()) + err := row.Scan(&pver) + + if err == sql.ErrNoRows { + return false + } + + if err != nil { + // ignore real errors? + log.Warnf("blkExistsSha: fail %v", err) + return false + } + return true +} + +// FetchBlockShaByIdx returns a block sha based on its height in the blockchain. +func (db *SqliteDb) FetchBlockShaByIdx(blkid int64) (sha *btcwire.ShaHash, err error) { + var row *sql.Row + db.dbLock.Lock() + defer db.dbLock.Unlock() + + blockidx := blkid + 1 // skew between btc blockid and sql + + row = db.blkStmts[blkFetchIdx].QueryRow(blockidx) + + var shabytes []byte + err = row.Scan(&shabytes) + if err != nil { + return + } + var shaval btcwire.ShaHash + shaval.SetBytes(shabytes) + return &shaval, nil +} + +// FetchIdxRange looks up a range of block by the start and ending ids. +// Fetch is inclusive of the start id and exclusive of the ending id. If the +// special id `AllShas' is provided as endid then FetchIdxRange will fetch all +// shas from startid until no more shas are present. +func (db *SqliteDb) FetchIdxRange(startid, endid int64) (rshalist []btcwire.ShaHash, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + startidx := startid + 1 // skew between btc blockid and sql + + var endidx int64 + if endid == btcdb.AllShas { + endidx = btcdb.AllShas // no skew if asking for all + } else { + endidx = endid + 1 // skew between btc blockid and sql + } + rows, err := db.blkStmts[blkFetchIdxList].Query(startidx, endidx) + if err != nil { + log.Warnf("query failed %v", err) + return + } + + var shalist []btcwire.ShaHash + for rows.Next() { + var sha btcwire.ShaHash + var shabytes []byte + err = rows.Scan(&shabytes) + if err != nil { + log.Warnf("wtf? %v", err) + break + } + sha.SetBytes(shabytes) + shalist = append(shalist, sha) + } + rows.Close() + if err == nil { + rshalist = shalist + } + log.Tracef("FetchIdxRange idx %v %v returned %v shas err %v", startid, endid, len(shalist), err) + return +} + +// NewestSha provides an interface to quickly look up the sha of +// the most recent (end) of the block chain. +func (db *SqliteDb) NewestSha() (sha *btcwire.ShaHash, blkid int64, err error) { + var row *sql.Row + var blockidx int64 + db.dbLock.Lock() + defer db.dbLock.Unlock() + + // answer may be cached + if db.lastBlkShaCached == true { + shacopy := db.lastBlkSha + sha = &shacopy + blkid = db.lastBlkIdx - 1 // skew between btc blockid and sql + return + } + + querystr := "SELECT key, blockid FROM block ORDER BY blockid DESC;" + + tx := &db.txState + if tx.tx != nil { + row = tx.tx.QueryRow(querystr) + } else { + row = db.sqldb.QueryRow(querystr) + } + + var shabytes []byte + err = row.Scan(&shabytes, &blockidx) + if err == nil { + var retsha btcwire.ShaHash + retsha.SetBytes(shabytes) + sha = &retsha + blkid = blockidx - 1 // skew between btc blockid and sql + + db.lastBlkSha = retsha + db.lastBlkIdx = blockidx + db.lastBlkShaCached = true + } + return +} + +type SqliteBlockIterator struct { + rows *sql.Rows + stmt *sql.Stmt + db *SqliteDb +} + +// NextRow iterates thru all blocks in database. +func (bi *SqliteBlockIterator) NextRow() bool { + return bi.rows.Next() +} + +// Row returns row data for block iterator. +func (bi *SqliteBlockIterator) Row() (key *btcwire.ShaHash, pver uint32, + buf []byte, err error) { + var keybytes []byte + + err = bi.rows.Scan(&keybytes, &pver, &buf) + if err == nil { + var retkey btcwire.ShaHash + retkey.SetBytes(keybytes) + key = &retkey + } + return +} + +// Close shuts down the iterator when done walking blocks in the database. +func (bi *SqliteBlockIterator) Close() { + bi.rows.Close() + bi.stmt.Close() +} + +// NewIterateBlocks prepares iterator for all blocks in database. +func (db *SqliteDb) NewIterateBlocks() (btcdb.BlockIterator, error) { + var bi SqliteBlockIterator + db.dbLock.Lock() + defer db.dbLock.Unlock() + + stmt, err := db.sqldb.Prepare("SELECT key, pver, data FROM block ORDER BY blockid;") + if err != nil { + return nil, err + } + tx := &db.txState + if tx.tx != nil { + txstmt := tx.tx.Stmt(stmt) + stmt.Close() + stmt = txstmt + } + bi.stmt = stmt + + bi.rows, err = bi.stmt.Query() + if err != nil { + return nil, err + } + bi.db = db + + return &bi, nil +} diff --git a/sqlite3/sqliteblock_test.go b/sqlite3/sqliteblock_test.go new file mode 100644 index 00000000..df71463d --- /dev/null +++ b/sqlite3/sqliteblock_test.go @@ -0,0 +1,302 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package sqlite3_test + +import ( + "bytes" + "fmt" + "github.com/conformal/btcdb" + "github.com/conformal/btcdb/db_sqlite" + "github.com/conformal/btcwire" + "github.com/conformal/seelog" + "os" + "testing" +) + +// array of shas +var testShas []btcwire.ShaHash = []btcwire.ShaHash{ + { + 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + }, + { + 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, + 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, + 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, + 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, + }, + { + 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, + 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, + 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, + 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, + }, + { + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, + }, + { + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, + }, +} + +// Work around stupid go vet bug where any non array should have named +// initializers. Since ShaHash is a glorified array it shouldn't matter. +var badShaArray = [32]byte{ + 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, +} +var badSha btcwire.ShaHash = btcwire.ShaHash(badShaArray) +var zeroSha = btcwire.ShaHash{} +var zeroBlock []byte = make([]byte, 32) + +func compareArray(t *testing.T, one, two []btcwire.ShaHash, test string, + sync string) { + if len(one) != len(two) { + t.Errorf("%s: lengths don't match for arrays (%s)", test, sync) + return + } + + for i := range one { + if !one[i].IsEqual(&two[i]) { + t.Errorf("%s: %dth sha doesn't match (%s)", test, i, + sync) + } + } +} + +func testNewestSha(t *testing.T, db btcdb.Db, expSha btcwire.ShaHash, + expBlk int64, situation string) { + + newestsha, blkid, err := db.NewestSha() + if err != nil { + t.Errorf("NewestSha failed %v (%s)", err, situation) + return + } + if blkid != expBlk { + t.Errorf("NewestSha blkid is %d not %d (%s)", blkid, expBlk, + situation) + } + if !newestsha.IsEqual(&expSha) { + t.Errorf("Newestsha isn't the last sha we inserted %v %v (%s)", + newestsha, &expSha, situation) + } +} + +type fetchIdxTest struct { + start int64 + end int64 + exp []btcwire.ShaHash + test string +} + +func testFetch(t *testing.T, db btcdb.Db, shas []btcwire.ShaHash, + sync string) { + + // Test the newest sha is what we expect and call it twice to ensure + // caching is working working properly. + numShas := int64(len(shas)) + newestSha := shas[numShas-1] + newestBlockID := int64(numShas) + testNewestSha(t, db, newestSha, newestBlockID, sync) + testNewestSha(t, db, newestSha, newestBlockID, sync+" cached") + + for i, sha := range shas { + // Add one for genesis block skew. + i = i + 1 + + // Ensure the sha exists in the db as expected. + if !db.ExistsSha(&sha) { + t.Errorf("testSha %d doesn't exists (%s)", i, sync) + break + } + + // Fetch the sha from the db and ensure all fields are expected + // values. + buf, pver, idx, err := sqlite3.FetchSha(db, &sha) + if err != nil { + t.Errorf("Failed to fetch testSha %d (%s)", i, sync) + } + if !bytes.Equal(zeroBlock, buf) { + t.Errorf("testSha %d incorrect block return (%s)", i, + sync) + } + if pver != 1 { + t.Errorf("pver is %d and not 1 for testSha %d (%s)", + pver, i, sync) + } + if idx != int64(i) { + t.Errorf("index isn't as expected %d vs %d (%s)", + idx, i, sync) + } + + // Fetch the sha by index and ensure it matches. + tsha, err := db.FetchBlockShaByIdx(int64(i)) + if err != nil { + t.Errorf("can't fetch sha at index %d: %v", i, err) + continue + } + if !tsha.IsEqual(&sha) { + t.Errorf("sha for index %d isn't shas[%d]", i, i) + } + } + + endBlockID := numShas + 1 + midBlockID := endBlockID / 2 + fetchIdxTests := []fetchIdxTest{ + // All shas. + {1, btcdb.AllShas, shas, "fetch all shas"}, + + //// All shas using known bounds. + {1, endBlockID, shas, "fetch all shas2"}, + + // Partial list starting at beginning. + {1, midBlockID, shas[:midBlockID-1], "fetch first half"}, + + // Partial list ending at end. + {midBlockID, endBlockID, shas[midBlockID-1 : endBlockID-1], + "fetch second half"}, + + // Nonexistant off the end. + {endBlockID, endBlockID * 2, []btcwire.ShaHash{}, + "fetch nonexistant"}, + } + + for _, test := range fetchIdxTests { + t.Logf("numSha: %d - Fetch from %d to %d\n", numShas, test.start, test.end) + if shalist, err := db.FetchIdxRange(test.start, test.end); err == nil { + compareArray(t, shalist, test.exp, test.test, sync) + } else { + t.Errorf("failed to fetch index range for %s (%s)", + test.test, sync) + } + } + + // Try and fetch nonexistant sha. + if db.ExistsSha(&badSha) { + t.Errorf("non existant sha exists (%s)!", sync) + } + _, _, _, err := sqlite3.FetchSha(db, &badSha) + if err == nil { + t.Errorf("Success when fetching a bad sha! (%s)", sync) + } + // XXX if not check to see it is the right value? + + testIterator(t, db, shas, sync) +} + +func testIterator(t *testing.T, db btcdb.Db, shas []btcwire.ShaHash, + sync string) { + + // Iterate over the whole list of shas. + iter, err := db.NewIterateBlocks() + if err != nil { + t.Errorf("failed to create iterated blocks") + return + } + + // Skip the genesis block. + _ = iter.NextRow() + + i := 0 + for ; iter.NextRow(); i++ { + key, pver, buf, err := iter.Row() + if err != nil { + t.Errorf("iter.NextRow() failed: %v (%s)", err, sync) + break + } + if i >= len(shas) { + t.Errorf("iterator returned more shas than "+ + "expected - %d (%s)", i, sync) + break + } + if !key.IsEqual(&shas[i]) { + t.Errorf("iterator test: %dth sha doesn't match (%s)", + i, sync) + } + if !bytes.Equal(zeroBlock, buf) { + t.Errorf("iterator test: %d buf incorrect (%s)", i, + sync) + } + if pver != 1 { + t.Errorf("iterator: %dth pver is %d and not 1 (%s)", + i, pver, sync) + } + } + if i < len(shas) { + t.Errorf("iterator got no rows on %dth loop, should have %d "+ + "(%s)", i, len(shas), sync) + } + if _, _, _, err = iter.Row(); err == nil { + t.Errorf("done iterator didn't return failure") + } + iter.Close() +} + +func TestBdb(t *testing.T) { + log, err := seelog.LoggerFromWriterWithMinLevel(os.Stdout, + seelog.InfoLvl) + if err != nil { + t.Errorf("failed to create logger: %v", err) + return + } + defer log.Flush() + btcdb.UseLogger(log) + + // Ignore db remove errors since it means we didn't have an old one. + _ = os.Remove("tstdb1") + db, err := btcdb.CreateDB("sqlite", "tstdb1") + if err != nil { + t.Errorf("Failed to open test database %v", err) + return + } + defer os.Remove("tstdb1") + + for i := range testShas { + var previous btcwire.ShaHash + if i == 0 { + previous = btcwire.GenesisHash + } else { + previous = testShas[i-1] + } + _, err := db.InsertBlockData(&testShas[i], &previous, 1, zeroBlock) + if err != nil { + t.Errorf("Failed to insert testSha %d. Error: %v", + i, err) + return + } + + testFetch(t, db, testShas[0:i+1], "pre sync ") + } + + // XXX insert enough so that we hit the transaction limit + // XXX try and insert a with a bad previous + + db.Sync() + + testFetch(t, db, testShas, "post sync") + + for i := len(testShas) - 1; i >= 0; i-- { + err := db.DropAfterBlockBySha(testShas[i]) + if err != nil { + t.Errorf("drop after %d failed %v", i, err) + break + } + testFetch(t, db, testShas[:i+1], + fmt.Sprintf("post DropAfter for sha %d", i)) + } + + // Just tests that it doesn't crash, no return value + db.Close() +} diff --git a/sqlite3/sqlitedbcache.go b/sqlite3/sqlitedbcache.go new file mode 100644 index 00000000..feba026d --- /dev/null +++ b/sqlite3/sqlitedbcache.go @@ -0,0 +1,261 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package sqlite3 + +import ( + "bytes" + "container/list" + "github.com/conformal/btcdb" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" + "sync" +) + +type txCache struct { + maxcount int + fifo list.List + // NOTE: the key is specifically ShaHash, not *ShaHash + txMap map[btcwire.ShaHash]*txCacheObj + cacheLock sync.RWMutex +} + +type txCacheObj struct { + next *txCacheObj + sha btcwire.ShaHash + blksha btcwire.ShaHash + pver uint32 + tx *btcwire.MsgTx + txbuf []byte +} + +type blockCache struct { + maxcount int + fifo list.List + blockMap map[btcwire.ShaHash]*blockCacheObj + cacheLock sync.RWMutex +} + +type blockCacheObj struct { + next *blockCacheObj + sha btcwire.ShaHash + blk *btcutil.Block +} + +// FetchBlockBySha - return a btcutil Block, object may be a cached. +func (db *SqliteDb) FetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) { + blkcache, ok := db.fetchBlockCache(sha) + if ok { + return blkcache.blk, nil + } + + buf, pver, height, err := db.fetchSha(*sha) + if err != nil { + return + } + + blk, err = btcutil.NewBlockFromBytes(buf, pver) + if err != nil { + return + } + blk.SetHeight(height) + db.insertBlockCache(sha, blk) + + return +} + +// fetchBlockCache check if a block is in the block cache, if so return it. +func (db *SqliteDb) fetchBlockCache(sha *btcwire.ShaHash) (*blockCacheObj, bool) { + + db.blockCache.cacheLock.RLock() + defer db.blockCache.cacheLock.RUnlock() + + blkobj, ok := db.blockCache.blockMap[*sha] + if !ok { // could this just return the map deref? + return nil, false + } + return blkobj, true +} + +// insertBlockCache insert the given sha/block into the cache map. +// If the block cache is determined to be full, it will release +// an old entry in FIFO order. +func (db *SqliteDb) insertBlockCache(sha *btcwire.ShaHash, blk *btcutil.Block) { + bc := &db.blockCache + + bc.cacheLock.Lock() + defer bc.cacheLock.Unlock() + + blkObj := blockCacheObj{sha: *sha, blk: blk} + bc.fifo.PushBack(&blkObj) + + if bc.fifo.Len() > bc.maxcount { + listobj := bc.fifo.Front() + bc.fifo.Remove(listobj) + tailObj, ok := listobj.Value.(*blockCacheObj) + if ok { + delete(bc.blockMap, tailObj.sha) + } else { + panic("invalid type pushed on blockCache list") + } + } + + bc.blockMap[blkObj.sha] = &blkObj +} + +type TxListReply struct { + Sha *btcwire.ShaHash + Tx *btcwire.MsgTx + Err error +} + +// FetchTxByShaList given a array of ShaHash, look up the transactions +// and return them in a TxListReply array. +func (db *SqliteDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { + var replies []*btcdb.TxListReply + for _, txsha := range txShaList { + tx, _, _, err := db.FetchTxBySha(txsha) + txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, Err: err} + replies = append(replies, &txlre) + } + return replies +} + +// FetchTxAllBySha returns several pieces of data regarding the given sha. +func (db *SqliteDb) FetchTxAllBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rtxbuf []byte, rpver uint32, rblksha *btcwire.ShaHash, err error) { + + // Check Tx cache + if txc, ok := db.fetchTxCache(txsha); ok { + return txc.tx, txc.txbuf, txc.pver, &txc.blksha, nil + } + + // If not cached load it + bidx, toff, tlen, err := db.FetchLocationBySha(txsha) + if err != nil { + log.Warnf("unable to find location of origin tx %v", txsha) + return + } + + blksha, err := db.FetchBlockShaByIdx(bidx) + if err != nil { + log.Warnf("block idx lookup %v to %v", bidx, err) + return + } + log.Tracef("transaction %v is at block %v %v tx %v", + txsha, blksha, bidx, toff) + + blk, err := db.FetchBlockBySha(blksha) + if err != nil { + log.Warnf("unable to fetch block %v %v ", + bidx, &blksha) + return + } + + blkbuf, pver, err := blk.Bytes() + if err != nil { + log.Warnf("unable to decode block %v %v", bidx, &blksha) + return + } + + txbuf := make([]byte, tlen) + copy(txbuf[:], blkbuf[toff:toff+tlen]) + rbuf := bytes.NewBuffer(txbuf) + + var tx btcwire.MsgTx + err = tx.BtcDecode(rbuf, pver) + if err != nil { + log.Warnf("unable to decode tx block %v %v txoff %v txlen %v", + bidx, &blksha, toff, tlen) + return + } + + // Shove data into TxCache + // XXX - + var txc txCacheObj + txc.sha = *txsha + txc.tx = &tx + txc.txbuf = txbuf + txc.pver = pver + txc.blksha = *blksha + db.insertTxCache(&txc) + + return &tx, txbuf, pver, blksha, nil +} + +// FetchTxBySha returns some data for the given Tx Sha. +func (db *SqliteDb) FetchTxBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rpver uint32, blksha *btcwire.ShaHash, err error) { + rtx, _, rpver, blksha, err = db.FetchTxAllBySha(txsha) + return +} + +// FetchTxBufBySha return the bytestream data and associated protocol version. +// for the given Tx Sha +func (db *SqliteDb) FetchTxBufBySha(txsha *btcwire.ShaHash) (txbuf []byte, rpver uint32, err error) { + _, txbuf, rpver, _, err = db.FetchTxAllBySha(txsha) + return +} + +// fetchTxCache look up the given transaction in the Tx cache. +func (db *SqliteDb) fetchTxCache(sha *btcwire.ShaHash) (*txCacheObj, bool) { + tc := &db.txCache + + tc.cacheLock.RLock() + defer tc.cacheLock.RUnlock() + + txObj, ok := tc.txMap[*sha] + if !ok { // could this just return the map deref? + return nil, false + } + return txObj, true +} + +// insertTxCache, insert the given txobj into the cache. +// if the tx cache is determined to be full, it will release +// an old entry in FIFO order. +func (db *SqliteDb) insertTxCache(txObj *txCacheObj) { + tc := &db.txCache + + tc.cacheLock.Lock() + defer tc.cacheLock.Unlock() + + tc.fifo.PushBack(txObj) + + if tc.fifo.Len() >= tc.maxcount { + listobj := tc.fifo.Front() + tc.fifo.Remove(listobj) + tailObj, ok := listobj.Value.(*txCacheObj) + if ok { + delete(tc.txMap, tailObj.sha) + } else { + panic("invalid type pushed on tx list") + } + + } + + tc.txMap[txObj.sha] = txObj +} + +// InvalidateTxCache clear/release all cached transactions. +func (db *SqliteDb) InvalidateTxCache() { + tc := &db.txCache + tc.cacheLock.Lock() + defer tc.cacheLock.Unlock() + tc.txMap = map[btcwire.ShaHash]*txCacheObj{} + tc.fifo = list.List{} +} + +// InvalidateTxCache clear/release all cached blocks. +func (db *SqliteDb) InvalidateBlockCache() { + bc := &db.blockCache + bc.cacheLock.Lock() + defer bc.cacheLock.Unlock() + bc.blockMap = map[btcwire.ShaHash]*blockCacheObj{} + bc.fifo = list.List{} +} + +// InvalidateCache clear/release all cached blocks and transactions. +func (db *SqliteDb) InvalidateCache() { + db.InvalidateTxCache() + db.InvalidateBlockCache() +} diff --git a/sqlite3/sqlitetx.go b/sqlite3/sqlitetx.go new file mode 100644 index 00000000..321d3a05 --- /dev/null +++ b/sqlite3/sqlitetx.go @@ -0,0 +1,253 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package sqlite3 + +import ( + "database/sql" + "github.com/conformal/btcdb" + "github.com/conformal/btcwire" + _ "github.com/mattn/go-sqlite3" +) + +// InsertTx inserts a tx hash and its associated data into the database. +func (db *SqliteDb) InsertTx(txsha *btcwire.ShaHash, blockidx int64, txoff int, txlen int, usedbuf []byte) (err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + return db.insertTx(txsha, blockidx, txoff, txlen, usedbuf) +} + +// insertTx inserts a tx hash and its associated data into the database. +// Must be called with db lock held. +func (db *SqliteDb) insertTx(txsha *btcwire.ShaHash, blockidx int64, txoff int, txlen int, usedbuf []byte) (err error) { + + tx := &db.txState + if tx.tx == nil { + err = db.startTx() + if err != nil { + return + } + } + blockid := blockidx + 1 + txd := tTxInsertData{txsha: txsha, blockid: blockid, txoff: txoff, txlen: txlen, usedbuf: usedbuf} + + log.Tracef("inserting tx %v for block %v off %v len %v", + txsha, blockid, txoff, txlen) + + rowBytes := txsha.String() + + var op int // which table to insert data into. + if db.UseTempTX { + var tblockid int64 + var ttxoff int + var ttxlen int + txop := db.txop(txFetchLocationByShaStmt) + row := txop.QueryRow(rowBytes) + err = row.Scan(&tblockid, &ttxoff, &ttxlen) + if err != sql.ErrNoRows { + // sha already present + err = btcdb.DuplicateSha + return + } + op = txtmpInsertStmt + } else { + op = txInsertStmt + } + + txop := db.txop(op) + _, err = txop.Exec(rowBytes, blockid, txoff, txlen, usedbuf) + if err != nil { + log.Warnf("failed to insert %v %v %v", txsha, blockid, err) + return + } + if db.UseTempTX { + db.TempTblSz++ + } + + // put in insert list for replay + tx.txInsertList = append(tx.txInsertList, txd) + + return +} + +// FetchLocationBySha looks up the Tx sha information by name. +func (db *SqliteDb) FetchLocationBySha(txsha *btcwire.ShaHash) (blockidx int64, txoff int, txlen int, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + return db.fetchLocationBySha(txsha) +} + +// fetchLocationBySha look up the Tx sha information by name. +// Must be called with db lock held. +func (db *SqliteDb) fetchLocationBySha(txsha *btcwire.ShaHash) (blockidx int64, txoff int, txlen int, err error) { + var row *sql.Row + var blockid int64 + var ttxoff int + var ttxlen int + + rowBytes := txsha.String() + txop := db.txop(txFetchLocationByShaStmt) + row = txop.QueryRow(rowBytes) + + err = row.Scan(&blockid, &ttxoff, &ttxlen) + if err == sql.ErrNoRows { + txop = db.txop(txtmpFetchLocationByShaStmt) + row = txop.QueryRow(rowBytes) + + err = row.Scan(&blockid, &ttxoff, &ttxlen) + if err == sql.ErrNoRows { + err = btcdb.TxShaMissing + return + } + if err != nil { + log.Warnf("txtmp FetchLocationBySha: fail %v", + err) + return + } + } + if err != nil { + log.Warnf("FetchLocationBySha: fail %v", err) + return + } + blockidx = blockid - 1 + txoff = ttxoff + txlen = ttxlen + return +} + +// FetchTxUsedBySha returns the used/spent buffer for a given transaction. +func (db *SqliteDb) FetchTxUsedBySha(txsha *btcwire.ShaHash) (spentbuf []byte, err error) { + var row *sql.Row + db.dbLock.Lock() + defer db.dbLock.Unlock() + + rowBytes := txsha.String() + txop := db.txop(txFetchUsedByShaStmt) + row = txop.QueryRow(rowBytes) + + var databytes []byte + err = row.Scan(&databytes) + if err == sql.ErrNoRows { + txop := db.txop(txtmpFetchUsedByShaStmt) + row = txop.QueryRow(rowBytes) + + err = row.Scan(&databytes) + if err == sql.ErrNoRows { + err = btcdb.TxShaMissing + return + } + if err != nil { + log.Warnf("txtmp FetchLocationBySha: fail %v", + err) + return + } + } + + if err != nil { + log.Warnf("FetchUsedBySha: fail %v", err) + return + } + spentbuf = databytes + return +} + +var vaccumDbNextMigrate bool + +// migrateTmpTable functions to perform internal db optimization when +// performing large numbers of database inserts. When in Fast operation +// mode, it inserts into txtmp, then when that table reaches a certain +// size limit it moves all tx in the txtmp table into the primary tx +// table and recomputes the index on the primary tx table. +func (db *SqliteDb) migrateTmpTable() error { + db.endTx(true) + db.startTx() // ??? + + db.UseTempTX = false + db.TempTblSz = 0 + + var doVacuum bool + var nsteps int + if vaccumDbNextMigrate { + nsteps = 6 + vaccumDbNextMigrate = false + doVacuum = true + } else { + nsteps = 5 + vaccumDbNextMigrate = true + } + + log.Infof("db compaction Stage 1/%v: Preparing", nsteps) + txop := db.txop(txMigratePrep) + _, err := txop.Exec() + if err != nil { + log.Warnf("Failed to prepare migrate - %v", err) + return err + } + + log.Infof("db compaction Stage 2/%v: Copying", nsteps) + txop = db.txop(txMigrateCopy) + _, err = txop.Exec() + if err != nil { + log.Warnf("Migrate read failed - %v", err) + return err + } + + log.Tracef("db compaction Stage 2a/%v: Enable db vacuum", nsteps) + txop = db.txop(txPragmaVacuumOn) + _, err = txop.Exec() + if err != nil { + log.Warnf("Migrate error trying to enable vacuum on "+ + "temporary transaction table - %v", err) + return err + } + + log.Infof("db compaction Stage 3/%v: Clearing old data", nsteps) + txop = db.txop(txMigrateClear) + _, err = txop.Exec() + if err != nil { + log.Warnf("Migrate error trying to clear temporary "+ + "transaction table - %v", err) + return err + } + + log.Tracef("db compaction Stage 3a/%v: Disable db vacuum", nsteps) + txop = db.txop(txPragmaVacuumOff) + _, err = txop.Exec() + if err != nil { + log.Warnf("Migrate error trying to disable vacuum on "+ + "temporary transaction table - %v", err) + return err + } + + log.Infof("db compaction Stage 4/%v: Rebuilding index", nsteps) + txop = db.txop(txMigrateFinish) + _, err = txop.Exec() + if err != nil { + log.Warnf("Migrate error trying to clear temporary "+ + "transaction table - %v", err) + return err + } + + log.Infof("db compaction Stage 5/%v: Finalizing transaction", nsteps) + db.endTx(true) // ??? + + if doVacuum { + log.Infof("db compaction Stage 6/%v: Optimizing database", nsteps) + txop = db.txop(txVacuum) + _, err = txop.Exec() + if err != nil { + log.Warnf("migrate error trying to clear txtmp tbl %v", err) + return err + } + } + + log.Infof("db compaction: Complete") + + // TODO(drahn) - determine if this should be turned back on or not + db.UseTempTX = true + + return nil +} diff --git a/sqlite3/testdata/blocks1-256.bz2 b/sqlite3/testdata/blocks1-256.bz2 new file mode 100644 index 0000000000000000000000000000000000000000..6b8bda4429200c0566bb13c28c35d6397272e475 GIT binary patch literal 37555 zcmV)8K*qm9T4*^jL0KkKSr@A=?*Lyo|NsC0|NsC0|NsC0|NsC0|NsC0|NsC0|NsC0 z|NsC0|Nr1%0`GtT00*bHz1|n5T)=M=pBnqU;dPxJy+dC1_BM0g?t47Wcf|GKdY#;B zx%Jb_&i2~vj_uc8@w++d_r2Xc+bQeWwB7AnH@m&s?|Z7VcLwh6_&)Ua-C7%VGQQc~ z>${h|?$>*(yBXJ;E3Q`cd%f+uRJ}XA?=#ch z^LF0cm$dfxz3JWU#m@D$*Rj3c>USOO?{`dUw52*PZ4) zZsq3L?C!nYb#Ajg-1geucDp>i-Q{VQx3xUI&gD6`Uawu6^=;dCb=vn_^?SRn>aKNl zE$pv*o-NzE&uh^0y|K@GzIV4+$bmfo002w?nqp#L0%cFrLlY(dm;eaCCJBH5N#bAt zO*GR4VqgTqU=ssCXc_@FgCjbAaCIAVQ4yFcx0gyDQ{z0Gs4FX^(;0dOf69QlWOd2#a z4FC-Q0fYo#1w0VRlT9>W35k^S(-Q<>m=TeQsvT87z%&|Zsek|g&_ zj3y%jGfhmMr>UAIk3kuxqef2@z$c;x$u^Pc4@lCq5G;R}q9*?O-;!gJMnwV1tNmyb zJ@fx!pSG8@UQWKV&A3<1{kZJNqQ08k z6D?(C9~^K8No6azD{saS5up0CIL)USziD_I?C)=&!gQRRl1VS{+J?XC_#bbw?xgcw zjlnWQhmuJmK$1=Nq>@QeY{iyDRa8VqDzR1~h>SsCD;S`vMFkaz#a1j>F^Y@@ixB}8 zR7F^*f~;V~WLU;U0gMLL5g4e*#6~J23L=XU6%+sgVk!!%C^905f(&B?Rz?W0j2J=! zkz&A5Bv=9>DkzG^F;zul6^s@Nf+(vMV51aRs)#D6ixx2yRYe3*Q4s|eF+^2@v4W~9 z3lL()AgmOMip7Gf76_n>7{x?j!Bi0#Dx^^qh$zHCR8?TA005$jqA-Mjq=2G;p(GR_ zERkRYhGl^yhX}|(fNQV-4dYK>n#ItplMB?#{#=cV!a#-X6#_w-Hww)~IQDocyC!q_ zP;QC6GWpfg{SRTvVKK%&;D83JAV?@cB9MTfNoG=P`RhZcW`%~VouXmT%%>Q`%gFiG znI{I2!l$s1vkuvTf^(_QFeC#yn%e`wLK@aVJW{_@>IkMu*Tz znP6aPZ9IqEsuH!41X8oO@$DzHuf2m$q6GXXz?)H=) z8Oo<|u~gic8#@ls4^23kc~Cpor9%wuOfhH~@7+O`A-6w1U~OG zKw{EgNhFbmD_5u(-)^8l4f&a^9R_dO&uKNTUsBR*+$IlA*>~OFPl4v}nJizqrf9w) zqe22=T4Ir+f2Wo>`sLWI3k8mhw?H3J4@8NVOJTC0;$We@CRKTKF-3MYHiCQ2x3BU( z)}O1pcK`t1iH=|ZvOy7K09X-1NkIVYB$7zO4T7V2@@RyR?lKpdotDOu@DKn100025 z8j|2a5i)c~hdx1-v5bPtJMzV|CO3sRfYNWYLAI?F)Ks#b?V*}=K)U~Rs~`0}`9jfr zq@K;^8mYzZ93voGNFdj?bzU>-R^QcyxFYAQ$zjzXUTxHHURb0-{WBrwQ(ZhX(djB$ z`|{b~h9T^fOv9~7I81ym3u$cwRR4{rh|WgWszLs;K?0NUxzv@B&3M~d!H89)Ki8|p zAx&K5^LeTg6__5a@+Dnc!orx%+h*yBU8H0mz)})hyp3;X`sbgdLE`m z?DM(0Suen3X-SJQGFHiJ0yCcr?|5*sFDL6h7#NVcZP%#TI4$qq*)#tOc#{r?EQZ$3 z@2aWLp$#GztTlv&s>7+oXsj0TsvJKrUA^1=xCTk?v&P>l6P;n&yXO^LN(W_@q8;<+ zrl^L0MY!5`jO_o#dV1AP-7?iu{WwR>bh*tHu{|g2=;Y@7OV2^dGW>g#(m%7hqc51| zzcxELdBOX6%VscLc4GgiyT#Xi{j2Y7xDyeuA-Ni0gT+xi?gf7B=ljdA!DA@CvcVa z0|L+;v>c%L7r?;)(hZ#!uzUkvOYL5>NO9@tvhh0$`rHzF$Y-2yK|#}W7;QdVP@gZ+ zsaaj}*SXBqyKiTPvBhP!4o2f%PFbD9m|>CUYfa|p_$#mW%GO>rXDT<-Io!O+ur6g+)r2!T9)G#e18LaZ{9x-(kfMX?ERZ|fqNIO zME-!VUTi4TBS-t#Yyc}w_xB<;JV`o%DIac@7nw_d$iX@5|TOOcDXyH_y!EDC<-4 zp|dc+urTQkwTFY@)y(Qp%58N!TtBnjTp9Kr7BaYo8#iI4v7ogav*%~Y0v&t_4WK;m zs=m@rZ7B+osQl1Akfx1co1S^%Ikray<`Wy)L8)zO^LY$qI#2!G2Dw%`Dr z5h{+fe|-9E$fmiB08zRveD1Yhcfwjt`wk1Iq2R`X1Q91A_9FYJ^&CP00aaVyE^zPKKH@`iDe(E?g>zB zM}%7D;S$eKl={FXK@rn&5>;QeQly&S%rH!KMK``jv;;EAMXa9wnFszdSnAk@(`fr1 z-F= zHj`aS%|-LLjNoBDsEib}NSwq5JLd6T7693-79 zDa>6lA#reFs2?X-X75sY!}Ai3TB3CwkYdOYjVslXg2Yd1enXi;DDq zzWn4VT)GEKJ7~9}y%gc+VRK3Rba`|;zJX#GA9Sf|N_4kmm@4D}qpUcsJR#}R86^G< zK3?Ss!&$nw>89d_t1|k&(zzOxU|5m0sk1dZMgpjHbS26>7RoL1{!7IW2-RKL0>wZ8 z&XXqs4L;1xknu*uwWRl0<1N;nGu^qkNAlVzq+3u3ApfQF#4g+wm$A|d!l1-_#J;yG zm3x<}PZ#pXr1xG8Mz6(W3cqYjn`{FN9tt7SJ^^!VBXJqHN)y}$W&%S+7+*{gN8(YL z5ohchr<3tZ=0#=0qo6$Obbi|LG=kcor0YLPK%_&LwJHl&e<-7n0%_7Ta;(=}?ZcGW z7T+s3xx;mUgx3d_b@*|}#dzL(mHFl|$h7?z$9%yOaT#1n%`PLi)RS9__>lx%bOy-Akf0XxoAo4bGs7|!&K zVdF4s+eNgQ3=(XFg^_SFGpAPBZD=(#bR(ZH3bbYqvRA4Zo{_Q@@-*w8gy+Zad|ABU zng4o-FaT?4s=bRnwy7h7^VJ49_PuGt6%Aw3^Ipp5ilKx@+Z*_<&m%won)`+4@VZhV zHg!;XUaY*>_#7oW>`w9d$demj4)B-L*Vf+Fq$4Eu;;nfzU&F4!7U$Y;%AhWCArS@4 zB!BjEb@@nDgj}0dN(C~)NpQOJmA$-mN*V2{*!6eX*_INd= z=fDLXe|P-UT#c#q@ZPo1k2PUb7tiD4X_n7hlBX;1ska~8f}x~aVURQY=<+tUzpc8Q zt}=$!+y>*`gZNj!5}6~H{CYxa&DUZ!e6P7Q>tOFrPa3)|*u*W5>(T_uBFMBznE3Ltoe!nHloq_n88!ROprYEC+-9f037NNO1u(LBUw5W@{UzH{ zvCt>#Fg;Vo6hJ%Fop`;K%e8Y>BbS@MN%9|0Q1jv&Cs+^WQ3dArTb$@NUIfB^_Etm{#jDAgQ26YLod%U4$7rV$nkip#Q; zvi&FM<=Wu3y>RFweN0i2m&BIh_6I)3sl3>%#?p>Oi$q6=fLPteH>3$a5%w<^?vPMx92tvP>l`P47rW>_nRV zfTuj5mXN3^1KHqn*R7R%+`@egteS@{*&XrjM}$b7JMs=%nfx8OU+@sd&h}0>q$MBB>Xtm9SQ3y(jn(R)=rk;Q9OyH<*O6iS5u|W4h`{~KFJ0>Vg z-T)so!4so}@3v1B6kS5-Vz>J_5MmX%qd4QCPk{?u2foQxU_6MN!AHoM(wxhl*b;TM z1Vc4s3M`q+D(3}9B?WVa2OXyh?MVLn(58ibwBV0>n8?IKhxzI6M3eipVcjT~Gg@sj zr3x+?rRe|yL6CuI#nExc0uEL0z;~%k$8}GWF4o}+u)v>uH8KGVcw;$W=TtOdp<1JY z>&0-7e74P}{5i5mO07Yj)vr2GA`vMe_~&8aJdLJv*Zvq)>xYGcx>e%fBr_I(A0_M8 z-2LHMP!X^vW`ssRiVCJe8%@HcwnK0tLj!mXR8aCCWjmL9=>r&1a;T&p5CGt3I6#)n zhJ$eIk5qwZ7)WCkTnYj$7IOL0cD_r5wZRr z+F$0YH)+(-acsHH3%sf~mV>de!rMD)&xx*pi?5t-?`=cTH&3+WJ_0#<$&N1s;tVj1 z$dZbT!Ut-R-|uHiCcl-y7L}JrexMik7fubi8A`5AQwV^<3O zvMAAehg72IDOeXh@D!UxlQsEJl=%P>u7uKX`3!JcHnGka85W3nE#PSINg{E0{wD=S zhbu{ICpZI4(!`e|;c+h^v9W_rsI((Fd5G*^&@x1f@g~_SaIHAVGpyN+_`OUdWwfb%tOm<=fAr+k#^ol)(mY~0;e6ub=ZYCNZ#N- zSNQxcn83j??A=7`CZ0T&ugeN+;a%HG&NJe)xG5?EB8&P(r;-b`nFqjTo4=QjzwCje zHv%x^VocPEEI8_gyF}U z9pvK~V+b=B5R9h2qxin_J4Xdc(z@@#F}NTA05l9fPEw?hhAjzx_ADd}+D-w64KyhY zD-2Tu&1aX5;B7dB|7ae%NN8CnroskUKKeuTG8&qB??RczamaLa)$>;Xcns?hoc{q* z9+M4;1OOmB{j6i;?=bEeCj>{!Ndv>{hTuK?$PU_V`q;=oS?|dw)c;^gbCJ@}{&ol( zG}`__Ye=$*XQ*$OKh8J%p~-)M$$^{R1wVz|BB4`wOXN{NrnbI*51{T&F}p(eEGcsd*hV!#dDi#q_s*aTQp>WFk}=m;i!mO@#$~$1$9Q z)DGq=0clhG9_ZIYpO1%oOOXCn9@L{E;4r4!r6aiu+H?x)UIj?Swau?7%vC_yz7;uw zrt09C8M>e143Sp^{62F%Y7cKvioYprA2YA`bO&hOf^B|EM;d3yQ%a&@eJ!T_R8agz zHhPGZCFS`JR!m?Xu<>>N0p^zY0%^=&eE6sE2t;-_DF`HSb^{%BhQ$r;{|s3<%TMHJ=< z2|zw&$9M{Q@6bFxgY)Pr{WDSmS9tx6(qg>iC`@J*O^=qiWa6wb>`x=paOV7;29 zV-v2R9sV)9Z$OA=;{Vh%5`ROu$rsj-RJs|{-y{i0!c%SF3;fZ?K;p6@pI>rjuC#f!b=sjjvC}>NP?9n+E)e&15F=|TC@q-uXa7u-Bg5j0PQo)`j zZ>sI6uXDvqO08=l=3gWk4;h$nL2?eU*{%W%^aTRLTQ5o{K@?D|^S*KJ0afG3xl;<< z`AWFU4nje8|G`DTW0H%|5#FYonpW5RaHNV}l>`v=l>xAdqQ8a;@-ZLES1y)nap^|N zIod@?T5cf9@Xl5fFaWqwn6+$OsG1-tMX10B6x1(glQ5#>XBtB$*!pSG>0O!ob;qZaXL6Ha7B;80j?8 z{}q;zbcgif=%+p44x-ZaH9&k3`(9`qpP1PT9|b4bsqA|LO!Nt_em#gvlaEL4(It8d z{!m={WQ44r9K>E;Th~1_A+S4*pXAl@ald22vvnJl-!F=%~@s9ixdn@!ZJ2y)eN8w!b_^*|_SCkF)X@Lc_ zuw z%2r~joj8{{}al_64lG#3>B!L0}UvOp^dlC?}*6T9gg>t3e3y-T;K&pW}WcNL3cv+tRdd}&IGy{^+PfR0r+x@iY*)P$?01PsOC(oy1 z#nV-XH;RVIrmexLdt5^@oZl~M2Pq|3lTiCp+~(TAv`vbqFsoFHAuLO>0_62)5V};3 zOoU&*!AT*Ssn8y)OE*&%QZ|;1KgR^NDcJ048K=(eRlPU%O$hf`=^u$tJ|lmjoAU@A z22e`ua92W66s3+L0MV-b1)c zp==w?W|Ijsh{zsEg;)*9q1CY2OX;-hEtF)9qaXXP(lKfS@vT8WbO@;Ctn`>awixJ5 zkV?)8pif(N0QQe{=&aezjmMHNo?ETY^11IR!EdZ_!IP58gB@qp<-~N+t*?b>RLn%g zBYyR+@6h4Mn-UrR_R;?*34M45Io7xf_*(Zp^<^_lSHz~6k_BPtIy9ZiK!4^Bs1SEO zL$hL`wZf=(T*2y~oWMIxy6PqMz(N{8P}4W|pB179@)(-XAjQRkU!41R;6Y}dy1iZH zl6bmLiQw?c>}CK2(KdPV8fM1N;+H;Sk9K)H<=O1uWB5zYcRRt=XKd?r9f0mUADj;B z1kD2Xk&Au~#!EdkP#pWw$}6b@&a38XB~2m?1X}Qyg-crDKjGbHl-QA*j?XLP02O3% zDqvOZPs?=4Ii!(#j%Q`3+QV`#&)scQXNmK`XHg!swI9*|7TH>Be-H?P+t3PoGl?WE zNcJP|@|^46ceBYkMF~TM^25yD__nO291fj)$oXZA9*cUzECm{cyBXYR*XKgpffH<&>{;tfFZqBo_tinP;NIski2aIz4{*5|TACBfs!izV=|dyCoG9 zz~!gb{RZ}x*h1SSvXSt_*rdAQ05^tmZPcKq-8^b8<}>mZn^a(S$=>fT_0@k(le0xB zK?dbLJY%G$iM7h{J9;n{$k1uCEb_d0Py8G(rOp>cf|7CMeG@dK{!;U2!)zL+_y}bA z{vZIA0^M#9j!2K%_}_HXFCUCjUg&#;sn(m=v-z5dK1I$IIt6L%GF+~z;sj%>iGs-d ztwx5ygjCKHN1|PuTfJ-I_%#Z|fvfH1>!bv8)_lRy@E429$}Gis61OSMxbSDz90@-Cpc5pAmFzC% zYat}Wlhp$p$6TauFQ=x^D4)icV=O~eLCeNu;Fb$?r*7OHQnkQS^TBhzu)J|9JlHor z?PK}DGx#cRf26S-3hMvZl^6^)kqvEki=jR_iW!vp(FbX$%oxxTSxvk6$5_(ADL%Ez z!Ueg?O*nIv)V)v;ai!VF!R*GG*C)mAP4+8xmZbJMta#t;W_i+fm4b~f*dUjha6a-Y zX2JI?JN?gjWF@k#tSrEV6khcjk}3V3Tp;b)Du|yCd_WE^D{IHURBOmvapg&qb>t+c z@@-?b-+xDKrnuU7Y4peX;Bg%xn0&YA4i$<>dUQ;t1V8EA6A(Fp zwrw(9N>E;KX!cv;Qwg@;0pX!)ETZj*harI_6CDaPvSq9GXW)OGvhu|)!;XB0(TJry zEA$e5YR3r`w()t@f{N}U7$8ZfOgOhgw}fEuKE0U7n@DSRoF=b?(t?0zr{N#2H;AhD zlhh|tg05^rc{EgFH-g3J>NVz27WM8Ok7qs;zO~8!3eqC22PPS(% z$I9%xedtE-HQmt@k77Q@Kf^Bj0K8LJt(EDkiPF#_agy4YO%tOr(y4JEzR4NF6i3%I zJ}zq8*%t9Psk`Bl&Umodgy>{zxN;@qfWNL%(_)wLP8N&oU=lgq9Dqt| z45sReb`|n^m5`3}J#aMnY&tKzZlfj69wy|%qeVR9L9=9aS6VJcn$fsqqGrGZ0c^t! zz!HaD_Brrds;Is!h4AnDj;w$`T$pz~3eJwFW^OlCp|v&VUr_`JrY_fv*wiB*{1HQ> zR9)Qq@r7O+)U>nyKGEI*OcRiEO-mEd2KKBpH?vYs7uhTuHSqby@?Ho|&=lDH8%gut z3izkU4iz>ozAqE>=S(RPr|;P8A|P6gr=^v%d8#W>c|X)r!}J#dn9~_T)~P6(6P%<` zE{)0DChf+U2F04m&PV&3%*-3+4`4C;;48GBRV=7DnNTimgz3Kx^pTbA%!zv9b5y;b z;d{+zn0p^~5_mVrU6ScAev{rYKQeCpCghkx5R^kwq#uCSlazx1u}>%7v5hENn$qC| zI+}DOBOI3rSn1uR!~SbHAwBm@Iu}+8arJHf0(R`XtN?2dIb-?9Peb2X9SQ#iHuF*W zU19p5XpS20Aupl(RS-xBO_Jw`rpj&CDRbz?c-bQHge57#@|=7^lGT3jF(C>YU&mCT;k>TZ&&`H!$wxFytAfws_qJ?=$8 z>`UJbOaBfBicz+->C*3^M)}d`Yzwnv00~p~NLFygppXX-Q4;XQV2szX6Xouw$#!pM zJMeu91+nXl3+=g^nVSKEu~g&8h*%dXFDPm5jk@=NEWGR&R>8ea)8I@e{=%+a^WVy5 zWT<32ar3O+*9548(8C~dP@wwfMrF{=qio`JGZ{2I63&2e`rj#S>+7)g+RvfFPN>~j zf6i*P#-AoVlyJM@VsZHvChWhU**l+~ykDY%Z z2Ncg4P0;z6=~g6?y1;>y`0RsZM)4#MYRjZ)Rc~F+Zqd) zpsIJ-n0KL%roT^=ziQw8%ZoKe=F2G=Vv{j748R*U~naE^xh~17J(XDZ;esTXq6@GOwywuuo3t@JQ zCA2c!7{lIA_5 z!5H9~7v%4w!st_~Uy(*9mTV6d;{XfU3W?XhM_{0EpLx+~S9FoPdCnpZ7j{_5e3d1% zKJIfK4LU5ZpUbtozoFsH4;`Up0Xo?u7K_-;;^L+`ZtNS2`<3pztx~bEQlFcyE|pzD zyMWzLIFR8v~Tg??*(FfRCxe=H^UvFViAfjX&96|Oup1Sc102c5Y8d|HhWhucJAAy9 zU%SlT$}N_Xw!888G70-zPy4(QBWI!AVdvw%HyPVt>IrzJXVRA!E3|N4Vi$$y!8;4U zY>*ka?;IBem#wf3hQtMZn>3UnJWar}D-?+o$msXO*k|%y@QYZMK8!y1a5hUqLmBC> zAdKG1;O7u0XA8z9X>izL{~5&vMf=|&!hTIp@%5TxFUyUGaZ95fM9EA)72K2>hDZdR zd*|7`S6C4|Q zN@h(Z`%r@zs(r0MlA1OnT9@BhQX{^W_#gE*7fEe`mSrC9dnbnxq}}i*?`PN)KebIA zu7;u6s)>U|Ao`4z3Oc@16lTJ z(f=$05I}$g0003fxZ=ctLVW>5=-9ogS{uLLHS~39zSq7vauxJOg=w{x8n@`@TJ1NA zNtwqK8fDEFTGX^QYqr{P+ck-*tIJhOD(Y48Q2t*R z_0B7+2_s?lYXSRf_|;=?dj1)Yv8TJT12Qk8q9%r;eFr`x0)CSGgxRk&cE1=L||Z}$GTNm<(>%Xz0QWjq=VrC``!OcoOPuj(6DP9E~cTQl(aFR=eMWZjrc z5yAkJgf-3G>EO77ky?(J%@Wh=5q{RNw)|^ApsmS2I$gSP_uH9@=?J6v-TRWTF^8g0 zY_q=d6nkitvXy$lKHTUSn2_tyF4BrZ*`TH6*+ZthdifoXP;vFOlCo<6H{mA)2mn9; z02j9j!|0f3icy5q?EPNRnqj|bkZTLfu8pz!Df&O9A#SnU{q=RLR7}_TUB=~2yPi3K zvqzi0n-JQUWKLx#3EOEozTg|QF>lV8eJ zCyanlg)xM9%++q6#WDrg(>b1AZ#Cf3$?Coh8DsQFqekXFX3WbDw0Eepi=Ix=T+!S%qE>XLnQ zJZ&Sl3e+7PQc1Oa44GCUz8I;2Htja%C=E8`$4`E(f4#Rg&1i_6T_@5S!Rr}s*+Ss2 zQ-oA?q4L^jv`d%EM_MPsF(J3_{TsR7Ym=g+bf~K)JR2^xQ;>fv3nKe4WrqbBKi>s2 zP%^agjI8kT0O=N_Kn3FhO9pLr_a#(jyn9Q)APeqwTkQDMY<#g{C}DD#sYfHsdsIGb zRAzou_V4l!=O8abKb7bEAn$m&9ELQzzpj7_%i&gR6|{7cawMrl1`9fTgtg=Dg^}vp zxJNY!t_BSD>>CCD&R+=a{E9c3-iXv;hYZP;joK47d=&1afrfOJoB11AkXjJts0xu@ zY0~2@ejy!Uje{J$nJ~Nqcvi4A8F~>ukDis6vXfewm5&pyDZmi=Kq%@O?-UO>Xh-!fmpQ!1v9L&ud^N+*mN@V{yu3K z5NBWm1`HY4Fkrx`<^wtfYL`_OM(eIEaaOWbt#emUw-01EAUTaHXb324v75*e9p-2o;v+=yY{nw`w!4G8 zpO%*!0tlNpnZygqt+;q|V_9q4Vk+eaGx+3g9CbRLgkxkH&`tGYp*&I!mLQo4=1JEE zq{szdU#@PKK24!vgzwp7EQbRP>0hbpx`)efOyy*(a3N;AkhrZDyoYqq&R|cFJSLP^ zXxDw_0&a+D8H5eJ-3saL+S4m6i2lawPW&srwf<}8%vBqoX4*6(*njt488~K|9<08R z`s^3eC&e=Tb_oVK+^L6wl}iQd>hB7OCCE`b+A3j#hZa0(2_l-C^vD*C{<2WceP!~+Q$2GN#BEnQPW zhFl~rJwC?r}F^8S^GiK z;ZBi1%eqBRD2c?n9>Qt7UW0Tr9&Debt`0>bK|!@?<~I`|wts7nAhI0Z1^Is7QwD7U zz#9}=ZYqS*x-9Ha;ovp8!xIUe{Z1s@B|ojO84og2OuJqtqwaF}of9I`J{xwICP!0H zRS9>k1R>hXcskaT;Oi@!W_Yb1X=tV7{17$u+uw5`tIAn=*n5;HZn@slpo@8+<)ndqG10G}rjN7*|V|Tc){W|sY zO_+E!RxrevJYB4P$I&~8u>NhS8^@0D?9i?;&4F3D7CVthKUqmI{fA$OX7eu`X~G&D zguCRcJC9;*swzJAp%4j4(B^uPzDI;sMRqc$E3z*SvCLJ#JM>@m!}Nk;j>TQQH)Ycg z;#%P`j%m^=M)PU|*BiQ;uPdHSl<0UZY#^_#Qp9}^!4m~fzUJ6pg>LGt?zo7O-A{^2 z*U)@#(N6mqPe)d6J$LTM^$z=kkb!&c#ecT>2S#W}qMO=;sQPEVuBo^iC*5-z}~vm7}YJWDEXDwan{MP?{aNBjMHZ;Malui4^RKu~#s2#LV=cSv5OPBExF zeuuF8WbU4r;pu~zlGIsB_^a4_Nt|Cf2|D-6t$I7QOm2<7bvwRpXvVhiw}s3LLY$1y z$rf*q05397&3(bmRSw8q_GRhSyQuf5Y<%u4J5dpD=h_bA4D;-Ej~25JP8v5dC|WcJ z;+vN=1=Qq1fLgxt@{*fKbf4KQ(5*WfrbnqN^w=6SRpI&%=^g#kmY}9yd2e}`1;UKh z37$4WI*Rsf_$LV}g~!h;du9PEAw#~oZ1!5Bf3`IFeXF)ZtnKdeatr1jUC2eFSogHn1z`8ZEF5EgtGM1pqN+{}U&g^u#WxbC2b>`(`OFvC3Im9u{A{@sHD&28a(LvAqW;0WXC6WK2Yn0l zQ2%Sa%r!P38F{?O6yx=p*tF-*RJX*w;NF<{pvBKP z7j$P3qEgs}5|rUsw7x64Y)v=p4lf^l^V}U=JUG6kj20TpW=Kd3BYJ(;Ykf=X1!OKs z7mi|0fG6#xelqZ!tWl5Mm*9JAi=1Y6?9)hb+nS`AVOA(SJ##`=(ZVKAoy*7D2{=k{hiNLd?)Xo=a1t#UOfkcGMH?mSG z|32j`0Wz-{2jC7rblzQEG@N>eoJmotLj|)FMpY;KrcGXe_Q32Xwq{s2&-!mB@{>wm z$mGA;u_A1(R5Ocz3a*v*L7_%jw`%d48>Ll!b;Z>eR#qV7vv8;wIcG(a!DT+0S#o1{o5J^ja_we^7ruY*;Lw1%4+G z0hT9?cyG70ZkYGJcx|#fTzCRcc6B`V?mV$Gcvq7Ic_Ht??Z`MnM|>2oIO@Sdf*uMf zdaOi-**5u4+#byzQSc}cRR8ooo=AQ_b$qj?Sg90fS=kF@)s>{78uHZjOP-x5oRFqj z&AB-49>CXD=x1q}6zAV&1X{x7%Y+k&a*AR2w}04P&F}^oBU%vF@8#lb5;Q|(;r5s< z2ZMIw7_WlTwl=W{Cn-C_0u!kyuyT}JPXmrVD)wEkg16xI(Nh!b*GA$eApNo$)%f~L z9}twgk{DB<9lXeS&tj%bSR#hyk2I06bL?@!!nU+xr~4JE=`ljs+oO;nxJSYsquI!| z4@|(ixO9nRgUf)APO zHC-1-Mu`F%?(-j}0=-sw0muI02t zHA8RuNt{wi9{ZW5dkLa6<(N z)}x;N`Q1=d|8?1|@?NLpN!Uo8!{kOuazfnSLaI?2H|AgGx5Z~db{Ofs$80l1emiyd zSX~ybB4ojmll(sa)E!B+rHIc~Wk$xSc?EUn_&k59v&Lok>V2yQgZpsF-FD%Wk zRmdEBoOiVbxE0iU{_4owaly5X0X6pEQ6;B!si1D^z>MI=`e12)>Spf(S<@mW_isCW z>6O9WA+`EZKPmEVy0XA~UdSdAscc&R`jr7>$y(uH6W_bvHwAZSiI(%XM2G@u(@Bfd zrIgIB=&;?LG#3mf`Nwv1+tRIYNAX(S=u?TebBvQyLVpRJFwLmD@-kBNQy--6B~JgE zqfb+Wj^SIw)y)`udeb~ywB52=L^LW$+X}@^sC%v4tn-Q zHBuA;60W?;bvcpCK& ztK$bw=YE|&Xw7J{879u;p%4HWI+HxTRK!^sJ?^z>Qgv~;EXw-H#9oW8Bc;a*K@>6iI_Lqu?wI8BItku z47lg)q6g5WOc8hg% z<0i&l_J2+33db%_TzgiRr_7buwYm&zSBo_P5zqvthL_SdePGke6cM6ZeG$PRGa2<$oOn9H5w8zj z0=lL)_~xL?mK_K3rM)Y))>6rXhSBA$K}8`ZL_l7@!K`t8cjH51H#8SrqN_p6%y46W z2LUf>z#Zd?D0U8hc7($3@_d9wBpX)PZAc{@x8Ns!sC~KgPyVE9g-*hrf9kpIv zyh;UF%DegG(*CtJTMHzO082o$zc+>ybHSVn8%+g9|8OzzRzTLTw)Me9d2Pg8TXa+< z22w{r!wa(HSxGei8fZT(f`-0S*<={VyX!9Ve7~I&=uSFR``F$_v+14h8RM76#cJRA zCEdoNM?lV_2Nb^bT=gM(iSB-ov)dmiK`V0t*5(Upy^OJBv?u9%%}nfvIiykRA8SXjhRGg7vd!qzZ>n+A&{<_s>G1{d)GB7Q2d7SIt z&&2~lSe5yM*)ANmM9^@fSo*kEsE({m_-IPBXV>IrbU!z^+8v-I(>I(TOIKwx|MDZN zlGutuHPLkAI*9d`dauC>t~tF8Z7>YX8XV1jX@6n->)=)cy%+oMl-<3@v{C%$dfoX! zjV!WU#8PO5Uhwxp1|DsAknnIBcGI&x(J^$Cn&P{6Z2a$40arZo;xPRJKMKs!6BIN3 zHb*-+0vy;9F%dGm&LM1%KD03$)*AhvoOlf%HBU6SW-*hhN=J8*%iI!&14gn?Gp^a`JjGMDEO;O8C^r!{2H0W3tVh z{eHX8vDlqR6sM+4X^n`(Ii$iIi$={kG4VYk-v-6dmzisw*rgWyBN<#B(8+Z^myEo0 zYV}v1)G@8c8qpRY0yO(6$kan?{JI&hDKK-z`+i?>rT5yp%Qc4q+qZ4Aqdtj5rqRL4$tUTSi70@QlO3JoRGjO zF~(1n%7$uR$y``c-)+&x1V%zcS!cqrm-zSUkn4h553EXxwh5w!jRu&CoH8=@GZ;j% z+=mMPpLJ3uYIQ@@ko@OglllIx?dbN;;S<>69a5XZi{fCktb+ptBToT?w6Op}GWRof z?3)S%{xcyz;r_0qYvwQZ%`MQUCD*EoyTkXW|3U>g`g)_SHMAEGh2P&D;ki#R2s^ZI zU|5p%%kE8$YhNP@>|UDz|2~QA!YPzK3PcA_d>nYa7N2@^_+4tRzp^_K3g!*`8GI*U zwvXF;97?Oah8_Y8CEKyN8=0b?a^jk-t4bT%f(lfx-IOY_F0Hq^!3Ft}R>B+z>=K3c>TA_Z2IsZ&)-2!0TLqHyyOMM)-?&k6 zPu?B|qAHr!NK0s~y;ukU9EO-VpMu<2h<|08-39tUz|!YWs-BQfcux%j-S+M)Xf{C( zy`k&PN#(C6w;D+@m#cOCLg%n&T?iC^60EBStoG0~s(l(Hsid2YYc!Y^yMVa38tK55 z#mm{MXm&VU%dsUBPUxBtE`{6`C<&|yTR9$$7IC8G%=s^%Zb<0ZZs;kG@`ry_=wIHUo5E;0Z^{?IaH zqt9Ql636QouvSA<>7YM?;&d*lW{Q*Fo*k8F z`x@E%EKYj~&Yvz;`vfH&Z9TgRC7TtzJGez~z}U|JCs2Pl+*D_LJ@oR0+!oVaCyvpH zzI_Uom{dSmGc18{{K7Njbp3)*;K=@!g*zgcIjw~nV#tmyM*6M)q2@!A-e-a zPa0i&LAeu8jp1&we@p84U}MCnIlslLHpxu3+dq^oWi7)q3{%6l5(j|jS~DYTskE2` z7?lrp2`5IhO}|-^wUx#!j8n#meoNdFi~Gd4Z=jmAF(y#abkCX{4)Pr zb!`0}{DpKq2EspI`F(BcV}3|H4aJ4OlM7uO;@WyG>@=zw?e#4J>Mo(zQm=TO?Cs53 zl>Gtw>>+aVW&dAt%Tq>(k%f(k2?4b)QH-gWfA4b2KZUP3SE%5|k!2lQ0;S_ibu)vN zp4I8@n%T+&VPq5=OmOCkSwVo4C%IGrc{0!QQhQ&Ni_jtrvZ0ig5U|sTzcq1cG{XQ! z_0Q$n$@K&?Y2P{6Lhw8;a#OuCpb-iZT<>n?>dkZw-mhTy8fyxZ(QC^dPHm&U69!=! zIqQ|RAz0OC`#UB2;KQ>hL%yF;qW{8nHY7atx@mG@y{3+U@d z(jeYFAuryR^57#adKhF_>=%HU=vZLhCMcCCA}$T+1r;}~(v_03(}Wg>OT7H4Rb5(^ z=X=TbH4Cw@WdBB6Y}A$2C6-9P#7AqfwT-K9@~KuVHe26uMM#fk2vA};Kc9z5==@Hz zF*t*&f9tUO8*{a&f>!q4CLMLjOO6IzICVY5FE6Y*0tcVCGhIUE!wVY~>oEhLe-WwM z-2Kf`eIOzK5H$blws1Gm#@RshqSsjH6A`-f<{Di1z7&kd^^2L?DkV19+haCUQ_b#? zoJ7)^1zdH1?L?iYgkcKIVc#EY6%E?*W~1akYwzC$`cj<~1e1{%inTz;@V!15ajP~5 zBZ^lvy$c-wELgSFG>q!K;p+51dwwl8O4ZwjeqBXXHKmkh%a(~x!U3L@B zp=SEB%uO*p0U0XyVA+d~I!-U*>*Eda+p7bhFr5EEo_oZ31#z(4aRQ@-%X2zhuSY#7 zK4GY)^QRrjgfVq1K?csCugofW1*X0rHA&5vJBPC`JMnqMZ*Y;!H2X);qko8Cb$JKE z1I*Vln5%AcWrakVMu*U+Jmb6w>2kLwx0HdcF*KudB<_l#<)(KomMGpS`(AO=nrQ9! zxf3~&k*W+zjBNzsuORV_Uw)@OB`Nh3dO^_j zncN`A^KknQZRzd0V7QNBK+bvKXnf9qukyUvK=SKfld7wz92r4NWDMa%;P13ysk z|B$xJG(!tTnyqUp$os%lp9Vqv1NnS~J}Q7k&1sFCOE*Mvu!7#xZn&v)LOJE9WiJM!Jl> z2$Hk}GUyhQEO73Uf`jK~V0Q1*d~0q0v>*3mi|wMjVTevS9Bqm)pzm^Z|1BB{1{gRy z=L`^u_MkO90a`iIGn)!)W;|q9#C5xl2$4u!b<%fTxJ3R^NET&~ z({=qmZU*^G=1^S-v2nyHuV)gBe9QmnuQ%aWoCjSQzIDp_Ip#{-%R#E1Sc5kLv3^LY zGx^x40@oCdZ*p^ytO#e=sX2u(ERg;5~sC<(C{*N++EEZ*9abR@i;L{of~m;U=oy@Mm99B(#7IMO(e zreEnwo0rq=b2)TgN}om{#z6=3f6!n9eKPI?@tRClMe?#v%{E}zh}ZTL4y z6{la2?*NHm(aBS)Tc?K|N{UOPcRRG%MiTU&k!yn1yeUmM(gOsMjAd;2ze9za#yaEN zp)=l3MNhHH9-T2&7@{j45-lNySFYz%hFSVzMSnzi3qiBj2|S_O6)o(SC!cb}W+`)}^130+~;@NJs9lO6LOe}Fr%%9lg19HLB44ap7YfuqCYJ1D2(& zlq-1Si;HMS?fNp>jYNar_2Y=yHRI^@;qZX(CyY82`gR+tOx5;%LaQi=B~_%jQpnPv zGY!KSO}rHolUd-e>4lcqXy4V(elD=W8HNgR9xJ8MA!{kX*FJ@p9e~_;zui(HxHSNT z>%R{2`lCA(sWiq9|77G8M?ITf6uA8W( zeLRsdj~@6tH~M}i%jV@sCIeYgNyFK*N7fLr$x`Wy^pkB*(4% zVBk@l$zA6Z&zF5X)s2t+Jy1m#pmy5#tXr_*1I(Ihzp6+TzNs~ukA4J)&qI?0KmAr4 z?nBJdwNpensuYJ$V%DzoJ^|_a{t*HHLB2*}U>h4j=CCrbz`x|8?el;O)cqx6%z!ef z#V$Pkzd$7)O>TKZ`;;(w6yAx)9G#{(J$(2N zhjuKs_o-B^{m2-5XKUVL|Lz#~1IHBdkTRq#1pa_uJ}RG7HIBz&Xl~YBCsp{+U%l1Z zoBKss)x|nWcGTxNlILtJ$xXl3_r{iXr6{dnm$*h~@iF2&AQEE`v2dMlDkIJj0Fxe0our_l0u?5(y7OmZ}QybYIG)(wUF!Be% znn}f1oS9g6Z$myLn#cK_KSe76+do+N|T|ue;q^nL+(cQK314@ApOmF~Io*W0q*IObCOU1>Nc%#7fb}vE0=rV@eclw)IVHmmcn&P`~S4<#iO;$;$@jXdQ-YKk!Sb3uKA$&HOYux)Q(s?Vk z$e(A*&pp|kQ3F56xS?r3Zl)^3OZ6LR9`PD({~#6htEl(ie%X&hNdM}L0LBlxkGtDH zKpA6wmG}Bu4w*NWzxK#oR8 z6BKx0iUdG?%s>SBz@z$M4zpFb$mU2tvY^}NXqDf%QF?XVM{jmyF&H|5Mn&Mv#ya>*O?~BpA z2o0jMsn=w#IpQb+EN-X%bVRFulM%wN`M+9BHnj$ET8YNt2m?qwc_{r4C5DiM?B;Yg z5Y{cYcF=;Fok(p*0G@3KgpQPb5-j<0Hk;T1=AZ2Ya}dD#S~8}Jsbw8T@>UwDm^c&_ zIihpB(tkyMrrUHBTMemXWHK+QaEFfhu9$QgHzD3eP5)Ja&x>o~Ntu=IFO}I9Zyhq^ z@NJ4e>%cMOK4G7mf#cvtQO*1cED@MsvBkP(adfCNirS%a25qg>wDZ-`@`K36bzW=i`@^Qpx@l^dek zmC;~c&qAE_Q-_RI*<|vChv_f*uqzUMhMi3Qs`Wf|E@K#QId)q&DMbQu5i)auGHV!5 z@ou%oZnHI)uj(dZ(bo+}`dngA04kQ&@*4KMkvix?qZLue)2g&p;EZ<1X?RzndQu@r z`Iw5E9Xh(YT@?N?%fcMRv_{UUqmofVn*4M@)2hc}oxvYx*inrahpH7Fw>7>L94&f0 z-l(lD~pSvR*4pC1uB>)kP33GjC_lLVk4gl(Kk!!S(KuI38i^UO%L@ z!Kq}KTXK!p72q}NdDpyNoh-cFOlucm}3c>8GVCN^`{2Xe4hs7q({ zlLzYFY?tRxApJy47o07h)O;@^n-);%x}uo!&|yOw&OqOuc=$u;>i)OMqi~SbBq<{; zA=Qc6oj%Usi0hdTD7PmULITA{?-Z>I@UzFGyJN#y=~!lk-o0m&lpF*9c+bPjCBZ4` zRxbb>ruTJV)9Iq?X)QYZGtu3_ty1N~*GEouKU`opuNcd-AQuv$e zR`2VsI)X3`M)%{KF^l9t=Ouaz0C|P}_t0c!r&8M-$-8xIAjvyrmLJ@1*oOFyGfeKt z8!5X=$H0}Hml!@lAx89hSxALh3q>WU^zKsZMr;?GzGfJ&+@zk1R!p40@``T}RH@WI z#P4Ev2I{!i)-l0PI@AT+au?GK1)Di7vaw=GkGi@g&@2vcBZ!JVylYq+w|T=-Sy(b4 zTrF#Gy9(02{r|#8pW5@TLOStCg@Z`v2VTcHH3p7l&Zk3MN! z8_!2PZ>&1E6NZVcLbiMwEr${R;u~Y7LZvS{GGA3s$$Lj$1h={7q`6L+`W z3B1b#*Tv9QhhvdPBuO3cu=#7g{=odNLX+4pTXJl|3V(txm1PSbk)4L;Agc>ojh%!I zA_1g6NhXD#YNB1fY(Iod!c7TSp<))t1#f>uYb_)GKMI~xSk|NG*)kd1JhXfrOR)M7 zW%k7c?aOang&+lFLK+{2BPe!8wnhP2tx6)`B(%RVyigoD3xs(~;}&m=`L*Lm3nrYL zK+vpY>01zZa%BABY2ehy>^G$Js*U@GnKLaqm#OB^2VUPy^_aD{f&;xG1GU5wwKB_V)hcBV#gp_0|nsh~Gs`8Kg38UkrXXTmku?Mo0 znb-S#$KFSVXSmwHDKk&~whhzJ6i(iFB0`jcVVlwmYq&ehP?${EBKdI`$JH2iAOecF z$g`2{TA|DE#aCT2yK?98LP814ftMbIA~{R9W%Z48!Ybba9dlRpC!4b{>>H$+Y!xr^ z+V-geM|Smk513B9e&5MrygSf3rj??q*HZ9MT}mA=hq%aHMn%U7muZn(rr@cD{k!9#d zM@K783Yjm9;ETRzgE^4*if;54FCWLDfA#Sfh@v+Ptz(HqSO!W0h^tCr4wh_(0Dq1v zgG~gj+D$A{7H{~p2m?*#11drO!5KY{$4!u0FoWou4)(j}AQTJ_^-7x;=;1wav!S~e zCcNph0}beNU`x0(*YMwI4^pZso*r#8c&Ah$bSQ|2_54TjyBJbRvA4@TEPFo2{8W09 zT1T9e(4v{~*Z)i;UsKc?I!qg#I6T$mg60$j>k4l(wq&@aH-}$tyDVI09X5`9FExC_ z-stj^{7unI)r36DlnV@3Pp%e#V)?`#bFWR&pXB{9t8>kUr3W{DmNSOMJ#VFD{`{FX zf}DuJZGK_)yrO4+fbvg85!Yjt?lzoma{UTz!Nr!u6}I_jY5uH}EwrzLeBjN1am5}v zW{L~3@>qN+o4Iiw(bfRSUrFtm!m$$mdZ=r5TN_Qpg>`Jh*Io?=PF=^&LO>^5DZ`f;X zA68@2k>3{kij=Ri3X}#&RJGuqd7~pl9Furt+b`?<{G}Jh2*MzoA7x3*=$J)Sb_WI~k-Y zk@m!Vf)nnFeLt@~0OunK`|wl)Qpg7+4p2rKQ4@RJx9yH}*0qxAyx@$ot98(J4x=6! z^5iUte;{!iZ7$@|n;}ehC;r!pwWECugI9+_D-6@SJu6buN2%^kbG94;dFmp!X7648 z5?C8>(GO20R{$oHv8PeVb6skEshgAnI+MM>vd(Ewr>N5k)WwaQL(Ge?-2b^GSiTos zaV)K}OKVd#x9VGFz7S?P^`mAwD)O9u4Ovx~TytbaHh!#TcW_dvE%qP1ba-jS!EL_k z{(W8v6T5&bE-atcW9p{9T~y~&5v*&md>#EF78O-OYq2mHQ-Fk}3~kk^c^0t4kD#$m zvwe@mJ>^6SQSJom&0yn?(kjeW%gen$#&|erp))8`Tq2^lY@^EkIAmN0C)l9+7^W(= zdDu{Il8|Flb8*V!bW=@NLEF+x)uc^Z3ZK^Dxl^qc+V|?;B}5*&S+{y-ym6ZRwoM?M zwVjdxF->KlgQ5aF zoT(%NF@${SRK}#yh53o@IC8I<#M?Dtr$@wvItW3eSbzZ!I4+wtOk+D$Htsl@p(PoO zh)mo{%zd<0i>(jXa``AL6UQz)S$=$_C!`J=!ZpUf+*?}t*fe~yO8XTu(|nw1wt|EFN?4dVvP)Ea?i!-`(T4+I@C z1ROJ!56fyCKrd!aqBXe6bM1+$=^VNTW&~na;HPkT*vN++;gxm z=Xh4s1P=oKy_yhaoC`TROa!CR9-(MXP7?^;6|y!Q1Z)^VpnIeApX#i63FA%uxq zUCnF3^$&mwYjW4?3gW%sXenZoxrA#i{3HI;GFRp=oG=0g!EgUaN1u-~9ujr3p_;{i z@0jC}dY*)cB3mxti~a*;Kh8??fD$2>pa#vYU>=-n=RY0fKF%oM4qrfbPY%*)_4I;||*9TNZdAz8qO30$O9>*-(N>ZlAuSP{pPv4rk0)uv7jGQcla z%C?@!5JR>bbZ4tB5Zx!4&Eq-FrJYw|lY5F;T8ogOfmA{$5j4M@9Kk=u1BJe9dfLjo z87|Pec8}|i9gQ#NIzRX3xJ;w<9oQ07NhrFb#POY@dwi=aH2gfnkPm#1wrB$j?X!Dg zY;tO!t=%ooBV}j+s?aKukPXBUEZy_39zCQ!G;n+bD~*N(-C1Pc6U~@-;H#S^5>yq+ zfnXRN*EA+{^-}kCvjS+Szu3!2jd)+oQ>b4I)7Sy9)m?ikawgZbbzmsuws?L|z&XvL zQ}C1MwP@;2 z9wp2KE^XtD6=`rzb4=`y>V-4Q#({+RG-I~T@{^mO6<1>KnoJ;>l?%!=QAqdqM5Wo) z2IyN~MX9y&an`Fo$b8(#11~v2eS=y*8wA9^M77rNhfH77N}}EUK3|bX4{f=OqKhuG&V2;p)@Nv?#Nhw_Ajm?7M@SB z;GV|Uk0Z!Du?Ho!3o9K8HxEB+pi7Pyb8jou4Ia77SMW9)fI?-b@Mo>+J!BJ%8{7l?*APU;%g|euGg}k4W8RGjiC1a#Pm^O14 zWyVQ9Xy4-_PM>8s$vvjIzHHt47U6v!75TrLU@L=-UGaqGq6H9OFt#{0bPVhSZcx2j zW3J+u(B8gyUb1d7!+7!og2vxs^1;S89c^ERkeUYDv)Eci^++`omB9xdUiJ?+<#LPZ zHJHX45PP+R7VQq7fU02^FhWMI?RooSzMHwfz(<+~I7!`frD`cu;o_0O7$gJu@05U3| z%gi?_O8j1r5Lo9%3I!?b&vA&+J%nHE$AIZOvK_7HR!mXIacl32w66J*964@Yr;X`u zR2e3v5PH10d8hrA7?w=sG0Vp!RIJu-W)N%)R&&fs3h({8s95e@6V+*-`uzR?S2M+C z6p7(is?dDO0KJ0$e*Gs)mRc1=05S~ZgSUAVf7}e`q>8L-)1+x8_OT5j+|XKjEH9FA z;>A2c*zMO%S=r0r%}X96-SmX2jhYt4swrkMLgg%7kk+5omjz3@mHXLfxDjtNxTaEt z18~qWb-)C?F8tpbwt%^HzQ&s9XI271$tPE)2PH(=nn2yQc(?NDUk)E3+wy>D%LzIa z`Hk;Vpiw#+Riz$BN77oYkt1coE_YBX?KwFG!?saP_~j4wgVVjyQYG_H=R2g5{L=g4v*-_R|CLpZV}JqJEz731h{rXufsb*6{+)7b(CJoj)^h zJ0=wO+>OBm4T63uBfsT`@>MtRKc_m2BLrSUGWn;ZJZkYVHbHH>}9BoarOfVgz|y zkM+!M1bLJZ&%nuoTl~K+D&Um<$NF2`02PYKD8*N%jWiir>t?}QNT$BxfoR!OL2J{c zT{^?(ZOLb;Q^6Ayyudk)THe-Hv!kqzQZq9RLT{JIjH-#$5IZYMP;#~g+9~n)%j&@# z{HZ%dFacNuWCrXFT3=1CF~0nbw11QvVm{;0r#gU24oTAZ`B|E(cWnF-Rc@$k2IaV7;uRk3mm8-OfS7eAF=2#(i8b zJHrtfZ~6yf|3P)E(zyD}5e7@b!lwJ$K99(o5KD4|S`sTyjeg47th!NA`tHS$fym>` z8>2Kb0oqNy_4H)2>hgzZ4)-}_^P{d+mOohk=wq->x?K;~IK!;%8!VERIPtqoNQhfs zuWXdEUhos>>CkXr&&Q~JvCjCH7dh`)3D-3-fofRKbh&66qZ$0ZS25ez37Ly_P&S&F z=t*6)0}andO}WJU`F@o51*8wpOMdITW$-=#!2k#lAV7w$pJjHh1$uw7VnV=YqC5z_S5*X*%nQ%AY=p*^=l-<4K_@WWwyzw4~f6n};az?yxk? zxuS14yu>F-*9(F88s^^;hKMRyzbtKiVH8@LH`Lr6QRyX3>O$t}x&meZz_9{Lh#<=S zbSEln8I#BzQu)K&KNz0hwZhs3$oYM4jQqJF5m>a=hibdWD-$EgIep7M8U4rojqlS> zG-=xqGS#7!MV54-o_&6i{b>@zBT5lhXAp-;>~PJdQq8DdU)WBAK2X}5-!y=`mmMjF zp0{8DPyg63%^OCR}?IH8C69`sZv3AXE>a)v8YBO46yl4McUq2q7qBS zA$1iI3#BmX>IqS|+Q8J8D1o(vTVe^0vpmbJAuf6fC29M5k? zq)3R!d(CpLD&0y06z4P7wDuU?cjceVYXEVMOXbM!c=VsdD}^flDyt?WMfl!z7%m#} zrPRx$!Q8noT&AC~xYD+D^?xkC!$zxIiKxk8Paa2AGcTW^T4yk(xvrwGTgj4u$JK$Q zrqdVF^!h>!`(Ae*az76^+f~M_{Jharpyr?P$d}^=(&%Jg-kX1>RQ087rT@cQ-AA#> zU)=if*Gz5Sjl>*X_@bQZn38rPmCCj1{8Uy~hi9+1ivVIMu{&X;qvTX^xJ&f#(Gtn0 z;$<3-Hz)9Um`ow|X()i+GHTYR*JI1E6eZYbMTAeBc%W+cs|p>1*Bv&I-=;b3s(k$e#>v> z16rC8Wa`H(|19JkNWa$V6^v(8a z^PI@M-E;MmBA-QLqtl4eT{J(&5;?xLAa{VlsGc!&7?B(vmtXZm&NkM_^1w-_JcCqO zZn8Xlq}Wb<t%rF+7}Yb~>sgz9hbVx)@bL_vRtNoK z3T#4@NUL=K<@j`GV{>tixFMU6IcmUnqt0`5=IfUW2ysPzvJ{kBaPY52AV$h~%t zZ>H#h@#G|j=hDAP)Fxe=8KA+d@mA}V(q^)ZCtjyum#HFq^a#WI^;-LmkYo<`6&~TD zmPI^698bu^1zRl`XE`!mCE(+)aytT(B`m@jsRGI}w{!^Xpo<%8d^UQ~*?$oCgVh1WT&k=*|pd9d<9H}Ip*H!vs0 z6gu1apbrQnHq4zF_quM)Y|>a1VO&?tMSboKEs>+h>5PH~7i zp?9%EZ%Gcb2Fl%HrPh*Mp3_yCm=^%(&0Q6+%(L#34Yl0KOFW=3vlM#Zo(h;O>!g!% zGa31?jP#y!LRZdjpQ7fvsV2%JZ$yc$b^X=&!hdlKHfSn;#l$x9d9I6hKbF$}kJ@4C zH7FUbs^K4hRns zP@$T_fxxL?BGJ{Yr8q7%v1s-!T+9!9x*UD6q;y-d6Hg=SNOzZI!!!-F`ldA~WS$|D z@LMuFm)m!Z4#JO-eCnocmB{TJ#;Ha>+NvWJ+!n=y#_iVAy&E}IYnc&O`679-DBK+L z922J7^iAz{Scf``_mo1l>meOecho~<$b?EHO~yQChr9by3cwwFFtP8|ExxvLqv4{N z{Gy{vGQdKdpkOWj6xa7Gn!D=E#bU&=QzHQSiNOob|H8{NiTB;Nq%BonVxM75grez& z_*_DEcU~L7p@x&`v@k$z_L$5CcX(*=-3V1}6$;{K1sO6=mj+LyF%ZHsfs1Q+IkAAT zQX;F~1Ps{MTZ4ZlbLw&_(15jHhEzmr6N z#bvCa6E&B@lu2ynaIKoi(bM(>Ba$xtVl*8yxF87gwA|`B$=g-l94a3?Qt7ub-UZgk zqM|z8@-=<;2!uEIcZCVi$C2n6tIAZl)`{R>PSOPbMHjRhYz2r385wrKheo zw8b$AvsPjfFLPhr>lZFC&$T{6Y&;>ihG#4tvmCG(k^_pT7OK*5J(_xW+=(;xV?@|G zyisP8S7z3V-ZcJyv-=et8(Dn1EXUmmv)+Dg`CWp&&t-`Oz@uWRc~sdU@g+C2Q=kb6 zaXLal7Jm|5${FfaKtQFwH zM+uqJq*WR8I>G-@f$n9XCRjVv#@}58@&l!8DP`jC%K(<`v1bp&SI@lm+VX9YR63@- zW_ll1bs(JSf!Ki6VT;LO@O2D}>7=ICPJR6NIlScj75yAO8Dn|YIUnM|-m7ca6xztK zBwlt>D&}7=>LBM`Deg%Fz(L6Ns8ERa>=?)=Z6ycsQ(Ci#^^sfs|60XX@Dw=~6@=-!#hvE~`O{XN~1kX2mqL1*NMMN_ybA3%J(Buh3JX5A~LC z{BEvRA`8tGC@&PeS~*+MMLHwvp^Y7d{4?E@T&_1E1k^U9AiJZ*`|%LeEsS``hV3|9 z*jIljz766qCHO*nHAB&7`kk?iay`6jBK`-$IzNlIJmk=GKp^tTTQm_6qN@9aVwlUW z&G68IB^tsmbFw(`valR5&IX&C{&LCxegD*DR}@XrJG>rchQuZ&OrXw(>PHNQHY4Z; zi9?N^)QxM*RU6l4$^SQSv62oPAH)^}b0&Mn+IPz%J!ELfB%lM@CP6RsJJ|@EQ+U7l z%08VU=#-jh{^Y9)Rtp8E;8FKGw=d8fY756g{_V;SonZLfa%Cr{3{YNr;do|R&YTb4 z1`TqtuU>>Zz0VWP{W>C4ux@vzn_pbMT-l-2?P*VTafrWa?ImH1JLp6(MSreqD0mM+ z0T&#G!XSXN|6~tun{C#w)3$cK(lnBvu(gM&Vw!#Aod6V5Si0-K(fW{FlJgDNz5;(J zfQv+^W`3VbK#tD6v@(0k6+>6PRSm)N7fL;+Cg+KTEvTEhS40r-(>yBTv1yg~O5EWg z2sioupNkO&s0OS>=7|2WroW1pwvyM$1emyELcsB{Iv@QukgjSmp>R-UH&!{rH+bs-chDY&8iDP*V~ zk1s?2s<`BQNsz^b_G&5HT8Qv%Uq5kO!l{KRs{aYX0Y;PlD{r@F>B&>58`LI(r{_`_ z6WZC9lH<#4-KoBQ5SGUrzDvAkZ{N0Rht=u6sO z`5eK2yA{pOj9*qY0v7UwOiqw%`9;Uj0eRO^*<*@WYxF+ebwZNL3fd&&<7Cv2I?C(72~0J)Q&l^mSIhpN}%>T8^> z6)e?`_wgA>cJ`#&zi#I5D+3UTS2S`gc&^-@%Db#kbInXSS4#3gF_R%l%* zS)%;p3RJx=bfu9B5$t$%I_yS=sT$t~F?MeHLt=^4O;eQ{dMovNU9axwC3f$BQEq?Vz zb1AKHgM$jvRuRa3Je;?EJJ+<&0NZcgGU67libhCcExzWv4W7vZwkt{hiqBd1#!X2; z%x^n_S?x0{vTWjN?_9DIADTu^owN4>Qi_}j)J&Fo zS)*;_MpBDWdE-OpP_^b7va1pR&>AZD%AqH68n-XAIVbEge3CE3IB^;mi9A~yHqcjG zw|>YL!FuIS?QQZ-kHj;2C_u=)@!~E>PjD=aGtsQ_eyl5#*WK4vw;F7Y&i2e9ygq5C zJrP_YuqO(VwdTAvf5TYe5T05CS0cB!?PAW<(%1G4295yK=u!nc_;&6KShJ8sU~Z6l z3M&$)$N;ZQ8H?e+M}}7vtS+2K6Q3Ed?|c-hcw_Ou>#uv#NBvc4{f;`}@O$MCcy5P{4IP$o zc|QEw|6ldXB6pCzNrBsdw?k+tUm;;cohB*gcQ0gDIV4Gmj%S0~=+0rN!2FUja`aY+ zAh)iF{wjvzu2usLfaurL*8gTcyXq%a(1amvZO|lB&?(B3rCc*@@DV*B)gP}dkCO$^ zCIKm1@6vx)QYr(*@#NkZ8IbYDzO=&sr7FfGq)ZnRc?`BXLPlOUx*!}{uS5WpZX?ZG zP446~FwAWZE5xZ}@BwiHAf~M9a%d__!8P{2p~jlT3}KWkX5O-?MwTaqAtNgDdM?pm zS4&N%Ajl<0|AhUysJ528<*8e617QCiTN&dCVZv9Yu-hb?mM zV|j%|U`Dr#4ih5r36F0yrV0FibHub`O|Ut^1#H~h3ahkp2d%b61G_wLagMl_Y1zzY z%Nx&hc+)NaF3+y>2|b+ujyeq&hzC3$EFm^6yB1i>Wlc;cpeN5-3`Gxodu$Pg;Ae7o z-6uGxE49kOmwhOz1L93c`hm={5tN8+Y}}pUfY4Gq>H<)T{GY%l6HN%2YoF!Atv4fn zs@9S`vIXvyLTC`U_DiHH%Pv(+hSuGq_pHJ+BMk! z66ccFpq_<=CC^#v=RbBbvlJ;Ouhz1uO{DT4_n%!|#Wz%ELrEiULDY|(1DMnQ*ZlVl zv{kRN9lYv*`|zF?-(*=}_p-zIN&pM4(KyE#*;9Q1N#XV)pA-!Ddw|&!i7Gl}ABj$a z@=45Aie*y))n-jUHKPSf;jUzS1334&WH^6E1UMi-fbRfTEb*5DgnwCVWt*#6yTyY1 z;N9S{w3!qfH@tRN0ztoTyiiiQ@CKZ?6@x6-pR;0c2l!`f* zCpO~1?K^_0MTaQ6fHv7KGM-vR$bI}CYW+_DcCS+KTKtYSbw|xjW&6uhZ#ugFFY?kG zUsy(JJqytZ9_o~_z-LekZ((&?4I?5L13PiKRA>^^wN(RPyYT1!`5umHevYHX`Y{kK z`5@APqY{yn{49!;WQVpZaq+XqZY|XAYo8fNw~?us0001i0tGMGJk>dcvUkfqNC*R$ zIG{DrsXgbBz$n*!eTMUfe+kq(uc-zESBDDu{){Mf5e}d>oB1tq@vV6dxdPLRU7aE% zT9s^M?U(C2uQ0*Nna}gW^|=}K2QsXCvQu>1TSd31oc|H=xnSbGHTlxLmTD7{;|~@5 zk5GXwxVJD+X(pIGhf)bvS}vq%CS#P(B5`!P6D8)BYdIfG>LKFdmftjg zw!P+dCxKITjfW49YT_Sh3;4p8U(Ye46phzYYj za=O$r2Tsw^#GA8?o>v--onQc{cr*r1e!vAs8a>XoxX-Q%Q#U(sY@kAz@ZsX4X>v-yF1!i7d1^5>~m7IU2XZ z{T4)qEA1p&Qkcx(yx&%cIS)f|pkeEtP&#kj7orFCkuw=XhBu@DC7h0d zRIBT4ezzk=^+yP2ONqj}wx&bQr^ls#FE_t)x=@KT>)lmbip;XQb?b4wQokIgt}w)` zn_L$p*j~vPYyhWXTpq)|TjqqO46MVj_&zwX<}Ctp=?kIWvoWUg)~Us4$K`fELv2(ddm#|pN;is z<(eYBgsRnXQ-qJqTwuiH(uMM6QCSMY>GcPDKOP*QMjvS!a^0K0+(LJgWQ_Ww74GV+ zuJ@NdP{G$O&(=UZyKuY982=pBKV@~4ySauD?!;=~%d|A=ZwCrl53yI%QWbh~`Sy;N zu;A$E67k|XLldB2`DL0b`ZD)gM<}D9Bc<3ZY6Ypa&W&0s>+?H5XAddO!=st5Vg-UI za87)OtrP98Nh5kT$TYDaJI@?J*zPdZYU3$7&6PAtuMD#Aso>Ie)tP54m76gge%DIR zh6c2l&{+C4Lm_&+v)6Ke)+q@9Fx32q&A=qXW8-d=Df82AZQW6zpMM~q7qJ4?ok*hA zh2B9kR!$iQ;fbzAb_)CjZ#TyDf;frDHwT@y&@F=NI#xhuu;KjJ6!Y5%L0TEt2!IQH zbumB6J4A=aRSft54tD%t^Iu+4-W^fSIy2D&%}m}h=Stb^h{c&`7e~Qyjm_tCFyL1a zf&F`{_+X|@bl8H6y3q@i=x94m9mLWuJ)0VdsM%)U!V63eB6T8cptp}sV=ax6@N`&7 z+TU=lQt;WKh>>Z%=BLh@+qB;fAoRmcn#x|}4oG{Wu1^0G!jYn(Z}Kox}m8(mO! zYV-Tt4jRXI?0{}^j-#dTJUX1*6Y=Db7#+iWJx1+4OqlJ>ECpfmEM|2|zvS-z z!QBtw!+eo@NOdOqX^4Hsj~z7e)xD*qe&ciIAqf}O%=KTT)4+*+??C?(-Pjqy9zi1KG=FQ7l5mvL+g-7pqbuPtvPXWCj3i0U_O?)#Uia1UAy#TXq>XYhNIJ zRht0?E2q?CX`!WtIdD?zADQ+g)Hk9+)BOP2G3wg?vE)`MslM6xDBwOzSG~&C-I_@8 zwE$7eqGn$Q9W8kEO`K4x=*!e(y0+*4-Q z=o=4Oi>47X3VAr1+I?ngFn`sx1CwTHQ>Uh~i7C+`wv`XV>5+h4h}Hs4qg-Fp2z7c# zFML>f=YK_4P0lN`5k_v=nLpFcOsDmC64Y{{-_u4d61O0*%BiY}31XWa;D(!DODJh? zwe$+Meg!_itom8!%t+8E$PrCbin!a}2O~kLBv`2-_k2oNSi61R*jUW~{Eaau55ap; zMw+byrw{ZqdkK~JqcEFk)*bKfP(%nMB%j!iQ0bXFx z-G0&^xtjEs-zMOVL9zz($UgpV@h$b!((-~Os7hng{ve%|VyhP%*Z}3%v`wy)NE5hs z&-&eEbqc-FGHR(De4x7vX}sEQcFzmVO=qoN7n{q(xOUMyK7aa*3W7ac)axZ9%L2u; zmyS-B7v?m~mlq9yoq&1jI*_wTfttl=xitk3p1x-5chIkOQ;^x>_*^?DidL5OyOWt6P&bJd2Y`QAN*{5I{bGqla08NTnR#Zvm=&1xx5A|k&R+L;zO z2jKpxUvM(F=v*0s8YkUu3@TcSL+p3zel7Lh(rqZhYsgG(PjRo4+Lwc(-Ae#lE0EzK zNf8mCZIceE7h1_}67=e*T=Kq8ojHqQZ|k@1UW9V>{XDc#&M+*}o{f6Yg+=!DqsX3?$g3apEB9Um(A zpjHH@HjG+3&Je(zGj?gjls1YunBhkjp!*zS9rALJ11`a7Uk%k%%ht2*Vcs33d!fwr z#F$srH^-7O%$kO)^PbZh#OAOEDl4{*%;hYFNNfv=D#}~i&3&FmC&?AvR|yf4dM>n5 z7oxjDHb1|5{Ok=o&XZBB#U>U93&J^eEitpyoHa)k4*j=5UPnJL)K4o#E0df{8$8#i z=-395O{xcysV3?wRJEL=*&rY{zMwAvIojiE`(A&;9@^J!b)nl8nu#GccFR_U|E`7k znSGicyf4};$zJ+CoM>iG!d-pO7R+|J1Jew9j@Ft5M1>wbU-!_+cd^nKYFbs8!Ufx_ ze6`KO0h}b(Dklk%TpvY3>HvWP1PBl#XIX>h$;lwV)^+>W0!dSU@4{rZj{BL`9$4GB z#n-P{2M!yBC`<>LVTT!cj6iV+`hCM*lhVtOEy6Ufk~6%PXQ*k%PfCs*#U@xhsKB;# zcHr6h$@A(bw2+=-vvZRfcNlFV3%fJpm?X{rdrjW%3}ZL}$Wl$D?%G)qW$=`A17Ly5 z#@CY}VaQ9A)>sxz;8+L|i8z06q8{Ctjq-n10c)_%d8E2-*sX3PeyJfEdV-jMaf)L| z=TY{j1&vwo`{)VyjfV)ir5&JZ^A$br7cJf#P?@vQc)NUv3E|J&_xS1bWLb^1dZQoF zGT?b)d&yL{PQAU1J!OfYeiW7uhv4vZPFJZSFc+~p3@BY#e+7o>A*SkD_ zHkw};T)O!G{Z}Fn{o}NJVt|umP_dtHEpZKF0o?b%+WB!R-QJB(I?N+Hn%E!qhB$~_ zu%qf}bo%1Xl8;1Xc5gwZy8B8l&(+IQtXBuXFpB|w16W#xsJtL@YN04}ndEZas`+kA zJX{zP*F?<3i~o;S00q{UbKYB-9m4xon{@9S%x!Q0Tp6j0r zKIw&zXEMUrXjP{%?uUs~_gj1EtJLzG#`RMFUR9W@mC6-JD%0K+R7kvNS{ z^FM*Ae7@7)D&9>uVM|3^dhcx`q?;GfI+1dv1UMc%Vij|9JB}q6b~u6U(cn zJH!o>Svo#SvV+Q{^uM;lUWZ`>2oNAZfmVK($@I1g01AQHixeeO*Q*t>!+N31Ph|Bx zGLu^twPg{Fy;*^cc@aY<@+%2+Ce_#)xgssQDo>BV1w*V- z(wleC=<-KC!5VanMvQ@OR<>*2EM8Ujq{5hxyKGwD1FFW0H&Jl=g$CH%z}mD7Yu(dv z?2E@s;|H(6(5tntnd#Vt4l&E-@91V4nYx=HX@fUWa_B8dBb<_h0YqKp8%M|n@%@39 zyo?$p`VvvxGRO||*!bi>pP6F<(jF~Um4}s~!)&&N$SS|I2TGiB*d|s;XxX|?GC`tM zuQb=MHv!)zuy1@-=uOje%Y8SQj+Vef1>*iy4`I@eqg&t@t9^lwQcCOk@{u z{|9a`=Z^5=1e2s2Z=h$d7CtpH6@G&+VZ9-md8{prEH*vI;M^y%)=gE4TPdt?&rcw4 zhX`ahwlX6J?J?&(xr{{0pmiA5mZPm?f&y8#5PE$*-PMGTvGoM@cQe|9a;L%MNxDGH zPItG>Np-(#^vbvO zYlkPNa&>c(=5}Q?L()|JtMBPnFnoSyD=J*2_@S`Qiju9l=kE+ZTVV+N^Z3Xx(^v)c z5WVqznmd(W+;tmQ62VK!J3k2qIZFsbAhIi~%bFMz5!+U`hjw<1C-76K!Acftr~L@= z_D5Y@Yn)aqA07kx#-{v8f~d&2n&!CuD)^OPgGu^YTVDori{ceo-4X2Yba!~bt<{<8 z>TMp5fJh4&P39#~7<(f<_lx%{Eh<;OYQM1@(<~|uA)4yU3Zl7ja{|%X%TtoMyYRpe z$(L@>daFcLhvldIS>r#cI06I!000000aUy?5J>D?Wl5R~m`n%HTUvVZJ{N*w8(Lh{ z^_#Ko~Xplcg?!2@UUQ z=1B;g&Dm9zy3=xYiF>)`U_;pefc-`yAIuk`d63N fn*wjQQLZMzvf=dzVx*Eu|Ha&qP81{s>dU*pKfkT7 literal 0 HcmV?d00001 From a2e3fd92b0e5fb20b70a0ab0db1d00bc81cc77c1 Mon Sep 17 00:00:00 2001 From: "Owain G. Ainsworth" Date: Wed, 29 May 2013 17:54:46 +0100 Subject: [PATCH 003/163] Only try to drop the uniquetx index if it exists. Solves a corner case after a crash. --- sqlite3/sqlite.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite3/sqlite.go b/sqlite3/sqlite.go index 2a152108..d8df4ae9 100644 --- a/sqlite3/sqlite.go +++ b/sqlite3/sqlite.go @@ -64,7 +64,7 @@ var txqueries []string = []string{ txtmpFetchLocationByShaStmt: "SELECT blockid, txoff, txlen FROM txtmp WHERE key = ?;", txMigrateCopy: "INSERT INTO tx (key, blockid, txoff, txlen, data) SELECT key, blockid, txoff, txlen, data FROM txtmp;", txMigrateClear: "DELETE from txtmp;", - txMigratePrep: "DROP index uniquetx;", + txMigratePrep: "DROP index IF EXISTS uniquetx;", txMigrateFinish: "CREATE UNIQUE INDEX IF NOT EXISTS uniquetx ON tx (key);", txMigrateCount: "SELECT COUNT(*) FROM txtmp;", txPragmaVacuumOn: "PRAGMA auto_vacuum = FULL;", From 184630782184135278dfa9979c40482043ee7715 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Wed, 29 May 2013 14:47:48 -0500 Subject: [PATCH 004/163] Use correct import path in btcdb tests. --- sqlite3/operational_test.go | 6 +++--- sqlite3/sqliteblock_test.go | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/sqlite3/operational_test.go b/sqlite3/operational_test.go index 81106cf7..864c2432 100644 --- a/sqlite3/operational_test.go +++ b/sqlite3/operational_test.go @@ -7,8 +7,8 @@ package sqlite3_test import ( "compress/bzip2" "encoding/binary" - "github.com/confomral/btcdb" - "github.com/confomral/btcdb/db_sqlite" + "github.com/conformal/btcdb" + "github.com/conformal/btcdb/sqlite3" "github.com/conformal/btcutil" "github.com/conformal/btcwire" "io" @@ -223,7 +223,7 @@ func testBackout(t *testing.T, mode int) { // db was closed at height 120, so no cleanup is possible. // reopen db - db, err = btcdb.NewDB("sqlite", dbname) + db, err = btcdb.OpenDB("sqlite", dbname) if err != nil { t.Errorf("Failed to open test database %v", err) return diff --git a/sqlite3/sqliteblock_test.go b/sqlite3/sqliteblock_test.go index df71463d..3fba73ba 100644 --- a/sqlite3/sqliteblock_test.go +++ b/sqlite3/sqliteblock_test.go @@ -8,7 +8,8 @@ import ( "bytes" "fmt" "github.com/conformal/btcdb" - "github.com/conformal/btcdb/db_sqlite" + "github.com/conformal/btcdb/sqlite3" + "github.com/conformal/btcutil" "github.com/conformal/btcwire" "github.com/conformal/seelog" "os" From 21b8011155d8148a85101e2adb587b4c49eb3e4a Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Wed, 29 May 2013 14:49:21 -0500 Subject: [PATCH 005/163] Remove log setup from tests as they don't need it. --- sqlite3/sqliteblock_test.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/sqlite3/sqliteblock_test.go b/sqlite3/sqliteblock_test.go index 3fba73ba..ec7a55fe 100644 --- a/sqlite3/sqliteblock_test.go +++ b/sqlite3/sqliteblock_test.go @@ -11,7 +11,6 @@ import ( "github.com/conformal/btcdb/sqlite3" "github.com/conformal/btcutil" "github.com/conformal/btcwire" - "github.com/conformal/seelog" "os" "testing" ) @@ -246,15 +245,6 @@ func testIterator(t *testing.T, db btcdb.Db, shas []btcwire.ShaHash, } func TestBdb(t *testing.T) { - log, err := seelog.LoggerFromWriterWithMinLevel(os.Stdout, - seelog.InfoLvl) - if err != nil { - t.Errorf("failed to create logger: %v", err) - return - } - defer log.Flush() - btcdb.UseLogger(log) - // Ignore db remove errors since it means we didn't have an old one. _ = os.Remove("tstdb1") db, err := btcdb.CreateDB("sqlite", "tstdb1") From 9bf708595d9a94ed5c2b68677b5febeeb7b26f0d Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Wed, 29 May 2013 15:08:07 -0500 Subject: [PATCH 006/163] Add InsertBlockData needed by tests to interface. The function is being added back in since the tests rely on it, but it is marked as DEPRECATED since it really should not be a part of the public generic db interface. --- db.go | 5 +++++ sqlite3/sqliteblock_test.go | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/db.go b/db.go index 226f3488..fe9fcb40 100644 --- a/db.go +++ b/db.go @@ -85,6 +85,11 @@ type Db interface { // into the database. InsertBlock(block *btcutil.Block) (blockid int64, err error) + // InsertBlockData stores a block hash and its associated data block with a + // previous sha of `prevSha' and a version of `pver'. This function is + // DEPRECATED and should not be used. + InsertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHash, pver uint32, buf []byte) (blockid int64, err error) + // InsertTx inserts a tx hash and its associated data into the database InsertTx(txsha *btcwire.ShaHash, blockidx int64, txoff int, txlen int, usedbuf []byte) (err error) diff --git a/sqlite3/sqliteblock_test.go b/sqlite3/sqliteblock_test.go index ec7a55fe..80622436 100644 --- a/sqlite3/sqliteblock_test.go +++ b/sqlite3/sqliteblock_test.go @@ -9,7 +9,6 @@ import ( "fmt" "github.com/conformal/btcdb" "github.com/conformal/btcdb/sqlite3" - "github.com/conformal/btcutil" "github.com/conformal/btcwire" "os" "testing" From e30ceb59475bd17e99776ec4325c4cd34c7313d6 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Wed, 29 May 2013 16:08:43 -0500 Subject: [PATCH 007/163] Cleanup and fix operational tests. This commit makes several changes to the operation tests as follows: - Ensure the database is closed between each operational mode test as otherwise the database is locked when trying to reopening it which makes it unusable - Make the insert tests error after a single failure since every test afterwards is guaranteed to fail as well - Remove some of the logging statements which cluttered up the failed test prints making it harder to see why a given test failed - Make a couple of the log messages more descriptive --- sqlite3/operational_test.go | 39 +++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/sqlite3/operational_test.go b/sqlite3/operational_test.go index 864c2432..e91161c2 100644 --- a/sqlite3/operational_test.go +++ b/sqlite3/operational_test.go @@ -49,6 +49,7 @@ func testOperationalMode(t *testing.T, mode int) { return } defer os.Remove(dbname) + defer db.Close() switch mode { case dbTmDefault: // default @@ -72,13 +73,16 @@ func testOperationalMode(t *testing.T, mode int) { testdatafile := filepath.Join("testdata", "blocks1-256.bz2") blocks, err := loadBlocks(t, testdatafile) + if err != nil { + t.Errorf("Unable to load blocks from test data for mode %v: %v", + mode, err) + return + } - var height = int64(1) err = nil - for ; height < int64(len(blocks)); height++ { - +out: + for height := int64(1); height < int64(len(blocks)); height++ { block := blocks[height] - if mode != dbTmNoVerify { // except for NoVerify which does not allow lookups check inputs mblock := block.MsgBlock() @@ -116,19 +120,20 @@ func testOperationalMode(t *testing.T, mode int) { for _, txe := range txlist { if txe.Err != nil { t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) + break out } } } - t.Logf("Inserting Block %v", height) newheight, err := db.InsertBlock(block) if err != nil { t.Errorf("failed to insert block %v err %v", height, err) + break out } if newheight != height { t.Errorf("height mismatch expect %v returned %v", height, newheight) - + break out } } @@ -165,6 +170,7 @@ func testBackout(t *testing.T, mode int) { return } defer os.Remove(dbname) + defer db.Close() switch mode { case dbTmDefault: // default @@ -186,22 +192,19 @@ func testBackout(t *testing.T, mode int) { testdatafile := filepath.Join("testdata", "blocks1-256.bz2") blocks, err := loadBlocks(t, testdatafile) - if len(blocks) < 120 { t.Errorf("test data too small") return } - var height = int64(1) err = nil - for ; height < int64(len(blocks)); height++ { - + for height := int64(1); height < int64(len(blocks)); height++ { if height == 100 { - t.Logf("sync") + t.Logf("Syncing at block height 100") db.Sync() } if height == 120 { - t.Logf("wha?") + t.Logf("Simulating unexpected application quit") // Simulate unexpected application quit db.RollbackClose() break @@ -209,14 +212,14 @@ func testBackout(t *testing.T, mode int) { block := blocks[height] - t.Logf("Inserting Block %v", height) newheight, err := db.InsertBlock(block) if err != nil { t.Errorf("failed to insert block %v err %v", height, err) + break } if newheight != height { t.Errorf("height mismatch expect %v returned %v", height, newheight) - + break } } @@ -228,6 +231,7 @@ func testBackout(t *testing.T, mode int) { t.Errorf("Failed to open test database %v", err) return } + defer db.Close() sha, err := blocks[99].Sha() if err != nil { @@ -249,12 +253,12 @@ func testBackout(t *testing.T, mode int) { _, err = db.FetchBlockBySha(sha) if err == nil { t.Errorf("loaded block 110 from db, failure expected") + return } block := blocks[110] mblock := block.MsgBlock() txsha, err := mblock.Transactions[0].TxSha(block.ProtocolVersion()) - t.Logf("txsha %v", txsha) _, _, _, err = db.FetchTxBySha(&txsha) _, err = db.FetchTxUsedBySha(&txsha) @@ -265,6 +269,7 @@ func testBackout(t *testing.T, mode int) { err = db.InsertTx(&txsha, 99, 1024, 1048, oldused) if err == nil { t.Errorf("dup insert of tx succeeded") + return } } @@ -294,14 +299,14 @@ func loadBlocks(t *testing.T, file string) (blocks []*btcutil.Block, err error) // block 0 isn't really there, put in nil blocks = append(blocks, block) - var height = int64(1) err = nil - for ; err == nil; height++ { + for height := int64(1); err == nil; height++ { var rintbuf uint32 err = binary.Read(dr, binary.LittleEndian, &rintbuf) if err == io.EOF { // hit end of file at expected offset: no warning height-- + err = nil break } if err != nil { From c97c8f17a43d3eebf8f431b0d51aeafdb33f38d0 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Wed, 29 May 2013 16:20:06 -0500 Subject: [PATCH 008/163] Add test coverage report and generate script. Both of these items were referenced in the README.md, but were not in the repository. --- cov_report.sh | 17 ++++++++++++++++ test_coverage.txt | 51 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 cov_report.sh create mode 100644 test_coverage.txt diff --git a/cov_report.sh b/cov_report.sh new file mode 100644 index 00000000..860fbc0d --- /dev/null +++ b/cov_report.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +# This script uses gocov to generate a test coverage report. +# The gocov tool my be obtained with the following command: +# go get github.com/axw/gocov/gocov +# +# It will be installed to $GOPATH/bin, so ensure that location is in your $PATH. + +# Check for gocov. +type gocov >/dev/null 2>&1 +if [ $? -ne 0 ]; then + echo >&2 "This script requires the gocov tool." + echo >&2 "You may obtain it with the following command:" + echo >&2 "go get github.com/axw/gocov/gocov" + exit 1 +fi +(cd sqlite3 && gocov test | gocov report) diff --git a/test_coverage.txt b/test_coverage.txt new file mode 100644 index 00000000..f1198084 --- /dev/null +++ b/test_coverage.txt @@ -0,0 +1,51 @@ + +github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.NewestSha 100.00% (25/25) +github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.endTx 100.00% (20/20) +github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.close 100.00% (9/9) +github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.txop 100.00% (7/7) +github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.fetchTxCache 100.00% (7/7) +github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteBlockIterator.Row 100.00% (7/7) +github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.FetchTxByShaList 100.00% (6/6) +github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.InvalidateTxCache 100.00% (5/5) +github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.InvalidateBlockCache 100.00% (5/5) +github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.Sync 100.00% (3/3) +github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.Close 100.00% (3/3) +github.com/conformal/btcdb/sqlite3/sqlitetx.go SqliteDb.FetchLocationBySha 100.00% (3/3) +github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.InsertBlockData 100.00% (3/3) +github.com/conformal/btcdb/sqlite3/sqlitetx.go SqliteDb.InsertTx 100.00% (3/3) +github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.FetchTxBufBySha 100.00% (2/2) +github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.InvalidateCache 100.00% (2/2) +github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteBlockIterator.Close 100.00% (2/2) +github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.FetchTxBySha 100.00% (2/2) +github.com/conformal/btcdb/sqlite3/sqlite.go OpenSqliteDB 100.00% (2/2) +github.com/conformal/btcdb/sqlite3/sqlite.go CreateSqliteDB 100.00% (2/2) +github.com/conformal/btcdb/sqlite3/sqlite.go init 100.00% (1/1) +github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteBlockIterator.NextRow 100.00% (1/1) +github.com/conformal/btcdb/sqlite3/sqlitetx.go SqliteDb.insertTx 96.77% (30/31) +github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.insertBlockCache 92.31% (12/13) +github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.insertTxCache 91.67% (11/12) +github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.FetchBlockShaByIdx 91.67% (11/12) +github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.rePlayTransaction 90.00% (18/20) +github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.NewIterateBlocks 88.24% (15/17) +github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.RollbackClose 87.50% (7/8) +github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.fetchSha 85.71% (12/14) +github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.ExistsSha 85.71% (6/7) +github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.FetchIdxRange 84.62% (22/26) +github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.insertBlockData 84.00% (21/25) +github.com/conformal/btcdb/sqlite3/sqlitetx.go SqliteDb.fetchLocationBySha 84.00% (21/25) +github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.FetchBlockBySha 83.33% (10/12) +github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.fetchBlockCache 83.33% (5/6) +github.com/conformal/btcdb/sqlite3/sqlitetx.go SqliteDb.FetchTxUsedBySha 82.61% (19/23) +github.com/conformal/btcdb/sqlite3/sqliteblock.go insertGenesisBlock 81.82% (9/11) +github.com/conformal/btcdb/sqlite3/sqlite.go newOrCreateSqliteDB 79.25% (42/53) +github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.startTx 78.57% (11/14) +github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.blkExistsSha 77.78% (7/9) +github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.FetchTxAllBySha 77.14% (27/35) +github.com/conformal/btcdb/sqlite3/sqlitetx.go SqliteDb.migrateTmpTable 76.67% (46/60) +github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.SetDBInsertMode 74.07% (20/27) +github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.syncPoint 71.43% (5/7) +github.com/conformal/btcdb/sqlite3/sqlite.go createDB 70.59% (12/17) +github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.DropAfterBlockBySha 68.57% (24/35) +github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.InsertBlock 46.00% (23/50) +github.com/conformal/btcdb/sqlite3 ----------------------------- 82.15% (566/689) + From f6c3d6d57afecdee8de6c150b4ce7c77c3082ed6 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Wed, 29 May 2013 16:36:45 -0500 Subject: [PATCH 009/163] Fix typo in README.md. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ac09b421..689861e8 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Package btcdb provides a database interface for the bitcoin block chain and transactions. There is a test suite which is aiming to reach 100% code coverage coverage. See `test_coverage.txt` for the current coverage (using gocov). On a UNIX-like OS, the script `cov_report.sh` can be used to generate the report. -Package btcjson is licensed under the liberal ISC license. +Package btcdb is licensed under the liberal ISC license. ## Sample Use From 6ac2a9b934029e4ba4c74298e80a3e7e547348e4 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Wed, 29 May 2013 16:41:51 -0500 Subject: [PATCH 010/163] Add note to README.md that Db interface may change. --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 689861e8..3a6c287d 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,9 @@ coverage. See `test_coverage.txt` for the current coverage (using gocov). On a UNIX-like OS, the script `cov_report.sh` can be used to generate the report. Package btcdb is licensed under the liberal ISC license. +Note that the main `Db` interface is not yet finalized, so it is subject to +change. + ## Sample Use ```Go From b4e38662ba9d8e57c37a1192c237d86efdce9fa6 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Wed, 29 May 2013 16:43:05 -0500 Subject: [PATCH 011/163] Update README.md. --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3a6c287d..9e7e3ec6 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,11 @@ btcdb ===== Package btcdb provides a database interface for the bitcoin block chain and -transactions. There is a test suite which is aiming to reach 100% code coverage -coverage. See `test_coverage.txt` for the current coverage (using gocov). On a -UNIX-like OS, the script `cov_report.sh` can be used to generate the report. -Package btcdb is licensed under the liberal ISC license. +transactions. There is a test suite with a high percentage of code coverage +coverage. See `test_coverage.txt` for the current coverage (using gocov). +Alternatively, if you are running a POSIX OS, you can run the cov_report.sh +script for a real-time report. Package btcdb is licensed under the liberal +ISC license. Note that the main `Db` interface is not yet finalized, so it is subject to change. From e76c83b27fc829f65d29bd6d511a7a57c93f1bc2 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Wed, 29 May 2013 18:11:27 -0500 Subject: [PATCH 012/163] Fix typo in README.md. --- README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9e7e3ec6..7dafbbc4 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,10 @@ btcdb ===== Package btcdb provides a database interface for the bitcoin block chain and -transactions. There is a test suite with a high percentage of code coverage -coverage. See `test_coverage.txt` for the current coverage (using gocov). -Alternatively, if you are running a POSIX OS, you can run the cov_report.sh -script for a real-time report. Package btcdb is licensed under the liberal -ISC license. +transactions. There is a test suite with a high percentage of code coverage. +See `test_coverage.txt` for the current coverage (using gocov). Alternatively, +if you are running a POSIX OS, you can run the cov_report.sh script for a +real-time report. Package btcdb is licensed under the liberal ISC license. Note that the main `Db` interface is not yet finalized, so it is subject to change. From 684582cde6fd07fe9655f07b2b9587462ffc59e4 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Thu, 30 May 2013 17:18:13 -0500 Subject: [PATCH 013/163] Cleanup the Db interface comments a bit. --- db.go | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/db.go b/db.go index fe9fcb40..f0f541cd 100644 --- a/db.go +++ b/db.go @@ -54,43 +54,44 @@ type Db interface { // cache the underlying object if desired. FetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) - // FetchBlockShaByIdx returns a block sha based on its height in the + // FetchBlockShaByIdx returns a block hash based on its height in the // blockchain. FetchBlockShaByIdx(blkid int64) (sha *btcwire.ShaHash, err error) - // FetchIdxRange looks up a range of block by the start and ending ids. - // Fetch is inclusive of the start id and exclusive of the ending id. If - // the special id `AllShas' is provided as endid then FetchIdxRange will - // fetch all shas from startid until no more shas are present. + // FetchIdxRange looks up a range of blocks by the start and ending + // heights. Fetch is inclusive of the start height and exclusive of the + // ending height. To fetch all hashes from the start height until no + // more are present, use the special id `AllShas'. FetchIdxRange(startid, endid int64) (rshalist []btcwire.ShaHash, err error) - // FetchTxAllBySha returns several pieces of data regarding the given sha. + // FetchTxAllBySha returns a FetchTxAllBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rtxbuf []byte, rpver uint32, rblksha *btcwire.ShaHash, err error) // FetchTxBufBySha returns the raw bytes and associated protocol version - // for the transaction with the requested sha. + // for the transaction with the requested hash. FetchTxBufBySha(txsha *btcwire.ShaHash) (txbuf []byte, rpver uint32, err error) - // FetchTxBySha returns some data for the given Tx Sha. + // FetchTxBySha returns some data for the given transaction hash. FetchTxBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rpver uint32, blksha *btcwire.ShaHash, err error) - // FetchTxByShaList returns a TxListReply given an array of ShaHash, look up the transactions - // and return them in a TxListReply array. + // FetchTxByShaList returns a TxListReply given an array of transaction + // hashes. FetchTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply - // FetchTxUsedBySha returns the used/spent buffer for a given transaction. + // FetchTxUsedBySha returns the used/spent buffer for a given + // transaction hash. FetchTxUsedBySha(txsha *btcwire.ShaHash) (spentbuf []byte, err error) - // InsertBlock inserts the block data and transaction data from a block + // InsertBlock inserts raw block and transaction data from a block // into the database. InsertBlock(block *btcutil.Block) (blockid int64, err error) - // InsertBlockData stores a block hash and its associated data block with a - // previous sha of `prevSha' and a version of `pver'. This function is - // DEPRECATED and should not be used. + // InsertBlockData stores a block hash and its associated data block + // with the given previous hash and protocol version into the database. InsertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHash, pver uint32, buf []byte) (blockid int64, err error) - // InsertTx inserts a tx hash and its associated data into the database + // InsertTx stores a transaction hash and its associated data into the + // database. InsertTx(txsha *btcwire.ShaHash, blockidx int64, txoff int, txlen int, usedbuf []byte) (err error) // InvalidateBlockCache releases all cached blocks. @@ -105,17 +106,17 @@ type Db interface { // NewIterateBlocks returns an iterator for all blocks in database. NewIterateBlocks() (pbi BlockIterator, err error) - // NewestSha provides an interface to quickly look up the sha of - // the most recent (end) of the block chain. + // NewestSha provides an interface to quickly look up the hash of + // the most recent (end) block of the block chain. NewestSha() (sha *btcwire.ShaHash, blkid int64, err error) // RollbackClose discards the recent database changes to the previously // saved data at last Sync and closes the database. RollbackClose() - // SetDBInsertMode provides hints to the database to how the application - // is running. This allows the database to work in optimized modes when - // the database may be very busy. + // SetDBInsertMode provides hints to the database about how the + // application is running. This allows the database to work in + // optimized modes when the database may be very busy. SetDBInsertMode(InsertMode) // Sync verifies that the database is coherent on disk and no From bea4ccbeef330274ebcfeb019694c34b0e8b1593 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Thu, 30 May 2013 17:21:15 -0500 Subject: [PATCH 014/163] Finish incomplete comment. --- db.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/db.go b/db.go index f0f541cd..91054895 100644 --- a/db.go +++ b/db.go @@ -64,7 +64,8 @@ type Db interface { // more are present, use the special id `AllShas'. FetchIdxRange(startid, endid int64) (rshalist []btcwire.ShaHash, err error) - // FetchTxAllBySha returns a + // FetchTxAllBySha returns several pieces of data for a given + // transaction hash. FetchTxAllBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rtxbuf []byte, rpver uint32, rblksha *btcwire.ShaHash, err error) // FetchTxBufBySha returns the raw bytes and associated protocol version From 7416e9a71d7791f0645667f6d76f1f961669ddb6 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Thu, 30 May 2013 17:27:39 -0500 Subject: [PATCH 015/163] Rename funcs and variables for Idx to Height. The Db interface is intended to work with block heights as opposed to specific database ids which may or may not be the same as the block height. This commits changes the function names to make that distinction a little more clear. --- db.go | 16 ++++++++-------- sqlite3/sqliteblock.go | 25 +++++++++++++------------ sqlite3/sqliteblock_test.go | 4 ++-- sqlite3/sqlitedbcache.go | 2 +- 4 files changed, 24 insertions(+), 23 deletions(-) diff --git a/db.go b/db.go index 91054895..5e1526bf 100644 --- a/db.go +++ b/db.go @@ -54,15 +54,15 @@ type Db interface { // cache the underlying object if desired. FetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) - // FetchBlockShaByIdx returns a block hash based on its height in the - // blockchain. - FetchBlockShaByIdx(blkid int64) (sha *btcwire.ShaHash, err error) + // FetchBlockShaByHeight returns a block hash based on its height in the + // block chain. + FetchBlockShaByHeight(height int64) (sha *btcwire.ShaHash, err error) - // FetchIdxRange looks up a range of blocks by the start and ending + // FetchHeightRange looks up a range of blocks by the start and ending // heights. Fetch is inclusive of the start height and exclusive of the // ending height. To fetch all hashes from the start height until no // more are present, use the special id `AllShas'. - FetchIdxRange(startid, endid int64) (rshalist []btcwire.ShaHash, err error) + FetchHeightRange(startHeight, endHeight int64) (rshalist []btcwire.ShaHash, err error) // FetchTxAllBySha returns several pieces of data for a given // transaction hash. @@ -85,7 +85,7 @@ type Db interface { // InsertBlock inserts raw block and transaction data from a block // into the database. - InsertBlock(block *btcutil.Block) (blockid int64, err error) + InsertBlock(block *btcutil.Block) (height int64, err error) // InsertBlockData stores a block hash and its associated data block // with the given previous hash and protocol version into the database. @@ -93,7 +93,7 @@ type Db interface { // InsertTx stores a transaction hash and its associated data into the // database. - InsertTx(txsha *btcwire.ShaHash, blockidx int64, txoff int, txlen int, usedbuf []byte) (err error) + InsertTx(txsha *btcwire.ShaHash, blockHeight int64, txoff int, txlen int, usedbuf []byte) (err error) // InvalidateBlockCache releases all cached blocks. InvalidateBlockCache() @@ -109,7 +109,7 @@ type Db interface { // NewestSha provides an interface to quickly look up the hash of // the most recent (end) block of the block chain. - NewestSha() (sha *btcwire.ShaHash, blkid int64, err error) + NewestSha() (sha *btcwire.ShaHash, height int64, err error) // RollbackClose discards the recent database changes to the previously // saved data at last Sync and closes the database. diff --git a/sqlite3/sqliteblock.go b/sqlite3/sqliteblock.go index 5416e160..b2c2b84e 100644 --- a/sqlite3/sqliteblock.go +++ b/sqlite3/sqliteblock.go @@ -151,13 +151,14 @@ func (db *SqliteDb) blkExistsSha(sha *btcwire.ShaHash) bool { return true } -// FetchBlockShaByIdx returns a block sha based on its height in the blockchain. -func (db *SqliteDb) FetchBlockShaByIdx(blkid int64) (sha *btcwire.ShaHash, err error) { +// FetchBlockShaByHeight returns a block hash based on its height in the +// block chain. +func (db *SqliteDb) FetchBlockShaByHeight(height int64) (sha *btcwire.ShaHash, err error) { var row *sql.Row db.dbLock.Lock() defer db.dbLock.Unlock() - blockidx := blkid + 1 // skew between btc blockid and sql + blockidx := height + 1 // skew between btc blockid and sql row = db.blkStmts[blkFetchIdx].QueryRow(blockidx) @@ -171,21 +172,21 @@ func (db *SqliteDb) FetchBlockShaByIdx(blkid int64) (sha *btcwire.ShaHash, err e return &shaval, nil } -// FetchIdxRange looks up a range of block by the start and ending ids. -// Fetch is inclusive of the start id and exclusive of the ending id. If the -// special id `AllShas' is provided as endid then FetchIdxRange will fetch all -// shas from startid until no more shas are present. -func (db *SqliteDb) FetchIdxRange(startid, endid int64) (rshalist []btcwire.ShaHash, err error) { +// FetchHeightRange looks up a range of blocks by the start and ending +// heights. Fetch is inclusive of the start height and exclusive of the +// ending height. To fetch all hashes from the start height until no +// more are present, use the special id `AllShas'. +func (db *SqliteDb) FetchHeightRange(startHeight, endHeight int64) (rshalist []btcwire.ShaHash, err error) { db.dbLock.Lock() defer db.dbLock.Unlock() - startidx := startid + 1 // skew between btc blockid and sql + startidx := startHeight + 1 // skew between btc block height and sql var endidx int64 - if endid == btcdb.AllShas { + if endHeight == btcdb.AllShas { endidx = btcdb.AllShas // no skew if asking for all } else { - endidx = endid + 1 // skew between btc blockid and sql + endidx = endHeight + 1 // skew between btc block height and sql } rows, err := db.blkStmts[blkFetchIdxList].Query(startidx, endidx) if err != nil { @@ -209,7 +210,7 @@ func (db *SqliteDb) FetchIdxRange(startid, endid int64) (rshalist []btcwire.ShaH if err == nil { rshalist = shalist } - log.Tracef("FetchIdxRange idx %v %v returned %v shas err %v", startid, endid, len(shalist), err) + log.Tracef("FetchIdxRange idx %v %v returned %v shas err %v", startHeight, endHeight, len(shalist), err) return } diff --git a/sqlite3/sqliteblock_test.go b/sqlite3/sqliteblock_test.go index 80622436..2b2d17b7 100644 --- a/sqlite3/sqliteblock_test.go +++ b/sqlite3/sqliteblock_test.go @@ -141,7 +141,7 @@ func testFetch(t *testing.T, db btcdb.Db, shas []btcwire.ShaHash, } // Fetch the sha by index and ensure it matches. - tsha, err := db.FetchBlockShaByIdx(int64(i)) + tsha, err := db.FetchBlockShaByHeight(int64(i)) if err != nil { t.Errorf("can't fetch sha at index %d: %v", i, err) continue @@ -174,7 +174,7 @@ func testFetch(t *testing.T, db btcdb.Db, shas []btcwire.ShaHash, for _, test := range fetchIdxTests { t.Logf("numSha: %d - Fetch from %d to %d\n", numShas, test.start, test.end) - if shalist, err := db.FetchIdxRange(test.start, test.end); err == nil { + if shalist, err := db.FetchHeightRange(test.start, test.end); err == nil { compareArray(t, shalist, test.exp, test.test, sync) } else { t.Errorf("failed to fetch index range for %s (%s)", diff --git a/sqlite3/sqlitedbcache.go b/sqlite3/sqlitedbcache.go index feba026d..45de6cee 100644 --- a/sqlite3/sqlitedbcache.go +++ b/sqlite3/sqlitedbcache.go @@ -137,7 +137,7 @@ func (db *SqliteDb) FetchTxAllBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, return } - blksha, err := db.FetchBlockShaByIdx(bidx) + blksha, err := db.FetchBlockShaByHeight(bidx) if err != nil { log.Warnf("block idx lookup %v to %v", bidx, err) return From fac055c24e653866946189f4e21beda82ceec3b5 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Fri, 31 May 2013 13:40:38 -0500 Subject: [PATCH 016/163] Add comments to clarify interface function caching. This commit attempts to clarify which functions in the Db interface may be returning cached data that the InvalidateCache, InvalidateBlockCache, and InvalidateTxCache functions are used to clear. --- db.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/db.go b/db.go index 5e1526bf..406a4846 100644 --- a/db.go +++ b/db.go @@ -51,7 +51,7 @@ type Db interface { ExistsSha(sha *btcwire.ShaHash) (exists bool) // FetchBlockBySha returns a btcutil Block. The implementation may - // cache the underlying object if desired. + // cache the underlying data if desired. FetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) // FetchBlockShaByHeight returns a block hash based on its height in the @@ -65,18 +65,21 @@ type Db interface { FetchHeightRange(startHeight, endHeight int64) (rshalist []btcwire.ShaHash, err error) // FetchTxAllBySha returns several pieces of data for a given - // transaction hash. + // transaction hash. The implementation may cache the underlying data + // if desired. FetchTxAllBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rtxbuf []byte, rpver uint32, rblksha *btcwire.ShaHash, err error) // FetchTxBufBySha returns the raw bytes and associated protocol version - // for the transaction with the requested hash. + // for the transaction with the requested hash. The implementation may + // cache the underlying data if desired. FetchTxBufBySha(txsha *btcwire.ShaHash) (txbuf []byte, rpver uint32, err error) - // FetchTxBySha returns some data for the given transaction hash. + // FetchTxBySha returns some data for the given transaction hash. The + // implementation may cache the underlying data if desired. FetchTxBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rpver uint32, blksha *btcwire.ShaHash, err error) // FetchTxByShaList returns a TxListReply given an array of transaction - // hashes. + // hashes. The implementation may cache the underlying data if desired. FetchTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply // FetchTxUsedBySha returns the used/spent buffer for a given @@ -131,6 +134,7 @@ type BlockIterator interface { // NextRow iterates thru all blocks in database. NextRow() bool + // Row returns row data for block iterator. Row() (key *btcwire.ShaHash, pver uint32, buf []byte, err error) } From b686bbacf58b21c05ea6b733be3bd87caa695e71 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Fri, 31 May 2013 13:50:23 -0500 Subject: [PATCH 017/163] Update and fix usage example. This commit updates the usage example as follows: - Add a defer db.Close since the database should be closed after the caller is done with it - Correct the import path for the btcdb/sqlite3 package - Add a db extension to the example database name - Make the error handling and comments match the standard style --- doc.go | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/doc.go b/doc.go index dc96fbf0..a93131fc 100644 --- a/doc.go +++ b/doc.go @@ -29,28 +29,27 @@ At the highest level, the use of this packages just requires that you import it, setup a database, insert some data into it, and optionally, query the data back. In a more concrete example: - // Import packages + // Import packages. import ( "github.com/conformal/btcdb" - _ "github.com/conformal/btcdb/db_sqlite" + _ "github.com/conformal/btcdb/sqlite3" ) - // Create a database - dbname := "dbexample" - db, err := btcdb.CreateDB("sqlite", dbname) + // Create a database and schedule it to be closed on exit. + dbName := "example.db" + db, err := btcdb.CreateDB("sqlite", dbName) if err != nil { - fmt.Printf("Failed to open database %v", err) - return + // Log and handle the error + } + defer db.Close() + + // Insert a block. + newHeight, err := db.InsertBlock(block) + if err != nil { + // Log and handle the error } - // Insert a block - newheight, err := db.InsertBlock(block) - if err != nil { - fmt.Printf("failed to insert block %v err %v", height, err) - } - - // Sync the database + // Sync the database. db.Sync() - */ package btcdb From 64568826f1c0b197ab52cb80c21eea97a161c6b5 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Fri, 31 May 2013 14:02:47 -0500 Subject: [PATCH 018/163] Correct spelling of nonexistent. --- db.go | 4 ++-- sqlite3/sqliteblock_test.go | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/db.go b/db.go index 406a4846..d7e5b4d8 100644 --- a/db.go +++ b/db.go @@ -14,8 +14,8 @@ var ( PrevShaMissing = errors.New("Previous sha missing from database") TxShaMissing = errors.New("Requested Tx does not exist") DuplicateSha = errors.New("Duplicate insert attempted") - DbDoesNotExist = errors.New("Non-existant database") - DbUnknownType = errors.New("Non-existant database type") + DbDoesNotExist = errors.New("Non-existent database") + DbUnknownType = errors.New("Non-existent database type") ) // AllShas is a special value that can be used as the final sha when requesting diff --git a/sqlite3/sqliteblock_test.go b/sqlite3/sqliteblock_test.go index 2b2d17b7..c129c15d 100644 --- a/sqlite3/sqliteblock_test.go +++ b/sqlite3/sqliteblock_test.go @@ -167,9 +167,9 @@ func testFetch(t *testing.T, db btcdb.Db, shas []btcwire.ShaHash, {midBlockID, endBlockID, shas[midBlockID-1 : endBlockID-1], "fetch second half"}, - // Nonexistant off the end. + // Nonexistent off the end. {endBlockID, endBlockID * 2, []btcwire.ShaHash{}, - "fetch nonexistant"}, + "fetch nonexistent"}, } for _, test := range fetchIdxTests { @@ -182,9 +182,9 @@ func testFetch(t *testing.T, db btcdb.Db, shas []btcwire.ShaHash, } } - // Try and fetch nonexistant sha. + // Try and fetch nonexistent sha. if db.ExistsSha(&badSha) { - t.Errorf("non existant sha exists (%s)!", sync) + t.Errorf("nonexistent sha exists (%s)!", sync) } _, _, _, err := sqlite3.FetchSha(db, &badSha) if err == nil { From 5882b3c79a805467b975313cbb0f46c5b7a74f83 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Fri, 31 May 2013 14:07:48 -0500 Subject: [PATCH 019/163] Change Tx to transaction in user facing error. --- db.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/db.go b/db.go index d7e5b4d8..8d8aaf38 100644 --- a/db.go +++ b/db.go @@ -10,9 +10,10 @@ import ( "github.com/conformal/btcwire" ) +// Errors that the various database functions may return. var ( PrevShaMissing = errors.New("Previous sha missing from database") - TxShaMissing = errors.New("Requested Tx does not exist") + TxShaMissing = errors.New("Requested transaction does not exist") DuplicateSha = errors.New("Duplicate insert attempted") DbDoesNotExist = errors.New("Non-existent database") DbUnknownType = errors.New("Non-existent database type") From 8d8fdf4fc1c169f90f9717f31eea2426065f8420 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Fri, 31 May 2013 15:12:20 -0500 Subject: [PATCH 020/163] Add documentation for Db interface. --- db.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/db.go b/db.go index 8d8aaf38..e6ef8e87 100644 --- a/db.go +++ b/db.go @@ -37,6 +37,10 @@ const ( InsertValidatedInput ) +// Db defines a generic interface that is used to request and insert data into +// the bitcoin block chain. This interface is intended to be agnostic to actual +// mechanism used for backend data storage. The AddDBDriver function can be +// used to add a new backend data storage method. type Db interface { // Close cleanly shuts down the database and syncs all data. Close() From 10d981974cf59f4e408e481de8c28b8a1ead3f69 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Fri, 31 May 2013 15:19:34 -0500 Subject: [PATCH 021/163] Add documentation for functions and types. --- db.go | 6 ++++++ log.go | 1 + 2 files changed, 7 insertions(+) diff --git a/db.go b/db.go index e6ef8e87..13c369aa 100644 --- a/db.go +++ b/db.go @@ -133,6 +133,8 @@ type Db interface { Sync() } +// BlockIterator defines a generic interface for an iterator through the block +// chain. type BlockIterator interface { // Close shuts down the iterator when done walking blocks in the database. Close() @@ -144,12 +146,16 @@ type BlockIterator interface { Row() (key *btcwire.ShaHash, pver uint32, buf []byte, err error) } +// DriverDB defines a structure for backend drivers to use when they registered +// themselves as a backend which implements the Db interface. type DriverDB struct { DbType string Create func(argstr string) (pbdb Db, err error) Open func(filepath string) (pbdb Db, err error) } +// TxListReply is used to return individual transaction information when +// data about multiple transactions is requested in a single call. type TxListReply struct { Sha *btcwire.ShaHash Tx *btcwire.MsgTx diff --git a/log.go b/log.go index 7701a92e..667e0bcb 100644 --- a/log.go +++ b/log.go @@ -51,6 +51,7 @@ func SetLogWriter(w io.Writer) error { return nil } +// GetLog returns the currently active logger. func GetLog() seelog.LoggerInterface { return log } From 2c0dc2d862e6ab636f89c27d43c4a4027e713bb6 Mon Sep 17 00:00:00 2001 From: Steven Roose Date: Sat, 1 Jun 2013 00:09:26 +0200 Subject: [PATCH 022/163] Small errors in documentation. I could not resist while reading. --- doc.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc.go b/doc.go index a93131fc..94b9e1af 100644 --- a/doc.go +++ b/doc.go @@ -3,10 +3,10 @@ // license that can be found in the LICENSE file. /* -Package btcdb provides a database interface for the bitcoin block chain. +Package btcdb provides a database interface for the Bitcoin block chain. -As of May 2013, there are over 235,000 blocks in the bitcoin block chain and -and over 17 million transactions (which turns out to be over 11Gb of data). +As of May 2013, there are over 235,000 blocks in the Bitcoin block chain and +and over 17 million transactions (which turns out to be over 11GB of data). btcdb provides a database layer to store and retrieve this data in a fairly simple and efficient manner. The use of this should not require specific knowledge of the database backend used although currently only db_sqlite is From 166a546078426798175bfae320dda2ed1f36b0d0 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Tue, 25 Jun 2013 10:15:58 -0500 Subject: [PATCH 023/163] Modify DropAfterBlockBySha to accept a pointer. Since all other functions take pointers, this makes the interface more consistent. --- db.go | 2 +- sqlite3/sqlite.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/db.go b/db.go index 13c369aa..2e181046 100644 --- a/db.go +++ b/db.go @@ -49,7 +49,7 @@ type Db interface { // the given block. It terminates any existing transaction and performs // its operations in an atomic transaction which is commited before // the function returns. - DropAfterBlockBySha(btcwire.ShaHash) (err error) + DropAfterBlockBySha(*btcwire.ShaHash) (err error) // ExistsSha returns whether or not the given block hash is present in // the database. diff --git a/sqlite3/sqlite.go b/sqlite3/sqlite.go index d8df4ae9..64e634db 100644 --- a/sqlite3/sqlite.go +++ b/sqlite3/sqlite.go @@ -468,7 +468,7 @@ func (db *SqliteDb) rePlayTransaction() (err error) { // DropAfterBlockBySha will remove any blocks from the database after the given block. // It terminates any existing transaction and performs its operations in an // atomic transaction, it is terminated (committed) before exit. -func (db *SqliteDb) DropAfterBlockBySha(sha btcwire.ShaHash) (err error) { +func (db *SqliteDb) DropAfterBlockBySha(sha *btcwire.ShaHash) (err error) { var row *sql.Row db.dbLock.Lock() defer db.dbLock.Unlock() From 1f773006f05aef67bbeaee15f3eb8ded5af9dd42 Mon Sep 17 00:00:00 2001 From: "Owain G. Ainsworth" Date: Tue, 9 Jul 2013 22:49:58 +0100 Subject: [PATCH 024/163] fix tests building from 166a546078426798175bfae320dda2ed1f36b0d0 --- sqlite3/sqliteblock_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite3/sqliteblock_test.go b/sqlite3/sqliteblock_test.go index c129c15d..12bac76d 100644 --- a/sqlite3/sqliteblock_test.go +++ b/sqlite3/sqliteblock_test.go @@ -278,7 +278,7 @@ func TestBdb(t *testing.T) { testFetch(t, db, testShas, "post sync") for i := len(testShas) - 1; i >= 0; i-- { - err := db.DropAfterBlockBySha(testShas[i]) + err := db.DropAfterBlockBySha(&testShas[i]) if err != nil { t.Errorf("drop after %d failed %v", i, err) break From a7a9023bc1bcbfe8063a22436c7c1cdd15da42c8 Mon Sep 17 00:00:00 2001 From: "Owain G. Ainsworth" Date: Wed, 10 Jul 2013 00:11:02 +0100 Subject: [PATCH 025/163] Add ExistsTxSha to db interface. This function may be used to check for existance of a tx sha without having to fetch the data from the db. --- db.go | 4 ++++ sqlite3/operational_test.go | 10 ++++++++ sqlite3/sqlite.go | 4 ++++ sqlite3/sqlitetx.go | 48 +++++++++++++++++++++++++++++++++++-- test_coverage.txt | 26 ++++++++++---------- 5 files changed, 78 insertions(+), 14 deletions(-) diff --git a/db.go b/db.go index 2e181046..7f2d0493 100644 --- a/db.go +++ b/db.go @@ -69,6 +69,10 @@ type Db interface { // more are present, use the special id `AllShas'. FetchHeightRange(startHeight, endHeight int64) (rshalist []btcwire.ShaHash, err error) + // ExistsTxSha returns whether or not the given tx hash is present in + // the database + ExistsTxSha(sha *btcwire.ShaHash) (exists bool) + // FetchTxAllBySha returns several pieces of data for a given // transaction hash. The implementation may cache the underlying data // if desired. diff --git a/sqlite3/operational_test.go b/sqlite3/operational_test.go index e91161c2..e731188d 100644 --- a/sqlite3/operational_test.go +++ b/sqlite3/operational_test.go @@ -94,6 +94,11 @@ out: } origintxsha := &txin.PreviousOutpoint.Hash txneededList = append(txneededList, origintxsha) + + if !db.ExistsTxSha(origintxsha) { + t.Errorf("referenced tx not found %v ", origintxsha) + } + _, _, _, _, err := db.FetchTxAllBySha(origintxsha) if err != nil { t.Errorf("referenced tx not found %v err %v ", origintxsha, err) @@ -259,6 +264,11 @@ func testBackout(t *testing.T, mode int) { block := blocks[110] mblock := block.MsgBlock() txsha, err := mblock.Transactions[0].TxSha(block.ProtocolVersion()) + exists := db.ExistsTxSha(&txsha) + if exists { + t.Errorf("tx %v exists in db, failure expected") + } + _, _, _, err = db.FetchTxBySha(&txsha) _, err = db.FetchTxUsedBySha(&txsha) diff --git a/sqlite3/sqlite.go b/sqlite3/sqlite.go index 64e634db..e41b7148 100644 --- a/sqlite3/sqlite.go +++ b/sqlite3/sqlite.go @@ -45,6 +45,8 @@ const ( txPragmaVacuumOn txPragmaVacuumOff txVacuum + txExistsShaStmt + txtmpExistsShaStmt ) var blkqueries []string = []string{ @@ -70,6 +72,8 @@ var txqueries []string = []string{ txPragmaVacuumOn: "PRAGMA auto_vacuum = FULL;", txPragmaVacuumOff: "PRAGMA auto_vacuum = NONE;", txVacuum: "VACUUM;", + txExistsShaStmt: "SELECT blockid FROM tx WHERE key = ?;", + txtmpExistsShaStmt: "SELECT blockid FROM txtmp WHERE key = ?;", } var log seelog.LoggerInterface = seelog.Disabled diff --git a/sqlite3/sqlitetx.go b/sqlite3/sqlitetx.go index 321d3a05..1c05aa7e 100644 --- a/sqlite3/sqlitetx.go +++ b/sqlite3/sqlitetx.go @@ -72,11 +72,55 @@ func (db *SqliteDb) insertTx(txsha *btcwire.ShaHash, blockidx int64, txoff int, return } -// FetchLocationBySha looks up the Tx sha information by name. -func (db *SqliteDb) FetchLocationBySha(txsha *btcwire.ShaHash) (blockidx int64, txoff int, txlen int, err error) { +// ExistsTxSha returns if the given tx sha exists in the database +func (db *SqliteDb) ExistsTxSha(txsha *btcwire.ShaHash) (exists bool) { db.dbLock.Lock() defer db.dbLock.Unlock() + if _, ok := db.fetchTxCache(txsha); ok { + return true + } + + return db.existsTxSha(txsha) +} + + +// existsTxSha returns if the given tx sha exists in the database.o +// Must be called with the db lock held. +func (db *SqliteDb) existsTxSha(txsha *btcwire.ShaHash) (exists bool) { + var blockid uint32 + + txop := db.txop(txExistsShaStmt) + row := txop.QueryRow(txsha.String()) + err := row.Scan(&blockid) + + if err == sql.ErrNoRows { + txop = db.txop(txtmpExistsShaStmt) + row = txop.QueryRow(txsha.String()) + err := row.Scan(&blockid) + + if err == sql.ErrNoRows { + return false + } + if err != nil { + log.Warnf("txTmpExistsTxSha: fail %v", err) + return false + } + log.Warnf("txtmpExistsTxSha: success") + return true + } + + if err != nil { + // ignore real errors? + log.Warnf("existsTxSha: fail %v", err) + return false + } + + return true +} + +// FetchLocationBySha looks up the Tx sha information by name. +func (db *SqliteDb) FetchLocationBySha(txsha *btcwire.ShaHash) (blockidx int64, txoff int, txlen int, err error) { return db.fetchLocationBySha(txsha) } diff --git a/test_coverage.txt b/test_coverage.txt index f1198084..fb0a17c5 100644 --- a/test_coverage.txt +++ b/test_coverage.txt @@ -3,34 +3,35 @@ github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.NewestSha 100.00% github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.endTx 100.00% (20/20) github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.close 100.00% (9/9) github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.txop 100.00% (7/7) -github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.fetchTxCache 100.00% (7/7) github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteBlockIterator.Row 100.00% (7/7) +github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.fetchTxCache 100.00% (7/7) github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.FetchTxByShaList 100.00% (6/6) -github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.InvalidateTxCache 100.00% (5/5) +github.com/conformal/btcdb/sqlite3/sqlitetx.go SqliteDb.ExistsTxSha 100.00% (5/5) github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.InvalidateBlockCache 100.00% (5/5) -github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.Sync 100.00% (3/3) -github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.Close 100.00% (3/3) -github.com/conformal/btcdb/sqlite3/sqlitetx.go SqliteDb.FetchLocationBySha 100.00% (3/3) +github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.InvalidateTxCache 100.00% (5/5) github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.InsertBlockData 100.00% (3/3) +github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.Close 100.00% (3/3) +github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.Sync 100.00% (3/3) github.com/conformal/btcdb/sqlite3/sqlitetx.go SqliteDb.InsertTx 100.00% (3/3) -github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.FetchTxBufBySha 100.00% (2/2) -github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.InvalidateCache 100.00% (2/2) github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteBlockIterator.Close 100.00% (2/2) -github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.FetchTxBySha 100.00% (2/2) -github.com/conformal/btcdb/sqlite3/sqlite.go OpenSqliteDB 100.00% (2/2) github.com/conformal/btcdb/sqlite3/sqlite.go CreateSqliteDB 100.00% (2/2) +github.com/conformal/btcdb/sqlite3/sqlite.go OpenSqliteDB 100.00% (2/2) +github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.InvalidateCache 100.00% (2/2) +github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.FetchTxBufBySha 100.00% (2/2) +github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.FetchTxBySha 100.00% (2/2) github.com/conformal/btcdb/sqlite3/sqlite.go init 100.00% (1/1) +github.com/conformal/btcdb/sqlite3/sqlitetx.go SqliteDb.FetchLocationBySha 100.00% (1/1) github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteBlockIterator.NextRow 100.00% (1/1) github.com/conformal/btcdb/sqlite3/sqlitetx.go SqliteDb.insertTx 96.77% (30/31) github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.insertBlockCache 92.31% (12/13) +github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.FetchBlockShaByHeight 91.67% (11/12) github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.insertTxCache 91.67% (11/12) -github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.FetchBlockShaByIdx 91.67% (11/12) github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.rePlayTransaction 90.00% (18/20) github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.NewIterateBlocks 88.24% (15/17) github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.RollbackClose 87.50% (7/8) github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.fetchSha 85.71% (12/14) github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.ExistsSha 85.71% (6/7) -github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.FetchIdxRange 84.62% (22/26) +github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.FetchHeightRange 84.62% (22/26) github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.insertBlockData 84.00% (21/25) github.com/conformal/btcdb/sqlite3/sqlitetx.go SqliteDb.fetchLocationBySha 84.00% (21/25) github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.FetchBlockBySha 83.33% (10/12) @@ -38,6 +39,7 @@ github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.fetchBlockCache 8 github.com/conformal/btcdb/sqlite3/sqlitetx.go SqliteDb.FetchTxUsedBySha 82.61% (19/23) github.com/conformal/btcdb/sqlite3/sqliteblock.go insertGenesisBlock 81.82% (9/11) github.com/conformal/btcdb/sqlite3/sqlite.go newOrCreateSqliteDB 79.25% (42/53) +github.com/conformal/btcdb/sqlite3/sqlitetx.go SqliteDb.existsTxSha 78.95% (15/19) github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.startTx 78.57% (11/14) github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.blkExistsSha 77.78% (7/9) github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.FetchTxAllBySha 77.14% (27/35) @@ -47,5 +49,5 @@ github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.syncPoint 71.43% (5/7) github.com/conformal/btcdb/sqlite3/sqlite.go createDB 70.59% (12/17) github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.DropAfterBlockBySha 68.57% (24/35) github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.InsertBlock 46.00% (23/50) -github.com/conformal/btcdb/sqlite3 ----------------------------- 82.15% (566/689) +github.com/conformal/btcdb/sqlite3 ------------------------------ 82.14% (584/711) From 66731c1a1ea36eb7723b9b106772a7e3580338cb Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Thu, 11 Jul 2013 16:35:50 -0400 Subject: [PATCH 026/163] Implement TxOut Spend tracking. Return spent data in TxListReply. Unspend coins when blocks are removed during DropAfterBlock. --- db.go | 8 +- sqlite3/insertremove_test.go | 217 ++++++++++++++++++++++++++ sqlite3/operational_test.go | 4 +- sqlite3/sqlite.go | 220 +++++++++++++++++++++++--- sqlite3/sqliteblock.go | 12 +- sqlite3/sqliteblock_test.go | 292 ----------------------------------- sqlite3/sqlitedbcache.go | 161 ++++++++++++++++--- sqlite3/sqlitetx.go | 50 +++++- 8 files changed, 611 insertions(+), 353 deletions(-) create mode 100644 sqlite3/insertremove_test.go delete mode 100644 sqlite3/sqliteblock_test.go diff --git a/db.go b/db.go index 7f2d0493..81cc9fee 100644 --- a/db.go +++ b/db.go @@ -161,9 +161,11 @@ type DriverDB struct { // TxListReply is used to return individual transaction information when // data about multiple transactions is requested in a single call. type TxListReply struct { - Sha *btcwire.ShaHash - Tx *btcwire.MsgTx - Err error + Sha *btcwire.ShaHash + Tx *btcwire.MsgTx + Height int64 + TxSpent []bool + Err error } // driverList holds all of the registered database backends. diff --git a/sqlite3/insertremove_test.go b/sqlite3/insertremove_test.go new file mode 100644 index 00000000..f639369e --- /dev/null +++ b/sqlite3/insertremove_test.go @@ -0,0 +1,217 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package sqlite3_test + +import ( + //"compress/bzip2" + //"encoding/binary" + "github.com/conformal/btcdb" + "github.com/conformal/btcdb/sqlite3" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" + //"io" + "fmt" + "os" + "path/filepath" + //"strings" + "testing" +) + +var tstBlocks []*btcutil.Block + +func loadblocks(t *testing.T) []*btcutil.Block { + if len(tstBlocks) != 0 { + return tstBlocks + } + + testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + blocks, err := loadBlocks(t, testdatafile) + if err != nil { + t.Errorf("Unable to load blocks from test data: %v", err) + return nil + } + tstBlocks = blocks + return blocks +} + +func TestUnspentInsert(t *testing.T) { + testUnspentInsert(t, dbTmDefault) + testUnspentInsert(t, dbTmNormal) + testUnspentInsert(t, dbTmFast) +} + +// insert every block in the test chain +// after each insert, fetch all the tx affected by the latest +// block and verify that the the tx is spent/unspent +// new tx should be fully unspent, referenced tx should have +// the associated txout set to spent. +func testUnspentInsert(t *testing.T, mode int) { + // Ignore db remove errors since it means we didn't have an old one. + dbname := "tstdbuspnt1" + _ = os.Remove(dbname) + db, err := btcdb.CreateDB("sqlite", dbname) + if err != nil { + t.Errorf("Failed to open test database %v", err) + return + } + defer os.Remove(dbname) + defer db.Close() + + switch mode { + case dbTmDefault: // default + // no setup + case dbTmNormal: // explicit normal + db.SetDBInsertMode(btcdb.InsertNormal) + case dbTmFast: // fast mode + db.SetDBInsertMode(btcdb.InsertFast) + if sqldb, ok := db.(*sqlite3.SqliteDb); ok { + sqldb.TempTblMax = 100 + } else { + t.Errorf("not right type") + } + case dbTmNoVerify: // validated block + t.Errorf("UnspentInsert test is not valid in NoVerify mode") + } + + // Since we are dealing with small dataset, reduce cache size + sqlite3.SetBlockCacheSize(db, 2) + sqlite3.SetTxCacheSize(db, 3) + + blocks := loadblocks(t) +endtest: + for height := int64(1); height < int64(len(blocks)); height++ { + + block := blocks[height] + // look up inputs to this x + mblock := block.MsgBlock() + var txneededList []*btcwire.ShaHash + var txlookupList []*btcwire.ShaHash + var txOutList []*btcwire.ShaHash + var txInList []*btcwire.OutPoint + for _, tx := range mblock.Transactions { + for _, txin := range tx.TxIn { + if txin.PreviousOutpoint.Index == uint32(4294967295) { + continue + } + origintxsha := &txin.PreviousOutpoint.Hash + + txInList = append(txInList, &txin.PreviousOutpoint) + txneededList = append(txneededList, origintxsha) + txlookupList = append(txlookupList, origintxsha) + + if !db.ExistsTxSha(origintxsha) { + t.Errorf("referenced tx not found %v ", origintxsha) + } + + } + txshaname, _ := tx.TxSha(block.ProtocolVersion()) + txlookupList = append(txlookupList, &txshaname) + txOutList = append(txOutList, &txshaname) + } + + txneededmap := map[btcwire.ShaHash]*btcdb.TxListReply{} + txlist := db.FetchTxByShaList(txneededList) + for _, txe := range txlist { + if txe.Err != nil { + t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) + break endtest + } + txneededmap[*txe.Sha] = txe + } + for _, spend := range txInList { + itxe := txneededmap[spend.Hash] + if itxe.TxSpent[spend.Index] == true { + t.Errorf("txin %v:%v is already spent", spend.Hash, spend.Index) + } + } + + newheight, err := db.InsertBlock(block) + if err != nil { + t.Errorf("failed to insert block %v err %v", height, err) + break endtest + } + if newheight != height { + t.Errorf("height mismatch expect %v returned %v", height, newheight) + break endtest + } + + txlookupmap := map[btcwire.ShaHash]*btcdb.TxListReply{} + txlist = db.FetchTxByShaList(txlookupList) + for _, txe := range txlist { + if txe.Err != nil { + t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) + break endtest + } + txlookupmap[*txe.Sha] = txe + } + for _, spend := range txInList { + itxe := txlookupmap[spend.Hash] + if itxe.TxSpent[spend.Index] == false { + fmt.Printf("?? txin %v:%v is unspent %v\n", spend.Hash, spend.Index, itxe.TxSpent) + t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent) + } + } + for _, txo := range txOutList { + itxe := txlookupmap[*txo] + for i, spent := range itxe.TxSpent { + if spent == true { + t.Errorf("freshly inserted tx %v already spent %v", txo, i) + } + } + + } + if len(txInList) == 0 { + continue + } + dropblock := blocks[height-1] + dropsha, _ := dropblock.Sha() + + err = db.DropAfterBlockBySha(dropsha) + if err != nil { + t.Errorf("failed to drop block %v err %v", height, err) + break endtest + } + + txlookupmap = map[btcwire.ShaHash]*btcdb.TxListReply{} + txlist = db.FetchTxByShaList(txlookupList) + for _, txe := range txlist { + if txe.Err != nil { + if _, ok := txneededmap[*txe.Sha]; ok { + t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) + break endtest + } + } + txlookupmap[*txe.Sha] = txe + } + for _, spend := range txInList { + itxe := txlookupmap[spend.Hash] + if itxe.TxSpent[spend.Index] == true { + fmt.Printf("?? txin %v:%v is spent %v\n", spend.Hash, spend.Index, itxe.TxSpent) + t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent) + } + } + newheight, err = db.InsertBlock(block) + if err != nil { + t.Errorf("failed to insert block %v err %v", height, err) + break endtest + } + txlookupmap = map[btcwire.ShaHash]*btcdb.TxListReply{} + txlist = db.FetchTxByShaList(txlookupList) + for _, txe := range txlist { + if txe.Err != nil { + t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) + break endtest + } + txlookupmap[*txe.Sha] = txe + } + for _, spend := range txInList { + itxe := txlookupmap[spend.Hash] + if itxe.TxSpent[spend.Index] == false { + fmt.Printf("?? txin %v:%v is unspent %v\n", spend.Hash, spend.Index, itxe.TxSpent) + t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent) + } + } + } +} diff --git a/sqlite3/operational_test.go b/sqlite3/operational_test.go index e731188d..5ddc0b3a 100644 --- a/sqlite3/operational_test.go +++ b/sqlite3/operational_test.go @@ -246,7 +246,7 @@ func testBackout(t *testing.T, mode int) { _ = db.ExistsSha(sha) _, err = db.FetchBlockBySha(sha) if err != nil { - t.Errorf("failed to load block 99 from db", err) + t.Errorf("failed to load block 99 from db %v", err) } sha, err = blocks[110].Sha() @@ -266,7 +266,7 @@ func testBackout(t *testing.T, mode int) { txsha, err := mblock.Transactions[0].TxSha(block.ProtocolVersion()) exists := db.ExistsTxSha(&txsha) if exists { - t.Errorf("tx %v exists in db, failure expected") + t.Errorf("tx %v exists in db, failure expected", txsha) } _, _, _, err = db.FetchTxBySha(&txsha) diff --git a/sqlite3/sqlite.go b/sqlite3/sqlite.go index e41b7148..69f52b79 100644 --- a/sqlite3/sqlite.go +++ b/sqlite3/sqlite.go @@ -34,9 +34,15 @@ const ( txInsertStmt = iota txFetchUsedByShaStmt txFetchLocationByShaStmt + txFetchLocUsedByShaStmt + txUpdateUsedByShaStmt + txtmpInsertStmt txtmpFetchUsedByShaStmt txtmpFetchLocationByShaStmt + txtmpFetchLocUsedByShaStmt + txtmpUpdateUsedByShaStmt + txMigrateCopy txMigrateClear txMigratePrep @@ -58,22 +64,28 @@ var blkqueries []string = []string{ } var txqueries []string = []string{ - txInsertStmt: "INSERT INTO tx (key, blockid, txoff, txlen, data) VALUES(?, ?, ?, ?, ?);", - txFetchUsedByShaStmt: "SELECT data FROM tx WHERE key = ?;", - txFetchLocationByShaStmt: "SELECT blockid, txoff, txlen FROM tx WHERE key = ?;", + txInsertStmt: "INSERT INTO tx (key, blockid, txoff, txlen, data) VALUES(?, ?, ?, ?, ?);", + txFetchUsedByShaStmt: "SELECT data FROM tx WHERE key = ?;", + txFetchLocationByShaStmt: "SELECT blockid, txoff, txlen FROM tx WHERE key = ?;", + txFetchLocUsedByShaStmt: "SELECT blockid, txoff, txlen, data FROM tx WHERE key = ?;", + txUpdateUsedByShaStmt: "UPDATE tx SET data = ? WHERE key = ?;", + txtmpInsertStmt: "INSERT INTO txtmp (key, blockid, txoff, txlen, data) VALUES(?, ?, ?, ?, ?);", txtmpFetchUsedByShaStmt: "SELECT data FROM txtmp WHERE key = ?;", txtmpFetchLocationByShaStmt: "SELECT blockid, txoff, txlen FROM txtmp WHERE key = ?;", - txMigrateCopy: "INSERT INTO tx (key, blockid, txoff, txlen, data) SELECT key, blockid, txoff, txlen, data FROM txtmp;", - txMigrateClear: "DELETE from txtmp;", - txMigratePrep: "DROP index IF EXISTS uniquetx;", - txMigrateFinish: "CREATE UNIQUE INDEX IF NOT EXISTS uniquetx ON tx (key);", - txMigrateCount: "SELECT COUNT(*) FROM txtmp;", - txPragmaVacuumOn: "PRAGMA auto_vacuum = FULL;", - txPragmaVacuumOff: "PRAGMA auto_vacuum = NONE;", - txVacuum: "VACUUM;", - txExistsShaStmt: "SELECT blockid FROM tx WHERE key = ?;", - txtmpExistsShaStmt: "SELECT blockid FROM txtmp WHERE key = ?;", + txtmpFetchLocUsedByShaStmt: "SELECT blockid, txoff, txlen, data FROM txtmp WHERE key = ?;", + txtmpUpdateUsedByShaStmt: "UPDATE txtmp SET data = ? WHERE key = ?;", + + txMigrateCopy: "INSERT INTO tx (key, blockid, txoff, txlen, data) SELECT key, blockid, txoff, txlen, data FROM txtmp;", + txMigrateClear: "DELETE from txtmp;", + txMigratePrep: "DROP index IF EXISTS uniquetx;", + txMigrateFinish: "CREATE UNIQUE INDEX IF NOT EXISTS uniquetx ON tx (key);", + txMigrateCount: "SELECT COUNT(*) FROM txtmp;", + txPragmaVacuumOn: "PRAGMA auto_vacuum = FULL;", + txPragmaVacuumOff: "PRAGMA auto_vacuum = NONE;", + txVacuum: "VACUUM;", + txExistsShaStmt: "SELECT blockid FROM tx WHERE key = ?;", + txtmpExistsShaStmt: "SELECT blockid FROM txtmp WHERE key = ?;", } var log seelog.LoggerInterface = seelog.Disabled @@ -269,6 +281,8 @@ func newOrCreateSqliteDB(filepath string, create bool) (pbdb btcdb.Db, err error bdb.blockCache.maxcount = 150 bdb.blockCache.blockMap = map[btcwire.ShaHash]*blockCacheObj{} + bdb.blockCache.blockMap = map[btcwire.ShaHash]*blockCacheObj{} + bdb.blockCache.blockHeightMap = map[int64]*blockCacheObj{} bdb.txCache.maxcount = 2000 bdb.txCache.txMap = map[btcwire.ShaHash]*txCacheObj{} @@ -325,6 +339,8 @@ func (db *SqliteDb) RollbackClose() { err := tx.tx.Rollback() if err != nil { log.Debugf("Rollback failed: %v", err) + } else { + tx.tx = nil } } db.close() @@ -477,8 +493,6 @@ func (db *SqliteDb) DropAfterBlockBySha(sha *btcwire.ShaHash) (err error) { db.dbLock.Lock() defer db.dbLock.Unlock() - db.InvalidateCache() - // This is a destructive operation and involves multiple requests // so requires a transaction, terminate any transaction to date // and start a new transaction @@ -491,6 +505,27 @@ func (db *SqliteDb) DropAfterBlockBySha(sha *btcwire.ShaHash) (err error) { return err } + var startheight int64 + + if db.lastBlkShaCached { + startheight = db.lastBlkIdx + } else { + querystr := "SELECT blockid FROM block ORDER BY blockid DESC;" + + tx := &db.txState + if tx.tx != nil { + row = tx.tx.QueryRow(querystr) + } else { + row = db.sqldb.QueryRow(querystr) + } + var startblkidx int64 + err = row.Scan(&startblkidx) + if err != nil { + log.Warnf("DropAfterBlockBySha:unable to fetch blockheight %v", err) + return err + } + startheight = startblkidx + } // also drop any cached sha data db.lastBlkShaCached = false @@ -499,22 +534,61 @@ func (db *SqliteDb) DropAfterBlockBySha(sha *btcwire.ShaHash) (err error) { tx := &db.txState row = tx.tx.QueryRow(querystr, sha.Bytes()) - var blockidx uint64 - err = row.Scan(&blockidx) + var keepidx int64 + err = row.Scan(&keepidx) if err != nil { // XXX db.endTx(false) return err } - _, err = tx.tx.Exec("DELETE FROM txtmp WHERE blockid > ?", blockidx) + for height := startheight; height > keepidx; height = height - 1 { + var blk *btcutil.Block + blkc, ok := db.fetchBlockHeightCache(height) + + if ok { + blk = blkc.blk + } else { + // must load the block from the db + sha, err = db.fetchBlockShaByHeight(height - 1) + if err != nil { + return + } + + var buf []byte + var pver uint32 + + buf, pver, _, err = db.fetchSha(*sha) + if err != nil { + return + } + + blk, err = btcutil.NewBlockFromBytes(buf, pver) + if err != nil { + return + } + } + + for _, tx := range blk.MsgBlock().Transactions { + err = db.unSpend(tx) + if err != nil { + return + } + } + } + + // invalidate the cache after possibly using cached entries for block + // lookup to unspend coins in them + db.InvalidateCache() + + _, err = tx.tx.Exec("DELETE FROM txtmp WHERE blockid > ?", keepidx) if err != nil { // XXX db.endTx(false) return err } - _, err = tx.tx.Exec("DELETE FROM tx WHERE blockid > ?", blockidx) + _, err = tx.tx.Exec("DELETE FROM tx WHERE blockid > ?", keepidx) if err != nil { // XXX db.endTx(false) @@ -522,7 +596,7 @@ func (db *SqliteDb) DropAfterBlockBySha(sha *btcwire.ShaHash) (err error) { } // delete from block last in case of foreign keys - _, err = tx.tx.Exec("DELETE FROM block WHERE blockid > ?", blockidx) + _, err = tx.tx.Exec("DELETE FROM block WHERE blockid > ?", keepidx) if err != nil { // XXX db.endTx(false) @@ -604,6 +678,9 @@ func (db *SqliteDb) InsertBlock(block *btcutil.Block) (height int64, err error) } spentbuflen := (len(tx.TxOut) + 7) / 8 spentbuf := make([]byte, spentbuflen, spentbuflen) + for i := uint(len(tx.TxOut) % 8); i < 8; i++ { + spentbuf[spentbuflen-1] |= (byte(1) << i) + } err = db.insertTx(&txsha, newheight, txloc[txidx].TxStart, txloc[txidx].TxLen, spentbuf) if err != nil { @@ -612,6 +689,12 @@ func (db *SqliteDb) InsertBlock(block *btcutil.Block) (height int64, err error) oBlkIdx, _, _, err = db.fetchLocationBySha(&txsha) log.Warnf("oblkidx %v err %v", oBlkIdx, err) + return + } + err = db.doSpend(tx) + if err != nil { + log.Warnf("block %v idx %v failed to spend tx %v %v err %v", &blocksha, newheight, &txsha, txidx, err) + return } } @@ -675,3 +758,100 @@ func (db *SqliteDb) SetDBInsertMode(newmode btcdb.InsertMode) { db.UseTempTX = false } } +func (db *SqliteDb) doSpend(tx *btcwire.MsgTx) error { + for txinidx := range tx.TxIn { + txin := tx.TxIn[txinidx] + + inTxSha := txin.PreviousOutpoint.Hash + inTxidx := txin.PreviousOutpoint.Index + + if inTxidx == ^uint32(0) { + continue + } + + //log.Infof("spending %v %v", &inTxSha, inTxidx) + + err := db.setSpentData(&inTxSha, inTxidx) + if err != nil { + return err + } + } + return nil +} + +func (db *SqliteDb) unSpend(tx *btcwire.MsgTx) error { + for txinidx := range tx.TxIn { + txin := tx.TxIn[txinidx] + + inTxSha := txin.PreviousOutpoint.Hash + inTxidx := txin.PreviousOutpoint.Index + + if inTxidx == ^uint32(0) { + continue + } + + err := db.clearSpentData(&inTxSha, inTxidx) + if err != nil { + return err + } + } + return nil +} + +func (db *SqliteDb) setSpentData(sha *btcwire.ShaHash, idx uint32) error { + return db.setclearSpentData(sha, idx, true) +} + +func (db *SqliteDb) clearSpentData(sha *btcwire.ShaHash, idx uint32) error { + return db.setclearSpentData(sha, idx, false) +} + +func (db *SqliteDb) setclearSpentData(txsha *btcwire.ShaHash, idx uint32, set bool) error { + var spentdata []byte + usingtmp := false + txop := db.txop(txFetchUsedByShaStmt) + row := txop.QueryRow(txsha.String()) + err := row.Scan(&spentdata) + if err != nil { + // if the error is simply didn't fine continue otherwise + // retun failure + + usingtmp = true + txop = db.txop(txtmpFetchUsedByShaStmt) + row := txop.QueryRow(txsha.String()) + err := row.Scan(&spentdata) + if err != nil { + log.Warnf("Failed to locate spent data - %v %v", txsha, err) + return err + } + } + byteidx := idx / 8 + byteoff := idx % 8 + + interestingsha := txsha.String() == "866504b0d6242c237e484b49c6a5db1c234e9a1547e96b7ba3d690cabccbf6b0" + + if interestingsha { + fmt.Printf("txdata was %v ", spentdata) + } + if set { + spentdata[byteidx] |= (byte(1) << byteoff) + } else { + spentdata[byteidx] &= ^(byte(1) << byteoff) + } + txc, cached := db.fetchTxCache(txsha) + if cached { + txc.spent = spentdata + } + if interestingsha { + fmt.Printf("now %v\n", spentdata) + } + + if usingtmp { + txop = db.txop(txtmpUpdateUsedByShaStmt) + } else { + txop = db.txop(txUpdateUsedByShaStmt) + } + _, err = txop.Exec(spentdata, txsha.String()) + + return err +} diff --git a/sqlite3/sqliteblock.go b/sqlite3/sqliteblock.go index b2c2b84e..5d9f5939 100644 --- a/sqlite3/sqliteblock.go +++ b/sqlite3/sqliteblock.go @@ -94,9 +94,6 @@ func (db *SqliteDb) insertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHa func (db *SqliteDb) fetchSha(sha btcwire.ShaHash) (buf []byte, pver uint32, blkid int64, err error) { - db.dbLock.Lock() - defer db.dbLock.Unlock() - row := db.blkStmts[blkFetchSha].QueryRow(sha.Bytes()) var blockidx int64 @@ -154,10 +151,17 @@ func (db *SqliteDb) blkExistsSha(sha *btcwire.ShaHash) bool { // FetchBlockShaByHeight returns a block hash based on its height in the // block chain. func (db *SqliteDb) FetchBlockShaByHeight(height int64) (sha *btcwire.ShaHash, err error) { - var row *sql.Row db.dbLock.Lock() defer db.dbLock.Unlock() + return db.fetchBlockShaByHeight(height) +} + +// fetchBlockShaByHeight returns a block hash based on its height in the +// block chain. +func (db *SqliteDb) fetchBlockShaByHeight(height int64) (sha *btcwire.ShaHash, err error) { + var row *sql.Row + blockidx := height + 1 // skew between btc blockid and sql row = db.blkStmts[blkFetchIdx].QueryRow(blockidx) diff --git a/sqlite3/sqliteblock_test.go b/sqlite3/sqliteblock_test.go deleted file mode 100644 index 12bac76d..00000000 --- a/sqlite3/sqliteblock_test.go +++ /dev/null @@ -1,292 +0,0 @@ -// Copyright (c) 2013 Conformal Systems LLC. -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package sqlite3_test - -import ( - "bytes" - "fmt" - "github.com/conformal/btcdb" - "github.com/conformal/btcdb/sqlite3" - "github.com/conformal/btcwire" - "os" - "testing" -) - -// array of shas -var testShas []btcwire.ShaHash = []btcwire.ShaHash{ - { - 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, - 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, - 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, - 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, - }, - { - 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, - 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, - 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, - 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, - }, - { - 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, - 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, - 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, - 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, - }, - { - 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, - 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, - 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, - 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, - }, - { - 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, - 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, - 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, - 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, - }, -} - -// Work around stupid go vet bug where any non array should have named -// initializers. Since ShaHash is a glorified array it shouldn't matter. -var badShaArray = [32]byte{ - 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, - 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, - 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, - 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, -} -var badSha btcwire.ShaHash = btcwire.ShaHash(badShaArray) -var zeroSha = btcwire.ShaHash{} -var zeroBlock []byte = make([]byte, 32) - -func compareArray(t *testing.T, one, two []btcwire.ShaHash, test string, - sync string) { - if len(one) != len(two) { - t.Errorf("%s: lengths don't match for arrays (%s)", test, sync) - return - } - - for i := range one { - if !one[i].IsEqual(&two[i]) { - t.Errorf("%s: %dth sha doesn't match (%s)", test, i, - sync) - } - } -} - -func testNewestSha(t *testing.T, db btcdb.Db, expSha btcwire.ShaHash, - expBlk int64, situation string) { - - newestsha, blkid, err := db.NewestSha() - if err != nil { - t.Errorf("NewestSha failed %v (%s)", err, situation) - return - } - if blkid != expBlk { - t.Errorf("NewestSha blkid is %d not %d (%s)", blkid, expBlk, - situation) - } - if !newestsha.IsEqual(&expSha) { - t.Errorf("Newestsha isn't the last sha we inserted %v %v (%s)", - newestsha, &expSha, situation) - } -} - -type fetchIdxTest struct { - start int64 - end int64 - exp []btcwire.ShaHash - test string -} - -func testFetch(t *testing.T, db btcdb.Db, shas []btcwire.ShaHash, - sync string) { - - // Test the newest sha is what we expect and call it twice to ensure - // caching is working working properly. - numShas := int64(len(shas)) - newestSha := shas[numShas-1] - newestBlockID := int64(numShas) - testNewestSha(t, db, newestSha, newestBlockID, sync) - testNewestSha(t, db, newestSha, newestBlockID, sync+" cached") - - for i, sha := range shas { - // Add one for genesis block skew. - i = i + 1 - - // Ensure the sha exists in the db as expected. - if !db.ExistsSha(&sha) { - t.Errorf("testSha %d doesn't exists (%s)", i, sync) - break - } - - // Fetch the sha from the db and ensure all fields are expected - // values. - buf, pver, idx, err := sqlite3.FetchSha(db, &sha) - if err != nil { - t.Errorf("Failed to fetch testSha %d (%s)", i, sync) - } - if !bytes.Equal(zeroBlock, buf) { - t.Errorf("testSha %d incorrect block return (%s)", i, - sync) - } - if pver != 1 { - t.Errorf("pver is %d and not 1 for testSha %d (%s)", - pver, i, sync) - } - if idx != int64(i) { - t.Errorf("index isn't as expected %d vs %d (%s)", - idx, i, sync) - } - - // Fetch the sha by index and ensure it matches. - tsha, err := db.FetchBlockShaByHeight(int64(i)) - if err != nil { - t.Errorf("can't fetch sha at index %d: %v", i, err) - continue - } - if !tsha.IsEqual(&sha) { - t.Errorf("sha for index %d isn't shas[%d]", i, i) - } - } - - endBlockID := numShas + 1 - midBlockID := endBlockID / 2 - fetchIdxTests := []fetchIdxTest{ - // All shas. - {1, btcdb.AllShas, shas, "fetch all shas"}, - - //// All shas using known bounds. - {1, endBlockID, shas, "fetch all shas2"}, - - // Partial list starting at beginning. - {1, midBlockID, shas[:midBlockID-1], "fetch first half"}, - - // Partial list ending at end. - {midBlockID, endBlockID, shas[midBlockID-1 : endBlockID-1], - "fetch second half"}, - - // Nonexistent off the end. - {endBlockID, endBlockID * 2, []btcwire.ShaHash{}, - "fetch nonexistent"}, - } - - for _, test := range fetchIdxTests { - t.Logf("numSha: %d - Fetch from %d to %d\n", numShas, test.start, test.end) - if shalist, err := db.FetchHeightRange(test.start, test.end); err == nil { - compareArray(t, shalist, test.exp, test.test, sync) - } else { - t.Errorf("failed to fetch index range for %s (%s)", - test.test, sync) - } - } - - // Try and fetch nonexistent sha. - if db.ExistsSha(&badSha) { - t.Errorf("nonexistent sha exists (%s)!", sync) - } - _, _, _, err := sqlite3.FetchSha(db, &badSha) - if err == nil { - t.Errorf("Success when fetching a bad sha! (%s)", sync) - } - // XXX if not check to see it is the right value? - - testIterator(t, db, shas, sync) -} - -func testIterator(t *testing.T, db btcdb.Db, shas []btcwire.ShaHash, - sync string) { - - // Iterate over the whole list of shas. - iter, err := db.NewIterateBlocks() - if err != nil { - t.Errorf("failed to create iterated blocks") - return - } - - // Skip the genesis block. - _ = iter.NextRow() - - i := 0 - for ; iter.NextRow(); i++ { - key, pver, buf, err := iter.Row() - if err != nil { - t.Errorf("iter.NextRow() failed: %v (%s)", err, sync) - break - } - if i >= len(shas) { - t.Errorf("iterator returned more shas than "+ - "expected - %d (%s)", i, sync) - break - } - if !key.IsEqual(&shas[i]) { - t.Errorf("iterator test: %dth sha doesn't match (%s)", - i, sync) - } - if !bytes.Equal(zeroBlock, buf) { - t.Errorf("iterator test: %d buf incorrect (%s)", i, - sync) - } - if pver != 1 { - t.Errorf("iterator: %dth pver is %d and not 1 (%s)", - i, pver, sync) - } - } - if i < len(shas) { - t.Errorf("iterator got no rows on %dth loop, should have %d "+ - "(%s)", i, len(shas), sync) - } - if _, _, _, err = iter.Row(); err == nil { - t.Errorf("done iterator didn't return failure") - } - iter.Close() -} - -func TestBdb(t *testing.T) { - // Ignore db remove errors since it means we didn't have an old one. - _ = os.Remove("tstdb1") - db, err := btcdb.CreateDB("sqlite", "tstdb1") - if err != nil { - t.Errorf("Failed to open test database %v", err) - return - } - defer os.Remove("tstdb1") - - for i := range testShas { - var previous btcwire.ShaHash - if i == 0 { - previous = btcwire.GenesisHash - } else { - previous = testShas[i-1] - } - _, err := db.InsertBlockData(&testShas[i], &previous, 1, zeroBlock) - if err != nil { - t.Errorf("Failed to insert testSha %d. Error: %v", - i, err) - return - } - - testFetch(t, db, testShas[0:i+1], "pre sync ") - } - - // XXX insert enough so that we hit the transaction limit - // XXX try and insert a with a bad previous - - db.Sync() - - testFetch(t, db, testShas, "post sync") - - for i := len(testShas) - 1; i >= 0; i-- { - err := db.DropAfterBlockBySha(&testShas[i]) - if err != nil { - t.Errorf("drop after %d failed %v", i, err) - break - } - testFetch(t, db, testShas[:i+1], - fmt.Sprintf("post DropAfter for sha %d", i)) - } - - // Just tests that it doesn't crash, no return value - db.Close() -} diff --git a/sqlite3/sqlitedbcache.go b/sqlite3/sqlitedbcache.go index 45de6cee..3875f3a2 100644 --- a/sqlite3/sqlitedbcache.go +++ b/sqlite3/sqlitedbcache.go @@ -7,6 +7,7 @@ package sqlite3 import ( "bytes" "container/list" + "fmt" "github.com/conformal/btcdb" "github.com/conformal/btcutil" "github.com/conformal/btcwire" @@ -27,14 +28,17 @@ type txCacheObj struct { blksha btcwire.ShaHash pver uint32 tx *btcwire.MsgTx + height int64 + spent []byte txbuf []byte } type blockCache struct { - maxcount int - fifo list.List - blockMap map[btcwire.ShaHash]*blockCacheObj - cacheLock sync.RWMutex + maxcount int + fifo list.List + blockMap map[btcwire.ShaHash]*blockCacheObj + blockHeightMap map[int64]*blockCacheObj + cacheLock sync.RWMutex } type blockCacheObj struct { @@ -45,6 +49,9 @@ type blockCacheObj struct { // FetchBlockBySha - return a btcutil Block, object may be a cached. func (db *SqliteDb) FetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + blkcache, ok := db.fetchBlockCache(sha) if ok { return blkcache.blk, nil @@ -78,6 +85,19 @@ func (db *SqliteDb) fetchBlockCache(sha *btcwire.ShaHash) (*blockCacheObj, bool) return blkobj, true } +// fetchBlockHeightCache check if a block is in the block cache, if so return it. +func (db *SqliteDb) fetchBlockHeightCache(height int64) (*blockCacheObj, bool) { + + db.blockCache.cacheLock.RLock() + defer db.blockCache.cacheLock.RUnlock() + + blkobj, ok := db.blockCache.blockHeightMap[height] + if !ok { // could this just return the map deref? + return nil, false + } + return blkobj, true +} + // insertBlockCache insert the given sha/block into the cache map. // If the block cache is determined to be full, it will release // an old entry in FIFO order. @@ -96,65 +116,85 @@ func (db *SqliteDb) insertBlockCache(sha *btcwire.ShaHash, blk *btcutil.Block) { tailObj, ok := listobj.Value.(*blockCacheObj) if ok { delete(bc.blockMap, tailObj.sha) + delete(bc.blockHeightMap, tailObj.blk.Height()) } else { panic("invalid type pushed on blockCache list") } } + bc.blockHeightMap[blk.Height()] = &blkObj bc.blockMap[blkObj.sha] = &blkObj } -type TxListReply struct { - Sha *btcwire.ShaHash - Tx *btcwire.MsgTx - Err error -} - // FetchTxByShaList given a array of ShaHash, look up the transactions // and return them in a TxListReply array. func (db *SqliteDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { var replies []*btcdb.TxListReply for _, txsha := range txShaList { - tx, _, _, err := db.FetchTxBySha(txsha) - txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, Err: err} + tx, _, _, _, height, txspent, err := db.fetchTxDataBySha(txsha) + btxspent := []bool{} + if err == nil { + btxspent = make([]bool, len(tx.TxOut), len(tx.TxOut)) + for idx := range tx.TxOut { + byteidx := idx / 8 + byteoff := uint(idx % 8) + btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0 + } + } + interestingsha := txsha.String() == "866504b0d6242c237e484b49c6a5db1c234e9a1547e96b7ba3d690cabccbf6b" + if interestingsha { + fmt.Printf("usage of sha %v is %v\n", txsha, btxspent) + } + fmt.Printf("usage of sha %v is %v\n", txsha, btxspent) + txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, Height: height, TxSpent: btxspent, Err: err} replies = append(replies, &txlre) } return replies } -// FetchTxAllBySha returns several pieces of data regarding the given sha. -func (db *SqliteDb) FetchTxAllBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rtxbuf []byte, rpver uint32, rblksha *btcwire.ShaHash, err error) { +// fetchTxDataBySha returns several pieces of data regarding the given sha. +func (db *SqliteDb) fetchTxDataBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rtxbuf []byte, rpver uint32, rblksha *btcwire.ShaHash, rheight int64, rtxspent []byte, err error) { + + var pver uint32 + var blksha *btcwire.ShaHash + var height int64 + var txspent []byte + var toff int + var tlen int + var blk *btcutil.Block + var blkbuf []byte // Check Tx cache if txc, ok := db.fetchTxCache(txsha); ok { - return txc.tx, txc.txbuf, txc.pver, &txc.blksha, nil + if txc.spent != nil { + return txc.tx, txc.txbuf, txc.pver, &txc.blksha, txc.height, txc.spent, nil + } } // If not cached load it - bidx, toff, tlen, err := db.FetchLocationBySha(txsha) + height, toff, tlen, txspent, err = db.fetchLocationUsedBySha(txsha) if err != nil { - log.Warnf("unable to find location of origin tx %v", txsha) return } - blksha, err := db.FetchBlockShaByHeight(bidx) + blksha, err = db.FetchBlockShaByHeight(height) if err != nil { - log.Warnf("block idx lookup %v to %v", bidx, err) + log.Warnf("block idx lookup %v to %v", height, err) return } log.Tracef("transaction %v is at block %v %v tx %v", - txsha, blksha, bidx, toff) + txsha, blksha, height, toff) - blk, err := db.FetchBlockBySha(blksha) + blk, err = db.FetchBlockBySha(blksha) if err != nil { log.Warnf("unable to fetch block %v %v ", - bidx, &blksha) + height, &blksha) return } - blkbuf, pver, err := blk.Bytes() + blkbuf, pver, err = blk.Bytes() if err != nil { - log.Warnf("unable to decode block %v %v", bidx, &blksha) + log.Warnf("unable to decode block %v %v", height, &blksha) return } @@ -166,7 +206,7 @@ func (db *SqliteDb) FetchTxAllBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, err = tx.BtcDecode(rbuf, pver) if err != nil { log.Warnf("unable to decode tx block %v %v txoff %v txlen %v", - bidx, &blksha, toff, tlen) + height, &blksha, toff, tlen) return } @@ -177,6 +217,76 @@ func (db *SqliteDb) FetchTxAllBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, txc.tx = &tx txc.txbuf = txbuf txc.pver = pver + txc.height = height + txc.spent = txspent + txc.blksha = *blksha + db.insertTxCache(&txc) + + return &tx, txbuf, pver, blksha, height, txspent, nil +} + +// FetchTxAllBySha returns several pieces of data regarding the given sha. +func (db *SqliteDb) FetchTxAllBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rtxbuf []byte, rpver uint32, rblksha *btcwire.ShaHash, err error) { + var pver uint32 + var blksha *btcwire.ShaHash + var height int64 + var toff int + var tlen int + var blk *btcutil.Block + var blkbuf []byte + + // Check Tx cache + if txc, ok := db.fetchTxCache(txsha); ok { + return txc.tx, txc.txbuf, txc.pver, &txc.blksha, nil + } + + // If not cached load it + height, toff, tlen, err = db.FetchLocationBySha(txsha) + if err != nil { + return + } + + blksha, err = db.FetchBlockShaByHeight(height) + if err != nil { + log.Warnf("block idx lookup %v to %v", height, err) + return + } + log.Tracef("transaction %v is at block %v %v tx %v", + txsha, blksha, height, toff) + + blk, err = db.FetchBlockBySha(blksha) + if err != nil { + log.Warnf("unable to fetch block %v %v ", + height, &blksha) + return + } + + blkbuf, pver, err = blk.Bytes() + if err != nil { + log.Warnf("unable to decode block %v %v", height, &blksha) + return + } + + txbuf := make([]byte, tlen) + copy(txbuf[:], blkbuf[toff:toff+tlen]) + rbuf := bytes.NewBuffer(txbuf) + + var tx btcwire.MsgTx + err = tx.BtcDecode(rbuf, pver) + if err != nil { + log.Warnf("unable to decode tx block %v %v txoff %v txlen %v", + height, &blksha, toff, tlen) + return + } + + // Shove data into TxCache + // XXX - + var txc txCacheObj + txc.sha = *txsha + txc.tx = &tx + txc.txbuf = txbuf + txc.pver = pver + txc.height = height txc.blksha = *blksha db.insertTxCache(&txc) @@ -251,6 +361,7 @@ func (db *SqliteDb) InvalidateBlockCache() { bc.cacheLock.Lock() defer bc.cacheLock.Unlock() bc.blockMap = map[btcwire.ShaHash]*blockCacheObj{} + bc.blockHeightMap = map[int64]*blockCacheObj{} bc.fifo = list.List{} } diff --git a/sqlite3/sqlitetx.go b/sqlite3/sqlitetx.go index 1c05aa7e..24a8e4cc 100644 --- a/sqlite3/sqlitetx.go +++ b/sqlite3/sqlitetx.go @@ -12,16 +12,16 @@ import ( ) // InsertTx inserts a tx hash and its associated data into the database. -func (db *SqliteDb) InsertTx(txsha *btcwire.ShaHash, blockidx int64, txoff int, txlen int, usedbuf []byte) (err error) { +func (db *SqliteDb) InsertTx(txsha *btcwire.ShaHash, height int64, txoff int, txlen int, usedbuf []byte) (err error) { db.dbLock.Lock() defer db.dbLock.Unlock() - return db.insertTx(txsha, blockidx, txoff, txlen, usedbuf) + return db.insertTx(txsha, height, txoff, txlen, usedbuf) } // insertTx inserts a tx hash and its associated data into the database. // Must be called with db lock held. -func (db *SqliteDb) insertTx(txsha *btcwire.ShaHash, blockidx int64, txoff int, txlen int, usedbuf []byte) (err error) { +func (db *SqliteDb) insertTx(txsha *btcwire.ShaHash, height int64, txoff int, txlen int, usedbuf []byte) (err error) { tx := &db.txState if tx.tx == nil { @@ -30,7 +30,7 @@ func (db *SqliteDb) insertTx(txsha *btcwire.ShaHash, blockidx int64, txoff int, return } } - blockid := blockidx + 1 + blockid := height + 1 txd := tTxInsertData{txsha: txsha, blockid: blockid, txoff: txoff, txlen: txlen, usedbuf: usedbuf} log.Tracef("inserting tx %v for block %v off %v len %v", @@ -84,7 +84,6 @@ func (db *SqliteDb) ExistsTxSha(txsha *btcwire.ShaHash) (exists bool) { return db.existsTxSha(txsha) } - // existsTxSha returns if the given tx sha exists in the database.o // Must be called with the db lock held. func (db *SqliteDb) existsTxSha(txsha *btcwire.ShaHash) (exists bool) { @@ -126,7 +125,7 @@ func (db *SqliteDb) FetchLocationBySha(txsha *btcwire.ShaHash) (blockidx int64, // fetchLocationBySha look up the Tx sha information by name. // Must be called with db lock held. -func (db *SqliteDb) fetchLocationBySha(txsha *btcwire.ShaHash) (blockidx int64, txoff int, txlen int, err error) { +func (db *SqliteDb) fetchLocationBySha(txsha *btcwire.ShaHash) (height int64, txoff int, txlen int, err error) { var row *sql.Row var blockid int64 var ttxoff int @@ -156,12 +155,49 @@ func (db *SqliteDb) fetchLocationBySha(txsha *btcwire.ShaHash) (blockidx int64, log.Warnf("FetchLocationBySha: fail %v", err) return } - blockidx = blockid - 1 + height = blockid - 1 txoff = ttxoff txlen = ttxlen return } +// fetchLocationUsedBySha look up the Tx sha information by name. +// Must be called with db lock held. +func (db *SqliteDb) fetchLocationUsedBySha(txsha *btcwire.ShaHash) (rheight int64, rtxoff int, rtxlen int, rspentbuf []byte, err error) { + var row *sql.Row + var blockid int64 + var txoff int + var txlen int + var txspent []byte + + rowBytes := txsha.String() + txop := db.txop(txFetchLocUsedByShaStmt) + row = txop.QueryRow(rowBytes) + + err = row.Scan(&blockid, &txoff, &txlen, &txspent) + if err == sql.ErrNoRows { + txop = db.txop(txtmpFetchLocUsedByShaStmt) + row = txop.QueryRow(rowBytes) + + err = row.Scan(&blockid, &txoff, &txlen, &txspent) + if err == sql.ErrNoRows { + err = btcdb.TxShaMissing + return + } + if err != nil { + log.Warnf("txtmp FetchLocationBySha: fail %v", + err) + return + } + } + if err != nil { + log.Warnf("FetchLocationBySha: fail %v", err) + return + } + height := blockid - 1 + return height, txoff, txlen, txspent, nil +} + // FetchTxUsedBySha returns the used/spent buffer for a given transaction. func (db *SqliteDb) FetchTxUsedBySha(txsha *btcwire.ShaHash) (spentbuf []byte, err error) { var row *sql.Row From bee05db6037b925e6bf1f8026ec06a27e2a03757 Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Thu, 18 Jul 2013 18:19:57 -0400 Subject: [PATCH 027/163] Exunge debug code. --- sqlite3/insertremove_test.go | 8 -------- sqlite3/sqlite.go | 8 -------- sqlite3/sqlitedbcache.go | 6 ------ 3 files changed, 22 deletions(-) diff --git a/sqlite3/insertremove_test.go b/sqlite3/insertremove_test.go index f639369e..4c26f10f 100644 --- a/sqlite3/insertremove_test.go +++ b/sqlite3/insertremove_test.go @@ -5,17 +5,12 @@ package sqlite3_test import ( - //"compress/bzip2" - //"encoding/binary" "github.com/conformal/btcdb" "github.com/conformal/btcdb/sqlite3" "github.com/conformal/btcutil" "github.com/conformal/btcwire" - //"io" - "fmt" "os" "path/filepath" - //"strings" "testing" ) @@ -149,7 +144,6 @@ endtest: for _, spend := range txInList { itxe := txlookupmap[spend.Hash] if itxe.TxSpent[spend.Index] == false { - fmt.Printf("?? txin %v:%v is unspent %v\n", spend.Hash, spend.Index, itxe.TxSpent) t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent) } } @@ -188,7 +182,6 @@ endtest: for _, spend := range txInList { itxe := txlookupmap[spend.Hash] if itxe.TxSpent[spend.Index] == true { - fmt.Printf("?? txin %v:%v is spent %v\n", spend.Hash, spend.Index, itxe.TxSpent) t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent) } } @@ -209,7 +202,6 @@ endtest: for _, spend := range txInList { itxe := txlookupmap[spend.Hash] if itxe.TxSpent[spend.Index] == false { - fmt.Printf("?? txin %v:%v is unspent %v\n", spend.Hash, spend.Index, itxe.TxSpent) t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent) } } diff --git a/sqlite3/sqlite.go b/sqlite3/sqlite.go index 69f52b79..6e16d70a 100644 --- a/sqlite3/sqlite.go +++ b/sqlite3/sqlite.go @@ -828,11 +828,6 @@ func (db *SqliteDb) setclearSpentData(txsha *btcwire.ShaHash, idx uint32, set bo byteidx := idx / 8 byteoff := idx % 8 - interestingsha := txsha.String() == "866504b0d6242c237e484b49c6a5db1c234e9a1547e96b7ba3d690cabccbf6b0" - - if interestingsha { - fmt.Printf("txdata was %v ", spentdata) - } if set { spentdata[byteidx] |= (byte(1) << byteoff) } else { @@ -842,9 +837,6 @@ func (db *SqliteDb) setclearSpentData(txsha *btcwire.ShaHash, idx uint32, set bo if cached { txc.spent = spentdata } - if interestingsha { - fmt.Printf("now %v\n", spentdata) - } if usingtmp { txop = db.txop(txtmpUpdateUsedByShaStmt) diff --git a/sqlite3/sqlitedbcache.go b/sqlite3/sqlitedbcache.go index 3875f3a2..40813afa 100644 --- a/sqlite3/sqlitedbcache.go +++ b/sqlite3/sqlitedbcache.go @@ -7,7 +7,6 @@ package sqlite3 import ( "bytes" "container/list" - "fmt" "github.com/conformal/btcdb" "github.com/conformal/btcutil" "github.com/conformal/btcwire" @@ -141,11 +140,6 @@ func (db *SqliteDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxLi btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0 } } - interestingsha := txsha.String() == "866504b0d6242c237e484b49c6a5db1c234e9a1547e96b7ba3d690cabccbf6b" - if interestingsha { - fmt.Printf("usage of sha %v is %v\n", txsha, btxspent) - } - fmt.Printf("usage of sha %v is %v\n", txsha, btxspent) txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, Height: height, TxSpent: btxspent, Err: err} replies = append(replies, &txlre) } From 92651c6d138cd0dc19824de7d54bed0cb2145e4f Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Thu, 18 Jul 2013 16:43:36 -0400 Subject: [PATCH 028/163] Fix spent computation on a multiple of 8 txout tx. --- sqlite3/sqlite.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sqlite3/sqlite.go b/sqlite3/sqlite.go index 6e16d70a..8807c0ae 100644 --- a/sqlite3/sqlite.go +++ b/sqlite3/sqlite.go @@ -678,8 +678,10 @@ func (db *SqliteDb) InsertBlock(block *btcutil.Block) (height int64, err error) } spentbuflen := (len(tx.TxOut) + 7) / 8 spentbuf := make([]byte, spentbuflen, spentbuflen) - for i := uint(len(tx.TxOut) % 8); i < 8; i++ { - spentbuf[spentbuflen-1] |= (byte(1) << i) + if len(tx.TxOut)%8 != 0 { + for i := uint(len(tx.TxOut) % 8); i < 8; i++ { + spentbuf[spentbuflen-1] |= (byte(1) << i) + } } err = db.insertTx(&txsha, newheight, txloc[txidx].TxStart, txloc[txidx].TxLen, spentbuf) From 584481280c961ee3877af324d8f010fbc7191d68 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Mon, 29 Jul 2013 10:52:39 -0500 Subject: [PATCH 029/163] Print actual hash on a couple of log warnings. The blocksha variable is already a pointer. --- sqlite3/sqlite.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlite3/sqlite.go b/sqlite3/sqlite.go index 8807c0ae..2bce6fe0 100644 --- a/sqlite3/sqlite.go +++ b/sqlite3/sqlite.go @@ -686,7 +686,7 @@ func (db *SqliteDb) InsertBlock(block *btcutil.Block) (height int64, err error) err = db.insertTx(&txsha, newheight, txloc[txidx].TxStart, txloc[txidx].TxLen, spentbuf) if err != nil { - log.Warnf("block %v idx %v failed to insert tx %v %v err %v", &blocksha, newheight, &txsha, txidx, err) + log.Warnf("block %v idx %v failed to insert tx %v %v err %v", blocksha, newheight, &txsha, txidx, err) var oBlkIdx int64 oBlkIdx, _, _, err = db.fetchLocationBySha(&txsha) log.Warnf("oblkidx %v err %v", oBlkIdx, err) @@ -695,7 +695,7 @@ func (db *SqliteDb) InsertBlock(block *btcutil.Block) (height int64, err error) } err = db.doSpend(tx) if err != nil { - log.Warnf("block %v idx %v failed to spend tx %v %v err %v", &blocksha, newheight, &txsha, txidx, err) + log.Warnf("block %v idx %v failed to spend tx %v %v err %v", blocksha, newheight, &txsha, txidx, err) return } From 25684a2ccb3c87ef8d4a9a7231efb95113b4d071 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Thu, 25 Jul 2013 16:44:18 -0500 Subject: [PATCH 030/163] Don't add a genesis block by default. This commit modifies the way initial database creation is handled. Previously, the genesis for the main chain was inserted automatically upon creation of the database. However, that approach caused an issue since other networks such as the test network don't use the same genesis block as the main network. The new approach introduced by this commit is to leave it up to the caller to insert the desired genesis block. In order to support this, the InsertBlock function has been modified to allow the first (and only the first) block to be inserted without having an existing parent. Also, the NewestSha function has been modified to return a zero hash, -1 for the height, and no error when the database does not yet have any blocks. This allows the caller to determine the difference between no blocks and only the genesis block (in which case the return values would be the genesis hash and 0 for the height). --- README.md | 25 +++++++++++++++-- db.go | 10 ++++--- doc.go | 32 ++++++++++++---------- sqlite3/insertremove_test.go | 2 +- sqlite3/operational_test.go | 11 ++++---- sqlite3/sqlite.go | 12 +++------ sqlite3/sqliteblock.go | 52 +++++++++++++----------------------- 7 files changed, 78 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 7dafbbc4..25a3f2db 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,30 @@ change. ## Sample Use ```Go - db, err := btcdb.CreateDB("sqlite", "dbexample") + // Import packages. + import ( + "github.com/conformal/btcdb" + _ "github.com/conformal/btcdb/sqlite3" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" + ) + + // Create a database and schedule it to be closed on exit. + dbName := "example.db" + db, err := btcdb.CreateDB("sqlite", dbName) + if err != nil { + // Log and handle the error + } + defer db.Close() + + + // Insert the main network genesis block. + pver := btcwire.ProtocolVersion + genesis := btcutil.NewBlock(&btcwire.GenesisBlock, pver) newHeight, err := db.InsertBlock(block) - db.Sync() + if err != nil { + // Log and handle the error + } ``` ## Documentation diff --git a/db.go b/db.go index 81cc9fee..808417ef 100644 --- a/db.go +++ b/db.go @@ -96,7 +96,9 @@ type Db interface { FetchTxUsedBySha(txsha *btcwire.ShaHash) (spentbuf []byte, err error) // InsertBlock inserts raw block and transaction data from a block - // into the database. + // into the database. The first block inserted into the database + // will be treated as the genesis block. Every subsequent block insert + // requires the referenced parent block to already exist. InsertBlock(block *btcutil.Block) (height int64, err error) // InsertBlockData stores a block hash and its associated data block @@ -119,8 +121,10 @@ type Db interface { // NewIterateBlocks returns an iterator for all blocks in database. NewIterateBlocks() (pbi BlockIterator, err error) - // NewestSha provides an interface to quickly look up the hash of - // the most recent (end) block of the block chain. + // NewestSha returns the hash and block height of the most recent (end) + // block of the block chain. It will return the zero hash, -1 for + // the block height, and no error (nil) if there are not any blocks in + // the database yet. NewestSha() (sha *btcwire.ShaHash, height int64, err error) // RollbackClose discards the recent database changes to the previously diff --git a/doc.go b/doc.go index 94b9e1af..1706110e 100644 --- a/doc.go +++ b/doc.go @@ -21,35 +21,39 @@ although a block can have a variable number of transactions. Along with these two items, several convenience functions for dealing with the database are provided as well as functions to query specific items that may be present in a block or tx (although many of these are in -the db_sqlite subpackage). +the sqlite3 subpackage). Usage At the highest level, the use of this packages just requires that you import it, setup a database, insert some data into it, and optionally, -query the data back. In a more concrete example: +query the data back. The first block inserted into the database will be +treated as the genesis block. Every subsequent block insert requires the +referenced parent block to already exist. In a more concrete example: - // Import packages. - import ( - "github.com/conformal/btcdb" - _ "github.com/conformal/btcdb/sqlite3" - ) + // Import packages. + import ( + "github.com/conformal/btcdb" + _ "github.com/conformal/btcdb/sqlite3" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" + ) // Create a database and schedule it to be closed on exit. - dbName := "example.db" - db, err := btcdb.CreateDB("sqlite", dbName) + dbName := "example.db" + db, err := btcdb.CreateDB("sqlite", dbName) if err != nil { // Log and handle the error } defer db.Close() - // Insert a block. - newHeight, err := db.InsertBlock(block) + + // Insert the main network genesis block. + pver := btcwire.ProtocolVersion + genesis := btcutil.NewBlock(&btcwire.GenesisBlock, pver) + newHeight, err := db.InsertBlock(block) if err != nil { // Log and handle the error } - - // Sync the database. - db.Sync() */ package btcdb diff --git a/sqlite3/insertremove_test.go b/sqlite3/insertremove_test.go index 4c26f10f..de27588a 100644 --- a/sqlite3/insertremove_test.go +++ b/sqlite3/insertremove_test.go @@ -76,7 +76,7 @@ func testUnspentInsert(t *testing.T, mode int) { blocks := loadblocks(t) endtest: - for height := int64(1); height < int64(len(blocks)); height++ { + for height := int64(0); height < int64(len(blocks)); height++ { block := blocks[height] // look up inputs to this x diff --git a/sqlite3/operational_test.go b/sqlite3/operational_test.go index 5ddc0b3a..8569ae15 100644 --- a/sqlite3/operational_test.go +++ b/sqlite3/operational_test.go @@ -81,7 +81,7 @@ func testOperationalMode(t *testing.T, mode int) { err = nil out: - for height := int64(1); height < int64(len(blocks)); height++ { + for height := int64(0); height < int64(len(blocks)); height++ { block := blocks[height] if mode != dbTmNoVerify { // except for NoVerify which does not allow lookups check inputs @@ -203,7 +203,7 @@ func testBackout(t *testing.T, mode int) { } err = nil - for height := int64(1); height < int64(len(blocks)); height++ { + for height := int64(0); height < int64(len(blocks)); height++ { if height == 100 { t.Logf("Syncing at block height 100") db.Sync() @@ -305,10 +305,11 @@ func loadBlocks(t *testing.T, file string) (blocks []*btcutil.Block, err error) } }() - var block *btcutil.Block - // block 0 isn't really there, put in nil - blocks = append(blocks, block) + // Set the first block as the genesis block. + genesis := btcutil.NewBlock(&btcwire.GenesisBlock, btcwire.ProtocolVersion) + blocks = append(blocks, genesis) + var block *btcutil.Block err = nil for height := int64(1); err == nil; height++ { var rintbuf uint32 diff --git a/sqlite3/sqlite.go b/sqlite3/sqlite.go index 2bce6fe0..b56e90a1 100644 --- a/sqlite3/sqlite.go +++ b/sqlite3/sqlite.go @@ -178,12 +178,6 @@ func createDB(db *sql.DB) error { } } - // Insert the genesis block. - err := insertGenesisBlock(db) - if err != nil { - return err - } - return nil } @@ -610,8 +604,10 @@ func (db *SqliteDb) DropAfterBlockBySha(sha *btcwire.ShaHash) (err error) { return } -// InsertBlock inserts the block data and transaction data from a block -// into the database. +// InsertBlock inserts raw block and transaction data from a block into the +// database. The first block inserted into the database will be treated as the +// genesis block. Every subsequent block insert requires the referenced parent +// block to already exist. func (db *SqliteDb) InsertBlock(block *btcutil.Block) (height int64, err error) { db.dbLock.Lock() defer db.dbLock.Unlock() diff --git a/sqlite3/sqliteblock.go b/sqlite3/sqliteblock.go index 5d9f5939..e5a43ddf 100644 --- a/sqlite3/sqliteblock.go +++ b/sqlite3/sqliteblock.go @@ -5,36 +5,12 @@ package sqlite3 import ( - "bytes" "database/sql" "github.com/conformal/btcdb" "github.com/conformal/btcwire" _ "github.com/mattn/go-sqlite3" ) -// insertGenesisBlock inserts the genesis block of the block chain into the -// database. -func insertGenesisBlock(db *sql.DB) error { - // Encode the genesis block to raw bytes. - pver := uint32(btcwire.ProtocolVersion) - var buf bytes.Buffer - err := btcwire.GenesisBlock.BtcEncode(&buf, pver) - if err != nil { - return err - } - - // Insert the genesis block along with its hash and protocol encoding - // version. - sql := blkqueries[blkInsertSha] - sha := btcwire.GenesisHash - _, err = db.Exec(sql, sha.Bytes(), pver, buf.Bytes()) - if err != nil { - return err - } - - return nil -} - // InsertBlockData stores a block hash and its associated data block with a // previous sha of `prevSha' and a version of `pver'. func (db *SqliteDb) InsertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHash, pver uint32, buf []byte) (blockid int64, err error) { @@ -56,12 +32,18 @@ func (db *SqliteDb) insertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHa } } - var prevOk bool - var blkid int64 - - prevOk = db.blkExistsSha(prevSha) // exists -> ok - if !prevOk { - return 0, btcdb.PrevShaMissing + // It is an error if the previous block does not already exist in the + // database, unless there are no blocks at all. + if prevOk := db.blkExistsSha(prevSha); !prevOk { + var numBlocks uint64 + querystr := "SELECT COUNT(blockid) FROM block;" + err := db.sqldb.QueryRow(querystr).Scan(&numBlocks) + if err != nil { + return 0, err + } + if numBlocks != 0 { + return 0, btcdb.PrevShaMissing + } } result, err := db.blkStmts[blkInsertSha].Exec(sha.Bytes(), pver, buf) @@ -69,7 +51,7 @@ func (db *SqliteDb) insertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHa return } - blkid, err = result.LastInsertId() + blkid, err := result.LastInsertId() if err != nil { return 0, err } @@ -218,8 +200,9 @@ func (db *SqliteDb) FetchHeightRange(startHeight, endHeight int64) (rshalist []b return } -// NewestSha provides an interface to quickly look up the sha of -// the most recent (end) of the block chain. +// NewestSha returns the hash and block height of the most recent (end) block of +// the block chain. It will return the zero hash, -1 for the block height, and +// no error (nil) if there are not any blocks in the database yet. func (db *SqliteDb) NewestSha() (sha *btcwire.ShaHash, blkid int64, err error) { var row *sql.Row var blockidx int64 @@ -245,6 +228,9 @@ func (db *SqliteDb) NewestSha() (sha *btcwire.ShaHash, blkid int64, err error) { var shabytes []byte err = row.Scan(&shabytes, &blockidx) + if err == sql.ErrNoRows { + return &btcwire.ShaHash{}, -1, nil + } if err == nil { var retsha btcwire.ShaHash retsha.SetBytes(shabytes) From dc3a2dbac0bf36ba9e129a7e7922d341da4d547b Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Mon, 29 Jul 2013 12:41:30 -0500 Subject: [PATCH 031/163] Correct example usage. The variable to insert is 'genesis', not 'block'. --- README.md | 2 +- doc.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 25a3f2db..22dc0674 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ change. // Insert the main network genesis block. pver := btcwire.ProtocolVersion genesis := btcutil.NewBlock(&btcwire.GenesisBlock, pver) - newHeight, err := db.InsertBlock(block) + newHeight, err := db.InsertBlock(genesis) if err != nil { // Log and handle the error } diff --git a/doc.go b/doc.go index 1706110e..933bd21d 100644 --- a/doc.go +++ b/doc.go @@ -51,7 +51,7 @@ referenced parent block to already exist. In a more concrete example: // Insert the main network genesis block. pver := btcwire.ProtocolVersion genesis := btcutil.NewBlock(&btcwire.GenesisBlock, pver) - newHeight, err := db.InsertBlock(block) + newHeight, err := db.InsertBlock(genesis) if err != nil { // Log and handle the error } From f205ff5ac01ff80c72789fb490ffe1b826ba8d77 Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Wed, 31 Jul 2013 10:46:28 -0400 Subject: [PATCH 032/163] External API functions need to grab locks before performing internal database operations. Untangle internal and external functions, to prevent recursive locks --- sqlite3/sqlitedbcache.go | 13 +++++++++++-- sqlite3/sqlitetx.go | 2 ++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/sqlite3/sqlitedbcache.go b/sqlite3/sqlitedbcache.go index 40813afa..3ed53318 100644 --- a/sqlite3/sqlitedbcache.go +++ b/sqlite3/sqlitedbcache.go @@ -50,6 +50,12 @@ type blockCacheObj struct { func (db *SqliteDb) FetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) { db.dbLock.Lock() defer db.dbLock.Unlock() + return db.fetchBlockBySha(sha) +} + +// fetchBlockBySha - return a btcutil Block, object may be a cached. +// Must be called with db lock held. +func (db *SqliteDb) fetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) { blkcache, ok := db.fetchBlockCache(sha) if ok { @@ -128,6 +134,9 @@ func (db *SqliteDb) insertBlockCache(sha *btcwire.ShaHash, blk *btcutil.Block) { // FetchTxByShaList given a array of ShaHash, look up the transactions // and return them in a TxListReply array. func (db *SqliteDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { + db.dbLock.Lock() + defer db.dbLock.Unlock() + var replies []*btcdb.TxListReply for _, txsha := range txShaList { tx, _, _, _, height, txspent, err := db.fetchTxDataBySha(txsha) @@ -171,7 +180,7 @@ func (db *SqliteDb) fetchTxDataBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx return } - blksha, err = db.FetchBlockShaByHeight(height) + blksha, err = db.fetchBlockShaByHeight(height) if err != nil { log.Warnf("block idx lookup %v to %v", height, err) return @@ -179,7 +188,7 @@ func (db *SqliteDb) fetchTxDataBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx log.Tracef("transaction %v is at block %v %v tx %v", txsha, blksha, height, toff) - blk, err = db.FetchBlockBySha(blksha) + blk, err = db.fetchBlockBySha(blksha) if err != nil { log.Warnf("unable to fetch block %v %v ", height, &blksha) diff --git a/sqlite3/sqlitetx.go b/sqlite3/sqlitetx.go index 24a8e4cc..c735f0f9 100644 --- a/sqlite3/sqlitetx.go +++ b/sqlite3/sqlitetx.go @@ -120,6 +120,8 @@ func (db *SqliteDb) existsTxSha(txsha *btcwire.ShaHash) (exists bool) { // FetchLocationBySha looks up the Tx sha information by name. func (db *SqliteDb) FetchLocationBySha(txsha *btcwire.ShaHash) (blockidx int64, txoff int, txlen int, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() return db.fetchLocationBySha(txsha) } From b46d53c18e16396bbe4b554d6c0935f3f9f1925a Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Thu, 1 Aug 2013 15:03:26 -0400 Subject: [PATCH 033/163] Test NewestSha() --- sqlite3/operational_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/sqlite3/operational_test.go b/sqlite3/operational_test.go index 8569ae15..b7208f97 100644 --- a/sqlite3/operational_test.go +++ b/sqlite3/operational_test.go @@ -140,6 +140,20 @@ out: t.Errorf("height mismatch expect %v returned %v", height, newheight) break out } + + newSha, blkid, err := db.NewestSha() + if err != nil { + t.Errorf("failed to obtain latest sha %v %v", height, err) + } + + if blkid != height { + t.Errorf("height doe not match latest block height %v %v", blkid, height, err) + } + + blkSha, _ := block.Sha() + if *newSha != *blkSha { + t.Errorf("Newest block sha does not match freshly inserted one %v %v ", newSha, blkSha, err) + } } switch mode { From ead14e5a12f9d6b458ffb3984686aaefa44aa3dd Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Thu, 1 Aug 2013 16:14:22 -0400 Subject: [PATCH 034/163] Testing code for FetchHeightRange() --- sqlite3/operational_test.go | 54 +++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/sqlite3/operational_test.go b/sqlite3/operational_test.go index b7208f97..164368ea 100644 --- a/sqlite3/operational_test.go +++ b/sqlite3/operational_test.go @@ -156,6 +156,10 @@ out: } } + + // now that db is populated, do some additional test + testFetchRangeHeight(t, db, blocks) + switch mode { case dbTmDefault: // default // no cleanup @@ -367,3 +371,53 @@ func loadBlocks(t *testing.T, file string) (blocks []*btcutil.Block, err error) } return } + + +func testFetchRangeHeight(t *testing.T, db btcdb.Db, blocks []*btcutil.Block) () { + + var testincrement int64 = 50 + var testcnt int64 = 100 + + shanames := make([]*btcwire.ShaHash, len(blocks)) + + nBlocks := int64(len(blocks)) + + for i := range blocks { + blockSha, err := blocks[i].Sha() + if err != nil { + t.Errorf("FetchRangeHeight: unexpected failure computing block sah %v", err) + } + shanames[i] = blockSha + } + + for startheight := int64(0); startheight < nBlocks; startheight += testincrement { + endheight := startheight + testcnt + + if endheight > nBlocks { + endheight = btcdb.AllShas + } + + shalist, err := db.FetchHeightRange(startheight, endheight) + if err != nil { + t.Errorf("FetchRangeHeight: unexpected failure looking up shas %v", err) + } + + if endheight == btcdb.AllShas { + if int64(len(shalist)) != nBlocks - startheight { + t.Errorf("FetchRangeHeight: expected A %v shas, got %v", nBlocks - startheight, len(shalist)) + } + } else { + if int64(len(shalist)) != testcnt { + t.Errorf("FetchRangeHeight: expected %v shas, got %v", testcnt, len(shalist)) + } + } + + for i := range shalist { + if *shanames[int64(i)+startheight] != shalist[i] { + t.Errorf("FetchRangeHeight: mismatch sha at %v requested range %v %v ", int64(i)+startheight, startheight, endheight) + } + } + } + + +} From 85d97d74369b8113f58045844cbd150008923dd8 Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Fri, 2 Aug 2013 18:31:21 -0400 Subject: [PATCH 035/163] block converted. --- leveldb/block.go | 293 ++++++++++++ leveldb/dbcache.go | 375 +++++++++++++++ leveldb/doc.go | 15 + leveldb/insertremove_test.go | 209 ++++++++ leveldb/internal_test.go | 48 ++ leveldb/leveldb.go | 798 +++++++++++++++++++++++++++++++ leveldb/operational_test.go | 420 ++++++++++++++++ leveldb/testdata/blocks1-256.bz2 | Bin 0 -> 37555 bytes leveldb/tx.go | 335 +++++++++++++ 9 files changed, 2493 insertions(+) create mode 100644 leveldb/block.go create mode 100644 leveldb/dbcache.go create mode 100644 leveldb/doc.go create mode 100644 leveldb/insertremove_test.go create mode 100644 leveldb/internal_test.go create mode 100644 leveldb/leveldb.go create mode 100644 leveldb/operational_test.go create mode 100644 leveldb/testdata/blocks1-256.bz2 create mode 100644 leveldb/tx.go diff --git a/leveldb/block.go b/leveldb/block.go new file mode 100644 index 00000000..145a89a7 --- /dev/null +++ b/leveldb/block.go @@ -0,0 +1,293 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ldb + +import ( + "bytes" + "fmt" + "encoding/binary" + "errors" + "github.com/conformal/btcdb" + "github.com/conformal/btcwire" +) + +// InsertBlockData stores a block hash and its associated data block with a +// previous sha of `prevSha' and a version of `pver'. +func (db *LevelDb) InsertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHash, pver uint32, buf []byte) (blockid int64, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + return db.insertBlockData(sha, prevSha, pver, buf) +} + +func (db *LevelDb) getBlkLoc(sha *btcwire.ShaHash) (int64, int, error) { + var blkHeight int64 + var blkFile int + + key := sha.Bytes() + + data, err := db.bShaDb.Get(key, db.ro) + + if err != nil { + return 0, 0, err + } + + // deserialize + dr := bytes.NewBuffer(data) + err = binary.Read(dr, binary.LittleEndian, &blkHeight) + if err != nil { + err = errors.New("Db Corrupt") + return 0, 0, err + } + err = binary.Read(dr, binary.LittleEndian, &blkFile) + if err != nil { + err = errors.New("Db Corrupt") + return 0, 0, err + } + return blkHeight, blkFile, nil +} + +func (db *LevelDb) getBlkByHeight(blkHeight int64, blkFile int) (rsha *btcwire.ShaHash, rbuf []byte, err error) { + var blkVal []byte + + key := fmt.Sprintf("%d",blkHeight) + + blkVal, err = db.bBlkDb[blkFile].Get([]byte(key), db.ro) + if err != nil { + return // exists ??? + } + + var sha btcwire.ShaHash + + sha.SetBytes(blkVal[0:31]) + + return &sha, blkVal[32:], nil +} + +func (db *LevelDb) getBlk(sha *btcwire.ShaHash) (rblkHeight int64, rblkFile int, rbuf []byte, err error) { + var blkHeight int64 + var blkFile int + + blkHeight, blkFile, err = db.getBlkLoc(sha) + if err != nil { + return + } + + var buf []byte + + _, buf, err = db.getBlkByHeight(blkHeight, blkFile) + if err != nil { + return + } + return blkHeight, blkFile, buf, nil +} + +func (db *LevelDb) setBlk(sha *btcwire.ShaHash, blkHeight int64, blkFile int, buf []byte) (error) { + + // serialize + var lw bytes.Buffer + err := binary.Write(&lw, binary.LittleEndian, &blkHeight) + if err != nil { + err = errors.New("Write fail") + return err + } + err = binary.Write(&lw, binary.LittleEndian, &blkFile) + if err != nil { + err = errors.New("Write fail") + return err + } + key := sha.Bytes() + + err = db.bShaDb.Put(key, lw.Bytes(), db.wo) + + if err != nil { + return err + } + + key = []byte(fmt.Sprintf("%d",blkHeight)) + + shaB := sha.Bytes() + blkVal := make([]byte, len(shaB) + len(buf)) + copy (blkVal[0:], shaB) + copy (blkVal[len(shaB):], buf) + err = db.bBlkDb[blkFile].Put(key, blkVal, db.wo) + + return nil +} + +// insertSha stores a block hash and its associated data block with a +// previous sha of `prevSha' and a version of `pver'. +// insertSha shall be called with db lock held +func (db *LevelDb) insertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHash, pver uint32, buf []byte) (blockid int64, err error) { + tx := &db.txState + if tx.tx == nil { + err = db.startTx() + if err != nil { + return + } + } + + oBlkHeight, _, err:= db.getBlkLoc(prevSha) + + if err != nil { + // check current block count + // if count != 0 { + // err = btcdb.PrevShaMissing + // return + // } + oBlkHeight = -1 + } + + // TODO(drahn) check curfile filesize, increment curfile if this puts it over + curFile := 0 + blkHeight := oBlkHeight - 1 + + err = db.setBlk(sha, blkHeight, curFile, buf) + + if err != nil { + return + } + + // update the last block cache + db.lastBlkShaCached = true + db.lastBlkSha = *sha + db.lastBlkIdx = blkHeight + + return blkHeight, nil +} + +// fetchSha returns the datablock and pver for the given ShaHash. +func (db *LevelDb) fetchSha(sha *btcwire.ShaHash) (rbuf []byte, rpver uint32, + rblkHeight int64, err error) { + var blkHeight int64 + var buf []byte + + blkHeight, _, buf, err = db.getBlk(sha) + if err != nil { + return + } + + fakepver := uint32(1) + + return buf, fakepver, blkHeight, nil +} + +// ExistsSha looks up the given block hash +// returns true if it is present in the database. +func (db *LevelDb) ExistsSha(sha *btcwire.ShaHash) (exists bool) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + _, exists = db.fetchBlockCache(sha) + if exists { + return + } + + // not in cache, try database + exists = db.blkExistsSha(sha) + return +} + +// blkExistsSha looks up the given block hash +// returns true if it is present in the database. +// CALLED WITH LOCK HELD +func (db *LevelDb) blkExistsSha(sha *btcwire.ShaHash) bool { + var pver uint32 + + oBlkHeight, _, err:= db.getBlkLoc(sha) + + if err != nil { + /* + should this warn if the failure is something besides does not exist ? + log.Warnf("blkExistsSha: fail %v", err) + */ + return false + } + return true +} + +// FetchBlockShaByHeight returns a block hash based on its height in the +// block chain. +func (db *LevelDb) FetchBlockShaByHeight(height int64) (sha *btcwire.ShaHash, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + return db.fetchBlockShaByHeight(height) +} + +// fetchBlockShaByHeight returns a block hash based on its height in the +// block chain. +func (db *LevelDb) fetchBlockShaByHeight(height int64) (sha *btcwire.ShaHash, err error) { + + // TODO(drahn) figure out which file blkHeight is located + blkFile := 0 + var buf []byte + + _, buf, err = db.getBlkByHeight(height, blkFile) + + var shaval btcwire.ShaHash + shaval.SetBytes(buf[0:31]) + return &shaval, nil +} + +// FetchHeightRange looks up a range of blocks by the start and ending +// heights. Fetch is inclusive of the start height and exclusive of the +// ending height. To fetch all hashes from the start height until no +// more are present, use the special id `AllShas'. +func (db *LevelDb) FetchHeightRange(startHeight, endHeight int64) (rshalist []btcwire.ShaHash, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + var endidx int64 + if endHeight == btcdb.AllShas { + endidx = startHeight + 500 + } else { + endidx = endHeight + } + + var shalist []btcwire.ShaHash + for height := startHeight; height < endidx; height++ { + // TODO(drahn) fix blkFile from height + blkFile := 0 + + key := fmt.Sprintf("%d", height) + blkVal, lerr := db.bBlkDb[blkFile].Get([]byte(key), db.ro) + if lerr != nil { + break + } + + var sha btcwire.ShaHash + sha.SetBytes(blkVal[0:31]) + shalist = append(shalist, sha) + } + + if err == nil { + return + } + log.Tracef("FetchIdxRange idx %v %v returned %v shas err %v", startHeight, endHeight, len(shalist), err) + + return shalist, nil +} + +// NewestSha returns the hash and block height of the most recent (end) block of +// the block chain. It will return the zero hash, -1 for the block height, and +// no error (nil) if there are not any blocks in the database yet. +func (db *LevelDb) NewestSha() (rsha *btcwire.ShaHash, rblkid int64, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + if db.lastBlkIdx == -1 { + err = errors.New("Empty Database") + return + } + sha := db.lastBlkSha + + return &sha, db.lastBlkIdx, nil +} + +func (db *LevelDb) NewIterateBlocks() (rbogus btcdb.BlockIterator, err error) { + err = errors.New("Not implemented") + return +} diff --git a/leveldb/dbcache.go b/leveldb/dbcache.go new file mode 100644 index 00000000..6c3bfafe --- /dev/null +++ b/leveldb/dbcache.go @@ -0,0 +1,375 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ldb + +import ( + "bytes" + "container/list" + "github.com/conformal/btcdb" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" + "sync" +) + +type txCache struct { + maxcount int + fifo list.List + // NOTE: the key is specifically ShaHash, not *ShaHash + txMap map[btcwire.ShaHash]*txCacheObj + cacheLock sync.RWMutex +} + +type txCacheObj struct { + next *txCacheObj + sha btcwire.ShaHash + blksha btcwire.ShaHash + pver uint32 + tx *btcwire.MsgTx + height int64 + spent []byte + txbuf []byte +} + +type blockCache struct { + maxcount int + fifo list.List + blockMap map[btcwire.ShaHash]*blockCacheObj + blockHeightMap map[int64]*blockCacheObj + cacheLock sync.RWMutex +} + +type blockCacheObj struct { + next *blockCacheObj + sha btcwire.ShaHash + blk *btcutil.Block +} + +// FetchBlockBySha - return a btcutil Block, object may be a cached. +func (db *LevelDb) FetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + return db.fetchBlockBySha(sha) +} + +// fetchBlockBySha - return a btcutil Block, object may be a cached. +// Must be called with db lock held. +func (db *LevelDb) fetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) { + + blkcache, ok := db.fetchBlockCache(sha) + if ok { + return blkcache.blk, nil + } + + buf, pver, height, err := db.fetchSha(sha) + if err != nil { + return + } + + blk, err = btcutil.NewBlockFromBytes(buf, pver) + if err != nil { + return + } + blk.SetHeight(height) + db.insertBlockCache(sha, blk) + + return +} + +// fetchBlockCache check if a block is in the block cache, if so return it. +func (db *LevelDb) fetchBlockCache(sha *btcwire.ShaHash) (*blockCacheObj, bool) { + + db.blockCache.cacheLock.RLock() + defer db.blockCache.cacheLock.RUnlock() + + blkobj, ok := db.blockCache.blockMap[*sha] + if !ok { // could this just return the map deref? + return nil, false + } + return blkobj, true +} + +// fetchBlockHeightCache check if a block is in the block cache, if so return it. +func (db *LevelDb) fetchBlockHeightCache(height int64) (*blockCacheObj, bool) { + + db.blockCache.cacheLock.RLock() + defer db.blockCache.cacheLock.RUnlock() + + blkobj, ok := db.blockCache.blockHeightMap[height] + if !ok { // could this just return the map deref? + return nil, false + } + return blkobj, true +} + +// insertBlockCache insert the given sha/block into the cache map. +// If the block cache is determined to be full, it will release +// an old entry in FIFO order. +func (db *LevelDb) insertBlockCache(sha *btcwire.ShaHash, blk *btcutil.Block) { + bc := &db.blockCache + + bc.cacheLock.Lock() + defer bc.cacheLock.Unlock() + + blkObj := blockCacheObj{sha: *sha, blk: blk} + bc.fifo.PushBack(&blkObj) + + if bc.fifo.Len() > bc.maxcount { + listobj := bc.fifo.Front() + bc.fifo.Remove(listobj) + tailObj, ok := listobj.Value.(*blockCacheObj) + if ok { + delete(bc.blockMap, tailObj.sha) + delete(bc.blockHeightMap, tailObj.blk.Height()) + } else { + panic("invalid type pushed on blockCache list") + } + } + + bc.blockHeightMap[blk.Height()] = &blkObj + bc.blockMap[blkObj.sha] = &blkObj +} + +// FetchTxByShaList given a array of ShaHash, look up the transactions +// and return them in a TxListReply array. +func (db *LevelDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + var replies []*btcdb.TxListReply + for _, txsha := range txShaList { + tx, _, _, _, height, txspent, err := db.fetchTxDataBySha(txsha) + btxspent := []bool{} + if err == nil { + btxspent = make([]bool, len(tx.TxOut), len(tx.TxOut)) + for idx := range tx.TxOut { + byteidx := idx / 8 + byteoff := uint(idx % 8) + btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0 + } + } + txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, Height: height, TxSpent: btxspent, Err: err} + replies = append(replies, &txlre) + } + return replies +} + +// fetchTxDataBySha returns several pieces of data regarding the given sha. +func (db *LevelDb) fetchTxDataBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rtxbuf []byte, rpver uint32, rblksha *btcwire.ShaHash, rheight int64, rtxspent []byte, err error) { + + var pver uint32 + var blksha *btcwire.ShaHash + var height int64 + var txspent []byte + var toff int + var tlen int + var blk *btcutil.Block + var blkbuf []byte + + // Check Tx cache + if txc, ok := db.fetchTxCache(txsha); ok { + if txc.spent != nil { + return txc.tx, txc.txbuf, txc.pver, &txc.blksha, txc.height, txc.spent, nil + } + } + + // If not cached load it + height, toff, tlen, txspent, err = db.fetchLocationUsedBySha(txsha) + if err != nil { + return + } + + blksha, err = db.fetchBlockShaByHeight(height) + if err != nil { + log.Warnf("block idx lookup %v to %v", height, err) + return + } + log.Tracef("transaction %v is at block %v %v tx %v", + txsha, blksha, height, toff) + + blk, err = db.fetchBlockBySha(blksha) + if err != nil { + log.Warnf("unable to fetch block %v %v ", + height, &blksha) + return + } + + blkbuf, pver, err = blk.Bytes() + if err != nil { + log.Warnf("unable to decode block %v %v", height, &blksha) + return + } + + txbuf := make([]byte, tlen) + copy(txbuf[:], blkbuf[toff:toff+tlen]) + rbuf := bytes.NewBuffer(txbuf) + + var tx btcwire.MsgTx + err = tx.BtcDecode(rbuf, pver) + if err != nil { + log.Warnf("unable to decode tx block %v %v txoff %v txlen %v", + height, &blksha, toff, tlen) + return + } + + // Shove data into TxCache + // XXX - + var txc txCacheObj + txc.sha = *txsha + txc.tx = &tx + txc.txbuf = txbuf + txc.pver = pver + txc.height = height + txc.spent = txspent + txc.blksha = *blksha + db.insertTxCache(&txc) + + return &tx, txbuf, pver, blksha, height, txspent, nil +} + +// FetchTxAllBySha returns several pieces of data regarding the given sha. +func (db *LevelDb) FetchTxAllBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rtxbuf []byte, rpver uint32, rblksha *btcwire.ShaHash, err error) { + var pver uint32 + var blksha *btcwire.ShaHash + var height int64 + var toff int + var tlen int + var blk *btcutil.Block + var blkbuf []byte + + // Check Tx cache + if txc, ok := db.fetchTxCache(txsha); ok { + return txc.tx, txc.txbuf, txc.pver, &txc.blksha, nil + } + + // If not cached load it + height, toff, tlen, err = db.FetchLocationBySha(txsha) + if err != nil { + return + } + + blksha, err = db.FetchBlockShaByHeight(height) + if err != nil { + log.Warnf("block idx lookup %v to %v", height, err) + return + } + log.Tracef("transaction %v is at block %v %v tx %v", + txsha, blksha, height, toff) + + blk, err = db.FetchBlockBySha(blksha) + if err != nil { + log.Warnf("unable to fetch block %v %v ", + height, &blksha) + return + } + + blkbuf, pver, err = blk.Bytes() + if err != nil { + log.Warnf("unable to decode block %v %v", height, &blksha) + return + } + + txbuf := make([]byte, tlen) + copy(txbuf[:], blkbuf[toff:toff+tlen]) + rbuf := bytes.NewBuffer(txbuf) + + var tx btcwire.MsgTx + err = tx.BtcDecode(rbuf, pver) + if err != nil { + log.Warnf("unable to decode tx block %v %v txoff %v txlen %v", + height, &blksha, toff, tlen) + return + } + + // Shove data into TxCache + // XXX - + var txc txCacheObj + txc.sha = *txsha + txc.tx = &tx + txc.txbuf = txbuf + txc.pver = pver + txc.height = height + txc.blksha = *blksha + db.insertTxCache(&txc) + + return &tx, txbuf, pver, blksha, nil +} + +// FetchTxBySha returns some data for the given Tx Sha. +func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rpver uint32, blksha *btcwire.ShaHash, err error) { + rtx, _, rpver, blksha, err = db.FetchTxAllBySha(txsha) + return +} + +// FetchTxBufBySha return the bytestream data and associated protocol version. +// for the given Tx Sha +func (db *LevelDb) FetchTxBufBySha(txsha *btcwire.ShaHash) (txbuf []byte, rpver uint32, err error) { + _, txbuf, rpver, _, err = db.FetchTxAllBySha(txsha) + return +} + +// fetchTxCache look up the given transaction in the Tx cache. +func (db *LevelDb) fetchTxCache(sha *btcwire.ShaHash) (*txCacheObj, bool) { + tc := &db.txCache + + tc.cacheLock.RLock() + defer tc.cacheLock.RUnlock() + + txObj, ok := tc.txMap[*sha] + if !ok { // could this just return the map deref? + return nil, false + } + return txObj, true +} + +// insertTxCache, insert the given txobj into the cache. +// if the tx cache is determined to be full, it will release +// an old entry in FIFO order. +func (db *LevelDb) insertTxCache(txObj *txCacheObj) { + tc := &db.txCache + + tc.cacheLock.Lock() + defer tc.cacheLock.Unlock() + + tc.fifo.PushBack(txObj) + + if tc.fifo.Len() >= tc.maxcount { + listobj := tc.fifo.Front() + tc.fifo.Remove(listobj) + tailObj, ok := listobj.Value.(*txCacheObj) + if ok { + delete(tc.txMap, tailObj.sha) + } else { + panic("invalid type pushed on tx list") + } + + } + + tc.txMap[txObj.sha] = txObj +} + +// InvalidateTxCache clear/release all cached transactions. +func (db *LevelDb) InvalidateTxCache() { + tc := &db.txCache + tc.cacheLock.Lock() + defer tc.cacheLock.Unlock() + tc.txMap = map[btcwire.ShaHash]*txCacheObj{} + tc.fifo = list.List{} +} + +// InvalidateTxCache clear/release all cached blocks. +func (db *LevelDb) InvalidateBlockCache() { + bc := &db.blockCache + bc.cacheLock.Lock() + defer bc.cacheLock.Unlock() + bc.blockMap = map[btcwire.ShaHash]*blockCacheObj{} + bc.blockHeightMap = map[int64]*blockCacheObj{} + bc.fifo = list.List{} +} + +// InvalidateCache clear/release all cached blocks and transactions. +func (db *LevelDb) InvalidateCache() { + db.InvalidateTxCache() + db.InvalidateBlockCache() +} diff --git a/leveldb/doc.go b/leveldb/doc.go new file mode 100644 index 00000000..65906a93 --- /dev/null +++ b/leveldb/doc.go @@ -0,0 +1,15 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +/* +Package sqlite3 implements a sqlite3 instance of btcdb. + +sqlite provides a zero setup, single file database. It requires cgo +and the presence of the sqlite library and headers, but nothing else. +The performance is generally high although it goes down with database +size. + +Many of the block or tx specific functions for btcdb are in this subpackage. +*/ +package ldb diff --git a/leveldb/insertremove_test.go b/leveldb/insertremove_test.go new file mode 100644 index 00000000..27fa248d --- /dev/null +++ b/leveldb/insertremove_test.go @@ -0,0 +1,209 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ldb_test + +import ( + "github.com/conformal/btcdb" + "github.com/conformal/btcdb/sqlite3" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" + "os" + "path/filepath" + "testing" +) + +var tstBlocks []*btcutil.Block + +func loadblocks(t *testing.T) []*btcutil.Block { + if len(tstBlocks) != 0 { + return tstBlocks + } + + testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + blocks, err := loadBlocks(t, testdatafile) + if err != nil { + t.Errorf("Unable to load blocks from test data: %v", err) + return nil + } + tstBlocks = blocks + return blocks +} + +func TestUnspentInsert(t *testing.T) { + testUnspentInsert(t, dbTmDefault) + testUnspentInsert(t, dbTmNormal) + testUnspentInsert(t, dbTmFast) +} + +// insert every block in the test chain +// after each insert, fetch all the tx affected by the latest +// block and verify that the the tx is spent/unspent +// new tx should be fully unspent, referenced tx should have +// the associated txout set to spent. +func testUnspentInsert(t *testing.T, mode int) { + // Ignore db remove errors since it means we didn't have an old one. + dbname := "tstdbuspnt1" + _ = os.Remove(dbname) + db, err := btcdb.CreateDB("sqlite", dbname) + if err != nil { + t.Errorf("Failed to open test database %v", err) + return + } + defer os.Remove(dbname) + defer db.Close() + + switch mode { + case dbTmDefault: // default + // no setup + case dbTmNormal: // explicit normal + db.SetDBInsertMode(btcdb.InsertNormal) + case dbTmFast: // fast mode + db.SetDBInsertMode(btcdb.InsertFast) + if sqldb, ok := db.(*sqlite3.LevelDb); ok { + sqldb.TempTblMax = 100 + } else { + t.Errorf("not right type") + } + case dbTmNoVerify: // validated block + t.Errorf("UnspentInsert test is not valid in NoVerify mode") + } + + // Since we are dealing with small dataset, reduce cache size + sqlite3.SetBlockCacheSize(db, 2) + sqlite3.SetTxCacheSize(db, 3) + + blocks := loadblocks(t) +endtest: + for height := int64(0); height < int64(len(blocks)); height++ { + + block := blocks[height] + // look up inputs to this x + mblock := block.MsgBlock() + var txneededList []*btcwire.ShaHash + var txlookupList []*btcwire.ShaHash + var txOutList []*btcwire.ShaHash + var txInList []*btcwire.OutPoint + for _, tx := range mblock.Transactions { + for _, txin := range tx.TxIn { + if txin.PreviousOutpoint.Index == uint32(4294967295) { + continue + } + origintxsha := &txin.PreviousOutpoint.Hash + + txInList = append(txInList, &txin.PreviousOutpoint) + txneededList = append(txneededList, origintxsha) + txlookupList = append(txlookupList, origintxsha) + + if !db.ExistsTxSha(origintxsha) { + t.Errorf("referenced tx not found %v ", origintxsha) + } + + } + txshaname, _ := tx.TxSha(block.ProtocolVersion()) + txlookupList = append(txlookupList, &txshaname) + txOutList = append(txOutList, &txshaname) + } + + txneededmap := map[btcwire.ShaHash]*btcdb.TxListReply{} + txlist := db.FetchTxByShaList(txneededList) + for _, txe := range txlist { + if txe.Err != nil { + t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) + break endtest + } + txneededmap[*txe.Sha] = txe + } + for _, spend := range txInList { + itxe := txneededmap[spend.Hash] + if itxe.TxSpent[spend.Index] == true { + t.Errorf("txin %v:%v is already spent", spend.Hash, spend.Index) + } + } + + newheight, err := db.InsertBlock(block) + if err != nil { + t.Errorf("failed to insert block %v err %v", height, err) + break endtest + } + if newheight != height { + t.Errorf("height mismatch expect %v returned %v", height, newheight) + break endtest + } + + txlookupmap := map[btcwire.ShaHash]*btcdb.TxListReply{} + txlist = db.FetchTxByShaList(txlookupList) + for _, txe := range txlist { + if txe.Err != nil { + t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) + break endtest + } + txlookupmap[*txe.Sha] = txe + } + for _, spend := range txInList { + itxe := txlookupmap[spend.Hash] + if itxe.TxSpent[spend.Index] == false { + t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent) + } + } + for _, txo := range txOutList { + itxe := txlookupmap[*txo] + for i, spent := range itxe.TxSpent { + if spent == true { + t.Errorf("freshly inserted tx %v already spent %v", txo, i) + } + } + + } + if len(txInList) == 0 { + continue + } + dropblock := blocks[height-1] + dropsha, _ := dropblock.Sha() + + err = db.DropAfterBlockBySha(dropsha) + if err != nil { + t.Errorf("failed to drop block %v err %v", height, err) + break endtest + } + + txlookupmap = map[btcwire.ShaHash]*btcdb.TxListReply{} + txlist = db.FetchTxByShaList(txlookupList) + for _, txe := range txlist { + if txe.Err != nil { + if _, ok := txneededmap[*txe.Sha]; ok { + t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) + break endtest + } + } + txlookupmap[*txe.Sha] = txe + } + for _, spend := range txInList { + itxe := txlookupmap[spend.Hash] + if itxe.TxSpent[spend.Index] == true { + t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent) + } + } + newheight, err = db.InsertBlock(block) + if err != nil { + t.Errorf("failed to insert block %v err %v", height, err) + break endtest + } + txlookupmap = map[btcwire.ShaHash]*btcdb.TxListReply{} + txlist = db.FetchTxByShaList(txlookupList) + for _, txe := range txlist { + if txe.Err != nil { + t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) + break endtest + } + txlookupmap[*txe.Sha] = txe + } + for _, spend := range txInList { + itxe := txlookupmap[spend.Hash] + if itxe.TxSpent[spend.Index] == false { + t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent) + } + } + } +} diff --git a/leveldb/internal_test.go b/leveldb/internal_test.go new file mode 100644 index 00000000..08706154 --- /dev/null +++ b/leveldb/internal_test.go @@ -0,0 +1,48 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ldb + +import ( + "fmt" + "github.com/conformal/btcdb" + "github.com/conformal/btcwire" +) + +// FetchSha returns the datablock and pver for the given ShaHash. +// This is a testing only interface. +func FetchSha(db btcdb.Db, sha *btcwire.ShaHash) (buf []byte, pver uint32, + blkid int64, err error) { + sqldb, ok := db.(*LevelDb) + if !ok { + err = fmt.Errorf("Invalid data type") + return + } + buf, pver, blkid, err = sqldb.fetchSha(*sha) + return +} + +// SetBlockCacheSize configures the maximum number of blocks in the cache to +// be the given size should be made before any fetching. +// This is a testing only interface. +func SetBlockCacheSize(db btcdb.Db, newsize int) { + sqldb, ok := db.(*LevelDb) + if !ok { + return + } + bc := &sqldb.blockCache + bc.maxcount = newsize +} + +// SetTxCacheSize configures the maximum number of tx in the cache to +// be the given size should be made before any fetching. +// This is a testing only interface. +func SetTxCacheSize(db btcdb.Db, newsize int) { + sqldb, ok := db.(*LevelDb) + if !ok { + return + } + tc := &sqldb.txCache + tc.maxcount = newsize +} diff --git a/leveldb/leveldb.go b/leveldb/leveldb.go new file mode 100644 index 00000000..d44d0e6e --- /dev/null +++ b/leveldb/leveldb.go @@ -0,0 +1,798 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ldb + +import ( + "database/sql" + "fmt" + "github.com/conformal/btcdb" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" + "github.com/conformal/seelog" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/opt" + _ "github.com/mattn/go-sqlite3" + "os" + "sync" +) + +const ( + dbVersion int = 2 + dbMaxTransCnt = 20000 + dbMaxTransMem = 64 * 1024 * 1024 // 64 MB +) + +var log seelog.LoggerInterface = seelog.Disabled + +type tBlockInsertData struct { + sha btcwire.ShaHash + pver uint32 + buf []byte +} +type tTxInsertData struct { + txsha *btcwire.ShaHash + blockid int64 + txoff int + txlen int + usedbuf []byte +} + +type txState struct { + tx *sql.Tx + writeCount int + txDataSz int + txInsertList []interface{} +} +type LevelDb struct { + // to be removed + sqldb *sql.DB + blkStmts []*sql.Stmt + blkBaseStmts []*sql.Stmt + txStmts []*sql.Stmt + txBaseStmts []*sql.Stmt + txState txState + + // lock preventing multiple entry + dbLock sync.Mutex + + // leveldb pieces + bShaDb *leveldb.DB + bBlkDb []*leveldb.DB + tShaDb *leveldb.DB + tLocDb []*leveldb.DB + tSpentDb []*leveldb.DB + blkOpen int + txOpen int + txSpentOpen int + ro *opt.ReadOptions + wo *opt.WriteOptions + + lastBlkShaCached bool + lastBlkSha btcwire.ShaHash + lastBlkIdx int64 + txCache txCache + blockCache blockCache + + UseTempTX bool + TempTblSz int + TempTblMax int + + dbInsertMode btcdb.InsertMode +} + +var self = btcdb.DriverDB{DbType: "sqlite", Create: CreateSqliteDB, Open: OpenSqliteDB} + +func init() { + btcdb.AddDBDriver(self) +} + +// createDB configure the database, setting up all tables to initial state. +func createDB(db *sql.DB) error { + log.Infof("Initializing new block database") + + // XXX check for old tables + buildTables := []string{ + "CREATE TABLE dbversion (version integer);", + "CREATE TABLE block ( blockid INTEGER PRIMARY KEY, key BLOB UNIQUE, " + + "pver INTEGER NOT NULL, data BLOB NOT NULL);", + "INSERT INTO dbversion (version) VALUES (" + fmt.Sprintf("%d", dbVersion) + + ");", + } + buildtxTables := []string{ + "CREATE TABLE tx (txidx INTEGER PRIMARY KEY, " + + "key TEXT, " + + "blockid INTEGER NOT NULL, " + + "txoff INTEGER NOT NULL, txlen INTEGER NOT NULL, " + + "data BLOB NOT NULL, " + + "FOREIGN KEY(blockid) REFERENCES block(blockid));", + "CREATE TABLE txtmp (key TEXT PRIMARY KEY, " + + "blockid INTEGER NOT NULL, " + + "txoff INTEGER NOT NULL, txlen INTEGER NOT NULL, " + + "data BLOB NOT NULL, " + + "FOREIGN KEY(blockid) REFERENCES block(blockid));", + "CREATE UNIQUE INDEX uniquetx ON tx (key);", + } + for _, sql := range buildTables { + _, err := db.Exec(sql) + if err != nil { + log.Warnf("sql table op failed %v [%v]", err, sql) + return err + } + } + for _, sql := range buildtxTables { + _, err := db.Exec(sql) + if err != nil { + log.Warnf("sql table op failed %v [%v]", err, sql) + return err + } + } + + return nil +} + +// OpenSqliteDB opens an existing database for use. +func OpenSqliteDB(filepath string) (pbdb btcdb.Db, err error) { + log = btcdb.GetLog() + return newOrCreateSqliteDB(filepath, false) +} + +// CreateSqliteDB creates, initializes and opens a database for use. +func CreateSqliteDB(filepath string) (pbdb btcdb.Db, err error) { + log = btcdb.GetLog() + return newOrCreateSqliteDB(filepath, true) +} + +// newOrCreateSqliteDB opens a database, either creating it or opens +// existing database based on flag. +func newOrCreateSqliteDB(filepath string, create bool) (pbdb btcdb.Db, err error) { + var bdb LevelDb + if create == false { + _, err = os.Stat(filepath) + if err != nil { + return nil, btcdb.DbDoesNotExist + } + } + + db, err := sql.Open("sqlite3", filepath) + if err != nil { + log.Warnf("db open failed %v\n", err) + return nil, err + } + + dbverstmt, err := db.Prepare("SELECT version FROM dbversion;") + if err != nil { + // about the only reason this would fail is that the database + // is not initialized + if create == false { + return nil, btcdb.DbDoesNotExist + } + err = createDB(db) + if err != nil { + // already warned in the called function + return nil, err + } + dbverstmt, err = db.Prepare("SELECT version FROM dbversion;") + if err != nil { + // if it failed this a second time, fail. + return nil, err + } + + } + row := dbverstmt.QueryRow() + var version int + err = row.Scan(&version) + if err != nil { + log.Warnf("unable to find db version: no row\n", err) + } + switch version { + case dbVersion: + // all good + default: + log.Warnf("mismatch db version: %v expected %v\n", version, dbVersion) + return nil, fmt.Errorf("Invalid version in database") + } + db.Exec("PRAGMA foreign_keys = ON;") + db.Exec("PRAGMA journal_mode=WAL;") + bdb.sqldb = db + + bdb.blkStmts = make([]*sql.Stmt, len(blkqueries)) + bdb.blkBaseStmts = make([]*sql.Stmt, len(blkqueries)) + for i := range blkqueries { + stmt, err := db.Prepare(blkqueries[i]) + if err != nil { + // XXX log/ + return nil, err + } + bdb.blkBaseStmts[i] = stmt + } + for i := range bdb.blkBaseStmts { + bdb.blkStmts[i] = bdb.blkBaseStmts[i] + } + + bdb.txBaseStmts = make([]*sql.Stmt, len(txqueries)) + for i := range txqueries { + stmt, err := db.Prepare(txqueries[i]) + if err != nil { + // XXX log/ + return nil, err + } + bdb.txBaseStmts[i] = stmt + } + // NOTE: all array entries in txStmts remain nil'ed + // tx statements are lazy bound + bdb.txStmts = make([]*sql.Stmt, len(txqueries)) + + bdb.blockCache.maxcount = 150 + bdb.blockCache.blockMap = map[btcwire.ShaHash]*blockCacheObj{} + bdb.blockCache.blockMap = map[btcwire.ShaHash]*blockCacheObj{} + bdb.blockCache.blockHeightMap = map[int64]*blockCacheObj{} + bdb.txCache.maxcount = 2000 + bdb.txCache.txMap = map[btcwire.ShaHash]*txCacheObj{} + + bdb.UseTempTX = true + bdb.TempTblMax = 1000000 + + return &bdb, nil +} + +// Sync verifies that the database is coherent on disk, +// and no outstanding transactions are in flight. +func (db *LevelDb) Sync() { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + db.endTx(true) +} + +// syncPoint notifies the db that this is a safe time to sync the database, +// if there are many outstanding transactions. +// Must be called with db lock held. +func (db *LevelDb) syncPoint() { + + tx := &db.txState + + if db.TempTblSz > db.TempTblMax { + err := db.migrateTmpTable() + if err != nil { + return + } + } else { + if len(tx.txInsertList) > dbMaxTransCnt || tx.txDataSz > dbMaxTransMem { + db.endTx(true) + } + } +} + +// Close cleanly shuts down database, syncing all data. +func (db *LevelDb) Close() { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + db.close() +} + +// RollbackClose discards the recent database changes to the previously +// saved data at last Sync. +func (db *LevelDb) RollbackClose() { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + tx := &db.txState + if tx.tx != nil { + err := tx.tx.Rollback() + if err != nil { + log.Debugf("Rollback failed: %v", err) + } else { + tx.tx = nil + } + } + db.close() +} + +// close performs the internal shutdown/close operation. +func (db *LevelDb) close() { + db.endTx(true) + + db.InvalidateCache() + + for i := range db.blkBaseStmts { + db.blkBaseStmts[i].Close() + } + for i := range db.txBaseStmts { + if db.txBaseStmts[i] != nil { + db.txBaseStmts[i].Close() + db.txBaseStmts[i] = nil + } + } + db.sqldb.Close() +} + +// txop returns the appropriately prepared statement, based on +// transaction state of the database. +func (db *LevelDb) txop(op int) *sql.Stmt { + if db.txStmts[op] != nil { + return db.txStmts[op] + } + if db.txState.tx == nil { + // we are not in a transaction, return the base statement + return db.txBaseStmts[op] + } + + if db.txStmts[op] == nil { + db.txStmts[op] = db.txState.tx.Stmt(db.txBaseStmts[op]) + } + + return db.txStmts[op] +} + +// startTx starts a transaction, preparing or scrubbing statements +// for proper operation inside a transaction. +func (db *LevelDb) startTx() (err error) { + tx := &db.txState + if tx.tx != nil { + // this shouldn't happen... + log.Warnf("Db startTx called while in a transaction") + return + } + tx.tx, err = db.sqldb.Begin() + if err != nil { + log.Warnf("Db startTx: begin failed %v", err) + tx.tx = nil + return + } + for i := range db.blkBaseStmts { + db.blkStmts[i] = tx.tx.Stmt(db.blkBaseStmts[i]) + } + for i := range db.txBaseStmts { + db.txStmts[i] = nil // these are lazily prepared + } + return +} + +// endTx commits the current active transaction, it zaps all of the prepared +// statements associated with the transaction. +func (db *LevelDb) endTx(recover bool) (err error) { + tx := &db.txState + + if tx.tx == nil { + return + } + + err = tx.tx.Commit() + if err != nil && recover { + // XXX - double check that the tx is dead after + // commit failure (rollback?) + + log.Warnf("Db endTx: commit failed %v", err) + err = db.rePlayTransaction() + if err != nil { + // We tried, return failure (after zeroing state) + // so the upper level can notice and restart + } + } + for i := range db.blkBaseStmts { + db.blkStmts[i].Close() + db.blkStmts[i] = db.blkBaseStmts[i] + } + for i := range db.txStmts { + if db.txStmts[i] != nil { + db.txStmts[i].Close() + db.txStmts[i] = nil + } + } + tx.tx = nil + var emptyTxList []interface{} + tx.txInsertList = emptyTxList + tx.txDataSz = 0 + return +} + +// rePlayTransaction will attempt to re-execute inserts performed +// sync the beginning of a transaction. This is to be used after +// a sql Commit operation fails to keep the database from losing data. +func (db *LevelDb) rePlayTransaction() (err error) { + err = db.startTx() + if err != nil { + return + } + tx := &db.txState + for _, ins := range tx.txInsertList { + switch v := ins.(type) { + case tBlockInsertData: + block := v + _, err = db.blkStmts[blkInsertSha].Exec(block.sha.Bytes(), + block.pver, block.buf) + if err != nil { + break + } + case tTxInsertData: + txd := v + txnamebytes := txd.txsha.Bytes() + txop := db.txop(txInsertStmt) + _, err = txop.Exec(txd.blockid, txnamebytes, txd.txoff, + txd.txlen, txd.usedbuf) + if err != nil { + break + } + } + } + // This function is called even if we have failed. + // We need to clean up so the database can be used again. + // However we want the original error not any new error, + // unless there was no original error but the commit fails. + err2 := db.endTx(false) + if err == nil && err2 != nil { + err = err2 + } + + return +} + +// DropAfterBlockBySha will remove any blocks from the database after the given block. +// It terminates any existing transaction and performs its operations in an +// atomic transaction, it is terminated (committed) before exit. +func (db *LevelDb) DropAfterBlockBySha(sha *btcwire.ShaHash) (err error) { + var row *sql.Row + db.dbLock.Lock() + defer db.dbLock.Unlock() + + // This is a destructive operation and involves multiple requests + // so requires a transaction, terminate any transaction to date + // and start a new transaction + err = db.endTx(true) + if err != nil { + return err + } + err = db.startTx() + if err != nil { + return err + } + + var startheight int64 + + if db.lastBlkShaCached { + startheight = db.lastBlkIdx + } else { + querystr := "SELECT blockid FROM block ORDER BY blockid DESC;" + + tx := &db.txState + if tx.tx != nil { + row = tx.tx.QueryRow(querystr) + } else { + row = db.sqldb.QueryRow(querystr) + } + var startblkidx int64 + err = row.Scan(&startblkidx) + if err != nil { + log.Warnf("DropAfterBlockBySha:unable to fetch blockheight %v", err) + return err + } + startheight = startblkidx + } + // also drop any cached sha data + db.lastBlkShaCached = false + + querystr := "SELECT blockid FROM block WHERE key = ?;" + + tx := &db.txState + row = tx.tx.QueryRow(querystr, sha.Bytes()) + + var keepidx int64 + err = row.Scan(&keepidx) + if err != nil { + // XXX + db.endTx(false) + return err + } + + for height := startheight; height > keepidx; height = height - 1 { + var blk *btcutil.Block + blkc, ok := db.fetchBlockHeightCache(height) + + if ok { + blk = blkc.blk + } else { + // must load the block from the db + sha, err = db.fetchBlockShaByHeight(height - 1) + if err != nil { + return + } + + var buf []byte + var pver uint32 + + buf, pver, _, err = db.fetchSha(*sha) + if err != nil { + return + } + + blk, err = btcutil.NewBlockFromBytes(buf, pver) + if err != nil { + return + } + } + + for _, tx := range blk.MsgBlock().Transactions { + err = db.unSpend(tx) + if err != nil { + return + } + } + } + + // invalidate the cache after possibly using cached entries for block + // lookup to unspend coins in them + db.InvalidateCache() + + _, err = tx.tx.Exec("DELETE FROM txtmp WHERE blockid > ?", keepidx) + if err != nil { + // XXX + db.endTx(false) + return err + } + + _, err = tx.tx.Exec("DELETE FROM tx WHERE blockid > ?", keepidx) + if err != nil { + // XXX + db.endTx(false) + return err + } + + // delete from block last in case of foreign keys + _, err = tx.tx.Exec("DELETE FROM block WHERE blockid > ?", keepidx) + if err != nil { + // XXX + db.endTx(false) + return err + } + + err = db.endTx(true) + if err != nil { + return err + } + return +} + +// InsertBlock inserts raw block and transaction data from a block into the +// database. The first block inserted into the database will be treated as the +// genesis block. Every subsequent block insert requires the referenced parent +// block to already exist. +func (db *LevelDb) InsertBlock(block *btcutil.Block) (height int64, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + blocksha, err := block.Sha() + if err != nil { + log.Warnf("Failed to compute block sha %v", blocksha) + return + } + mblock := block.MsgBlock() + rawMsg, pver, err := block.Bytes() + if err != nil { + log.Warnf("Failed to obtain raw block sha %v", blocksha) + return + } + txloc, err := block.TxLoc() + if err != nil { + log.Warnf("Failed to obtain raw block sha %v", blocksha) + return + } + + // Insert block into database + newheight, err := db.insertBlockData(blocksha, &mblock.Header.PrevBlock, + pver, rawMsg) + if err != nil { + log.Warnf("Failed to insert block %v %v %v", blocksha, + &mblock.Header.PrevBlock, err) + return + } + + // At least two blocks in the long past were generated by faulty + // miners, the sha of the transaction exists in a previous block, + // detect this condition and 'accept' the block. + for txidx, tx := range mblock.Transactions { + var txsha btcwire.ShaHash + txsha, err = tx.TxSha(pver) + if err != nil { + log.Warnf("failed to compute tx name block %v idx %v err %v", blocksha, txidx, err) + return + } + // Some old blocks contain duplicate transactions + // Attempt to cleanly bypass this problem + // http://blockexplorer.com/b/91842 + // http://blockexplorer.com/b/91880 + if newheight == 91842 { + dupsha, err := btcwire.NewShaHashFromStr("d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599") + if err != nil { + panic("invalid sha string in source") + } + if txsha == *dupsha { + log.Tracef("skipping sha %v %v", dupsha, newheight) + continue + } + } + if newheight == 91880 { + dupsha, err := btcwire.NewShaHashFromStr("e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468") + if err != nil { + panic("invalid sha string in source") + } + if txsha == *dupsha { + log.Tracef("skipping sha %v %v", dupsha, newheight) + continue + } + } + spentbuflen := (len(tx.TxOut) + 7) / 8 + spentbuf := make([]byte, spentbuflen, spentbuflen) + if len(tx.TxOut)%8 != 0 { + for i := uint(len(tx.TxOut) % 8); i < 8; i++ { + spentbuf[spentbuflen-1] |= (byte(1) << i) + } + } + + err = db.insertTx(&txsha, newheight, txloc[txidx].TxStart, txloc[txidx].TxLen, spentbuf) + if err != nil { + log.Warnf("block %v idx %v failed to insert tx %v %v err %v", blocksha, newheight, &txsha, txidx, err) + var oBlkIdx int64 + oBlkIdx, _, _, err = db.fetchLocationBySha(&txsha) + log.Warnf("oblkidx %v err %v", oBlkIdx, err) + + return + } + err = db.doSpend(tx) + if err != nil { + log.Warnf("block %v idx %v failed to spend tx %v %v err %v", blocksha, newheight, &txsha, txidx, err) + + return + } + } + db.syncPoint() + return newheight, nil +} + +// SetDBInsertMode provides hints to the database to how the application +// is running this allows the database to work in optimized modes when the +// database may be very busy. +func (db *LevelDb) SetDBInsertMode(newmode btcdb.InsertMode) { + + oldMode := db.dbInsertMode + switch newmode { + case btcdb.InsertNormal: + // Normal mode inserts tx directly into the tx table + db.UseTempTX = false + db.dbInsertMode = newmode + switch oldMode { + case btcdb.InsertFast: + if db.TempTblSz != 0 { + err := db.migrateTmpTable() + if err != nil { + return + } + } + case btcdb.InsertValidatedInput: + // generate tx indexes + txop := db.txop(txMigrateFinish) + _, err := txop.Exec() + if err != nil { + log.Warnf("Failed to create tx table index - %v", err) + } + } + case btcdb.InsertFast: + // Fast mode inserts tx into txtmp with validation, + // then dumps to tx then rebuilds indexes at thresholds + db.UseTempTX = true + if oldMode != btcdb.InsertNormal { + log.Warnf("switching between invalid DB modes") + break + } + db.dbInsertMode = newmode + case btcdb.InsertValidatedInput: + // ValidatedInput mode inserts into tx table with + // no duplicate checks, then builds index on exit from + // ValidatedInput mode + if oldMode != btcdb.InsertNormal { + log.Warnf("switching between invalid DB modes") + break + } + // remove tx table index + txop := db.txop(txMigratePrep) + _, err := txop.Exec() + if err != nil { + log.Warnf("Failed to clear tx table index - %v", err) + } + db.dbInsertMode = newmode + + // XXX + db.UseTempTX = false + } +} +func (db *LevelDb) doSpend(tx *btcwire.MsgTx) error { + for txinidx := range tx.TxIn { + txin := tx.TxIn[txinidx] + + inTxSha := txin.PreviousOutpoint.Hash + inTxidx := txin.PreviousOutpoint.Index + + if inTxidx == ^uint32(0) { + continue + } + + //log.Infof("spending %v %v", &inTxSha, inTxidx) + + err := db.setSpentData(&inTxSha, inTxidx) + if err != nil { + return err + } + } + return nil +} + +func (db *LevelDb) unSpend(tx *btcwire.MsgTx) error { + for txinidx := range tx.TxIn { + txin := tx.TxIn[txinidx] + + inTxSha := txin.PreviousOutpoint.Hash + inTxidx := txin.PreviousOutpoint.Index + + if inTxidx == ^uint32(0) { + continue + } + + err := db.clearSpentData(&inTxSha, inTxidx) + if err != nil { + return err + } + } + return nil +} + +func (db *LevelDb) setSpentData(sha *btcwire.ShaHash, idx uint32) error { + return db.setclearSpentData(sha, idx, true) +} + +func (db *LevelDb) clearSpentData(sha *btcwire.ShaHash, idx uint32) error { + return db.setclearSpentData(sha, idx, false) +} + +func (db *LevelDb) setclearSpentData(txsha *btcwire.ShaHash, idx uint32, set bool) error { + var spentdata []byte + usingtmp := false + txop := db.txop(txFetchUsedByShaStmt) + row := txop.QueryRow(txsha.String()) + err := row.Scan(&spentdata) + if err != nil { + // if the error is simply didn't fine continue otherwise + // retun failure + + usingtmp = true + txop = db.txop(txtmpFetchUsedByShaStmt) + row := txop.QueryRow(txsha.String()) + err := row.Scan(&spentdata) + if err != nil { + log.Warnf("Failed to locate spent data - %v %v", txsha, err) + return err + } + } + byteidx := idx / 8 + byteoff := idx % 8 + + if set { + spentdata[byteidx] |= (byte(1) << byteoff) + } else { + spentdata[byteidx] &= ^(byte(1) << byteoff) + } + txc, cached := db.fetchTxCache(txsha) + if cached { + txc.spent = spentdata + } + + if usingtmp { + txop = db.txop(txtmpUpdateUsedByShaStmt) + } else { + txop = db.txop(txUpdateUsedByShaStmt) + } + _, err = txop.Exec(spentdata, txsha.String()) + + return err +} diff --git a/leveldb/operational_test.go b/leveldb/operational_test.go new file mode 100644 index 00000000..71c92430 --- /dev/null +++ b/leveldb/operational_test.go @@ -0,0 +1,420 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ldb_test + +import ( + "compress/bzip2" + "encoding/binary" + "github.com/conformal/btcdb" + "github.com/conformal/btcdb/sqlite3" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +var network = btcwire.MainNet + +const ( + dbTmDefault = iota + dbTmNormal + dbTmFast + dbTmNoVerify +) + +func TestOperational(t *testing.T) { + testOperationalMode(t, dbTmDefault) + testOperationalMode(t, dbTmNormal) + testOperationalMode(t, dbTmFast) + testOperationalMode(t, dbTmNoVerify) +} + +func testOperationalMode(t *testing.T, mode int) { + // simplified basic operation is: + // 1) fetch block from remote server + // 2) look up all txin (except coinbase in db) + // 3) insert block + + // Ignore db remove errors since it means we didn't have an old one. + dbname := "tstdbop1" + _ = os.Remove(dbname) + db, err := btcdb.CreateDB("sqlite", dbname) + if err != nil { + t.Errorf("Failed to open test database %v", err) + return + } + defer os.Remove(dbname) + defer db.Close() + + switch mode { + case dbTmDefault: // default + // no setup + case dbTmNormal: // explicit normal + db.SetDBInsertMode(btcdb.InsertNormal) + case dbTmFast: // fast mode + db.SetDBInsertMode(btcdb.InsertFast) + if sqldb, ok := db.(*sqlite3.LevelDb); ok { + sqldb.TempTblMax = 100 + } else { + t.Errorf("not right type") + } + case dbTmNoVerify: // validated block + db.SetDBInsertMode(btcdb.InsertValidatedInput) + } + + // Since we are dealing with small dataset, reduce cache size + sqlite3.SetBlockCacheSize(db, 2) + sqlite3.SetTxCacheSize(db, 3) + + testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + blocks, err := loadBlocks(t, testdatafile) + if err != nil { + t.Errorf("Unable to load blocks from test data for mode %v: %v", + mode, err) + return + } + + err = nil +out: + for height := int64(0); height < int64(len(blocks)); height++ { + block := blocks[height] + if mode != dbTmNoVerify { + // except for NoVerify which does not allow lookups check inputs + mblock := block.MsgBlock() + var txneededList []*btcwire.ShaHash + for _, tx := range mblock.Transactions { + for _, txin := range tx.TxIn { + if txin.PreviousOutpoint.Index == uint32(4294967295) { + continue + } + origintxsha := &txin.PreviousOutpoint.Hash + txneededList = append(txneededList, origintxsha) + + if !db.ExistsTxSha(origintxsha) { + t.Errorf("referenced tx not found %v ", origintxsha) + } + + _, _, _, _, err := db.FetchTxAllBySha(origintxsha) + if err != nil { + t.Errorf("referenced tx not found %v err %v ", origintxsha, err) + } + _, _, _, _, err = db.FetchTxAllBySha(origintxsha) + if err != nil { + t.Errorf("referenced tx not found %v err %v ", origintxsha, err) + } + _, _, _, err = db.FetchTxBySha(origintxsha) + if err != nil { + t.Errorf("referenced tx not found %v err %v ", origintxsha, err) + } + _, _, err = db.FetchTxBufBySha(origintxsha) + if err != nil { + t.Errorf("referenced tx not found %v err %v ", origintxsha, err) + } + _, err = db.FetchTxUsedBySha(origintxsha) + if err != nil { + t.Errorf("tx used fetch fail %v err %v ", origintxsha, err) + } + } + } + txlist := db.FetchTxByShaList(txneededList) + for _, txe := range txlist { + if txe.Err != nil { + t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) + break out + } + } + + } + + newheight, err := db.InsertBlock(block) + if err != nil { + t.Errorf("failed to insert block %v err %v", height, err) + break out + } + if newheight != height { + t.Errorf("height mismatch expect %v returned %v", height, newheight) + break out + } + + newSha, blkid, err := db.NewestSha() + if err != nil { + t.Errorf("failed to obtain latest sha %v %v", height, err) + } + + if blkid != height { + t.Errorf("height doe not match latest block height %v %v", blkid, height, err) + } + + blkSha, _ := block.Sha() + if *newSha != *blkSha { + t.Errorf("Newest block sha does not match freshly inserted one %v %v ", newSha, blkSha, err) + } + } + + // now that db is populated, do some additional test + testFetchRangeHeight(t, db, blocks) + + switch mode { + case dbTmDefault: // default + // no cleanup + case dbTmNormal: // explicit normal + // no cleanup + case dbTmFast: // fast mode + db.SetDBInsertMode(btcdb.InsertNormal) + case dbTmNoVerify: // validated block + db.SetDBInsertMode(btcdb.InsertNormal) + } +} + +func TestBackout(t *testing.T) { + testBackout(t, dbTmDefault) + testBackout(t, dbTmNormal) + testBackout(t, dbTmFast) +} + +func testBackout(t *testing.T, mode int) { + // simplified basic operation is: + // 1) fetch block from remote server + // 2) look up all txin (except coinbase in db) + // 3) insert block + + // Ignore db remove errors since it means we didn't have an old one. + dbname := "tstdbop2" + _ = os.Remove(dbname) + db, err := btcdb.CreateDB("sqlite", dbname) + if err != nil { + t.Errorf("Failed to open test database %v", err) + return + } + defer os.Remove(dbname) + defer db.Close() + + switch mode { + case dbTmDefault: // default + // no setup + case dbTmNormal: // explicit normal + db.SetDBInsertMode(btcdb.InsertNormal) + case dbTmFast: // fast mode + db.SetDBInsertMode(btcdb.InsertFast) + if sqldb, ok := db.(*sqlite3.LevelDb); ok { + sqldb.TempTblMax = 100 + } else { + t.Errorf("not right type") + } + } + + // Since we are dealing with small dataset, reduce cache size + sqlite3.SetBlockCacheSize(db, 2) + sqlite3.SetTxCacheSize(db, 3) + + testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + blocks, err := loadBlocks(t, testdatafile) + if len(blocks) < 120 { + t.Errorf("test data too small") + return + } + + err = nil + for height := int64(0); height < int64(len(blocks)); height++ { + if height == 100 { + t.Logf("Syncing at block height 100") + db.Sync() + } + if height == 120 { + t.Logf("Simulating unexpected application quit") + // Simulate unexpected application quit + db.RollbackClose() + break + } + + block := blocks[height] + + newheight, err := db.InsertBlock(block) + if err != nil { + t.Errorf("failed to insert block %v err %v", height, err) + break + } + if newheight != height { + t.Errorf("height mismatch expect %v returned %v", height, newheight) + break + } + } + + // db was closed at height 120, so no cleanup is possible. + + // reopen db + db, err = btcdb.OpenDB("sqlite", dbname) + if err != nil { + t.Errorf("Failed to open test database %v", err) + return + } + defer db.Close() + + sha, err := blocks[99].Sha() + if err != nil { + t.Errorf("failed to get block 99 sha err %v", err) + return + } + _ = db.ExistsSha(sha) + _, err = db.FetchBlockBySha(sha) + if err != nil { + t.Errorf("failed to load block 99 from db %v", err) + } + + sha, err = blocks[110].Sha() + if err != nil { + t.Errorf("failed to get block 110 sha err %v", err) + return + } + _ = db.ExistsSha(sha) + _, err = db.FetchBlockBySha(sha) + if err == nil { + t.Errorf("loaded block 110 from db, failure expected") + return + } + + block := blocks[110] + mblock := block.MsgBlock() + txsha, err := mblock.Transactions[0].TxSha(block.ProtocolVersion()) + exists := db.ExistsTxSha(&txsha) + if exists { + t.Errorf("tx %v exists in db, failure expected", txsha) + } + + _, _, _, err = db.FetchTxBySha(&txsha) + _, err = db.FetchTxUsedBySha(&txsha) + + block = blocks[99] + mblock = block.MsgBlock() + txsha, err = mblock.Transactions[0].TxSha(block.ProtocolVersion()) + oldused, err := db.FetchTxUsedBySha(&txsha) + err = db.InsertTx(&txsha, 99, 1024, 1048, oldused) + if err == nil { + t.Errorf("dup insert of tx succeeded") + return + } +} + +func loadBlocks(t *testing.T, file string) (blocks []*btcutil.Block, err error) { + testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + var dr io.Reader + var fi io.ReadCloser + fi, err = os.Open(testdatafile) + if err != nil { + t.Errorf("failed to open file %v, err %v", testdatafile, err) + return + } + if strings.HasSuffix(testdatafile, ".bz2") { + z := bzip2.NewReader(fi) + dr = z + } else { + dr = fi + } + + defer func() { + if err := fi.Close(); err != nil { + t.Errorf("failed to close file %v %v", testdatafile, err) + } + }() + + // Set the first block as the genesis block. + genesis := btcutil.NewBlock(&btcwire.GenesisBlock, btcwire.ProtocolVersion) + blocks = append(blocks, genesis) + + var block *btcutil.Block + err = nil + for height := int64(1); err == nil; height++ { + var rintbuf uint32 + err = binary.Read(dr, binary.LittleEndian, &rintbuf) + if err == io.EOF { + // hit end of file at expected offset: no warning + height-- + err = nil + break + } + if err != nil { + t.Errorf("failed to load network type, err %v", err) + break + } + if rintbuf != uint32(network) { + t.Errorf("Block doesn't match network: %v expects %v", + rintbuf, network) + break + } + err = binary.Read(dr, binary.LittleEndian, &rintbuf) + blocklen := rintbuf + + rbytes := make([]byte, blocklen) + + // read block + dr.Read(rbytes) + + var pver uint32 + switch { + case height < 200000: + pver = 1 + case height >= 200000: + pver = 2 + } + block, err = btcutil.NewBlockFromBytes(rbytes, pver) + if err != nil { + t.Errorf("failed to parse block %v", height) + return + } + blocks = append(blocks, block) + } + return +} + +func testFetchRangeHeight(t *testing.T, db btcdb.Db, blocks []*btcutil.Block) { + + var testincrement int64 = 50 + var testcnt int64 = 100 + + shanames := make([]*btcwire.ShaHash, len(blocks)) + + nBlocks := int64(len(blocks)) + + for i := range blocks { + blockSha, err := blocks[i].Sha() + if err != nil { + t.Errorf("FetchRangeHeight: unexpected failure computing block sah %v", err) + } + shanames[i] = blockSha + } + + for startheight := int64(0); startheight < nBlocks; startheight += testincrement { + endheight := startheight + testcnt + + if endheight > nBlocks { + endheight = btcdb.AllShas + } + + shalist, err := db.FetchHeightRange(startheight, endheight) + if err != nil { + t.Errorf("FetchRangeHeight: unexpected failure looking up shas %v", err) + } + + if endheight == btcdb.AllShas { + if int64(len(shalist)) != nBlocks-startheight { + t.Errorf("FetchRangeHeight: expected A %v shas, got %v", nBlocks-startheight, len(shalist)) + } + } else { + if int64(len(shalist)) != testcnt { + t.Errorf("FetchRangeHeight: expected %v shas, got %v", testcnt, len(shalist)) + } + } + + for i := range shalist { + if *shanames[int64(i)+startheight] != shalist[i] { + t.Errorf("FetchRangeHeight: mismatch sha at %v requested range %v %v ", int64(i)+startheight, startheight, endheight) + } + } + } + +} diff --git a/leveldb/testdata/blocks1-256.bz2 b/leveldb/testdata/blocks1-256.bz2 new file mode 100644 index 0000000000000000000000000000000000000000..6b8bda4429200c0566bb13c28c35d6397272e475 GIT binary patch literal 37555 zcmV)8K*qm9T4*^jL0KkKSr@A=?*Lyo|NsC0|NsC0|NsC0|NsC0|NsC0|NsC0|NsC0 z|NsC0|Nr1%0`GtT00*bHz1|n5T)=M=pBnqU;dPxJy+dC1_BM0g?t47Wcf|GKdY#;B zx%Jb_&i2~vj_uc8@w++d_r2Xc+bQeWwB7AnH@m&s?|Z7VcLwh6_&)Ua-C7%VGQQc~ z>${h|?$>*(yBXJ;E3Q`cd%f+uRJ}XA?=#ch z^LF0cm$dfxz3JWU#m@D$*Rj3c>USOO?{`dUw52*PZ4) zZsq3L?C!nYb#Ajg-1geucDp>i-Q{VQx3xUI&gD6`Uawu6^=;dCb=vn_^?SRn>aKNl zE$pv*o-NzE&uh^0y|K@GzIV4+$bmfo002w?nqp#L0%cFrLlY(dm;eaCCJBH5N#bAt zO*GR4VqgTqU=ssCXc_@FgCjbAaCIAVQ4yFcx0gyDQ{z0Gs4FX^(;0dOf69QlWOd2#a z4FC-Q0fYo#1w0VRlT9>W35k^S(-Q<>m=TeQsvT87z%&|Zsek|g&_ zj3y%jGfhmMr>UAIk3kuxqef2@z$c;x$u^Pc4@lCq5G;R}q9*?O-;!gJMnwV1tNmyb zJ@fx!pSG8@UQWKV&A3<1{kZJNqQ08k z6D?(C9~^K8No6azD{saS5up0CIL)USziD_I?C)=&!gQRRl1VS{+J?XC_#bbw?xgcw zjlnWQhmuJmK$1=Nq>@QeY{iyDRa8VqDzR1~h>SsCD;S`vMFkaz#a1j>F^Y@@ixB}8 zR7F^*f~;V~WLU;U0gMLL5g4e*#6~J23L=XU6%+sgVk!!%C^905f(&B?Rz?W0j2J=! zkz&A5Bv=9>DkzG^F;zul6^s@Nf+(vMV51aRs)#D6ixx2yRYe3*Q4s|eF+^2@v4W~9 z3lL()AgmOMip7Gf76_n>7{x?j!Bi0#Dx^^qh$zHCR8?TA005$jqA-Mjq=2G;p(GR_ zERkRYhGl^yhX}|(fNQV-4dYK>n#ItplMB?#{#=cV!a#-X6#_w-Hww)~IQDocyC!q_ zP;QC6GWpfg{SRTvVKK%&;D83JAV?@cB9MTfNoG=P`RhZcW`%~VouXmT%%>Q`%gFiG znI{I2!l$s1vkuvTf^(_QFeC#yn%e`wLK@aVJW{_@>IkMu*Tz znP6aPZ9IqEsuH!41X8oO@$DzHuf2m$q6GXXz?)H=) z8Oo<|u~gic8#@ls4^23kc~Cpor9%wuOfhH~@7+O`A-6w1U~OG zKw{EgNhFbmD_5u(-)^8l4f&a^9R_dO&uKNTUsBR*+$IlA*>~OFPl4v}nJizqrf9w) zqe22=T4Ir+f2Wo>`sLWI3k8mhw?H3J4@8NVOJTC0;$We@CRKTKF-3MYHiCQ2x3BU( z)}O1pcK`t1iH=|ZvOy7K09X-1NkIVYB$7zO4T7V2@@RyR?lKpdotDOu@DKn100025 z8j|2a5i)c~hdx1-v5bPtJMzV|CO3sRfYNWYLAI?F)Ks#b?V*}=K)U~Rs~`0}`9jfr zq@K;^8mYzZ93voGNFdj?bzU>-R^QcyxFYAQ$zjzXUTxHHURb0-{WBrwQ(ZhX(djB$ z`|{b~h9T^fOv9~7I81ym3u$cwRR4{rh|WgWszLs;K?0NUxzv@B&3M~d!H89)Ki8|p zAx&K5^LeTg6__5a@+Dnc!orx%+h*yBU8H0mz)})hyp3;X`sbgdLE`m z?DM(0Suen3X-SJQGFHiJ0yCcr?|5*sFDL6h7#NVcZP%#TI4$qq*)#tOc#{r?EQZ$3 z@2aWLp$#GztTlv&s>7+oXsj0TsvJKrUA^1=xCTk?v&P>l6P;n&yXO^LN(W_@q8;<+ zrl^L0MY!5`jO_o#dV1AP-7?iu{WwR>bh*tHu{|g2=;Y@7OV2^dGW>g#(m%7hqc51| zzcxELdBOX6%VscLc4GgiyT#Xi{j2Y7xDyeuA-Ni0gT+xi?gf7B=ljdA!DA@CvcVa z0|L+;v>c%L7r?;)(hZ#!uzUkvOYL5>NO9@tvhh0$`rHzF$Y-2yK|#}W7;QdVP@gZ+ zsaaj}*SXBqyKiTPvBhP!4o2f%PFbD9m|>CUYfa|p_$#mW%GO>rXDT<-Io!O+ur6g+)r2!T9)G#e18LaZ{9x-(kfMX?ERZ|fqNIO zME-!VUTi4TBS-t#Yyc}w_xB<;JV`o%DIac@7nw_d$iX@5|TOOcDXyH_y!EDC<-4 zp|dc+urTQkwTFY@)y(Qp%58N!TtBnjTp9Kr7BaYo8#iI4v7ogav*%~Y0v&t_4WK;m zs=m@rZ7B+osQl1Akfx1co1S^%Ikray<`Wy)L8)zO^LY$qI#2!G2Dw%`Dr z5h{+fe|-9E$fmiB08zRveD1Yhcfwjt`wk1Iq2R`X1Q91A_9FYJ^&CP00aaVyE^zPKKH@`iDe(E?g>zB zM}%7D;S$eKl={FXK@rn&5>;QeQly&S%rH!KMK``jv;;EAMXa9wnFszdSnAk@(`fr1 z-F= zHj`aS%|-LLjNoBDsEib}NSwq5JLd6T7693-79 zDa>6lA#reFs2?X-X75sY!}Ai3TB3CwkYdOYjVslXg2Yd1enXi;DDq zzWn4VT)GEKJ7~9}y%gc+VRK3Rba`|;zJX#GA9Sf|N_4kmm@4D}qpUcsJR#}R86^G< zK3?Ss!&$nw>89d_t1|k&(zzOxU|5m0sk1dZMgpjHbS26>7RoL1{!7IW2-RKL0>wZ8 z&XXqs4L;1xknu*uwWRl0<1N;nGu^qkNAlVzq+3u3ApfQF#4g+wm$A|d!l1-_#J;yG zm3x<}PZ#pXr1xG8Mz6(W3cqYjn`{FN9tt7SJ^^!VBXJqHN)y}$W&%S+7+*{gN8(YL z5ohchr<3tZ=0#=0qo6$Obbi|LG=kcor0YLPK%_&LwJHl&e<-7n0%_7Ta;(=}?ZcGW z7T+s3xx;mUgx3d_b@*|}#dzL(mHFl|$h7?z$9%yOaT#1n%`PLi)RS9__>lx%bOy-Akf0XxoAo4bGs7|!&K zVdF4s+eNgQ3=(XFg^_SFGpAPBZD=(#bR(ZH3bbYqvRA4Zo{_Q@@-*w8gy+Zad|ABU zng4o-FaT?4s=bRnwy7h7^VJ49_PuGt6%Aw3^Ipp5ilKx@+Z*_<&m%won)`+4@VZhV zHg!;XUaY*>_#7oW>`w9d$demj4)B-L*Vf+Fq$4Eu;;nfzU&F4!7U$Y;%AhWCArS@4 zB!BjEb@@nDgj}0dN(C~)NpQOJmA$-mN*V2{*!6eX*_INd= z=fDLXe|P-UT#c#q@ZPo1k2PUb7tiD4X_n7hlBX;1ska~8f}x~aVURQY=<+tUzpc8Q zt}=$!+y>*`gZNj!5}6~H{CYxa&DUZ!e6P7Q>tOFrPa3)|*u*W5>(T_uBFMBznE3Ltoe!nHloq_n88!ROprYEC+-9f037NNO1u(LBUw5W@{UzH{ zvCt>#Fg;Vo6hJ%Fop`;K%e8Y>BbS@MN%9|0Q1jv&Cs+^WQ3dArTb$@NUIfB^_Etm{#jDAgQ26YLod%U4$7rV$nkip#Q; zvi&FM<=Wu3y>RFweN0i2m&BIh_6I)3sl3>%#?p>Oi$q6=fLPteH>3$a5%w<^?vPMx92tvP>l`P47rW>_nRV zfTuj5mXN3^1KHqn*R7R%+`@egteS@{*&XrjM}$b7JMs=%nfx8OU+@sd&h}0>q$MBB>Xtm9SQ3y(jn(R)=rk;Q9OyH<*O6iS5u|W4h`{~KFJ0>Vg z-T)so!4so}@3v1B6kS5-Vz>J_5MmX%qd4QCPk{?u2foQxU_6MN!AHoM(wxhl*b;TM z1Vc4s3M`q+D(3}9B?WVa2OXyh?MVLn(58ibwBV0>n8?IKhxzI6M3eipVcjT~Gg@sj zr3x+?rRe|yL6CuI#nExc0uEL0z;~%k$8}GWF4o}+u)v>uH8KGVcw;$W=TtOdp<1JY z>&0-7e74P}{5i5mO07Yj)vr2GA`vMe_~&8aJdLJv*Zvq)>xYGcx>e%fBr_I(A0_M8 z-2LHMP!X^vW`ssRiVCJe8%@HcwnK0tLj!mXR8aCCWjmL9=>r&1a;T&p5CGt3I6#)n zhJ$eIk5qwZ7)WCkTnYj$7IOL0cD_r5wZRr z+F$0YH)+(-acsHH3%sf~mV>de!rMD)&xx*pi?5t-?`=cTH&3+WJ_0#<$&N1s;tVj1 z$dZbT!Ut-R-|uHiCcl-y7L}JrexMik7fubi8A`5AQwV^<3O zvMAAehg72IDOeXh@D!UxlQsEJl=%P>u7uKX`3!JcHnGka85W3nE#PSINg{E0{wD=S zhbu{ICpZI4(!`e|;c+h^v9W_rsI((Fd5G*^&@x1f@g~_SaIHAVGpyN+_`OUdWwfb%tOm<=fAr+k#^ol)(mY~0;e6ub=ZYCNZ#N- zSNQxcn83j??A=7`CZ0T&ugeN+;a%HG&NJe)xG5?EB8&P(r;-b`nFqjTo4=QjzwCje zHv%x^VocPEEI8_gyF}U z9pvK~V+b=B5R9h2qxin_J4Xdc(z@@#F}NTA05l9fPEw?hhAjzx_ADd}+D-w64KyhY zD-2Tu&1aX5;B7dB|7ae%NN8CnroskUKKeuTG8&qB??RczamaLa)$>;Xcns?hoc{q* z9+M4;1OOmB{j6i;?=bEeCj>{!Ndv>{hTuK?$PU_V`q;=oS?|dw)c;^gbCJ@}{&ol( zG}`__Ye=$*XQ*$OKh8J%p~-)M$$^{R1wVz|BB4`wOXN{NrnbI*51{T&F}p(eEGcsd*hV!#dDi#q_s*aTQp>WFk}=m;i!mO@#$~$1$9Q z)DGq=0clhG9_ZIYpO1%oOOXCn9@L{E;4r4!r6aiu+H?x)UIj?Swau?7%vC_yz7;uw zrt09C8M>e143Sp^{62F%Y7cKvioYprA2YA`bO&hOf^B|EM;d3yQ%a&@eJ!T_R8agz zHhPGZCFS`JR!m?Xu<>>N0p^zY0%^=&eE6sE2t;-_DF`HSb^{%BhQ$r;{|s3<%TMHJ=< z2|zw&$9M{Q@6bFxgY)Pr{WDSmS9tx6(qg>iC`@J*O^=qiWa6wb>`x=paOV7;29 zV-v2R9sV)9Z$OA=;{Vh%5`ROu$rsj-RJs|{-y{i0!c%SF3;fZ?K;p6@pI>rjuC#f!b=sjjvC}>NP?9n+E)e&15F=|TC@q-uXa7u-Bg5j0PQo)`j zZ>sI6uXDvqO08=l=3gWk4;h$nL2?eU*{%W%^aTRLTQ5o{K@?D|^S*KJ0afG3xl;<< z`AWFU4nje8|G`DTW0H%|5#FYonpW5RaHNV}l>`v=l>xAdqQ8a;@-ZLES1y)nap^|N zIod@?T5cf9@Xl5fFaWqwn6+$OsG1-tMX10B6x1(glQ5#>XBtB$*!pSG>0O!ob;qZaXL6Ha7B;80j?8 z{}q;zbcgif=%+p44x-ZaH9&k3`(9`qpP1PT9|b4bsqA|LO!Nt_em#gvlaEL4(It8d z{!m={WQ44r9K>E;Th~1_A+S4*pXAl@ald22vvnJl-!F=%~@s9ixdn@!ZJ2y)eN8w!b_^*|_SCkF)X@Lc_ zuw z%2r~joj8{{}al_64lG#3>B!L0}UvOp^dlC?}*6T9gg>t3e3y-T;K&pW}WcNL3cv+tRdd}&IGy{^+PfR0r+x@iY*)P$?01PsOC(oy1 z#nV-XH;RVIrmexLdt5^@oZl~M2Pq|3lTiCp+~(TAv`vbqFsoFHAuLO>0_62)5V};3 zOoU&*!AT*Ssn8y)OE*&%QZ|;1KgR^NDcJ048K=(eRlPU%O$hf`=^u$tJ|lmjoAU@A z22e`ua92W66s3+L0MV-b1)c zp==w?W|Ijsh{zsEg;)*9q1CY2OX;-hEtF)9qaXXP(lKfS@vT8WbO@;Ctn`>awixJ5 zkV?)8pif(N0QQe{=&aezjmMHNo?ETY^11IR!EdZ_!IP58gB@qp<-~N+t*?b>RLn%g zBYyR+@6h4Mn-UrR_R;?*34M45Io7xf_*(Zp^<^_lSHz~6k_BPtIy9ZiK!4^Bs1SEO zL$hL`wZf=(T*2y~oWMIxy6PqMz(N{8P}4W|pB179@)(-XAjQRkU!41R;6Y}dy1iZH zl6bmLiQw?c>}CK2(KdPV8fM1N;+H;Sk9K)H<=O1uWB5zYcRRt=XKd?r9f0mUADj;B z1kD2Xk&Au~#!EdkP#pWw$}6b@&a38XB~2m?1X}Qyg-crDKjGbHl-QA*j?XLP02O3% zDqvOZPs?=4Ii!(#j%Q`3+QV`#&)scQXNmK`XHg!swI9*|7TH>Be-H?P+t3PoGl?WE zNcJP|@|^46ceBYkMF~TM^25yD__nO291fj)$oXZA9*cUzECm{cyBXYR*XKgpffH<&>{;tfFZqBo_tinP;NIski2aIz4{*5|TACBfs!izV=|dyCoG9 zz~!gb{RZ}x*h1SSvXSt_*rdAQ05^tmZPcKq-8^b8<}>mZn^a(S$=>fT_0@k(le0xB zK?dbLJY%G$iM7h{J9;n{$k1uCEb_d0Py8G(rOp>cf|7CMeG@dK{!;U2!)zL+_y}bA z{vZIA0^M#9j!2K%_}_HXFCUCjUg&#;sn(m=v-z5dK1I$IIt6L%GF+~z;sj%>iGs-d ztwx5ygjCKHN1|PuTfJ-I_%#Z|fvfH1>!bv8)_lRy@E429$}Gis61OSMxbSDz90@-Cpc5pAmFzC% zYat}Wlhp$p$6TauFQ=x^D4)icV=O~eLCeNu;Fb$?r*7OHQnkQS^TBhzu)J|9JlHor z?PK}DGx#cRf26S-3hMvZl^6^)kqvEki=jR_iW!vp(FbX$%oxxTSxvk6$5_(ADL%Ez z!Ueg?O*nIv)V)v;ai!VF!R*GG*C)mAP4+8xmZbJMta#t;W_i+fm4b~f*dUjha6a-Y zX2JI?JN?gjWF@k#tSrEV6khcjk}3V3Tp;b)Du|yCd_WE^D{IHURBOmvapg&qb>t+c z@@-?b-+xDKrnuU7Y4peX;Bg%xn0&YA4i$<>dUQ;t1V8EA6A(Fp zwrw(9N>E;KX!cv;Qwg@;0pX!)ETZj*harI_6CDaPvSq9GXW)OGvhu|)!;XB0(TJry zEA$e5YR3r`w()t@f{N}U7$8ZfOgOhgw}fEuKE0U7n@DSRoF=b?(t?0zr{N#2H;AhD zlhh|tg05^rc{EgFH-g3J>NVz27WM8Ok7qs;zO~8!3eqC22PPS(% z$I9%xedtE-HQmt@k77Q@Kf^Bj0K8LJt(EDkiPF#_agy4YO%tOr(y4JEzR4NF6i3%I zJ}zq8*%t9Psk`Bl&Umodgy>{zxN;@qfWNL%(_)wLP8N&oU=lgq9Dqt| z45sReb`|n^m5`3}J#aMnY&tKzZlfj69wy|%qeVR9L9=9aS6VJcn$fsqqGrGZ0c^t! zz!HaD_Brrds;Is!h4AnDj;w$`T$pz~3eJwFW^OlCp|v&VUr_`JrY_fv*wiB*{1HQ> zR9)Qq@r7O+)U>nyKGEI*OcRiEO-mEd2KKBpH?vYs7uhTuHSqby@?Ho|&=lDH8%gut z3izkU4iz>ozAqE>=S(RPr|;P8A|P6gr=^v%d8#W>c|X)r!}J#dn9~_T)~P6(6P%<` zE{)0DChf+U2F04m&PV&3%*-3+4`4C;;48GBRV=7DnNTimgz3Kx^pTbA%!zv9b5y;b z;d{+zn0p^~5_mVrU6ScAev{rYKQeCpCghkx5R^kwq#uCSlazx1u}>%7v5hENn$qC| zI+}DOBOI3rSn1uR!~SbHAwBm@Iu}+8arJHf0(R`XtN?2dIb-?9Peb2X9SQ#iHuF*W zU19p5XpS20Aupl(RS-xBO_Jw`rpj&CDRbz?c-bQHge57#@|=7^lGT3jF(C>YU&mCT;k>TZ&&`H!$wxFytAfws_qJ?=$8 z>`UJbOaBfBicz+->C*3^M)}d`Yzwnv00~p~NLFygppXX-Q4;XQV2szX6Xouw$#!pM zJMeu91+nXl3+=g^nVSKEu~g&8h*%dXFDPm5jk@=NEWGR&R>8ea)8I@e{=%+a^WVy5 zWT<32ar3O+*9548(8C~dP@wwfMrF{=qio`JGZ{2I63&2e`rj#S>+7)g+RvfFPN>~j zf6i*P#-AoVlyJM@VsZHvChWhU**l+~ykDY%Z z2Ncg4P0;z6=~g6?y1;>y`0RsZM)4#MYRjZ)Rc~F+Zqd) zpsIJ-n0KL%roT^=ziQw8%ZoKe=F2G=Vv{j748R*U~naE^xh~17J(XDZ;esTXq6@GOwywuuo3t@JQ zCA2c!7{lIA_5 z!5H9~7v%4w!st_~Uy(*9mTV6d;{XfU3W?XhM_{0EpLx+~S9FoPdCnpZ7j{_5e3d1% zKJIfK4LU5ZpUbtozoFsH4;`Up0Xo?u7K_-;;^L+`ZtNS2`<3pztx~bEQlFcyE|pzD zyMWzLIFR8v~Tg??*(FfRCxe=H^UvFViAfjX&96|Oup1Sc102c5Y8d|HhWhucJAAy9 zU%SlT$}N_Xw!888G70-zPy4(QBWI!AVdvw%HyPVt>IrzJXVRA!E3|N4Vi$$y!8;4U zY>*ka?;IBem#wf3hQtMZn>3UnJWar}D-?+o$msXO*k|%y@QYZMK8!y1a5hUqLmBC> zAdKG1;O7u0XA8z9X>izL{~5&vMf=|&!hTIp@%5TxFUyUGaZ95fM9EA)72K2>hDZdR zd*|7`S6C4|Q zN@h(Z`%r@zs(r0MlA1OnT9@BhQX{^W_#gE*7fEe`mSrC9dnbnxq}}i*?`PN)KebIA zu7;u6s)>U|Ao`4z3Oc@16lTJ z(f=$05I}$g0003fxZ=ctLVW>5=-9ogS{uLLHS~39zSq7vauxJOg=w{x8n@`@TJ1NA zNtwqK8fDEFTGX^QYqr{P+ck-*tIJhOD(Y48Q2t*R z_0B7+2_s?lYXSRf_|;=?dj1)Yv8TJT12Qk8q9%r;eFr`x0)CSGgxRk&cE1=L||Z}$GTNm<(>%Xz0QWjq=VrC``!OcoOPuj(6DP9E~cTQl(aFR=eMWZjrc z5yAkJgf-3G>EO77ky?(J%@Wh=5q{RNw)|^ApsmS2I$gSP_uH9@=?J6v-TRWTF^8g0 zY_q=d6nkitvXy$lKHTUSn2_tyF4BrZ*`TH6*+ZthdifoXP;vFOlCo<6H{mA)2mn9; z02j9j!|0f3icy5q?EPNRnqj|bkZTLfu8pz!Df&O9A#SnU{q=RLR7}_TUB=~2yPi3K zvqzi0n-JQUWKLx#3EOEozTg|QF>lV8eJ zCyanlg)xM9%++q6#WDrg(>b1AZ#Cf3$?Coh8DsQFqekXFX3WbDw0Eepi=Ix=T+!S%qE>XLnQ zJZ&Sl3e+7PQc1Oa44GCUz8I;2Htja%C=E8`$4`E(f4#Rg&1i_6T_@5S!Rr}s*+Ss2 zQ-oA?q4L^jv`d%EM_MPsF(J3_{TsR7Ym=g+bf~K)JR2^xQ;>fv3nKe4WrqbBKi>s2 zP%^agjI8kT0O=N_Kn3FhO9pLr_a#(jyn9Q)APeqwTkQDMY<#g{C}DD#sYfHsdsIGb zRAzou_V4l!=O8abKb7bEAn$m&9ELQzzpj7_%i&gR6|{7cawMrl1`9fTgtg=Dg^}vp zxJNY!t_BSD>>CCD&R+=a{E9c3-iXv;hYZP;joK47d=&1afrfOJoB11AkXjJts0xu@ zY0~2@ejy!Uje{J$nJ~Nqcvi4A8F~>ukDis6vXfewm5&pyDZmi=Kq%@O?-UO>Xh-!fmpQ!1v9L&ud^N+*mN@V{yu3K z5NBWm1`HY4Fkrx`<^wtfYL`_OM(eIEaaOWbt#emUw-01EAUTaHXb324v75*e9p-2o;v+=yY{nw`w!4G8 zpO%*!0tlNpnZygqt+;q|V_9q4Vk+eaGx+3g9CbRLgkxkH&`tGYp*&I!mLQo4=1JEE zq{szdU#@PKK24!vgzwp7EQbRP>0hbpx`)efOyy*(a3N;AkhrZDyoYqq&R|cFJSLP^ zXxDw_0&a+D8H5eJ-3saL+S4m6i2lawPW&srwf<}8%vBqoX4*6(*njt488~K|9<08R z`s^3eC&e=Tb_oVK+^L6wl}iQd>hB7OCCE`b+A3j#hZa0(2_l-C^vD*C{<2WceP!~+Q$2GN#BEnQPW zhFl~rJwC?r}F^8S^GiK z;ZBi1%eqBRD2c?n9>Qt7UW0Tr9&Debt`0>bK|!@?<~I`|wts7nAhI0Z1^Is7QwD7U zz#9}=ZYqS*x-9Ha;ovp8!xIUe{Z1s@B|ojO84og2OuJqtqwaF}of9I`J{xwICP!0H zRS9>k1R>hXcskaT;Oi@!W_Yb1X=tV7{17$u+uw5`tIAn=*n5;HZn@slpo@8+<)ndqG10G}rjN7*|V|Tc){W|sY zO_+E!RxrevJYB4P$I&~8u>NhS8^@0D?9i?;&4F3D7CVthKUqmI{fA$OX7eu`X~G&D zguCRcJC9;*swzJAp%4j4(B^uPzDI;sMRqc$E3z*SvCLJ#JM>@m!}Nk;j>TQQH)Ycg z;#%P`j%m^=M)PU|*BiQ;uPdHSl<0UZY#^_#Qp9}^!4m~fzUJ6pg>LGt?zo7O-A{^2 z*U)@#(N6mqPe)d6J$LTM^$z=kkb!&c#ecT>2S#W}qMO=;sQPEVuBo^iC*5-z}~vm7}YJWDEXDwan{MP?{aNBjMHZ;Malui4^RKu~#s2#LV=cSv5OPBExF zeuuF8WbU4r;pu~zlGIsB_^a4_Nt|Cf2|D-6t$I7QOm2<7bvwRpXvVhiw}s3LLY$1y z$rf*q05397&3(bmRSw8q_GRhSyQuf5Y<%u4J5dpD=h_bA4D;-Ej~25JP8v5dC|WcJ z;+vN=1=Qq1fLgxt@{*fKbf4KQ(5*WfrbnqN^w=6SRpI&%=^g#kmY}9yd2e}`1;UKh z37$4WI*Rsf_$LV}g~!h;du9PEAw#~oZ1!5Bf3`IFeXF)ZtnKdeatr1jUC2eFSogHn1z`8ZEF5EgtGM1pqN+{}U&g^u#WxbC2b>`(`OFvC3Im9u{A{@sHD&28a(LvAqW;0WXC6WK2Yn0l zQ2%Sa%r!P38F{?O6yx=p*tF-*RJX*w;NF<{pvBKP z7j$P3qEgs}5|rUsw7x64Y)v=p4lf^l^V}U=JUG6kj20TpW=Kd3BYJ(;Ykf=X1!OKs z7mi|0fG6#xelqZ!tWl5Mm*9JAi=1Y6?9)hb+nS`AVOA(SJ##`=(ZVKAoy*7D2{=k{hiNLd?)Xo=a1t#UOfkcGMH?mSG z|32j`0Wz-{2jC7rblzQEG@N>eoJmotLj|)FMpY;KrcGXe_Q32Xwq{s2&-!mB@{>wm z$mGA;u_A1(R5Ocz3a*v*L7_%jw`%d48>Ll!b;Z>eR#qV7vv8;wIcG(a!DT+0S#o1{o5J^ja_we^7ruY*;Lw1%4+G z0hT9?cyG70ZkYGJcx|#fTzCRcc6B`V?mV$Gcvq7Ic_Ht??Z`MnM|>2oIO@Sdf*uMf zdaOi-**5u4+#byzQSc}cRR8ooo=AQ_b$qj?Sg90fS=kF@)s>{78uHZjOP-x5oRFqj z&AB-49>CXD=x1q}6zAV&1X{x7%Y+k&a*AR2w}04P&F}^oBU%vF@8#lb5;Q|(;r5s< z2ZMIw7_WlTwl=W{Cn-C_0u!kyuyT}JPXmrVD)wEkg16xI(Nh!b*GA$eApNo$)%f~L z9}twgk{DB<9lXeS&tj%bSR#hyk2I06bL?@!!nU+xr~4JE=`ljs+oO;nxJSYsquI!| z4@|(ixO9nRgUf)APO zHC-1-Mu`F%?(-j}0=-sw0muI02t zHA8RuNt{wi9{ZW5dkLa6<(N z)}x;N`Q1=d|8?1|@?NLpN!Uo8!{kOuazfnSLaI?2H|AgGx5Z~db{Ofs$80l1emiyd zSX~ybB4ojmll(sa)E!B+rHIc~Wk$xSc?EUn_&k59v&Lok>V2yQgZpsF-FD%Wk zRmdEBoOiVbxE0iU{_4owaly5X0X6pEQ6;B!si1D^z>MI=`e12)>Spf(S<@mW_isCW z>6O9WA+`EZKPmEVy0XA~UdSdAscc&R`jr7>$y(uH6W_bvHwAZSiI(%XM2G@u(@Bfd zrIgIB=&;?LG#3mf`Nwv1+tRIYNAX(S=u?TebBvQyLVpRJFwLmD@-kBNQy--6B~JgE zqfb+Wj^SIw)y)`udeb~ywB52=L^LW$+X}@^sC%v4tn-Q zHBuA;60W?;bvcpCK& ztK$bw=YE|&Xw7J{879u;p%4HWI+HxTRK!^sJ?^z>Qgv~;EXw-H#9oW8Bc;a*K@>6iI_Lqu?wI8BItku z47lg)q6g5WOc8hg% z<0i&l_J2+33db%_TzgiRr_7buwYm&zSBo_P5zqvthL_SdePGke6cM6ZeG$PRGa2<$oOn9H5w8zj z0=lL)_~xL?mK_K3rM)Y))>6rXhSBA$K}8`ZL_l7@!K`t8cjH51H#8SrqN_p6%y46W z2LUf>z#Zd?D0U8hc7($3@_d9wBpX)PZAc{@x8Ns!sC~KgPyVE9g-*hrf9kpIv zyh;UF%DegG(*CtJTMHzO082o$zc+>ybHSVn8%+g9|8OzzRzTLTw)Me9d2Pg8TXa+< z22w{r!wa(HSxGei8fZT(f`-0S*<={VyX!9Ve7~I&=uSFR``F$_v+14h8RM76#cJRA zCEdoNM?lV_2Nb^bT=gM(iSB-ov)dmiK`V0t*5(Upy^OJBv?u9%%}nfvIiykRA8SXjhRGg7vd!qzZ>n+A&{<_s>G1{d)GB7Q2d7SIt z&&2~lSe5yM*)ANmM9^@fSo*kEsE({m_-IPBXV>IrbU!z^+8v-I(>I(TOIKwx|MDZN zlGutuHPLkAI*9d`dauC>t~tF8Z7>YX8XV1jX@6n->)=)cy%+oMl-<3@v{C%$dfoX! zjV!WU#8PO5Uhwxp1|DsAknnIBcGI&x(J^$Cn&P{6Z2a$40arZo;xPRJKMKs!6BIN3 zHb*-+0vy;9F%dGm&LM1%KD03$)*AhvoOlf%HBU6SW-*hhN=J8*%iI!&14gn?Gp^a`JjGMDEO;O8C^r!{2H0W3tVh z{eHX8vDlqR6sM+4X^n`(Ii$iIi$={kG4VYk-v-6dmzisw*rgWyBN<#B(8+Z^myEo0 zYV}v1)G@8c8qpRY0yO(6$kan?{JI&hDKK-z`+i?>rT5yp%Qc4q+qZ4Aqdtj5rqRL4$tUTSi70@QlO3JoRGjO zF~(1n%7$uR$y``c-)+&x1V%zcS!cqrm-zSUkn4h553EXxwh5w!jRu&CoH8=@GZ;j% z+=mMPpLJ3uYIQ@@ko@OglllIx?dbN;;S<>69a5XZi{fCktb+ptBToT?w6Op}GWRof z?3)S%{xcyz;r_0qYvwQZ%`MQUCD*EoyTkXW|3U>g`g)_SHMAEGh2P&D;ki#R2s^ZI zU|5p%%kE8$YhNP@>|UDz|2~QA!YPzK3PcA_d>nYa7N2@^_+4tRzp^_K3g!*`8GI*U zwvXF;97?Oah8_Y8CEKyN8=0b?a^jk-t4bT%f(lfx-IOY_F0Hq^!3Ft}R>B+z>=K3c>TA_Z2IsZ&)-2!0TLqHyyOMM)-?&k6 zPu?B|qAHr!NK0s~y;ukU9EO-VpMu<2h<|08-39tUz|!YWs-BQfcux%j-S+M)Xf{C( zy`k&PN#(C6w;D+@m#cOCLg%n&T?iC^60EBStoG0~s(l(Hsid2YYc!Y^yMVa38tK55 z#mm{MXm&VU%dsUBPUxBtE`{6`C<&|yTR9$$7IC8G%=s^%Zb<0ZZs;kG@`ry_=wIHUo5E;0Z^{?IaH zqt9Ql636QouvSA<>7YM?;&d*lW{Q*Fo*k8F z`x@E%EKYj~&Yvz;`vfH&Z9TgRC7TtzJGez~z}U|JCs2Pl+*D_LJ@oR0+!oVaCyvpH zzI_Uom{dSmGc18{{K7Njbp3)*;K=@!g*zgcIjw~nV#tmyM*6M)q2@!A-e-a zPa0i&LAeu8jp1&we@p84U}MCnIlslLHpxu3+dq^oWi7)q3{%6l5(j|jS~DYTskE2` z7?lrp2`5IhO}|-^wUx#!j8n#meoNdFi~Gd4Z=jmAF(y#abkCX{4)Pr zb!`0}{DpKq2EspI`F(BcV}3|H4aJ4OlM7uO;@WyG>@=zw?e#4J>Mo(zQm=TO?Cs53 zl>Gtw>>+aVW&dAt%Tq>(k%f(k2?4b)QH-gWfA4b2KZUP3SE%5|k!2lQ0;S_ibu)vN zp4I8@n%T+&VPq5=OmOCkSwVo4C%IGrc{0!QQhQ&Ni_jtrvZ0ig5U|sTzcq1cG{XQ! z_0Q$n$@K&?Y2P{6Lhw8;a#OuCpb-iZT<>n?>dkZw-mhTy8fyxZ(QC^dPHm&U69!=! zIqQ|RAz0OC`#UB2;KQ>hL%yF;qW{8nHY7atx@mG@y{3+U@d z(jeYFAuryR^57#adKhF_>=%HU=vZLhCMcCCA}$T+1r;}~(v_03(}Wg>OT7H4Rb5(^ z=X=TbH4Cw@WdBB6Y}A$2C6-9P#7AqfwT-K9@~KuVHe26uMM#fk2vA};Kc9z5==@Hz zF*t*&f9tUO8*{a&f>!q4CLMLjOO6IzICVY5FE6Y*0tcVCGhIUE!wVY~>oEhLe-WwM z-2Kf`eIOzK5H$blws1Gm#@RshqSsjH6A`-f<{Di1z7&kd^^2L?DkV19+haCUQ_b#? zoJ7)^1zdH1?L?iYgkcKIVc#EY6%E?*W~1akYwzC$`cj<~1e1{%inTz;@V!15ajP~5 zBZ^lvy$c-wELgSFG>q!K;p+51dwwl8O4ZwjeqBXXHKmkh%a(~x!U3L@B zp=SEB%uO*p0U0XyVA+d~I!-U*>*Eda+p7bhFr5EEo_oZ31#z(4aRQ@-%X2zhuSY#7 zK4GY)^QRrjgfVq1K?csCugofW1*X0rHA&5vJBPC`JMnqMZ*Y;!H2X);qko8Cb$JKE z1I*Vln5%AcWrakVMu*U+Jmb6w>2kLwx0HdcF*KudB<_l#<)(KomMGpS`(AO=nrQ9! zxf3~&k*W+zjBNzsuORV_Uw)@OB`Nh3dO^_j zncN`A^KknQZRzd0V7QNBK+bvKXnf9qukyUvK=SKfld7wz92r4NWDMa%;P13ysk z|B$xJG(!tTnyqUp$os%lp9Vqv1NnS~J}Q7k&1sFCOE*Mvu!7#xZn&v)LOJE9WiJM!Jl> z2$Hk}GUyhQEO73Uf`jK~V0Q1*d~0q0v>*3mi|wMjVTevS9Bqm)pzm^Z|1BB{1{gRy z=L`^u_MkO90a`iIGn)!)W;|q9#C5xl2$4u!b<%fTxJ3R^NET&~ z({=qmZU*^G=1^S-v2nyHuV)gBe9QmnuQ%aWoCjSQzIDp_Ip#{-%R#E1Sc5kLv3^LY zGx^x40@oCdZ*p^ytO#e=sX2u(ERg;5~sC<(C{*N++EEZ*9abR@i;L{of~m;U=oy@Mm99B(#7IMO(e zreEnwo0rq=b2)TgN}om{#z6=3f6!n9eKPI?@tRClMe?#v%{E}zh}ZTL4y z6{la2?*NHm(aBS)Tc?K|N{UOPcRRG%MiTU&k!yn1yeUmM(gOsMjAd;2ze9za#yaEN zp)=l3MNhHH9-T2&7@{j45-lNySFYz%hFSVzMSnzi3qiBj2|S_O6)o(SC!cb}W+`)}^130+~;@NJs9lO6LOe}Fr%%9lg19HLB44ap7YfuqCYJ1D2(& zlq-1Si;HMS?fNp>jYNar_2Y=yHRI^@;qZX(CyY82`gR+tOx5;%LaQi=B~_%jQpnPv zGY!KSO}rHolUd-e>4lcqXy4V(elD=W8HNgR9xJ8MA!{kX*FJ@p9e~_;zui(HxHSNT z>%R{2`lCA(sWiq9|77G8M?ITf6uA8W( zeLRsdj~@6tH~M}i%jV@sCIeYgNyFK*N7fLr$x`Wy^pkB*(4% zVBk@l$zA6Z&zF5X)s2t+Jy1m#pmy5#tXr_*1I(Ihzp6+TzNs~ukA4J)&qI?0KmAr4 z?nBJdwNpensuYJ$V%DzoJ^|_a{t*HHLB2*}U>h4j=CCrbz`x|8?el;O)cqx6%z!ef z#V$Pkzd$7)O>TKZ`;;(w6yAx)9G#{(J$(2N zhjuKs_o-B^{m2-5XKUVL|Lz#~1IHBdkTRq#1pa_uJ}RG7HIBz&Xl~YBCsp{+U%l1Z zoBKss)x|nWcGTxNlILtJ$xXl3_r{iXr6{dnm$*h~@iF2&AQEE`v2dMlDkIJj0Fxe0our_l0u?5(y7OmZ}QybYIG)(wUF!Be% znn}f1oS9g6Z$myLn#cK_KSe76+do+N|T|ue;q^nL+(cQK314@ApOmF~Io*W0q*IObCOU1>Nc%#7fb}vE0=rV@eclw)IVHmmcn&P`~S4<#iO;$;$@jXdQ-YKk!Sb3uKA$&HOYux)Q(s?Vk z$e(A*&pp|kQ3F56xS?r3Zl)^3OZ6LR9`PD({~#6htEl(ie%X&hNdM}L0LBlxkGtDH zKpA6wmG}Bu4w*NWzxK#oR8 z6BKx0iUdG?%s>SBz@z$M4zpFb$mU2tvY^}NXqDf%QF?XVM{jmyF&H|5Mn&Mv#ya>*O?~BpA z2o0jMsn=w#IpQb+EN-X%bVRFulM%wN`M+9BHnj$ET8YNt2m?qwc_{r4C5DiM?B;Yg z5Y{cYcF=;Fok(p*0G@3KgpQPb5-j<0Hk;T1=AZ2Ya}dD#S~8}Jsbw8T@>UwDm^c&_ zIihpB(tkyMrrUHBTMemXWHK+QaEFfhu9$QgHzD3eP5)Ja&x>o~Ntu=IFO}I9Zyhq^ z@NJ4e>%cMOK4G7mf#cvtQO*1cED@MsvBkP(adfCNirS%a25qg>wDZ-`@`K36bzW=i`@^Qpx@l^dek zmC;~c&qAE_Q-_RI*<|vChv_f*uqzUMhMi3Qs`Wf|E@K#QId)q&DMbQu5i)auGHV!5 z@ou%oZnHI)uj(dZ(bo+}`dngA04kQ&@*4KMkvix?qZLue)2g&p;EZ<1X?RzndQu@r z`Iw5E9Xh(YT@?N?%fcMRv_{UUqmofVn*4M@)2hc}oxvYx*inrahpH7Fw>7>L94&f0 z-l(lD~pSvR*4pC1uB>)kP33GjC_lLVk4gl(Kk!!S(KuI38i^UO%L@ z!Kq}KTXK!p72q}NdDpyNoh-cFOlucm}3c>8GVCN^`{2Xe4hs7q({ zlLzYFY?tRxApJy47o07h)O;@^n-);%x}uo!&|yOw&OqOuc=$u;>i)OMqi~SbBq<{; zA=Qc6oj%Usi0hdTD7PmULITA{?-Z>I@UzFGyJN#y=~!lk-o0m&lpF*9c+bPjCBZ4` zRxbb>ruTJV)9Iq?X)QYZGtu3_ty1N~*GEouKU`opuNcd-AQuv$e zR`2VsI)X3`M)%{KF^l9t=Ouaz0C|P}_t0c!r&8M-$-8xIAjvyrmLJ@1*oOFyGfeKt z8!5X=$H0}Hml!@lAx89hSxALh3q>WU^zKsZMr;?GzGfJ&+@zk1R!p40@``T}RH@WI z#P4Ev2I{!i)-l0PI@AT+au?GK1)Di7vaw=GkGi@g&@2vcBZ!JVylYq+w|T=-Sy(b4 zTrF#Gy9(02{r|#8pW5@TLOStCg@Z`v2VTcHH3p7l&Zk3MN! z8_!2PZ>&1E6NZVcLbiMwEr${R;u~Y7LZvS{GGA3s$$Lj$1h={7q`6L+`W z3B1b#*Tv9QhhvdPBuO3cu=#7g{=odNLX+4pTXJl|3V(txm1PSbk)4L;Agc>ojh%!I zA_1g6NhXD#YNB1fY(Iod!c7TSp<))t1#f>uYb_)GKMI~xSk|NG*)kd1JhXfrOR)M7 zW%k7c?aOang&+lFLK+{2BPe!8wnhP2tx6)`B(%RVyigoD3xs(~;}&m=`L*Lm3nrYL zK+vpY>01zZa%BABY2ehy>^G$Js*U@GnKLaqm#OB^2VUPy^_aD{f&;xG1GU5wwKB_V)hcBV#gp_0|nsh~Gs`8Kg38UkrXXTmku?Mo0 znb-S#$KFSVXSmwHDKk&~whhzJ6i(iFB0`jcVVlwmYq&ehP?${EBKdI`$JH2iAOecF z$g`2{TA|DE#aCT2yK?98LP814ftMbIA~{R9W%Z48!Ybba9dlRpC!4b{>>H$+Y!xr^ z+V-geM|Smk513B9e&5MrygSf3rj??q*HZ9MT}mA=hq%aHMn%U7muZn(rr@cD{k!9#d zM@K783Yjm9;ETRzgE^4*if;54FCWLDfA#Sfh@v+Ptz(HqSO!W0h^tCr4wh_(0Dq1v zgG~gj+D$A{7H{~p2m?*#11drO!5KY{$4!u0FoWou4)(j}AQTJ_^-7x;=;1wav!S~e zCcNph0}beNU`x0(*YMwI4^pZso*r#8c&Ah$bSQ|2_54TjyBJbRvA4@TEPFo2{8W09 zT1T9e(4v{~*Z)i;UsKc?I!qg#I6T$mg60$j>k4l(wq&@aH-}$tyDVI09X5`9FExC_ z-stj^{7unI)r36DlnV@3Pp%e#V)?`#bFWR&pXB{9t8>kUr3W{DmNSOMJ#VFD{`{FX zf}DuJZGK_)yrO4+fbvg85!Yjt?lzoma{UTz!Nr!u6}I_jY5uH}EwrzLeBjN1am5}v zW{L~3@>qN+o4Iiw(bfRSUrFtm!m$$mdZ=r5TN_Qpg>`Jh*Io?=PF=^&LO>^5DZ`f;X zA68@2k>3{kij=Ri3X}#&RJGuqd7~pl9Furt+b`?<{G}Jh2*MzoA7x3*=$J)Sb_WI~k-Y zk@m!Vf)nnFeLt@~0OunK`|wl)Qpg7+4p2rKQ4@RJx9yH}*0qxAyx@$ot98(J4x=6! z^5iUte;{!iZ7$@|n;}ehC;r!pwWECugI9+_D-6@SJu6buN2%^kbG94;dFmp!X7648 z5?C8>(GO20R{$oHv8PeVb6skEshgAnI+MM>vd(Ewr>N5k)WwaQL(Ge?-2b^GSiTos zaV)K}OKVd#x9VGFz7S?P^`mAwD)O9u4Ovx~TytbaHh!#TcW_dvE%qP1ba-jS!EL_k z{(W8v6T5&bE-atcW9p{9T~y~&5v*&md>#EF78O-OYq2mHQ-Fk}3~kk^c^0t4kD#$m zvwe@mJ>^6SQSJom&0yn?(kjeW%gen$#&|erp))8`Tq2^lY@^EkIAmN0C)l9+7^W(= zdDu{Il8|Flb8*V!bW=@NLEF+x)uc^Z3ZK^Dxl^qc+V|?;B}5*&S+{y-ym6ZRwoM?M zwVjdxF->KlgQ5aF zoT(%NF@${SRK}#yh53o@IC8I<#M?Dtr$@wvItW3eSbzZ!I4+wtOk+D$Htsl@p(PoO zh)mo{%zd<0i>(jXa``AL6UQz)S$=$_C!`J=!ZpUf+*?}t*fe~yO8XTu(|nw1wt|EFN?4dVvP)Ea?i!-`(T4+I@C z1ROJ!56fyCKrd!aqBXe6bM1+$=^VNTW&~na;HPkT*vN++;gxm z=Xh4s1P=oKy_yhaoC`TROa!CR9-(MXP7?^;6|y!Q1Z)^VpnIeApX#i63FA%uxq zUCnF3^$&mwYjW4?3gW%sXenZoxrA#i{3HI;GFRp=oG=0g!EgUaN1u-~9ujr3p_;{i z@0jC}dY*)cB3mxti~a*;Kh8??fD$2>pa#vYU>=-n=RY0fKF%oM4qrfbPY%*)_4I;||*9TNZdAz8qO30$O9>*-(N>ZlAuSP{pPv4rk0)uv7jGQcla z%C?@!5JR>bbZ4tB5Zx!4&Eq-FrJYw|lY5F;T8ogOfmA{$5j4M@9Kk=u1BJe9dfLjo z87|Pec8}|i9gQ#NIzRX3xJ;w<9oQ07NhrFb#POY@dwi=aH2gfnkPm#1wrB$j?X!Dg zY;tO!t=%ooBV}j+s?aKukPXBUEZy_39zCQ!G;n+bD~*N(-C1Pc6U~@-;H#S^5>yq+ zfnXRN*EA+{^-}kCvjS+Szu3!2jd)+oQ>b4I)7Sy9)m?ikawgZbbzmsuws?L|z&XvL zQ}C1MwP@;2 z9wp2KE^XtD6=`rzb4=`y>V-4Q#({+RG-I~T@{^mO6<1>KnoJ;>l?%!=QAqdqM5Wo) z2IyN~MX9y&an`Fo$b8(#11~v2eS=y*8wA9^M77rNhfH77N}}EUK3|bX4{f=OqKhuG&V2;p)@Nv?#Nhw_Ajm?7M@SB z;GV|Uk0Z!Du?Ho!3o9K8HxEB+pi7Pyb8jou4Ia77SMW9)fI?-b@Mo>+J!BJ%8{7l?*APU;%g|euGg}k4W8RGjiC1a#Pm^O14 zWyVQ9Xy4-_PM>8s$vvjIzHHt47U6v!75TrLU@L=-UGaqGq6H9OFt#{0bPVhSZcx2j zW3J+u(B8gyUb1d7!+7!og2vxs^1;S89c^ERkeUYDv)Eci^++`omB9xdUiJ?+<#LPZ zHJHX45PP+R7VQq7fU02^FhWMI?RooSzMHwfz(<+~I7!`frD`cu;o_0O7$gJu@05U3| z%gi?_O8j1r5Lo9%3I!?b&vA&+J%nHE$AIZOvK_7HR!mXIacl32w66J*964@Yr;X`u zR2e3v5PH10d8hrA7?w=sG0Vp!RIJu-W)N%)R&&fs3h({8s95e@6V+*-`uzR?S2M+C z6p7(is?dDO0KJ0$e*Gs)mRc1=05S~ZgSUAVf7}e`q>8L-)1+x8_OT5j+|XKjEH9FA z;>A2c*zMO%S=r0r%}X96-SmX2jhYt4swrkMLgg%7kk+5omjz3@mHXLfxDjtNxTaEt z18~qWb-)C?F8tpbwt%^HzQ&s9XI271$tPE)2PH(=nn2yQc(?NDUk)E3+wy>D%LzIa z`Hk;Vpiw#+Riz$BN77oYkt1coE_YBX?KwFG!?saP_~j4wgVVjyQYG_H=R2g5{L=g4v*-_R|CLpZV}JqJEz731h{rXufsb*6{+)7b(CJoj)^h zJ0=wO+>OBm4T63uBfsT`@>MtRKc_m2BLrSUGWn;ZJZkYVHbHH>}9BoarOfVgz|y zkM+!M1bLJZ&%nuoTl~K+D&Um<$NF2`02PYKD8*N%jWiir>t?}QNT$BxfoR!OL2J{c zT{^?(ZOLb;Q^6Ayyudk)THe-Hv!kqzQZq9RLT{JIjH-#$5IZYMP;#~g+9~n)%j&@# z{HZ%dFacNuWCrXFT3=1CF~0nbw11QvVm{;0r#gU24oTAZ`B|E(cWnF-Rc@$k2IaV7;uRk3mm8-OfS7eAF=2#(i8b zJHrtfZ~6yf|3P)E(zyD}5e7@b!lwJ$K99(o5KD4|S`sTyjeg47th!NA`tHS$fym>` z8>2Kb0oqNy_4H)2>hgzZ4)-}_^P{d+mOohk=wq->x?K;~IK!;%8!VERIPtqoNQhfs zuWXdEUhos>>CkXr&&Q~JvCjCH7dh`)3D-3-fofRKbh&66qZ$0ZS25ez37Ly_P&S&F z=t*6)0}andO}WJU`F@o51*8wpOMdITW$-=#!2k#lAV7w$pJjHh1$uw7VnV=YqC5z_S5*X*%nQ%AY=p*^=l-<4K_@WWwyzw4~f6n};az?yxk? zxuS14yu>F-*9(F88s^^;hKMRyzbtKiVH8@LH`Lr6QRyX3>O$t}x&meZz_9{Lh#<=S zbSEln8I#BzQu)K&KNz0hwZhs3$oYM4jQqJF5m>a=hibdWD-$EgIep7M8U4rojqlS> zG-=xqGS#7!MV54-o_&6i{b>@zBT5lhXAp-;>~PJdQq8DdU)WBAK2X}5-!y=`mmMjF zp0{8DPyg63%^OCR}?IH8C69`sZv3AXE>a)v8YBO46yl4McUq2q7qBS zA$1iI3#BmX>IqS|+Q8J8D1o(vTVe^0vpmbJAuf6fC29M5k? zq)3R!d(CpLD&0y06z4P7wDuU?cjceVYXEVMOXbM!c=VsdD}^flDyt?WMfl!z7%m#} zrPRx$!Q8noT&AC~xYD+D^?xkC!$zxIiKxk8Paa2AGcTW^T4yk(xvrwGTgj4u$JK$Q zrqdVF^!h>!`(Ae*az76^+f~M_{Jharpyr?P$d}^=(&%Jg-kX1>RQ087rT@cQ-AA#> zU)=if*Gz5Sjl>*X_@bQZn38rPmCCj1{8Uy~hi9+1ivVIMu{&X;qvTX^xJ&f#(Gtn0 z;$<3-Hz)9Um`ow|X()i+GHTYR*JI1E6eZYbMTAeBc%W+cs|p>1*Bv&I-=;b3s(k$e#>v> z16rC8Wa`H(|19JkNWa$V6^v(8a z^PI@M-E;MmBA-QLqtl4eT{J(&5;?xLAa{VlsGc!&7?B(vmtXZm&NkM_^1w-_JcCqO zZn8Xlq}Wb<t%rF+7}Yb~>sgz9hbVx)@bL_vRtNoK z3T#4@NUL=K<@j`GV{>tixFMU6IcmUnqt0`5=IfUW2ysPzvJ{kBaPY52AV$h~%t zZ>H#h@#G|j=hDAP)Fxe=8KA+d@mA}V(q^)ZCtjyum#HFq^a#WI^;-LmkYo<`6&~TD zmPI^698bu^1zRl`XE`!mCE(+)aytT(B`m@jsRGI}w{!^Xpo<%8d^UQ~*?$oCgVh1WT&k=*|pd9d<9H}Ip*H!vs0 z6gu1apbrQnHq4zF_quM)Y|>a1VO&?tMSboKEs>+h>5PH~7i zp?9%EZ%Gcb2Fl%HrPh*Mp3_yCm=^%(&0Q6+%(L#34Yl0KOFW=3vlM#Zo(h;O>!g!% zGa31?jP#y!LRZdjpQ7fvsV2%JZ$yc$b^X=&!hdlKHfSn;#l$x9d9I6hKbF$}kJ@4C zH7FUbs^K4hRns zP@$T_fxxL?BGJ{Yr8q7%v1s-!T+9!9x*UD6q;y-d6Hg=SNOzZI!!!-F`ldA~WS$|D z@LMuFm)m!Z4#JO-eCnocmB{TJ#;Ha>+NvWJ+!n=y#_iVAy&E}IYnc&O`679-DBK+L z922J7^iAz{Scf``_mo1l>meOecho~<$b?EHO~yQChr9by3cwwFFtP8|ExxvLqv4{N z{Gy{vGQdKdpkOWj6xa7Gn!D=E#bU&=QzHQSiNOob|H8{NiTB;Nq%BonVxM75grez& z_*_DEcU~L7p@x&`v@k$z_L$5CcX(*=-3V1}6$;{K1sO6=mj+LyF%ZHsfs1Q+IkAAT zQX;F~1Ps{MTZ4ZlbLw&_(15jHhEzmr6N z#bvCa6E&B@lu2ynaIKoi(bM(>Ba$xtVl*8yxF87gwA|`B$=g-l94a3?Qt7ub-UZgk zqM|z8@-=<;2!uEIcZCVi$C2n6tIAZl)`{R>PSOPbMHjRhYz2r385wrKheo zw8b$AvsPjfFLPhr>lZFC&$T{6Y&;>ihG#4tvmCG(k^_pT7OK*5J(_xW+=(;xV?@|G zyisP8S7z3V-ZcJyv-=et8(Dn1EXUmmv)+Dg`CWp&&t-`Oz@uWRc~sdU@g+C2Q=kb6 zaXLal7Jm|5${FfaKtQFwH zM+uqJq*WR8I>G-@f$n9XCRjVv#@}58@&l!8DP`jC%K(<`v1bp&SI@lm+VX9YR63@- zW_ll1bs(JSf!Ki6VT;LO@O2D}>7=ICPJR6NIlScj75yAO8Dn|YIUnM|-m7ca6xztK zBwlt>D&}7=>LBM`Deg%Fz(L6Ns8ERa>=?)=Z6ycsQ(Ci#^^sfs|60XX@Dw=~6@=-!#hvE~`O{XN~1kX2mqL1*NMMN_ybA3%J(Buh3JX5A~LC z{BEvRA`8tGC@&PeS~*+MMLHwvp^Y7d{4?E@T&_1E1k^U9AiJZ*`|%LeEsS``hV3|9 z*jIljz766qCHO*nHAB&7`kk?iay`6jBK`-$IzNlIJmk=GKp^tTTQm_6qN@9aVwlUW z&G68IB^tsmbFw(`valR5&IX&C{&LCxegD*DR}@XrJG>rchQuZ&OrXw(>PHNQHY4Z; zi9?N^)QxM*RU6l4$^SQSv62oPAH)^}b0&Mn+IPz%J!ELfB%lM@CP6RsJJ|@EQ+U7l z%08VU=#-jh{^Y9)Rtp8E;8FKGw=d8fY756g{_V;SonZLfa%Cr{3{YNr;do|R&YTb4 z1`TqtuU>>Zz0VWP{W>C4ux@vzn_pbMT-l-2?P*VTafrWa?ImH1JLp6(MSreqD0mM+ z0T&#G!XSXN|6~tun{C#w)3$cK(lnBvu(gM&Vw!#Aod6V5Si0-K(fW{FlJgDNz5;(J zfQv+^W`3VbK#tD6v@(0k6+>6PRSm)N7fL;+Cg+KTEvTEhS40r-(>yBTv1yg~O5EWg z2sioupNkO&s0OS>=7|2WroW1pwvyM$1emyELcsB{Iv@QukgjSmp>R-UH&!{rH+bs-chDY&8iDP*V~ zk1s?2s<`BQNsz^b_G&5HT8Qv%Uq5kO!l{KRs{aYX0Y;PlD{r@F>B&>58`LI(r{_`_ z6WZC9lH<#4-KoBQ5SGUrzDvAkZ{N0Rht=u6sO z`5eK2yA{pOj9*qY0v7UwOiqw%`9;Uj0eRO^*<*@WYxF+ebwZNL3fd&&<7Cv2I?C(72~0J)Q&l^mSIhpN}%>T8^> z6)e?`_wgA>cJ`#&zi#I5D+3UTS2S`gc&^-@%Db#kbInXSS4#3gF_R%l%* zS)%;p3RJx=bfu9B5$t$%I_yS=sT$t~F?MeHLt=^4O;eQ{dMovNU9axwC3f$BQEq?Vz zb1AKHgM$jvRuRa3Je;?EJJ+<&0NZcgGU67libhCcExzWv4W7vZwkt{hiqBd1#!X2; z%x^n_S?x0{vTWjN?_9DIADTu^owN4>Qi_}j)J&Fo zS)*;_MpBDWdE-OpP_^b7va1pR&>AZD%AqH68n-XAIVbEge3CE3IB^;mi9A~yHqcjG zw|>YL!FuIS?QQZ-kHj;2C_u=)@!~E>PjD=aGtsQ_eyl5#*WK4vw;F7Y&i2e9ygq5C zJrP_YuqO(VwdTAvf5TYe5T05CS0cB!?PAW<(%1G4295yK=u!nc_;&6KShJ8sU~Z6l z3M&$)$N;ZQ8H?e+M}}7vtS+2K6Q3Ed?|c-hcw_Ou>#uv#NBvc4{f;`}@O$MCcy5P{4IP$o zc|QEw|6ldXB6pCzNrBsdw?k+tUm;;cohB*gcQ0gDIV4Gmj%S0~=+0rN!2FUja`aY+ zAh)iF{wjvzu2usLfaurL*8gTcyXq%a(1amvZO|lB&?(B3rCc*@@DV*B)gP}dkCO$^ zCIKm1@6vx)QYr(*@#NkZ8IbYDzO=&sr7FfGq)ZnRc?`BXLPlOUx*!}{uS5WpZX?ZG zP446~FwAWZE5xZ}@BwiHAf~M9a%d__!8P{2p~jlT3}KWkX5O-?MwTaqAtNgDdM?pm zS4&N%Ajl<0|AhUysJ528<*8e617QCiTN&dCVZv9Yu-hb?mM zV|j%|U`Dr#4ih5r36F0yrV0FibHub`O|Ut^1#H~h3ahkp2d%b61G_wLagMl_Y1zzY z%Nx&hc+)NaF3+y>2|b+ujyeq&hzC3$EFm^6yB1i>Wlc;cpeN5-3`Gxodu$Pg;Ae7o z-6uGxE49kOmwhOz1L93c`hm={5tN8+Y}}pUfY4Gq>H<)T{GY%l6HN%2YoF!Atv4fn zs@9S`vIXvyLTC`U_DiHH%Pv(+hSuGq_pHJ+BMk! z66ccFpq_<=CC^#v=RbBbvlJ;Ouhz1uO{DT4_n%!|#Wz%ELrEiULDY|(1DMnQ*ZlVl zv{kRN9lYv*`|zF?-(*=}_p-zIN&pM4(KyE#*;9Q1N#XV)pA-!Ddw|&!i7Gl}ABj$a z@=45Aie*y))n-jUHKPSf;jUzS1334&WH^6E1UMi-fbRfTEb*5DgnwCVWt*#6yTyY1 z;N9S{w3!qfH@tRN0ztoTyiiiQ@CKZ?6@x6-pR;0c2l!`f* zCpO~1?K^_0MTaQ6fHv7KGM-vR$bI}CYW+_DcCS+KTKtYSbw|xjW&6uhZ#ugFFY?kG zUsy(JJqytZ9_o~_z-LekZ((&?4I?5L13PiKRA>^^wN(RPyYT1!`5umHevYHX`Y{kK z`5@APqY{yn{49!;WQVpZaq+XqZY|XAYo8fNw~?us0001i0tGMGJk>dcvUkfqNC*R$ zIG{DrsXgbBz$n*!eTMUfe+kq(uc-zESBDDu{){Mf5e}d>oB1tq@vV6dxdPLRU7aE% zT9s^M?U(C2uQ0*Nna}gW^|=}K2QsXCvQu>1TSd31oc|H=xnSbGHTlxLmTD7{;|~@5 zk5GXwxVJD+X(pIGhf)bvS}vq%CS#P(B5`!P6D8)BYdIfG>LKFdmftjg zw!P+dCxKITjfW49YT_Sh3;4p8U(Ye46phzYYj za=O$r2Tsw^#GA8?o>v--onQc{cr*r1e!vAs8a>XoxX-Q%Q#U(sY@kAz@ZsX4X>v-yF1!i7d1^5>~m7IU2XZ z{T4)qEA1p&Qkcx(yx&%cIS)f|pkeEtP&#kj7orFCkuw=XhBu@DC7h0d zRIBT4ezzk=^+yP2ONqj}wx&bQr^ls#FE_t)x=@KT>)lmbip;XQb?b4wQokIgt}w)` zn_L$p*j~vPYyhWXTpq)|TjqqO46MVj_&zwX<}Ctp=?kIWvoWUg)~Us4$K`fELv2(ddm#|pN;is z<(eYBgsRnXQ-qJqTwuiH(uMM6QCSMY>GcPDKOP*QMjvS!a^0K0+(LJgWQ_Ww74GV+ zuJ@NdP{G$O&(=UZyKuY982=pBKV@~4ySauD?!;=~%d|A=ZwCrl53yI%QWbh~`Sy;N zu;A$E67k|XLldB2`DL0b`ZD)gM<}D9Bc<3ZY6Ypa&W&0s>+?H5XAddO!=st5Vg-UI za87)OtrP98Nh5kT$TYDaJI@?J*zPdZYU3$7&6PAtuMD#Aso>Ie)tP54m76gge%DIR zh6c2l&{+C4Lm_&+v)6Ke)+q@9Fx32q&A=qXW8-d=Df82AZQW6zpMM~q7qJ4?ok*hA zh2B9kR!$iQ;fbzAb_)CjZ#TyDf;frDHwT@y&@F=NI#xhuu;KjJ6!Y5%L0TEt2!IQH zbumB6J4A=aRSft54tD%t^Iu+4-W^fSIy2D&%}m}h=Stb^h{c&`7e~Qyjm_tCFyL1a zf&F`{_+X|@bl8H6y3q@i=x94m9mLWuJ)0VdsM%)U!V63eB6T8cptp}sV=ax6@N`&7 z+TU=lQt;WKh>>Z%=BLh@+qB;fAoRmcn#x|}4oG{Wu1^0G!jYn(Z}Kox}m8(mO! zYV-Tt4jRXI?0{}^j-#dTJUX1*6Y=Db7#+iWJx1+4OqlJ>ECpfmEM|2|zvS-z z!QBtw!+eo@NOdOqX^4Hsj~z7e)xD*qe&ciIAqf}O%=KTT)4+*+??C?(-Pjqy9zi1KG=FQ7l5mvL+g-7pqbuPtvPXWCj3i0U_O?)#Uia1UAy#TXq>XYhNIJ zRht0?E2q?CX`!WtIdD?zADQ+g)Hk9+)BOP2G3wg?vE)`MslM6xDBwOzSG~&C-I_@8 zwE$7eqGn$Q9W8kEO`K4x=*!e(y0+*4-Q z=o=4Oi>47X3VAr1+I?ngFn`sx1CwTHQ>Uh~i7C+`wv`XV>5+h4h}Hs4qg-Fp2z7c# zFML>f=YK_4P0lN`5k_v=nLpFcOsDmC64Y{{-_u4d61O0*%BiY}31XWa;D(!DODJh? zwe$+Meg!_itom8!%t+8E$PrCbin!a}2O~kLBv`2-_k2oNSi61R*jUW~{Eaau55ap; zMw+byrw{ZqdkK~JqcEFk)*bKfP(%nMB%j!iQ0bXFx z-G0&^xtjEs-zMOVL9zz($UgpV@h$b!((-~Os7hng{ve%|VyhP%*Z}3%v`wy)NE5hs z&-&eEbqc-FGHR(De4x7vX}sEQcFzmVO=qoN7n{q(xOUMyK7aa*3W7ac)axZ9%L2u; zmyS-B7v?m~mlq9yoq&1jI*_wTfttl=xitk3p1x-5chIkOQ;^x>_*^?DidL5OyOWt6P&bJd2Y`QAN*{5I{bGqla08NTnR#Zvm=&1xx5A|k&R+L;zO z2jKpxUvM(F=v*0s8YkUu3@TcSL+p3zel7Lh(rqZhYsgG(PjRo4+Lwc(-Ae#lE0EzK zNf8mCZIceE7h1_}67=e*T=Kq8ojHqQZ|k@1UW9V>{XDc#&M+*}o{f6Yg+=!DqsX3?$g3apEB9Um(A zpjHH@HjG+3&Je(zGj?gjls1YunBhkjp!*zS9rALJ11`a7Uk%k%%ht2*Vcs33d!fwr z#F$srH^-7O%$kO)^PbZh#OAOEDl4{*%;hYFNNfv=D#}~i&3&FmC&?AvR|yf4dM>n5 z7oxjDHb1|5{Ok=o&XZBB#U>U93&J^eEitpyoHa)k4*j=5UPnJL)K4o#E0df{8$8#i z=-395O{xcysV3?wRJEL=*&rY{zMwAvIojiE`(A&;9@^J!b)nl8nu#GccFR_U|E`7k znSGicyf4};$zJ+CoM>iG!d-pO7R+|J1Jew9j@Ft5M1>wbU-!_+cd^nKYFbs8!Ufx_ ze6`KO0h}b(Dklk%TpvY3>HvWP1PBl#XIX>h$;lwV)^+>W0!dSU@4{rZj{BL`9$4GB z#n-P{2M!yBC`<>LVTT!cj6iV+`hCM*lhVtOEy6Ufk~6%PXQ*k%PfCs*#U@xhsKB;# zcHr6h$@A(bw2+=-vvZRfcNlFV3%fJpm?X{rdrjW%3}ZL}$Wl$D?%G)qW$=`A17Ly5 z#@CY}VaQ9A)>sxz;8+L|i8z06q8{Ctjq-n10c)_%d8E2-*sX3PeyJfEdV-jMaf)L| z=TY{j1&vwo`{)VyjfV)ir5&JZ^A$br7cJf#P?@vQc)NUv3E|J&_xS1bWLb^1dZQoF zGT?b)d&yL{PQAU1J!OfYeiW7uhv4vZPFJZSFc+~p3@BY#e+7o>A*SkD_ zHkw};T)O!G{Z}Fn{o}NJVt|umP_dtHEpZKF0o?b%+WB!R-QJB(I?N+Hn%E!qhB$~_ zu%qf}bo%1Xl8;1Xc5gwZy8B8l&(+IQtXBuXFpB|w16W#xsJtL@YN04}ndEZas`+kA zJX{zP*F?<3i~o;S00q{UbKYB-9m4xon{@9S%x!Q0Tp6j0r zKIw&zXEMUrXjP{%?uUs~_gj1EtJLzG#`RMFUR9W@mC6-JD%0K+R7kvNS{ z^FM*Ae7@7)D&9>uVM|3^dhcx`q?;GfI+1dv1UMc%Vij|9JB}q6b~u6U(cn zJH!o>Svo#SvV+Q{^uM;lUWZ`>2oNAZfmVK($@I1g01AQHixeeO*Q*t>!+N31Ph|Bx zGLu^twPg{Fy;*^cc@aY<@+%2+Ce_#)xgssQDo>BV1w*V- z(wleC=<-KC!5VanMvQ@OR<>*2EM8Ujq{5hxyKGwD1FFW0H&Jl=g$CH%z}mD7Yu(dv z?2E@s;|H(6(5tntnd#Vt4l&E-@91V4nYx=HX@fUWa_B8dBb<_h0YqKp8%M|n@%@39 zyo?$p`VvvxGRO||*!bi>pP6F<(jF~Um4}s~!)&&N$SS|I2TGiB*d|s;XxX|?GC`tM zuQb=MHv!)zuy1@-=uOje%Y8SQj+Vef1>*iy4`I@eqg&t@t9^lwQcCOk@{u z{|9a`=Z^5=1e2s2Z=h$d7CtpH6@G&+VZ9-md8{prEH*vI;M^y%)=gE4TPdt?&rcw4 zhX`ahwlX6J?J?&(xr{{0pmiA5mZPm?f&y8#5PE$*-PMGTvGoM@cQe|9a;L%MNxDGH zPItG>Np-(#^vbvO zYlkPNa&>c(=5}Q?L()|JtMBPnFnoSyD=J*2_@S`Qiju9l=kE+ZTVV+N^Z3Xx(^v)c z5WVqznmd(W+;tmQ62VK!J3k2qIZFsbAhIi~%bFMz5!+U`hjw<1C-76K!Acftr~L@= z_D5Y@Yn)aqA07kx#-{v8f~d&2n&!CuD)^OPgGu^YTVDori{ceo-4X2Yba!~bt<{<8 z>TMp5fJh4&P39#~7<(f<_lx%{Eh<;OYQM1@(<~|uA)4yU3Zl7ja{|%X%TtoMyYRpe z$(L@>daFcLhvldIS>r#cI06I!000000aUy?5J>D?Wl5R~m`n%HTUvVZJ{N*w8(Lh{ z^_#Ko~Xplcg?!2@UUQ z=1B;g&Dm9zy3=xYiF>)`U_;pefc-`yAIuk`d63N fn*wjQQLZMzvf=dzVx*Eu|Ha&qP81{s>dU*pKfkT7 literal 0 HcmV?d00001 diff --git a/leveldb/tx.go b/leveldb/tx.go new file mode 100644 index 00000000..c233777b --- /dev/null +++ b/leveldb/tx.go @@ -0,0 +1,335 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ldb + +import ( + "database/sql" + "github.com/conformal/btcdb" + "github.com/conformal/btcwire" + _ "github.com/mattn/go-sqlite3" +) + +// InsertTx inserts a tx hash and its associated data into the database. +func (db *LevelDb) InsertTx(txsha *btcwire.ShaHash, height int64, txoff int, txlen int, usedbuf []byte) (err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + return db.insertTx(txsha, height, txoff, txlen, usedbuf) +} + +// insertTx inserts a tx hash and its associated data into the database. +// Must be called with db lock held. +func (db *LevelDb) insertTx(txsha *btcwire.ShaHash, height int64, txoff int, txlen int, usedbuf []byte) (err error) { + + tx := &db.txState + if tx.tx == nil { + err = db.startTx() + if err != nil { + return + } + } + blockid := height + 1 + txd := tTxInsertData{txsha: txsha, blockid: blockid, txoff: txoff, txlen: txlen, usedbuf: usedbuf} + + log.Tracef("inserting tx %v for block %v off %v len %v", + txsha, blockid, txoff, txlen) + + rowBytes := txsha.String() + + var op int // which table to insert data into. + if db.UseTempTX { + var tblockid int64 + var ttxoff int + var ttxlen int + txop := db.txop(txFetchLocationByShaStmt) + row := txop.QueryRow(rowBytes) + err = row.Scan(&tblockid, &ttxoff, &ttxlen) + if err != sql.ErrNoRows { + // sha already present + err = btcdb.DuplicateSha + return + } + op = txtmpInsertStmt + } else { + op = txInsertStmt + } + + txop := db.txop(op) + _, err = txop.Exec(rowBytes, blockid, txoff, txlen, usedbuf) + if err != nil { + log.Warnf("failed to insert %v %v %v", txsha, blockid, err) + return + } + if db.UseTempTX { + db.TempTblSz++ + } + + // put in insert list for replay + tx.txInsertList = append(tx.txInsertList, txd) + + return +} + +// ExistsTxSha returns if the given tx sha exists in the database +func (db *LevelDb) ExistsTxSha(txsha *btcwire.ShaHash) (exists bool) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + if _, ok := db.fetchTxCache(txsha); ok { + return true + } + + return db.existsTxSha(txsha) +} + +// existsTxSha returns if the given tx sha exists in the database.o +// Must be called with the db lock held. +func (db *LevelDb) existsTxSha(txsha *btcwire.ShaHash) (exists bool) { + var blockid uint32 + + txop := db.txop(txExistsShaStmt) + row := txop.QueryRow(txsha.String()) + err := row.Scan(&blockid) + + if err == sql.ErrNoRows { + txop = db.txop(txtmpExistsShaStmt) + row = txop.QueryRow(txsha.String()) + err := row.Scan(&blockid) + + if err == sql.ErrNoRows { + return false + } + if err != nil { + log.Warnf("txTmpExistsTxSha: fail %v", err) + return false + } + log.Warnf("txtmpExistsTxSha: success") + return true + } + + if err != nil { + // ignore real errors? + log.Warnf("existsTxSha: fail %v", err) + return false + } + + return true +} + +// FetchLocationBySha looks up the Tx sha information by name. +func (db *LevelDb) FetchLocationBySha(txsha *btcwire.ShaHash) (blockidx int64, txoff int, txlen int, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + return db.fetchLocationBySha(txsha) +} + +// fetchLocationBySha look up the Tx sha information by name. +// Must be called with db lock held. +func (db *LevelDb) fetchLocationBySha(txsha *btcwire.ShaHash) (height int64, txoff int, txlen int, err error) { + var row *sql.Row + var blockid int64 + var ttxoff int + var ttxlen int + + rowBytes := txsha.String() + txop := db.txop(txFetchLocationByShaStmt) + row = txop.QueryRow(rowBytes) + + err = row.Scan(&blockid, &ttxoff, &ttxlen) + if err == sql.ErrNoRows { + txop = db.txop(txtmpFetchLocationByShaStmt) + row = txop.QueryRow(rowBytes) + + err = row.Scan(&blockid, &ttxoff, &ttxlen) + if err == sql.ErrNoRows { + err = btcdb.TxShaMissing + return + } + if err != nil { + log.Warnf("txtmp FetchLocationBySha: fail %v", + err) + return + } + } + if err != nil { + log.Warnf("FetchLocationBySha: fail %v", err) + return + } + height = blockid - 1 + txoff = ttxoff + txlen = ttxlen + return +} + +// fetchLocationUsedBySha look up the Tx sha information by name. +// Must be called with db lock held. +func (db *LevelDb) fetchLocationUsedBySha(txsha *btcwire.ShaHash) (rheight int64, rtxoff int, rtxlen int, rspentbuf []byte, err error) { + var row *sql.Row + var blockid int64 + var txoff int + var txlen int + var txspent []byte + + rowBytes := txsha.String() + txop := db.txop(txFetchLocUsedByShaStmt) + row = txop.QueryRow(rowBytes) + + err = row.Scan(&blockid, &txoff, &txlen, &txspent) + if err == sql.ErrNoRows { + txop = db.txop(txtmpFetchLocUsedByShaStmt) + row = txop.QueryRow(rowBytes) + + err = row.Scan(&blockid, &txoff, &txlen, &txspent) + if err == sql.ErrNoRows { + err = btcdb.TxShaMissing + return + } + if err != nil { + log.Warnf("txtmp FetchLocationBySha: fail %v", + err) + return + } + } + if err != nil { + log.Warnf("FetchLocationBySha: fail %v", err) + return + } + height := blockid - 1 + return height, txoff, txlen, txspent, nil +} + +// FetchTxUsedBySha returns the used/spent buffer for a given transaction. +func (db *LevelDb) FetchTxUsedBySha(txsha *btcwire.ShaHash) (spentbuf []byte, err error) { + var row *sql.Row + db.dbLock.Lock() + defer db.dbLock.Unlock() + + rowBytes := txsha.String() + txop := db.txop(txFetchUsedByShaStmt) + row = txop.QueryRow(rowBytes) + + var databytes []byte + err = row.Scan(&databytes) + if err == sql.ErrNoRows { + txop := db.txop(txtmpFetchUsedByShaStmt) + row = txop.QueryRow(rowBytes) + + err = row.Scan(&databytes) + if err == sql.ErrNoRows { + err = btcdb.TxShaMissing + return + } + if err != nil { + log.Warnf("txtmp FetchLocationBySha: fail %v", + err) + return + } + } + + if err != nil { + log.Warnf("FetchUsedBySha: fail %v", err) + return + } + spentbuf = databytes + return +} + +var vaccumDbNextMigrate bool + +// migrateTmpTable functions to perform internal db optimization when +// performing large numbers of database inserts. When in Fast operation +// mode, it inserts into txtmp, then when that table reaches a certain +// size limit it moves all tx in the txtmp table into the primary tx +// table and recomputes the index on the primary tx table. +func (db *LevelDb) migrateTmpTable() error { + db.endTx(true) + db.startTx() // ??? + + db.UseTempTX = false + db.TempTblSz = 0 + + var doVacuum bool + var nsteps int + if vaccumDbNextMigrate { + nsteps = 6 + vaccumDbNextMigrate = false + doVacuum = true + } else { + nsteps = 5 + vaccumDbNextMigrate = true + } + + log.Infof("db compaction Stage 1/%v: Preparing", nsteps) + txop := db.txop(txMigratePrep) + _, err := txop.Exec() + if err != nil { + log.Warnf("Failed to prepare migrate - %v", err) + return err + } + + log.Infof("db compaction Stage 2/%v: Copying", nsteps) + txop = db.txop(txMigrateCopy) + _, err = txop.Exec() + if err != nil { + log.Warnf("Migrate read failed - %v", err) + return err + } + + log.Tracef("db compaction Stage 2a/%v: Enable db vacuum", nsteps) + txop = db.txop(txPragmaVacuumOn) + _, err = txop.Exec() + if err != nil { + log.Warnf("Migrate error trying to enable vacuum on "+ + "temporary transaction table - %v", err) + return err + } + + log.Infof("db compaction Stage 3/%v: Clearing old data", nsteps) + txop = db.txop(txMigrateClear) + _, err = txop.Exec() + if err != nil { + log.Warnf("Migrate error trying to clear temporary "+ + "transaction table - %v", err) + return err + } + + log.Tracef("db compaction Stage 3a/%v: Disable db vacuum", nsteps) + txop = db.txop(txPragmaVacuumOff) + _, err = txop.Exec() + if err != nil { + log.Warnf("Migrate error trying to disable vacuum on "+ + "temporary transaction table - %v", err) + return err + } + + log.Infof("db compaction Stage 4/%v: Rebuilding index", nsteps) + txop = db.txop(txMigrateFinish) + _, err = txop.Exec() + if err != nil { + log.Warnf("Migrate error trying to clear temporary "+ + "transaction table - %v", err) + return err + } + + log.Infof("db compaction Stage 5/%v: Finalizing transaction", nsteps) + db.endTx(true) // ??? + + if doVacuum { + log.Infof("db compaction Stage 6/%v: Optimizing database", nsteps) + txop = db.txop(txVacuum) + _, err = txop.Exec() + if err != nil { + log.Warnf("migrate error trying to clear txtmp tbl %v", err) + return err + } + } + + log.Infof("db compaction: Complete") + + // TODO(drahn) - determine if this should be turned back on or not + db.UseTempTX = true + + return nil +} From d8c3213d999f70d99428ecd40f45d516205f06e6 Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Sat, 3 Aug 2013 11:09:50 -0400 Subject: [PATCH 036/163] more --- leveldb/block.go | 293 ------------ leveldb/dbcache.go | 375 --------------- leveldb/doc.go | 15 - leveldb/insertremove_test.go | 209 -------- leveldb/internal_test.go | 48 -- leveldb/leveldb.go | 798 ------------------------------- leveldb/operational_test.go | 420 ---------------- leveldb/testdata/blocks1-256.bz2 | Bin 37555 -> 0 bytes leveldb/tx.go | 335 ------------- 9 files changed, 2493 deletions(-) delete mode 100644 leveldb/block.go delete mode 100644 leveldb/dbcache.go delete mode 100644 leveldb/doc.go delete mode 100644 leveldb/insertremove_test.go delete mode 100644 leveldb/internal_test.go delete mode 100644 leveldb/leveldb.go delete mode 100644 leveldb/operational_test.go delete mode 100644 leveldb/testdata/blocks1-256.bz2 delete mode 100644 leveldb/tx.go diff --git a/leveldb/block.go b/leveldb/block.go deleted file mode 100644 index 145a89a7..00000000 --- a/leveldb/block.go +++ /dev/null @@ -1,293 +0,0 @@ -// Copyright (c) 2013 Conformal Systems LLC. -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package ldb - -import ( - "bytes" - "fmt" - "encoding/binary" - "errors" - "github.com/conformal/btcdb" - "github.com/conformal/btcwire" -) - -// InsertBlockData stores a block hash and its associated data block with a -// previous sha of `prevSha' and a version of `pver'. -func (db *LevelDb) InsertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHash, pver uint32, buf []byte) (blockid int64, err error) { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - return db.insertBlockData(sha, prevSha, pver, buf) -} - -func (db *LevelDb) getBlkLoc(sha *btcwire.ShaHash) (int64, int, error) { - var blkHeight int64 - var blkFile int - - key := sha.Bytes() - - data, err := db.bShaDb.Get(key, db.ro) - - if err != nil { - return 0, 0, err - } - - // deserialize - dr := bytes.NewBuffer(data) - err = binary.Read(dr, binary.LittleEndian, &blkHeight) - if err != nil { - err = errors.New("Db Corrupt") - return 0, 0, err - } - err = binary.Read(dr, binary.LittleEndian, &blkFile) - if err != nil { - err = errors.New("Db Corrupt") - return 0, 0, err - } - return blkHeight, blkFile, nil -} - -func (db *LevelDb) getBlkByHeight(blkHeight int64, blkFile int) (rsha *btcwire.ShaHash, rbuf []byte, err error) { - var blkVal []byte - - key := fmt.Sprintf("%d",blkHeight) - - blkVal, err = db.bBlkDb[blkFile].Get([]byte(key), db.ro) - if err != nil { - return // exists ??? - } - - var sha btcwire.ShaHash - - sha.SetBytes(blkVal[0:31]) - - return &sha, blkVal[32:], nil -} - -func (db *LevelDb) getBlk(sha *btcwire.ShaHash) (rblkHeight int64, rblkFile int, rbuf []byte, err error) { - var blkHeight int64 - var blkFile int - - blkHeight, blkFile, err = db.getBlkLoc(sha) - if err != nil { - return - } - - var buf []byte - - _, buf, err = db.getBlkByHeight(blkHeight, blkFile) - if err != nil { - return - } - return blkHeight, blkFile, buf, nil -} - -func (db *LevelDb) setBlk(sha *btcwire.ShaHash, blkHeight int64, blkFile int, buf []byte) (error) { - - // serialize - var lw bytes.Buffer - err := binary.Write(&lw, binary.LittleEndian, &blkHeight) - if err != nil { - err = errors.New("Write fail") - return err - } - err = binary.Write(&lw, binary.LittleEndian, &blkFile) - if err != nil { - err = errors.New("Write fail") - return err - } - key := sha.Bytes() - - err = db.bShaDb.Put(key, lw.Bytes(), db.wo) - - if err != nil { - return err - } - - key = []byte(fmt.Sprintf("%d",blkHeight)) - - shaB := sha.Bytes() - blkVal := make([]byte, len(shaB) + len(buf)) - copy (blkVal[0:], shaB) - copy (blkVal[len(shaB):], buf) - err = db.bBlkDb[blkFile].Put(key, blkVal, db.wo) - - return nil -} - -// insertSha stores a block hash and its associated data block with a -// previous sha of `prevSha' and a version of `pver'. -// insertSha shall be called with db lock held -func (db *LevelDb) insertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHash, pver uint32, buf []byte) (blockid int64, err error) { - tx := &db.txState - if tx.tx == nil { - err = db.startTx() - if err != nil { - return - } - } - - oBlkHeight, _, err:= db.getBlkLoc(prevSha) - - if err != nil { - // check current block count - // if count != 0 { - // err = btcdb.PrevShaMissing - // return - // } - oBlkHeight = -1 - } - - // TODO(drahn) check curfile filesize, increment curfile if this puts it over - curFile := 0 - blkHeight := oBlkHeight - 1 - - err = db.setBlk(sha, blkHeight, curFile, buf) - - if err != nil { - return - } - - // update the last block cache - db.lastBlkShaCached = true - db.lastBlkSha = *sha - db.lastBlkIdx = blkHeight - - return blkHeight, nil -} - -// fetchSha returns the datablock and pver for the given ShaHash. -func (db *LevelDb) fetchSha(sha *btcwire.ShaHash) (rbuf []byte, rpver uint32, - rblkHeight int64, err error) { - var blkHeight int64 - var buf []byte - - blkHeight, _, buf, err = db.getBlk(sha) - if err != nil { - return - } - - fakepver := uint32(1) - - return buf, fakepver, blkHeight, nil -} - -// ExistsSha looks up the given block hash -// returns true if it is present in the database. -func (db *LevelDb) ExistsSha(sha *btcwire.ShaHash) (exists bool) { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - _, exists = db.fetchBlockCache(sha) - if exists { - return - } - - // not in cache, try database - exists = db.blkExistsSha(sha) - return -} - -// blkExistsSha looks up the given block hash -// returns true if it is present in the database. -// CALLED WITH LOCK HELD -func (db *LevelDb) blkExistsSha(sha *btcwire.ShaHash) bool { - var pver uint32 - - oBlkHeight, _, err:= db.getBlkLoc(sha) - - if err != nil { - /* - should this warn if the failure is something besides does not exist ? - log.Warnf("blkExistsSha: fail %v", err) - */ - return false - } - return true -} - -// FetchBlockShaByHeight returns a block hash based on its height in the -// block chain. -func (db *LevelDb) FetchBlockShaByHeight(height int64) (sha *btcwire.ShaHash, err error) { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - return db.fetchBlockShaByHeight(height) -} - -// fetchBlockShaByHeight returns a block hash based on its height in the -// block chain. -func (db *LevelDb) fetchBlockShaByHeight(height int64) (sha *btcwire.ShaHash, err error) { - - // TODO(drahn) figure out which file blkHeight is located - blkFile := 0 - var buf []byte - - _, buf, err = db.getBlkByHeight(height, blkFile) - - var shaval btcwire.ShaHash - shaval.SetBytes(buf[0:31]) - return &shaval, nil -} - -// FetchHeightRange looks up a range of blocks by the start and ending -// heights. Fetch is inclusive of the start height and exclusive of the -// ending height. To fetch all hashes from the start height until no -// more are present, use the special id `AllShas'. -func (db *LevelDb) FetchHeightRange(startHeight, endHeight int64) (rshalist []btcwire.ShaHash, err error) { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - var endidx int64 - if endHeight == btcdb.AllShas { - endidx = startHeight + 500 - } else { - endidx = endHeight - } - - var shalist []btcwire.ShaHash - for height := startHeight; height < endidx; height++ { - // TODO(drahn) fix blkFile from height - blkFile := 0 - - key := fmt.Sprintf("%d", height) - blkVal, lerr := db.bBlkDb[blkFile].Get([]byte(key), db.ro) - if lerr != nil { - break - } - - var sha btcwire.ShaHash - sha.SetBytes(blkVal[0:31]) - shalist = append(shalist, sha) - } - - if err == nil { - return - } - log.Tracef("FetchIdxRange idx %v %v returned %v shas err %v", startHeight, endHeight, len(shalist), err) - - return shalist, nil -} - -// NewestSha returns the hash and block height of the most recent (end) block of -// the block chain. It will return the zero hash, -1 for the block height, and -// no error (nil) if there are not any blocks in the database yet. -func (db *LevelDb) NewestSha() (rsha *btcwire.ShaHash, rblkid int64, err error) { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - if db.lastBlkIdx == -1 { - err = errors.New("Empty Database") - return - } - sha := db.lastBlkSha - - return &sha, db.lastBlkIdx, nil -} - -func (db *LevelDb) NewIterateBlocks() (rbogus btcdb.BlockIterator, err error) { - err = errors.New("Not implemented") - return -} diff --git a/leveldb/dbcache.go b/leveldb/dbcache.go deleted file mode 100644 index 6c3bfafe..00000000 --- a/leveldb/dbcache.go +++ /dev/null @@ -1,375 +0,0 @@ -// Copyright (c) 2013 Conformal Systems LLC. -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package ldb - -import ( - "bytes" - "container/list" - "github.com/conformal/btcdb" - "github.com/conformal/btcutil" - "github.com/conformal/btcwire" - "sync" -) - -type txCache struct { - maxcount int - fifo list.List - // NOTE: the key is specifically ShaHash, not *ShaHash - txMap map[btcwire.ShaHash]*txCacheObj - cacheLock sync.RWMutex -} - -type txCacheObj struct { - next *txCacheObj - sha btcwire.ShaHash - blksha btcwire.ShaHash - pver uint32 - tx *btcwire.MsgTx - height int64 - spent []byte - txbuf []byte -} - -type blockCache struct { - maxcount int - fifo list.List - blockMap map[btcwire.ShaHash]*blockCacheObj - blockHeightMap map[int64]*blockCacheObj - cacheLock sync.RWMutex -} - -type blockCacheObj struct { - next *blockCacheObj - sha btcwire.ShaHash - blk *btcutil.Block -} - -// FetchBlockBySha - return a btcutil Block, object may be a cached. -func (db *LevelDb) FetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) { - db.dbLock.Lock() - defer db.dbLock.Unlock() - return db.fetchBlockBySha(sha) -} - -// fetchBlockBySha - return a btcutil Block, object may be a cached. -// Must be called with db lock held. -func (db *LevelDb) fetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) { - - blkcache, ok := db.fetchBlockCache(sha) - if ok { - return blkcache.blk, nil - } - - buf, pver, height, err := db.fetchSha(sha) - if err != nil { - return - } - - blk, err = btcutil.NewBlockFromBytes(buf, pver) - if err != nil { - return - } - blk.SetHeight(height) - db.insertBlockCache(sha, blk) - - return -} - -// fetchBlockCache check if a block is in the block cache, if so return it. -func (db *LevelDb) fetchBlockCache(sha *btcwire.ShaHash) (*blockCacheObj, bool) { - - db.blockCache.cacheLock.RLock() - defer db.blockCache.cacheLock.RUnlock() - - blkobj, ok := db.blockCache.blockMap[*sha] - if !ok { // could this just return the map deref? - return nil, false - } - return blkobj, true -} - -// fetchBlockHeightCache check if a block is in the block cache, if so return it. -func (db *LevelDb) fetchBlockHeightCache(height int64) (*blockCacheObj, bool) { - - db.blockCache.cacheLock.RLock() - defer db.blockCache.cacheLock.RUnlock() - - blkobj, ok := db.blockCache.blockHeightMap[height] - if !ok { // could this just return the map deref? - return nil, false - } - return blkobj, true -} - -// insertBlockCache insert the given sha/block into the cache map. -// If the block cache is determined to be full, it will release -// an old entry in FIFO order. -func (db *LevelDb) insertBlockCache(sha *btcwire.ShaHash, blk *btcutil.Block) { - bc := &db.blockCache - - bc.cacheLock.Lock() - defer bc.cacheLock.Unlock() - - blkObj := blockCacheObj{sha: *sha, blk: blk} - bc.fifo.PushBack(&blkObj) - - if bc.fifo.Len() > bc.maxcount { - listobj := bc.fifo.Front() - bc.fifo.Remove(listobj) - tailObj, ok := listobj.Value.(*blockCacheObj) - if ok { - delete(bc.blockMap, tailObj.sha) - delete(bc.blockHeightMap, tailObj.blk.Height()) - } else { - panic("invalid type pushed on blockCache list") - } - } - - bc.blockHeightMap[blk.Height()] = &blkObj - bc.blockMap[blkObj.sha] = &blkObj -} - -// FetchTxByShaList given a array of ShaHash, look up the transactions -// and return them in a TxListReply array. -func (db *LevelDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - var replies []*btcdb.TxListReply - for _, txsha := range txShaList { - tx, _, _, _, height, txspent, err := db.fetchTxDataBySha(txsha) - btxspent := []bool{} - if err == nil { - btxspent = make([]bool, len(tx.TxOut), len(tx.TxOut)) - for idx := range tx.TxOut { - byteidx := idx / 8 - byteoff := uint(idx % 8) - btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0 - } - } - txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, Height: height, TxSpent: btxspent, Err: err} - replies = append(replies, &txlre) - } - return replies -} - -// fetchTxDataBySha returns several pieces of data regarding the given sha. -func (db *LevelDb) fetchTxDataBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rtxbuf []byte, rpver uint32, rblksha *btcwire.ShaHash, rheight int64, rtxspent []byte, err error) { - - var pver uint32 - var blksha *btcwire.ShaHash - var height int64 - var txspent []byte - var toff int - var tlen int - var blk *btcutil.Block - var blkbuf []byte - - // Check Tx cache - if txc, ok := db.fetchTxCache(txsha); ok { - if txc.spent != nil { - return txc.tx, txc.txbuf, txc.pver, &txc.blksha, txc.height, txc.spent, nil - } - } - - // If not cached load it - height, toff, tlen, txspent, err = db.fetchLocationUsedBySha(txsha) - if err != nil { - return - } - - blksha, err = db.fetchBlockShaByHeight(height) - if err != nil { - log.Warnf("block idx lookup %v to %v", height, err) - return - } - log.Tracef("transaction %v is at block %v %v tx %v", - txsha, blksha, height, toff) - - blk, err = db.fetchBlockBySha(blksha) - if err != nil { - log.Warnf("unable to fetch block %v %v ", - height, &blksha) - return - } - - blkbuf, pver, err = blk.Bytes() - if err != nil { - log.Warnf("unable to decode block %v %v", height, &blksha) - return - } - - txbuf := make([]byte, tlen) - copy(txbuf[:], blkbuf[toff:toff+tlen]) - rbuf := bytes.NewBuffer(txbuf) - - var tx btcwire.MsgTx - err = tx.BtcDecode(rbuf, pver) - if err != nil { - log.Warnf("unable to decode tx block %v %v txoff %v txlen %v", - height, &blksha, toff, tlen) - return - } - - // Shove data into TxCache - // XXX - - var txc txCacheObj - txc.sha = *txsha - txc.tx = &tx - txc.txbuf = txbuf - txc.pver = pver - txc.height = height - txc.spent = txspent - txc.blksha = *blksha - db.insertTxCache(&txc) - - return &tx, txbuf, pver, blksha, height, txspent, nil -} - -// FetchTxAllBySha returns several pieces of data regarding the given sha. -func (db *LevelDb) FetchTxAllBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rtxbuf []byte, rpver uint32, rblksha *btcwire.ShaHash, err error) { - var pver uint32 - var blksha *btcwire.ShaHash - var height int64 - var toff int - var tlen int - var blk *btcutil.Block - var blkbuf []byte - - // Check Tx cache - if txc, ok := db.fetchTxCache(txsha); ok { - return txc.tx, txc.txbuf, txc.pver, &txc.blksha, nil - } - - // If not cached load it - height, toff, tlen, err = db.FetchLocationBySha(txsha) - if err != nil { - return - } - - blksha, err = db.FetchBlockShaByHeight(height) - if err != nil { - log.Warnf("block idx lookup %v to %v", height, err) - return - } - log.Tracef("transaction %v is at block %v %v tx %v", - txsha, blksha, height, toff) - - blk, err = db.FetchBlockBySha(blksha) - if err != nil { - log.Warnf("unable to fetch block %v %v ", - height, &blksha) - return - } - - blkbuf, pver, err = blk.Bytes() - if err != nil { - log.Warnf("unable to decode block %v %v", height, &blksha) - return - } - - txbuf := make([]byte, tlen) - copy(txbuf[:], blkbuf[toff:toff+tlen]) - rbuf := bytes.NewBuffer(txbuf) - - var tx btcwire.MsgTx - err = tx.BtcDecode(rbuf, pver) - if err != nil { - log.Warnf("unable to decode tx block %v %v txoff %v txlen %v", - height, &blksha, toff, tlen) - return - } - - // Shove data into TxCache - // XXX - - var txc txCacheObj - txc.sha = *txsha - txc.tx = &tx - txc.txbuf = txbuf - txc.pver = pver - txc.height = height - txc.blksha = *blksha - db.insertTxCache(&txc) - - return &tx, txbuf, pver, blksha, nil -} - -// FetchTxBySha returns some data for the given Tx Sha. -func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rpver uint32, blksha *btcwire.ShaHash, err error) { - rtx, _, rpver, blksha, err = db.FetchTxAllBySha(txsha) - return -} - -// FetchTxBufBySha return the bytestream data and associated protocol version. -// for the given Tx Sha -func (db *LevelDb) FetchTxBufBySha(txsha *btcwire.ShaHash) (txbuf []byte, rpver uint32, err error) { - _, txbuf, rpver, _, err = db.FetchTxAllBySha(txsha) - return -} - -// fetchTxCache look up the given transaction in the Tx cache. -func (db *LevelDb) fetchTxCache(sha *btcwire.ShaHash) (*txCacheObj, bool) { - tc := &db.txCache - - tc.cacheLock.RLock() - defer tc.cacheLock.RUnlock() - - txObj, ok := tc.txMap[*sha] - if !ok { // could this just return the map deref? - return nil, false - } - return txObj, true -} - -// insertTxCache, insert the given txobj into the cache. -// if the tx cache is determined to be full, it will release -// an old entry in FIFO order. -func (db *LevelDb) insertTxCache(txObj *txCacheObj) { - tc := &db.txCache - - tc.cacheLock.Lock() - defer tc.cacheLock.Unlock() - - tc.fifo.PushBack(txObj) - - if tc.fifo.Len() >= tc.maxcount { - listobj := tc.fifo.Front() - tc.fifo.Remove(listobj) - tailObj, ok := listobj.Value.(*txCacheObj) - if ok { - delete(tc.txMap, tailObj.sha) - } else { - panic("invalid type pushed on tx list") - } - - } - - tc.txMap[txObj.sha] = txObj -} - -// InvalidateTxCache clear/release all cached transactions. -func (db *LevelDb) InvalidateTxCache() { - tc := &db.txCache - tc.cacheLock.Lock() - defer tc.cacheLock.Unlock() - tc.txMap = map[btcwire.ShaHash]*txCacheObj{} - tc.fifo = list.List{} -} - -// InvalidateTxCache clear/release all cached blocks. -func (db *LevelDb) InvalidateBlockCache() { - bc := &db.blockCache - bc.cacheLock.Lock() - defer bc.cacheLock.Unlock() - bc.blockMap = map[btcwire.ShaHash]*blockCacheObj{} - bc.blockHeightMap = map[int64]*blockCacheObj{} - bc.fifo = list.List{} -} - -// InvalidateCache clear/release all cached blocks and transactions. -func (db *LevelDb) InvalidateCache() { - db.InvalidateTxCache() - db.InvalidateBlockCache() -} diff --git a/leveldb/doc.go b/leveldb/doc.go deleted file mode 100644 index 65906a93..00000000 --- a/leveldb/doc.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) 2013 Conformal Systems LLC. -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -/* -Package sqlite3 implements a sqlite3 instance of btcdb. - -sqlite provides a zero setup, single file database. It requires cgo -and the presence of the sqlite library and headers, but nothing else. -The performance is generally high although it goes down with database -size. - -Many of the block or tx specific functions for btcdb are in this subpackage. -*/ -package ldb diff --git a/leveldb/insertremove_test.go b/leveldb/insertremove_test.go deleted file mode 100644 index 27fa248d..00000000 --- a/leveldb/insertremove_test.go +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright (c) 2013 Conformal Systems LLC. -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package ldb_test - -import ( - "github.com/conformal/btcdb" - "github.com/conformal/btcdb/sqlite3" - "github.com/conformal/btcutil" - "github.com/conformal/btcwire" - "os" - "path/filepath" - "testing" -) - -var tstBlocks []*btcutil.Block - -func loadblocks(t *testing.T) []*btcutil.Block { - if len(tstBlocks) != 0 { - return tstBlocks - } - - testdatafile := filepath.Join("testdata", "blocks1-256.bz2") - blocks, err := loadBlocks(t, testdatafile) - if err != nil { - t.Errorf("Unable to load blocks from test data: %v", err) - return nil - } - tstBlocks = blocks - return blocks -} - -func TestUnspentInsert(t *testing.T) { - testUnspentInsert(t, dbTmDefault) - testUnspentInsert(t, dbTmNormal) - testUnspentInsert(t, dbTmFast) -} - -// insert every block in the test chain -// after each insert, fetch all the tx affected by the latest -// block and verify that the the tx is spent/unspent -// new tx should be fully unspent, referenced tx should have -// the associated txout set to spent. -func testUnspentInsert(t *testing.T, mode int) { - // Ignore db remove errors since it means we didn't have an old one. - dbname := "tstdbuspnt1" - _ = os.Remove(dbname) - db, err := btcdb.CreateDB("sqlite", dbname) - if err != nil { - t.Errorf("Failed to open test database %v", err) - return - } - defer os.Remove(dbname) - defer db.Close() - - switch mode { - case dbTmDefault: // default - // no setup - case dbTmNormal: // explicit normal - db.SetDBInsertMode(btcdb.InsertNormal) - case dbTmFast: // fast mode - db.SetDBInsertMode(btcdb.InsertFast) - if sqldb, ok := db.(*sqlite3.LevelDb); ok { - sqldb.TempTblMax = 100 - } else { - t.Errorf("not right type") - } - case dbTmNoVerify: // validated block - t.Errorf("UnspentInsert test is not valid in NoVerify mode") - } - - // Since we are dealing with small dataset, reduce cache size - sqlite3.SetBlockCacheSize(db, 2) - sqlite3.SetTxCacheSize(db, 3) - - blocks := loadblocks(t) -endtest: - for height := int64(0); height < int64(len(blocks)); height++ { - - block := blocks[height] - // look up inputs to this x - mblock := block.MsgBlock() - var txneededList []*btcwire.ShaHash - var txlookupList []*btcwire.ShaHash - var txOutList []*btcwire.ShaHash - var txInList []*btcwire.OutPoint - for _, tx := range mblock.Transactions { - for _, txin := range tx.TxIn { - if txin.PreviousOutpoint.Index == uint32(4294967295) { - continue - } - origintxsha := &txin.PreviousOutpoint.Hash - - txInList = append(txInList, &txin.PreviousOutpoint) - txneededList = append(txneededList, origintxsha) - txlookupList = append(txlookupList, origintxsha) - - if !db.ExistsTxSha(origintxsha) { - t.Errorf("referenced tx not found %v ", origintxsha) - } - - } - txshaname, _ := tx.TxSha(block.ProtocolVersion()) - txlookupList = append(txlookupList, &txshaname) - txOutList = append(txOutList, &txshaname) - } - - txneededmap := map[btcwire.ShaHash]*btcdb.TxListReply{} - txlist := db.FetchTxByShaList(txneededList) - for _, txe := range txlist { - if txe.Err != nil { - t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) - break endtest - } - txneededmap[*txe.Sha] = txe - } - for _, spend := range txInList { - itxe := txneededmap[spend.Hash] - if itxe.TxSpent[spend.Index] == true { - t.Errorf("txin %v:%v is already spent", spend.Hash, spend.Index) - } - } - - newheight, err := db.InsertBlock(block) - if err != nil { - t.Errorf("failed to insert block %v err %v", height, err) - break endtest - } - if newheight != height { - t.Errorf("height mismatch expect %v returned %v", height, newheight) - break endtest - } - - txlookupmap := map[btcwire.ShaHash]*btcdb.TxListReply{} - txlist = db.FetchTxByShaList(txlookupList) - for _, txe := range txlist { - if txe.Err != nil { - t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) - break endtest - } - txlookupmap[*txe.Sha] = txe - } - for _, spend := range txInList { - itxe := txlookupmap[spend.Hash] - if itxe.TxSpent[spend.Index] == false { - t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent) - } - } - for _, txo := range txOutList { - itxe := txlookupmap[*txo] - for i, spent := range itxe.TxSpent { - if spent == true { - t.Errorf("freshly inserted tx %v already spent %v", txo, i) - } - } - - } - if len(txInList) == 0 { - continue - } - dropblock := blocks[height-1] - dropsha, _ := dropblock.Sha() - - err = db.DropAfterBlockBySha(dropsha) - if err != nil { - t.Errorf("failed to drop block %v err %v", height, err) - break endtest - } - - txlookupmap = map[btcwire.ShaHash]*btcdb.TxListReply{} - txlist = db.FetchTxByShaList(txlookupList) - for _, txe := range txlist { - if txe.Err != nil { - if _, ok := txneededmap[*txe.Sha]; ok { - t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) - break endtest - } - } - txlookupmap[*txe.Sha] = txe - } - for _, spend := range txInList { - itxe := txlookupmap[spend.Hash] - if itxe.TxSpent[spend.Index] == true { - t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent) - } - } - newheight, err = db.InsertBlock(block) - if err != nil { - t.Errorf("failed to insert block %v err %v", height, err) - break endtest - } - txlookupmap = map[btcwire.ShaHash]*btcdb.TxListReply{} - txlist = db.FetchTxByShaList(txlookupList) - for _, txe := range txlist { - if txe.Err != nil { - t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) - break endtest - } - txlookupmap[*txe.Sha] = txe - } - for _, spend := range txInList { - itxe := txlookupmap[spend.Hash] - if itxe.TxSpent[spend.Index] == false { - t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent) - } - } - } -} diff --git a/leveldb/internal_test.go b/leveldb/internal_test.go deleted file mode 100644 index 08706154..00000000 --- a/leveldb/internal_test.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) 2013 Conformal Systems LLC. -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package ldb - -import ( - "fmt" - "github.com/conformal/btcdb" - "github.com/conformal/btcwire" -) - -// FetchSha returns the datablock and pver for the given ShaHash. -// This is a testing only interface. -func FetchSha(db btcdb.Db, sha *btcwire.ShaHash) (buf []byte, pver uint32, - blkid int64, err error) { - sqldb, ok := db.(*LevelDb) - if !ok { - err = fmt.Errorf("Invalid data type") - return - } - buf, pver, blkid, err = sqldb.fetchSha(*sha) - return -} - -// SetBlockCacheSize configures the maximum number of blocks in the cache to -// be the given size should be made before any fetching. -// This is a testing only interface. -func SetBlockCacheSize(db btcdb.Db, newsize int) { - sqldb, ok := db.(*LevelDb) - if !ok { - return - } - bc := &sqldb.blockCache - bc.maxcount = newsize -} - -// SetTxCacheSize configures the maximum number of tx in the cache to -// be the given size should be made before any fetching. -// This is a testing only interface. -func SetTxCacheSize(db btcdb.Db, newsize int) { - sqldb, ok := db.(*LevelDb) - if !ok { - return - } - tc := &sqldb.txCache - tc.maxcount = newsize -} diff --git a/leveldb/leveldb.go b/leveldb/leveldb.go deleted file mode 100644 index d44d0e6e..00000000 --- a/leveldb/leveldb.go +++ /dev/null @@ -1,798 +0,0 @@ -// Copyright (c) 2013 Conformal Systems LLC. -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package ldb - -import ( - "database/sql" - "fmt" - "github.com/conformal/btcdb" - "github.com/conformal/btcutil" - "github.com/conformal/btcwire" - "github.com/conformal/seelog" - "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/opt" - _ "github.com/mattn/go-sqlite3" - "os" - "sync" -) - -const ( - dbVersion int = 2 - dbMaxTransCnt = 20000 - dbMaxTransMem = 64 * 1024 * 1024 // 64 MB -) - -var log seelog.LoggerInterface = seelog.Disabled - -type tBlockInsertData struct { - sha btcwire.ShaHash - pver uint32 - buf []byte -} -type tTxInsertData struct { - txsha *btcwire.ShaHash - blockid int64 - txoff int - txlen int - usedbuf []byte -} - -type txState struct { - tx *sql.Tx - writeCount int - txDataSz int - txInsertList []interface{} -} -type LevelDb struct { - // to be removed - sqldb *sql.DB - blkStmts []*sql.Stmt - blkBaseStmts []*sql.Stmt - txStmts []*sql.Stmt - txBaseStmts []*sql.Stmt - txState txState - - // lock preventing multiple entry - dbLock sync.Mutex - - // leveldb pieces - bShaDb *leveldb.DB - bBlkDb []*leveldb.DB - tShaDb *leveldb.DB - tLocDb []*leveldb.DB - tSpentDb []*leveldb.DB - blkOpen int - txOpen int - txSpentOpen int - ro *opt.ReadOptions - wo *opt.WriteOptions - - lastBlkShaCached bool - lastBlkSha btcwire.ShaHash - lastBlkIdx int64 - txCache txCache - blockCache blockCache - - UseTempTX bool - TempTblSz int - TempTblMax int - - dbInsertMode btcdb.InsertMode -} - -var self = btcdb.DriverDB{DbType: "sqlite", Create: CreateSqliteDB, Open: OpenSqliteDB} - -func init() { - btcdb.AddDBDriver(self) -} - -// createDB configure the database, setting up all tables to initial state. -func createDB(db *sql.DB) error { - log.Infof("Initializing new block database") - - // XXX check for old tables - buildTables := []string{ - "CREATE TABLE dbversion (version integer);", - "CREATE TABLE block ( blockid INTEGER PRIMARY KEY, key BLOB UNIQUE, " + - "pver INTEGER NOT NULL, data BLOB NOT NULL);", - "INSERT INTO dbversion (version) VALUES (" + fmt.Sprintf("%d", dbVersion) + - ");", - } - buildtxTables := []string{ - "CREATE TABLE tx (txidx INTEGER PRIMARY KEY, " + - "key TEXT, " + - "blockid INTEGER NOT NULL, " + - "txoff INTEGER NOT NULL, txlen INTEGER NOT NULL, " + - "data BLOB NOT NULL, " + - "FOREIGN KEY(blockid) REFERENCES block(blockid));", - "CREATE TABLE txtmp (key TEXT PRIMARY KEY, " + - "blockid INTEGER NOT NULL, " + - "txoff INTEGER NOT NULL, txlen INTEGER NOT NULL, " + - "data BLOB NOT NULL, " + - "FOREIGN KEY(blockid) REFERENCES block(blockid));", - "CREATE UNIQUE INDEX uniquetx ON tx (key);", - } - for _, sql := range buildTables { - _, err := db.Exec(sql) - if err != nil { - log.Warnf("sql table op failed %v [%v]", err, sql) - return err - } - } - for _, sql := range buildtxTables { - _, err := db.Exec(sql) - if err != nil { - log.Warnf("sql table op failed %v [%v]", err, sql) - return err - } - } - - return nil -} - -// OpenSqliteDB opens an existing database for use. -func OpenSqliteDB(filepath string) (pbdb btcdb.Db, err error) { - log = btcdb.GetLog() - return newOrCreateSqliteDB(filepath, false) -} - -// CreateSqliteDB creates, initializes and opens a database for use. -func CreateSqliteDB(filepath string) (pbdb btcdb.Db, err error) { - log = btcdb.GetLog() - return newOrCreateSqliteDB(filepath, true) -} - -// newOrCreateSqliteDB opens a database, either creating it or opens -// existing database based on flag. -func newOrCreateSqliteDB(filepath string, create bool) (pbdb btcdb.Db, err error) { - var bdb LevelDb - if create == false { - _, err = os.Stat(filepath) - if err != nil { - return nil, btcdb.DbDoesNotExist - } - } - - db, err := sql.Open("sqlite3", filepath) - if err != nil { - log.Warnf("db open failed %v\n", err) - return nil, err - } - - dbverstmt, err := db.Prepare("SELECT version FROM dbversion;") - if err != nil { - // about the only reason this would fail is that the database - // is not initialized - if create == false { - return nil, btcdb.DbDoesNotExist - } - err = createDB(db) - if err != nil { - // already warned in the called function - return nil, err - } - dbverstmt, err = db.Prepare("SELECT version FROM dbversion;") - if err != nil { - // if it failed this a second time, fail. - return nil, err - } - - } - row := dbverstmt.QueryRow() - var version int - err = row.Scan(&version) - if err != nil { - log.Warnf("unable to find db version: no row\n", err) - } - switch version { - case dbVersion: - // all good - default: - log.Warnf("mismatch db version: %v expected %v\n", version, dbVersion) - return nil, fmt.Errorf("Invalid version in database") - } - db.Exec("PRAGMA foreign_keys = ON;") - db.Exec("PRAGMA journal_mode=WAL;") - bdb.sqldb = db - - bdb.blkStmts = make([]*sql.Stmt, len(blkqueries)) - bdb.blkBaseStmts = make([]*sql.Stmt, len(blkqueries)) - for i := range blkqueries { - stmt, err := db.Prepare(blkqueries[i]) - if err != nil { - // XXX log/ - return nil, err - } - bdb.blkBaseStmts[i] = stmt - } - for i := range bdb.blkBaseStmts { - bdb.blkStmts[i] = bdb.blkBaseStmts[i] - } - - bdb.txBaseStmts = make([]*sql.Stmt, len(txqueries)) - for i := range txqueries { - stmt, err := db.Prepare(txqueries[i]) - if err != nil { - // XXX log/ - return nil, err - } - bdb.txBaseStmts[i] = stmt - } - // NOTE: all array entries in txStmts remain nil'ed - // tx statements are lazy bound - bdb.txStmts = make([]*sql.Stmt, len(txqueries)) - - bdb.blockCache.maxcount = 150 - bdb.blockCache.blockMap = map[btcwire.ShaHash]*blockCacheObj{} - bdb.blockCache.blockMap = map[btcwire.ShaHash]*blockCacheObj{} - bdb.blockCache.blockHeightMap = map[int64]*blockCacheObj{} - bdb.txCache.maxcount = 2000 - bdb.txCache.txMap = map[btcwire.ShaHash]*txCacheObj{} - - bdb.UseTempTX = true - bdb.TempTblMax = 1000000 - - return &bdb, nil -} - -// Sync verifies that the database is coherent on disk, -// and no outstanding transactions are in flight. -func (db *LevelDb) Sync() { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - db.endTx(true) -} - -// syncPoint notifies the db that this is a safe time to sync the database, -// if there are many outstanding transactions. -// Must be called with db lock held. -func (db *LevelDb) syncPoint() { - - tx := &db.txState - - if db.TempTblSz > db.TempTblMax { - err := db.migrateTmpTable() - if err != nil { - return - } - } else { - if len(tx.txInsertList) > dbMaxTransCnt || tx.txDataSz > dbMaxTransMem { - db.endTx(true) - } - } -} - -// Close cleanly shuts down database, syncing all data. -func (db *LevelDb) Close() { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - db.close() -} - -// RollbackClose discards the recent database changes to the previously -// saved data at last Sync. -func (db *LevelDb) RollbackClose() { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - tx := &db.txState - if tx.tx != nil { - err := tx.tx.Rollback() - if err != nil { - log.Debugf("Rollback failed: %v", err) - } else { - tx.tx = nil - } - } - db.close() -} - -// close performs the internal shutdown/close operation. -func (db *LevelDb) close() { - db.endTx(true) - - db.InvalidateCache() - - for i := range db.blkBaseStmts { - db.blkBaseStmts[i].Close() - } - for i := range db.txBaseStmts { - if db.txBaseStmts[i] != nil { - db.txBaseStmts[i].Close() - db.txBaseStmts[i] = nil - } - } - db.sqldb.Close() -} - -// txop returns the appropriately prepared statement, based on -// transaction state of the database. -func (db *LevelDb) txop(op int) *sql.Stmt { - if db.txStmts[op] != nil { - return db.txStmts[op] - } - if db.txState.tx == nil { - // we are not in a transaction, return the base statement - return db.txBaseStmts[op] - } - - if db.txStmts[op] == nil { - db.txStmts[op] = db.txState.tx.Stmt(db.txBaseStmts[op]) - } - - return db.txStmts[op] -} - -// startTx starts a transaction, preparing or scrubbing statements -// for proper operation inside a transaction. -func (db *LevelDb) startTx() (err error) { - tx := &db.txState - if tx.tx != nil { - // this shouldn't happen... - log.Warnf("Db startTx called while in a transaction") - return - } - tx.tx, err = db.sqldb.Begin() - if err != nil { - log.Warnf("Db startTx: begin failed %v", err) - tx.tx = nil - return - } - for i := range db.blkBaseStmts { - db.blkStmts[i] = tx.tx.Stmt(db.blkBaseStmts[i]) - } - for i := range db.txBaseStmts { - db.txStmts[i] = nil // these are lazily prepared - } - return -} - -// endTx commits the current active transaction, it zaps all of the prepared -// statements associated with the transaction. -func (db *LevelDb) endTx(recover bool) (err error) { - tx := &db.txState - - if tx.tx == nil { - return - } - - err = tx.tx.Commit() - if err != nil && recover { - // XXX - double check that the tx is dead after - // commit failure (rollback?) - - log.Warnf("Db endTx: commit failed %v", err) - err = db.rePlayTransaction() - if err != nil { - // We tried, return failure (after zeroing state) - // so the upper level can notice and restart - } - } - for i := range db.blkBaseStmts { - db.blkStmts[i].Close() - db.blkStmts[i] = db.blkBaseStmts[i] - } - for i := range db.txStmts { - if db.txStmts[i] != nil { - db.txStmts[i].Close() - db.txStmts[i] = nil - } - } - tx.tx = nil - var emptyTxList []interface{} - tx.txInsertList = emptyTxList - tx.txDataSz = 0 - return -} - -// rePlayTransaction will attempt to re-execute inserts performed -// sync the beginning of a transaction. This is to be used after -// a sql Commit operation fails to keep the database from losing data. -func (db *LevelDb) rePlayTransaction() (err error) { - err = db.startTx() - if err != nil { - return - } - tx := &db.txState - for _, ins := range tx.txInsertList { - switch v := ins.(type) { - case tBlockInsertData: - block := v - _, err = db.blkStmts[blkInsertSha].Exec(block.sha.Bytes(), - block.pver, block.buf) - if err != nil { - break - } - case tTxInsertData: - txd := v - txnamebytes := txd.txsha.Bytes() - txop := db.txop(txInsertStmt) - _, err = txop.Exec(txd.blockid, txnamebytes, txd.txoff, - txd.txlen, txd.usedbuf) - if err != nil { - break - } - } - } - // This function is called even if we have failed. - // We need to clean up so the database can be used again. - // However we want the original error not any new error, - // unless there was no original error but the commit fails. - err2 := db.endTx(false) - if err == nil && err2 != nil { - err = err2 - } - - return -} - -// DropAfterBlockBySha will remove any blocks from the database after the given block. -// It terminates any existing transaction and performs its operations in an -// atomic transaction, it is terminated (committed) before exit. -func (db *LevelDb) DropAfterBlockBySha(sha *btcwire.ShaHash) (err error) { - var row *sql.Row - db.dbLock.Lock() - defer db.dbLock.Unlock() - - // This is a destructive operation and involves multiple requests - // so requires a transaction, terminate any transaction to date - // and start a new transaction - err = db.endTx(true) - if err != nil { - return err - } - err = db.startTx() - if err != nil { - return err - } - - var startheight int64 - - if db.lastBlkShaCached { - startheight = db.lastBlkIdx - } else { - querystr := "SELECT blockid FROM block ORDER BY blockid DESC;" - - tx := &db.txState - if tx.tx != nil { - row = tx.tx.QueryRow(querystr) - } else { - row = db.sqldb.QueryRow(querystr) - } - var startblkidx int64 - err = row.Scan(&startblkidx) - if err != nil { - log.Warnf("DropAfterBlockBySha:unable to fetch blockheight %v", err) - return err - } - startheight = startblkidx - } - // also drop any cached sha data - db.lastBlkShaCached = false - - querystr := "SELECT blockid FROM block WHERE key = ?;" - - tx := &db.txState - row = tx.tx.QueryRow(querystr, sha.Bytes()) - - var keepidx int64 - err = row.Scan(&keepidx) - if err != nil { - // XXX - db.endTx(false) - return err - } - - for height := startheight; height > keepidx; height = height - 1 { - var blk *btcutil.Block - blkc, ok := db.fetchBlockHeightCache(height) - - if ok { - blk = blkc.blk - } else { - // must load the block from the db - sha, err = db.fetchBlockShaByHeight(height - 1) - if err != nil { - return - } - - var buf []byte - var pver uint32 - - buf, pver, _, err = db.fetchSha(*sha) - if err != nil { - return - } - - blk, err = btcutil.NewBlockFromBytes(buf, pver) - if err != nil { - return - } - } - - for _, tx := range blk.MsgBlock().Transactions { - err = db.unSpend(tx) - if err != nil { - return - } - } - } - - // invalidate the cache after possibly using cached entries for block - // lookup to unspend coins in them - db.InvalidateCache() - - _, err = tx.tx.Exec("DELETE FROM txtmp WHERE blockid > ?", keepidx) - if err != nil { - // XXX - db.endTx(false) - return err - } - - _, err = tx.tx.Exec("DELETE FROM tx WHERE blockid > ?", keepidx) - if err != nil { - // XXX - db.endTx(false) - return err - } - - // delete from block last in case of foreign keys - _, err = tx.tx.Exec("DELETE FROM block WHERE blockid > ?", keepidx) - if err != nil { - // XXX - db.endTx(false) - return err - } - - err = db.endTx(true) - if err != nil { - return err - } - return -} - -// InsertBlock inserts raw block and transaction data from a block into the -// database. The first block inserted into the database will be treated as the -// genesis block. Every subsequent block insert requires the referenced parent -// block to already exist. -func (db *LevelDb) InsertBlock(block *btcutil.Block) (height int64, err error) { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - blocksha, err := block.Sha() - if err != nil { - log.Warnf("Failed to compute block sha %v", blocksha) - return - } - mblock := block.MsgBlock() - rawMsg, pver, err := block.Bytes() - if err != nil { - log.Warnf("Failed to obtain raw block sha %v", blocksha) - return - } - txloc, err := block.TxLoc() - if err != nil { - log.Warnf("Failed to obtain raw block sha %v", blocksha) - return - } - - // Insert block into database - newheight, err := db.insertBlockData(blocksha, &mblock.Header.PrevBlock, - pver, rawMsg) - if err != nil { - log.Warnf("Failed to insert block %v %v %v", blocksha, - &mblock.Header.PrevBlock, err) - return - } - - // At least two blocks in the long past were generated by faulty - // miners, the sha of the transaction exists in a previous block, - // detect this condition and 'accept' the block. - for txidx, tx := range mblock.Transactions { - var txsha btcwire.ShaHash - txsha, err = tx.TxSha(pver) - if err != nil { - log.Warnf("failed to compute tx name block %v idx %v err %v", blocksha, txidx, err) - return - } - // Some old blocks contain duplicate transactions - // Attempt to cleanly bypass this problem - // http://blockexplorer.com/b/91842 - // http://blockexplorer.com/b/91880 - if newheight == 91842 { - dupsha, err := btcwire.NewShaHashFromStr("d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599") - if err != nil { - panic("invalid sha string in source") - } - if txsha == *dupsha { - log.Tracef("skipping sha %v %v", dupsha, newheight) - continue - } - } - if newheight == 91880 { - dupsha, err := btcwire.NewShaHashFromStr("e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468") - if err != nil { - panic("invalid sha string in source") - } - if txsha == *dupsha { - log.Tracef("skipping sha %v %v", dupsha, newheight) - continue - } - } - spentbuflen := (len(tx.TxOut) + 7) / 8 - spentbuf := make([]byte, spentbuflen, spentbuflen) - if len(tx.TxOut)%8 != 0 { - for i := uint(len(tx.TxOut) % 8); i < 8; i++ { - spentbuf[spentbuflen-1] |= (byte(1) << i) - } - } - - err = db.insertTx(&txsha, newheight, txloc[txidx].TxStart, txloc[txidx].TxLen, spentbuf) - if err != nil { - log.Warnf("block %v idx %v failed to insert tx %v %v err %v", blocksha, newheight, &txsha, txidx, err) - var oBlkIdx int64 - oBlkIdx, _, _, err = db.fetchLocationBySha(&txsha) - log.Warnf("oblkidx %v err %v", oBlkIdx, err) - - return - } - err = db.doSpend(tx) - if err != nil { - log.Warnf("block %v idx %v failed to spend tx %v %v err %v", blocksha, newheight, &txsha, txidx, err) - - return - } - } - db.syncPoint() - return newheight, nil -} - -// SetDBInsertMode provides hints to the database to how the application -// is running this allows the database to work in optimized modes when the -// database may be very busy. -func (db *LevelDb) SetDBInsertMode(newmode btcdb.InsertMode) { - - oldMode := db.dbInsertMode - switch newmode { - case btcdb.InsertNormal: - // Normal mode inserts tx directly into the tx table - db.UseTempTX = false - db.dbInsertMode = newmode - switch oldMode { - case btcdb.InsertFast: - if db.TempTblSz != 0 { - err := db.migrateTmpTable() - if err != nil { - return - } - } - case btcdb.InsertValidatedInput: - // generate tx indexes - txop := db.txop(txMigrateFinish) - _, err := txop.Exec() - if err != nil { - log.Warnf("Failed to create tx table index - %v", err) - } - } - case btcdb.InsertFast: - // Fast mode inserts tx into txtmp with validation, - // then dumps to tx then rebuilds indexes at thresholds - db.UseTempTX = true - if oldMode != btcdb.InsertNormal { - log.Warnf("switching between invalid DB modes") - break - } - db.dbInsertMode = newmode - case btcdb.InsertValidatedInput: - // ValidatedInput mode inserts into tx table with - // no duplicate checks, then builds index on exit from - // ValidatedInput mode - if oldMode != btcdb.InsertNormal { - log.Warnf("switching between invalid DB modes") - break - } - // remove tx table index - txop := db.txop(txMigratePrep) - _, err := txop.Exec() - if err != nil { - log.Warnf("Failed to clear tx table index - %v", err) - } - db.dbInsertMode = newmode - - // XXX - db.UseTempTX = false - } -} -func (db *LevelDb) doSpend(tx *btcwire.MsgTx) error { - for txinidx := range tx.TxIn { - txin := tx.TxIn[txinidx] - - inTxSha := txin.PreviousOutpoint.Hash - inTxidx := txin.PreviousOutpoint.Index - - if inTxidx == ^uint32(0) { - continue - } - - //log.Infof("spending %v %v", &inTxSha, inTxidx) - - err := db.setSpentData(&inTxSha, inTxidx) - if err != nil { - return err - } - } - return nil -} - -func (db *LevelDb) unSpend(tx *btcwire.MsgTx) error { - for txinidx := range tx.TxIn { - txin := tx.TxIn[txinidx] - - inTxSha := txin.PreviousOutpoint.Hash - inTxidx := txin.PreviousOutpoint.Index - - if inTxidx == ^uint32(0) { - continue - } - - err := db.clearSpentData(&inTxSha, inTxidx) - if err != nil { - return err - } - } - return nil -} - -func (db *LevelDb) setSpentData(sha *btcwire.ShaHash, idx uint32) error { - return db.setclearSpentData(sha, idx, true) -} - -func (db *LevelDb) clearSpentData(sha *btcwire.ShaHash, idx uint32) error { - return db.setclearSpentData(sha, idx, false) -} - -func (db *LevelDb) setclearSpentData(txsha *btcwire.ShaHash, idx uint32, set bool) error { - var spentdata []byte - usingtmp := false - txop := db.txop(txFetchUsedByShaStmt) - row := txop.QueryRow(txsha.String()) - err := row.Scan(&spentdata) - if err != nil { - // if the error is simply didn't fine continue otherwise - // retun failure - - usingtmp = true - txop = db.txop(txtmpFetchUsedByShaStmt) - row := txop.QueryRow(txsha.String()) - err := row.Scan(&spentdata) - if err != nil { - log.Warnf("Failed to locate spent data - %v %v", txsha, err) - return err - } - } - byteidx := idx / 8 - byteoff := idx % 8 - - if set { - spentdata[byteidx] |= (byte(1) << byteoff) - } else { - spentdata[byteidx] &= ^(byte(1) << byteoff) - } - txc, cached := db.fetchTxCache(txsha) - if cached { - txc.spent = spentdata - } - - if usingtmp { - txop = db.txop(txtmpUpdateUsedByShaStmt) - } else { - txop = db.txop(txUpdateUsedByShaStmt) - } - _, err = txop.Exec(spentdata, txsha.String()) - - return err -} diff --git a/leveldb/operational_test.go b/leveldb/operational_test.go deleted file mode 100644 index 71c92430..00000000 --- a/leveldb/operational_test.go +++ /dev/null @@ -1,420 +0,0 @@ -// Copyright (c) 2013 Conformal Systems LLC. -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package ldb_test - -import ( - "compress/bzip2" - "encoding/binary" - "github.com/conformal/btcdb" - "github.com/conformal/btcdb/sqlite3" - "github.com/conformal/btcutil" - "github.com/conformal/btcwire" - "io" - "os" - "path/filepath" - "strings" - "testing" -) - -var network = btcwire.MainNet - -const ( - dbTmDefault = iota - dbTmNormal - dbTmFast - dbTmNoVerify -) - -func TestOperational(t *testing.T) { - testOperationalMode(t, dbTmDefault) - testOperationalMode(t, dbTmNormal) - testOperationalMode(t, dbTmFast) - testOperationalMode(t, dbTmNoVerify) -} - -func testOperationalMode(t *testing.T, mode int) { - // simplified basic operation is: - // 1) fetch block from remote server - // 2) look up all txin (except coinbase in db) - // 3) insert block - - // Ignore db remove errors since it means we didn't have an old one. - dbname := "tstdbop1" - _ = os.Remove(dbname) - db, err := btcdb.CreateDB("sqlite", dbname) - if err != nil { - t.Errorf("Failed to open test database %v", err) - return - } - defer os.Remove(dbname) - defer db.Close() - - switch mode { - case dbTmDefault: // default - // no setup - case dbTmNormal: // explicit normal - db.SetDBInsertMode(btcdb.InsertNormal) - case dbTmFast: // fast mode - db.SetDBInsertMode(btcdb.InsertFast) - if sqldb, ok := db.(*sqlite3.LevelDb); ok { - sqldb.TempTblMax = 100 - } else { - t.Errorf("not right type") - } - case dbTmNoVerify: // validated block - db.SetDBInsertMode(btcdb.InsertValidatedInput) - } - - // Since we are dealing with small dataset, reduce cache size - sqlite3.SetBlockCacheSize(db, 2) - sqlite3.SetTxCacheSize(db, 3) - - testdatafile := filepath.Join("testdata", "blocks1-256.bz2") - blocks, err := loadBlocks(t, testdatafile) - if err != nil { - t.Errorf("Unable to load blocks from test data for mode %v: %v", - mode, err) - return - } - - err = nil -out: - for height := int64(0); height < int64(len(blocks)); height++ { - block := blocks[height] - if mode != dbTmNoVerify { - // except for NoVerify which does not allow lookups check inputs - mblock := block.MsgBlock() - var txneededList []*btcwire.ShaHash - for _, tx := range mblock.Transactions { - for _, txin := range tx.TxIn { - if txin.PreviousOutpoint.Index == uint32(4294967295) { - continue - } - origintxsha := &txin.PreviousOutpoint.Hash - txneededList = append(txneededList, origintxsha) - - if !db.ExistsTxSha(origintxsha) { - t.Errorf("referenced tx not found %v ", origintxsha) - } - - _, _, _, _, err := db.FetchTxAllBySha(origintxsha) - if err != nil { - t.Errorf("referenced tx not found %v err %v ", origintxsha, err) - } - _, _, _, _, err = db.FetchTxAllBySha(origintxsha) - if err != nil { - t.Errorf("referenced tx not found %v err %v ", origintxsha, err) - } - _, _, _, err = db.FetchTxBySha(origintxsha) - if err != nil { - t.Errorf("referenced tx not found %v err %v ", origintxsha, err) - } - _, _, err = db.FetchTxBufBySha(origintxsha) - if err != nil { - t.Errorf("referenced tx not found %v err %v ", origintxsha, err) - } - _, err = db.FetchTxUsedBySha(origintxsha) - if err != nil { - t.Errorf("tx used fetch fail %v err %v ", origintxsha, err) - } - } - } - txlist := db.FetchTxByShaList(txneededList) - for _, txe := range txlist { - if txe.Err != nil { - t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) - break out - } - } - - } - - newheight, err := db.InsertBlock(block) - if err != nil { - t.Errorf("failed to insert block %v err %v", height, err) - break out - } - if newheight != height { - t.Errorf("height mismatch expect %v returned %v", height, newheight) - break out - } - - newSha, blkid, err := db.NewestSha() - if err != nil { - t.Errorf("failed to obtain latest sha %v %v", height, err) - } - - if blkid != height { - t.Errorf("height doe not match latest block height %v %v", blkid, height, err) - } - - blkSha, _ := block.Sha() - if *newSha != *blkSha { - t.Errorf("Newest block sha does not match freshly inserted one %v %v ", newSha, blkSha, err) - } - } - - // now that db is populated, do some additional test - testFetchRangeHeight(t, db, blocks) - - switch mode { - case dbTmDefault: // default - // no cleanup - case dbTmNormal: // explicit normal - // no cleanup - case dbTmFast: // fast mode - db.SetDBInsertMode(btcdb.InsertNormal) - case dbTmNoVerify: // validated block - db.SetDBInsertMode(btcdb.InsertNormal) - } -} - -func TestBackout(t *testing.T) { - testBackout(t, dbTmDefault) - testBackout(t, dbTmNormal) - testBackout(t, dbTmFast) -} - -func testBackout(t *testing.T, mode int) { - // simplified basic operation is: - // 1) fetch block from remote server - // 2) look up all txin (except coinbase in db) - // 3) insert block - - // Ignore db remove errors since it means we didn't have an old one. - dbname := "tstdbop2" - _ = os.Remove(dbname) - db, err := btcdb.CreateDB("sqlite", dbname) - if err != nil { - t.Errorf("Failed to open test database %v", err) - return - } - defer os.Remove(dbname) - defer db.Close() - - switch mode { - case dbTmDefault: // default - // no setup - case dbTmNormal: // explicit normal - db.SetDBInsertMode(btcdb.InsertNormal) - case dbTmFast: // fast mode - db.SetDBInsertMode(btcdb.InsertFast) - if sqldb, ok := db.(*sqlite3.LevelDb); ok { - sqldb.TempTblMax = 100 - } else { - t.Errorf("not right type") - } - } - - // Since we are dealing with small dataset, reduce cache size - sqlite3.SetBlockCacheSize(db, 2) - sqlite3.SetTxCacheSize(db, 3) - - testdatafile := filepath.Join("testdata", "blocks1-256.bz2") - blocks, err := loadBlocks(t, testdatafile) - if len(blocks) < 120 { - t.Errorf("test data too small") - return - } - - err = nil - for height := int64(0); height < int64(len(blocks)); height++ { - if height == 100 { - t.Logf("Syncing at block height 100") - db.Sync() - } - if height == 120 { - t.Logf("Simulating unexpected application quit") - // Simulate unexpected application quit - db.RollbackClose() - break - } - - block := blocks[height] - - newheight, err := db.InsertBlock(block) - if err != nil { - t.Errorf("failed to insert block %v err %v", height, err) - break - } - if newheight != height { - t.Errorf("height mismatch expect %v returned %v", height, newheight) - break - } - } - - // db was closed at height 120, so no cleanup is possible. - - // reopen db - db, err = btcdb.OpenDB("sqlite", dbname) - if err != nil { - t.Errorf("Failed to open test database %v", err) - return - } - defer db.Close() - - sha, err := blocks[99].Sha() - if err != nil { - t.Errorf("failed to get block 99 sha err %v", err) - return - } - _ = db.ExistsSha(sha) - _, err = db.FetchBlockBySha(sha) - if err != nil { - t.Errorf("failed to load block 99 from db %v", err) - } - - sha, err = blocks[110].Sha() - if err != nil { - t.Errorf("failed to get block 110 sha err %v", err) - return - } - _ = db.ExistsSha(sha) - _, err = db.FetchBlockBySha(sha) - if err == nil { - t.Errorf("loaded block 110 from db, failure expected") - return - } - - block := blocks[110] - mblock := block.MsgBlock() - txsha, err := mblock.Transactions[0].TxSha(block.ProtocolVersion()) - exists := db.ExistsTxSha(&txsha) - if exists { - t.Errorf("tx %v exists in db, failure expected", txsha) - } - - _, _, _, err = db.FetchTxBySha(&txsha) - _, err = db.FetchTxUsedBySha(&txsha) - - block = blocks[99] - mblock = block.MsgBlock() - txsha, err = mblock.Transactions[0].TxSha(block.ProtocolVersion()) - oldused, err := db.FetchTxUsedBySha(&txsha) - err = db.InsertTx(&txsha, 99, 1024, 1048, oldused) - if err == nil { - t.Errorf("dup insert of tx succeeded") - return - } -} - -func loadBlocks(t *testing.T, file string) (blocks []*btcutil.Block, err error) { - testdatafile := filepath.Join("testdata", "blocks1-256.bz2") - var dr io.Reader - var fi io.ReadCloser - fi, err = os.Open(testdatafile) - if err != nil { - t.Errorf("failed to open file %v, err %v", testdatafile, err) - return - } - if strings.HasSuffix(testdatafile, ".bz2") { - z := bzip2.NewReader(fi) - dr = z - } else { - dr = fi - } - - defer func() { - if err := fi.Close(); err != nil { - t.Errorf("failed to close file %v %v", testdatafile, err) - } - }() - - // Set the first block as the genesis block. - genesis := btcutil.NewBlock(&btcwire.GenesisBlock, btcwire.ProtocolVersion) - blocks = append(blocks, genesis) - - var block *btcutil.Block - err = nil - for height := int64(1); err == nil; height++ { - var rintbuf uint32 - err = binary.Read(dr, binary.LittleEndian, &rintbuf) - if err == io.EOF { - // hit end of file at expected offset: no warning - height-- - err = nil - break - } - if err != nil { - t.Errorf("failed to load network type, err %v", err) - break - } - if rintbuf != uint32(network) { - t.Errorf("Block doesn't match network: %v expects %v", - rintbuf, network) - break - } - err = binary.Read(dr, binary.LittleEndian, &rintbuf) - blocklen := rintbuf - - rbytes := make([]byte, blocklen) - - // read block - dr.Read(rbytes) - - var pver uint32 - switch { - case height < 200000: - pver = 1 - case height >= 200000: - pver = 2 - } - block, err = btcutil.NewBlockFromBytes(rbytes, pver) - if err != nil { - t.Errorf("failed to parse block %v", height) - return - } - blocks = append(blocks, block) - } - return -} - -func testFetchRangeHeight(t *testing.T, db btcdb.Db, blocks []*btcutil.Block) { - - var testincrement int64 = 50 - var testcnt int64 = 100 - - shanames := make([]*btcwire.ShaHash, len(blocks)) - - nBlocks := int64(len(blocks)) - - for i := range blocks { - blockSha, err := blocks[i].Sha() - if err != nil { - t.Errorf("FetchRangeHeight: unexpected failure computing block sah %v", err) - } - shanames[i] = blockSha - } - - for startheight := int64(0); startheight < nBlocks; startheight += testincrement { - endheight := startheight + testcnt - - if endheight > nBlocks { - endheight = btcdb.AllShas - } - - shalist, err := db.FetchHeightRange(startheight, endheight) - if err != nil { - t.Errorf("FetchRangeHeight: unexpected failure looking up shas %v", err) - } - - if endheight == btcdb.AllShas { - if int64(len(shalist)) != nBlocks-startheight { - t.Errorf("FetchRangeHeight: expected A %v shas, got %v", nBlocks-startheight, len(shalist)) - } - } else { - if int64(len(shalist)) != testcnt { - t.Errorf("FetchRangeHeight: expected %v shas, got %v", testcnt, len(shalist)) - } - } - - for i := range shalist { - if *shanames[int64(i)+startheight] != shalist[i] { - t.Errorf("FetchRangeHeight: mismatch sha at %v requested range %v %v ", int64(i)+startheight, startheight, endheight) - } - } - } - -} diff --git a/leveldb/testdata/blocks1-256.bz2 b/leveldb/testdata/blocks1-256.bz2 deleted file mode 100644 index 6b8bda4429200c0566bb13c28c35d6397272e475..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 37555 zcmV)8K*qm9T4*^jL0KkKSr@A=?*Lyo|NsC0|NsC0|NsC0|NsC0|NsC0|NsC0|NsC0 z|NsC0|Nr1%0`GtT00*bHz1|n5T)=M=pBnqU;dPxJy+dC1_BM0g?t47Wcf|GKdY#;B zx%Jb_&i2~vj_uc8@w++d_r2Xc+bQeWwB7AnH@m&s?|Z7VcLwh6_&)Ua-C7%VGQQc~ z>${h|?$>*(yBXJ;E3Q`cd%f+uRJ}XA?=#ch z^LF0cm$dfxz3JWU#m@D$*Rj3c>USOO?{`dUw52*PZ4) zZsq3L?C!nYb#Ajg-1geucDp>i-Q{VQx3xUI&gD6`Uawu6^=;dCb=vn_^?SRn>aKNl zE$pv*o-NzE&uh^0y|K@GzIV4+$bmfo002w?nqp#L0%cFrLlY(dm;eaCCJBH5N#bAt zO*GR4VqgTqU=ssCXc_@FgCjbAaCIAVQ4yFcx0gyDQ{z0Gs4FX^(;0dOf69QlWOd2#a z4FC-Q0fYo#1w0VRlT9>W35k^S(-Q<>m=TeQsvT87z%&|Zsek|g&_ zj3y%jGfhmMr>UAIk3kuxqef2@z$c;x$u^Pc4@lCq5G;R}q9*?O-;!gJMnwV1tNmyb zJ@fx!pSG8@UQWKV&A3<1{kZJNqQ08k z6D?(C9~^K8No6azD{saS5up0CIL)USziD_I?C)=&!gQRRl1VS{+J?XC_#bbw?xgcw zjlnWQhmuJmK$1=Nq>@QeY{iyDRa8VqDzR1~h>SsCD;S`vMFkaz#a1j>F^Y@@ixB}8 zR7F^*f~;V~WLU;U0gMLL5g4e*#6~J23L=XU6%+sgVk!!%C^905f(&B?Rz?W0j2J=! zkz&A5Bv=9>DkzG^F;zul6^s@Nf+(vMV51aRs)#D6ixx2yRYe3*Q4s|eF+^2@v4W~9 z3lL()AgmOMip7Gf76_n>7{x?j!Bi0#Dx^^qh$zHCR8?TA005$jqA-Mjq=2G;p(GR_ zERkRYhGl^yhX}|(fNQV-4dYK>n#ItplMB?#{#=cV!a#-X6#_w-Hww)~IQDocyC!q_ zP;QC6GWpfg{SRTvVKK%&;D83JAV?@cB9MTfNoG=P`RhZcW`%~VouXmT%%>Q`%gFiG znI{I2!l$s1vkuvTf^(_QFeC#yn%e`wLK@aVJW{_@>IkMu*Tz znP6aPZ9IqEsuH!41X8oO@$DzHuf2m$q6GXXz?)H=) z8Oo<|u~gic8#@ls4^23kc~Cpor9%wuOfhH~@7+O`A-6w1U~OG zKw{EgNhFbmD_5u(-)^8l4f&a^9R_dO&uKNTUsBR*+$IlA*>~OFPl4v}nJizqrf9w) zqe22=T4Ir+f2Wo>`sLWI3k8mhw?H3J4@8NVOJTC0;$We@CRKTKF-3MYHiCQ2x3BU( z)}O1pcK`t1iH=|ZvOy7K09X-1NkIVYB$7zO4T7V2@@RyR?lKpdotDOu@DKn100025 z8j|2a5i)c~hdx1-v5bPtJMzV|CO3sRfYNWYLAI?F)Ks#b?V*}=K)U~Rs~`0}`9jfr zq@K;^8mYzZ93voGNFdj?bzU>-R^QcyxFYAQ$zjzXUTxHHURb0-{WBrwQ(ZhX(djB$ z`|{b~h9T^fOv9~7I81ym3u$cwRR4{rh|WgWszLs;K?0NUxzv@B&3M~d!H89)Ki8|p zAx&K5^LeTg6__5a@+Dnc!orx%+h*yBU8H0mz)})hyp3;X`sbgdLE`m z?DM(0Suen3X-SJQGFHiJ0yCcr?|5*sFDL6h7#NVcZP%#TI4$qq*)#tOc#{r?EQZ$3 z@2aWLp$#GztTlv&s>7+oXsj0TsvJKrUA^1=xCTk?v&P>l6P;n&yXO^LN(W_@q8;<+ zrl^L0MY!5`jO_o#dV1AP-7?iu{WwR>bh*tHu{|g2=;Y@7OV2^dGW>g#(m%7hqc51| zzcxELdBOX6%VscLc4GgiyT#Xi{j2Y7xDyeuA-Ni0gT+xi?gf7B=ljdA!DA@CvcVa z0|L+;v>c%L7r?;)(hZ#!uzUkvOYL5>NO9@tvhh0$`rHzF$Y-2yK|#}W7;QdVP@gZ+ zsaaj}*SXBqyKiTPvBhP!4o2f%PFbD9m|>CUYfa|p_$#mW%GO>rXDT<-Io!O+ur6g+)r2!T9)G#e18LaZ{9x-(kfMX?ERZ|fqNIO zME-!VUTi4TBS-t#Yyc}w_xB<;JV`o%DIac@7nw_d$iX@5|TOOcDXyH_y!EDC<-4 zp|dc+urTQkwTFY@)y(Qp%58N!TtBnjTp9Kr7BaYo8#iI4v7ogav*%~Y0v&t_4WK;m zs=m@rZ7B+osQl1Akfx1co1S^%Ikray<`Wy)L8)zO^LY$qI#2!G2Dw%`Dr z5h{+fe|-9E$fmiB08zRveD1Yhcfwjt`wk1Iq2R`X1Q91A_9FYJ^&CP00aaVyE^zPKKH@`iDe(E?g>zB zM}%7D;S$eKl={FXK@rn&5>;QeQly&S%rH!KMK``jv;;EAMXa9wnFszdSnAk@(`fr1 z-F= zHj`aS%|-LLjNoBDsEib}NSwq5JLd6T7693-79 zDa>6lA#reFs2?X-X75sY!}Ai3TB3CwkYdOYjVslXg2Yd1enXi;DDq zzWn4VT)GEKJ7~9}y%gc+VRK3Rba`|;zJX#GA9Sf|N_4kmm@4D}qpUcsJR#}R86^G< zK3?Ss!&$nw>89d_t1|k&(zzOxU|5m0sk1dZMgpjHbS26>7RoL1{!7IW2-RKL0>wZ8 z&XXqs4L;1xknu*uwWRl0<1N;nGu^qkNAlVzq+3u3ApfQF#4g+wm$A|d!l1-_#J;yG zm3x<}PZ#pXr1xG8Mz6(W3cqYjn`{FN9tt7SJ^^!VBXJqHN)y}$W&%S+7+*{gN8(YL z5ohchr<3tZ=0#=0qo6$Obbi|LG=kcor0YLPK%_&LwJHl&e<-7n0%_7Ta;(=}?ZcGW z7T+s3xx;mUgx3d_b@*|}#dzL(mHFl|$h7?z$9%yOaT#1n%`PLi)RS9__>lx%bOy-Akf0XxoAo4bGs7|!&K zVdF4s+eNgQ3=(XFg^_SFGpAPBZD=(#bR(ZH3bbYqvRA4Zo{_Q@@-*w8gy+Zad|ABU zng4o-FaT?4s=bRnwy7h7^VJ49_PuGt6%Aw3^Ipp5ilKx@+Z*_<&m%won)`+4@VZhV zHg!;XUaY*>_#7oW>`w9d$demj4)B-L*Vf+Fq$4Eu;;nfzU&F4!7U$Y;%AhWCArS@4 zB!BjEb@@nDgj}0dN(C~)NpQOJmA$-mN*V2{*!6eX*_INd= z=fDLXe|P-UT#c#q@ZPo1k2PUb7tiD4X_n7hlBX;1ska~8f}x~aVURQY=<+tUzpc8Q zt}=$!+y>*`gZNj!5}6~H{CYxa&DUZ!e6P7Q>tOFrPa3)|*u*W5>(T_uBFMBznE3Ltoe!nHloq_n88!ROprYEC+-9f037NNO1u(LBUw5W@{UzH{ zvCt>#Fg;Vo6hJ%Fop`;K%e8Y>BbS@MN%9|0Q1jv&Cs+^WQ3dArTb$@NUIfB^_Etm{#jDAgQ26YLod%U4$7rV$nkip#Q; zvi&FM<=Wu3y>RFweN0i2m&BIh_6I)3sl3>%#?p>Oi$q6=fLPteH>3$a5%w<^?vPMx92tvP>l`P47rW>_nRV zfTuj5mXN3^1KHqn*R7R%+`@egteS@{*&XrjM}$b7JMs=%nfx8OU+@sd&h}0>q$MBB>Xtm9SQ3y(jn(R)=rk;Q9OyH<*O6iS5u|W4h`{~KFJ0>Vg z-T)so!4so}@3v1B6kS5-Vz>J_5MmX%qd4QCPk{?u2foQxU_6MN!AHoM(wxhl*b;TM z1Vc4s3M`q+D(3}9B?WVa2OXyh?MVLn(58ibwBV0>n8?IKhxzI6M3eipVcjT~Gg@sj zr3x+?rRe|yL6CuI#nExc0uEL0z;~%k$8}GWF4o}+u)v>uH8KGVcw;$W=TtOdp<1JY z>&0-7e74P}{5i5mO07Yj)vr2GA`vMe_~&8aJdLJv*Zvq)>xYGcx>e%fBr_I(A0_M8 z-2LHMP!X^vW`ssRiVCJe8%@HcwnK0tLj!mXR8aCCWjmL9=>r&1a;T&p5CGt3I6#)n zhJ$eIk5qwZ7)WCkTnYj$7IOL0cD_r5wZRr z+F$0YH)+(-acsHH3%sf~mV>de!rMD)&xx*pi?5t-?`=cTH&3+WJ_0#<$&N1s;tVj1 z$dZbT!Ut-R-|uHiCcl-y7L}JrexMik7fubi8A`5AQwV^<3O zvMAAehg72IDOeXh@D!UxlQsEJl=%P>u7uKX`3!JcHnGka85W3nE#PSINg{E0{wD=S zhbu{ICpZI4(!`e|;c+h^v9W_rsI((Fd5G*^&@x1f@g~_SaIHAVGpyN+_`OUdWwfb%tOm<=fAr+k#^ol)(mY~0;e6ub=ZYCNZ#N- zSNQxcn83j??A=7`CZ0T&ugeN+;a%HG&NJe)xG5?EB8&P(r;-b`nFqjTo4=QjzwCje zHv%x^VocPEEI8_gyF}U z9pvK~V+b=B5R9h2qxin_J4Xdc(z@@#F}NTA05l9fPEw?hhAjzx_ADd}+D-w64KyhY zD-2Tu&1aX5;B7dB|7ae%NN8CnroskUKKeuTG8&qB??RczamaLa)$>;Xcns?hoc{q* z9+M4;1OOmB{j6i;?=bEeCj>{!Ndv>{hTuK?$PU_V`q;=oS?|dw)c;^gbCJ@}{&ol( zG}`__Ye=$*XQ*$OKh8J%p~-)M$$^{R1wVz|BB4`wOXN{NrnbI*51{T&F}p(eEGcsd*hV!#dDi#q_s*aTQp>WFk}=m;i!mO@#$~$1$9Q z)DGq=0clhG9_ZIYpO1%oOOXCn9@L{E;4r4!r6aiu+H?x)UIj?Swau?7%vC_yz7;uw zrt09C8M>e143Sp^{62F%Y7cKvioYprA2YA`bO&hOf^B|EM;d3yQ%a&@eJ!T_R8agz zHhPGZCFS`JR!m?Xu<>>N0p^zY0%^=&eE6sE2t;-_DF`HSb^{%BhQ$r;{|s3<%TMHJ=< z2|zw&$9M{Q@6bFxgY)Pr{WDSmS9tx6(qg>iC`@J*O^=qiWa6wb>`x=paOV7;29 zV-v2R9sV)9Z$OA=;{Vh%5`ROu$rsj-RJs|{-y{i0!c%SF3;fZ?K;p6@pI>rjuC#f!b=sjjvC}>NP?9n+E)e&15F=|TC@q-uXa7u-Bg5j0PQo)`j zZ>sI6uXDvqO08=l=3gWk4;h$nL2?eU*{%W%^aTRLTQ5o{K@?D|^S*KJ0afG3xl;<< z`AWFU4nje8|G`DTW0H%|5#FYonpW5RaHNV}l>`v=l>xAdqQ8a;@-ZLES1y)nap^|N zIod@?T5cf9@Xl5fFaWqwn6+$OsG1-tMX10B6x1(glQ5#>XBtB$*!pSG>0O!ob;qZaXL6Ha7B;80j?8 z{}q;zbcgif=%+p44x-ZaH9&k3`(9`qpP1PT9|b4bsqA|LO!Nt_em#gvlaEL4(It8d z{!m={WQ44r9K>E;Th~1_A+S4*pXAl@ald22vvnJl-!F=%~@s9ixdn@!ZJ2y)eN8w!b_^*|_SCkF)X@Lc_ zuw z%2r~joj8{{}al_64lG#3>B!L0}UvOp^dlC?}*6T9gg>t3e3y-T;K&pW}WcNL3cv+tRdd}&IGy{^+PfR0r+x@iY*)P$?01PsOC(oy1 z#nV-XH;RVIrmexLdt5^@oZl~M2Pq|3lTiCp+~(TAv`vbqFsoFHAuLO>0_62)5V};3 zOoU&*!AT*Ssn8y)OE*&%QZ|;1KgR^NDcJ048K=(eRlPU%O$hf`=^u$tJ|lmjoAU@A z22e`ua92W66s3+L0MV-b1)c zp==w?W|Ijsh{zsEg;)*9q1CY2OX;-hEtF)9qaXXP(lKfS@vT8WbO@;Ctn`>awixJ5 zkV?)8pif(N0QQe{=&aezjmMHNo?ETY^11IR!EdZ_!IP58gB@qp<-~N+t*?b>RLn%g zBYyR+@6h4Mn-UrR_R;?*34M45Io7xf_*(Zp^<^_lSHz~6k_BPtIy9ZiK!4^Bs1SEO zL$hL`wZf=(T*2y~oWMIxy6PqMz(N{8P}4W|pB179@)(-XAjQRkU!41R;6Y}dy1iZH zl6bmLiQw?c>}CK2(KdPV8fM1N;+H;Sk9K)H<=O1uWB5zYcRRt=XKd?r9f0mUADj;B z1kD2Xk&Au~#!EdkP#pWw$}6b@&a38XB~2m?1X}Qyg-crDKjGbHl-QA*j?XLP02O3% zDqvOZPs?=4Ii!(#j%Q`3+QV`#&)scQXNmK`XHg!swI9*|7TH>Be-H?P+t3PoGl?WE zNcJP|@|^46ceBYkMF~TM^25yD__nO291fj)$oXZA9*cUzECm{cyBXYR*XKgpffH<&>{;tfFZqBo_tinP;NIski2aIz4{*5|TACBfs!izV=|dyCoG9 zz~!gb{RZ}x*h1SSvXSt_*rdAQ05^tmZPcKq-8^b8<}>mZn^a(S$=>fT_0@k(le0xB zK?dbLJY%G$iM7h{J9;n{$k1uCEb_d0Py8G(rOp>cf|7CMeG@dK{!;U2!)zL+_y}bA z{vZIA0^M#9j!2K%_}_HXFCUCjUg&#;sn(m=v-z5dK1I$IIt6L%GF+~z;sj%>iGs-d ztwx5ygjCKHN1|PuTfJ-I_%#Z|fvfH1>!bv8)_lRy@E429$}Gis61OSMxbSDz90@-Cpc5pAmFzC% zYat}Wlhp$p$6TauFQ=x^D4)icV=O~eLCeNu;Fb$?r*7OHQnkQS^TBhzu)J|9JlHor z?PK}DGx#cRf26S-3hMvZl^6^)kqvEki=jR_iW!vp(FbX$%oxxTSxvk6$5_(ADL%Ez z!Ueg?O*nIv)V)v;ai!VF!R*GG*C)mAP4+8xmZbJMta#t;W_i+fm4b~f*dUjha6a-Y zX2JI?JN?gjWF@k#tSrEV6khcjk}3V3Tp;b)Du|yCd_WE^D{IHURBOmvapg&qb>t+c z@@-?b-+xDKrnuU7Y4peX;Bg%xn0&YA4i$<>dUQ;t1V8EA6A(Fp zwrw(9N>E;KX!cv;Qwg@;0pX!)ETZj*harI_6CDaPvSq9GXW)OGvhu|)!;XB0(TJry zEA$e5YR3r`w()t@f{N}U7$8ZfOgOhgw}fEuKE0U7n@DSRoF=b?(t?0zr{N#2H;AhD zlhh|tg05^rc{EgFH-g3J>NVz27WM8Ok7qs;zO~8!3eqC22PPS(% z$I9%xedtE-HQmt@k77Q@Kf^Bj0K8LJt(EDkiPF#_agy4YO%tOr(y4JEzR4NF6i3%I zJ}zq8*%t9Psk`Bl&Umodgy>{zxN;@qfWNL%(_)wLP8N&oU=lgq9Dqt| z45sReb`|n^m5`3}J#aMnY&tKzZlfj69wy|%qeVR9L9=9aS6VJcn$fsqqGrGZ0c^t! zz!HaD_Brrds;Is!h4AnDj;w$`T$pz~3eJwFW^OlCp|v&VUr_`JrY_fv*wiB*{1HQ> zR9)Qq@r7O+)U>nyKGEI*OcRiEO-mEd2KKBpH?vYs7uhTuHSqby@?Ho|&=lDH8%gut z3izkU4iz>ozAqE>=S(RPr|;P8A|P6gr=^v%d8#W>c|X)r!}J#dn9~_T)~P6(6P%<` zE{)0DChf+U2F04m&PV&3%*-3+4`4C;;48GBRV=7DnNTimgz3Kx^pTbA%!zv9b5y;b z;d{+zn0p^~5_mVrU6ScAev{rYKQeCpCghkx5R^kwq#uCSlazx1u}>%7v5hENn$qC| zI+}DOBOI3rSn1uR!~SbHAwBm@Iu}+8arJHf0(R`XtN?2dIb-?9Peb2X9SQ#iHuF*W zU19p5XpS20Aupl(RS-xBO_Jw`rpj&CDRbz?c-bQHge57#@|=7^lGT3jF(C>YU&mCT;k>TZ&&`H!$wxFytAfws_qJ?=$8 z>`UJbOaBfBicz+->C*3^M)}d`Yzwnv00~p~NLFygppXX-Q4;XQV2szX6Xouw$#!pM zJMeu91+nXl3+=g^nVSKEu~g&8h*%dXFDPm5jk@=NEWGR&R>8ea)8I@e{=%+a^WVy5 zWT<32ar3O+*9548(8C~dP@wwfMrF{=qio`JGZ{2I63&2e`rj#S>+7)g+RvfFPN>~j zf6i*P#-AoVlyJM@VsZHvChWhU**l+~ykDY%Z z2Ncg4P0;z6=~g6?y1;>y`0RsZM)4#MYRjZ)Rc~F+Zqd) zpsIJ-n0KL%roT^=ziQw8%ZoKe=F2G=Vv{j748R*U~naE^xh~17J(XDZ;esTXq6@GOwywuuo3t@JQ zCA2c!7{lIA_5 z!5H9~7v%4w!st_~Uy(*9mTV6d;{XfU3W?XhM_{0EpLx+~S9FoPdCnpZ7j{_5e3d1% zKJIfK4LU5ZpUbtozoFsH4;`Up0Xo?u7K_-;;^L+`ZtNS2`<3pztx~bEQlFcyE|pzD zyMWzLIFR8v~Tg??*(FfRCxe=H^UvFViAfjX&96|Oup1Sc102c5Y8d|HhWhucJAAy9 zU%SlT$}N_Xw!888G70-zPy4(QBWI!AVdvw%HyPVt>IrzJXVRA!E3|N4Vi$$y!8;4U zY>*ka?;IBem#wf3hQtMZn>3UnJWar}D-?+o$msXO*k|%y@QYZMK8!y1a5hUqLmBC> zAdKG1;O7u0XA8z9X>izL{~5&vMf=|&!hTIp@%5TxFUyUGaZ95fM9EA)72K2>hDZdR zd*|7`S6C4|Q zN@h(Z`%r@zs(r0MlA1OnT9@BhQX{^W_#gE*7fEe`mSrC9dnbnxq}}i*?`PN)KebIA zu7;u6s)>U|Ao`4z3Oc@16lTJ z(f=$05I}$g0003fxZ=ctLVW>5=-9ogS{uLLHS~39zSq7vauxJOg=w{x8n@`@TJ1NA zNtwqK8fDEFTGX^QYqr{P+ck-*tIJhOD(Y48Q2t*R z_0B7+2_s?lYXSRf_|;=?dj1)Yv8TJT12Qk8q9%r;eFr`x0)CSGgxRk&cE1=L||Z}$GTNm<(>%Xz0QWjq=VrC``!OcoOPuj(6DP9E~cTQl(aFR=eMWZjrc z5yAkJgf-3G>EO77ky?(J%@Wh=5q{RNw)|^ApsmS2I$gSP_uH9@=?J6v-TRWTF^8g0 zY_q=d6nkitvXy$lKHTUSn2_tyF4BrZ*`TH6*+ZthdifoXP;vFOlCo<6H{mA)2mn9; z02j9j!|0f3icy5q?EPNRnqj|bkZTLfu8pz!Df&O9A#SnU{q=RLR7}_TUB=~2yPi3K zvqzi0n-JQUWKLx#3EOEozTg|QF>lV8eJ zCyanlg)xM9%++q6#WDrg(>b1AZ#Cf3$?Coh8DsQFqekXFX3WbDw0Eepi=Ix=T+!S%qE>XLnQ zJZ&Sl3e+7PQc1Oa44GCUz8I;2Htja%C=E8`$4`E(f4#Rg&1i_6T_@5S!Rr}s*+Ss2 zQ-oA?q4L^jv`d%EM_MPsF(J3_{TsR7Ym=g+bf~K)JR2^xQ;>fv3nKe4WrqbBKi>s2 zP%^agjI8kT0O=N_Kn3FhO9pLr_a#(jyn9Q)APeqwTkQDMY<#g{C}DD#sYfHsdsIGb zRAzou_V4l!=O8abKb7bEAn$m&9ELQzzpj7_%i&gR6|{7cawMrl1`9fTgtg=Dg^}vp zxJNY!t_BSD>>CCD&R+=a{E9c3-iXv;hYZP;joK47d=&1afrfOJoB11AkXjJts0xu@ zY0~2@ejy!Uje{J$nJ~Nqcvi4A8F~>ukDis6vXfewm5&pyDZmi=Kq%@O?-UO>Xh-!fmpQ!1v9L&ud^N+*mN@V{yu3K z5NBWm1`HY4Fkrx`<^wtfYL`_OM(eIEaaOWbt#emUw-01EAUTaHXb324v75*e9p-2o;v+=yY{nw`w!4G8 zpO%*!0tlNpnZygqt+;q|V_9q4Vk+eaGx+3g9CbRLgkxkH&`tGYp*&I!mLQo4=1JEE zq{szdU#@PKK24!vgzwp7EQbRP>0hbpx`)efOyy*(a3N;AkhrZDyoYqq&R|cFJSLP^ zXxDw_0&a+D8H5eJ-3saL+S4m6i2lawPW&srwf<}8%vBqoX4*6(*njt488~K|9<08R z`s^3eC&e=Tb_oVK+^L6wl}iQd>hB7OCCE`b+A3j#hZa0(2_l-C^vD*C{<2WceP!~+Q$2GN#BEnQPW zhFl~rJwC?r}F^8S^GiK z;ZBi1%eqBRD2c?n9>Qt7UW0Tr9&Debt`0>bK|!@?<~I`|wts7nAhI0Z1^Is7QwD7U zz#9}=ZYqS*x-9Ha;ovp8!xIUe{Z1s@B|ojO84og2OuJqtqwaF}of9I`J{xwICP!0H zRS9>k1R>hXcskaT;Oi@!W_Yb1X=tV7{17$u+uw5`tIAn=*n5;HZn@slpo@8+<)ndqG10G}rjN7*|V|Tc){W|sY zO_+E!RxrevJYB4P$I&~8u>NhS8^@0D?9i?;&4F3D7CVthKUqmI{fA$OX7eu`X~G&D zguCRcJC9;*swzJAp%4j4(B^uPzDI;sMRqc$E3z*SvCLJ#JM>@m!}Nk;j>TQQH)Ycg z;#%P`j%m^=M)PU|*BiQ;uPdHSl<0UZY#^_#Qp9}^!4m~fzUJ6pg>LGt?zo7O-A{^2 z*U)@#(N6mqPe)d6J$LTM^$z=kkb!&c#ecT>2S#W}qMO=;sQPEVuBo^iC*5-z}~vm7}YJWDEXDwan{MP?{aNBjMHZ;Malui4^RKu~#s2#LV=cSv5OPBExF zeuuF8WbU4r;pu~zlGIsB_^a4_Nt|Cf2|D-6t$I7QOm2<7bvwRpXvVhiw}s3LLY$1y z$rf*q05397&3(bmRSw8q_GRhSyQuf5Y<%u4J5dpD=h_bA4D;-Ej~25JP8v5dC|WcJ z;+vN=1=Qq1fLgxt@{*fKbf4KQ(5*WfrbnqN^w=6SRpI&%=^g#kmY}9yd2e}`1;UKh z37$4WI*Rsf_$LV}g~!h;du9PEAw#~oZ1!5Bf3`IFeXF)ZtnKdeatr1jUC2eFSogHn1z`8ZEF5EgtGM1pqN+{}U&g^u#WxbC2b>`(`OFvC3Im9u{A{@sHD&28a(LvAqW;0WXC6WK2Yn0l zQ2%Sa%r!P38F{?O6yx=p*tF-*RJX*w;NF<{pvBKP z7j$P3qEgs}5|rUsw7x64Y)v=p4lf^l^V}U=JUG6kj20TpW=Kd3BYJ(;Ykf=X1!OKs z7mi|0fG6#xelqZ!tWl5Mm*9JAi=1Y6?9)hb+nS`AVOA(SJ##`=(ZVKAoy*7D2{=k{hiNLd?)Xo=a1t#UOfkcGMH?mSG z|32j`0Wz-{2jC7rblzQEG@N>eoJmotLj|)FMpY;KrcGXe_Q32Xwq{s2&-!mB@{>wm z$mGA;u_A1(R5Ocz3a*v*L7_%jw`%d48>Ll!b;Z>eR#qV7vv8;wIcG(a!DT+0S#o1{o5J^ja_we^7ruY*;Lw1%4+G z0hT9?cyG70ZkYGJcx|#fTzCRcc6B`V?mV$Gcvq7Ic_Ht??Z`MnM|>2oIO@Sdf*uMf zdaOi-**5u4+#byzQSc}cRR8ooo=AQ_b$qj?Sg90fS=kF@)s>{78uHZjOP-x5oRFqj z&AB-49>CXD=x1q}6zAV&1X{x7%Y+k&a*AR2w}04P&F}^oBU%vF@8#lb5;Q|(;r5s< z2ZMIw7_WlTwl=W{Cn-C_0u!kyuyT}JPXmrVD)wEkg16xI(Nh!b*GA$eApNo$)%f~L z9}twgk{DB<9lXeS&tj%bSR#hyk2I06bL?@!!nU+xr~4JE=`ljs+oO;nxJSYsquI!| z4@|(ixO9nRgUf)APO zHC-1-Mu`F%?(-j}0=-sw0muI02t zHA8RuNt{wi9{ZW5dkLa6<(N z)}x;N`Q1=d|8?1|@?NLpN!Uo8!{kOuazfnSLaI?2H|AgGx5Z~db{Ofs$80l1emiyd zSX~ybB4ojmll(sa)E!B+rHIc~Wk$xSc?EUn_&k59v&Lok>V2yQgZpsF-FD%Wk zRmdEBoOiVbxE0iU{_4owaly5X0X6pEQ6;B!si1D^z>MI=`e12)>Spf(S<@mW_isCW z>6O9WA+`EZKPmEVy0XA~UdSdAscc&R`jr7>$y(uH6W_bvHwAZSiI(%XM2G@u(@Bfd zrIgIB=&;?LG#3mf`Nwv1+tRIYNAX(S=u?TebBvQyLVpRJFwLmD@-kBNQy--6B~JgE zqfb+Wj^SIw)y)`udeb~ywB52=L^LW$+X}@^sC%v4tn-Q zHBuA;60W?;bvcpCK& ztK$bw=YE|&Xw7J{879u;p%4HWI+HxTRK!^sJ?^z>Qgv~;EXw-H#9oW8Bc;a*K@>6iI_Lqu?wI8BItku z47lg)q6g5WOc8hg% z<0i&l_J2+33db%_TzgiRr_7buwYm&zSBo_P5zqvthL_SdePGke6cM6ZeG$PRGa2<$oOn9H5w8zj z0=lL)_~xL?mK_K3rM)Y))>6rXhSBA$K}8`ZL_l7@!K`t8cjH51H#8SrqN_p6%y46W z2LUf>z#Zd?D0U8hc7($3@_d9wBpX)PZAc{@x8Ns!sC~KgPyVE9g-*hrf9kpIv zyh;UF%DegG(*CtJTMHzO082o$zc+>ybHSVn8%+g9|8OzzRzTLTw)Me9d2Pg8TXa+< z22w{r!wa(HSxGei8fZT(f`-0S*<={VyX!9Ve7~I&=uSFR``F$_v+14h8RM76#cJRA zCEdoNM?lV_2Nb^bT=gM(iSB-ov)dmiK`V0t*5(Upy^OJBv?u9%%}nfvIiykRA8SXjhRGg7vd!qzZ>n+A&{<_s>G1{d)GB7Q2d7SIt z&&2~lSe5yM*)ANmM9^@fSo*kEsE({m_-IPBXV>IrbU!z^+8v-I(>I(TOIKwx|MDZN zlGutuHPLkAI*9d`dauC>t~tF8Z7>YX8XV1jX@6n->)=)cy%+oMl-<3@v{C%$dfoX! zjV!WU#8PO5Uhwxp1|DsAknnIBcGI&x(J^$Cn&P{6Z2a$40arZo;xPRJKMKs!6BIN3 zHb*-+0vy;9F%dGm&LM1%KD03$)*AhvoOlf%HBU6SW-*hhN=J8*%iI!&14gn?Gp^a`JjGMDEO;O8C^r!{2H0W3tVh z{eHX8vDlqR6sM+4X^n`(Ii$iIi$={kG4VYk-v-6dmzisw*rgWyBN<#B(8+Z^myEo0 zYV}v1)G@8c8qpRY0yO(6$kan?{JI&hDKK-z`+i?>rT5yp%Qc4q+qZ4Aqdtj5rqRL4$tUTSi70@QlO3JoRGjO zF~(1n%7$uR$y``c-)+&x1V%zcS!cqrm-zSUkn4h553EXxwh5w!jRu&CoH8=@GZ;j% z+=mMPpLJ3uYIQ@@ko@OglllIx?dbN;;S<>69a5XZi{fCktb+ptBToT?w6Op}GWRof z?3)S%{xcyz;r_0qYvwQZ%`MQUCD*EoyTkXW|3U>g`g)_SHMAEGh2P&D;ki#R2s^ZI zU|5p%%kE8$YhNP@>|UDz|2~QA!YPzK3PcA_d>nYa7N2@^_+4tRzp^_K3g!*`8GI*U zwvXF;97?Oah8_Y8CEKyN8=0b?a^jk-t4bT%f(lfx-IOY_F0Hq^!3Ft}R>B+z>=K3c>TA_Z2IsZ&)-2!0TLqHyyOMM)-?&k6 zPu?B|qAHr!NK0s~y;ukU9EO-VpMu<2h<|08-39tUz|!YWs-BQfcux%j-S+M)Xf{C( zy`k&PN#(C6w;D+@m#cOCLg%n&T?iC^60EBStoG0~s(l(Hsid2YYc!Y^yMVa38tK55 z#mm{MXm&VU%dsUBPUxBtE`{6`C<&|yTR9$$7IC8G%=s^%Zb<0ZZs;kG@`ry_=wIHUo5E;0Z^{?IaH zqt9Ql636QouvSA<>7YM?;&d*lW{Q*Fo*k8F z`x@E%EKYj~&Yvz;`vfH&Z9TgRC7TtzJGez~z}U|JCs2Pl+*D_LJ@oR0+!oVaCyvpH zzI_Uom{dSmGc18{{K7Njbp3)*;K=@!g*zgcIjw~nV#tmyM*6M)q2@!A-e-a zPa0i&LAeu8jp1&we@p84U}MCnIlslLHpxu3+dq^oWi7)q3{%6l5(j|jS~DYTskE2` z7?lrp2`5IhO}|-^wUx#!j8n#meoNdFi~Gd4Z=jmAF(y#abkCX{4)Pr zb!`0}{DpKq2EspI`F(BcV}3|H4aJ4OlM7uO;@WyG>@=zw?e#4J>Mo(zQm=TO?Cs53 zl>Gtw>>+aVW&dAt%Tq>(k%f(k2?4b)QH-gWfA4b2KZUP3SE%5|k!2lQ0;S_ibu)vN zp4I8@n%T+&VPq5=OmOCkSwVo4C%IGrc{0!QQhQ&Ni_jtrvZ0ig5U|sTzcq1cG{XQ! z_0Q$n$@K&?Y2P{6Lhw8;a#OuCpb-iZT<>n?>dkZw-mhTy8fyxZ(QC^dPHm&U69!=! zIqQ|RAz0OC`#UB2;KQ>hL%yF;qW{8nHY7atx@mG@y{3+U@d z(jeYFAuryR^57#adKhF_>=%HU=vZLhCMcCCA}$T+1r;}~(v_03(}Wg>OT7H4Rb5(^ z=X=TbH4Cw@WdBB6Y}A$2C6-9P#7AqfwT-K9@~KuVHe26uMM#fk2vA};Kc9z5==@Hz zF*t*&f9tUO8*{a&f>!q4CLMLjOO6IzICVY5FE6Y*0tcVCGhIUE!wVY~>oEhLe-WwM z-2Kf`eIOzK5H$blws1Gm#@RshqSsjH6A`-f<{Di1z7&kd^^2L?DkV19+haCUQ_b#? zoJ7)^1zdH1?L?iYgkcKIVc#EY6%E?*W~1akYwzC$`cj<~1e1{%inTz;@V!15ajP~5 zBZ^lvy$c-wELgSFG>q!K;p+51dwwl8O4ZwjeqBXXHKmkh%a(~x!U3L@B zp=SEB%uO*p0U0XyVA+d~I!-U*>*Eda+p7bhFr5EEo_oZ31#z(4aRQ@-%X2zhuSY#7 zK4GY)^QRrjgfVq1K?csCugofW1*X0rHA&5vJBPC`JMnqMZ*Y;!H2X);qko8Cb$JKE z1I*Vln5%AcWrakVMu*U+Jmb6w>2kLwx0HdcF*KudB<_l#<)(KomMGpS`(AO=nrQ9! zxf3~&k*W+zjBNzsuORV_Uw)@OB`Nh3dO^_j zncN`A^KknQZRzd0V7QNBK+bvKXnf9qukyUvK=SKfld7wz92r4NWDMa%;P13ysk z|B$xJG(!tTnyqUp$os%lp9Vqv1NnS~J}Q7k&1sFCOE*Mvu!7#xZn&v)LOJE9WiJM!Jl> z2$Hk}GUyhQEO73Uf`jK~V0Q1*d~0q0v>*3mi|wMjVTevS9Bqm)pzm^Z|1BB{1{gRy z=L`^u_MkO90a`iIGn)!)W;|q9#C5xl2$4u!b<%fTxJ3R^NET&~ z({=qmZU*^G=1^S-v2nyHuV)gBe9QmnuQ%aWoCjSQzIDp_Ip#{-%R#E1Sc5kLv3^LY zGx^x40@oCdZ*p^ytO#e=sX2u(ERg;5~sC<(C{*N++EEZ*9abR@i;L{of~m;U=oy@Mm99B(#7IMO(e zreEnwo0rq=b2)TgN}om{#z6=3f6!n9eKPI?@tRClMe?#v%{E}zh}ZTL4y z6{la2?*NHm(aBS)Tc?K|N{UOPcRRG%MiTU&k!yn1yeUmM(gOsMjAd;2ze9za#yaEN zp)=l3MNhHH9-T2&7@{j45-lNySFYz%hFSVzMSnzi3qiBj2|S_O6)o(SC!cb}W+`)}^130+~;@NJs9lO6LOe}Fr%%9lg19HLB44ap7YfuqCYJ1D2(& zlq-1Si;HMS?fNp>jYNar_2Y=yHRI^@;qZX(CyY82`gR+tOx5;%LaQi=B~_%jQpnPv zGY!KSO}rHolUd-e>4lcqXy4V(elD=W8HNgR9xJ8MA!{kX*FJ@p9e~_;zui(HxHSNT z>%R{2`lCA(sWiq9|77G8M?ITf6uA8W( zeLRsdj~@6tH~M}i%jV@sCIeYgNyFK*N7fLr$x`Wy^pkB*(4% zVBk@l$zA6Z&zF5X)s2t+Jy1m#pmy5#tXr_*1I(Ihzp6+TzNs~ukA4J)&qI?0KmAr4 z?nBJdwNpensuYJ$V%DzoJ^|_a{t*HHLB2*}U>h4j=CCrbz`x|8?el;O)cqx6%z!ef z#V$Pkzd$7)O>TKZ`;;(w6yAx)9G#{(J$(2N zhjuKs_o-B^{m2-5XKUVL|Lz#~1IHBdkTRq#1pa_uJ}RG7HIBz&Xl~YBCsp{+U%l1Z zoBKss)x|nWcGTxNlILtJ$xXl3_r{iXr6{dnm$*h~@iF2&AQEE`v2dMlDkIJj0Fxe0our_l0u?5(y7OmZ}QybYIG)(wUF!Be% znn}f1oS9g6Z$myLn#cK_KSe76+do+N|T|ue;q^nL+(cQK314@ApOmF~Io*W0q*IObCOU1>Nc%#7fb}vE0=rV@eclw)IVHmmcn&P`~S4<#iO;$;$@jXdQ-YKk!Sb3uKA$&HOYux)Q(s?Vk z$e(A*&pp|kQ3F56xS?r3Zl)^3OZ6LR9`PD({~#6htEl(ie%X&hNdM}L0LBlxkGtDH zKpA6wmG}Bu4w*NWzxK#oR8 z6BKx0iUdG?%s>SBz@z$M4zpFb$mU2tvY^}NXqDf%QF?XVM{jmyF&H|5Mn&Mv#ya>*O?~BpA z2o0jMsn=w#IpQb+EN-X%bVRFulM%wN`M+9BHnj$ET8YNt2m?qwc_{r4C5DiM?B;Yg z5Y{cYcF=;Fok(p*0G@3KgpQPb5-j<0Hk;T1=AZ2Ya}dD#S~8}Jsbw8T@>UwDm^c&_ zIihpB(tkyMrrUHBTMemXWHK+QaEFfhu9$QgHzD3eP5)Ja&x>o~Ntu=IFO}I9Zyhq^ z@NJ4e>%cMOK4G7mf#cvtQO*1cED@MsvBkP(adfCNirS%a25qg>wDZ-`@`K36bzW=i`@^Qpx@l^dek zmC;~c&qAE_Q-_RI*<|vChv_f*uqzUMhMi3Qs`Wf|E@K#QId)q&DMbQu5i)auGHV!5 z@ou%oZnHI)uj(dZ(bo+}`dngA04kQ&@*4KMkvix?qZLue)2g&p;EZ<1X?RzndQu@r z`Iw5E9Xh(YT@?N?%fcMRv_{UUqmofVn*4M@)2hc}oxvYx*inrahpH7Fw>7>L94&f0 z-l(lD~pSvR*4pC1uB>)kP33GjC_lLVk4gl(Kk!!S(KuI38i^UO%L@ z!Kq}KTXK!p72q}NdDpyNoh-cFOlucm}3c>8GVCN^`{2Xe4hs7q({ zlLzYFY?tRxApJy47o07h)O;@^n-);%x}uo!&|yOw&OqOuc=$u;>i)OMqi~SbBq<{; zA=Qc6oj%Usi0hdTD7PmULITA{?-Z>I@UzFGyJN#y=~!lk-o0m&lpF*9c+bPjCBZ4` zRxbb>ruTJV)9Iq?X)QYZGtu3_ty1N~*GEouKU`opuNcd-AQuv$e zR`2VsI)X3`M)%{KF^l9t=Ouaz0C|P}_t0c!r&8M-$-8xIAjvyrmLJ@1*oOFyGfeKt z8!5X=$H0}Hml!@lAx89hSxALh3q>WU^zKsZMr;?GzGfJ&+@zk1R!p40@``T}RH@WI z#P4Ev2I{!i)-l0PI@AT+au?GK1)Di7vaw=GkGi@g&@2vcBZ!JVylYq+w|T=-Sy(b4 zTrF#Gy9(02{r|#8pW5@TLOStCg@Z`v2VTcHH3p7l&Zk3MN! z8_!2PZ>&1E6NZVcLbiMwEr${R;u~Y7LZvS{GGA3s$$Lj$1h={7q`6L+`W z3B1b#*Tv9QhhvdPBuO3cu=#7g{=odNLX+4pTXJl|3V(txm1PSbk)4L;Agc>ojh%!I zA_1g6NhXD#YNB1fY(Iod!c7TSp<))t1#f>uYb_)GKMI~xSk|NG*)kd1JhXfrOR)M7 zW%k7c?aOang&+lFLK+{2BPe!8wnhP2tx6)`B(%RVyigoD3xs(~;}&m=`L*Lm3nrYL zK+vpY>01zZa%BABY2ehy>^G$Js*U@GnKLaqm#OB^2VUPy^_aD{f&;xG1GU5wwKB_V)hcBV#gp_0|nsh~Gs`8Kg38UkrXXTmku?Mo0 znb-S#$KFSVXSmwHDKk&~whhzJ6i(iFB0`jcVVlwmYq&ehP?${EBKdI`$JH2iAOecF z$g`2{TA|DE#aCT2yK?98LP814ftMbIA~{R9W%Z48!Ybba9dlRpC!4b{>>H$+Y!xr^ z+V-geM|Smk513B9e&5MrygSf3rj??q*HZ9MT}mA=hq%aHMn%U7muZn(rr@cD{k!9#d zM@K783Yjm9;ETRzgE^4*if;54FCWLDfA#Sfh@v+Ptz(HqSO!W0h^tCr4wh_(0Dq1v zgG~gj+D$A{7H{~p2m?*#11drO!5KY{$4!u0FoWou4)(j}AQTJ_^-7x;=;1wav!S~e zCcNph0}beNU`x0(*YMwI4^pZso*r#8c&Ah$bSQ|2_54TjyBJbRvA4@TEPFo2{8W09 zT1T9e(4v{~*Z)i;UsKc?I!qg#I6T$mg60$j>k4l(wq&@aH-}$tyDVI09X5`9FExC_ z-stj^{7unI)r36DlnV@3Pp%e#V)?`#bFWR&pXB{9t8>kUr3W{DmNSOMJ#VFD{`{FX zf}DuJZGK_)yrO4+fbvg85!Yjt?lzoma{UTz!Nr!u6}I_jY5uH}EwrzLeBjN1am5}v zW{L~3@>qN+o4Iiw(bfRSUrFtm!m$$mdZ=r5TN_Qpg>`Jh*Io?=PF=^&LO>^5DZ`f;X zA68@2k>3{kij=Ri3X}#&RJGuqd7~pl9Furt+b`?<{G}Jh2*MzoA7x3*=$J)Sb_WI~k-Y zk@m!Vf)nnFeLt@~0OunK`|wl)Qpg7+4p2rKQ4@RJx9yH}*0qxAyx@$ot98(J4x=6! z^5iUte;{!iZ7$@|n;}ehC;r!pwWECugI9+_D-6@SJu6buN2%^kbG94;dFmp!X7648 z5?C8>(GO20R{$oHv8PeVb6skEshgAnI+MM>vd(Ewr>N5k)WwaQL(Ge?-2b^GSiTos zaV)K}OKVd#x9VGFz7S?P^`mAwD)O9u4Ovx~TytbaHh!#TcW_dvE%qP1ba-jS!EL_k z{(W8v6T5&bE-atcW9p{9T~y~&5v*&md>#EF78O-OYq2mHQ-Fk}3~kk^c^0t4kD#$m zvwe@mJ>^6SQSJom&0yn?(kjeW%gen$#&|erp))8`Tq2^lY@^EkIAmN0C)l9+7^W(= zdDu{Il8|Flb8*V!bW=@NLEF+x)uc^Z3ZK^Dxl^qc+V|?;B}5*&S+{y-ym6ZRwoM?M zwVjdxF->KlgQ5aF zoT(%NF@${SRK}#yh53o@IC8I<#M?Dtr$@wvItW3eSbzZ!I4+wtOk+D$Htsl@p(PoO zh)mo{%zd<0i>(jXa``AL6UQz)S$=$_C!`J=!ZpUf+*?}t*fe~yO8XTu(|nw1wt|EFN?4dVvP)Ea?i!-`(T4+I@C z1ROJ!56fyCKrd!aqBXe6bM1+$=^VNTW&~na;HPkT*vN++;gxm z=Xh4s1P=oKy_yhaoC`TROa!CR9-(MXP7?^;6|y!Q1Z)^VpnIeApX#i63FA%uxq zUCnF3^$&mwYjW4?3gW%sXenZoxrA#i{3HI;GFRp=oG=0g!EgUaN1u-~9ujr3p_;{i z@0jC}dY*)cB3mxti~a*;Kh8??fD$2>pa#vYU>=-n=RY0fKF%oM4qrfbPY%*)_4I;||*9TNZdAz8qO30$O9>*-(N>ZlAuSP{pPv4rk0)uv7jGQcla z%C?@!5JR>bbZ4tB5Zx!4&Eq-FrJYw|lY5F;T8ogOfmA{$5j4M@9Kk=u1BJe9dfLjo z87|Pec8}|i9gQ#NIzRX3xJ;w<9oQ07NhrFb#POY@dwi=aH2gfnkPm#1wrB$j?X!Dg zY;tO!t=%ooBV}j+s?aKukPXBUEZy_39zCQ!G;n+bD~*N(-C1Pc6U~@-;H#S^5>yq+ zfnXRN*EA+{^-}kCvjS+Szu3!2jd)+oQ>b4I)7Sy9)m?ikawgZbbzmsuws?L|z&XvL zQ}C1MwP@;2 z9wp2KE^XtD6=`rzb4=`y>V-4Q#({+RG-I~T@{^mO6<1>KnoJ;>l?%!=QAqdqM5Wo) z2IyN~MX9y&an`Fo$b8(#11~v2eS=y*8wA9^M77rNhfH77N}}EUK3|bX4{f=OqKhuG&V2;p)@Nv?#Nhw_Ajm?7M@SB z;GV|Uk0Z!Du?Ho!3o9K8HxEB+pi7Pyb8jou4Ia77SMW9)fI?-b@Mo>+J!BJ%8{7l?*APU;%g|euGg}k4W8RGjiC1a#Pm^O14 zWyVQ9Xy4-_PM>8s$vvjIzHHt47U6v!75TrLU@L=-UGaqGq6H9OFt#{0bPVhSZcx2j zW3J+u(B8gyUb1d7!+7!og2vxs^1;S89c^ERkeUYDv)Eci^++`omB9xdUiJ?+<#LPZ zHJHX45PP+R7VQq7fU02^FhWMI?RooSzMHwfz(<+~I7!`frD`cu;o_0O7$gJu@05U3| z%gi?_O8j1r5Lo9%3I!?b&vA&+J%nHE$AIZOvK_7HR!mXIacl32w66J*964@Yr;X`u zR2e3v5PH10d8hrA7?w=sG0Vp!RIJu-W)N%)R&&fs3h({8s95e@6V+*-`uzR?S2M+C z6p7(is?dDO0KJ0$e*Gs)mRc1=05S~ZgSUAVf7}e`q>8L-)1+x8_OT5j+|XKjEH9FA z;>A2c*zMO%S=r0r%}X96-SmX2jhYt4swrkMLgg%7kk+5omjz3@mHXLfxDjtNxTaEt z18~qWb-)C?F8tpbwt%^HzQ&s9XI271$tPE)2PH(=nn2yQc(?NDUk)E3+wy>D%LzIa z`Hk;Vpiw#+Riz$BN77oYkt1coE_YBX?KwFG!?saP_~j4wgVVjyQYG_H=R2g5{L=g4v*-_R|CLpZV}JqJEz731h{rXufsb*6{+)7b(CJoj)^h zJ0=wO+>OBm4T63uBfsT`@>MtRKc_m2BLrSUGWn;ZJZkYVHbHH>}9BoarOfVgz|y zkM+!M1bLJZ&%nuoTl~K+D&Um<$NF2`02PYKD8*N%jWiir>t?}QNT$BxfoR!OL2J{c zT{^?(ZOLb;Q^6Ayyudk)THe-Hv!kqzQZq9RLT{JIjH-#$5IZYMP;#~g+9~n)%j&@# z{HZ%dFacNuWCrXFT3=1CF~0nbw11QvVm{;0r#gU24oTAZ`B|E(cWnF-Rc@$k2IaV7;uRk3mm8-OfS7eAF=2#(i8b zJHrtfZ~6yf|3P)E(zyD}5e7@b!lwJ$K99(o5KD4|S`sTyjeg47th!NA`tHS$fym>` z8>2Kb0oqNy_4H)2>hgzZ4)-}_^P{d+mOohk=wq->x?K;~IK!;%8!VERIPtqoNQhfs zuWXdEUhos>>CkXr&&Q~JvCjCH7dh`)3D-3-fofRKbh&66qZ$0ZS25ez37Ly_P&S&F z=t*6)0}andO}WJU`F@o51*8wpOMdITW$-=#!2k#lAV7w$pJjHh1$uw7VnV=YqC5z_S5*X*%nQ%AY=p*^=l-<4K_@WWwyzw4~f6n};az?yxk? zxuS14yu>F-*9(F88s^^;hKMRyzbtKiVH8@LH`Lr6QRyX3>O$t}x&meZz_9{Lh#<=S zbSEln8I#BzQu)K&KNz0hwZhs3$oYM4jQqJF5m>a=hibdWD-$EgIep7M8U4rojqlS> zG-=xqGS#7!MV54-o_&6i{b>@zBT5lhXAp-;>~PJdQq8DdU)WBAK2X}5-!y=`mmMjF zp0{8DPyg63%^OCR}?IH8C69`sZv3AXE>a)v8YBO46yl4McUq2q7qBS zA$1iI3#BmX>IqS|+Q8J8D1o(vTVe^0vpmbJAuf6fC29M5k? zq)3R!d(CpLD&0y06z4P7wDuU?cjceVYXEVMOXbM!c=VsdD}^flDyt?WMfl!z7%m#} zrPRx$!Q8noT&AC~xYD+D^?xkC!$zxIiKxk8Paa2AGcTW^T4yk(xvrwGTgj4u$JK$Q zrqdVF^!h>!`(Ae*az76^+f~M_{Jharpyr?P$d}^=(&%Jg-kX1>RQ087rT@cQ-AA#> zU)=if*Gz5Sjl>*X_@bQZn38rPmCCj1{8Uy~hi9+1ivVIMu{&X;qvTX^xJ&f#(Gtn0 z;$<3-Hz)9Um`ow|X()i+GHTYR*JI1E6eZYbMTAeBc%W+cs|p>1*Bv&I-=;b3s(k$e#>v> z16rC8Wa`H(|19JkNWa$V6^v(8a z^PI@M-E;MmBA-QLqtl4eT{J(&5;?xLAa{VlsGc!&7?B(vmtXZm&NkM_^1w-_JcCqO zZn8Xlq}Wb<t%rF+7}Yb~>sgz9hbVx)@bL_vRtNoK z3T#4@NUL=K<@j`GV{>tixFMU6IcmUnqt0`5=IfUW2ysPzvJ{kBaPY52AV$h~%t zZ>H#h@#G|j=hDAP)Fxe=8KA+d@mA}V(q^)ZCtjyum#HFq^a#WI^;-LmkYo<`6&~TD zmPI^698bu^1zRl`XE`!mCE(+)aytT(B`m@jsRGI}w{!^Xpo<%8d^UQ~*?$oCgVh1WT&k=*|pd9d<9H}Ip*H!vs0 z6gu1apbrQnHq4zF_quM)Y|>a1VO&?tMSboKEs>+h>5PH~7i zp?9%EZ%Gcb2Fl%HrPh*Mp3_yCm=^%(&0Q6+%(L#34Yl0KOFW=3vlM#Zo(h;O>!g!% zGa31?jP#y!LRZdjpQ7fvsV2%JZ$yc$b^X=&!hdlKHfSn;#l$x9d9I6hKbF$}kJ@4C zH7FUbs^K4hRns zP@$T_fxxL?BGJ{Yr8q7%v1s-!T+9!9x*UD6q;y-d6Hg=SNOzZI!!!-F`ldA~WS$|D z@LMuFm)m!Z4#JO-eCnocmB{TJ#;Ha>+NvWJ+!n=y#_iVAy&E}IYnc&O`679-DBK+L z922J7^iAz{Scf``_mo1l>meOecho~<$b?EHO~yQChr9by3cwwFFtP8|ExxvLqv4{N z{Gy{vGQdKdpkOWj6xa7Gn!D=E#bU&=QzHQSiNOob|H8{NiTB;Nq%BonVxM75grez& z_*_DEcU~L7p@x&`v@k$z_L$5CcX(*=-3V1}6$;{K1sO6=mj+LyF%ZHsfs1Q+IkAAT zQX;F~1Ps{MTZ4ZlbLw&_(15jHhEzmr6N z#bvCa6E&B@lu2ynaIKoi(bM(>Ba$xtVl*8yxF87gwA|`B$=g-l94a3?Qt7ub-UZgk zqM|z8@-=<;2!uEIcZCVi$C2n6tIAZl)`{R>PSOPbMHjRhYz2r385wrKheo zw8b$AvsPjfFLPhr>lZFC&$T{6Y&;>ihG#4tvmCG(k^_pT7OK*5J(_xW+=(;xV?@|G zyisP8S7z3V-ZcJyv-=et8(Dn1EXUmmv)+Dg`CWp&&t-`Oz@uWRc~sdU@g+C2Q=kb6 zaXLal7Jm|5${FfaKtQFwH zM+uqJq*WR8I>G-@f$n9XCRjVv#@}58@&l!8DP`jC%K(<`v1bp&SI@lm+VX9YR63@- zW_ll1bs(JSf!Ki6VT;LO@O2D}>7=ICPJR6NIlScj75yAO8Dn|YIUnM|-m7ca6xztK zBwlt>D&}7=>LBM`Deg%Fz(L6Ns8ERa>=?)=Z6ycsQ(Ci#^^sfs|60XX@Dw=~6@=-!#hvE~`O{XN~1kX2mqL1*NMMN_ybA3%J(Buh3JX5A~LC z{BEvRA`8tGC@&PeS~*+MMLHwvp^Y7d{4?E@T&_1E1k^U9AiJZ*`|%LeEsS``hV3|9 z*jIljz766qCHO*nHAB&7`kk?iay`6jBK`-$IzNlIJmk=GKp^tTTQm_6qN@9aVwlUW z&G68IB^tsmbFw(`valR5&IX&C{&LCxegD*DR}@XrJG>rchQuZ&OrXw(>PHNQHY4Z; zi9?N^)QxM*RU6l4$^SQSv62oPAH)^}b0&Mn+IPz%J!ELfB%lM@CP6RsJJ|@EQ+U7l z%08VU=#-jh{^Y9)Rtp8E;8FKGw=d8fY756g{_V;SonZLfa%Cr{3{YNr;do|R&YTb4 z1`TqtuU>>Zz0VWP{W>C4ux@vzn_pbMT-l-2?P*VTafrWa?ImH1JLp6(MSreqD0mM+ z0T&#G!XSXN|6~tun{C#w)3$cK(lnBvu(gM&Vw!#Aod6V5Si0-K(fW{FlJgDNz5;(J zfQv+^W`3VbK#tD6v@(0k6+>6PRSm)N7fL;+Cg+KTEvTEhS40r-(>yBTv1yg~O5EWg z2sioupNkO&s0OS>=7|2WroW1pwvyM$1emyELcsB{Iv@QukgjSmp>R-UH&!{rH+bs-chDY&8iDP*V~ zk1s?2s<`BQNsz^b_G&5HT8Qv%Uq5kO!l{KRs{aYX0Y;PlD{r@F>B&>58`LI(r{_`_ z6WZC9lH<#4-KoBQ5SGUrzDvAkZ{N0Rht=u6sO z`5eK2yA{pOj9*qY0v7UwOiqw%`9;Uj0eRO^*<*@WYxF+ebwZNL3fd&&<7Cv2I?C(72~0J)Q&l^mSIhpN}%>T8^> z6)e?`_wgA>cJ`#&zi#I5D+3UTS2S`gc&^-@%Db#kbInXSS4#3gF_R%l%* zS)%;p3RJx=bfu9B5$t$%I_yS=sT$t~F?MeHLt=^4O;eQ{dMovNU9axwC3f$BQEq?Vz zb1AKHgM$jvRuRa3Je;?EJJ+<&0NZcgGU67libhCcExzWv4W7vZwkt{hiqBd1#!X2; z%x^n_S?x0{vTWjN?_9DIADTu^owN4>Qi_}j)J&Fo zS)*;_MpBDWdE-OpP_^b7va1pR&>AZD%AqH68n-XAIVbEge3CE3IB^;mi9A~yHqcjG zw|>YL!FuIS?QQZ-kHj;2C_u=)@!~E>PjD=aGtsQ_eyl5#*WK4vw;F7Y&i2e9ygq5C zJrP_YuqO(VwdTAvf5TYe5T05CS0cB!?PAW<(%1G4295yK=u!nc_;&6KShJ8sU~Z6l z3M&$)$N;ZQ8H?e+M}}7vtS+2K6Q3Ed?|c-hcw_Ou>#uv#NBvc4{f;`}@O$MCcy5P{4IP$o zc|QEw|6ldXB6pCzNrBsdw?k+tUm;;cohB*gcQ0gDIV4Gmj%S0~=+0rN!2FUja`aY+ zAh)iF{wjvzu2usLfaurL*8gTcyXq%a(1amvZO|lB&?(B3rCc*@@DV*B)gP}dkCO$^ zCIKm1@6vx)QYr(*@#NkZ8IbYDzO=&sr7FfGq)ZnRc?`BXLPlOUx*!}{uS5WpZX?ZG zP446~FwAWZE5xZ}@BwiHAf~M9a%d__!8P{2p~jlT3}KWkX5O-?MwTaqAtNgDdM?pm zS4&N%Ajl<0|AhUysJ528<*8e617QCiTN&dCVZv9Yu-hb?mM zV|j%|U`Dr#4ih5r36F0yrV0FibHub`O|Ut^1#H~h3ahkp2d%b61G_wLagMl_Y1zzY z%Nx&hc+)NaF3+y>2|b+ujyeq&hzC3$EFm^6yB1i>Wlc;cpeN5-3`Gxodu$Pg;Ae7o z-6uGxE49kOmwhOz1L93c`hm={5tN8+Y}}pUfY4Gq>H<)T{GY%l6HN%2YoF!Atv4fn zs@9S`vIXvyLTC`U_DiHH%Pv(+hSuGq_pHJ+BMk! z66ccFpq_<=CC^#v=RbBbvlJ;Ouhz1uO{DT4_n%!|#Wz%ELrEiULDY|(1DMnQ*ZlVl zv{kRN9lYv*`|zF?-(*=}_p-zIN&pM4(KyE#*;9Q1N#XV)pA-!Ddw|&!i7Gl}ABj$a z@=45Aie*y))n-jUHKPSf;jUzS1334&WH^6E1UMi-fbRfTEb*5DgnwCVWt*#6yTyY1 z;N9S{w3!qfH@tRN0ztoTyiiiQ@CKZ?6@x6-pR;0c2l!`f* zCpO~1?K^_0MTaQ6fHv7KGM-vR$bI}CYW+_DcCS+KTKtYSbw|xjW&6uhZ#ugFFY?kG zUsy(JJqytZ9_o~_z-LekZ((&?4I?5L13PiKRA>^^wN(RPyYT1!`5umHevYHX`Y{kK z`5@APqY{yn{49!;WQVpZaq+XqZY|XAYo8fNw~?us0001i0tGMGJk>dcvUkfqNC*R$ zIG{DrsXgbBz$n*!eTMUfe+kq(uc-zESBDDu{){Mf5e}d>oB1tq@vV6dxdPLRU7aE% zT9s^M?U(C2uQ0*Nna}gW^|=}K2QsXCvQu>1TSd31oc|H=xnSbGHTlxLmTD7{;|~@5 zk5GXwxVJD+X(pIGhf)bvS}vq%CS#P(B5`!P6D8)BYdIfG>LKFdmftjg zw!P+dCxKITjfW49YT_Sh3;4p8U(Ye46phzYYj za=O$r2Tsw^#GA8?o>v--onQc{cr*r1e!vAs8a>XoxX-Q%Q#U(sY@kAz@ZsX4X>v-yF1!i7d1^5>~m7IU2XZ z{T4)qEA1p&Qkcx(yx&%cIS)f|pkeEtP&#kj7orFCkuw=XhBu@DC7h0d zRIBT4ezzk=^+yP2ONqj}wx&bQr^ls#FE_t)x=@KT>)lmbip;XQb?b4wQokIgt}w)` zn_L$p*j~vPYyhWXTpq)|TjqqO46MVj_&zwX<}Ctp=?kIWvoWUg)~Us4$K`fELv2(ddm#|pN;is z<(eYBgsRnXQ-qJqTwuiH(uMM6QCSMY>GcPDKOP*QMjvS!a^0K0+(LJgWQ_Ww74GV+ zuJ@NdP{G$O&(=UZyKuY982=pBKV@~4ySauD?!;=~%d|A=ZwCrl53yI%QWbh~`Sy;N zu;A$E67k|XLldB2`DL0b`ZD)gM<}D9Bc<3ZY6Ypa&W&0s>+?H5XAddO!=st5Vg-UI za87)OtrP98Nh5kT$TYDaJI@?J*zPdZYU3$7&6PAtuMD#Aso>Ie)tP54m76gge%DIR zh6c2l&{+C4Lm_&+v)6Ke)+q@9Fx32q&A=qXW8-d=Df82AZQW6zpMM~q7qJ4?ok*hA zh2B9kR!$iQ;fbzAb_)CjZ#TyDf;frDHwT@y&@F=NI#xhuu;KjJ6!Y5%L0TEt2!IQH zbumB6J4A=aRSft54tD%t^Iu+4-W^fSIy2D&%}m}h=Stb^h{c&`7e~Qyjm_tCFyL1a zf&F`{_+X|@bl8H6y3q@i=x94m9mLWuJ)0VdsM%)U!V63eB6T8cptp}sV=ax6@N`&7 z+TU=lQt;WKh>>Z%=BLh@+qB;fAoRmcn#x|}4oG{Wu1^0G!jYn(Z}Kox}m8(mO! zYV-Tt4jRXI?0{}^j-#dTJUX1*6Y=Db7#+iWJx1+4OqlJ>ECpfmEM|2|zvS-z z!QBtw!+eo@NOdOqX^4Hsj~z7e)xD*qe&ciIAqf}O%=KTT)4+*+??C?(-Pjqy9zi1KG=FQ7l5mvL+g-7pqbuPtvPXWCj3i0U_O?)#Uia1UAy#TXq>XYhNIJ zRht0?E2q?CX`!WtIdD?zADQ+g)Hk9+)BOP2G3wg?vE)`MslM6xDBwOzSG~&C-I_@8 zwE$7eqGn$Q9W8kEO`K4x=*!e(y0+*4-Q z=o=4Oi>47X3VAr1+I?ngFn`sx1CwTHQ>Uh~i7C+`wv`XV>5+h4h}Hs4qg-Fp2z7c# zFML>f=YK_4P0lN`5k_v=nLpFcOsDmC64Y{{-_u4d61O0*%BiY}31XWa;D(!DODJh? zwe$+Meg!_itom8!%t+8E$PrCbin!a}2O~kLBv`2-_k2oNSi61R*jUW~{Eaau55ap; zMw+byrw{ZqdkK~JqcEFk)*bKfP(%nMB%j!iQ0bXFx z-G0&^xtjEs-zMOVL9zz($UgpV@h$b!((-~Os7hng{ve%|VyhP%*Z}3%v`wy)NE5hs z&-&eEbqc-FGHR(De4x7vX}sEQcFzmVO=qoN7n{q(xOUMyK7aa*3W7ac)axZ9%L2u; zmyS-B7v?m~mlq9yoq&1jI*_wTfttl=xitk3p1x-5chIkOQ;^x>_*^?DidL5OyOWt6P&bJd2Y`QAN*{5I{bGqla08NTnR#Zvm=&1xx5A|k&R+L;zO z2jKpxUvM(F=v*0s8YkUu3@TcSL+p3zel7Lh(rqZhYsgG(PjRo4+Lwc(-Ae#lE0EzK zNf8mCZIceE7h1_}67=e*T=Kq8ojHqQZ|k@1UW9V>{XDc#&M+*}o{f6Yg+=!DqsX3?$g3apEB9Um(A zpjHH@HjG+3&Je(zGj?gjls1YunBhkjp!*zS9rALJ11`a7Uk%k%%ht2*Vcs33d!fwr z#F$srH^-7O%$kO)^PbZh#OAOEDl4{*%;hYFNNfv=D#}~i&3&FmC&?AvR|yf4dM>n5 z7oxjDHb1|5{Ok=o&XZBB#U>U93&J^eEitpyoHa)k4*j=5UPnJL)K4o#E0df{8$8#i z=-395O{xcysV3?wRJEL=*&rY{zMwAvIojiE`(A&;9@^J!b)nl8nu#GccFR_U|E`7k znSGicyf4};$zJ+CoM>iG!d-pO7R+|J1Jew9j@Ft5M1>wbU-!_+cd^nKYFbs8!Ufx_ ze6`KO0h}b(Dklk%TpvY3>HvWP1PBl#XIX>h$;lwV)^+>W0!dSU@4{rZj{BL`9$4GB z#n-P{2M!yBC`<>LVTT!cj6iV+`hCM*lhVtOEy6Ufk~6%PXQ*k%PfCs*#U@xhsKB;# zcHr6h$@A(bw2+=-vvZRfcNlFV3%fJpm?X{rdrjW%3}ZL}$Wl$D?%G)qW$=`A17Ly5 z#@CY}VaQ9A)>sxz;8+L|i8z06q8{Ctjq-n10c)_%d8E2-*sX3PeyJfEdV-jMaf)L| z=TY{j1&vwo`{)VyjfV)ir5&JZ^A$br7cJf#P?@vQc)NUv3E|J&_xS1bWLb^1dZQoF zGT?b)d&yL{PQAU1J!OfYeiW7uhv4vZPFJZSFc+~p3@BY#e+7o>A*SkD_ zHkw};T)O!G{Z}Fn{o}NJVt|umP_dtHEpZKF0o?b%+WB!R-QJB(I?N+Hn%E!qhB$~_ zu%qf}bo%1Xl8;1Xc5gwZy8B8l&(+IQtXBuXFpB|w16W#xsJtL@YN04}ndEZas`+kA zJX{zP*F?<3i~o;S00q{UbKYB-9m4xon{@9S%x!Q0Tp6j0r zKIw&zXEMUrXjP{%?uUs~_gj1EtJLzG#`RMFUR9W@mC6-JD%0K+R7kvNS{ z^FM*Ae7@7)D&9>uVM|3^dhcx`q?;GfI+1dv1UMc%Vij|9JB}q6b~u6U(cn zJH!o>Svo#SvV+Q{^uM;lUWZ`>2oNAZfmVK($@I1g01AQHixeeO*Q*t>!+N31Ph|Bx zGLu^twPg{Fy;*^cc@aY<@+%2+Ce_#)xgssQDo>BV1w*V- z(wleC=<-KC!5VanMvQ@OR<>*2EM8Ujq{5hxyKGwD1FFW0H&Jl=g$CH%z}mD7Yu(dv z?2E@s;|H(6(5tntnd#Vt4l&E-@91V4nYx=HX@fUWa_B8dBb<_h0YqKp8%M|n@%@39 zyo?$p`VvvxGRO||*!bi>pP6F<(jF~Um4}s~!)&&N$SS|I2TGiB*d|s;XxX|?GC`tM zuQb=MHv!)zuy1@-=uOje%Y8SQj+Vef1>*iy4`I@eqg&t@t9^lwQcCOk@{u z{|9a`=Z^5=1e2s2Z=h$d7CtpH6@G&+VZ9-md8{prEH*vI;M^y%)=gE4TPdt?&rcw4 zhX`ahwlX6J?J?&(xr{{0pmiA5mZPm?f&y8#5PE$*-PMGTvGoM@cQe|9a;L%MNxDGH zPItG>Np-(#^vbvO zYlkPNa&>c(=5}Q?L()|JtMBPnFnoSyD=J*2_@S`Qiju9l=kE+ZTVV+N^Z3Xx(^v)c z5WVqznmd(W+;tmQ62VK!J3k2qIZFsbAhIi~%bFMz5!+U`hjw<1C-76K!Acftr~L@= z_D5Y@Yn)aqA07kx#-{v8f~d&2n&!CuD)^OPgGu^YTVDori{ceo-4X2Yba!~bt<{<8 z>TMp5fJh4&P39#~7<(f<_lx%{Eh<;OYQM1@(<~|uA)4yU3Zl7ja{|%X%TtoMyYRpe z$(L@>daFcLhvldIS>r#cI06I!000000aUy?5J>D?Wl5R~m`n%HTUvVZJ{N*w8(Lh{ z^_#Ko~Xplcg?!2@UUQ z=1B;g&Dm9zy3=xYiF>)`U_;pefc-`yAIuk`d63N fn*wjQQLZMzvf=dzVx*Eu|Ha&qP81{s>dU*pKfkT7 diff --git a/leveldb/tx.go b/leveldb/tx.go deleted file mode 100644 index c233777b..00000000 --- a/leveldb/tx.go +++ /dev/null @@ -1,335 +0,0 @@ -// Copyright (c) 2013 Conformal Systems LLC. -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package ldb - -import ( - "database/sql" - "github.com/conformal/btcdb" - "github.com/conformal/btcwire" - _ "github.com/mattn/go-sqlite3" -) - -// InsertTx inserts a tx hash and its associated data into the database. -func (db *LevelDb) InsertTx(txsha *btcwire.ShaHash, height int64, txoff int, txlen int, usedbuf []byte) (err error) { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - return db.insertTx(txsha, height, txoff, txlen, usedbuf) -} - -// insertTx inserts a tx hash and its associated data into the database. -// Must be called with db lock held. -func (db *LevelDb) insertTx(txsha *btcwire.ShaHash, height int64, txoff int, txlen int, usedbuf []byte) (err error) { - - tx := &db.txState - if tx.tx == nil { - err = db.startTx() - if err != nil { - return - } - } - blockid := height + 1 - txd := tTxInsertData{txsha: txsha, blockid: blockid, txoff: txoff, txlen: txlen, usedbuf: usedbuf} - - log.Tracef("inserting tx %v for block %v off %v len %v", - txsha, blockid, txoff, txlen) - - rowBytes := txsha.String() - - var op int // which table to insert data into. - if db.UseTempTX { - var tblockid int64 - var ttxoff int - var ttxlen int - txop := db.txop(txFetchLocationByShaStmt) - row := txop.QueryRow(rowBytes) - err = row.Scan(&tblockid, &ttxoff, &ttxlen) - if err != sql.ErrNoRows { - // sha already present - err = btcdb.DuplicateSha - return - } - op = txtmpInsertStmt - } else { - op = txInsertStmt - } - - txop := db.txop(op) - _, err = txop.Exec(rowBytes, blockid, txoff, txlen, usedbuf) - if err != nil { - log.Warnf("failed to insert %v %v %v", txsha, blockid, err) - return - } - if db.UseTempTX { - db.TempTblSz++ - } - - // put in insert list for replay - tx.txInsertList = append(tx.txInsertList, txd) - - return -} - -// ExistsTxSha returns if the given tx sha exists in the database -func (db *LevelDb) ExistsTxSha(txsha *btcwire.ShaHash) (exists bool) { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - if _, ok := db.fetchTxCache(txsha); ok { - return true - } - - return db.existsTxSha(txsha) -} - -// existsTxSha returns if the given tx sha exists in the database.o -// Must be called with the db lock held. -func (db *LevelDb) existsTxSha(txsha *btcwire.ShaHash) (exists bool) { - var blockid uint32 - - txop := db.txop(txExistsShaStmt) - row := txop.QueryRow(txsha.String()) - err := row.Scan(&blockid) - - if err == sql.ErrNoRows { - txop = db.txop(txtmpExistsShaStmt) - row = txop.QueryRow(txsha.String()) - err := row.Scan(&blockid) - - if err == sql.ErrNoRows { - return false - } - if err != nil { - log.Warnf("txTmpExistsTxSha: fail %v", err) - return false - } - log.Warnf("txtmpExistsTxSha: success") - return true - } - - if err != nil { - // ignore real errors? - log.Warnf("existsTxSha: fail %v", err) - return false - } - - return true -} - -// FetchLocationBySha looks up the Tx sha information by name. -func (db *LevelDb) FetchLocationBySha(txsha *btcwire.ShaHash) (blockidx int64, txoff int, txlen int, err error) { - db.dbLock.Lock() - defer db.dbLock.Unlock() - return db.fetchLocationBySha(txsha) -} - -// fetchLocationBySha look up the Tx sha information by name. -// Must be called with db lock held. -func (db *LevelDb) fetchLocationBySha(txsha *btcwire.ShaHash) (height int64, txoff int, txlen int, err error) { - var row *sql.Row - var blockid int64 - var ttxoff int - var ttxlen int - - rowBytes := txsha.String() - txop := db.txop(txFetchLocationByShaStmt) - row = txop.QueryRow(rowBytes) - - err = row.Scan(&blockid, &ttxoff, &ttxlen) - if err == sql.ErrNoRows { - txop = db.txop(txtmpFetchLocationByShaStmt) - row = txop.QueryRow(rowBytes) - - err = row.Scan(&blockid, &ttxoff, &ttxlen) - if err == sql.ErrNoRows { - err = btcdb.TxShaMissing - return - } - if err != nil { - log.Warnf("txtmp FetchLocationBySha: fail %v", - err) - return - } - } - if err != nil { - log.Warnf("FetchLocationBySha: fail %v", err) - return - } - height = blockid - 1 - txoff = ttxoff - txlen = ttxlen - return -} - -// fetchLocationUsedBySha look up the Tx sha information by name. -// Must be called with db lock held. -func (db *LevelDb) fetchLocationUsedBySha(txsha *btcwire.ShaHash) (rheight int64, rtxoff int, rtxlen int, rspentbuf []byte, err error) { - var row *sql.Row - var blockid int64 - var txoff int - var txlen int - var txspent []byte - - rowBytes := txsha.String() - txop := db.txop(txFetchLocUsedByShaStmt) - row = txop.QueryRow(rowBytes) - - err = row.Scan(&blockid, &txoff, &txlen, &txspent) - if err == sql.ErrNoRows { - txop = db.txop(txtmpFetchLocUsedByShaStmt) - row = txop.QueryRow(rowBytes) - - err = row.Scan(&blockid, &txoff, &txlen, &txspent) - if err == sql.ErrNoRows { - err = btcdb.TxShaMissing - return - } - if err != nil { - log.Warnf("txtmp FetchLocationBySha: fail %v", - err) - return - } - } - if err != nil { - log.Warnf("FetchLocationBySha: fail %v", err) - return - } - height := blockid - 1 - return height, txoff, txlen, txspent, nil -} - -// FetchTxUsedBySha returns the used/spent buffer for a given transaction. -func (db *LevelDb) FetchTxUsedBySha(txsha *btcwire.ShaHash) (spentbuf []byte, err error) { - var row *sql.Row - db.dbLock.Lock() - defer db.dbLock.Unlock() - - rowBytes := txsha.String() - txop := db.txop(txFetchUsedByShaStmt) - row = txop.QueryRow(rowBytes) - - var databytes []byte - err = row.Scan(&databytes) - if err == sql.ErrNoRows { - txop := db.txop(txtmpFetchUsedByShaStmt) - row = txop.QueryRow(rowBytes) - - err = row.Scan(&databytes) - if err == sql.ErrNoRows { - err = btcdb.TxShaMissing - return - } - if err != nil { - log.Warnf("txtmp FetchLocationBySha: fail %v", - err) - return - } - } - - if err != nil { - log.Warnf("FetchUsedBySha: fail %v", err) - return - } - spentbuf = databytes - return -} - -var vaccumDbNextMigrate bool - -// migrateTmpTable functions to perform internal db optimization when -// performing large numbers of database inserts. When in Fast operation -// mode, it inserts into txtmp, then when that table reaches a certain -// size limit it moves all tx in the txtmp table into the primary tx -// table and recomputes the index on the primary tx table. -func (db *LevelDb) migrateTmpTable() error { - db.endTx(true) - db.startTx() // ??? - - db.UseTempTX = false - db.TempTblSz = 0 - - var doVacuum bool - var nsteps int - if vaccumDbNextMigrate { - nsteps = 6 - vaccumDbNextMigrate = false - doVacuum = true - } else { - nsteps = 5 - vaccumDbNextMigrate = true - } - - log.Infof("db compaction Stage 1/%v: Preparing", nsteps) - txop := db.txop(txMigratePrep) - _, err := txop.Exec() - if err != nil { - log.Warnf("Failed to prepare migrate - %v", err) - return err - } - - log.Infof("db compaction Stage 2/%v: Copying", nsteps) - txop = db.txop(txMigrateCopy) - _, err = txop.Exec() - if err != nil { - log.Warnf("Migrate read failed - %v", err) - return err - } - - log.Tracef("db compaction Stage 2a/%v: Enable db vacuum", nsteps) - txop = db.txop(txPragmaVacuumOn) - _, err = txop.Exec() - if err != nil { - log.Warnf("Migrate error trying to enable vacuum on "+ - "temporary transaction table - %v", err) - return err - } - - log.Infof("db compaction Stage 3/%v: Clearing old data", nsteps) - txop = db.txop(txMigrateClear) - _, err = txop.Exec() - if err != nil { - log.Warnf("Migrate error trying to clear temporary "+ - "transaction table - %v", err) - return err - } - - log.Tracef("db compaction Stage 3a/%v: Disable db vacuum", nsteps) - txop = db.txop(txPragmaVacuumOff) - _, err = txop.Exec() - if err != nil { - log.Warnf("Migrate error trying to disable vacuum on "+ - "temporary transaction table - %v", err) - return err - } - - log.Infof("db compaction Stage 4/%v: Rebuilding index", nsteps) - txop = db.txop(txMigrateFinish) - _, err = txop.Exec() - if err != nil { - log.Warnf("Migrate error trying to clear temporary "+ - "transaction table - %v", err) - return err - } - - log.Infof("db compaction Stage 5/%v: Finalizing transaction", nsteps) - db.endTx(true) // ??? - - if doVacuum { - log.Infof("db compaction Stage 6/%v: Optimizing database", nsteps) - txop = db.txop(txVacuum) - _, err = txop.Exec() - if err != nil { - log.Warnf("migrate error trying to clear txtmp tbl %v", err) - return err - } - } - - log.Infof("db compaction: Complete") - - // TODO(drahn) - determine if this should be turned back on or not - db.UseTempTX = true - - return nil -} From 3b743e4cfc6c84801c7cd0019f617632a30f02e1 Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Mon, 29 Jul 2013 16:39:48 -0400 Subject: [PATCH 037/163] Cleanup after insert failure, do not leave inconsistant db. Fix error returns in InsertBlock and FetchBlockBySha Give up on return by name in InsertBlock() and return explicit err one location in FetchBlockBySha to return proper error value --- sqlite3/insertfail_test.go | 141 +++++++++++++++++++++++++++++++++++++ sqlite3/internal_test.go | 36 ++++++++++ sqlite3/sqlite.go | 54 +++++++++++--- sqlite3/sqlitedbcache.go | 2 +- 4 files changed, 222 insertions(+), 11 deletions(-) create mode 100644 sqlite3/insertfail_test.go diff --git a/sqlite3/insertfail_test.go b/sqlite3/insertfail_test.go new file mode 100644 index 00000000..705b5c64 --- /dev/null +++ b/sqlite3/insertfail_test.go @@ -0,0 +1,141 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package sqlite3_test + +import ( + "github.com/conformal/btcdb" + "github.com/conformal/btcdb/sqlite3" + "os" + "path/filepath" + "testing" +) + +func TestFailOperational(t *testing.T) { + sqlite3.SetTestingT(t) + failtestOperationalMode(t, dbTmDefault) + failtestOperationalMode(t, dbTmNormal) + failtestOperationalMode(t, dbTmFast) + failtestOperationalMode(t, dbTmNoVerify) +} + +func failtestOperationalMode(t *testing.T, mode int) { + // simplified basic operation is: + // 1) fetch block from remote server + // 2) look up all txin (except coinbase in db) + // 3) insert block + + // Ignore db remove errors since it means we didn't have an old one. + dbname := "tstdbop1" + _ = os.Remove(dbname) + db, err := btcdb.CreateDB("sqlite", dbname) + if err != nil { + t.Errorf("Failed to open test database %v", err) + return + } + defer os.Remove(dbname) + defer db.Close() + + switch mode { + case dbTmDefault: // default + // no setup + case dbTmNormal: // explicit normal + db.SetDBInsertMode(btcdb.InsertNormal) + case dbTmFast: // fast mode + db.SetDBInsertMode(btcdb.InsertFast) + if sqldb, ok := db.(*sqlite3.SqliteDb); ok { + sqldb.TempTblMax = 100 + } else { + t.Errorf("not right type") + } + case dbTmNoVerify: // validated block + // no point in testing this + return + } + + // Since we are dealing with small dataset, reduce cache size + sqlite3.SetBlockCacheSize(db, 2) + sqlite3.SetTxCacheSize(db, 3) + + testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + blocks, err := loadBlocks(t, testdatafile) + if err != nil { + t.Errorf("Unable to load blocks from test data for mode %v: %v", + mode, err) + return + } + + err = nil +out: + for height := int64(0); height < int64(len(blocks)); height++ { + block := blocks[height] + + mblock := block.MsgBlock() + blockname, _ := block.Sha() + + if height == 248 { + // time to corrupt the datbase, to see if it leaves the block or tx in the db + if len(mblock.Transactions) != 2 { + t.Errorf("transaction #248 should have two transactions txid %v ?= 828ef3b079f9c23829c56fe86e85b4a69d9e06e5b54ea597eef5fb3ffef509fe", blockname) + return + } + tx := mblock.Transactions[1] + txin := tx.TxIn[0] + origintxsha := &txin.PreviousOutpoint.Hash + sqlite3.KillTx(db, origintxsha) + _, _, _, _, err = db.FetchTxAllBySha(origintxsha) + if err == nil { + t.Errorf("deleted tx found %v", origintxsha) + } + } + + + if height == 248 { + } + newheight, err := db.InsertBlock(block) + if err != nil { + if height != 248 { + t.Errorf("failed to insert block %v err %v", height, err) + break out + } + } else { + if height == 248 { + t.Errorf("block insert with missing input tx succeeded block %v err %v", height, err) + break out + } + } + if height == 248 { + for _, tx := range mblock.Transactions { + txsha, err := tx.TxSha(block.ProtocolVersion()) + _, _, _, _, err = db.FetchTxAllBySha(&txsha) + if err == nil { + t.Errorf("referenced tx found, should not have been %v, ", txsha) + } + } + } + if height == 248 { + exists := db.ExistsSha(blockname) + if exists == true { + t.Errorf("block still present after failed insert") + } + // if we got here with no error, testing was successful + break out + } + if newheight != height { + t.Errorf("height mismatch expect %v returned %v", height, newheight) + break out + } + } + + switch mode { + case dbTmDefault: // default + // no cleanup + case dbTmNormal: // explicit normal + // no cleanup + case dbTmFast: // fast mode + db.SetDBInsertMode(btcdb.InsertNormal) + case dbTmNoVerify: // validated block + db.SetDBInsertMode(btcdb.InsertNormal) + } +} diff --git a/sqlite3/internal_test.go b/sqlite3/internal_test.go index 90cb425d..64e9f0dd 100644 --- a/sqlite3/internal_test.go +++ b/sqlite3/internal_test.go @@ -8,8 +8,15 @@ import ( "fmt" "github.com/conformal/btcdb" "github.com/conformal/btcwire" + "testing" ) +var t *testing.T + +func SetTestingT(t_arg *testing.T) { + t = t_arg +} + // FetchSha returns the datablock and pver for the given ShaHash. // This is a testing only interface. func FetchSha(db btcdb.Db, sha *btcwire.ShaHash) (buf []byte, pver uint32, @@ -46,3 +53,32 @@ func SetTxCacheSize(db btcdb.Db, newsize int) { tc := &sqldb.txCache tc.maxcount = newsize } + +// KillTx is a function that deletes a transaction from the database +// this should only be used for testing purposes to valiate error paths +// in the database. This is _expected_ to leave the database in an +// inconsistant state. +func KillTx(dbarg btcdb.Db, txsha *btcwire.ShaHash) { + db, ok := dbarg.(*SqliteDb) + if !ok { + return + } + db.endTx(false) + db.startTx() + tx := &db.txState + key := txsha.String() + _, err := tx.tx.Exec("DELETE FROM txtmp WHERE key == ?", key) + if err != nil { + log.Warnf("error deleting tx %v from txtmp", txsha) + } + _, err = tx.tx.Exec("DELETE FROM tx WHERE key == ?", key) + if err != nil { + log.Warnf("error deleting tx %v from tx (%v)", txsha, key) + } + err = db.endTx(true) + if err != nil { + // XXX + db.endTx(false) + } + db.InvalidateCache() +} diff --git a/sqlite3/sqlite.go b/sqlite3/sqlite.go index b56e90a1..cee14f72 100644 --- a/sqlite3/sqlite.go +++ b/sqlite3/sqlite.go @@ -575,7 +575,12 @@ func (db *SqliteDb) DropAfterBlockBySha(sha *btcwire.ShaHash) (err error) { // lookup to unspend coins in them db.InvalidateCache() - _, err = tx.tx.Exec("DELETE FROM txtmp WHERE blockid > ?", keepidx) + return db.delFromDB(keepidx) +} + +func (db *SqliteDb) delFromDB(keepidx int64) (error) { + tx := &db.txState + _, err := tx.tx.Exec("DELETE FROM txtmp WHERE blockid > ?", keepidx) if err != nil { // XXX db.endTx(false) @@ -601,32 +606,33 @@ func (db *SqliteDb) DropAfterBlockBySha(sha *btcwire.ShaHash) (err error) { if err != nil { return err } - return + return err } // InsertBlock inserts raw block and transaction data from a block into the // database. The first block inserted into the database will be treated as the // genesis block. Every subsequent block insert requires the referenced parent // block to already exist. -func (db *SqliteDb) InsertBlock(block *btcutil.Block) (height int64, err error) { +func (db *SqliteDb) InsertBlock(block *btcutil.Block) (int64, error) { db.dbLock.Lock() defer db.dbLock.Unlock() blocksha, err := block.Sha() if err != nil { log.Warnf("Failed to compute block sha %v", blocksha) - return + return -1, err } + mblock := block.MsgBlock() rawMsg, pver, err := block.Bytes() if err != nil { log.Warnf("Failed to obtain raw block sha %v", blocksha) - return + return -1, err } txloc, err := block.TxLoc() if err != nil { log.Warnf("Failed to obtain raw block sha %v", blocksha) - return + return -1, err } // Insert block into database @@ -635,9 +641,32 @@ func (db *SqliteDb) InsertBlock(block *btcutil.Block) (height int64, err error) if err != nil { log.Warnf("Failed to insert block %v %v %v", blocksha, &mblock.Header.PrevBlock, err) - return + return -1, err } + txinsertidx := -1 + success := false + + defer func() { + if success { + return + } + + for txidx := 0; txidx <= txinsertidx; txidx++ { + tx := mblock.Transactions[txidx] + + err = db.unSpend(tx) + if err != nil { + log.Warnf("unSpend error during block insert unwind %v %v %v", blocksha, txidx, err) + } + } + + err = db.delFromDB(newheight -1) + if err != nil { + log.Warnf("Error during block insert unwind %v %v", blocksha, err) + } + }() + // At least two blocks in the long past were generated by faulty // miners, the sha of the transaction exists in a previous block, // detect this condition and 'accept' the block. @@ -646,8 +675,12 @@ func (db *SqliteDb) InsertBlock(block *btcutil.Block) (height int64, err error) txsha, err = tx.TxSha(pver) if err != nil { log.Warnf("failed to compute tx name block %v idx %v err %v", blocksha, txidx, err) - return + return -1, err } + + // num tx inserted, thus would need unwind if failure occurs + txinsertidx = txidx + // Some old blocks contain duplicate transactions // Attempt to cleanly bypass this problem // http://blockexplorer.com/b/91842 @@ -687,15 +720,16 @@ func (db *SqliteDb) InsertBlock(block *btcutil.Block) (height int64, err error) oBlkIdx, _, _, err = db.fetchLocationBySha(&txsha) log.Warnf("oblkidx %v err %v", oBlkIdx, err) - return + return -1, err } err = db.doSpend(tx) if err != nil { log.Warnf("block %v idx %v failed to spend tx %v %v err %v", blocksha, newheight, &txsha, txidx, err) - return + return -1, err } } + success = true db.syncPoint() return newheight, nil } diff --git a/sqlite3/sqlitedbcache.go b/sqlite3/sqlitedbcache.go index 3ed53318..d2d8d7d1 100644 --- a/sqlite3/sqlitedbcache.go +++ b/sqlite3/sqlitedbcache.go @@ -64,7 +64,7 @@ func (db *SqliteDb) fetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, e buf, pver, height, err := db.fetchSha(*sha) if err != nil { - return + return nil, err } blk, err = btcutil.NewBlockFromBytes(buf, pver) From 8062889d043a806a671a41c63a8cec39287a3759 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Mon, 5 Aug 2013 13:03:41 -0500 Subject: [PATCH 038/163] Update for latest btcutil and btcwire changes. This commit updates the calls into btcutil and btcwire for the latest API changes which remove the need for the protocol version for serialization and deserialization of blocks and transactions. --- README.md | 3 +-- doc.go | 3 +-- sqlite3/insertremove_test.go | 2 +- sqlite3/operational_test.go | 15 ++++----------- sqlite3/sqlite.go | 12 +++++------- sqlite3/sqlitedbcache.go | 8 ++++---- 6 files changed, 16 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 22dc0674..565f3f18 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,7 @@ change. // Insert the main network genesis block. - pver := btcwire.ProtocolVersion - genesis := btcutil.NewBlock(&btcwire.GenesisBlock, pver) + genesis := btcutil.NewBlock(&btcwire.GenesisBlock) newHeight, err := db.InsertBlock(genesis) if err != nil { // Log and handle the error diff --git a/doc.go b/doc.go index 933bd21d..f3065900 100644 --- a/doc.go +++ b/doc.go @@ -49,8 +49,7 @@ referenced parent block to already exist. In a more concrete example: // Insert the main network genesis block. - pver := btcwire.ProtocolVersion - genesis := btcutil.NewBlock(&btcwire.GenesisBlock, pver) + genesis := btcutil.NewBlock(&btcwire.GenesisBlock) newHeight, err := db.InsertBlock(genesis) if err != nil { // Log and handle the error diff --git a/sqlite3/insertremove_test.go b/sqlite3/insertremove_test.go index de27588a..ca8baf5b 100644 --- a/sqlite3/insertremove_test.go +++ b/sqlite3/insertremove_test.go @@ -101,7 +101,7 @@ endtest: } } - txshaname, _ := tx.TxSha(block.ProtocolVersion()) + txshaname, _ := tx.TxSha() txlookupList = append(txlookupList, &txshaname) txOutList = append(txOutList, &txshaname) } diff --git a/sqlite3/operational_test.go b/sqlite3/operational_test.go index 164368ea..9cfa7b6e 100644 --- a/sqlite3/operational_test.go +++ b/sqlite3/operational_test.go @@ -281,7 +281,7 @@ func testBackout(t *testing.T, mode int) { block := blocks[110] mblock := block.MsgBlock() - txsha, err := mblock.Transactions[0].TxSha(block.ProtocolVersion()) + txsha, err := mblock.Transactions[0].TxSha() exists := db.ExistsTxSha(&txsha) if exists { t.Errorf("tx %v exists in db, failure expected", txsha) @@ -292,7 +292,7 @@ func testBackout(t *testing.T, mode int) { block = blocks[99] mblock = block.MsgBlock() - txsha, err = mblock.Transactions[0].TxSha(block.ProtocolVersion()) + txsha, err = mblock.Transactions[0].TxSha() oldused, err := db.FetchTxUsedBySha(&txsha) err = db.InsertTx(&txsha, 99, 1024, 1048, oldused) if err == nil { @@ -324,7 +324,7 @@ func loadBlocks(t *testing.T, file string) (blocks []*btcutil.Block, err error) }() // Set the first block as the genesis block. - genesis := btcutil.NewBlock(&btcwire.GenesisBlock, btcwire.ProtocolVersion) + genesis := btcutil.NewBlock(&btcwire.GenesisBlock) blocks = append(blocks, genesis) var block *btcutil.Block @@ -355,14 +355,7 @@ func loadBlocks(t *testing.T, file string) (blocks []*btcutil.Block, err error) // read block dr.Read(rbytes) - var pver uint32 - switch { - case height < 200000: - pver = 1 - case height >= 200000: - pver = 2 - } - block, err = btcutil.NewBlockFromBytes(rbytes, pver) + block, err = btcutil.NewBlockFromBytes(rbytes) if err != nil { t.Errorf("failed to parse block %v", height) return diff --git a/sqlite3/sqlite.go b/sqlite3/sqlite.go index cee14f72..5b6a79c6 100644 --- a/sqlite3/sqlite.go +++ b/sqlite3/sqlite.go @@ -550,14 +550,12 @@ func (db *SqliteDb) DropAfterBlockBySha(sha *btcwire.ShaHash) (err error) { } var buf []byte - var pver uint32 - - buf, pver, _, err = db.fetchSha(*sha) + buf, _, _, err = db.fetchSha(*sha) if err != nil { return } - blk, err = btcutil.NewBlockFromBytes(buf, pver) + blk, err = btcutil.NewBlockFromBytes(buf) if err != nil { return } @@ -624,7 +622,7 @@ func (db *SqliteDb) InsertBlock(block *btcutil.Block) (int64, error) { } mblock := block.MsgBlock() - rawMsg, pver, err := block.Bytes() + rawMsg, err := block.Bytes() if err != nil { log.Warnf("Failed to obtain raw block sha %v", blocksha) return -1, err @@ -637,7 +635,7 @@ func (db *SqliteDb) InsertBlock(block *btcutil.Block) (int64, error) { // Insert block into database newheight, err := db.insertBlockData(blocksha, &mblock.Header.PrevBlock, - pver, rawMsg) + 0, rawMsg) if err != nil { log.Warnf("Failed to insert block %v %v %v", blocksha, &mblock.Header.PrevBlock, err) @@ -672,7 +670,7 @@ func (db *SqliteDb) InsertBlock(block *btcutil.Block) (int64, error) { // detect this condition and 'accept' the block. for txidx, tx := range mblock.Transactions { var txsha btcwire.ShaHash - txsha, err = tx.TxSha(pver) + txsha, err = tx.TxSha() if err != nil { log.Warnf("failed to compute tx name block %v idx %v err %v", blocksha, txidx, err) return -1, err diff --git a/sqlite3/sqlitedbcache.go b/sqlite3/sqlitedbcache.go index d2d8d7d1..0553c54b 100644 --- a/sqlite3/sqlitedbcache.go +++ b/sqlite3/sqlitedbcache.go @@ -62,12 +62,12 @@ func (db *SqliteDb) fetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, e return blkcache.blk, nil } - buf, pver, height, err := db.fetchSha(*sha) + buf, _, height, err := db.fetchSha(*sha) if err != nil { return nil, err } - blk, err = btcutil.NewBlockFromBytes(buf, pver) + blk, err = btcutil.NewBlockFromBytes(buf) if err != nil { return } @@ -195,7 +195,7 @@ func (db *SqliteDb) fetchTxDataBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx return } - blkbuf, pver, err = blk.Bytes() + blkbuf, err = blk.Bytes() if err != nil { log.Warnf("unable to decode block %v %v", height, &blksha) return @@ -264,7 +264,7 @@ func (db *SqliteDb) FetchTxAllBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, return } - blkbuf, pver, err = blk.Bytes() + blkbuf, err = blk.Bytes() if err != nil { log.Warnf("unable to decode block %v %v", height, &blksha) return From 519e183151ca64b0731f620c3542bedb881d7482 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Tue, 6 Aug 2013 11:48:23 -0500 Subject: [PATCH 039/163] Update the insertfail test for the latest changes. --- sqlite3/insertfail_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite3/insertfail_test.go b/sqlite3/insertfail_test.go index 705b5c64..e956617f 100644 --- a/sqlite3/insertfail_test.go +++ b/sqlite3/insertfail_test.go @@ -107,7 +107,7 @@ out: } if height == 248 { for _, tx := range mblock.Transactions { - txsha, err := tx.TxSha(block.ProtocolVersion()) + txsha, err := tx.TxSha() _, _, _, _, err = db.FetchTxAllBySha(&txsha) if err == nil { t.Errorf("referenced tx found, should not have been %v, ", txsha) From 840708a3e7abe7c33913c50e499f00b7c6b5cc40 Mon Sep 17 00:00:00 2001 From: "Owain G. Ainsworth" Date: Tue, 6 Aug 2013 18:00:47 +0100 Subject: [PATCH 040/163] Fix up some format strings to appease go vet. Extra argument present when no format directive for it (and not needed, either). --- sqlite3/operational_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlite3/operational_test.go b/sqlite3/operational_test.go index 9cfa7b6e..6caae7f5 100644 --- a/sqlite3/operational_test.go +++ b/sqlite3/operational_test.go @@ -147,12 +147,12 @@ out: } if blkid != height { - t.Errorf("height doe not match latest block height %v %v", blkid, height, err) + t.Errorf("height does not match latest block height %v %v", blkid, height) } blkSha, _ := block.Sha() if *newSha != *blkSha { - t.Errorf("Newest block sha does not match freshly inserted one %v %v ", newSha, blkSha, err) + t.Errorf("Newest block sha does not match freshly inserted one %v %v ", newSha, blkSha) } } From b4c6a5b8abe7fa009747ee37a4704c4be6886d90 Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Thu, 22 Aug 2013 09:32:23 -0400 Subject: [PATCH 041/163] fmt --- sqlite3/insertfail_test.go | 7 +++---- sqlite3/operational_test.go | 13 +++++-------- sqlite3/sqlite.go | 6 +++--- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/sqlite3/insertfail_test.go b/sqlite3/insertfail_test.go index e956617f..18961443 100644 --- a/sqlite3/insertfail_test.go +++ b/sqlite3/insertfail_test.go @@ -50,7 +50,7 @@ func failtestOperationalMode(t *testing.T, mode int) { t.Errorf("not right type") } case dbTmNoVerify: // validated block - // no point in testing this + // no point in testing this return } @@ -72,7 +72,7 @@ out: block := blocks[height] mblock := block.MsgBlock() - blockname, _ := block.Sha() + blockname, _ := block.Sha() if height == 248 { // time to corrupt the datbase, to see if it leaves the block or tx in the db @@ -90,7 +90,6 @@ out: } } - if height == 248 { } newheight, err := db.InsertBlock(block) @@ -120,7 +119,7 @@ out: t.Errorf("block still present after failed insert") } // if we got here with no error, testing was successful - break out + break out } if newheight != height { t.Errorf("height mismatch expect %v returned %v", height, newheight) diff --git a/sqlite3/operational_test.go b/sqlite3/operational_test.go index 6caae7f5..8132f91b 100644 --- a/sqlite3/operational_test.go +++ b/sqlite3/operational_test.go @@ -152,11 +152,10 @@ out: blkSha, _ := block.Sha() if *newSha != *blkSha { - t.Errorf("Newest block sha does not match freshly inserted one %v %v ", newSha, blkSha) + t.Errorf("Newest block sha does not match freshly inserted one %v %v ", newSha, blkSha) } } - // now that db is populated, do some additional test testFetchRangeHeight(t, db, blocks) @@ -365,11 +364,10 @@ func loadBlocks(t *testing.T, file string) (blocks []*btcutil.Block, err error) return } - -func testFetchRangeHeight(t *testing.T, db btcdb.Db, blocks []*btcutil.Block) () { +func testFetchRangeHeight(t *testing.T, db btcdb.Db, blocks []*btcutil.Block) { var testincrement int64 = 50 - var testcnt int64 = 100 + var testcnt int64 = 100 shanames := make([]*btcwire.ShaHash, len(blocks)) @@ -396,8 +394,8 @@ func testFetchRangeHeight(t *testing.T, db btcdb.Db, blocks []*btcutil.Block) () } if endheight == btcdb.AllShas { - if int64(len(shalist)) != nBlocks - startheight { - t.Errorf("FetchRangeHeight: expected A %v shas, got %v", nBlocks - startheight, len(shalist)) + if int64(len(shalist)) != nBlocks-startheight { + t.Errorf("FetchRangeHeight: expected A %v shas, got %v", nBlocks-startheight, len(shalist)) } } else { if int64(len(shalist)) != testcnt { @@ -412,5 +410,4 @@ func testFetchRangeHeight(t *testing.T, db btcdb.Db, blocks []*btcutil.Block) () } } - } diff --git a/sqlite3/sqlite.go b/sqlite3/sqlite.go index 5b6a79c6..8f68d352 100644 --- a/sqlite3/sqlite.go +++ b/sqlite3/sqlite.go @@ -576,7 +576,7 @@ func (db *SqliteDb) DropAfterBlockBySha(sha *btcwire.ShaHash) (err error) { return db.delFromDB(keepidx) } -func (db *SqliteDb) delFromDB(keepidx int64) (error) { +func (db *SqliteDb) delFromDB(keepidx int64) error { tx := &db.txState _, err := tx.tx.Exec("DELETE FROM txtmp WHERE blockid > ?", keepidx) if err != nil { @@ -649,7 +649,7 @@ func (db *SqliteDb) InsertBlock(block *btcutil.Block) (int64, error) { if success { return } - + for txidx := 0; txidx <= txinsertidx; txidx++ { tx := mblock.Transactions[txidx] @@ -659,7 +659,7 @@ func (db *SqliteDb) InsertBlock(block *btcutil.Block) (int64, error) { } } - err = db.delFromDB(newheight -1) + err = db.delFromDB(newheight - 1) if err != nil { log.Warnf("Error during block insert unwind %v %v", blocksha, err) } From b66abdf6ba9da677560cdc58b4c5578a5b242bf1 Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Sat, 3 Aug 2013 11:20:05 -0400 Subject: [PATCH 042/163] Initial leveldb code. This code is still prototype at this time. It appears to function correctly but something consumes more memory than is considered reasonable for the dataset comprised of the full bitcoind chain. Not recommened for use at this time. --- ldb/block.go | 262 +++++++++++++++++ ldb/dbcache.go | 136 +++++++++ ldb/dbtest/dbtst.go | 56 ++++ ldb/doc.go | 15 + ldb/insertremove_test.go | 202 ++++++++++++++ ldb/internal_test.go | 24 ++ ldb/leveldb.go | 528 +++++++++++++++++++++++++++++++++++ ldb/operational_test.go | 394 ++++++++++++++++++++++++++ ldb/testdata/blocks1-256.bz2 | Bin 0 -> 37555 bytes ldb/tx.go | 175 ++++++++++++ 10 files changed, 1792 insertions(+) create mode 100644 ldb/block.go create mode 100644 ldb/dbcache.go create mode 100644 ldb/dbtest/dbtst.go create mode 100644 ldb/doc.go create mode 100644 ldb/insertremove_test.go create mode 100644 ldb/internal_test.go create mode 100644 ldb/leveldb.go create mode 100644 ldb/operational_test.go create mode 100644 ldb/testdata/blocks1-256.bz2 create mode 100644 ldb/tx.go diff --git a/ldb/block.go b/ldb/block.go new file mode 100644 index 00000000..8d211e75 --- /dev/null +++ b/ldb/block.go @@ -0,0 +1,262 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ldb + +import ( + "bytes" + "encoding/binary" + "fmt" + "github.com/conformal/btcdb" + "github.com/conformal/btcwire" +) + +// InsertBlockData stores a block hash and its associated data block with a +// previous sha of `prevSha' and a version of `pver'. +func (db *LevelDb) InsertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHash, pver uint32, buf []byte) (blockid int64, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + return db.insertBlockData(sha, prevSha, buf) +} + +func (db *LevelDb) getBlkLoc(sha *btcwire.ShaHash) (int64, error) { + var blkHeight int64 + + key := sha.Bytes() + + data, err := db.bShaDb.Get(key, db.ro) + + if err != nil { + return 0, err + } + + // deserialize + dr := bytes.NewBuffer(data) + err = binary.Read(dr, binary.LittleEndian, &blkHeight) + if err != nil { + fmt.Printf("get getBlkLoc len %v\n", len(data)) + err = fmt.Errorf("Db Corrupt 0") + return 0, err + } + return blkHeight, nil +} + +func (db *LevelDb) getBlkByHeight(blkHeight int64) (rsha *btcwire.ShaHash, rbuf []byte, err error) { + var blkVal []byte + + key := int64ToKey(blkHeight) + + blkVal, err = db.bBlkDb.Get(key, db.ro) + if err != nil { + return // exists ??? + } + + var sha btcwire.ShaHash + + sha.SetBytes(blkVal[0:32]) + + blockdata := make([]byte, len(blkVal[32:])) + copy(blockdata[:], blkVal[32:]) + + return &sha, blockdata, nil +} + +func (db *LevelDb) getBlk(sha *btcwire.ShaHash) (rblkHeight int64, rbuf []byte, err error) { + var blkHeight int64 + + blkHeight, err = db.getBlkLoc(sha) + if err != nil { + return + } + + var buf []byte + + _, buf, err = db.getBlkByHeight(blkHeight) + if err != nil { + return + } + return blkHeight, buf, nil +} + +func (db *LevelDb) setBlk(sha *btcwire.ShaHash, blkHeight int64, buf []byte) error { + + // serialize + var lw bytes.Buffer + err := binary.Write(&lw, binary.LittleEndian, blkHeight) + if err != nil { + err = fmt.Errorf("Write Fail") + return err + } + shaKey := sha.Bytes() + + blkKey := int64ToKey(blkHeight) + + shaB := sha.Bytes() + blkVal := make([]byte, len(shaB)+len(buf)) + copy(blkVal[0:], shaB) + copy(blkVal[len(shaB):], buf) + + db.bShaBatch().Put(shaKey, lw.Bytes()) + + db.bBlkBatch().Put(blkKey, blkVal) + + return nil +} + +// insertSha stores a block hash and its associated data block with a +// previous sha of `prevSha' and a version of `pver'. +// insertSha shall be called with db lock held +func (db *LevelDb) insertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHash, buf []byte) (blockid int64, err error) { + + oBlkHeight, err := db.getBlkLoc(prevSha) + + if err != nil { + // check current block count + // if count != 0 { + // err = btcdb.PrevShaMissing + // return + // } + oBlkHeight = -1 + } + + // TODO(drahn) check curfile filesize, increment curfile if this puts it over + blkHeight := oBlkHeight + 1 + + err = db.setBlk(sha, blkHeight, buf) + + if err != nil { + return + } + + // update the last block cache + db.lastBlkShaCached = true + db.lastBlkSha = *sha + db.lastBlkIdx = blkHeight + db.nextBlock = blkHeight + 1 + + return blkHeight, nil +} + +// fetchSha returns the datablock and pver for the given ShaHash. +func (db *LevelDb) fetchSha(sha *btcwire.ShaHash) (rbuf []byte, + rblkHeight int64, err error) { + var blkHeight int64 + var buf []byte + + blkHeight, buf, err = db.getBlk(sha) + if err != nil { + return + } + + return buf, blkHeight, nil +} + +// ExistsSha looks up the given block hash +// returns true if it is present in the database. +func (db *LevelDb) ExistsSha(sha *btcwire.ShaHash) (exists bool) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + // not in cache, try database + exists = db.blkExistsSha(sha) + return +} + +// blkExistsSha looks up the given block hash +// returns true if it is present in the database. +// CALLED WITH LOCK HELD +func (db *LevelDb) blkExistsSha(sha *btcwire.ShaHash) bool { + + _, err := db.getBlkLoc(sha) + + if err != nil { + /* + should this warn if the failure is something besides does not exist ? + log.Warnf("blkExistsSha: fail %v", err) + */ + return false + } + return true +} + +// FetchBlockShaByHeight returns a block hash based on its height in the +// block chain. +func (db *LevelDb) FetchBlockShaByHeight(height int64) (sha *btcwire.ShaHash, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + return db.fetchBlockShaByHeight(height) +} + +// fetchBlockShaByHeight returns a block hash based on its height in the +// block chain. +func (db *LevelDb) fetchBlockShaByHeight(height int64) (rsha *btcwire.ShaHash, err error) { + var sha *btcwire.ShaHash + sha, _, err = db.getBlkByHeight(height) + if err != nil { + return + } + + return sha, nil +} + +// FetchHeightRange looks up a range of blocks by the start and ending +// heights. Fetch is inclusive of the start height and exclusive of the +// ending height. To fetch all hashes from the start height until no +// more are present, use the special id `AllShas'. +func (db *LevelDb) FetchHeightRange(startHeight, endHeight int64) (rshalist []btcwire.ShaHash, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + var endidx int64 + if endHeight == btcdb.AllShas { + endidx = startHeight + 500 + } else { + endidx = endHeight + } + + var shalist []btcwire.ShaHash + for height := startHeight; height < endidx; height++ { + // TODO(drahn) fix blkFile from height + + key := int64ToKey(height) + blkVal, lerr := db.bBlkDb.Get(key, db.ro) + if lerr != nil { + break + } + + var sha btcwire.ShaHash + sha.SetBytes(blkVal[0:32]) + shalist = append(shalist, sha) + } + + if err != nil { + return + } + //log.Tracef("FetchIdxRange idx %v %v returned %v shas err %v", startHeight, endHeight, len(shalist), err) + + return shalist, nil +} + +// NewestSha returns the hash and block height of the most recent (end) block of +// the block chain. It will return the zero hash, -1 for the block height, and +// no error (nil) if there are not any blocks in the database yet. +func (db *LevelDb) NewestSha() (rsha *btcwire.ShaHash, rblkid int64, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + if db.lastBlkIdx == -1 { + err = fmt.Errorf("Empty Database") + return + } + sha := db.lastBlkSha + + return &sha, db.lastBlkIdx, nil +} + +func (db *LevelDb) NewIterateBlocks() (rbogus btcdb.BlockIterator, err error) { + err = fmt.Errorf("Not implemented") + return +} diff --git a/ldb/dbcache.go b/ldb/dbcache.go new file mode 100644 index 00000000..c5f74c90 --- /dev/null +++ b/ldb/dbcache.go @@ -0,0 +1,136 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ldb + +import ( + "bytes" + "fmt" + "github.com/conformal/btcdb" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" +) + +// FetchBlockBySha - return a btcutil Block, object may be a cached. +func (db *LevelDb) FetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + return db.fetchBlockBySha(sha) +} + +// fetchBlockBySha - return a btcutil Block, object may be a cached. +// Must be called with db lock held. +func (db *LevelDb) fetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) { + + buf, height, err := db.fetchSha(sha) + if err != nil { + return + } + + blk, err = btcutil.NewBlockFromBytes(buf) + if err != nil { + return + } + blk.SetHeight(height) + + return +} + +// FetchTxByShaList given a array of ShaHash, look up the transactions +// and return them in a TxListReply array. +func (db *LevelDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + replies := make([]*btcdb.TxListReply, len(txShaList)) + for i, txsha := range txShaList { + tx, _, _, _, height, txspent, err := db.fetchTxDataBySha(txsha) + btxspent := []bool{} + if err == nil { + btxspent = make([]bool, len(tx.TxOut), len(tx.TxOut)) + for idx := range tx.TxOut { + byteidx := idx / 8 + byteoff := uint(idx % 8) + btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0 + } + } + txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, Height: height, TxSpent: btxspent, Err: err} + replies[i] = &txlre + } + return replies +} + +// fetchTxDataBySha returns several pieces of data regarding the given sha. +func (db *LevelDb) fetchTxDataBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rtxbuf []byte, rpver uint32, rblksha *btcwire.ShaHash, rheight int64, rtxspent []byte, err error) { + var pver uint32 + var blksha *btcwire.ShaHash + var blkHeight int64 + var txspent []byte + var txOff, txLen int + var blkbuf []byte + + blkHeight, txOff, txLen, txspent, err = db.getTxData(txsha) + if err != nil { + err = btcdb.TxShaMissing + return + } + + blksha, blkbuf, err = db.getBlkByHeight(blkHeight) + if err != nil { + fmt.Printf("failed to get block %v %v\n", blkHeight, err) + return + } + + //log.Trace("transaction %v is at block %v %v txoff %v, txlen %v\n", + // txsha, blksha, blkHeight, txOff, txLen) + + txbuf := make([]byte, txLen) + copy(txbuf[:], blkbuf[txOff:txOff+txLen]) + rbuf := bytes.NewBuffer(txbuf) + + var tx btcwire.MsgTx + err = tx.Deserialize(rbuf) + if err != nil { + log.Warnf("unable to decode tx block %v %v txoff %v txlen %v", + blkHeight, blksha, txOff, txLen) + return + } + + return &tx, txbuf, pver, blksha, blkHeight, txspent, nil +} + +// FetchTxAllBySha returns several pieces of data regarding the given sha. +func (db *LevelDb) FetchTxAllBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rtxbuf []byte, rpver uint32, rblksha *btcwire.ShaHash, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + tx, txbuf, pver, blksha, _, _, err := db.fetchTxDataBySha(txsha) + + return tx, txbuf, pver, blksha, nil +} + +// FetchTxBySha returns some data for the given Tx Sha. +func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rpver uint32, blksha *btcwire.ShaHash, err error) { + rtx, _, rpver, blksha, err = db.FetchTxAllBySha(txsha) + return +} + +// FetchTxBufBySha return the bytestream data and associated protocol version. +// for the given Tx Sha +func (db *LevelDb) FetchTxBufBySha(txsha *btcwire.ShaHash) (txbuf []byte, rpver uint32, err error) { + _, txbuf, rpver, _, err = db.FetchTxAllBySha(txsha) + return +} + +// InvalidateTxCache clear/release all cached transactions. +func (db *LevelDb) InvalidateTxCache() { +} + +// InvalidateTxCache clear/release all cached blocks. +func (db *LevelDb) InvalidateBlockCache() { +} + +// InvalidateCache clear/release all cached blocks and transactions. +func (db *LevelDb) InvalidateCache() { +} diff --git a/ldb/dbtest/dbtst.go b/ldb/dbtest/dbtst.go new file mode 100644 index 00000000..15d905bb --- /dev/null +++ b/ldb/dbtest/dbtst.go @@ -0,0 +1,56 @@ +// +package main + +import ( + "fmt" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/opt" +) + +type tst struct { + key int + value string +} + +var dataset = []tst{ + //var dataset = []struct { key int, value string } { + {1, "one"}, + {2, "two"}, + {3, "three"}, + {4, "four"}, + {5, "five"}, +} + +func main() { + + ro := &opt.ReadOptions{} + wo := &opt.WriteOptions{} + + ldb, err := leveldb.OpenFile("dbfile", &opt.Options{Flag: opt.OFCreateIfMissing}) + if err != nil { + fmt.Printf("db open failed %v\n", err) + return + } + + batch := new(leveldb.Batch) + for _, datum := range dataset { + key := fmt.Sprintf("%v", datum.key) + batch.Put([]byte(key), []byte(datum.value)) + } + err = ldb.Write(batch, wo) + + for _, datum := range dataset { + key := fmt.Sprintf("%v", datum.key) + data, err := ldb.Get([]byte(key), ro) + + if err != nil { + fmt.Printf("db read failed %v\n", err) + } + + if string(data) != datum.value { + fmt.Printf("mismatched data from db key %v val %v db %v", key, datum.value, data) + } + } + fmt.Printf("completed\n") + ldb.Close() +} diff --git a/ldb/doc.go b/ldb/doc.go new file mode 100644 index 00000000..65906a93 --- /dev/null +++ b/ldb/doc.go @@ -0,0 +1,15 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +/* +Package sqlite3 implements a sqlite3 instance of btcdb. + +sqlite provides a zero setup, single file database. It requires cgo +and the presence of the sqlite library and headers, but nothing else. +The performance is generally high although it goes down with database +size. + +Many of the block or tx specific functions for btcdb are in this subpackage. +*/ +package ldb diff --git a/ldb/insertremove_test.go b/ldb/insertremove_test.go new file mode 100644 index 00000000..c6b6cafa --- /dev/null +++ b/ldb/insertremove_test.go @@ -0,0 +1,202 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ldb_test + +import ( + "github.com/conformal/btcdb" + _ "github.com/conformal/btcdb/ldb" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" + "os" + "path/filepath" + "testing" +) + +var tstBlocks []*btcutil.Block + +func loadblocks(t *testing.T) []*btcutil.Block { + if len(tstBlocks) != 0 { + return tstBlocks + } + + testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + blocks, err := loadBlocks(t, testdatafile) + if err != nil { + t.Errorf("Unable to load blocks from test data: %v", err) + return nil + } + tstBlocks = blocks + return blocks +} + +func TestUnspentInsert(t *testing.T) { + testUnspentInsert(t, dbTmDefault) + testUnspentInsert(t, dbTmNormal) + testUnspentInsert(t, dbTmFast) +} + +// insert every block in the test chain +// after each insert, fetch all the tx affected by the latest +// block and verify that the the tx is spent/unspent +// new tx should be fully unspent, referenced tx should have +// the associated txout set to spent. +func testUnspentInsert(t *testing.T, mode int) { + // Ignore db remove errors since it means we didn't have an old one. + dbname := "tstdbuspnt1" + _ = os.RemoveAll(dbname) + db, err := btcdb.CreateDB("leveldb", dbname) + if err != nil { + t.Errorf("Failed to open test database %v", err) + return + } + defer os.RemoveAll(dbname) + defer db.Close() + + switch mode { + case dbTmDefault: // default + // no setup + case dbTmNormal: // explicit normal + db.SetDBInsertMode(btcdb.InsertNormal) + case dbTmFast: // fast mode + + case dbTmNoVerify: // validated block + t.Errorf("UnspentInsert test is not valid in NoVerify mode") + } + + // Since we are dealing with small dataset, reduce cache size + + blocks := loadblocks(t) +endtest: + for height := int64(0); height < int64(len(blocks)); height++ { + + block := blocks[height] + // look up inputs to this x + mblock := block.MsgBlock() + var txneededList []*btcwire.ShaHash + var txlookupList []*btcwire.ShaHash + var txOutList []*btcwire.ShaHash + var txInList []*btcwire.OutPoint + for _, tx := range mblock.Transactions { + for _, txin := range tx.TxIn { + if txin.PreviousOutpoint.Index == uint32(4294967295) { + continue + } + origintxsha := &txin.PreviousOutpoint.Hash + + txInList = append(txInList, &txin.PreviousOutpoint) + txneededList = append(txneededList, origintxsha) + txlookupList = append(txlookupList, origintxsha) + + if !db.ExistsTxSha(origintxsha) { + t.Errorf("referenced tx not found %v ", origintxsha) + } + + } + txshaname, _ := tx.TxSha() + txlookupList = append(txlookupList, &txshaname) + txOutList = append(txOutList, &txshaname) + } + + txneededmap := map[btcwire.ShaHash]*btcdb.TxListReply{} + txlist := db.FetchTxByShaList(txneededList) + for _, txe := range txlist { + if txe.Err != nil { + t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) + break endtest + } + txneededmap[*txe.Sha] = txe + } + for _, spend := range txInList { + itxe := txneededmap[spend.Hash] + if itxe.TxSpent[spend.Index] == true { + t.Errorf("txin %v:%v is already spent", spend.Hash, spend.Index) + } + } + + newheight, err := db.InsertBlock(block) + if err != nil { + t.Errorf("failed to insert block %v err %v", height, err) + break endtest + } + if newheight != height { + t.Errorf("height mismatch expect %v returned %v", height, newheight) + break endtest + } + + txlookupmap := map[btcwire.ShaHash]*btcdb.TxListReply{} + txlist = db.FetchTxByShaList(txlookupList) + for _, txe := range txlist { + if txe.Err != nil { + t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) + break endtest + } + txlookupmap[*txe.Sha] = txe + } + for _, spend := range txInList { + itxe := txlookupmap[spend.Hash] + if itxe.TxSpent[spend.Index] == false { + t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent) + } + } + for _, txo := range txOutList { + itxe := txlookupmap[*txo] + for i, spent := range itxe.TxSpent { + if spent == true { + t.Errorf("freshly inserted tx %v already spent %v", txo, i) + } + } + + } + if len(txInList) == 0 { + continue + } + dropblock := blocks[height-1] + dropsha, _ := dropblock.Sha() + + err = db.DropAfterBlockBySha(dropsha) + if err != nil { + t.Errorf("failed to drop block %v err %v", height, err) + break endtest + } + + txlookupmap = map[btcwire.ShaHash]*btcdb.TxListReply{} + txlist = db.FetchTxByShaList(txlookupList) + for _, txe := range txlist { + if txe.Err != nil { + if _, ok := txneededmap[*txe.Sha]; ok { + t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) + break endtest + } + } + txlookupmap[*txe.Sha] = txe + } + for _, spend := range txInList { + itxe := txlookupmap[spend.Hash] + if itxe.TxSpent[spend.Index] == true { + t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent) + } + } + newheight, err = db.InsertBlock(block) + if err != nil { + t.Errorf("failed to insert block %v err %v", height, err) + break endtest + } + txlookupmap = map[btcwire.ShaHash]*btcdb.TxListReply{} + txlist = db.FetchTxByShaList(txlookupList) + for _, txe := range txlist { + if txe.Err != nil { + t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) + break endtest + } + txlookupmap[*txe.Sha] = txe + } + for _, spend := range txInList { + itxe := txlookupmap[spend.Hash] + if itxe.TxSpent[spend.Index] == false { + t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent) + } + } + } +} diff --git a/ldb/internal_test.go b/ldb/internal_test.go new file mode 100644 index 00000000..3d6cabf3 --- /dev/null +++ b/ldb/internal_test.go @@ -0,0 +1,24 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ldb + +import ( + "fmt" + "github.com/conformal/btcdb" + "github.com/conformal/btcwire" +) + +// FetchSha returns the datablock and pver for the given ShaHash. +// This is a testing only interface. +func FetchSha(db btcdb.Db, sha *btcwire.ShaHash) (buf []byte, pver uint32, + blkid int64, err error) { + sqldb, ok := db.(*LevelDb) + if !ok { + err = fmt.Errorf("Invalid data type") + return + } + buf, blkid, err = sqldb.fetchSha(sha) + return +} diff --git a/ldb/leveldb.go b/ldb/leveldb.go new file mode 100644 index 00000000..6605b368 --- /dev/null +++ b/ldb/leveldb.go @@ -0,0 +1,528 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ldb + +import ( + "fmt" + "github.com/conformal/btcdb" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" + "github.com/conformal/seelog" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/opt" + "os" + "path/filepath" + "runtime" + "sync" +) + +const ( + dbVersion int = 2 + dbMaxTransCnt = 20000 + dbMaxTransMem = 64 * 1024 * 1024 // 64 MB +) + +var log seelog.LoggerInterface = seelog.Disabled + +type tBlockInsertData struct { + sha btcwire.ShaHash + pver uint32 + buf []byte +} +type tTxInsertData struct { + txsha *btcwire.ShaHash + blockid int64 + txoff int + txlen int + usedbuf []byte +} + +type LevelDb struct { + // lock preventing multiple entry + dbLock sync.Mutex + + // leveldb pieces + bShaDb *leveldb.DB + bBlkDb *leveldb.DB + tShaDb *leveldb.DB + ro *opt.ReadOptions + wo *opt.WriteOptions + + bShabatch *leveldb.Batch + bBlkbatch *leveldb.Batch + + nextBlock int64 + + lastBlkShaCached bool + lastBlkSha btcwire.ShaHash + lastBlkIdx int64 + + txUpdateMap map[btcwire.ShaHash]*txUpdateObj +} + +var self = btcdb.DriverDB{DbType: "leveldb", Create: CreateDB, Open: OpenDB} + +func init() { + btcdb.AddDBDriver(self) +} + +// OpenDB opens an existing database for use. +func OpenDB(dbpath string) (btcdb.Db, error) { + log = btcdb.GetLog() + + db, err := openDB(dbpath, 0) + if err != nil { + return nil, err + } + log.Info("Opening DB\n") + + // Need to find last block and tx + + var lastknownblock, nextunknownblock, testblock int64 + + increment := int64(100000) + ldb := db.(*LevelDb) + + var lastSha *btcwire.ShaHash + // forward scan +blockforward: + for { + + sha, _, err := ldb.getBlkByHeight(testblock) + if err == nil { + // block is found + lastSha = sha + lastknownblock = testblock + testblock += increment + } else { + if testblock == 0 { + //no blocks in db, odd but ok. + return db, nil + } + nextunknownblock = testblock + break blockforward + } + } + + // narrow search +blocknarrow: + for { + testblock = (lastknownblock + nextunknownblock) / 2 + sha, _, err := ldb.getBlkByHeight(testblock) + if err == nil { + lastknownblock = testblock + lastSha = sha + } else { + nextunknownblock = testblock + } + if lastknownblock+1 == nextunknownblock { + break blocknarrow + } + } + + ldb.lastBlkSha = *lastSha + ldb.lastBlkIdx = lastknownblock + ldb.nextBlock = lastknownblock + 1 + + log.Info("Opening DB: completed\n") + + return db, nil +} + +func openDB(dbpath string, flag opt.OptionsFlag) (pbdb btcdb.Db, err error) { + var db LevelDb + var tbShaDb, ttShaDb, tbBlkDb *leveldb.DB + defer func() { + if err == nil { + db.bShaDb = tbShaDb + db.bBlkDb = tbBlkDb + db.tShaDb = ttShaDb + + db.txUpdateMap = map[btcwire.ShaHash]*txUpdateObj{} + + pbdb = &db + } + }() + + if flag&opt.OFCreateIfMissing == opt.OFCreateIfMissing { + err = os.Mkdir(dbpath, 0750) + if err != nil { + log.Errorf("mkdir failed %v %v", dbpath, err) + return + } + } else { + _, err = os.Stat(dbpath) + if err != nil { + err = btcdb.DbDoesNotExist + return + } + } + + bShaName := filepath.Join(dbpath, "bSha.ldb") + tbShaDb, err = leveldb.OpenFile(bShaName, &opt.Options{Flag: flag}) + if err != nil { + return + } + bBlkName := filepath.Join(dbpath, "bBlk.ldb") + tbBlkDb, err = leveldb.OpenFile(bBlkName, &opt.Options{Flag: flag}) + if err != nil { + return + } + tShaName := filepath.Join(dbpath, "tSha.ldb") + ttShaDb, err = leveldb.OpenFile(tShaName, &opt.Options{Flag: flag}) + if err != nil { + return + } + + return +} + +// CreateDB creates, initializes and opens a database for use. +func CreateDB(dbpath string) (btcdb.Db, error) { + log = btcdb.GetLog() + + // No special setup needed, just OpenBB + return openDB(dbpath, opt.OFCreateIfMissing) +} + +func (db *LevelDb) close() { + db.bShaDb.Close() + db.bBlkDb.Close() + db.tShaDb.Close() +} + +// Sync verifies that the database is coherent on disk, +// and no outstanding transactions are in flight. +func (db *LevelDb) Sync() { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + // while specified by the API, does nothing + // however does grab lock to verify it does not return until other operations are complete. +} + +// Close cleanly shuts down database, syncing all data. +func (db *LevelDb) Close() { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + db.close() +} + +// DropAfterBlockBySha will remove any blocks from the database after +// the given block. +func (db *LevelDb) DropAfterBlockBySha(sha *btcwire.ShaHash) error { + db.dbLock.Lock() + defer db.dbLock.Unlock() + defer db.processBatches() + + startheight := db.nextBlock - 1 + + keepidx, err := db.getBlkLoc(sha) + if err != nil { + // should the error here be normalized ? + log.Infof("block loc failed %v ", sha) + return err + } + + for height := startheight; height > keepidx; height = height - 1 { + var blk *btcutil.Block + blksha, buf, err := db.getBlkByHeight(height) + if err != nil { + return err + } + blk, err = btcutil.NewBlockFromBytes(buf) + if err != nil { + return err + } + + for _, tx := range blk.MsgBlock().Transactions { + err = db.unSpend(tx) + if err != nil { + return err + } + } + // rather than iterate the list of tx backward, do it twice. + for _, tx := range blk.MsgBlock().Transactions { + txSha, _ := tx.TxSha() + var txUo txUpdateObj + txUo.delete = true + db.txUpdateMap[txSha] = &txUo + } + db.bShaBatch().Delete(shaToKey(blksha)) + db.bBlkBatch().Delete(int64ToKey(height)) + } + + db.nextBlock = keepidx + 1 + + return nil +} + +// InsertBlock inserts raw block and transaction data from a block into the +// database. The first block inserted into the database will be treated as the +// genesis block. Every subsequent block insert requires the referenced parent +// block to already exist. +func (db *LevelDb) InsertBlock(block *btcutil.Block) (height int64, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + defer db.processBatches() + + blocksha, err := block.Sha() + if err != nil { + log.Warnf("Failed to compute block sha %v", blocksha) + return + } + mblock := block.MsgBlock() + rawMsg, err := block.Bytes() + if err != nil { + log.Warnf("Failed to obtain raw block sha %v", blocksha) + return + } + txloc, err := block.TxLoc() + if err != nil { + log.Warnf("Failed to obtain raw block sha %v", blocksha) + return + } + + // Insert block into database + newheight, err := db.insertBlockData(blocksha, &mblock.Header.PrevBlock, + rawMsg) + if err != nil { + log.Warnf("Failed to insert block %v %v %v", blocksha, + &mblock.Header.PrevBlock, err) + return + } + + // At least two blocks in the long past were generated by faulty + // miners, the sha of the transaction exists in a previous block, + // detect this condition and 'accept' the block. + for txidx, tx := range mblock.Transactions { + var txsha btcwire.ShaHash + txsha, err = tx.TxSha() + if err != nil { + log.Warnf("failed to compute tx name block %v idx %v err %v", blocksha, txidx, err) + return + } + // Some old blocks contain duplicate transactions + // Attempt to cleanly bypass this problem + // http://blockexplorer.com/b/91842 + // http://blockexplorer.com/b/91880 + if newheight == 91842 { + dupsha, err := btcwire.NewShaHashFromStr("d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599") + if err != nil { + panic("invalid sha string in source") + } + if txsha == *dupsha { + //log.Tracef("skipping sha %v %v", dupsha, newheight) + continue + } + } + if newheight == 91880 { + dupsha, err := btcwire.NewShaHashFromStr("e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468") + if err != nil { + panic("invalid sha string in source") + } + if txsha == *dupsha { + //log.Tracef("skipping sha %v %v", dupsha, newheight) + continue + } + } + spentbuflen := (len(tx.TxOut) + 7) / 8 + spentbuf := make([]byte, spentbuflen, spentbuflen) + if len(tx.TxOut)%8 != 0 { + for i := uint(len(tx.TxOut) % 8); i < 8; i++ { + spentbuf[spentbuflen-1] |= (byte(1) << i) + } + } + + err = db.insertTx(&txsha, newheight, txloc[txidx].TxStart, txloc[txidx].TxLen, spentbuf) + if err != nil { + log.Warnf("block %v idx %v failed to insert tx %v %v err %v", blocksha, newheight, &txsha, txidx, err) + + return + } + err = db.doSpend(tx) + if err != nil { + log.Warnf("block %v idx %v failed to spend tx %v %v err %v", blocksha, newheight, &txsha, txidx, err) + + return + } + } + return newheight, nil +} + +// SetDBInsertMode provides hints to the database to how the application +// is running this allows the database to work in optimized modes when the +// database may be very busy. +func (db *LevelDb) SetDBInsertMode(newmode btcdb.InsertMode) { + + // special modes are not supported +} + +// doSpend iterates all TxIn in a bitcoin transaction marking each associated +// TxOut as spent. +func (db *LevelDb) doSpend(tx *btcwire.MsgTx) error { + for txinidx := range tx.TxIn { + txin := tx.TxIn[txinidx] + + inTxSha := txin.PreviousOutpoint.Hash + inTxidx := txin.PreviousOutpoint.Index + + if inTxidx == ^uint32(0) { + continue + } + + //log.Infof("spending %v %v", &inTxSha, inTxidx) + + err := db.setSpentData(&inTxSha, inTxidx) + if err != nil { + return err + } + } + return nil +} + +// unSpend iterates all TxIn in a bitcoin transaction marking each associated +// TxOut as unspent. +func (db *LevelDb) unSpend(tx *btcwire.MsgTx) error { + for txinidx := range tx.TxIn { + txin := tx.TxIn[txinidx] + + inTxSha := txin.PreviousOutpoint.Hash + inTxidx := txin.PreviousOutpoint.Index + + if inTxidx == ^uint32(0) { + continue + } + + err := db.clearSpentData(&inTxSha, inTxidx) + if err != nil { + return err + } + } + return nil +} + +func (db *LevelDb) setSpentData(sha *btcwire.ShaHash, idx uint32) error { + return db.setclearSpentData(sha, idx, true) +} + +func (db *LevelDb) clearSpentData(sha *btcwire.ShaHash, idx uint32) error { + return db.setclearSpentData(sha, idx, false) +} + +func (db *LevelDb) setclearSpentData(txsha *btcwire.ShaHash, idx uint32, set bool) error { + var txUo *txUpdateObj + var ok bool + + if txUo, ok = db.txUpdateMap[*txsha]; !ok { + // not cached, load from db + var txU txUpdateObj + blkHeight, txOff, txLen, spentData, err := db.getTxData(txsha) + if err != nil { + return err + } + + txU.txSha = txsha + txU.blkHeight = blkHeight + txU.txoff = txOff + txU.txlen = txLen + txU.spentData = spentData + + txUo = &txU + } + + byteidx := idx / 8 + byteoff := idx % 8 + + if set { + txUo.spentData[byteidx] |= (byte(1) << byteoff) + } else { + txUo.spentData[byteidx] &= ^(byte(1) << byteoff) + } + + db.txUpdateMap[*txsha] = txUo + + return nil +} + +func intToKey(keyint int) []byte { + key := fmt.Sprintf("%d", keyint) + return []byte(key) +} +func int64ToKey(keyint int64) []byte { + key := fmt.Sprintf("%d", keyint) + return []byte(key) +} + +func shaToKey(sha *btcwire.ShaHash) []byte { + return sha.Bytes() +} + +func (db *LevelDb) bShaBatch() *leveldb.Batch { + if db.bShabatch == nil { + db.bShabatch = new(leveldb.Batch) + } + return db.bShabatch +} + +func (db *LevelDb) bBlkBatch() *leveldb.Batch { + if db.bBlkbatch == nil { + db.bBlkbatch = new(leveldb.Batch) + } + return db.bBlkbatch +} + +func (db *LevelDb) processBatches() { + var err error + if db.bShabatch != nil { + err = db.bShaDb.Write(db.bShabatch, db.wo) + db.bShabatch.Reset() + db.bShabatch = nil + if err != nil { + return + } + } + if db.bBlkbatch != nil { + err = db.bBlkDb.Write(db.bBlkbatch, db.wo) + db.bBlkbatch.Reset() + db.bBlkbatch = nil + if err != nil { + return + } + } + + if len(db.txUpdateMap) != 0 { + tShabatch := new(leveldb.Batch) + + for txSha, txU := range db.txUpdateMap { + key := shaToKey(&txSha) + if txU.delete { + //log.Infof("deleting tx %v", txSha) + tShabatch.Delete(key) + } else { + //log.Infof("inserting tx %v", txSha) + txdat, err := db.formatTx(txU) + if err != nil { + return + } + tShabatch.Put(key, txdat) + } + } + + err = db.tShaDb.Write(tShabatch, db.wo) + tShabatch.Reset() + if err != nil { + return + } + db.txUpdateMap = map[btcwire.ShaHash]*txUpdateObj{} + runtime.GC() + } + +} + +func (db *LevelDb) RollbackClose() { + db.close() +} diff --git a/ldb/operational_test.go b/ldb/operational_test.go new file mode 100644 index 00000000..1000bd66 --- /dev/null +++ b/ldb/operational_test.go @@ -0,0 +1,394 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ldb_test + +import ( + "compress/bzip2" + "encoding/binary" + "github.com/conformal/btcdb" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +var network = btcwire.MainNet + +const ( + dbTmDefault = iota + dbTmNormal + dbTmFast + dbTmNoVerify +) + +func TestOperational(t *testing.T) { + testOperationalMode(t, dbTmDefault) + testOperationalMode(t, dbTmNormal) + testOperationalMode(t, dbTmFast) + testOperationalMode(t, dbTmNoVerify) +} + +func testOperationalMode(t *testing.T, mode int) { + // simplified basic operation is: + // 1) fetch block from remote server + // 2) look up all txin (except coinbase in db) + // 3) insert block + + // Ignore db remove errors since it means we didn't have an old one. + dbname := "tstdbop1" + _ = os.RemoveAll(dbname) + db, err := btcdb.CreateDB("leveldb", dbname) + if err != nil { + t.Errorf("Failed to open test database %v", err) + return + } + defer os.RemoveAll(dbname) + defer db.Close() + + switch mode { + case dbTmDefault: // default + // no setup + case dbTmNormal: // explicit normal + db.SetDBInsertMode(btcdb.InsertNormal) + case dbTmFast: // fast mode + + case dbTmNoVerify: // validated block + db.SetDBInsertMode(btcdb.InsertValidatedInput) + } + + testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + blocks, err := loadBlocks(t, testdatafile) + if err != nil { + t.Errorf("Unable to load blocks from test data for mode %v: %v", + mode, err) + return + } + + err = nil +out: + for height := int64(0); height < int64(len(blocks)); height++ { + block := blocks[height] + if mode != dbTmNoVerify { + // except for NoVerify which does not allow lookups check inputs + mblock := block.MsgBlock() + var txneededList []*btcwire.ShaHash + for _, tx := range mblock.Transactions { + for _, txin := range tx.TxIn { + if txin.PreviousOutpoint.Index == uint32(4294967295) { + continue + } + origintxsha := &txin.PreviousOutpoint.Hash + txneededList = append(txneededList, origintxsha) + + if !db.ExistsTxSha(origintxsha) { + t.Errorf("referenced tx not found %v ", origintxsha) + } + + _, _, _, _, err := db.FetchTxAllBySha(origintxsha) + if err != nil { + t.Errorf("referenced tx not found %v err %v ", origintxsha, err) + } + _, _, _, _, err = db.FetchTxAllBySha(origintxsha) + if err != nil { + t.Errorf("referenced tx not found %v err %v ", origintxsha, err) + } + _, _, _, err = db.FetchTxBySha(origintxsha) + if err != nil { + t.Errorf("referenced tx not found %v err %v ", origintxsha, err) + } + _, _, err = db.FetchTxBufBySha(origintxsha) + if err != nil { + t.Errorf("referenced tx not found %v err %v ", origintxsha, err) + } + _, err = db.FetchTxUsedBySha(origintxsha) + if err != nil { + t.Errorf("tx used fetch fail %v err %v ", origintxsha, err) + } + } + } + txlist := db.FetchTxByShaList(txneededList) + for _, txe := range txlist { + if txe.Err != nil { + t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) + break out + } + } + + } + + newheight, err := db.InsertBlock(block) + if err != nil { + t.Errorf("failed to insert block %v err %v", height, err) + break out + } + if newheight != height { + t.Errorf("height mismatch expect %v returned %v", height, newheight) + break out + } + + newSha, blkid, err := db.NewestSha() + if err != nil { + t.Errorf("failed to obtain latest sha %v %v", height, err) + } + + if blkid != height { + t.Errorf("height doe not match latest block height %v %v", blkid, height, err) + } + + blkSha, _ := block.Sha() + if *newSha != *blkSha { + t.Errorf("Newest block sha does not match freshly inserted one %v %v ", newSha, blkSha, err) + } + } + + // now that db is populated, do some additional test + testFetchRangeHeight(t, db, blocks) + + switch mode { + case dbTmDefault: // default + // no cleanup + case dbTmNormal: // explicit normal + // no cleanup + case dbTmFast: // fast mode + db.SetDBInsertMode(btcdb.InsertNormal) + case dbTmNoVerify: // validated block + db.SetDBInsertMode(btcdb.InsertNormal) + } +} + +func TestBackout(t *testing.T) { + testBackout(t, dbTmDefault) + testBackout(t, dbTmNormal) + testBackout(t, dbTmFast) +} + +func testBackout(t *testing.T, mode int) { + // simplified basic operation is: + // 1) fetch block from remote server + // 2) look up all txin (except coinbase in db) + // 3) insert block + + // Ignore db remove errors since it means we didn't have an old one. + dbname := "tstdbop2" + _ = os.RemoveAll(dbname) + db, err := btcdb.CreateDB("leveldb", dbname) + if err != nil { + t.Errorf("Failed to open test database %v", err) + return + } + defer os.RemoveAll(dbname) + defer db.Close() + + switch mode { + case dbTmDefault: // default + // no setup + case dbTmNormal: // explicit normal + db.SetDBInsertMode(btcdb.InsertNormal) + case dbTmFast: // fast mode + + } + + testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + blocks, err := loadBlocks(t, testdatafile) + if len(blocks) < 120 { + t.Errorf("test data too small") + return + } + + err = nil + for height := int64(0); height < int64(len(blocks)); height++ { + if height == 100 { + t.Logf("Syncing at block height 100") + db.Sync() + } + if height == 120 { + t.Logf("Simulating unexpected application quit") + // Simulate unexpected application quit + db.RollbackClose() + break + } + + block := blocks[height] + + newheight, err := db.InsertBlock(block) + if err != nil { + t.Errorf("failed to insert block %v err %v", height, err) + break + } + if newheight != height { + t.Errorf("height mismatch expect %v returned %v", height, newheight) + break + } + } + + // db was closed at height 120, so no cleanup is possible. + + // reopen db + db, err = btcdb.OpenDB("leveldb", dbname) + if err != nil { + t.Errorf("Failed to open test database %v", err) + return + } + defer db.Close() + + sha, err := blocks[99].Sha() + if err != nil { + t.Errorf("failed to get block 99 sha err %v", err) + return + } + _ = db.ExistsSha(sha) + _, err = db.FetchBlockBySha(sha) + if err != nil { + t.Errorf("failed to load block 99 from db %v", err) + } + + sha, err = blocks[119].Sha() + if err != nil { + t.Errorf("failed to get block 110 sha err %v", err) + return + } + _ = db.ExistsSha(sha) + _, err = db.FetchBlockBySha(sha) + if err != nil { + t.Errorf("loaded block 119 from db") + return + } + + block := blocks[119] + mblock := block.MsgBlock() + txsha, err := mblock.Transactions[0].TxSha() + exists := db.ExistsTxSha(&txsha) + if !exists { + t.Errorf("tx %v not located db\n", txsha) + } + + _, _, _, err = db.FetchTxBySha(&txsha) + if err != nil { + t.Errorf("tx %v not located db\n", txsha) + return + } + _, err = db.FetchTxUsedBySha(&txsha) + if err != nil { + t.Errorf("tx %v not located db\n", txsha) + return + } +} + +func loadBlocks(t *testing.T, file string) (blocks []*btcutil.Block, err error) { + testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + var dr io.Reader + var fi io.ReadCloser + fi, err = os.Open(testdatafile) + if err != nil { + t.Errorf("failed to open file %v, err %v", testdatafile, err) + return + } + if strings.HasSuffix(testdatafile, ".bz2") { + z := bzip2.NewReader(fi) + dr = z + } else { + dr = fi + } + + defer func() { + if err := fi.Close(); err != nil { + t.Errorf("failed to close file %v %v", testdatafile, err) + } + }() + + // Set the first block as the genesis block. + genesis := btcutil.NewBlock(&btcwire.GenesisBlock) + blocks = append(blocks, genesis) + + var block *btcutil.Block + err = nil + for height := int64(1); err == nil; height++ { + var rintbuf uint32 + err = binary.Read(dr, binary.LittleEndian, &rintbuf) + if err == io.EOF { + // hit end of file at expected offset: no warning + height-- + err = nil + break + } + if err != nil { + t.Errorf("failed to load network type, err %v", err) + break + } + if rintbuf != uint32(network) { + t.Errorf("Block doesn't match network: %v expects %v", + rintbuf, network) + break + } + err = binary.Read(dr, binary.LittleEndian, &rintbuf) + blocklen := rintbuf + + rbytes := make([]byte, blocklen) + + // read block + dr.Read(rbytes) + + block, err = btcutil.NewBlockFromBytes(rbytes) + if err != nil { + t.Errorf("failed to parse block %v", height) + return + } + blocks = append(blocks, block) + } + return +} + +func testFetchRangeHeight(t *testing.T, db btcdb.Db, blocks []*btcutil.Block) { + + var testincrement int64 = 50 + var testcnt int64 = 100 + + shanames := make([]*btcwire.ShaHash, len(blocks)) + + nBlocks := int64(len(blocks)) + + for i := range blocks { + blockSha, err := blocks[i].Sha() + if err != nil { + t.Errorf("FetchRangeHeight: unexpected failure computing block sah %v", err) + } + shanames[i] = blockSha + } + + for startheight := int64(0); startheight < nBlocks; startheight += testincrement { + endheight := startheight + testcnt + + if endheight > nBlocks { + endheight = btcdb.AllShas + } + + shalist, err := db.FetchHeightRange(startheight, endheight) + if err != nil { + t.Errorf("FetchRangeHeight: unexpected failure looking up shas %v", err) + } + + if endheight == btcdb.AllShas { + if int64(len(shalist)) != nBlocks-startheight { + t.Errorf("FetchRangeHeight: expected A %v shas, got %v", nBlocks-startheight, len(shalist)) + } + } else { + if int64(len(shalist)) != testcnt { + t.Errorf("FetchRangeHeight: expected %v shas, got %v", testcnt, len(shalist)) + } + } + + for i := range shalist { + sha0 := *shanames[int64(i)+startheight] + sha1 := shalist[i] + if sha0 != sha1 { + t.Errorf("FetchRangeHeight: mismatch sha at %v requested range %v %v: %v %v ", int64(i)+startheight, startheight, endheight, sha0, sha1) + } + } + } + +} diff --git a/ldb/testdata/blocks1-256.bz2 b/ldb/testdata/blocks1-256.bz2 new file mode 100644 index 0000000000000000000000000000000000000000..6b8bda4429200c0566bb13c28c35d6397272e475 GIT binary patch literal 37555 zcmV)8K*qm9T4*^jL0KkKSr@A=?*Lyo|NsC0|NsC0|NsC0|NsC0|NsC0|NsC0|NsC0 z|NsC0|Nr1%0`GtT00*bHz1|n5T)=M=pBnqU;dPxJy+dC1_BM0g?t47Wcf|GKdY#;B zx%Jb_&i2~vj_uc8@w++d_r2Xc+bQeWwB7AnH@m&s?|Z7VcLwh6_&)Ua-C7%VGQQc~ z>${h|?$>*(yBXJ;E3Q`cd%f+uRJ}XA?=#ch z^LF0cm$dfxz3JWU#m@D$*Rj3c>USOO?{`dUw52*PZ4) zZsq3L?C!nYb#Ajg-1geucDp>i-Q{VQx3xUI&gD6`Uawu6^=;dCb=vn_^?SRn>aKNl zE$pv*o-NzE&uh^0y|K@GzIV4+$bmfo002w?nqp#L0%cFrLlY(dm;eaCCJBH5N#bAt zO*GR4VqgTqU=ssCXc_@FgCjbAaCIAVQ4yFcx0gyDQ{z0Gs4FX^(;0dOf69QlWOd2#a z4FC-Q0fYo#1w0VRlT9>W35k^S(-Q<>m=TeQsvT87z%&|Zsek|g&_ zj3y%jGfhmMr>UAIk3kuxqef2@z$c;x$u^Pc4@lCq5G;R}q9*?O-;!gJMnwV1tNmyb zJ@fx!pSG8@UQWKV&A3<1{kZJNqQ08k z6D?(C9~^K8No6azD{saS5up0CIL)USziD_I?C)=&!gQRRl1VS{+J?XC_#bbw?xgcw zjlnWQhmuJmK$1=Nq>@QeY{iyDRa8VqDzR1~h>SsCD;S`vMFkaz#a1j>F^Y@@ixB}8 zR7F^*f~;V~WLU;U0gMLL5g4e*#6~J23L=XU6%+sgVk!!%C^905f(&B?Rz?W0j2J=! zkz&A5Bv=9>DkzG^F;zul6^s@Nf+(vMV51aRs)#D6ixx2yRYe3*Q4s|eF+^2@v4W~9 z3lL()AgmOMip7Gf76_n>7{x?j!Bi0#Dx^^qh$zHCR8?TA005$jqA-Mjq=2G;p(GR_ zERkRYhGl^yhX}|(fNQV-4dYK>n#ItplMB?#{#=cV!a#-X6#_w-Hww)~IQDocyC!q_ zP;QC6GWpfg{SRTvVKK%&;D83JAV?@cB9MTfNoG=P`RhZcW`%~VouXmT%%>Q`%gFiG znI{I2!l$s1vkuvTf^(_QFeC#yn%e`wLK@aVJW{_@>IkMu*Tz znP6aPZ9IqEsuH!41X8oO@$DzHuf2m$q6GXXz?)H=) z8Oo<|u~gic8#@ls4^23kc~Cpor9%wuOfhH~@7+O`A-6w1U~OG zKw{EgNhFbmD_5u(-)^8l4f&a^9R_dO&uKNTUsBR*+$IlA*>~OFPl4v}nJizqrf9w) zqe22=T4Ir+f2Wo>`sLWI3k8mhw?H3J4@8NVOJTC0;$We@CRKTKF-3MYHiCQ2x3BU( z)}O1pcK`t1iH=|ZvOy7K09X-1NkIVYB$7zO4T7V2@@RyR?lKpdotDOu@DKn100025 z8j|2a5i)c~hdx1-v5bPtJMzV|CO3sRfYNWYLAI?F)Ks#b?V*}=K)U~Rs~`0}`9jfr zq@K;^8mYzZ93voGNFdj?bzU>-R^QcyxFYAQ$zjzXUTxHHURb0-{WBrwQ(ZhX(djB$ z`|{b~h9T^fOv9~7I81ym3u$cwRR4{rh|WgWszLs;K?0NUxzv@B&3M~d!H89)Ki8|p zAx&K5^LeTg6__5a@+Dnc!orx%+h*yBU8H0mz)})hyp3;X`sbgdLE`m z?DM(0Suen3X-SJQGFHiJ0yCcr?|5*sFDL6h7#NVcZP%#TI4$qq*)#tOc#{r?EQZ$3 z@2aWLp$#GztTlv&s>7+oXsj0TsvJKrUA^1=xCTk?v&P>l6P;n&yXO^LN(W_@q8;<+ zrl^L0MY!5`jO_o#dV1AP-7?iu{WwR>bh*tHu{|g2=;Y@7OV2^dGW>g#(m%7hqc51| zzcxELdBOX6%VscLc4GgiyT#Xi{j2Y7xDyeuA-Ni0gT+xi?gf7B=ljdA!DA@CvcVa z0|L+;v>c%L7r?;)(hZ#!uzUkvOYL5>NO9@tvhh0$`rHzF$Y-2yK|#}W7;QdVP@gZ+ zsaaj}*SXBqyKiTPvBhP!4o2f%PFbD9m|>CUYfa|p_$#mW%GO>rXDT<-Io!O+ur6g+)r2!T9)G#e18LaZ{9x-(kfMX?ERZ|fqNIO zME-!VUTi4TBS-t#Yyc}w_xB<;JV`o%DIac@7nw_d$iX@5|TOOcDXyH_y!EDC<-4 zp|dc+urTQkwTFY@)y(Qp%58N!TtBnjTp9Kr7BaYo8#iI4v7ogav*%~Y0v&t_4WK;m zs=m@rZ7B+osQl1Akfx1co1S^%Ikray<`Wy)L8)zO^LY$qI#2!G2Dw%`Dr z5h{+fe|-9E$fmiB08zRveD1Yhcfwjt`wk1Iq2R`X1Q91A_9FYJ^&CP00aaVyE^zPKKH@`iDe(E?g>zB zM}%7D;S$eKl={FXK@rn&5>;QeQly&S%rH!KMK``jv;;EAMXa9wnFszdSnAk@(`fr1 z-F= zHj`aS%|-LLjNoBDsEib}NSwq5JLd6T7693-79 zDa>6lA#reFs2?X-X75sY!}Ai3TB3CwkYdOYjVslXg2Yd1enXi;DDq zzWn4VT)GEKJ7~9}y%gc+VRK3Rba`|;zJX#GA9Sf|N_4kmm@4D}qpUcsJR#}R86^G< zK3?Ss!&$nw>89d_t1|k&(zzOxU|5m0sk1dZMgpjHbS26>7RoL1{!7IW2-RKL0>wZ8 z&XXqs4L;1xknu*uwWRl0<1N;nGu^qkNAlVzq+3u3ApfQF#4g+wm$A|d!l1-_#J;yG zm3x<}PZ#pXr1xG8Mz6(W3cqYjn`{FN9tt7SJ^^!VBXJqHN)y}$W&%S+7+*{gN8(YL z5ohchr<3tZ=0#=0qo6$Obbi|LG=kcor0YLPK%_&LwJHl&e<-7n0%_7Ta;(=}?ZcGW z7T+s3xx;mUgx3d_b@*|}#dzL(mHFl|$h7?z$9%yOaT#1n%`PLi)RS9__>lx%bOy-Akf0XxoAo4bGs7|!&K zVdF4s+eNgQ3=(XFg^_SFGpAPBZD=(#bR(ZH3bbYqvRA4Zo{_Q@@-*w8gy+Zad|ABU zng4o-FaT?4s=bRnwy7h7^VJ49_PuGt6%Aw3^Ipp5ilKx@+Z*_<&m%won)`+4@VZhV zHg!;XUaY*>_#7oW>`w9d$demj4)B-L*Vf+Fq$4Eu;;nfzU&F4!7U$Y;%AhWCArS@4 zB!BjEb@@nDgj}0dN(C~)NpQOJmA$-mN*V2{*!6eX*_INd= z=fDLXe|P-UT#c#q@ZPo1k2PUb7tiD4X_n7hlBX;1ska~8f}x~aVURQY=<+tUzpc8Q zt}=$!+y>*`gZNj!5}6~H{CYxa&DUZ!e6P7Q>tOFrPa3)|*u*W5>(T_uBFMBznE3Ltoe!nHloq_n88!ROprYEC+-9f037NNO1u(LBUw5W@{UzH{ zvCt>#Fg;Vo6hJ%Fop`;K%e8Y>BbS@MN%9|0Q1jv&Cs+^WQ3dArTb$@NUIfB^_Etm{#jDAgQ26YLod%U4$7rV$nkip#Q; zvi&FM<=Wu3y>RFweN0i2m&BIh_6I)3sl3>%#?p>Oi$q6=fLPteH>3$a5%w<^?vPMx92tvP>l`P47rW>_nRV zfTuj5mXN3^1KHqn*R7R%+`@egteS@{*&XrjM}$b7JMs=%nfx8OU+@sd&h}0>q$MBB>Xtm9SQ3y(jn(R)=rk;Q9OyH<*O6iS5u|W4h`{~KFJ0>Vg z-T)so!4so}@3v1B6kS5-Vz>J_5MmX%qd4QCPk{?u2foQxU_6MN!AHoM(wxhl*b;TM z1Vc4s3M`q+D(3}9B?WVa2OXyh?MVLn(58ibwBV0>n8?IKhxzI6M3eipVcjT~Gg@sj zr3x+?rRe|yL6CuI#nExc0uEL0z;~%k$8}GWF4o}+u)v>uH8KGVcw;$W=TtOdp<1JY z>&0-7e74P}{5i5mO07Yj)vr2GA`vMe_~&8aJdLJv*Zvq)>xYGcx>e%fBr_I(A0_M8 z-2LHMP!X^vW`ssRiVCJe8%@HcwnK0tLj!mXR8aCCWjmL9=>r&1a;T&p5CGt3I6#)n zhJ$eIk5qwZ7)WCkTnYj$7IOL0cD_r5wZRr z+F$0YH)+(-acsHH3%sf~mV>de!rMD)&xx*pi?5t-?`=cTH&3+WJ_0#<$&N1s;tVj1 z$dZbT!Ut-R-|uHiCcl-y7L}JrexMik7fubi8A`5AQwV^<3O zvMAAehg72IDOeXh@D!UxlQsEJl=%P>u7uKX`3!JcHnGka85W3nE#PSINg{E0{wD=S zhbu{ICpZI4(!`e|;c+h^v9W_rsI((Fd5G*^&@x1f@g~_SaIHAVGpyN+_`OUdWwfb%tOm<=fAr+k#^ol)(mY~0;e6ub=ZYCNZ#N- zSNQxcn83j??A=7`CZ0T&ugeN+;a%HG&NJe)xG5?EB8&P(r;-b`nFqjTo4=QjzwCje zHv%x^VocPEEI8_gyF}U z9pvK~V+b=B5R9h2qxin_J4Xdc(z@@#F}NTA05l9fPEw?hhAjzx_ADd}+D-w64KyhY zD-2Tu&1aX5;B7dB|7ae%NN8CnroskUKKeuTG8&qB??RczamaLa)$>;Xcns?hoc{q* z9+M4;1OOmB{j6i;?=bEeCj>{!Ndv>{hTuK?$PU_V`q;=oS?|dw)c;^gbCJ@}{&ol( zG}`__Ye=$*XQ*$OKh8J%p~-)M$$^{R1wVz|BB4`wOXN{NrnbI*51{T&F}p(eEGcsd*hV!#dDi#q_s*aTQp>WFk}=m;i!mO@#$~$1$9Q z)DGq=0clhG9_ZIYpO1%oOOXCn9@L{E;4r4!r6aiu+H?x)UIj?Swau?7%vC_yz7;uw zrt09C8M>e143Sp^{62F%Y7cKvioYprA2YA`bO&hOf^B|EM;d3yQ%a&@eJ!T_R8agz zHhPGZCFS`JR!m?Xu<>>N0p^zY0%^=&eE6sE2t;-_DF`HSb^{%BhQ$r;{|s3<%TMHJ=< z2|zw&$9M{Q@6bFxgY)Pr{WDSmS9tx6(qg>iC`@J*O^=qiWa6wb>`x=paOV7;29 zV-v2R9sV)9Z$OA=;{Vh%5`ROu$rsj-RJs|{-y{i0!c%SF3;fZ?K;p6@pI>rjuC#f!b=sjjvC}>NP?9n+E)e&15F=|TC@q-uXa7u-Bg5j0PQo)`j zZ>sI6uXDvqO08=l=3gWk4;h$nL2?eU*{%W%^aTRLTQ5o{K@?D|^S*KJ0afG3xl;<< z`AWFU4nje8|G`DTW0H%|5#FYonpW5RaHNV}l>`v=l>xAdqQ8a;@-ZLES1y)nap^|N zIod@?T5cf9@Xl5fFaWqwn6+$OsG1-tMX10B6x1(glQ5#>XBtB$*!pSG>0O!ob;qZaXL6Ha7B;80j?8 z{}q;zbcgif=%+p44x-ZaH9&k3`(9`qpP1PT9|b4bsqA|LO!Nt_em#gvlaEL4(It8d z{!m={WQ44r9K>E;Th~1_A+S4*pXAl@ald22vvnJl-!F=%~@s9ixdn@!ZJ2y)eN8w!b_^*|_SCkF)X@Lc_ zuw z%2r~joj8{{}al_64lG#3>B!L0}UvOp^dlC?}*6T9gg>t3e3y-T;K&pW}WcNL3cv+tRdd}&IGy{^+PfR0r+x@iY*)P$?01PsOC(oy1 z#nV-XH;RVIrmexLdt5^@oZl~M2Pq|3lTiCp+~(TAv`vbqFsoFHAuLO>0_62)5V};3 zOoU&*!AT*Ssn8y)OE*&%QZ|;1KgR^NDcJ048K=(eRlPU%O$hf`=^u$tJ|lmjoAU@A z22e`ua92W66s3+L0MV-b1)c zp==w?W|Ijsh{zsEg;)*9q1CY2OX;-hEtF)9qaXXP(lKfS@vT8WbO@;Ctn`>awixJ5 zkV?)8pif(N0QQe{=&aezjmMHNo?ETY^11IR!EdZ_!IP58gB@qp<-~N+t*?b>RLn%g zBYyR+@6h4Mn-UrR_R;?*34M45Io7xf_*(Zp^<^_lSHz~6k_BPtIy9ZiK!4^Bs1SEO zL$hL`wZf=(T*2y~oWMIxy6PqMz(N{8P}4W|pB179@)(-XAjQRkU!41R;6Y}dy1iZH zl6bmLiQw?c>}CK2(KdPV8fM1N;+H;Sk9K)H<=O1uWB5zYcRRt=XKd?r9f0mUADj;B z1kD2Xk&Au~#!EdkP#pWw$}6b@&a38XB~2m?1X}Qyg-crDKjGbHl-QA*j?XLP02O3% zDqvOZPs?=4Ii!(#j%Q`3+QV`#&)scQXNmK`XHg!swI9*|7TH>Be-H?P+t3PoGl?WE zNcJP|@|^46ceBYkMF~TM^25yD__nO291fj)$oXZA9*cUzECm{cyBXYR*XKgpffH<&>{;tfFZqBo_tinP;NIski2aIz4{*5|TACBfs!izV=|dyCoG9 zz~!gb{RZ}x*h1SSvXSt_*rdAQ05^tmZPcKq-8^b8<}>mZn^a(S$=>fT_0@k(le0xB zK?dbLJY%G$iM7h{J9;n{$k1uCEb_d0Py8G(rOp>cf|7CMeG@dK{!;U2!)zL+_y}bA z{vZIA0^M#9j!2K%_}_HXFCUCjUg&#;sn(m=v-z5dK1I$IIt6L%GF+~z;sj%>iGs-d ztwx5ygjCKHN1|PuTfJ-I_%#Z|fvfH1>!bv8)_lRy@E429$}Gis61OSMxbSDz90@-Cpc5pAmFzC% zYat}Wlhp$p$6TauFQ=x^D4)icV=O~eLCeNu;Fb$?r*7OHQnkQS^TBhzu)J|9JlHor z?PK}DGx#cRf26S-3hMvZl^6^)kqvEki=jR_iW!vp(FbX$%oxxTSxvk6$5_(ADL%Ez z!Ueg?O*nIv)V)v;ai!VF!R*GG*C)mAP4+8xmZbJMta#t;W_i+fm4b~f*dUjha6a-Y zX2JI?JN?gjWF@k#tSrEV6khcjk}3V3Tp;b)Du|yCd_WE^D{IHURBOmvapg&qb>t+c z@@-?b-+xDKrnuU7Y4peX;Bg%xn0&YA4i$<>dUQ;t1V8EA6A(Fp zwrw(9N>E;KX!cv;Qwg@;0pX!)ETZj*harI_6CDaPvSq9GXW)OGvhu|)!;XB0(TJry zEA$e5YR3r`w()t@f{N}U7$8ZfOgOhgw}fEuKE0U7n@DSRoF=b?(t?0zr{N#2H;AhD zlhh|tg05^rc{EgFH-g3J>NVz27WM8Ok7qs;zO~8!3eqC22PPS(% z$I9%xedtE-HQmt@k77Q@Kf^Bj0K8LJt(EDkiPF#_agy4YO%tOr(y4JEzR4NF6i3%I zJ}zq8*%t9Psk`Bl&Umodgy>{zxN;@qfWNL%(_)wLP8N&oU=lgq9Dqt| z45sReb`|n^m5`3}J#aMnY&tKzZlfj69wy|%qeVR9L9=9aS6VJcn$fsqqGrGZ0c^t! zz!HaD_Brrds;Is!h4AnDj;w$`T$pz~3eJwFW^OlCp|v&VUr_`JrY_fv*wiB*{1HQ> zR9)Qq@r7O+)U>nyKGEI*OcRiEO-mEd2KKBpH?vYs7uhTuHSqby@?Ho|&=lDH8%gut z3izkU4iz>ozAqE>=S(RPr|;P8A|P6gr=^v%d8#W>c|X)r!}J#dn9~_T)~P6(6P%<` zE{)0DChf+U2F04m&PV&3%*-3+4`4C;;48GBRV=7DnNTimgz3Kx^pTbA%!zv9b5y;b z;d{+zn0p^~5_mVrU6ScAev{rYKQeCpCghkx5R^kwq#uCSlazx1u}>%7v5hENn$qC| zI+}DOBOI3rSn1uR!~SbHAwBm@Iu}+8arJHf0(R`XtN?2dIb-?9Peb2X9SQ#iHuF*W zU19p5XpS20Aupl(RS-xBO_Jw`rpj&CDRbz?c-bQHge57#@|=7^lGT3jF(C>YU&mCT;k>TZ&&`H!$wxFytAfws_qJ?=$8 z>`UJbOaBfBicz+->C*3^M)}d`Yzwnv00~p~NLFygppXX-Q4;XQV2szX6Xouw$#!pM zJMeu91+nXl3+=g^nVSKEu~g&8h*%dXFDPm5jk@=NEWGR&R>8ea)8I@e{=%+a^WVy5 zWT<32ar3O+*9548(8C~dP@wwfMrF{=qio`JGZ{2I63&2e`rj#S>+7)g+RvfFPN>~j zf6i*P#-AoVlyJM@VsZHvChWhU**l+~ykDY%Z z2Ncg4P0;z6=~g6?y1;>y`0RsZM)4#MYRjZ)Rc~F+Zqd) zpsIJ-n0KL%roT^=ziQw8%ZoKe=F2G=Vv{j748R*U~naE^xh~17J(XDZ;esTXq6@GOwywuuo3t@JQ zCA2c!7{lIA_5 z!5H9~7v%4w!st_~Uy(*9mTV6d;{XfU3W?XhM_{0EpLx+~S9FoPdCnpZ7j{_5e3d1% zKJIfK4LU5ZpUbtozoFsH4;`Up0Xo?u7K_-;;^L+`ZtNS2`<3pztx~bEQlFcyE|pzD zyMWzLIFR8v~Tg??*(FfRCxe=H^UvFViAfjX&96|Oup1Sc102c5Y8d|HhWhucJAAy9 zU%SlT$}N_Xw!888G70-zPy4(QBWI!AVdvw%HyPVt>IrzJXVRA!E3|N4Vi$$y!8;4U zY>*ka?;IBem#wf3hQtMZn>3UnJWar}D-?+o$msXO*k|%y@QYZMK8!y1a5hUqLmBC> zAdKG1;O7u0XA8z9X>izL{~5&vMf=|&!hTIp@%5TxFUyUGaZ95fM9EA)72K2>hDZdR zd*|7`S6C4|Q zN@h(Z`%r@zs(r0MlA1OnT9@BhQX{^W_#gE*7fEe`mSrC9dnbnxq}}i*?`PN)KebIA zu7;u6s)>U|Ao`4z3Oc@16lTJ z(f=$05I}$g0003fxZ=ctLVW>5=-9ogS{uLLHS~39zSq7vauxJOg=w{x8n@`@TJ1NA zNtwqK8fDEFTGX^QYqr{P+ck-*tIJhOD(Y48Q2t*R z_0B7+2_s?lYXSRf_|;=?dj1)Yv8TJT12Qk8q9%r;eFr`x0)CSGgxRk&cE1=L||Z}$GTNm<(>%Xz0QWjq=VrC``!OcoOPuj(6DP9E~cTQl(aFR=eMWZjrc z5yAkJgf-3G>EO77ky?(J%@Wh=5q{RNw)|^ApsmS2I$gSP_uH9@=?J6v-TRWTF^8g0 zY_q=d6nkitvXy$lKHTUSn2_tyF4BrZ*`TH6*+ZthdifoXP;vFOlCo<6H{mA)2mn9; z02j9j!|0f3icy5q?EPNRnqj|bkZTLfu8pz!Df&O9A#SnU{q=RLR7}_TUB=~2yPi3K zvqzi0n-JQUWKLx#3EOEozTg|QF>lV8eJ zCyanlg)xM9%++q6#WDrg(>b1AZ#Cf3$?Coh8DsQFqekXFX3WbDw0Eepi=Ix=T+!S%qE>XLnQ zJZ&Sl3e+7PQc1Oa44GCUz8I;2Htja%C=E8`$4`E(f4#Rg&1i_6T_@5S!Rr}s*+Ss2 zQ-oA?q4L^jv`d%EM_MPsF(J3_{TsR7Ym=g+bf~K)JR2^xQ;>fv3nKe4WrqbBKi>s2 zP%^agjI8kT0O=N_Kn3FhO9pLr_a#(jyn9Q)APeqwTkQDMY<#g{C}DD#sYfHsdsIGb zRAzou_V4l!=O8abKb7bEAn$m&9ELQzzpj7_%i&gR6|{7cawMrl1`9fTgtg=Dg^}vp zxJNY!t_BSD>>CCD&R+=a{E9c3-iXv;hYZP;joK47d=&1afrfOJoB11AkXjJts0xu@ zY0~2@ejy!Uje{J$nJ~Nqcvi4A8F~>ukDis6vXfewm5&pyDZmi=Kq%@O?-UO>Xh-!fmpQ!1v9L&ud^N+*mN@V{yu3K z5NBWm1`HY4Fkrx`<^wtfYL`_OM(eIEaaOWbt#emUw-01EAUTaHXb324v75*e9p-2o;v+=yY{nw`w!4G8 zpO%*!0tlNpnZygqt+;q|V_9q4Vk+eaGx+3g9CbRLgkxkH&`tGYp*&I!mLQo4=1JEE zq{szdU#@PKK24!vgzwp7EQbRP>0hbpx`)efOyy*(a3N;AkhrZDyoYqq&R|cFJSLP^ zXxDw_0&a+D8H5eJ-3saL+S4m6i2lawPW&srwf<}8%vBqoX4*6(*njt488~K|9<08R z`s^3eC&e=Tb_oVK+^L6wl}iQd>hB7OCCE`b+A3j#hZa0(2_l-C^vD*C{<2WceP!~+Q$2GN#BEnQPW zhFl~rJwC?r}F^8S^GiK z;ZBi1%eqBRD2c?n9>Qt7UW0Tr9&Debt`0>bK|!@?<~I`|wts7nAhI0Z1^Is7QwD7U zz#9}=ZYqS*x-9Ha;ovp8!xIUe{Z1s@B|ojO84og2OuJqtqwaF}of9I`J{xwICP!0H zRS9>k1R>hXcskaT;Oi@!W_Yb1X=tV7{17$u+uw5`tIAn=*n5;HZn@slpo@8+<)ndqG10G}rjN7*|V|Tc){W|sY zO_+E!RxrevJYB4P$I&~8u>NhS8^@0D?9i?;&4F3D7CVthKUqmI{fA$OX7eu`X~G&D zguCRcJC9;*swzJAp%4j4(B^uPzDI;sMRqc$E3z*SvCLJ#JM>@m!}Nk;j>TQQH)Ycg z;#%P`j%m^=M)PU|*BiQ;uPdHSl<0UZY#^_#Qp9}^!4m~fzUJ6pg>LGt?zo7O-A{^2 z*U)@#(N6mqPe)d6J$LTM^$z=kkb!&c#ecT>2S#W}qMO=;sQPEVuBo^iC*5-z}~vm7}YJWDEXDwan{MP?{aNBjMHZ;Malui4^RKu~#s2#LV=cSv5OPBExF zeuuF8WbU4r;pu~zlGIsB_^a4_Nt|Cf2|D-6t$I7QOm2<7bvwRpXvVhiw}s3LLY$1y z$rf*q05397&3(bmRSw8q_GRhSyQuf5Y<%u4J5dpD=h_bA4D;-Ej~25JP8v5dC|WcJ z;+vN=1=Qq1fLgxt@{*fKbf4KQ(5*WfrbnqN^w=6SRpI&%=^g#kmY}9yd2e}`1;UKh z37$4WI*Rsf_$LV}g~!h;du9PEAw#~oZ1!5Bf3`IFeXF)ZtnKdeatr1jUC2eFSogHn1z`8ZEF5EgtGM1pqN+{}U&g^u#WxbC2b>`(`OFvC3Im9u{A{@sHD&28a(LvAqW;0WXC6WK2Yn0l zQ2%Sa%r!P38F{?O6yx=p*tF-*RJX*w;NF<{pvBKP z7j$P3qEgs}5|rUsw7x64Y)v=p4lf^l^V}U=JUG6kj20TpW=Kd3BYJ(;Ykf=X1!OKs z7mi|0fG6#xelqZ!tWl5Mm*9JAi=1Y6?9)hb+nS`AVOA(SJ##`=(ZVKAoy*7D2{=k{hiNLd?)Xo=a1t#UOfkcGMH?mSG z|32j`0Wz-{2jC7rblzQEG@N>eoJmotLj|)FMpY;KrcGXe_Q32Xwq{s2&-!mB@{>wm z$mGA;u_A1(R5Ocz3a*v*L7_%jw`%d48>Ll!b;Z>eR#qV7vv8;wIcG(a!DT+0S#o1{o5J^ja_we^7ruY*;Lw1%4+G z0hT9?cyG70ZkYGJcx|#fTzCRcc6B`V?mV$Gcvq7Ic_Ht??Z`MnM|>2oIO@Sdf*uMf zdaOi-**5u4+#byzQSc}cRR8ooo=AQ_b$qj?Sg90fS=kF@)s>{78uHZjOP-x5oRFqj z&AB-49>CXD=x1q}6zAV&1X{x7%Y+k&a*AR2w}04P&F}^oBU%vF@8#lb5;Q|(;r5s< z2ZMIw7_WlTwl=W{Cn-C_0u!kyuyT}JPXmrVD)wEkg16xI(Nh!b*GA$eApNo$)%f~L z9}twgk{DB<9lXeS&tj%bSR#hyk2I06bL?@!!nU+xr~4JE=`ljs+oO;nxJSYsquI!| z4@|(ixO9nRgUf)APO zHC-1-Mu`F%?(-j}0=-sw0muI02t zHA8RuNt{wi9{ZW5dkLa6<(N z)}x;N`Q1=d|8?1|@?NLpN!Uo8!{kOuazfnSLaI?2H|AgGx5Z~db{Ofs$80l1emiyd zSX~ybB4ojmll(sa)E!B+rHIc~Wk$xSc?EUn_&k59v&Lok>V2yQgZpsF-FD%Wk zRmdEBoOiVbxE0iU{_4owaly5X0X6pEQ6;B!si1D^z>MI=`e12)>Spf(S<@mW_isCW z>6O9WA+`EZKPmEVy0XA~UdSdAscc&R`jr7>$y(uH6W_bvHwAZSiI(%XM2G@u(@Bfd zrIgIB=&;?LG#3mf`Nwv1+tRIYNAX(S=u?TebBvQyLVpRJFwLmD@-kBNQy--6B~JgE zqfb+Wj^SIw)y)`udeb~ywB52=L^LW$+X}@^sC%v4tn-Q zHBuA;60W?;bvcpCK& ztK$bw=YE|&Xw7J{879u;p%4HWI+HxTRK!^sJ?^z>Qgv~;EXw-H#9oW8Bc;a*K@>6iI_Lqu?wI8BItku z47lg)q6g5WOc8hg% z<0i&l_J2+33db%_TzgiRr_7buwYm&zSBo_P5zqvthL_SdePGke6cM6ZeG$PRGa2<$oOn9H5w8zj z0=lL)_~xL?mK_K3rM)Y))>6rXhSBA$K}8`ZL_l7@!K`t8cjH51H#8SrqN_p6%y46W z2LUf>z#Zd?D0U8hc7($3@_d9wBpX)PZAc{@x8Ns!sC~KgPyVE9g-*hrf9kpIv zyh;UF%DegG(*CtJTMHzO082o$zc+>ybHSVn8%+g9|8OzzRzTLTw)Me9d2Pg8TXa+< z22w{r!wa(HSxGei8fZT(f`-0S*<={VyX!9Ve7~I&=uSFR``F$_v+14h8RM76#cJRA zCEdoNM?lV_2Nb^bT=gM(iSB-ov)dmiK`V0t*5(Upy^OJBv?u9%%}nfvIiykRA8SXjhRGg7vd!qzZ>n+A&{<_s>G1{d)GB7Q2d7SIt z&&2~lSe5yM*)ANmM9^@fSo*kEsE({m_-IPBXV>IrbU!z^+8v-I(>I(TOIKwx|MDZN zlGutuHPLkAI*9d`dauC>t~tF8Z7>YX8XV1jX@6n->)=)cy%+oMl-<3@v{C%$dfoX! zjV!WU#8PO5Uhwxp1|DsAknnIBcGI&x(J^$Cn&P{6Z2a$40arZo;xPRJKMKs!6BIN3 zHb*-+0vy;9F%dGm&LM1%KD03$)*AhvoOlf%HBU6SW-*hhN=J8*%iI!&14gn?Gp^a`JjGMDEO;O8C^r!{2H0W3tVh z{eHX8vDlqR6sM+4X^n`(Ii$iIi$={kG4VYk-v-6dmzisw*rgWyBN<#B(8+Z^myEo0 zYV}v1)G@8c8qpRY0yO(6$kan?{JI&hDKK-z`+i?>rT5yp%Qc4q+qZ4Aqdtj5rqRL4$tUTSi70@QlO3JoRGjO zF~(1n%7$uR$y``c-)+&x1V%zcS!cqrm-zSUkn4h553EXxwh5w!jRu&CoH8=@GZ;j% z+=mMPpLJ3uYIQ@@ko@OglllIx?dbN;;S<>69a5XZi{fCktb+ptBToT?w6Op}GWRof z?3)S%{xcyz;r_0qYvwQZ%`MQUCD*EoyTkXW|3U>g`g)_SHMAEGh2P&D;ki#R2s^ZI zU|5p%%kE8$YhNP@>|UDz|2~QA!YPzK3PcA_d>nYa7N2@^_+4tRzp^_K3g!*`8GI*U zwvXF;97?Oah8_Y8CEKyN8=0b?a^jk-t4bT%f(lfx-IOY_F0Hq^!3Ft}R>B+z>=K3c>TA_Z2IsZ&)-2!0TLqHyyOMM)-?&k6 zPu?B|qAHr!NK0s~y;ukU9EO-VpMu<2h<|08-39tUz|!YWs-BQfcux%j-S+M)Xf{C( zy`k&PN#(C6w;D+@m#cOCLg%n&T?iC^60EBStoG0~s(l(Hsid2YYc!Y^yMVa38tK55 z#mm{MXm&VU%dsUBPUxBtE`{6`C<&|yTR9$$7IC8G%=s^%Zb<0ZZs;kG@`ry_=wIHUo5E;0Z^{?IaH zqt9Ql636QouvSA<>7YM?;&d*lW{Q*Fo*k8F z`x@E%EKYj~&Yvz;`vfH&Z9TgRC7TtzJGez~z}U|JCs2Pl+*D_LJ@oR0+!oVaCyvpH zzI_Uom{dSmGc18{{K7Njbp3)*;K=@!g*zgcIjw~nV#tmyM*6M)q2@!A-e-a zPa0i&LAeu8jp1&we@p84U}MCnIlslLHpxu3+dq^oWi7)q3{%6l5(j|jS~DYTskE2` z7?lrp2`5IhO}|-^wUx#!j8n#meoNdFi~Gd4Z=jmAF(y#abkCX{4)Pr zb!`0}{DpKq2EspI`F(BcV}3|H4aJ4OlM7uO;@WyG>@=zw?e#4J>Mo(zQm=TO?Cs53 zl>Gtw>>+aVW&dAt%Tq>(k%f(k2?4b)QH-gWfA4b2KZUP3SE%5|k!2lQ0;S_ibu)vN zp4I8@n%T+&VPq5=OmOCkSwVo4C%IGrc{0!QQhQ&Ni_jtrvZ0ig5U|sTzcq1cG{XQ! z_0Q$n$@K&?Y2P{6Lhw8;a#OuCpb-iZT<>n?>dkZw-mhTy8fyxZ(QC^dPHm&U69!=! zIqQ|RAz0OC`#UB2;KQ>hL%yF;qW{8nHY7atx@mG@y{3+U@d z(jeYFAuryR^57#adKhF_>=%HU=vZLhCMcCCA}$T+1r;}~(v_03(}Wg>OT7H4Rb5(^ z=X=TbH4Cw@WdBB6Y}A$2C6-9P#7AqfwT-K9@~KuVHe26uMM#fk2vA};Kc9z5==@Hz zF*t*&f9tUO8*{a&f>!q4CLMLjOO6IzICVY5FE6Y*0tcVCGhIUE!wVY~>oEhLe-WwM z-2Kf`eIOzK5H$blws1Gm#@RshqSsjH6A`-f<{Di1z7&kd^^2L?DkV19+haCUQ_b#? zoJ7)^1zdH1?L?iYgkcKIVc#EY6%E?*W~1akYwzC$`cj<~1e1{%inTz;@V!15ajP~5 zBZ^lvy$c-wELgSFG>q!K;p+51dwwl8O4ZwjeqBXXHKmkh%a(~x!U3L@B zp=SEB%uO*p0U0XyVA+d~I!-U*>*Eda+p7bhFr5EEo_oZ31#z(4aRQ@-%X2zhuSY#7 zK4GY)^QRrjgfVq1K?csCugofW1*X0rHA&5vJBPC`JMnqMZ*Y;!H2X);qko8Cb$JKE z1I*Vln5%AcWrakVMu*U+Jmb6w>2kLwx0HdcF*KudB<_l#<)(KomMGpS`(AO=nrQ9! zxf3~&k*W+zjBNzsuORV_Uw)@OB`Nh3dO^_j zncN`A^KknQZRzd0V7QNBK+bvKXnf9qukyUvK=SKfld7wz92r4NWDMa%;P13ysk z|B$xJG(!tTnyqUp$os%lp9Vqv1NnS~J}Q7k&1sFCOE*Mvu!7#xZn&v)LOJE9WiJM!Jl> z2$Hk}GUyhQEO73Uf`jK~V0Q1*d~0q0v>*3mi|wMjVTevS9Bqm)pzm^Z|1BB{1{gRy z=L`^u_MkO90a`iIGn)!)W;|q9#C5xl2$4u!b<%fTxJ3R^NET&~ z({=qmZU*^G=1^S-v2nyHuV)gBe9QmnuQ%aWoCjSQzIDp_Ip#{-%R#E1Sc5kLv3^LY zGx^x40@oCdZ*p^ytO#e=sX2u(ERg;5~sC<(C{*N++EEZ*9abR@i;L{of~m;U=oy@Mm99B(#7IMO(e zreEnwo0rq=b2)TgN}om{#z6=3f6!n9eKPI?@tRClMe?#v%{E}zh}ZTL4y z6{la2?*NHm(aBS)Tc?K|N{UOPcRRG%MiTU&k!yn1yeUmM(gOsMjAd;2ze9za#yaEN zp)=l3MNhHH9-T2&7@{j45-lNySFYz%hFSVzMSnzi3qiBj2|S_O6)o(SC!cb}W+`)}^130+~;@NJs9lO6LOe}Fr%%9lg19HLB44ap7YfuqCYJ1D2(& zlq-1Si;HMS?fNp>jYNar_2Y=yHRI^@;qZX(CyY82`gR+tOx5;%LaQi=B~_%jQpnPv zGY!KSO}rHolUd-e>4lcqXy4V(elD=W8HNgR9xJ8MA!{kX*FJ@p9e~_;zui(HxHSNT z>%R{2`lCA(sWiq9|77G8M?ITf6uA8W( zeLRsdj~@6tH~M}i%jV@sCIeYgNyFK*N7fLr$x`Wy^pkB*(4% zVBk@l$zA6Z&zF5X)s2t+Jy1m#pmy5#tXr_*1I(Ihzp6+TzNs~ukA4J)&qI?0KmAr4 z?nBJdwNpensuYJ$V%DzoJ^|_a{t*HHLB2*}U>h4j=CCrbz`x|8?el;O)cqx6%z!ef z#V$Pkzd$7)O>TKZ`;;(w6yAx)9G#{(J$(2N zhjuKs_o-B^{m2-5XKUVL|Lz#~1IHBdkTRq#1pa_uJ}RG7HIBz&Xl~YBCsp{+U%l1Z zoBKss)x|nWcGTxNlILtJ$xXl3_r{iXr6{dnm$*h~@iF2&AQEE`v2dMlDkIJj0Fxe0our_l0u?5(y7OmZ}QybYIG)(wUF!Be% znn}f1oS9g6Z$myLn#cK_KSe76+do+N|T|ue;q^nL+(cQK314@ApOmF~Io*W0q*IObCOU1>Nc%#7fb}vE0=rV@eclw)IVHmmcn&P`~S4<#iO;$;$@jXdQ-YKk!Sb3uKA$&HOYux)Q(s?Vk z$e(A*&pp|kQ3F56xS?r3Zl)^3OZ6LR9`PD({~#6htEl(ie%X&hNdM}L0LBlxkGtDH zKpA6wmG}Bu4w*NWzxK#oR8 z6BKx0iUdG?%s>SBz@z$M4zpFb$mU2tvY^}NXqDf%QF?XVM{jmyF&H|5Mn&Mv#ya>*O?~BpA z2o0jMsn=w#IpQb+EN-X%bVRFulM%wN`M+9BHnj$ET8YNt2m?qwc_{r4C5DiM?B;Yg z5Y{cYcF=;Fok(p*0G@3KgpQPb5-j<0Hk;T1=AZ2Ya}dD#S~8}Jsbw8T@>UwDm^c&_ zIihpB(tkyMrrUHBTMemXWHK+QaEFfhu9$QgHzD3eP5)Ja&x>o~Ntu=IFO}I9Zyhq^ z@NJ4e>%cMOK4G7mf#cvtQO*1cED@MsvBkP(adfCNirS%a25qg>wDZ-`@`K36bzW=i`@^Qpx@l^dek zmC;~c&qAE_Q-_RI*<|vChv_f*uqzUMhMi3Qs`Wf|E@K#QId)q&DMbQu5i)auGHV!5 z@ou%oZnHI)uj(dZ(bo+}`dngA04kQ&@*4KMkvix?qZLue)2g&p;EZ<1X?RzndQu@r z`Iw5E9Xh(YT@?N?%fcMRv_{UUqmofVn*4M@)2hc}oxvYx*inrahpH7Fw>7>L94&f0 z-l(lD~pSvR*4pC1uB>)kP33GjC_lLVk4gl(Kk!!S(KuI38i^UO%L@ z!Kq}KTXK!p72q}NdDpyNoh-cFOlucm}3c>8GVCN^`{2Xe4hs7q({ zlLzYFY?tRxApJy47o07h)O;@^n-);%x}uo!&|yOw&OqOuc=$u;>i)OMqi~SbBq<{; zA=Qc6oj%Usi0hdTD7PmULITA{?-Z>I@UzFGyJN#y=~!lk-o0m&lpF*9c+bPjCBZ4` zRxbb>ruTJV)9Iq?X)QYZGtu3_ty1N~*GEouKU`opuNcd-AQuv$e zR`2VsI)X3`M)%{KF^l9t=Ouaz0C|P}_t0c!r&8M-$-8xIAjvyrmLJ@1*oOFyGfeKt z8!5X=$H0}Hml!@lAx89hSxALh3q>WU^zKsZMr;?GzGfJ&+@zk1R!p40@``T}RH@WI z#P4Ev2I{!i)-l0PI@AT+au?GK1)Di7vaw=GkGi@g&@2vcBZ!JVylYq+w|T=-Sy(b4 zTrF#Gy9(02{r|#8pW5@TLOStCg@Z`v2VTcHH3p7l&Zk3MN! z8_!2PZ>&1E6NZVcLbiMwEr${R;u~Y7LZvS{GGA3s$$Lj$1h={7q`6L+`W z3B1b#*Tv9QhhvdPBuO3cu=#7g{=odNLX+4pTXJl|3V(txm1PSbk)4L;Agc>ojh%!I zA_1g6NhXD#YNB1fY(Iod!c7TSp<))t1#f>uYb_)GKMI~xSk|NG*)kd1JhXfrOR)M7 zW%k7c?aOang&+lFLK+{2BPe!8wnhP2tx6)`B(%RVyigoD3xs(~;}&m=`L*Lm3nrYL zK+vpY>01zZa%BABY2ehy>^G$Js*U@GnKLaqm#OB^2VUPy^_aD{f&;xG1GU5wwKB_V)hcBV#gp_0|nsh~Gs`8Kg38UkrXXTmku?Mo0 znb-S#$KFSVXSmwHDKk&~whhzJ6i(iFB0`jcVVlwmYq&ehP?${EBKdI`$JH2iAOecF z$g`2{TA|DE#aCT2yK?98LP814ftMbIA~{R9W%Z48!Ybba9dlRpC!4b{>>H$+Y!xr^ z+V-geM|Smk513B9e&5MrygSf3rj??q*HZ9MT}mA=hq%aHMn%U7muZn(rr@cD{k!9#d zM@K783Yjm9;ETRzgE^4*if;54FCWLDfA#Sfh@v+Ptz(HqSO!W0h^tCr4wh_(0Dq1v zgG~gj+D$A{7H{~p2m?*#11drO!5KY{$4!u0FoWou4)(j}AQTJ_^-7x;=;1wav!S~e zCcNph0}beNU`x0(*YMwI4^pZso*r#8c&Ah$bSQ|2_54TjyBJbRvA4@TEPFo2{8W09 zT1T9e(4v{~*Z)i;UsKc?I!qg#I6T$mg60$j>k4l(wq&@aH-}$tyDVI09X5`9FExC_ z-stj^{7unI)r36DlnV@3Pp%e#V)?`#bFWR&pXB{9t8>kUr3W{DmNSOMJ#VFD{`{FX zf}DuJZGK_)yrO4+fbvg85!Yjt?lzoma{UTz!Nr!u6}I_jY5uH}EwrzLeBjN1am5}v zW{L~3@>qN+o4Iiw(bfRSUrFtm!m$$mdZ=r5TN_Qpg>`Jh*Io?=PF=^&LO>^5DZ`f;X zA68@2k>3{kij=Ri3X}#&RJGuqd7~pl9Furt+b`?<{G}Jh2*MzoA7x3*=$J)Sb_WI~k-Y zk@m!Vf)nnFeLt@~0OunK`|wl)Qpg7+4p2rKQ4@RJx9yH}*0qxAyx@$ot98(J4x=6! z^5iUte;{!iZ7$@|n;}ehC;r!pwWECugI9+_D-6@SJu6buN2%^kbG94;dFmp!X7648 z5?C8>(GO20R{$oHv8PeVb6skEshgAnI+MM>vd(Ewr>N5k)WwaQL(Ge?-2b^GSiTos zaV)K}OKVd#x9VGFz7S?P^`mAwD)O9u4Ovx~TytbaHh!#TcW_dvE%qP1ba-jS!EL_k z{(W8v6T5&bE-atcW9p{9T~y~&5v*&md>#EF78O-OYq2mHQ-Fk}3~kk^c^0t4kD#$m zvwe@mJ>^6SQSJom&0yn?(kjeW%gen$#&|erp))8`Tq2^lY@^EkIAmN0C)l9+7^W(= zdDu{Il8|Flb8*V!bW=@NLEF+x)uc^Z3ZK^Dxl^qc+V|?;B}5*&S+{y-ym6ZRwoM?M zwVjdxF->KlgQ5aF zoT(%NF@${SRK}#yh53o@IC8I<#M?Dtr$@wvItW3eSbzZ!I4+wtOk+D$Htsl@p(PoO zh)mo{%zd<0i>(jXa``AL6UQz)S$=$_C!`J=!ZpUf+*?}t*fe~yO8XTu(|nw1wt|EFN?4dVvP)Ea?i!-`(T4+I@C z1ROJ!56fyCKrd!aqBXe6bM1+$=^VNTW&~na;HPkT*vN++;gxm z=Xh4s1P=oKy_yhaoC`TROa!CR9-(MXP7?^;6|y!Q1Z)^VpnIeApX#i63FA%uxq zUCnF3^$&mwYjW4?3gW%sXenZoxrA#i{3HI;GFRp=oG=0g!EgUaN1u-~9ujr3p_;{i z@0jC}dY*)cB3mxti~a*;Kh8??fD$2>pa#vYU>=-n=RY0fKF%oM4qrfbPY%*)_4I;||*9TNZdAz8qO30$O9>*-(N>ZlAuSP{pPv4rk0)uv7jGQcla z%C?@!5JR>bbZ4tB5Zx!4&Eq-FrJYw|lY5F;T8ogOfmA{$5j4M@9Kk=u1BJe9dfLjo z87|Pec8}|i9gQ#NIzRX3xJ;w<9oQ07NhrFb#POY@dwi=aH2gfnkPm#1wrB$j?X!Dg zY;tO!t=%ooBV}j+s?aKukPXBUEZy_39zCQ!G;n+bD~*N(-C1Pc6U~@-;H#S^5>yq+ zfnXRN*EA+{^-}kCvjS+Szu3!2jd)+oQ>b4I)7Sy9)m?ikawgZbbzmsuws?L|z&XvL zQ}C1MwP@;2 z9wp2KE^XtD6=`rzb4=`y>V-4Q#({+RG-I~T@{^mO6<1>KnoJ;>l?%!=QAqdqM5Wo) z2IyN~MX9y&an`Fo$b8(#11~v2eS=y*8wA9^M77rNhfH77N}}EUK3|bX4{f=OqKhuG&V2;p)@Nv?#Nhw_Ajm?7M@SB z;GV|Uk0Z!Du?Ho!3o9K8HxEB+pi7Pyb8jou4Ia77SMW9)fI?-b@Mo>+J!BJ%8{7l?*APU;%g|euGg}k4W8RGjiC1a#Pm^O14 zWyVQ9Xy4-_PM>8s$vvjIzHHt47U6v!75TrLU@L=-UGaqGq6H9OFt#{0bPVhSZcx2j zW3J+u(B8gyUb1d7!+7!og2vxs^1;S89c^ERkeUYDv)Eci^++`omB9xdUiJ?+<#LPZ zHJHX45PP+R7VQq7fU02^FhWMI?RooSzMHwfz(<+~I7!`frD`cu;o_0O7$gJu@05U3| z%gi?_O8j1r5Lo9%3I!?b&vA&+J%nHE$AIZOvK_7HR!mXIacl32w66J*964@Yr;X`u zR2e3v5PH10d8hrA7?w=sG0Vp!RIJu-W)N%)R&&fs3h({8s95e@6V+*-`uzR?S2M+C z6p7(is?dDO0KJ0$e*Gs)mRc1=05S~ZgSUAVf7}e`q>8L-)1+x8_OT5j+|XKjEH9FA z;>A2c*zMO%S=r0r%}X96-SmX2jhYt4swrkMLgg%7kk+5omjz3@mHXLfxDjtNxTaEt z18~qWb-)C?F8tpbwt%^HzQ&s9XI271$tPE)2PH(=nn2yQc(?NDUk)E3+wy>D%LzIa z`Hk;Vpiw#+Riz$BN77oYkt1coE_YBX?KwFG!?saP_~j4wgVVjyQYG_H=R2g5{L=g4v*-_R|CLpZV}JqJEz731h{rXufsb*6{+)7b(CJoj)^h zJ0=wO+>OBm4T63uBfsT`@>MtRKc_m2BLrSUGWn;ZJZkYVHbHH>}9BoarOfVgz|y zkM+!M1bLJZ&%nuoTl~K+D&Um<$NF2`02PYKD8*N%jWiir>t?}QNT$BxfoR!OL2J{c zT{^?(ZOLb;Q^6Ayyudk)THe-Hv!kqzQZq9RLT{JIjH-#$5IZYMP;#~g+9~n)%j&@# z{HZ%dFacNuWCrXFT3=1CF~0nbw11QvVm{;0r#gU24oTAZ`B|E(cWnF-Rc@$k2IaV7;uRk3mm8-OfS7eAF=2#(i8b zJHrtfZ~6yf|3P)E(zyD}5e7@b!lwJ$K99(o5KD4|S`sTyjeg47th!NA`tHS$fym>` z8>2Kb0oqNy_4H)2>hgzZ4)-}_^P{d+mOohk=wq->x?K;~IK!;%8!VERIPtqoNQhfs zuWXdEUhos>>CkXr&&Q~JvCjCH7dh`)3D-3-fofRKbh&66qZ$0ZS25ez37Ly_P&S&F z=t*6)0}andO}WJU`F@o51*8wpOMdITW$-=#!2k#lAV7w$pJjHh1$uw7VnV=YqC5z_S5*X*%nQ%AY=p*^=l-<4K_@WWwyzw4~f6n};az?yxk? zxuS14yu>F-*9(F88s^^;hKMRyzbtKiVH8@LH`Lr6QRyX3>O$t}x&meZz_9{Lh#<=S zbSEln8I#BzQu)K&KNz0hwZhs3$oYM4jQqJF5m>a=hibdWD-$EgIep7M8U4rojqlS> zG-=xqGS#7!MV54-o_&6i{b>@zBT5lhXAp-;>~PJdQq8DdU)WBAK2X}5-!y=`mmMjF zp0{8DPyg63%^OCR}?IH8C69`sZv3AXE>a)v8YBO46yl4McUq2q7qBS zA$1iI3#BmX>IqS|+Q8J8D1o(vTVe^0vpmbJAuf6fC29M5k? zq)3R!d(CpLD&0y06z4P7wDuU?cjceVYXEVMOXbM!c=VsdD}^flDyt?WMfl!z7%m#} zrPRx$!Q8noT&AC~xYD+D^?xkC!$zxIiKxk8Paa2AGcTW^T4yk(xvrwGTgj4u$JK$Q zrqdVF^!h>!`(Ae*az76^+f~M_{Jharpyr?P$d}^=(&%Jg-kX1>RQ087rT@cQ-AA#> zU)=if*Gz5Sjl>*X_@bQZn38rPmCCj1{8Uy~hi9+1ivVIMu{&X;qvTX^xJ&f#(Gtn0 z;$<3-Hz)9Um`ow|X()i+GHTYR*JI1E6eZYbMTAeBc%W+cs|p>1*Bv&I-=;b3s(k$e#>v> z16rC8Wa`H(|19JkNWa$V6^v(8a z^PI@M-E;MmBA-QLqtl4eT{J(&5;?xLAa{VlsGc!&7?B(vmtXZm&NkM_^1w-_JcCqO zZn8Xlq}Wb<t%rF+7}Yb~>sgz9hbVx)@bL_vRtNoK z3T#4@NUL=K<@j`GV{>tixFMU6IcmUnqt0`5=IfUW2ysPzvJ{kBaPY52AV$h~%t zZ>H#h@#G|j=hDAP)Fxe=8KA+d@mA}V(q^)ZCtjyum#HFq^a#WI^;-LmkYo<`6&~TD zmPI^698bu^1zRl`XE`!mCE(+)aytT(B`m@jsRGI}w{!^Xpo<%8d^UQ~*?$oCgVh1WT&k=*|pd9d<9H}Ip*H!vs0 z6gu1apbrQnHq4zF_quM)Y|>a1VO&?tMSboKEs>+h>5PH~7i zp?9%EZ%Gcb2Fl%HrPh*Mp3_yCm=^%(&0Q6+%(L#34Yl0KOFW=3vlM#Zo(h;O>!g!% zGa31?jP#y!LRZdjpQ7fvsV2%JZ$yc$b^X=&!hdlKHfSn;#l$x9d9I6hKbF$}kJ@4C zH7FUbs^K4hRns zP@$T_fxxL?BGJ{Yr8q7%v1s-!T+9!9x*UD6q;y-d6Hg=SNOzZI!!!-F`ldA~WS$|D z@LMuFm)m!Z4#JO-eCnocmB{TJ#;Ha>+NvWJ+!n=y#_iVAy&E}IYnc&O`679-DBK+L z922J7^iAz{Scf``_mo1l>meOecho~<$b?EHO~yQChr9by3cwwFFtP8|ExxvLqv4{N z{Gy{vGQdKdpkOWj6xa7Gn!D=E#bU&=QzHQSiNOob|H8{NiTB;Nq%BonVxM75grez& z_*_DEcU~L7p@x&`v@k$z_L$5CcX(*=-3V1}6$;{K1sO6=mj+LyF%ZHsfs1Q+IkAAT zQX;F~1Ps{MTZ4ZlbLw&_(15jHhEzmr6N z#bvCa6E&B@lu2ynaIKoi(bM(>Ba$xtVl*8yxF87gwA|`B$=g-l94a3?Qt7ub-UZgk zqM|z8@-=<;2!uEIcZCVi$C2n6tIAZl)`{R>PSOPbMHjRhYz2r385wrKheo zw8b$AvsPjfFLPhr>lZFC&$T{6Y&;>ihG#4tvmCG(k^_pT7OK*5J(_xW+=(;xV?@|G zyisP8S7z3V-ZcJyv-=et8(Dn1EXUmmv)+Dg`CWp&&t-`Oz@uWRc~sdU@g+C2Q=kb6 zaXLal7Jm|5${FfaKtQFwH zM+uqJq*WR8I>G-@f$n9XCRjVv#@}58@&l!8DP`jC%K(<`v1bp&SI@lm+VX9YR63@- zW_ll1bs(JSf!Ki6VT;LO@O2D}>7=ICPJR6NIlScj75yAO8Dn|YIUnM|-m7ca6xztK zBwlt>D&}7=>LBM`Deg%Fz(L6Ns8ERa>=?)=Z6ycsQ(Ci#^^sfs|60XX@Dw=~6@=-!#hvE~`O{XN~1kX2mqL1*NMMN_ybA3%J(Buh3JX5A~LC z{BEvRA`8tGC@&PeS~*+MMLHwvp^Y7d{4?E@T&_1E1k^U9AiJZ*`|%LeEsS``hV3|9 z*jIljz766qCHO*nHAB&7`kk?iay`6jBK`-$IzNlIJmk=GKp^tTTQm_6qN@9aVwlUW z&G68IB^tsmbFw(`valR5&IX&C{&LCxegD*DR}@XrJG>rchQuZ&OrXw(>PHNQHY4Z; zi9?N^)QxM*RU6l4$^SQSv62oPAH)^}b0&Mn+IPz%J!ELfB%lM@CP6RsJJ|@EQ+U7l z%08VU=#-jh{^Y9)Rtp8E;8FKGw=d8fY756g{_V;SonZLfa%Cr{3{YNr;do|R&YTb4 z1`TqtuU>>Zz0VWP{W>C4ux@vzn_pbMT-l-2?P*VTafrWa?ImH1JLp6(MSreqD0mM+ z0T&#G!XSXN|6~tun{C#w)3$cK(lnBvu(gM&Vw!#Aod6V5Si0-K(fW{FlJgDNz5;(J zfQv+^W`3VbK#tD6v@(0k6+>6PRSm)N7fL;+Cg+KTEvTEhS40r-(>yBTv1yg~O5EWg z2sioupNkO&s0OS>=7|2WroW1pwvyM$1emyELcsB{Iv@QukgjSmp>R-UH&!{rH+bs-chDY&8iDP*V~ zk1s?2s<`BQNsz^b_G&5HT8Qv%Uq5kO!l{KRs{aYX0Y;PlD{r@F>B&>58`LI(r{_`_ z6WZC9lH<#4-KoBQ5SGUrzDvAkZ{N0Rht=u6sO z`5eK2yA{pOj9*qY0v7UwOiqw%`9;Uj0eRO^*<*@WYxF+ebwZNL3fd&&<7Cv2I?C(72~0J)Q&l^mSIhpN}%>T8^> z6)e?`_wgA>cJ`#&zi#I5D+3UTS2S`gc&^-@%Db#kbInXSS4#3gF_R%l%* zS)%;p3RJx=bfu9B5$t$%I_yS=sT$t~F?MeHLt=^4O;eQ{dMovNU9axwC3f$BQEq?Vz zb1AKHgM$jvRuRa3Je;?EJJ+<&0NZcgGU67libhCcExzWv4W7vZwkt{hiqBd1#!X2; z%x^n_S?x0{vTWjN?_9DIADTu^owN4>Qi_}j)J&Fo zS)*;_MpBDWdE-OpP_^b7va1pR&>AZD%AqH68n-XAIVbEge3CE3IB^;mi9A~yHqcjG zw|>YL!FuIS?QQZ-kHj;2C_u=)@!~E>PjD=aGtsQ_eyl5#*WK4vw;F7Y&i2e9ygq5C zJrP_YuqO(VwdTAvf5TYe5T05CS0cB!?PAW<(%1G4295yK=u!nc_;&6KShJ8sU~Z6l z3M&$)$N;ZQ8H?e+M}}7vtS+2K6Q3Ed?|c-hcw_Ou>#uv#NBvc4{f;`}@O$MCcy5P{4IP$o zc|QEw|6ldXB6pCzNrBsdw?k+tUm;;cohB*gcQ0gDIV4Gmj%S0~=+0rN!2FUja`aY+ zAh)iF{wjvzu2usLfaurL*8gTcyXq%a(1amvZO|lB&?(B3rCc*@@DV*B)gP}dkCO$^ zCIKm1@6vx)QYr(*@#NkZ8IbYDzO=&sr7FfGq)ZnRc?`BXLPlOUx*!}{uS5WpZX?ZG zP446~FwAWZE5xZ}@BwiHAf~M9a%d__!8P{2p~jlT3}KWkX5O-?MwTaqAtNgDdM?pm zS4&N%Ajl<0|AhUysJ528<*8e617QCiTN&dCVZv9Yu-hb?mM zV|j%|U`Dr#4ih5r36F0yrV0FibHub`O|Ut^1#H~h3ahkp2d%b61G_wLagMl_Y1zzY z%Nx&hc+)NaF3+y>2|b+ujyeq&hzC3$EFm^6yB1i>Wlc;cpeN5-3`Gxodu$Pg;Ae7o z-6uGxE49kOmwhOz1L93c`hm={5tN8+Y}}pUfY4Gq>H<)T{GY%l6HN%2YoF!Atv4fn zs@9S`vIXvyLTC`U_DiHH%Pv(+hSuGq_pHJ+BMk! z66ccFpq_<=CC^#v=RbBbvlJ;Ouhz1uO{DT4_n%!|#Wz%ELrEiULDY|(1DMnQ*ZlVl zv{kRN9lYv*`|zF?-(*=}_p-zIN&pM4(KyE#*;9Q1N#XV)pA-!Ddw|&!i7Gl}ABj$a z@=45Aie*y))n-jUHKPSf;jUzS1334&WH^6E1UMi-fbRfTEb*5DgnwCVWt*#6yTyY1 z;N9S{w3!qfH@tRN0ztoTyiiiQ@CKZ?6@x6-pR;0c2l!`f* zCpO~1?K^_0MTaQ6fHv7KGM-vR$bI}CYW+_DcCS+KTKtYSbw|xjW&6uhZ#ugFFY?kG zUsy(JJqytZ9_o~_z-LekZ((&?4I?5L13PiKRA>^^wN(RPyYT1!`5umHevYHX`Y{kK z`5@APqY{yn{49!;WQVpZaq+XqZY|XAYo8fNw~?us0001i0tGMGJk>dcvUkfqNC*R$ zIG{DrsXgbBz$n*!eTMUfe+kq(uc-zESBDDu{){Mf5e}d>oB1tq@vV6dxdPLRU7aE% zT9s^M?U(C2uQ0*Nna}gW^|=}K2QsXCvQu>1TSd31oc|H=xnSbGHTlxLmTD7{;|~@5 zk5GXwxVJD+X(pIGhf)bvS}vq%CS#P(B5`!P6D8)BYdIfG>LKFdmftjg zw!P+dCxKITjfW49YT_Sh3;4p8U(Ye46phzYYj za=O$r2Tsw^#GA8?o>v--onQc{cr*r1e!vAs8a>XoxX-Q%Q#U(sY@kAz@ZsX4X>v-yF1!i7d1^5>~m7IU2XZ z{T4)qEA1p&Qkcx(yx&%cIS)f|pkeEtP&#kj7orFCkuw=XhBu@DC7h0d zRIBT4ezzk=^+yP2ONqj}wx&bQr^ls#FE_t)x=@KT>)lmbip;XQb?b4wQokIgt}w)` zn_L$p*j~vPYyhWXTpq)|TjqqO46MVj_&zwX<}Ctp=?kIWvoWUg)~Us4$K`fELv2(ddm#|pN;is z<(eYBgsRnXQ-qJqTwuiH(uMM6QCSMY>GcPDKOP*QMjvS!a^0K0+(LJgWQ_Ww74GV+ zuJ@NdP{G$O&(=UZyKuY982=pBKV@~4ySauD?!;=~%d|A=ZwCrl53yI%QWbh~`Sy;N zu;A$E67k|XLldB2`DL0b`ZD)gM<}D9Bc<3ZY6Ypa&W&0s>+?H5XAddO!=st5Vg-UI za87)OtrP98Nh5kT$TYDaJI@?J*zPdZYU3$7&6PAtuMD#Aso>Ie)tP54m76gge%DIR zh6c2l&{+C4Lm_&+v)6Ke)+q@9Fx32q&A=qXW8-d=Df82AZQW6zpMM~q7qJ4?ok*hA zh2B9kR!$iQ;fbzAb_)CjZ#TyDf;frDHwT@y&@F=NI#xhuu;KjJ6!Y5%L0TEt2!IQH zbumB6J4A=aRSft54tD%t^Iu+4-W^fSIy2D&%}m}h=Stb^h{c&`7e~Qyjm_tCFyL1a zf&F`{_+X|@bl8H6y3q@i=x94m9mLWuJ)0VdsM%)U!V63eB6T8cptp}sV=ax6@N`&7 z+TU=lQt;WKh>>Z%=BLh@+qB;fAoRmcn#x|}4oG{Wu1^0G!jYn(Z}Kox}m8(mO! zYV-Tt4jRXI?0{}^j-#dTJUX1*6Y=Db7#+iWJx1+4OqlJ>ECpfmEM|2|zvS-z z!QBtw!+eo@NOdOqX^4Hsj~z7e)xD*qe&ciIAqf}O%=KTT)4+*+??C?(-Pjqy9zi1KG=FQ7l5mvL+g-7pqbuPtvPXWCj3i0U_O?)#Uia1UAy#TXq>XYhNIJ zRht0?E2q?CX`!WtIdD?zADQ+g)Hk9+)BOP2G3wg?vE)`MslM6xDBwOzSG~&C-I_@8 zwE$7eqGn$Q9W8kEO`K4x=*!e(y0+*4-Q z=o=4Oi>47X3VAr1+I?ngFn`sx1CwTHQ>Uh~i7C+`wv`XV>5+h4h}Hs4qg-Fp2z7c# zFML>f=YK_4P0lN`5k_v=nLpFcOsDmC64Y{{-_u4d61O0*%BiY}31XWa;D(!DODJh? zwe$+Meg!_itom8!%t+8E$PrCbin!a}2O~kLBv`2-_k2oNSi61R*jUW~{Eaau55ap; zMw+byrw{ZqdkK~JqcEFk)*bKfP(%nMB%j!iQ0bXFx z-G0&^xtjEs-zMOVL9zz($UgpV@h$b!((-~Os7hng{ve%|VyhP%*Z}3%v`wy)NE5hs z&-&eEbqc-FGHR(De4x7vX}sEQcFzmVO=qoN7n{q(xOUMyK7aa*3W7ac)axZ9%L2u; zmyS-B7v?m~mlq9yoq&1jI*_wTfttl=xitk3p1x-5chIkOQ;^x>_*^?DidL5OyOWt6P&bJd2Y`QAN*{5I{bGqla08NTnR#Zvm=&1xx5A|k&R+L;zO z2jKpxUvM(F=v*0s8YkUu3@TcSL+p3zel7Lh(rqZhYsgG(PjRo4+Lwc(-Ae#lE0EzK zNf8mCZIceE7h1_}67=e*T=Kq8ojHqQZ|k@1UW9V>{XDc#&M+*}o{f6Yg+=!DqsX3?$g3apEB9Um(A zpjHH@HjG+3&Je(zGj?gjls1YunBhkjp!*zS9rALJ11`a7Uk%k%%ht2*Vcs33d!fwr z#F$srH^-7O%$kO)^PbZh#OAOEDl4{*%;hYFNNfv=D#}~i&3&FmC&?AvR|yf4dM>n5 z7oxjDHb1|5{Ok=o&XZBB#U>U93&J^eEitpyoHa)k4*j=5UPnJL)K4o#E0df{8$8#i z=-395O{xcysV3?wRJEL=*&rY{zMwAvIojiE`(A&;9@^J!b)nl8nu#GccFR_U|E`7k znSGicyf4};$zJ+CoM>iG!d-pO7R+|J1Jew9j@Ft5M1>wbU-!_+cd^nKYFbs8!Ufx_ ze6`KO0h}b(Dklk%TpvY3>HvWP1PBl#XIX>h$;lwV)^+>W0!dSU@4{rZj{BL`9$4GB z#n-P{2M!yBC`<>LVTT!cj6iV+`hCM*lhVtOEy6Ufk~6%PXQ*k%PfCs*#U@xhsKB;# zcHr6h$@A(bw2+=-vvZRfcNlFV3%fJpm?X{rdrjW%3}ZL}$Wl$D?%G)qW$=`A17Ly5 z#@CY}VaQ9A)>sxz;8+L|i8z06q8{Ctjq-n10c)_%d8E2-*sX3PeyJfEdV-jMaf)L| z=TY{j1&vwo`{)VyjfV)ir5&JZ^A$br7cJf#P?@vQc)NUv3E|J&_xS1bWLb^1dZQoF zGT?b)d&yL{PQAU1J!OfYeiW7uhv4vZPFJZSFc+~p3@BY#e+7o>A*SkD_ zHkw};T)O!G{Z}Fn{o}NJVt|umP_dtHEpZKF0o?b%+WB!R-QJB(I?N+Hn%E!qhB$~_ zu%qf}bo%1Xl8;1Xc5gwZy8B8l&(+IQtXBuXFpB|w16W#xsJtL@YN04}ndEZas`+kA zJX{zP*F?<3i~o;S00q{UbKYB-9m4xon{@9S%x!Q0Tp6j0r zKIw&zXEMUrXjP{%?uUs~_gj1EtJLzG#`RMFUR9W@mC6-JD%0K+R7kvNS{ z^FM*Ae7@7)D&9>uVM|3^dhcx`q?;GfI+1dv1UMc%Vij|9JB}q6b~u6U(cn zJH!o>Svo#SvV+Q{^uM;lUWZ`>2oNAZfmVK($@I1g01AQHixeeO*Q*t>!+N31Ph|Bx zGLu^twPg{Fy;*^cc@aY<@+%2+Ce_#)xgssQDo>BV1w*V- z(wleC=<-KC!5VanMvQ@OR<>*2EM8Ujq{5hxyKGwD1FFW0H&Jl=g$CH%z}mD7Yu(dv z?2E@s;|H(6(5tntnd#Vt4l&E-@91V4nYx=HX@fUWa_B8dBb<_h0YqKp8%M|n@%@39 zyo?$p`VvvxGRO||*!bi>pP6F<(jF~Um4}s~!)&&N$SS|I2TGiB*d|s;XxX|?GC`tM zuQb=MHv!)zuy1@-=uOje%Y8SQj+Vef1>*iy4`I@eqg&t@t9^lwQcCOk@{u z{|9a`=Z^5=1e2s2Z=h$d7CtpH6@G&+VZ9-md8{prEH*vI;M^y%)=gE4TPdt?&rcw4 zhX`ahwlX6J?J?&(xr{{0pmiA5mZPm?f&y8#5PE$*-PMGTvGoM@cQe|9a;L%MNxDGH zPItG>Np-(#^vbvO zYlkPNa&>c(=5}Q?L()|JtMBPnFnoSyD=J*2_@S`Qiju9l=kE+ZTVV+N^Z3Xx(^v)c z5WVqznmd(W+;tmQ62VK!J3k2qIZFsbAhIi~%bFMz5!+U`hjw<1C-76K!Acftr~L@= z_D5Y@Yn)aqA07kx#-{v8f~d&2n&!CuD)^OPgGu^YTVDori{ceo-4X2Yba!~bt<{<8 z>TMp5fJh4&P39#~7<(f<_lx%{Eh<;OYQM1@(<~|uA)4yU3Zl7ja{|%X%TtoMyYRpe z$(L@>daFcLhvldIS>r#cI06I!000000aUy?5J>D?Wl5R~m`n%HTUvVZJ{N*w8(Lh{ z^_#Ko~Xplcg?!2@UUQ z=1B;g&Dm9zy3=xYiF>)`U_;pefc-`yAIuk`d63N fn*wjQQLZMzvf=dzVx*Eu|Ha&qP81{s>dU*pKfkT7 literal 0 HcmV?d00001 diff --git a/ldb/tx.go b/ldb/tx.go new file mode 100644 index 00000000..300f599f --- /dev/null +++ b/ldb/tx.go @@ -0,0 +1,175 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ldb + +import ( + "bytes" + "encoding/binary" + "fmt" + //"github.com/conformal/btcdb" + "github.com/conformal/btcwire" +) + +type txUpdateObj struct { + txSha *btcwire.ShaHash + blkHeight int64 + txoff int + txlen int + spentData []byte + delete bool +} + +// InsertTx inserts a tx hash and its associated data into the database. +func (db *LevelDb) InsertTx(txsha *btcwire.ShaHash, height int64, txoff int, txlen int, spentbuf []byte) (err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + return db.insertTx(txsha, height, txoff, txlen, spentbuf) +} + +// insertTx inserts a tx hash and its associated data into the database. +// Must be called with db lock held. +func (db *LevelDb) insertTx(txSha *btcwire.ShaHash, height int64, txoff int, txlen int, spentbuf []byte) (err error) { + var txU txUpdateObj + + txU.txSha = txSha + txU.blkHeight = height + txU.txoff = txoff + txU.txlen = txlen + txU.spentData = spentbuf + + db.txUpdateMap[*txSha] = &txU + + return nil +} + +// formatTx generates the value buffer for the Tx db. +func (db *LevelDb) formatTx(txu *txUpdateObj) ([]byte, error) { + + blkHeight := txu.blkHeight + txoff := txu.txoff + txlen := txu.txlen + spentbuf := txu.spentData + + txOff := int32(txoff) + txLen := int32(txlen) + + var txW bytes.Buffer + + err := binary.Write(&txW, binary.LittleEndian, blkHeight) + if err != nil { + fmt.Printf("fail encoding blkHeight %v\n", err) + err = fmt.Errorf("Write fail") + return nil, err + } + + err = binary.Write(&txW, binary.LittleEndian, txOff) + if err != nil { + fmt.Printf("fail encoding txoff %v\n", err) + err = fmt.Errorf("Write fail") + return nil, err + } + + err = binary.Write(&txW, binary.LittleEndian, txLen) + if err != nil { + fmt.Printf("fail encoding txlen %v\n", err) + err = fmt.Errorf("Write fail") + return nil, err + } + + err = binary.Write(&txW, binary.LittleEndian, spentbuf) + if err != nil { + fmt.Printf("fail encoding spentbuf %v\n", err) + err = fmt.Errorf("Write fail") + return nil, err + } + + return txW.Bytes(), nil +} + +func (db *LevelDb) getTxData(txsha *btcwire.ShaHash) (rblkHeight int64, + rtxOff int, rtxLen int, rspentBuf []byte, err error) { + var buf []byte + + key := shaToKey(txsha) + buf, err = db.tShaDb.Get(key, db.ro) + if err != nil { + return + } + + var blkHeight int64 + var txOff, txLen int32 + dr := bytes.NewBuffer(buf) + err = binary.Read(dr, binary.LittleEndian, &blkHeight) + if err != nil { + err = fmt.Errorf("Db Corrupt 1") + return + } + err = binary.Read(dr, binary.LittleEndian, &txOff) + if err != nil { + err = fmt.Errorf("Db Corrupt 2") + return + } + err = binary.Read(dr, binary.LittleEndian, &txLen) + if err != nil { + err = fmt.Errorf("Db Corrupt 3") + return + } + // remainder of buffer is spentbuf + spentBuf := make([]byte, dr.Len()) + err = binary.Read(dr, binary.LittleEndian, spentBuf) + if err != nil { + fmt.Printf("fail encoding spentbuf %v\n", err) + err = fmt.Errorf("Db Corrupt 4") + return + } + return blkHeight, int(txOff), int(txLen), spentBuf, nil +} + +// ExistsTxSha returns if the given tx sha exists in the database +func (db *LevelDb) ExistsTxSha(txsha *btcwire.ShaHash) (exists bool) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + return db.existsTxSha(txsha) +} + +// existsTxSha returns if the given tx sha exists in the database.o +// Must be called with the db lock held. +func (db *LevelDb) existsTxSha(txSha *btcwire.ShaHash) (exists bool) { + _, _, _, _, err := db.getTxData(txSha) + if err == nil { + return true + } + + // BUG(drahn) If there was an error beside non-existant deal with it. + + return false +} + +// FetchLocationBySha looks up the Tx sha information by name. +func (db *LevelDb) FetchLocationBySha(txsha *btcwire.ShaHash) (blockidx int64, txoff int, txlen int, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + err = fmt.Errorf("obsolete function") + return +} + +// FetchTxUsedBySha returns the used/spent buffer for a given transaction. +func (db *LevelDb) FetchTxUsedBySha(txSha *btcwire.ShaHash) (spentbuf []byte, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + _, _, _, spentbuf, err = db.getTxData(txSha) + if err != nil { + return + } + return // spentbuf has the value already +} + +func (db *LevelDb) fetchLocationUsedBySha(txsha *btcwire.ShaHash) error { + // delete me + return fmt.Errorf("Deleted function") +} From 96f7305c292ccbea7006d0bfd1b9118535e271f6 Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Thu, 22 Aug 2013 11:40:59 -0400 Subject: [PATCH 043/163] One DB instad of many. improved error handling Fix testing, no point in running tests multiple times Fix error when no blocks in database, reopen would misreport NewstSha() return --- ldb/block.go | 24 ++++--- ldb/boundary_test.go | 44 ++++++++++++ ldb/dbcache.go | 2 - ldb/insertremove_test.go | 7 +- ldb/leveldb.go | 148 +++++++++++++++++---------------------- ldb/operational_test.go | 29 +++++--- ldb/tx.go | 9 +-- 7 files changed, 151 insertions(+), 112 deletions(-) create mode 100644 ldb/boundary_test.go diff --git a/ldb/block.go b/ldb/block.go index 8d211e75..8a5a9523 100644 --- a/ldb/block.go +++ b/ldb/block.go @@ -24,9 +24,9 @@ func (db *LevelDb) InsertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHas func (db *LevelDb) getBlkLoc(sha *btcwire.ShaHash) (int64, error) { var blkHeight int64 - key := sha.Bytes() + key := shaBlkToKey(sha) - data, err := db.bShaDb.Get(key, db.ro) + data, err := db.lDb.Get(key, db.ro) if err != nil { return 0, err @@ -48,8 +48,9 @@ func (db *LevelDb) getBlkByHeight(blkHeight int64) (rsha *btcwire.ShaHash, rbuf key := int64ToKey(blkHeight) - blkVal, err = db.bBlkDb.Get(key, db.ro) + blkVal, err = db.lDb.Get(key, db.ro) if err != nil { + log.Tracef("failed to find height %v", blkHeight) return // exists ??? } @@ -89,7 +90,7 @@ func (db *LevelDb) setBlk(sha *btcwire.ShaHash, blkHeight int64, buf []byte) err err = fmt.Errorf("Write Fail") return err } - shaKey := sha.Bytes() + shaKey := shaBlkToKey(sha) blkKey := int64ToKey(blkHeight) @@ -98,9 +99,9 @@ func (db *LevelDb) setBlk(sha *btcwire.ShaHash, blkHeight int64, buf []byte) err copy(blkVal[0:], shaB) copy(blkVal[len(shaB):], buf) - db.bShaBatch().Put(shaKey, lw.Bytes()) + db.lBatch().Put(shaKey, lw.Bytes()) - db.bBlkBatch().Put(blkKey, blkVal) + db.lBatch().Put(blkKey, blkVal) return nil } @@ -110,7 +111,8 @@ func (db *LevelDb) setBlk(sha *btcwire.ShaHash, blkHeight int64, buf []byte) err // insertSha shall be called with db lock held func (db *LevelDb) insertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHash, buf []byte) (blockid int64, err error) { - oBlkHeight, err := db.getBlkLoc(prevSha) + var oBlkHeight int64 + oBlkHeight, err = db.getBlkLoc(prevSha) if err != nil { // check current block count @@ -119,6 +121,9 @@ func (db *LevelDb) insertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHas // return // } oBlkHeight = -1 + if db.nextBlock != 0 { + return 0, err + } } // TODO(drahn) check curfile filesize, increment curfile if this puts it over @@ -222,7 +227,7 @@ func (db *LevelDb) FetchHeightRange(startHeight, endHeight int64) (rshalist []bt // TODO(drahn) fix blkFile from height key := int64ToKey(height) - blkVal, lerr := db.bBlkDb.Get(key, db.ro) + blkVal, lerr := db.lDb.Get(key, db.ro) if lerr != nil { break } @@ -248,8 +253,9 @@ func (db *LevelDb) NewestSha() (rsha *btcwire.ShaHash, rblkid int64, err error) defer db.dbLock.Unlock() if db.lastBlkIdx == -1 { + rblkid = db.lastBlkIdx err = fmt.Errorf("Empty Database") - return + return } sha := db.lastBlkSha diff --git a/ldb/boundary_test.go b/ldb/boundary_test.go new file mode 100644 index 00000000..3e48f8f0 --- /dev/null +++ b/ldb/boundary_test.go @@ -0,0 +1,44 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ldb_test + +import ( + "github.com/conformal/btcdb" + "os" + "testing" +) + +// we need to test for empty databas and make certain it returns proper value + +func TestEmptyDB(t *testing.T) { + + dbname := "tstdbempty" + _ = os.RemoveAll(dbname) + db, err := btcdb.CreateDB("leveldb", dbname) + if err != nil { + t.Errorf("Failed to open test database %v", err) + return + } + defer os.RemoveAll(dbname) + + // This is a reopen test + db.Close() + + db, err = btcdb.OpenDB("leveldb", dbname) + if err != nil { + t.Errorf("Failed to open test database %v", err) + return + } + defer db.Close() + + sha, height, err := db.NewestSha() + + if sha != nil { + t.Errorf("sha not nil") + } + if height != -1 { + t.Errorf("height not -1 %v", height) + } +} diff --git a/ldb/dbcache.go b/ldb/dbcache.go index c5f74c90..6a9b824a 100644 --- a/ldb/dbcache.go +++ b/ldb/dbcache.go @@ -6,7 +6,6 @@ package ldb import ( "bytes" - "fmt" "github.com/conformal/btcdb" "github.com/conformal/btcutil" "github.com/conformal/btcwire" @@ -78,7 +77,6 @@ func (db *LevelDb) fetchTxDataBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, blksha, blkbuf, err = db.getBlkByHeight(blkHeight) if err != nil { - fmt.Printf("failed to get block %v %v\n", blkHeight, err) return } diff --git a/ldb/insertremove_test.go b/ldb/insertremove_test.go index c6b6cafa..5de70aba 100644 --- a/ldb/insertremove_test.go +++ b/ldb/insertremove_test.go @@ -5,6 +5,7 @@ package ldb_test import ( + "fmt" "github.com/conformal/btcdb" _ "github.com/conformal/btcdb/ldb" "github.com/conformal/btcutil" @@ -33,8 +34,8 @@ func loadblocks(t *testing.T) []*btcutil.Block { func TestUnspentInsert(t *testing.T) { testUnspentInsert(t, dbTmDefault) - testUnspentInsert(t, dbTmNormal) - testUnspentInsert(t, dbTmFast) + //testUnspentInsert(t, dbTmNormal) + //testUnspentInsert(t, dbTmFast) } // insert every block in the test chain @@ -44,7 +45,7 @@ func TestUnspentInsert(t *testing.T) { // the associated txout set to spent. func testUnspentInsert(t *testing.T, mode int) { // Ignore db remove errors since it means we didn't have an old one. - dbname := "tstdbuspnt1" + dbname := fmt.Sprintf("tstdbuspnt1.%d", mode) _ = os.RemoveAll(dbname) db, err := btcdb.CreateDB("leveldb", dbname) if err != nil { diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 6605b368..488be28f 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -44,14 +44,11 @@ type LevelDb struct { dbLock sync.Mutex // leveldb pieces - bShaDb *leveldb.DB - bBlkDb *leveldb.DB - tShaDb *leveldb.DB + lDb *leveldb.DB ro *opt.ReadOptions wo *opt.WriteOptions - bShabatch *leveldb.Batch - bBlkbatch *leveldb.Batch + lbatch *leveldb.Batch nextBlock int64 @@ -99,16 +96,20 @@ blockforward: } else { if testblock == 0 { //no blocks in db, odd but ok. - return db, nil + lastknownblock = -1 + nextunknownblock = 0 + var emptysha btcwire.ShaHash + lastSha = &emptysha + } else { + nextunknownblock = testblock } - nextunknownblock = testblock break blockforward } } // narrow search blocknarrow: - for { + for lastknownblock != -1 { testblock = (lastknownblock + nextunknownblock) / 2 sha, _, err := ldb.getBlkByHeight(testblock) if err == nil { @@ -133,12 +134,10 @@ blocknarrow: func openDB(dbpath string, flag opt.OptionsFlag) (pbdb btcdb.Db, err error) { var db LevelDb - var tbShaDb, ttShaDb, tbBlkDb *leveldb.DB + var tlDb *leveldb.DB defer func() { if err == nil { - db.bShaDb = tbShaDb - db.bBlkDb = tbBlkDb - db.tShaDb = ttShaDb + db.lDb = tlDb db.txUpdateMap = map[btcwire.ShaHash]*txUpdateObj{} @@ -160,18 +159,8 @@ func openDB(dbpath string, flag opt.OptionsFlag) (pbdb btcdb.Db, err error) { } } - bShaName := filepath.Join(dbpath, "bSha.ldb") - tbShaDb, err = leveldb.OpenFile(bShaName, &opt.Options{Flag: flag}) - if err != nil { - return - } - bBlkName := filepath.Join(dbpath, "bBlk.ldb") - tbBlkDb, err = leveldb.OpenFile(bBlkName, &opt.Options{Flag: flag}) - if err != nil { - return - } - tShaName := filepath.Join(dbpath, "tSha.ldb") - ttShaDb, err = leveldb.OpenFile(tShaName, &opt.Options{Flag: flag}) + lDbName := filepath.Join(dbpath, "btcd.ldb") + tlDb, err = leveldb.OpenFile(lDbName, &opt.Options{Flag: flag}) if err != nil { return } @@ -188,9 +177,7 @@ func CreateDB(dbpath string) (btcdb.Db, error) { } func (db *LevelDb) close() { - db.bShaDb.Close() - db.bBlkDb.Close() - db.tShaDb.Close() + db.lDb.Close() } // Sync verifies that the database is coherent on disk, @@ -213,17 +200,22 @@ func (db *LevelDb) Close() { // DropAfterBlockBySha will remove any blocks from the database after // the given block. -func (db *LevelDb) DropAfterBlockBySha(sha *btcwire.ShaHash) error { +func (db *LevelDb) DropAfterBlockBySha(sha *btcwire.ShaHash) (rerr error) { db.dbLock.Lock() defer db.dbLock.Unlock() - defer db.processBatches() + defer func () { + if rerr == nil { + rerr = db.processBatches() + + } + } () startheight := db.nextBlock - 1 keepidx, err := db.getBlkLoc(sha) if err != nil { // should the error here be normalized ? - log.Infof("block loc failed %v ", sha) + log.Tracef("block loc failed %v ", sha) return err } @@ -251,8 +243,8 @@ func (db *LevelDb) DropAfterBlockBySha(sha *btcwire.ShaHash) error { txUo.delete = true db.txUpdateMap[txSha] = &txUo } - db.bShaBatch().Delete(shaToKey(blksha)) - db.bBlkBatch().Delete(int64ToKey(height)) + db.lBatch().Delete(shaBlkToKey(blksha)) + db.lBatch().Delete(int64ToKey(height)) } db.nextBlock = keepidx + 1 @@ -264,26 +256,31 @@ func (db *LevelDb) DropAfterBlockBySha(sha *btcwire.ShaHash) error { // database. The first block inserted into the database will be treated as the // genesis block. Every subsequent block insert requires the referenced parent // block to already exist. -func (db *LevelDb) InsertBlock(block *btcutil.Block) (height int64, err error) { +func (db *LevelDb) InsertBlock(block *btcutil.Block) (height int64, rerr error) { db.dbLock.Lock() defer db.dbLock.Unlock() - defer db.processBatches() + defer func () { + if rerr == nil { + rerr = db.processBatches() + } + } () + blocksha, err := block.Sha() if err != nil { log.Warnf("Failed to compute block sha %v", blocksha) - return + return 0, err } mblock := block.MsgBlock() rawMsg, err := block.Bytes() if err != nil { log.Warnf("Failed to obtain raw block sha %v", blocksha) - return + return 0, err } txloc, err := block.TxLoc() if err != nil { log.Warnf("Failed to obtain raw block sha %v", blocksha) - return + return 0, err } // Insert block into database @@ -292,7 +289,7 @@ func (db *LevelDb) InsertBlock(block *btcutil.Block) (height int64, err error) { if err != nil { log.Warnf("Failed to insert block %v %v %v", blocksha, &mblock.Header.PrevBlock, err) - return + return 0, err } // At least two blocks in the long past were generated by faulty @@ -303,7 +300,7 @@ func (db *LevelDb) InsertBlock(block *btcutil.Block) (height int64, err error) { txsha, err = tx.TxSha() if err != nil { log.Warnf("failed to compute tx name block %v idx %v err %v", blocksha, txidx, err) - return + return 0, err } // Some old blocks contain duplicate transactions // Attempt to cleanly bypass this problem @@ -448,81 +445,68 @@ func (db *LevelDb) setclearSpentData(txsha *btcwire.ShaHash, idx uint32, set boo return nil } -func intToKey(keyint int) []byte { - key := fmt.Sprintf("%d", keyint) - return []byte(key) -} func int64ToKey(keyint int64) []byte { key := fmt.Sprintf("%d", keyint) return []byte(key) } -func shaToKey(sha *btcwire.ShaHash) []byte { - return sha.Bytes() +func shaBlkToKey(sha *btcwire.ShaHash) []byte { + shaB := sha.Bytes() + return shaB } -func (db *LevelDb) bShaBatch() *leveldb.Batch { - if db.bShabatch == nil { - db.bShabatch = new(leveldb.Batch) +func shaTxToKey(sha *btcwire.ShaHash) []byte { + shaB := sha.Bytes() + shaB = append(shaB, "tx"...) + return shaB +} + +func (db *LevelDb) lBatch() *leveldb.Batch { + if db.lbatch == nil { + db.lbatch = new(leveldb.Batch) } - return db.bShabatch + return db.lbatch } -func (db *LevelDb) bBlkBatch() *leveldb.Batch { - if db.bBlkbatch == nil { - db.bBlkbatch = new(leveldb.Batch) - } - return db.bBlkbatch -} - -func (db *LevelDb) processBatches() { +func (db *LevelDb) processBatches() error { var err error - if db.bShabatch != nil { - err = db.bShaDb.Write(db.bShabatch, db.wo) - db.bShabatch.Reset() - db.bShabatch = nil - if err != nil { - return - } - } - if db.bBlkbatch != nil { - err = db.bBlkDb.Write(db.bBlkbatch, db.wo) - db.bBlkbatch.Reset() - db.bBlkbatch = nil - if err != nil { - return - } - } - if len(db.txUpdateMap) != 0 { - tShabatch := new(leveldb.Batch) + if len(db.txUpdateMap) != 0 || db.lbatch != nil { + if db.lbatch == nil { + db.lbatch = new(leveldb.Batch) + } for txSha, txU := range db.txUpdateMap { - key := shaToKey(&txSha) + key := shaTxToKey(&txSha) if txU.delete { //log.Infof("deleting tx %v", txSha) - tShabatch.Delete(key) + db.lbatch.Delete(key) } else { //log.Infof("inserting tx %v", txSha) txdat, err := db.formatTx(txU) if err != nil { - return + return err } - tShabatch.Put(key, txdat) + db.lbatch.Put(key, txdat) } } - err = db.tShaDb.Write(tShabatch, db.wo) - tShabatch.Reset() + err = db.lDb.Write(db.lbatch, db.wo) if err != nil { - return + fmt.Printf("batch failed %v\n", err) + return err } + db.lbatch.Reset() db.txUpdateMap = map[btcwire.ShaHash]*txUpdateObj{} runtime.GC() } + return nil } func (db *LevelDb) RollbackClose() { + db.dbLock.Lock() + defer db.dbLock.Unlock() + db.close() } diff --git a/ldb/operational_test.go b/ldb/operational_test.go index 1000bd66..eb52db46 100644 --- a/ldb/operational_test.go +++ b/ldb/operational_test.go @@ -7,6 +7,7 @@ package ldb_test import ( "compress/bzip2" "encoding/binary" + "fmt" "github.com/conformal/btcdb" "github.com/conformal/btcutil" "github.com/conformal/btcwire" @@ -28,9 +29,9 @@ const ( func TestOperational(t *testing.T) { testOperationalMode(t, dbTmDefault) - testOperationalMode(t, dbTmNormal) - testOperationalMode(t, dbTmFast) - testOperationalMode(t, dbTmNoVerify) + //testOperationalMode(t, dbTmNormal) + //testOperationalMode(t, dbTmFast) + //testOperationalMode(t, dbTmNoVerify) } func testOperationalMode(t *testing.T, mode int) { @@ -40,7 +41,7 @@ func testOperationalMode(t *testing.T, mode int) { // 3) insert block // Ignore db remove errors since it means we didn't have an old one. - dbname := "tstdbop1" + dbname := fmt.Sprintf("tstdbop1.%d", mode) _ = os.RemoveAll(dbname) db, err := btcdb.CreateDB("leveldb", dbname) if err != nil { @@ -163,8 +164,8 @@ out: func TestBackout(t *testing.T) { testBackout(t, dbTmDefault) - testBackout(t, dbTmNormal) - testBackout(t, dbTmFast) + //testBackout(t, dbTmNormal) + //testBackout(t, dbTmFast) } func testBackout(t *testing.T, mode int) { @@ -173,8 +174,9 @@ func testBackout(t *testing.T, mode int) { // 2) look up all txin (except coinbase in db) // 3) insert block + t.Logf("mode %v", mode) // Ignore db remove errors since it means we didn't have an old one. - dbname := "tstdbop2" + dbname := fmt.Sprintf("tstdbop2.%d", mode) _ = os.RemoveAll(dbname) db, err := btcdb.CreateDB("leveldb", dbname) if err != nil { @@ -218,11 +220,11 @@ func testBackout(t *testing.T, mode int) { newheight, err := db.InsertBlock(block) if err != nil { t.Errorf("failed to insert block %v err %v", height, err) - break + return } if newheight != height { t.Errorf("height mismatch expect %v returned %v", height, newheight) - break + return } } @@ -245,6 +247,7 @@ func testBackout(t *testing.T, mode int) { _, err = db.FetchBlockBySha(sha) if err != nil { t.Errorf("failed to load block 99 from db %v", err) + return } sha, err = blocks[119].Sha() @@ -279,7 +282,14 @@ func testBackout(t *testing.T, mode int) { } } + +var savedblocks []*btcutil.Block + func loadBlocks(t *testing.T, file string) (blocks []*btcutil.Block, err error) { + if len(savedblocks) != 0 { + blocks = savedblocks + return + } testdatafile := filepath.Join("testdata", "blocks1-256.bz2") var dr io.Reader var fi io.ReadCloser @@ -340,6 +350,7 @@ func loadBlocks(t *testing.T, file string) (blocks []*btcutil.Block, err error) } blocks = append(blocks, block) } + savedblocks = blocks return } diff --git a/ldb/tx.go b/ldb/tx.go index 300f599f..7bc00300 100644 --- a/ldb/tx.go +++ b/ldb/tx.go @@ -60,28 +60,24 @@ func (db *LevelDb) formatTx(txu *txUpdateObj) ([]byte, error) { err := binary.Write(&txW, binary.LittleEndian, blkHeight) if err != nil { - fmt.Printf("fail encoding blkHeight %v\n", err) err = fmt.Errorf("Write fail") return nil, err } err = binary.Write(&txW, binary.LittleEndian, txOff) if err != nil { - fmt.Printf("fail encoding txoff %v\n", err) err = fmt.Errorf("Write fail") return nil, err } err = binary.Write(&txW, binary.LittleEndian, txLen) if err != nil { - fmt.Printf("fail encoding txlen %v\n", err) err = fmt.Errorf("Write fail") return nil, err } err = binary.Write(&txW, binary.LittleEndian, spentbuf) if err != nil { - fmt.Printf("fail encoding spentbuf %v\n", err) err = fmt.Errorf("Write fail") return nil, err } @@ -93,8 +89,8 @@ func (db *LevelDb) getTxData(txsha *btcwire.ShaHash) (rblkHeight int64, rtxOff int, rtxLen int, rspentBuf []byte, err error) { var buf []byte - key := shaToKey(txsha) - buf, err = db.tShaDb.Get(key, db.ro) + key := shaTxToKey(txsha) + buf, err = db.lDb.Get(key, db.ro) if err != nil { return } @@ -121,7 +117,6 @@ func (db *LevelDb) getTxData(txsha *btcwire.ShaHash) (rblkHeight int64, spentBuf := make([]byte, dr.Len()) err = binary.Read(dr, binary.LittleEndian, spentBuf) if err != nil { - fmt.Printf("fail encoding spentbuf %v\n", err) err = fmt.Errorf("Db Corrupt 4") return } From 66d6f10d5dcbf6ece82070514a24c63e513e687f Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Mon, 9 Sep 2013 15:47:46 -0500 Subject: [PATCH 044/163] Fix NewestSha. --- ldb/block.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ldb/block.go b/ldb/block.go index 8a5a9523..ab7e500f 100644 --- a/ldb/block.go +++ b/ldb/block.go @@ -253,9 +253,7 @@ func (db *LevelDb) NewestSha() (rsha *btcwire.ShaHash, rblkid int64, err error) defer db.dbLock.Unlock() if db.lastBlkIdx == -1 { - rblkid = db.lastBlkIdx - err = fmt.Errorf("Empty Database") - return + return &btcwire.ShaHash{}, -1, nil } sha := db.lastBlkSha From 30f20033014bab65e729740221d6ce4a372e591c Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Mon, 9 Sep 2013 15:48:18 -0500 Subject: [PATCH 045/163] Remove spurious log.Info prints. --- ldb/leveldb.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 488be28f..0c8937f8 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -73,7 +73,6 @@ func OpenDB(dbpath string) (btcdb.Db, error) { if err != nil { return nil, err } - log.Info("Opening DB\n") // Need to find last block and tx @@ -127,8 +126,6 @@ blocknarrow: ldb.lastBlkIdx = lastknownblock ldb.nextBlock = lastknownblock + 1 - log.Info("Opening DB: completed\n") - return db, nil } From 85553133ce04cc8809d37f6aab929f003650057a Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Mon, 9 Sep 2013 15:50:13 -0500 Subject: [PATCH 046/163] gofmt. --- ldb/boundary_test.go | 2 +- ldb/leveldb.go | 19 +++++++++---------- ldb/operational_test.go | 1 - 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/ldb/boundary_test.go b/ldb/boundary_test.go index 3e48f8f0..baffe339 100644 --- a/ldb/boundary_test.go +++ b/ldb/boundary_test.go @@ -13,7 +13,7 @@ import ( // we need to test for empty databas and make certain it returns proper value func TestEmptyDB(t *testing.T) { - + dbname := "tstdbempty" _ = os.RemoveAll(dbname) db, err := btcdb.CreateDB("leveldb", dbname) diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 0c8937f8..8d445029 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -45,8 +45,8 @@ type LevelDb struct { // leveldb pieces lDb *leveldb.DB - ro *opt.ReadOptions - wo *opt.WriteOptions + ro *opt.ReadOptions + wo *opt.WriteOptions lbatch *leveldb.Batch @@ -134,7 +134,7 @@ func openDB(dbpath string, flag opt.OptionsFlag) (pbdb btcdb.Db, err error) { var tlDb *leveldb.DB defer func() { if err == nil { - db.lDb = tlDb + db.lDb = tlDb db.txUpdateMap = map[btcwire.ShaHash]*txUpdateObj{} @@ -200,12 +200,12 @@ func (db *LevelDb) Close() { func (db *LevelDb) DropAfterBlockBySha(sha *btcwire.ShaHash) (rerr error) { db.dbLock.Lock() defer db.dbLock.Unlock() - defer func () { + defer func() { if rerr == nil { rerr = db.processBatches() } - } () + }() startheight := db.nextBlock - 1 @@ -256,12 +256,11 @@ func (db *LevelDb) DropAfterBlockBySha(sha *btcwire.ShaHash) (rerr error) { func (db *LevelDb) InsertBlock(block *btcutil.Block) (height int64, rerr error) { db.dbLock.Lock() defer db.dbLock.Unlock() - defer func () { + defer func() { if rerr == nil { rerr = db.processBatches() } - } () - + }() blocksha, err := block.Sha() if err != nil { @@ -448,12 +447,12 @@ func int64ToKey(keyint int64) []byte { } func shaBlkToKey(sha *btcwire.ShaHash) []byte { - shaB := sha.Bytes() + shaB := sha.Bytes() return shaB } func shaTxToKey(sha *btcwire.ShaHash) []byte { - shaB := sha.Bytes() + shaB := sha.Bytes() shaB = append(shaB, "tx"...) return shaB } diff --git a/ldb/operational_test.go b/ldb/operational_test.go index eb52db46..de1f9bab 100644 --- a/ldb/operational_test.go +++ b/ldb/operational_test.go @@ -282,7 +282,6 @@ func testBackout(t *testing.T, mode int) { } } - var savedblocks []*btcutil.Block func loadBlocks(t *testing.T, file string) (blocks []*btcutil.Block, err error) { From adc73cbc9daafde824571f847c15d4565dda2b3e Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Tue, 10 Sep 2013 11:25:13 -0400 Subject: [PATCH 047/163] quiet. --- ldb/block.go | 2 +- ldb/leveldb.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ldb/block.go b/ldb/block.go index ab7e500f..c7471cb4 100644 --- a/ldb/block.go +++ b/ldb/block.go @@ -36,7 +36,7 @@ func (db *LevelDb) getBlkLoc(sha *btcwire.ShaHash) (int64, error) { dr := bytes.NewBuffer(data) err = binary.Read(dr, binary.LittleEndian, &blkHeight) if err != nil { - fmt.Printf("get getBlkLoc len %v\n", len(data)) + log.Tracef("get getBlkLoc len %v\n", len(data)) err = fmt.Errorf("Db Corrupt 0") return 0, err } diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 8d445029..ed22318f 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -489,7 +489,7 @@ func (db *LevelDb) processBatches() error { err = db.lDb.Write(db.lbatch, db.wo) if err != nil { - fmt.Printf("batch failed %v\n", err) + log.Tracef("batch failed %v\n", err) return err } db.lbatch.Reset() From d9404fe88017921072bcfc15305e0700c35adb47 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Tue, 10 Sep 2013 10:44:26 -0500 Subject: [PATCH 048/163] Correct test for zero hash versus nil on empty db. --- ldb/boundary_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ldb/boundary_test.go b/ldb/boundary_test.go index baffe339..198fb87b 100644 --- a/ldb/boundary_test.go +++ b/ldb/boundary_test.go @@ -6,6 +6,7 @@ package ldb_test import ( "github.com/conformal/btcdb" + "github.com/conformal/btcwire" "os" "testing" ) @@ -34,8 +35,7 @@ func TestEmptyDB(t *testing.T) { defer db.Close() sha, height, err := db.NewestSha() - - if sha != nil { + if !sha.IsEqual(&btcwire.ShaHash{}) { t.Errorf("sha not nil") } if height != -1 { From 9049bcaceccfcbe02e5c4b79e1d661c60866c0d7 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Tue, 10 Sep 2013 10:47:49 -0500 Subject: [PATCH 049/163] Update error message for zero hash change. --- ldb/boundary_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ldb/boundary_test.go b/ldb/boundary_test.go index 198fb87b..dc0bdd06 100644 --- a/ldb/boundary_test.go +++ b/ldb/boundary_test.go @@ -36,7 +36,7 @@ func TestEmptyDB(t *testing.T) { sha, height, err := db.NewestSha() if !sha.IsEqual(&btcwire.ShaHash{}) { - t.Errorf("sha not nil") + t.Errorf("sha not zero hash") } if height != -1 { t.Errorf("height not -1 %v", height) From 7d679a6228d65c1a7dca81e0fe30e0423db009a8 Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Tue, 10 Sep 2013 14:17:24 -0400 Subject: [PATCH 050/163] Adjust Block height indicators properly when database is created. --- ldb/boundary_test.go | 10 +++++++++- ldb/leveldb.go | 8 +++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/ldb/boundary_test.go b/ldb/boundary_test.go index dc0bdd06..df2a6ebd 100644 --- a/ldb/boundary_test.go +++ b/ldb/boundary_test.go @@ -24,6 +24,14 @@ func TestEmptyDB(t *testing.T) { } defer os.RemoveAll(dbname) + sha, height, err := db.NewestSha() + if !sha.IsEqual(&btcwire.ShaHash{}) { + t.Errorf("sha not zero hash") + } + if height != -1 { + t.Errorf("height not -1 %v", height) + } + // This is a reopen test db.Close() @@ -34,7 +42,7 @@ func TestEmptyDB(t *testing.T) { } defer db.Close() - sha, height, err := db.NewestSha() + sha, height, err = db.NewestSha() if !sha.IsEqual(&btcwire.ShaHash{}) { t.Errorf("sha not zero hash") } diff --git a/ldb/leveldb.go b/ldb/leveldb.go index ed22318f..7713c5ce 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -170,7 +170,13 @@ func CreateDB(dbpath string) (btcdb.Db, error) { log = btcdb.GetLog() // No special setup needed, just OpenBB - return openDB(dbpath, opt.OFCreateIfMissing) + db, err := openDB(dbpath, opt.OFCreateIfMissing) + if err == nil { + ldb := db.(*LevelDb) + ldb.lastBlkIdx = -1 + ldb.nextBlock = 0 + } + return db, err } func (db *LevelDb) close() { From 5e4c78a1b701d0a9809ab039f2baa62af0bb1cc7 Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Tue, 10 Sep 2013 14:58:36 -0400 Subject: [PATCH 051/163] Fix a shadowed return parameter. --- sqlite3/sqliteblock.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/sqlite3/sqliteblock.go b/sqlite3/sqliteblock.go index e5a43ddf..43ef3410 100644 --- a/sqlite3/sqliteblock.go +++ b/sqlite3/sqliteblock.go @@ -23,12 +23,12 @@ func (db *SqliteDb) InsertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHa // insertSha stores a block hash and its associated data block with a // previous sha of `prevSha' and a version of `pver'. // insertSha shall be called with db lock held -func (db *SqliteDb) insertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHash, pver uint32, buf []byte) (blockid int64, err error) { +func (db *SqliteDb) insertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHash, pver uint32, buf []byte) (int64, error) { tx := &db.txState if tx.tx == nil { - err = db.startTx() + err := db.startTx() if err != nil { - return + return 0, err } } @@ -48,7 +48,7 @@ func (db *SqliteDb) insertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHa result, err := db.blkStmts[blkInsertSha].Exec(sha.Bytes(), pver, buf) if err != nil { - return + return 0, err } blkid, err := result.LastInsertId() @@ -68,8 +68,7 @@ func (db *SqliteDb) insertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHa tx.txInsertList = append(tx.txInsertList, bid) tx.txDataSz += len(buf) - blockid = blkid - return + return blkid, nil } // fetchSha returns the datablock and pver for the given ShaHash. From 9e27c82a231fc2073e4e7d72a67889fa98f71b60 Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Fri, 13 Sep 2013 11:49:30 -0400 Subject: [PATCH 052/163] Fix error messages (pointed out by go vet) --- ldb/operational_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ldb/operational_test.go b/ldb/operational_test.go index de1f9bab..4434c9e1 100644 --- a/ldb/operational_test.go +++ b/ldb/operational_test.go @@ -138,12 +138,12 @@ out: } if blkid != height { - t.Errorf("height doe not match latest block height %v %v", blkid, height, err) + t.Errorf("height doe not match latest block height %v %v %v", blkid, height, err) } blkSha, _ := block.Sha() if *newSha != *blkSha { - t.Errorf("Newest block sha does not match freshly inserted one %v %v ", newSha, blkSha, err) + t.Errorf("Newest block sha does not match freshly inserted one %v %v %v ", newSha, blkSha, err) } } From 1530141ba0a0a027fb905eaf70e7380b61b730be Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Wed, 11 Sep 2013 22:43:08 -0400 Subject: [PATCH 053/163] This query should be performed as part of the transaction, not outside it. --- sqlite3/sqliteblock.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite3/sqliteblock.go b/sqlite3/sqliteblock.go index 43ef3410..48b9f177 100644 --- a/sqlite3/sqliteblock.go +++ b/sqlite3/sqliteblock.go @@ -37,7 +37,7 @@ func (db *SqliteDb) insertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHa if prevOk := db.blkExistsSha(prevSha); !prevOk { var numBlocks uint64 querystr := "SELECT COUNT(blockid) FROM block;" - err := db.sqldb.QueryRow(querystr).Scan(&numBlocks) + err := tx.tx.QueryRow(querystr).Scan(&numBlocks) if err != nil { return 0, err } From a2946ea14bbe097e23ae51661c54552b373a5907 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Fri, 13 Sep 2013 17:31:00 -0500 Subject: [PATCH 054/163] Don't add "btcd.ldb" directory to provided path. --- ldb/leveldb.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 7713c5ce..683ca6cc 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -13,7 +13,6 @@ import ( "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/opt" "os" - "path/filepath" "runtime" "sync" ) @@ -156,8 +155,7 @@ func openDB(dbpath string, flag opt.OptionsFlag) (pbdb btcdb.Db, err error) { } } - lDbName := filepath.Join(dbpath, "btcd.ldb") - tlDb, err = leveldb.OpenFile(lDbName, &opt.Options{Flag: flag}) + tlDb, err = leveldb.OpenFile(dbpath, &opt.Options{Flag: flag}) if err != nil { return } From 62e38e29e539b1eb33b2c3cc79415d0beded57d8 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sun, 15 Sep 2013 15:18:46 -0500 Subject: [PATCH 055/163] Add a new function named SupportedDBs. This function allows the callers to programatically ascertain which database backend drivers are registered and therefore supported. --- db.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/db.go b/db.go index 808417ef..4c830df0 100644 --- a/db.go +++ b/db.go @@ -206,3 +206,13 @@ func OpenDB(dbtype string, argstr string) (pbdb Db, err error) { } return nil, DbUnknownType } + +// SupportedDBs returns a slice of strings that represent the database drivers +// that have been registered and are therefore supported. +func SupportedDBs() []string { + var supportedDBs []string + for _, drv := range driverList { + supportedDBs = append(supportedDBs, drv.DbType) + } + return supportedDBs +} From 18b3e86179ce32cb42be6f4ea42764b03e0bfa2c Mon Sep 17 00:00:00 2001 From: David Hill Date: Mon, 23 Sep 2013 15:39:53 -0400 Subject: [PATCH 056/163] FetchTxAllBySha needs to return err to the caller. ok drahn@ --- ldb/dbcache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ldb/dbcache.go b/ldb/dbcache.go index 6a9b824a..98cccd14 100644 --- a/ldb/dbcache.go +++ b/ldb/dbcache.go @@ -105,7 +105,7 @@ func (db *LevelDb) FetchTxAllBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, tx, txbuf, pver, blksha, _, _, err := db.fetchTxDataBySha(txsha) - return tx, txbuf, pver, blksha, nil + return tx, txbuf, pver, blksha, err } // FetchTxBySha returns some data for the given Tx Sha. From 0e98349c9818d93aa0154e8c1a207869ab75ea65 Mon Sep 17 00:00:00 2001 From: Marco Peereboom Date: Tue, 17 Sep 2013 10:38:56 -0500 Subject: [PATCH 057/163] disable compression, don't use cache and limit open files --- ldb/leveldb.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 683ca6cc..b0acfe1d 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -11,6 +11,7 @@ import ( "github.com/conformal/btcwire" "github.com/conformal/seelog" "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/cache" "github.com/syndtr/goleveldb/leveldb/opt" "os" "runtime" @@ -155,7 +156,10 @@ func openDB(dbpath string, flag opt.OptionsFlag) (pbdb btcdb.Db, err error) { } } - tlDb, err = leveldb.OpenFile(dbpath, &opt.Options{Flag: flag}) + tlDb, err = leveldb.OpenFile(dbpath, &opt.Options{Flag: flag, + BlockCache: cache.EmptyCache{}, + MaxOpenFiles: 256, + CompressionType: opt.NoCompression}) if err != nil { return } From 3640f6d37c224d0510d91e3aa604beacd606f069 Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Wed, 25 Sep 2013 16:18:35 -0400 Subject: [PATCH 058/163] Version the leveldb (and keep compatibility with pre-versioned uncompressed databases. --- ldb/boundary_test.go | 3 +++ ldb/doc.go | 7 +++++ ldb/insertremove_test.go | 3 +++ ldb/leveldb.go | 55 ++++++++++++++++++++++++++++++++++++++-- ldb/operational_test.go | 6 +++++ 5 files changed, 72 insertions(+), 2 deletions(-) diff --git a/ldb/boundary_test.go b/ldb/boundary_test.go index df2a6ebd..9d5c15f3 100644 --- a/ldb/boundary_test.go +++ b/ldb/boundary_test.go @@ -16,13 +16,16 @@ import ( func TestEmptyDB(t *testing.T) { dbname := "tstdbempty" + dbnamever := dbname + ".ver" _ = os.RemoveAll(dbname) + _ = os.RemoveAll(dbnamever) db, err := btcdb.CreateDB("leveldb", dbname) if err != nil { t.Errorf("Failed to open test database %v", err) return } defer os.RemoveAll(dbname) + defer os.RemoveAll(dbnamever) sha, height, err := db.NewestSha() if !sha.IsEqual(&btcwire.ShaHash{}) { diff --git a/ldb/doc.go b/ldb/doc.go index 65906a93..9ee98f21 100644 --- a/ldb/doc.go +++ b/ldb/doc.go @@ -11,5 +11,12 @@ The performance is generally high although it goes down with database size. Many of the block or tx specific functions for btcdb are in this subpackage. + +Database version number is stored in a flat file .ver +Currently a single (littlendian) integer in the file. If there is +additional data to save in the future, the presense of additional +data can be indicated by changing the version number, then parsing the +file differently. + */ package ldb diff --git a/ldb/insertremove_test.go b/ldb/insertremove_test.go index 5de70aba..26e9ad2b 100644 --- a/ldb/insertremove_test.go +++ b/ldb/insertremove_test.go @@ -46,13 +46,16 @@ func TestUnspentInsert(t *testing.T) { func testUnspentInsert(t *testing.T, mode int) { // Ignore db remove errors since it means we didn't have an old one. dbname := fmt.Sprintf("tstdbuspnt1.%d", mode) + dbnamever := dbname + ".ver" _ = os.RemoveAll(dbname) + _ = os.RemoveAll(dbnamever) db, err := btcdb.CreateDB("leveldb", dbname) if err != nil { t.Errorf("Failed to open test database %v", err) return } defer os.RemoveAll(dbname) + defer os.RemoveAll(dbnamever) defer db.Close() switch mode { diff --git a/ldb/leveldb.go b/ldb/leveldb.go index b0acfe1d..5389d5d5 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -5,6 +5,7 @@ package ldb import ( + "encoding/binary" "fmt" "github.com/conformal/btcdb" "github.com/conformal/btcutil" @@ -129,9 +130,13 @@ blocknarrow: return db, nil } +var CurrentDBVersion int32 = 1 + func openDB(dbpath string, flag opt.OptionsFlag) (pbdb btcdb.Db, err error) { var db LevelDb var tlDb *leveldb.DB + var dbversion int32 + defer func() { if err == nil { db.lDb = tlDb @@ -156,14 +161,60 @@ func openDB(dbpath string, flag opt.OptionsFlag) (pbdb btcdb.Db, err error) { } } - tlDb, err = leveldb.OpenFile(dbpath, &opt.Options{Flag: flag, + needVersionFile := false + verfile := dbpath + ".ver" + fi, ferr := os.Open(verfile) + if ferr == nil { + defer fi.Close() + + ferr = binary.Read(fi, binary.LittleEndian, &dbversion) + if ferr != nil { + dbversion = ^0 + } + } else { + if flag&opt.OFCreateIfMissing != 0 { + needVersionFile = true + } + } + + opts := &opt.Options{Flag: flag, BlockCache: cache.EmptyCache{}, MaxOpenFiles: 256, - CompressionType: opt.NoCompression}) + CompressionType: opt.NoCompression, + } + + switch dbversion { + case 0: + opts = &opt.Options{Flag: flag} + case 1: + // uses defaults from above + default: + err = fmt.Errorf("unsupported db version %v", dbversion) + return + } + + tlDb, err = leveldb.OpenFile(dbpath, opts) if err != nil { return } + // If we opened the database successfully on 'create' + // update the + if needVersionFile { + fo, ferr := os.Create(verfile) + if ferr != nil { + // TODO(design) close and delete database? + err = ferr + return + } + defer fo.Close() + dbversion = CurrentDBVersion + err = binary.Write(fo, binary.LittleEndian, dbversion) + if err != nil { + return + } + } + return } diff --git a/ldb/operational_test.go b/ldb/operational_test.go index 4434c9e1..d42fd5f2 100644 --- a/ldb/operational_test.go +++ b/ldb/operational_test.go @@ -42,13 +42,16 @@ func testOperationalMode(t *testing.T, mode int) { // Ignore db remove errors since it means we didn't have an old one. dbname := fmt.Sprintf("tstdbop1.%d", mode) + dbnamever := dbname + ".ver" _ = os.RemoveAll(dbname) + _ = os.RemoveAll(dbnamever) db, err := btcdb.CreateDB("leveldb", dbname) if err != nil { t.Errorf("Failed to open test database %v", err) return } defer os.RemoveAll(dbname) + defer os.RemoveAll(dbnamever) defer db.Close() switch mode { @@ -177,13 +180,16 @@ func testBackout(t *testing.T, mode int) { t.Logf("mode %v", mode) // Ignore db remove errors since it means we didn't have an old one. dbname := fmt.Sprintf("tstdbop2.%d", mode) + dbnamever := dbname + ".ver" _ = os.RemoveAll(dbname) + _ = os.RemoveAll(dbnamever) db, err := btcdb.CreateDB("leveldb", dbname) if err != nil { t.Errorf("Failed to open test database %v", err) return } defer os.RemoveAll(dbname) + defer os.RemoveAll(dbnamever) defer db.Close() switch mode { From e840c8314ce062cdf7dc4a1c445f6d1bd09f4f6e Mon Sep 17 00:00:00 2001 From: Marco Peereboom Date: Thu, 26 Sep 2013 10:15:14 -0500 Subject: [PATCH 059/163] catch up with api changes --- ldb/leveldb.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 5389d5d5..c1e77388 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -177,8 +177,9 @@ func openDB(dbpath string, flag opt.OptionsFlag) (pbdb btcdb.Db, err error) { } } + myCache := cache.NewEmptyCache() opts := &opt.Options{Flag: flag, - BlockCache: cache.EmptyCache{}, + BlockCache: myCache, MaxOpenFiles: 256, CompressionType: opt.NoCompression, } From 8d1db93c82651897222218b1eff3af8da7ae93ba Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Wed, 2 Oct 2013 16:12:02 -0400 Subject: [PATCH 060/163] Remove unused interfaces. --- db.go | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/db.go b/db.go index 4c830df0..06ea8e31 100644 --- a/db.go +++ b/db.go @@ -73,16 +73,6 @@ type Db interface { // the database ExistsTxSha(sha *btcwire.ShaHash) (exists bool) - // FetchTxAllBySha returns several pieces of data for a given - // transaction hash. The implementation may cache the underlying data - // if desired. - FetchTxAllBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rtxbuf []byte, rpver uint32, rblksha *btcwire.ShaHash, err error) - - // FetchTxBufBySha returns the raw bytes and associated protocol version - // for the transaction with the requested hash. The implementation may - // cache the underlying data if desired. - FetchTxBufBySha(txsha *btcwire.ShaHash) (txbuf []byte, rpver uint32, err error) - // FetchTxBySha returns some data for the given transaction hash. The // implementation may cache the underlying data if desired. FetchTxBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rpver uint32, blksha *btcwire.ShaHash, err error) @@ -91,24 +81,12 @@ type Db interface { // hashes. The implementation may cache the underlying data if desired. FetchTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply - // FetchTxUsedBySha returns the used/spent buffer for a given - // transaction hash. - FetchTxUsedBySha(txsha *btcwire.ShaHash) (spentbuf []byte, err error) - // InsertBlock inserts raw block and transaction data from a block // into the database. The first block inserted into the database // will be treated as the genesis block. Every subsequent block insert // requires the referenced parent block to already exist. InsertBlock(block *btcutil.Block) (height int64, err error) - // InsertBlockData stores a block hash and its associated data block - // with the given previous hash and protocol version into the database. - InsertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHash, pver uint32, buf []byte) (blockid int64, err error) - - // InsertTx stores a transaction hash and its associated data into the - // database. - InsertTx(txsha *btcwire.ShaHash, blockHeight int64, txoff int, txlen int, usedbuf []byte) (err error) - // InvalidateBlockCache releases all cached blocks. InvalidateBlockCache() From 63f8993163c558b7561f1d0b4799d261b7a34f09 Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Wed, 2 Oct 2013 18:10:00 -0400 Subject: [PATCH 061/163] Delete testing operations from obsolete functions. --- ldb/operational_test.go | 21 --------------------- sqlite3/insertfail_test.go | 4 ++-- sqlite3/operational_test.go | 28 +--------------------------- 3 files changed, 3 insertions(+), 50 deletions(-) diff --git a/ldb/operational_test.go b/ldb/operational_test.go index d42fd5f2..44d34bd3 100644 --- a/ldb/operational_test.go +++ b/ldb/operational_test.go @@ -93,26 +93,10 @@ out: t.Errorf("referenced tx not found %v ", origintxsha) } - _, _, _, _, err := db.FetchTxAllBySha(origintxsha) - if err != nil { - t.Errorf("referenced tx not found %v err %v ", origintxsha, err) - } - _, _, _, _, err = db.FetchTxAllBySha(origintxsha) - if err != nil { - t.Errorf("referenced tx not found %v err %v ", origintxsha, err) - } _, _, _, err = db.FetchTxBySha(origintxsha) if err != nil { t.Errorf("referenced tx not found %v err %v ", origintxsha, err) } - _, _, err = db.FetchTxBufBySha(origintxsha) - if err != nil { - t.Errorf("referenced tx not found %v err %v ", origintxsha, err) - } - _, err = db.FetchTxUsedBySha(origintxsha) - if err != nil { - t.Errorf("tx used fetch fail %v err %v ", origintxsha, err) - } } } txlist := db.FetchTxByShaList(txneededList) @@ -281,11 +265,6 @@ func testBackout(t *testing.T, mode int) { t.Errorf("tx %v not located db\n", txsha) return } - _, err = db.FetchTxUsedBySha(&txsha) - if err != nil { - t.Errorf("tx %v not located db\n", txsha) - return - } } var savedblocks []*btcutil.Block diff --git a/sqlite3/insertfail_test.go b/sqlite3/insertfail_test.go index 18961443..d6838034 100644 --- a/sqlite3/insertfail_test.go +++ b/sqlite3/insertfail_test.go @@ -84,7 +84,7 @@ out: txin := tx.TxIn[0] origintxsha := &txin.PreviousOutpoint.Hash sqlite3.KillTx(db, origintxsha) - _, _, _, _, err = db.FetchTxAllBySha(origintxsha) + _, _, _, err = db.FetchTxBySha(origintxsha) if err == nil { t.Errorf("deleted tx found %v", origintxsha) } @@ -107,7 +107,7 @@ out: if height == 248 { for _, tx := range mblock.Transactions { txsha, err := tx.TxSha() - _, _, _, _, err = db.FetchTxAllBySha(&txsha) + _, _, _, err = db.FetchTxBySha(&txsha) if err == nil { t.Errorf("referenced tx found, should not have been %v, ", txsha) } diff --git a/sqlite3/operational_test.go b/sqlite3/operational_test.go index 8132f91b..58cb40bb 100644 --- a/sqlite3/operational_test.go +++ b/sqlite3/operational_test.go @@ -99,26 +99,10 @@ out: t.Errorf("referenced tx not found %v ", origintxsha) } - _, _, _, _, err := db.FetchTxAllBySha(origintxsha) - if err != nil { - t.Errorf("referenced tx not found %v err %v ", origintxsha, err) - } - _, _, _, _, err = db.FetchTxAllBySha(origintxsha) - if err != nil { - t.Errorf("referenced tx not found %v err %v ", origintxsha, err) - } _, _, _, err = db.FetchTxBySha(origintxsha) if err != nil { t.Errorf("referenced tx not found %v err %v ", origintxsha, err) } - _, _, err = db.FetchTxBufBySha(origintxsha) - if err != nil { - t.Errorf("referenced tx not found %v err %v ", origintxsha, err) - } - _, err = db.FetchTxUsedBySha(origintxsha) - if err != nil { - t.Errorf("tx used fetch fail %v err %v ", origintxsha, err) - } } } txlist := db.FetchTxByShaList(txneededList) @@ -287,17 +271,7 @@ func testBackout(t *testing.T, mode int) { } _, _, _, err = db.FetchTxBySha(&txsha) - _, err = db.FetchTxUsedBySha(&txsha) - - block = blocks[99] - mblock = block.MsgBlock() - txsha, err = mblock.Transactions[0].TxSha() - oldused, err := db.FetchTxUsedBySha(&txsha) - err = db.InsertTx(&txsha, 99, 1024, 1048, oldused) - if err == nil { - t.Errorf("dup insert of tx succeeded") - return - } + _, _, _, err = db.FetchTxBySha(&txsha) } func loadBlocks(t *testing.T, file string) (blocks []*btcutil.Block, err error) { From 9abf071308d43f714b41be5dbafd5b5d7f53d80e Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 3 Oct 2013 10:32:19 -0400 Subject: [PATCH 062/163] increase default pagesize to 4096 on sqlite --- sqlite3/sqlite.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sqlite3/sqlite.go b/sqlite3/sqlite.go index 8f68d352..364c98d4 100644 --- a/sqlite3/sqlite.go +++ b/sqlite3/sqlite.go @@ -210,6 +210,10 @@ func newOrCreateSqliteDB(filepath string, create bool) (pbdb btcdb.Db, err error return nil, err } + db.Exec("PRAGMA page_size=4096;") + db.Exec("PRAGMA foreign_keys=ON;") + db.Exec("PRAGMA journal_mode=WAL;") + dbverstmt, err := db.Prepare("SELECT version FROM dbversion;") if err != nil { // about the only reason this would fail is that the database @@ -242,8 +246,6 @@ func newOrCreateSqliteDB(filepath string, create bool) (pbdb btcdb.Db, err error log.Warnf("mismatch db version: %v expected %v\n", version, dbVersion) return nil, fmt.Errorf("Invalid version in database") } - db.Exec("PRAGMA foreign_keys = ON;") - db.Exec("PRAGMA journal_mode=WAL;") bdb.sqldb = db bdb.blkStmts = make([]*sql.Stmt, len(blkqueries)) From 78d4bfecd411d0681abef28a4e348e96ca4c070d Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Wed, 2 Oct 2013 18:29:37 -0400 Subject: [PATCH 063/163] Remove obsolete API functions --- sqlite3/sqliteblock.go | 9 --------- sqlite3/sqlitedbcache.go | 21 ++++----------------- sqlite3/sqlitetx.go | 8 -------- 3 files changed, 4 insertions(+), 34 deletions(-) diff --git a/sqlite3/sqliteblock.go b/sqlite3/sqliteblock.go index 48b9f177..0be4bcd9 100644 --- a/sqlite3/sqliteblock.go +++ b/sqlite3/sqliteblock.go @@ -11,15 +11,6 @@ import ( _ "github.com/mattn/go-sqlite3" ) -// InsertBlockData stores a block hash and its associated data block with a -// previous sha of `prevSha' and a version of `pver'. -func (db *SqliteDb) InsertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHash, pver uint32, buf []byte) (blockid int64, err error) { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - return db.insertBlockData(sha, prevSha, pver, buf) -} - // insertSha stores a block hash and its associated data block with a // previous sha of `prevSha' and a version of `pver'. // insertSha shall be called with db lock held diff --git a/sqlite3/sqlitedbcache.go b/sqlite3/sqlitedbcache.go index 0553c54b..97079cf6 100644 --- a/sqlite3/sqlitedbcache.go +++ b/sqlite3/sqlitedbcache.go @@ -228,8 +228,8 @@ func (db *SqliteDb) fetchTxDataBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx return &tx, txbuf, pver, blksha, height, txspent, nil } -// FetchTxAllBySha returns several pieces of data regarding the given sha. -func (db *SqliteDb) FetchTxAllBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rtxbuf []byte, rpver uint32, rblksha *btcwire.ShaHash, err error) { +// FetchTxBySha returns several pieces of data regarding the given sha. +func (db *SqliteDb) FetchTxBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rpver uint32, rblksha *btcwire.ShaHash, err error) { var pver uint32 var blksha *btcwire.ShaHash var height int64 @@ -240,7 +240,7 @@ func (db *SqliteDb) FetchTxAllBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, // Check Tx cache if txc, ok := db.fetchTxCache(txsha); ok { - return txc.tx, txc.txbuf, txc.pver, &txc.blksha, nil + return txc.tx, txc.pver, &txc.blksha, nil } // If not cached load it @@ -293,20 +293,7 @@ func (db *SqliteDb) FetchTxAllBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, txc.blksha = *blksha db.insertTxCache(&txc) - return &tx, txbuf, pver, blksha, nil -} - -// FetchTxBySha returns some data for the given Tx Sha. -func (db *SqliteDb) FetchTxBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rpver uint32, blksha *btcwire.ShaHash, err error) { - rtx, _, rpver, blksha, err = db.FetchTxAllBySha(txsha) - return -} - -// FetchTxBufBySha return the bytestream data and associated protocol version. -// for the given Tx Sha -func (db *SqliteDb) FetchTxBufBySha(txsha *btcwire.ShaHash) (txbuf []byte, rpver uint32, err error) { - _, txbuf, rpver, _, err = db.FetchTxAllBySha(txsha) - return + return &tx, pver, blksha, nil } // fetchTxCache look up the given transaction in the Tx cache. diff --git a/sqlite3/sqlitetx.go b/sqlite3/sqlitetx.go index c735f0f9..2e4ccaa3 100644 --- a/sqlite3/sqlitetx.go +++ b/sqlite3/sqlitetx.go @@ -11,14 +11,6 @@ import ( _ "github.com/mattn/go-sqlite3" ) -// InsertTx inserts a tx hash and its associated data into the database. -func (db *SqliteDb) InsertTx(txsha *btcwire.ShaHash, height int64, txoff int, txlen int, usedbuf []byte) (err error) { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - return db.insertTx(txsha, height, txoff, txlen, usedbuf) -} - // insertTx inserts a tx hash and its associated data into the database. // Must be called with db lock held. func (db *SqliteDb) insertTx(txsha *btcwire.ShaHash, height int64, txoff int, txlen int, usedbuf []byte) (err error) { From 07d634c76aac7d574787c81f9f1691c402175799 Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Thu, 3 Oct 2013 10:15:13 -0400 Subject: [PATCH 064/163] remove old API function. --- ldb/dbcache.go | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/ldb/dbcache.go b/ldb/dbcache.go index 98cccd14..a9667846 100644 --- a/ldb/dbcache.go +++ b/ldb/dbcache.go @@ -98,26 +98,9 @@ func (db *LevelDb) fetchTxDataBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, return &tx, txbuf, pver, blksha, blkHeight, txspent, nil } -// FetchTxAllBySha returns several pieces of data regarding the given sha. -func (db *LevelDb) FetchTxAllBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rtxbuf []byte, rpver uint32, rblksha *btcwire.ShaHash, err error) { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - tx, txbuf, pver, blksha, _, _, err := db.fetchTxDataBySha(txsha) - - return tx, txbuf, pver, blksha, err -} - // FetchTxBySha returns some data for the given Tx Sha. func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rpver uint32, blksha *btcwire.ShaHash, err error) { - rtx, _, rpver, blksha, err = db.FetchTxAllBySha(txsha) - return -} - -// FetchTxBufBySha return the bytestream data and associated protocol version. -// for the given Tx Sha -func (db *LevelDb) FetchTxBufBySha(txsha *btcwire.ShaHash) (txbuf []byte, rpver uint32, err error) { - _, txbuf, rpver, _, err = db.FetchTxAllBySha(txsha) + rtx, _, rpver, blksha, _, _, err = db.fetchTxDataBySha(txsha) return } From cda0b10082332fdc0f2596b03af67ee4a151cf2e Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Thu, 3 Oct 2013 15:21:45 -0400 Subject: [PATCH 065/163] Introduce an API change for btcdb FetchTxBySha changes what it returns, it can now return a TxListReply and and error if none are found. FetchTxByShaList is renamed to FetchUnSpentTxByShaList to indicate that it will (likey/eventually) only return Tx that have some unspent TxOuts. Tx which are fully spent may not be (reliably) looked up using this API. --- db.go | 5 ++-- ldb/block.go | 13 ++-------- ldb/dbcache.go | 38 ++++++++++++++++++---------- ldb/insertremove_test.go | 8 +++--- ldb/leveldb.go | 5 ---- ldb/operational_test.go | 6 ++--- sqlite3/insertfail_test.go | 4 +-- sqlite3/insertremove_test.go | 8 +++--- sqlite3/operational_test.go | 8 +++--- sqlite3/sqlitedbcache.go | 48 +++++++++++++++++++++++++++--------- 10 files changed, 84 insertions(+), 59 deletions(-) diff --git a/db.go b/db.go index 06ea8e31..b15c95b6 100644 --- a/db.go +++ b/db.go @@ -75,11 +75,11 @@ type Db interface { // FetchTxBySha returns some data for the given transaction hash. The // implementation may cache the underlying data if desired. - FetchTxBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rpver uint32, blksha *btcwire.ShaHash, err error) + FetchTxBySha(txsha *btcwire.ShaHash) ([]*TxListReply, error) // FetchTxByShaList returns a TxListReply given an array of transaction // hashes. The implementation may cache the underlying data if desired. - FetchTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply + FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) ([]*TxListReply) // InsertBlock inserts raw block and transaction data from a block // into the database. The first block inserted into the database @@ -145,6 +145,7 @@ type DriverDB struct { type TxListReply struct { Sha *btcwire.ShaHash Tx *btcwire.MsgTx + BlkSha *btcwire.ShaHash Height int64 TxSpent []bool Err error diff --git a/ldb/block.go b/ldb/block.go index c7471cb4..6423ac1c 100644 --- a/ldb/block.go +++ b/ldb/block.go @@ -12,15 +12,6 @@ import ( "github.com/conformal/btcwire" ) -// InsertBlockData stores a block hash and its associated data block with a -// previous sha of `prevSha' and a version of `pver'. -func (db *LevelDb) InsertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHash, pver uint32, buf []byte) (blockid int64, err error) { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - return db.insertBlockData(sha, prevSha, buf) -} - func (db *LevelDb) getBlkLoc(sha *btcwire.ShaHash) (int64, error) { var blkHeight int64 @@ -107,7 +98,7 @@ func (db *LevelDb) setBlk(sha *btcwire.ShaHash, blkHeight int64, buf []byte) err } // insertSha stores a block hash and its associated data block with a -// previous sha of `prevSha' and a version of `pver'. +// previous sha of `prevSha'. // insertSha shall be called with db lock held func (db *LevelDb) insertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHash, buf []byte) (blockid int64, err error) { @@ -144,7 +135,7 @@ func (db *LevelDb) insertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHas return blkHeight, nil } -// fetchSha returns the datablock and pver for the given ShaHash. +// fetchSha returns the datablock for the given ShaHash. func (db *LevelDb) fetchSha(sha *btcwire.ShaHash) (rbuf []byte, rblkHeight int64, err error) { var blkHeight int64 diff --git a/ldb/dbcache.go b/ldb/dbcache.go index a9667846..0ac0506e 100644 --- a/ldb/dbcache.go +++ b/ldb/dbcache.go @@ -36,15 +36,15 @@ func (db *LevelDb) fetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, er return } -// FetchTxByShaList given a array of ShaHash, look up the transactions +// FetchUnSpentTxByShaList given a array of ShaHash, look up the transactions // and return them in a TxListReply array. -func (db *LevelDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { +func (db *LevelDb) FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { db.dbLock.Lock() defer db.dbLock.Unlock() replies := make([]*btcdb.TxListReply, len(txShaList)) for i, txsha := range txShaList { - tx, _, _, _, height, txspent, err := db.fetchTxDataBySha(txsha) + tx, blockSha, height, txspent, err := db.fetchTxDataBySha(txsha) btxspent := []bool{} if err == nil { btxspent = make([]bool, len(tx.TxOut), len(tx.TxOut)) @@ -54,15 +54,14 @@ func (db *LevelDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxLis btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0 } } - txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, Height: height, TxSpent: btxspent, Err: err} + txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, BlkSha: blockSha, Height: height, TxSpent: btxspent, Err: err} replies[i] = &txlre } return replies } // fetchTxDataBySha returns several pieces of data regarding the given sha. -func (db *LevelDb) fetchTxDataBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rtxbuf []byte, rpver uint32, rblksha *btcwire.ShaHash, rheight int64, rtxspent []byte, err error) { - var pver uint32 +func (db *LevelDb) fetchTxDataBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rblksha *btcwire.ShaHash, rheight int64, rtxspent []byte, err error) { var blksha *btcwire.ShaHash var blkHeight int64 var txspent []byte @@ -83,9 +82,7 @@ func (db *LevelDb) fetchTxDataBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, //log.Trace("transaction %v is at block %v %v txoff %v, txlen %v\n", // txsha, blksha, blkHeight, txOff, txLen) - txbuf := make([]byte, txLen) - copy(txbuf[:], blkbuf[txOff:txOff+txLen]) - rbuf := bytes.NewBuffer(txbuf) + rbuf := bytes.NewBuffer(blkbuf[txOff:txOff+txLen]) var tx btcwire.MsgTx err = tx.Deserialize(rbuf) @@ -95,13 +92,28 @@ func (db *LevelDb) fetchTxDataBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, return } - return &tx, txbuf, pver, blksha, blkHeight, txspent, nil + return &tx, blksha, blkHeight, txspent, nil } // FetchTxBySha returns some data for the given Tx Sha. -func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rpver uint32, blksha *btcwire.ShaHash, err error) { - rtx, _, rpver, blksha, _, _, err = db.fetchTxDataBySha(txsha) - return +func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) ([]*btcdb.TxListReply, error) { + tx, blksha, height, txspent, err := db.fetchTxDataBySha(txsha) + if err != nil { + return []*btcdb.TxListReply{}, err + } + + replies := make ([]*btcdb.TxListReply, 1) + + btxspent := make([]bool, len(tx.TxOut), len(tx.TxOut)) + for idx := range tx.TxOut { + byteidx := idx / 8 + byteoff := uint(idx % 8) + btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0 + } + + txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, BlkSha: blksha, Height: height, TxSpent: btxspent, Err: err} + replies[0] = &txlre + return replies, nil } // InvalidateTxCache clear/release all cached transactions. diff --git a/ldb/insertremove_test.go b/ldb/insertremove_test.go index 26e9ad2b..fadf4799 100644 --- a/ldb/insertremove_test.go +++ b/ldb/insertremove_test.go @@ -104,7 +104,7 @@ endtest: } txneededmap := map[btcwire.ShaHash]*btcdb.TxListReply{} - txlist := db.FetchTxByShaList(txneededList) + txlist := db.FetchUnSpentTxByShaList(txneededList) for _, txe := range txlist { if txe.Err != nil { t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) @@ -130,7 +130,7 @@ endtest: } txlookupmap := map[btcwire.ShaHash]*btcdb.TxListReply{} - txlist = db.FetchTxByShaList(txlookupList) + txlist = db.FetchUnSpentTxByShaList(txlookupList) for _, txe := range txlist { if txe.Err != nil { t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) @@ -166,7 +166,7 @@ endtest: } txlookupmap = map[btcwire.ShaHash]*btcdb.TxListReply{} - txlist = db.FetchTxByShaList(txlookupList) + txlist = db.FetchUnSpentTxByShaList(txlookupList) for _, txe := range txlist { if txe.Err != nil { if _, ok := txneededmap[*txe.Sha]; ok { @@ -188,7 +188,7 @@ endtest: break endtest } txlookupmap = map[btcwire.ShaHash]*btcdb.TxListReply{} - txlist = db.FetchTxByShaList(txlookupList) + txlist = db.FetchUnSpentTxByShaList(txlookupList) for _, txe := range txlist { if txe.Err != nil { t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) diff --git a/ldb/leveldb.go b/ldb/leveldb.go index c1e77388..24f9b3a1 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -27,11 +27,6 @@ const ( var log seelog.LoggerInterface = seelog.Disabled -type tBlockInsertData struct { - sha btcwire.ShaHash - pver uint32 - buf []byte -} type tTxInsertData struct { txsha *btcwire.ShaHash blockid int64 diff --git a/ldb/operational_test.go b/ldb/operational_test.go index 44d34bd3..0eab65e9 100644 --- a/ldb/operational_test.go +++ b/ldb/operational_test.go @@ -93,13 +93,13 @@ out: t.Errorf("referenced tx not found %v ", origintxsha) } - _, _, _, err = db.FetchTxBySha(origintxsha) + _, err = db.FetchTxBySha(origintxsha) if err != nil { t.Errorf("referenced tx not found %v err %v ", origintxsha, err) } } } - txlist := db.FetchTxByShaList(txneededList) + txlist := db.FetchUnSpentTxByShaList(txneededList) for _, txe := range txlist { if txe.Err != nil { t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) @@ -260,7 +260,7 @@ func testBackout(t *testing.T, mode int) { t.Errorf("tx %v not located db\n", txsha) } - _, _, _, err = db.FetchTxBySha(&txsha) + _, err = db.FetchTxBySha(&txsha) if err != nil { t.Errorf("tx %v not located db\n", txsha) return diff --git a/sqlite3/insertfail_test.go b/sqlite3/insertfail_test.go index d6838034..4c27e2c8 100644 --- a/sqlite3/insertfail_test.go +++ b/sqlite3/insertfail_test.go @@ -84,7 +84,7 @@ out: txin := tx.TxIn[0] origintxsha := &txin.PreviousOutpoint.Hash sqlite3.KillTx(db, origintxsha) - _, _, _, err = db.FetchTxBySha(origintxsha) + _, err = db.FetchTxBySha(origintxsha) if err == nil { t.Errorf("deleted tx found %v", origintxsha) } @@ -107,7 +107,7 @@ out: if height == 248 { for _, tx := range mblock.Transactions { txsha, err := tx.TxSha() - _, _, _, err = db.FetchTxBySha(&txsha) + _, err = db.FetchTxBySha(&txsha) if err == nil { t.Errorf("referenced tx found, should not have been %v, ", txsha) } diff --git a/sqlite3/insertremove_test.go b/sqlite3/insertremove_test.go index ca8baf5b..c49595c6 100644 --- a/sqlite3/insertremove_test.go +++ b/sqlite3/insertremove_test.go @@ -107,7 +107,7 @@ endtest: } txneededmap := map[btcwire.ShaHash]*btcdb.TxListReply{} - txlist := db.FetchTxByShaList(txneededList) + txlist := db.FetchUnSpentTxByShaList(txneededList) for _, txe := range txlist { if txe.Err != nil { t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) @@ -133,7 +133,7 @@ endtest: } txlookupmap := map[btcwire.ShaHash]*btcdb.TxListReply{} - txlist = db.FetchTxByShaList(txlookupList) + txlist = db.FetchUnSpentTxByShaList(txlookupList) for _, txe := range txlist { if txe.Err != nil { t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) @@ -169,7 +169,7 @@ endtest: } txlookupmap = map[btcwire.ShaHash]*btcdb.TxListReply{} - txlist = db.FetchTxByShaList(txlookupList) + txlist = db.FetchUnSpentTxByShaList(txlookupList) for _, txe := range txlist { if txe.Err != nil { if _, ok := txneededmap[*txe.Sha]; ok { @@ -191,7 +191,7 @@ endtest: break endtest } txlookupmap = map[btcwire.ShaHash]*btcdb.TxListReply{} - txlist = db.FetchTxByShaList(txlookupList) + txlist = db.FetchUnSpentTxByShaList(txlookupList) for _, txe := range txlist { if txe.Err != nil { t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) diff --git a/sqlite3/operational_test.go b/sqlite3/operational_test.go index 58cb40bb..b9eba899 100644 --- a/sqlite3/operational_test.go +++ b/sqlite3/operational_test.go @@ -99,13 +99,13 @@ out: t.Errorf("referenced tx not found %v ", origintxsha) } - _, _, _, err = db.FetchTxBySha(origintxsha) + _, err = db.FetchTxBySha(origintxsha) if err != nil { t.Errorf("referenced tx not found %v err %v ", origintxsha, err) } } } - txlist := db.FetchTxByShaList(txneededList) + txlist := db.FetchUnSpentTxByShaList(txneededList) for _, txe := range txlist { if txe.Err != nil { t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) @@ -270,8 +270,8 @@ func testBackout(t *testing.T, mode int) { t.Errorf("tx %v exists in db, failure expected", txsha) } - _, _, _, err = db.FetchTxBySha(&txsha) - _, _, _, err = db.FetchTxBySha(&txsha) + _, err = db.FetchTxBySha(&txsha) + _, err = db.FetchTxBySha(&txsha) } func loadBlocks(t *testing.T, file string) (blocks []*btcutil.Block, err error) { diff --git a/sqlite3/sqlitedbcache.go b/sqlite3/sqlitedbcache.go index 97079cf6..daac3ec2 100644 --- a/sqlite3/sqlitedbcache.go +++ b/sqlite3/sqlitedbcache.go @@ -131,9 +131,9 @@ func (db *SqliteDb) insertBlockCache(sha *btcwire.ShaHash, blk *btcutil.Block) { bc.blockMap[blkObj.sha] = &blkObj } -// FetchTxByShaList given a array of ShaHash, look up the transactions +// FetchUnSpentTxByShaList given a array of ShaHash, look up the transactions // and return them in a TxListReply array. -func (db *SqliteDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { +func (db *SqliteDb) FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { db.dbLock.Lock() defer db.dbLock.Unlock() @@ -229,30 +229,44 @@ func (db *SqliteDb) fetchTxDataBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx } // FetchTxBySha returns several pieces of data regarding the given sha. -func (db *SqliteDb) FetchTxBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rpver uint32, rblksha *btcwire.ShaHash, err error) { +func (db *SqliteDb) FetchTxBySha(txsha *btcwire.ShaHash) ([]*btcdb.TxListReply, error) { var pver uint32 var blksha *btcwire.ShaHash var height int64 var toff int var tlen int + var txspent []byte var blk *btcutil.Block var blkbuf []byte + var err error // Check Tx cache if txc, ok := db.fetchTxCache(txsha); ok { - return txc.tx, txc.pver, &txc.blksha, nil + replies := make ([]*btcdb.TxListReply, 1) + + tx := txc.tx + btxspent := make([]bool, len(tx.TxOut), len(tx.TxOut)) + for idx := range tx.TxOut { + byteidx := idx / 8 + byteoff := uint(idx % 8) + btxspent[idx] = (txc.spent[byteidx] & (byte(1) << byteoff)) != 0 + } + + txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, BlkSha: &txc.blksha, Height: txc.height, TxSpent: btxspent, Err: nil} + replies[0] = &txlre + return replies, nil } // If not cached load it - height, toff, tlen, err = db.FetchLocationBySha(txsha) + height, toff, tlen, txspent, err = db.fetchLocationUsedBySha(txsha) if err != nil { - return + return []*btcdb.TxListReply{}, err } blksha, err = db.FetchBlockShaByHeight(height) if err != nil { log.Warnf("block idx lookup %v to %v", height, err) - return + return []*btcdb.TxListReply{}, err } log.Tracef("transaction %v is at block %v %v tx %v", txsha, blksha, height, toff) @@ -261,13 +275,13 @@ func (db *SqliteDb) FetchTxBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rp if err != nil { log.Warnf("unable to fetch block %v %v ", height, &blksha) - return + return []*btcdb.TxListReply{}, err } blkbuf, err = blk.Bytes() if err != nil { log.Warnf("unable to decode block %v %v", height, &blksha) - return + return []*btcdb.TxListReply{}, err } txbuf := make([]byte, tlen) @@ -279,7 +293,7 @@ func (db *SqliteDb) FetchTxBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rp if err != nil { log.Warnf("unable to decode tx block %v %v txoff %v txlen %v", height, &blksha, toff, tlen) - return + return []*btcdb.TxListReply{}, err } // Shove data into TxCache @@ -290,10 +304,22 @@ func (db *SqliteDb) FetchTxBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rp txc.txbuf = txbuf txc.pver = pver txc.height = height + txc.spent = txspent txc.blksha = *blksha db.insertTxCache(&txc) - return &tx, pver, blksha, nil + + btxspent := make([]bool, len(tx.TxOut), len(tx.TxOut)) + for idx := range tx.TxOut { + byteidx := idx / 8 + byteoff := uint(idx % 8) + btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0 + } + + replies := make ([]*btcdb.TxListReply, 1) + txlre := btcdb.TxListReply{Sha: txsha, Tx: &tx, BlkSha: blksha, Height: height, TxSpent: btxspent, Err: err} + replies[0] = &txlre + return replies, nil } // fetchTxCache look up the given transaction in the Tx cache. From e4d3f259912ab65f1ee64635bab536ded54b6ef9 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Fri, 4 Oct 2013 11:40:41 -0500 Subject: [PATCH 066/163] Update imports for goleveldb to Conformal fork. --- ldb/dbtest/dbtst.go | 4 ++-- ldb/leveldb.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ldb/dbtest/dbtst.go b/ldb/dbtest/dbtst.go index 15d905bb..7fccb0cd 100644 --- a/ldb/dbtest/dbtst.go +++ b/ldb/dbtest/dbtst.go @@ -3,8 +3,8 @@ package main import ( "fmt" - "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/opt" + "github.com/conformal/goleveldb/leveldb" + "github.com/conformal/goleveldb/leveldb/opt" ) type tst struct { diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 24f9b3a1..eb393ac8 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -11,9 +11,9 @@ import ( "github.com/conformal/btcutil" "github.com/conformal/btcwire" "github.com/conformal/seelog" - "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/cache" - "github.com/syndtr/goleveldb/leveldb/opt" + "github.com/conformal/goleveldb/leveldb" + "github.com/conformal/goleveldb/leveldb/cache" + "github.com/conformal/goleveldb/leveldb/opt" "os" "runtime" "sync" From 6260dc959d9d2d37a38389313385dc17066af2f5 Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Fri, 11 Oct 2013 07:35:12 -0400 Subject: [PATCH 067/163] When creating database, set version 1 so the correct options are set. --- ldb/leveldb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ldb/leveldb.go b/ldb/leveldb.go index eb393ac8..a74b321a 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -169,6 +169,7 @@ func openDB(dbpath string, flag opt.OptionsFlag) (pbdb btcdb.Db, err error) { } else { if flag&opt.OFCreateIfMissing != 0 { needVersionFile = true + dbversion = CurrentDBVersion } } @@ -204,7 +205,6 @@ func openDB(dbpath string, flag opt.OptionsFlag) (pbdb btcdb.Db, err error) { return } defer fo.Close() - dbversion = CurrentDBVersion err = binary.Write(fo, binary.LittleEndian, dbversion) if err != nil { return From 1f87ee217aafe9a20c226401b98c5390f4754b5c Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sat, 12 Oct 2013 01:48:27 -0500 Subject: [PATCH 068/163] Move testdata to root btcd directory. Rather than duplicating the same data, just put it under testsdata at the root and access it from the modules from the parent directory. --- ldb/insertremove_test.go | 2 +- ldb/operational_test.go | 6 +++--- sqlite3/insertfail_test.go | 2 +- sqlite3/insertremove_test.go | 2 +- sqlite3/operational_test.go | 6 +++--- sqlite3/testdata/blocks1-256.bz2 | Bin 37555 -> 0 bytes {ldb/testdata => testdata}/blocks1-256.bz2 | Bin 7 files changed, 9 insertions(+), 9 deletions(-) delete mode 100644 sqlite3/testdata/blocks1-256.bz2 rename {ldb/testdata => testdata}/blocks1-256.bz2 (100%) diff --git a/ldb/insertremove_test.go b/ldb/insertremove_test.go index fadf4799..b5ce2dc5 100644 --- a/ldb/insertremove_test.go +++ b/ldb/insertremove_test.go @@ -22,7 +22,7 @@ func loadblocks(t *testing.T) []*btcutil.Block { return tstBlocks } - testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + testdatafile := filepath.Join("..", "testdata", "blocks1-256.bz2") blocks, err := loadBlocks(t, testdatafile) if err != nil { t.Errorf("Unable to load blocks from test data: %v", err) diff --git a/ldb/operational_test.go b/ldb/operational_test.go index 0eab65e9..89fb1425 100644 --- a/ldb/operational_test.go +++ b/ldb/operational_test.go @@ -65,7 +65,7 @@ func testOperationalMode(t *testing.T, mode int) { db.SetDBInsertMode(btcdb.InsertValidatedInput) } - testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + testdatafile := filepath.Join("..", "testdata", "blocks1-256.bz2") blocks, err := loadBlocks(t, testdatafile) if err != nil { t.Errorf("Unable to load blocks from test data for mode %v: %v", @@ -185,7 +185,7 @@ func testBackout(t *testing.T, mode int) { } - testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + testdatafile := filepath.Join("..", "testdata", "blocks1-256.bz2") blocks, err := loadBlocks(t, testdatafile) if len(blocks) < 120 { t.Errorf("test data too small") @@ -274,7 +274,7 @@ func loadBlocks(t *testing.T, file string) (blocks []*btcutil.Block, err error) blocks = savedblocks return } - testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + testdatafile := filepath.Join("..", "testdata", "blocks1-256.bz2") var dr io.Reader var fi io.ReadCloser fi, err = os.Open(testdatafile) diff --git a/sqlite3/insertfail_test.go b/sqlite3/insertfail_test.go index 4c27e2c8..391c3146 100644 --- a/sqlite3/insertfail_test.go +++ b/sqlite3/insertfail_test.go @@ -58,7 +58,7 @@ func failtestOperationalMode(t *testing.T, mode int) { sqlite3.SetBlockCacheSize(db, 2) sqlite3.SetTxCacheSize(db, 3) - testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + testdatafile := filepath.Join("..", "testdata", "blocks1-256.bz2") blocks, err := loadBlocks(t, testdatafile) if err != nil { t.Errorf("Unable to load blocks from test data for mode %v: %v", diff --git a/sqlite3/insertremove_test.go b/sqlite3/insertremove_test.go index c49595c6..4b0f586e 100644 --- a/sqlite3/insertremove_test.go +++ b/sqlite3/insertremove_test.go @@ -21,7 +21,7 @@ func loadblocks(t *testing.T) []*btcutil.Block { return tstBlocks } - testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + testdatafile := filepath.Join("..", "testdata", "blocks1-256.bz2") blocks, err := loadBlocks(t, testdatafile) if err != nil { t.Errorf("Unable to load blocks from test data: %v", err) diff --git a/sqlite3/operational_test.go b/sqlite3/operational_test.go index b9eba899..da73a962 100644 --- a/sqlite3/operational_test.go +++ b/sqlite3/operational_test.go @@ -71,7 +71,7 @@ func testOperationalMode(t *testing.T, mode int) { sqlite3.SetBlockCacheSize(db, 2) sqlite3.SetTxCacheSize(db, 3) - testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + testdatafile := filepath.Join("..", "testdata", "blocks1-256.bz2") blocks, err := loadBlocks(t, testdatafile) if err != nil { t.Errorf("Unable to load blocks from test data for mode %v: %v", @@ -196,7 +196,7 @@ func testBackout(t *testing.T, mode int) { sqlite3.SetBlockCacheSize(db, 2) sqlite3.SetTxCacheSize(db, 3) - testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + testdatafile := filepath.Join("..", "testdata", "blocks1-256.bz2") blocks, err := loadBlocks(t, testdatafile) if len(blocks) < 120 { t.Errorf("test data too small") @@ -275,7 +275,7 @@ func testBackout(t *testing.T, mode int) { } func loadBlocks(t *testing.T, file string) (blocks []*btcutil.Block, err error) { - testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + testdatafile := filepath.Join("..", "testdata", "blocks1-256.bz2") var dr io.Reader var fi io.ReadCloser fi, err = os.Open(testdatafile) diff --git a/sqlite3/testdata/blocks1-256.bz2 b/sqlite3/testdata/blocks1-256.bz2 deleted file mode 100644 index 6b8bda4429200c0566bb13c28c35d6397272e475..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 37555 zcmV)8K*qm9T4*^jL0KkKSr@A=?*Lyo|NsC0|NsC0|NsC0|NsC0|NsC0|NsC0|NsC0 z|NsC0|Nr1%0`GtT00*bHz1|n5T)=M=pBnqU;dPxJy+dC1_BM0g?t47Wcf|GKdY#;B zx%Jb_&i2~vj_uc8@w++d_r2Xc+bQeWwB7AnH@m&s?|Z7VcLwh6_&)Ua-C7%VGQQc~ z>${h|?$>*(yBXJ;E3Q`cd%f+uRJ}XA?=#ch z^LF0cm$dfxz3JWU#m@D$*Rj3c>USOO?{`dUw52*PZ4) zZsq3L?C!nYb#Ajg-1geucDp>i-Q{VQx3xUI&gD6`Uawu6^=;dCb=vn_^?SRn>aKNl zE$pv*o-NzE&uh^0y|K@GzIV4+$bmfo002w?nqp#L0%cFrLlY(dm;eaCCJBH5N#bAt zO*GR4VqgTqU=ssCXc_@FgCjbAaCIAVQ4yFcx0gyDQ{z0Gs4FX^(;0dOf69QlWOd2#a z4FC-Q0fYo#1w0VRlT9>W35k^S(-Q<>m=TeQsvT87z%&|Zsek|g&_ zj3y%jGfhmMr>UAIk3kuxqef2@z$c;x$u^Pc4@lCq5G;R}q9*?O-;!gJMnwV1tNmyb zJ@fx!pSG8@UQWKV&A3<1{kZJNqQ08k z6D?(C9~^K8No6azD{saS5up0CIL)USziD_I?C)=&!gQRRl1VS{+J?XC_#bbw?xgcw zjlnWQhmuJmK$1=Nq>@QeY{iyDRa8VqDzR1~h>SsCD;S`vMFkaz#a1j>F^Y@@ixB}8 zR7F^*f~;V~WLU;U0gMLL5g4e*#6~J23L=XU6%+sgVk!!%C^905f(&B?Rz?W0j2J=! zkz&A5Bv=9>DkzG^F;zul6^s@Nf+(vMV51aRs)#D6ixx2yRYe3*Q4s|eF+^2@v4W~9 z3lL()AgmOMip7Gf76_n>7{x?j!Bi0#Dx^^qh$zHCR8?TA005$jqA-Mjq=2G;p(GR_ zERkRYhGl^yhX}|(fNQV-4dYK>n#ItplMB?#{#=cV!a#-X6#_w-Hww)~IQDocyC!q_ zP;QC6GWpfg{SRTvVKK%&;D83JAV?@cB9MTfNoG=P`RhZcW`%~VouXmT%%>Q`%gFiG znI{I2!l$s1vkuvTf^(_QFeC#yn%e`wLK@aVJW{_@>IkMu*Tz znP6aPZ9IqEsuH!41X8oO@$DzHuf2m$q6GXXz?)H=) z8Oo<|u~gic8#@ls4^23kc~Cpor9%wuOfhH~@7+O`A-6w1U~OG zKw{EgNhFbmD_5u(-)^8l4f&a^9R_dO&uKNTUsBR*+$IlA*>~OFPl4v}nJizqrf9w) zqe22=T4Ir+f2Wo>`sLWI3k8mhw?H3J4@8NVOJTC0;$We@CRKTKF-3MYHiCQ2x3BU( z)}O1pcK`t1iH=|ZvOy7K09X-1NkIVYB$7zO4T7V2@@RyR?lKpdotDOu@DKn100025 z8j|2a5i)c~hdx1-v5bPtJMzV|CO3sRfYNWYLAI?F)Ks#b?V*}=K)U~Rs~`0}`9jfr zq@K;^8mYzZ93voGNFdj?bzU>-R^QcyxFYAQ$zjzXUTxHHURb0-{WBrwQ(ZhX(djB$ z`|{b~h9T^fOv9~7I81ym3u$cwRR4{rh|WgWszLs;K?0NUxzv@B&3M~d!H89)Ki8|p zAx&K5^LeTg6__5a@+Dnc!orx%+h*yBU8H0mz)})hyp3;X`sbgdLE`m z?DM(0Suen3X-SJQGFHiJ0yCcr?|5*sFDL6h7#NVcZP%#TI4$qq*)#tOc#{r?EQZ$3 z@2aWLp$#GztTlv&s>7+oXsj0TsvJKrUA^1=xCTk?v&P>l6P;n&yXO^LN(W_@q8;<+ zrl^L0MY!5`jO_o#dV1AP-7?iu{WwR>bh*tHu{|g2=;Y@7OV2^dGW>g#(m%7hqc51| zzcxELdBOX6%VscLc4GgiyT#Xi{j2Y7xDyeuA-Ni0gT+xi?gf7B=ljdA!DA@CvcVa z0|L+;v>c%L7r?;)(hZ#!uzUkvOYL5>NO9@tvhh0$`rHzF$Y-2yK|#}W7;QdVP@gZ+ zsaaj}*SXBqyKiTPvBhP!4o2f%PFbD9m|>CUYfa|p_$#mW%GO>rXDT<-Io!O+ur6g+)r2!T9)G#e18LaZ{9x-(kfMX?ERZ|fqNIO zME-!VUTi4TBS-t#Yyc}w_xB<;JV`o%DIac@7nw_d$iX@5|TOOcDXyH_y!EDC<-4 zp|dc+urTQkwTFY@)y(Qp%58N!TtBnjTp9Kr7BaYo8#iI4v7ogav*%~Y0v&t_4WK;m zs=m@rZ7B+osQl1Akfx1co1S^%Ikray<`Wy)L8)zO^LY$qI#2!G2Dw%`Dr z5h{+fe|-9E$fmiB08zRveD1Yhcfwjt`wk1Iq2R`X1Q91A_9FYJ^&CP00aaVyE^zPKKH@`iDe(E?g>zB zM}%7D;S$eKl={FXK@rn&5>;QeQly&S%rH!KMK``jv;;EAMXa9wnFszdSnAk@(`fr1 z-F= zHj`aS%|-LLjNoBDsEib}NSwq5JLd6T7693-79 zDa>6lA#reFs2?X-X75sY!}Ai3TB3CwkYdOYjVslXg2Yd1enXi;DDq zzWn4VT)GEKJ7~9}y%gc+VRK3Rba`|;zJX#GA9Sf|N_4kmm@4D}qpUcsJR#}R86^G< zK3?Ss!&$nw>89d_t1|k&(zzOxU|5m0sk1dZMgpjHbS26>7RoL1{!7IW2-RKL0>wZ8 z&XXqs4L;1xknu*uwWRl0<1N;nGu^qkNAlVzq+3u3ApfQF#4g+wm$A|d!l1-_#J;yG zm3x<}PZ#pXr1xG8Mz6(W3cqYjn`{FN9tt7SJ^^!VBXJqHN)y}$W&%S+7+*{gN8(YL z5ohchr<3tZ=0#=0qo6$Obbi|LG=kcor0YLPK%_&LwJHl&e<-7n0%_7Ta;(=}?ZcGW z7T+s3xx;mUgx3d_b@*|}#dzL(mHFl|$h7?z$9%yOaT#1n%`PLi)RS9__>lx%bOy-Akf0XxoAo4bGs7|!&K zVdF4s+eNgQ3=(XFg^_SFGpAPBZD=(#bR(ZH3bbYqvRA4Zo{_Q@@-*w8gy+Zad|ABU zng4o-FaT?4s=bRnwy7h7^VJ49_PuGt6%Aw3^Ipp5ilKx@+Z*_<&m%won)`+4@VZhV zHg!;XUaY*>_#7oW>`w9d$demj4)B-L*Vf+Fq$4Eu;;nfzU&F4!7U$Y;%AhWCArS@4 zB!BjEb@@nDgj}0dN(C~)NpQOJmA$-mN*V2{*!6eX*_INd= z=fDLXe|P-UT#c#q@ZPo1k2PUb7tiD4X_n7hlBX;1ska~8f}x~aVURQY=<+tUzpc8Q zt}=$!+y>*`gZNj!5}6~H{CYxa&DUZ!e6P7Q>tOFrPa3)|*u*W5>(T_uBFMBznE3Ltoe!nHloq_n88!ROprYEC+-9f037NNO1u(LBUw5W@{UzH{ zvCt>#Fg;Vo6hJ%Fop`;K%e8Y>BbS@MN%9|0Q1jv&Cs+^WQ3dArTb$@NUIfB^_Etm{#jDAgQ26YLod%U4$7rV$nkip#Q; zvi&FM<=Wu3y>RFweN0i2m&BIh_6I)3sl3>%#?p>Oi$q6=fLPteH>3$a5%w<^?vPMx92tvP>l`P47rW>_nRV zfTuj5mXN3^1KHqn*R7R%+`@egteS@{*&XrjM}$b7JMs=%nfx8OU+@sd&h}0>q$MBB>Xtm9SQ3y(jn(R)=rk;Q9OyH<*O6iS5u|W4h`{~KFJ0>Vg z-T)so!4so}@3v1B6kS5-Vz>J_5MmX%qd4QCPk{?u2foQxU_6MN!AHoM(wxhl*b;TM z1Vc4s3M`q+D(3}9B?WVa2OXyh?MVLn(58ibwBV0>n8?IKhxzI6M3eipVcjT~Gg@sj zr3x+?rRe|yL6CuI#nExc0uEL0z;~%k$8}GWF4o}+u)v>uH8KGVcw;$W=TtOdp<1JY z>&0-7e74P}{5i5mO07Yj)vr2GA`vMe_~&8aJdLJv*Zvq)>xYGcx>e%fBr_I(A0_M8 z-2LHMP!X^vW`ssRiVCJe8%@HcwnK0tLj!mXR8aCCWjmL9=>r&1a;T&p5CGt3I6#)n zhJ$eIk5qwZ7)WCkTnYj$7IOL0cD_r5wZRr z+F$0YH)+(-acsHH3%sf~mV>de!rMD)&xx*pi?5t-?`=cTH&3+WJ_0#<$&N1s;tVj1 z$dZbT!Ut-R-|uHiCcl-y7L}JrexMik7fubi8A`5AQwV^<3O zvMAAehg72IDOeXh@D!UxlQsEJl=%P>u7uKX`3!JcHnGka85W3nE#PSINg{E0{wD=S zhbu{ICpZI4(!`e|;c+h^v9W_rsI((Fd5G*^&@x1f@g~_SaIHAVGpyN+_`OUdWwfb%tOm<=fAr+k#^ol)(mY~0;e6ub=ZYCNZ#N- zSNQxcn83j??A=7`CZ0T&ugeN+;a%HG&NJe)xG5?EB8&P(r;-b`nFqjTo4=QjzwCje zHv%x^VocPEEI8_gyF}U z9pvK~V+b=B5R9h2qxin_J4Xdc(z@@#F}NTA05l9fPEw?hhAjzx_ADd}+D-w64KyhY zD-2Tu&1aX5;B7dB|7ae%NN8CnroskUKKeuTG8&qB??RczamaLa)$>;Xcns?hoc{q* z9+M4;1OOmB{j6i;?=bEeCj>{!Ndv>{hTuK?$PU_V`q;=oS?|dw)c;^gbCJ@}{&ol( zG}`__Ye=$*XQ*$OKh8J%p~-)M$$^{R1wVz|BB4`wOXN{NrnbI*51{T&F}p(eEGcsd*hV!#dDi#q_s*aTQp>WFk}=m;i!mO@#$~$1$9Q z)DGq=0clhG9_ZIYpO1%oOOXCn9@L{E;4r4!r6aiu+H?x)UIj?Swau?7%vC_yz7;uw zrt09C8M>e143Sp^{62F%Y7cKvioYprA2YA`bO&hOf^B|EM;d3yQ%a&@eJ!T_R8agz zHhPGZCFS`JR!m?Xu<>>N0p^zY0%^=&eE6sE2t;-_DF`HSb^{%BhQ$r;{|s3<%TMHJ=< z2|zw&$9M{Q@6bFxgY)Pr{WDSmS9tx6(qg>iC`@J*O^=qiWa6wb>`x=paOV7;29 zV-v2R9sV)9Z$OA=;{Vh%5`ROu$rsj-RJs|{-y{i0!c%SF3;fZ?K;p6@pI>rjuC#f!b=sjjvC}>NP?9n+E)e&15F=|TC@q-uXa7u-Bg5j0PQo)`j zZ>sI6uXDvqO08=l=3gWk4;h$nL2?eU*{%W%^aTRLTQ5o{K@?D|^S*KJ0afG3xl;<< z`AWFU4nje8|G`DTW0H%|5#FYonpW5RaHNV}l>`v=l>xAdqQ8a;@-ZLES1y)nap^|N zIod@?T5cf9@Xl5fFaWqwn6+$OsG1-tMX10B6x1(glQ5#>XBtB$*!pSG>0O!ob;qZaXL6Ha7B;80j?8 z{}q;zbcgif=%+p44x-ZaH9&k3`(9`qpP1PT9|b4bsqA|LO!Nt_em#gvlaEL4(It8d z{!m={WQ44r9K>E;Th~1_A+S4*pXAl@ald22vvnJl-!F=%~@s9ixdn@!ZJ2y)eN8w!b_^*|_SCkF)X@Lc_ zuw z%2r~joj8{{}al_64lG#3>B!L0}UvOp^dlC?}*6T9gg>t3e3y-T;K&pW}WcNL3cv+tRdd}&IGy{^+PfR0r+x@iY*)P$?01PsOC(oy1 z#nV-XH;RVIrmexLdt5^@oZl~M2Pq|3lTiCp+~(TAv`vbqFsoFHAuLO>0_62)5V};3 zOoU&*!AT*Ssn8y)OE*&%QZ|;1KgR^NDcJ048K=(eRlPU%O$hf`=^u$tJ|lmjoAU@A z22e`ua92W66s3+L0MV-b1)c zp==w?W|Ijsh{zsEg;)*9q1CY2OX;-hEtF)9qaXXP(lKfS@vT8WbO@;Ctn`>awixJ5 zkV?)8pif(N0QQe{=&aezjmMHNo?ETY^11IR!EdZ_!IP58gB@qp<-~N+t*?b>RLn%g zBYyR+@6h4Mn-UrR_R;?*34M45Io7xf_*(Zp^<^_lSHz~6k_BPtIy9ZiK!4^Bs1SEO zL$hL`wZf=(T*2y~oWMIxy6PqMz(N{8P}4W|pB179@)(-XAjQRkU!41R;6Y}dy1iZH zl6bmLiQw?c>}CK2(KdPV8fM1N;+H;Sk9K)H<=O1uWB5zYcRRt=XKd?r9f0mUADj;B z1kD2Xk&Au~#!EdkP#pWw$}6b@&a38XB~2m?1X}Qyg-crDKjGbHl-QA*j?XLP02O3% zDqvOZPs?=4Ii!(#j%Q`3+QV`#&)scQXNmK`XHg!swI9*|7TH>Be-H?P+t3PoGl?WE zNcJP|@|^46ceBYkMF~TM^25yD__nO291fj)$oXZA9*cUzECm{cyBXYR*XKgpffH<&>{;tfFZqBo_tinP;NIski2aIz4{*5|TACBfs!izV=|dyCoG9 zz~!gb{RZ}x*h1SSvXSt_*rdAQ05^tmZPcKq-8^b8<}>mZn^a(S$=>fT_0@k(le0xB zK?dbLJY%G$iM7h{J9;n{$k1uCEb_d0Py8G(rOp>cf|7CMeG@dK{!;U2!)zL+_y}bA z{vZIA0^M#9j!2K%_}_HXFCUCjUg&#;sn(m=v-z5dK1I$IIt6L%GF+~z;sj%>iGs-d ztwx5ygjCKHN1|PuTfJ-I_%#Z|fvfH1>!bv8)_lRy@E429$}Gis61OSMxbSDz90@-Cpc5pAmFzC% zYat}Wlhp$p$6TauFQ=x^D4)icV=O~eLCeNu;Fb$?r*7OHQnkQS^TBhzu)J|9JlHor z?PK}DGx#cRf26S-3hMvZl^6^)kqvEki=jR_iW!vp(FbX$%oxxTSxvk6$5_(ADL%Ez z!Ueg?O*nIv)V)v;ai!VF!R*GG*C)mAP4+8xmZbJMta#t;W_i+fm4b~f*dUjha6a-Y zX2JI?JN?gjWF@k#tSrEV6khcjk}3V3Tp;b)Du|yCd_WE^D{IHURBOmvapg&qb>t+c z@@-?b-+xDKrnuU7Y4peX;Bg%xn0&YA4i$<>dUQ;t1V8EA6A(Fp zwrw(9N>E;KX!cv;Qwg@;0pX!)ETZj*harI_6CDaPvSq9GXW)OGvhu|)!;XB0(TJry zEA$e5YR3r`w()t@f{N}U7$8ZfOgOhgw}fEuKE0U7n@DSRoF=b?(t?0zr{N#2H;AhD zlhh|tg05^rc{EgFH-g3J>NVz27WM8Ok7qs;zO~8!3eqC22PPS(% z$I9%xedtE-HQmt@k77Q@Kf^Bj0K8LJt(EDkiPF#_agy4YO%tOr(y4JEzR4NF6i3%I zJ}zq8*%t9Psk`Bl&Umodgy>{zxN;@qfWNL%(_)wLP8N&oU=lgq9Dqt| z45sReb`|n^m5`3}J#aMnY&tKzZlfj69wy|%qeVR9L9=9aS6VJcn$fsqqGrGZ0c^t! zz!HaD_Brrds;Is!h4AnDj;w$`T$pz~3eJwFW^OlCp|v&VUr_`JrY_fv*wiB*{1HQ> zR9)Qq@r7O+)U>nyKGEI*OcRiEO-mEd2KKBpH?vYs7uhTuHSqby@?Ho|&=lDH8%gut z3izkU4iz>ozAqE>=S(RPr|;P8A|P6gr=^v%d8#W>c|X)r!}J#dn9~_T)~P6(6P%<` zE{)0DChf+U2F04m&PV&3%*-3+4`4C;;48GBRV=7DnNTimgz3Kx^pTbA%!zv9b5y;b z;d{+zn0p^~5_mVrU6ScAev{rYKQeCpCghkx5R^kwq#uCSlazx1u}>%7v5hENn$qC| zI+}DOBOI3rSn1uR!~SbHAwBm@Iu}+8arJHf0(R`XtN?2dIb-?9Peb2X9SQ#iHuF*W zU19p5XpS20Aupl(RS-xBO_Jw`rpj&CDRbz?c-bQHge57#@|=7^lGT3jF(C>YU&mCT;k>TZ&&`H!$wxFytAfws_qJ?=$8 z>`UJbOaBfBicz+->C*3^M)}d`Yzwnv00~p~NLFygppXX-Q4;XQV2szX6Xouw$#!pM zJMeu91+nXl3+=g^nVSKEu~g&8h*%dXFDPm5jk@=NEWGR&R>8ea)8I@e{=%+a^WVy5 zWT<32ar3O+*9548(8C~dP@wwfMrF{=qio`JGZ{2I63&2e`rj#S>+7)g+RvfFPN>~j zf6i*P#-AoVlyJM@VsZHvChWhU**l+~ykDY%Z z2Ncg4P0;z6=~g6?y1;>y`0RsZM)4#MYRjZ)Rc~F+Zqd) zpsIJ-n0KL%roT^=ziQw8%ZoKe=F2G=Vv{j748R*U~naE^xh~17J(XDZ;esTXq6@GOwywuuo3t@JQ zCA2c!7{lIA_5 z!5H9~7v%4w!st_~Uy(*9mTV6d;{XfU3W?XhM_{0EpLx+~S9FoPdCnpZ7j{_5e3d1% zKJIfK4LU5ZpUbtozoFsH4;`Up0Xo?u7K_-;;^L+`ZtNS2`<3pztx~bEQlFcyE|pzD zyMWzLIFR8v~Tg??*(FfRCxe=H^UvFViAfjX&96|Oup1Sc102c5Y8d|HhWhucJAAy9 zU%SlT$}N_Xw!888G70-zPy4(QBWI!AVdvw%HyPVt>IrzJXVRA!E3|N4Vi$$y!8;4U zY>*ka?;IBem#wf3hQtMZn>3UnJWar}D-?+o$msXO*k|%y@QYZMK8!y1a5hUqLmBC> zAdKG1;O7u0XA8z9X>izL{~5&vMf=|&!hTIp@%5TxFUyUGaZ95fM9EA)72K2>hDZdR zd*|7`S6C4|Q zN@h(Z`%r@zs(r0MlA1OnT9@BhQX{^W_#gE*7fEe`mSrC9dnbnxq}}i*?`PN)KebIA zu7;u6s)>U|Ao`4z3Oc@16lTJ z(f=$05I}$g0003fxZ=ctLVW>5=-9ogS{uLLHS~39zSq7vauxJOg=w{x8n@`@TJ1NA zNtwqK8fDEFTGX^QYqr{P+ck-*tIJhOD(Y48Q2t*R z_0B7+2_s?lYXSRf_|;=?dj1)Yv8TJT12Qk8q9%r;eFr`x0)CSGgxRk&cE1=L||Z}$GTNm<(>%Xz0QWjq=VrC``!OcoOPuj(6DP9E~cTQl(aFR=eMWZjrc z5yAkJgf-3G>EO77ky?(J%@Wh=5q{RNw)|^ApsmS2I$gSP_uH9@=?J6v-TRWTF^8g0 zY_q=d6nkitvXy$lKHTUSn2_tyF4BrZ*`TH6*+ZthdifoXP;vFOlCo<6H{mA)2mn9; z02j9j!|0f3icy5q?EPNRnqj|bkZTLfu8pz!Df&O9A#SnU{q=RLR7}_TUB=~2yPi3K zvqzi0n-JQUWKLx#3EOEozTg|QF>lV8eJ zCyanlg)xM9%++q6#WDrg(>b1AZ#Cf3$?Coh8DsQFqekXFX3WbDw0Eepi=Ix=T+!S%qE>XLnQ zJZ&Sl3e+7PQc1Oa44GCUz8I;2Htja%C=E8`$4`E(f4#Rg&1i_6T_@5S!Rr}s*+Ss2 zQ-oA?q4L^jv`d%EM_MPsF(J3_{TsR7Ym=g+bf~K)JR2^xQ;>fv3nKe4WrqbBKi>s2 zP%^agjI8kT0O=N_Kn3FhO9pLr_a#(jyn9Q)APeqwTkQDMY<#g{C}DD#sYfHsdsIGb zRAzou_V4l!=O8abKb7bEAn$m&9ELQzzpj7_%i&gR6|{7cawMrl1`9fTgtg=Dg^}vp zxJNY!t_BSD>>CCD&R+=a{E9c3-iXv;hYZP;joK47d=&1afrfOJoB11AkXjJts0xu@ zY0~2@ejy!Uje{J$nJ~Nqcvi4A8F~>ukDis6vXfewm5&pyDZmi=Kq%@O?-UO>Xh-!fmpQ!1v9L&ud^N+*mN@V{yu3K z5NBWm1`HY4Fkrx`<^wtfYL`_OM(eIEaaOWbt#emUw-01EAUTaHXb324v75*e9p-2o;v+=yY{nw`w!4G8 zpO%*!0tlNpnZygqt+;q|V_9q4Vk+eaGx+3g9CbRLgkxkH&`tGYp*&I!mLQo4=1JEE zq{szdU#@PKK24!vgzwp7EQbRP>0hbpx`)efOyy*(a3N;AkhrZDyoYqq&R|cFJSLP^ zXxDw_0&a+D8H5eJ-3saL+S4m6i2lawPW&srwf<}8%vBqoX4*6(*njt488~K|9<08R z`s^3eC&e=Tb_oVK+^L6wl}iQd>hB7OCCE`b+A3j#hZa0(2_l-C^vD*C{<2WceP!~+Q$2GN#BEnQPW zhFl~rJwC?r}F^8S^GiK z;ZBi1%eqBRD2c?n9>Qt7UW0Tr9&Debt`0>bK|!@?<~I`|wts7nAhI0Z1^Is7QwD7U zz#9}=ZYqS*x-9Ha;ovp8!xIUe{Z1s@B|ojO84og2OuJqtqwaF}of9I`J{xwICP!0H zRS9>k1R>hXcskaT;Oi@!W_Yb1X=tV7{17$u+uw5`tIAn=*n5;HZn@slpo@8+<)ndqG10G}rjN7*|V|Tc){W|sY zO_+E!RxrevJYB4P$I&~8u>NhS8^@0D?9i?;&4F3D7CVthKUqmI{fA$OX7eu`X~G&D zguCRcJC9;*swzJAp%4j4(B^uPzDI;sMRqc$E3z*SvCLJ#JM>@m!}Nk;j>TQQH)Ycg z;#%P`j%m^=M)PU|*BiQ;uPdHSl<0UZY#^_#Qp9}^!4m~fzUJ6pg>LGt?zo7O-A{^2 z*U)@#(N6mqPe)d6J$LTM^$z=kkb!&c#ecT>2S#W}qMO=;sQPEVuBo^iC*5-z}~vm7}YJWDEXDwan{MP?{aNBjMHZ;Malui4^RKu~#s2#LV=cSv5OPBExF zeuuF8WbU4r;pu~zlGIsB_^a4_Nt|Cf2|D-6t$I7QOm2<7bvwRpXvVhiw}s3LLY$1y z$rf*q05397&3(bmRSw8q_GRhSyQuf5Y<%u4J5dpD=h_bA4D;-Ej~25JP8v5dC|WcJ z;+vN=1=Qq1fLgxt@{*fKbf4KQ(5*WfrbnqN^w=6SRpI&%=^g#kmY}9yd2e}`1;UKh z37$4WI*Rsf_$LV}g~!h;du9PEAw#~oZ1!5Bf3`IFeXF)ZtnKdeatr1jUC2eFSogHn1z`8ZEF5EgtGM1pqN+{}U&g^u#WxbC2b>`(`OFvC3Im9u{A{@sHD&28a(LvAqW;0WXC6WK2Yn0l zQ2%Sa%r!P38F{?O6yx=p*tF-*RJX*w;NF<{pvBKP z7j$P3qEgs}5|rUsw7x64Y)v=p4lf^l^V}U=JUG6kj20TpW=Kd3BYJ(;Ykf=X1!OKs z7mi|0fG6#xelqZ!tWl5Mm*9JAi=1Y6?9)hb+nS`AVOA(SJ##`=(ZVKAoy*7D2{=k{hiNLd?)Xo=a1t#UOfkcGMH?mSG z|32j`0Wz-{2jC7rblzQEG@N>eoJmotLj|)FMpY;KrcGXe_Q32Xwq{s2&-!mB@{>wm z$mGA;u_A1(R5Ocz3a*v*L7_%jw`%d48>Ll!b;Z>eR#qV7vv8;wIcG(a!DT+0S#o1{o5J^ja_we^7ruY*;Lw1%4+G z0hT9?cyG70ZkYGJcx|#fTzCRcc6B`V?mV$Gcvq7Ic_Ht??Z`MnM|>2oIO@Sdf*uMf zdaOi-**5u4+#byzQSc}cRR8ooo=AQ_b$qj?Sg90fS=kF@)s>{78uHZjOP-x5oRFqj z&AB-49>CXD=x1q}6zAV&1X{x7%Y+k&a*AR2w}04P&F}^oBU%vF@8#lb5;Q|(;r5s< z2ZMIw7_WlTwl=W{Cn-C_0u!kyuyT}JPXmrVD)wEkg16xI(Nh!b*GA$eApNo$)%f~L z9}twgk{DB<9lXeS&tj%bSR#hyk2I06bL?@!!nU+xr~4JE=`ljs+oO;nxJSYsquI!| z4@|(ixO9nRgUf)APO zHC-1-Mu`F%?(-j}0=-sw0muI02t zHA8RuNt{wi9{ZW5dkLa6<(N z)}x;N`Q1=d|8?1|@?NLpN!Uo8!{kOuazfnSLaI?2H|AgGx5Z~db{Ofs$80l1emiyd zSX~ybB4ojmll(sa)E!B+rHIc~Wk$xSc?EUn_&k59v&Lok>V2yQgZpsF-FD%Wk zRmdEBoOiVbxE0iU{_4owaly5X0X6pEQ6;B!si1D^z>MI=`e12)>Spf(S<@mW_isCW z>6O9WA+`EZKPmEVy0XA~UdSdAscc&R`jr7>$y(uH6W_bvHwAZSiI(%XM2G@u(@Bfd zrIgIB=&;?LG#3mf`Nwv1+tRIYNAX(S=u?TebBvQyLVpRJFwLmD@-kBNQy--6B~JgE zqfb+Wj^SIw)y)`udeb~ywB52=L^LW$+X}@^sC%v4tn-Q zHBuA;60W?;bvcpCK& ztK$bw=YE|&Xw7J{879u;p%4HWI+HxTRK!^sJ?^z>Qgv~;EXw-H#9oW8Bc;a*K@>6iI_Lqu?wI8BItku z47lg)q6g5WOc8hg% z<0i&l_J2+33db%_TzgiRr_7buwYm&zSBo_P5zqvthL_SdePGke6cM6ZeG$PRGa2<$oOn9H5w8zj z0=lL)_~xL?mK_K3rM)Y))>6rXhSBA$K}8`ZL_l7@!K`t8cjH51H#8SrqN_p6%y46W z2LUf>z#Zd?D0U8hc7($3@_d9wBpX)PZAc{@x8Ns!sC~KgPyVE9g-*hrf9kpIv zyh;UF%DegG(*CtJTMHzO082o$zc+>ybHSVn8%+g9|8OzzRzTLTw)Me9d2Pg8TXa+< z22w{r!wa(HSxGei8fZT(f`-0S*<={VyX!9Ve7~I&=uSFR``F$_v+14h8RM76#cJRA zCEdoNM?lV_2Nb^bT=gM(iSB-ov)dmiK`V0t*5(Upy^OJBv?u9%%}nfvIiykRA8SXjhRGg7vd!qzZ>n+A&{<_s>G1{d)GB7Q2d7SIt z&&2~lSe5yM*)ANmM9^@fSo*kEsE({m_-IPBXV>IrbU!z^+8v-I(>I(TOIKwx|MDZN zlGutuHPLkAI*9d`dauC>t~tF8Z7>YX8XV1jX@6n->)=)cy%+oMl-<3@v{C%$dfoX! zjV!WU#8PO5Uhwxp1|DsAknnIBcGI&x(J^$Cn&P{6Z2a$40arZo;xPRJKMKs!6BIN3 zHb*-+0vy;9F%dGm&LM1%KD03$)*AhvoOlf%HBU6SW-*hhN=J8*%iI!&14gn?Gp^a`JjGMDEO;O8C^r!{2H0W3tVh z{eHX8vDlqR6sM+4X^n`(Ii$iIi$={kG4VYk-v-6dmzisw*rgWyBN<#B(8+Z^myEo0 zYV}v1)G@8c8qpRY0yO(6$kan?{JI&hDKK-z`+i?>rT5yp%Qc4q+qZ4Aqdtj5rqRL4$tUTSi70@QlO3JoRGjO zF~(1n%7$uR$y``c-)+&x1V%zcS!cqrm-zSUkn4h553EXxwh5w!jRu&CoH8=@GZ;j% z+=mMPpLJ3uYIQ@@ko@OglllIx?dbN;;S<>69a5XZi{fCktb+ptBToT?w6Op}GWRof z?3)S%{xcyz;r_0qYvwQZ%`MQUCD*EoyTkXW|3U>g`g)_SHMAEGh2P&D;ki#R2s^ZI zU|5p%%kE8$YhNP@>|UDz|2~QA!YPzK3PcA_d>nYa7N2@^_+4tRzp^_K3g!*`8GI*U zwvXF;97?Oah8_Y8CEKyN8=0b?a^jk-t4bT%f(lfx-IOY_F0Hq^!3Ft}R>B+z>=K3c>TA_Z2IsZ&)-2!0TLqHyyOMM)-?&k6 zPu?B|qAHr!NK0s~y;ukU9EO-VpMu<2h<|08-39tUz|!YWs-BQfcux%j-S+M)Xf{C( zy`k&PN#(C6w;D+@m#cOCLg%n&T?iC^60EBStoG0~s(l(Hsid2YYc!Y^yMVa38tK55 z#mm{MXm&VU%dsUBPUxBtE`{6`C<&|yTR9$$7IC8G%=s^%Zb<0ZZs;kG@`ry_=wIHUo5E;0Z^{?IaH zqt9Ql636QouvSA<>7YM?;&d*lW{Q*Fo*k8F z`x@E%EKYj~&Yvz;`vfH&Z9TgRC7TtzJGez~z}U|JCs2Pl+*D_LJ@oR0+!oVaCyvpH zzI_Uom{dSmGc18{{K7Njbp3)*;K=@!g*zgcIjw~nV#tmyM*6M)q2@!A-e-a zPa0i&LAeu8jp1&we@p84U}MCnIlslLHpxu3+dq^oWi7)q3{%6l5(j|jS~DYTskE2` z7?lrp2`5IhO}|-^wUx#!j8n#meoNdFi~Gd4Z=jmAF(y#abkCX{4)Pr zb!`0}{DpKq2EspI`F(BcV}3|H4aJ4OlM7uO;@WyG>@=zw?e#4J>Mo(zQm=TO?Cs53 zl>Gtw>>+aVW&dAt%Tq>(k%f(k2?4b)QH-gWfA4b2KZUP3SE%5|k!2lQ0;S_ibu)vN zp4I8@n%T+&VPq5=OmOCkSwVo4C%IGrc{0!QQhQ&Ni_jtrvZ0ig5U|sTzcq1cG{XQ! z_0Q$n$@K&?Y2P{6Lhw8;a#OuCpb-iZT<>n?>dkZw-mhTy8fyxZ(QC^dPHm&U69!=! zIqQ|RAz0OC`#UB2;KQ>hL%yF;qW{8nHY7atx@mG@y{3+U@d z(jeYFAuryR^57#adKhF_>=%HU=vZLhCMcCCA}$T+1r;}~(v_03(}Wg>OT7H4Rb5(^ z=X=TbH4Cw@WdBB6Y}A$2C6-9P#7AqfwT-K9@~KuVHe26uMM#fk2vA};Kc9z5==@Hz zF*t*&f9tUO8*{a&f>!q4CLMLjOO6IzICVY5FE6Y*0tcVCGhIUE!wVY~>oEhLe-WwM z-2Kf`eIOzK5H$blws1Gm#@RshqSsjH6A`-f<{Di1z7&kd^^2L?DkV19+haCUQ_b#? zoJ7)^1zdH1?L?iYgkcKIVc#EY6%E?*W~1akYwzC$`cj<~1e1{%inTz;@V!15ajP~5 zBZ^lvy$c-wELgSFG>q!K;p+51dwwl8O4ZwjeqBXXHKmkh%a(~x!U3L@B zp=SEB%uO*p0U0XyVA+d~I!-U*>*Eda+p7bhFr5EEo_oZ31#z(4aRQ@-%X2zhuSY#7 zK4GY)^QRrjgfVq1K?csCugofW1*X0rHA&5vJBPC`JMnqMZ*Y;!H2X);qko8Cb$JKE z1I*Vln5%AcWrakVMu*U+Jmb6w>2kLwx0HdcF*KudB<_l#<)(KomMGpS`(AO=nrQ9! zxf3~&k*W+zjBNzsuORV_Uw)@OB`Nh3dO^_j zncN`A^KknQZRzd0V7QNBK+bvKXnf9qukyUvK=SKfld7wz92r4NWDMa%;P13ysk z|B$xJG(!tTnyqUp$os%lp9Vqv1NnS~J}Q7k&1sFCOE*Mvu!7#xZn&v)LOJE9WiJM!Jl> z2$Hk}GUyhQEO73Uf`jK~V0Q1*d~0q0v>*3mi|wMjVTevS9Bqm)pzm^Z|1BB{1{gRy z=L`^u_MkO90a`iIGn)!)W;|q9#C5xl2$4u!b<%fTxJ3R^NET&~ z({=qmZU*^G=1^S-v2nyHuV)gBe9QmnuQ%aWoCjSQzIDp_Ip#{-%R#E1Sc5kLv3^LY zGx^x40@oCdZ*p^ytO#e=sX2u(ERg;5~sC<(C{*N++EEZ*9abR@i;L{of~m;U=oy@Mm99B(#7IMO(e zreEnwo0rq=b2)TgN}om{#z6=3f6!n9eKPI?@tRClMe?#v%{E}zh}ZTL4y z6{la2?*NHm(aBS)Tc?K|N{UOPcRRG%MiTU&k!yn1yeUmM(gOsMjAd;2ze9za#yaEN zp)=l3MNhHH9-T2&7@{j45-lNySFYz%hFSVzMSnzi3qiBj2|S_O6)o(SC!cb}W+`)}^130+~;@NJs9lO6LOe}Fr%%9lg19HLB44ap7YfuqCYJ1D2(& zlq-1Si;HMS?fNp>jYNar_2Y=yHRI^@;qZX(CyY82`gR+tOx5;%LaQi=B~_%jQpnPv zGY!KSO}rHolUd-e>4lcqXy4V(elD=W8HNgR9xJ8MA!{kX*FJ@p9e~_;zui(HxHSNT z>%R{2`lCA(sWiq9|77G8M?ITf6uA8W( zeLRsdj~@6tH~M}i%jV@sCIeYgNyFK*N7fLr$x`Wy^pkB*(4% zVBk@l$zA6Z&zF5X)s2t+Jy1m#pmy5#tXr_*1I(Ihzp6+TzNs~ukA4J)&qI?0KmAr4 z?nBJdwNpensuYJ$V%DzoJ^|_a{t*HHLB2*}U>h4j=CCrbz`x|8?el;O)cqx6%z!ef z#V$Pkzd$7)O>TKZ`;;(w6yAx)9G#{(J$(2N zhjuKs_o-B^{m2-5XKUVL|Lz#~1IHBdkTRq#1pa_uJ}RG7HIBz&Xl~YBCsp{+U%l1Z zoBKss)x|nWcGTxNlILtJ$xXl3_r{iXr6{dnm$*h~@iF2&AQEE`v2dMlDkIJj0Fxe0our_l0u?5(y7OmZ}QybYIG)(wUF!Be% znn}f1oS9g6Z$myLn#cK_KSe76+do+N|T|ue;q^nL+(cQK314@ApOmF~Io*W0q*IObCOU1>Nc%#7fb}vE0=rV@eclw)IVHmmcn&P`~S4<#iO;$;$@jXdQ-YKk!Sb3uKA$&HOYux)Q(s?Vk z$e(A*&pp|kQ3F56xS?r3Zl)^3OZ6LR9`PD({~#6htEl(ie%X&hNdM}L0LBlxkGtDH zKpA6wmG}Bu4w*NWzxK#oR8 z6BKx0iUdG?%s>SBz@z$M4zpFb$mU2tvY^}NXqDf%QF?XVM{jmyF&H|5Mn&Mv#ya>*O?~BpA z2o0jMsn=w#IpQb+EN-X%bVRFulM%wN`M+9BHnj$ET8YNt2m?qwc_{r4C5DiM?B;Yg z5Y{cYcF=;Fok(p*0G@3KgpQPb5-j<0Hk;T1=AZ2Ya}dD#S~8}Jsbw8T@>UwDm^c&_ zIihpB(tkyMrrUHBTMemXWHK+QaEFfhu9$QgHzD3eP5)Ja&x>o~Ntu=IFO}I9Zyhq^ z@NJ4e>%cMOK4G7mf#cvtQO*1cED@MsvBkP(adfCNirS%a25qg>wDZ-`@`K36bzW=i`@^Qpx@l^dek zmC;~c&qAE_Q-_RI*<|vChv_f*uqzUMhMi3Qs`Wf|E@K#QId)q&DMbQu5i)auGHV!5 z@ou%oZnHI)uj(dZ(bo+}`dngA04kQ&@*4KMkvix?qZLue)2g&p;EZ<1X?RzndQu@r z`Iw5E9Xh(YT@?N?%fcMRv_{UUqmofVn*4M@)2hc}oxvYx*inrahpH7Fw>7>L94&f0 z-l(lD~pSvR*4pC1uB>)kP33GjC_lLVk4gl(Kk!!S(KuI38i^UO%L@ z!Kq}KTXK!p72q}NdDpyNoh-cFOlucm}3c>8GVCN^`{2Xe4hs7q({ zlLzYFY?tRxApJy47o07h)O;@^n-);%x}uo!&|yOw&OqOuc=$u;>i)OMqi~SbBq<{; zA=Qc6oj%Usi0hdTD7PmULITA{?-Z>I@UzFGyJN#y=~!lk-o0m&lpF*9c+bPjCBZ4` zRxbb>ruTJV)9Iq?X)QYZGtu3_ty1N~*GEouKU`opuNcd-AQuv$e zR`2VsI)X3`M)%{KF^l9t=Ouaz0C|P}_t0c!r&8M-$-8xIAjvyrmLJ@1*oOFyGfeKt z8!5X=$H0}Hml!@lAx89hSxALh3q>WU^zKsZMr;?GzGfJ&+@zk1R!p40@``T}RH@WI z#P4Ev2I{!i)-l0PI@AT+au?GK1)Di7vaw=GkGi@g&@2vcBZ!JVylYq+w|T=-Sy(b4 zTrF#Gy9(02{r|#8pW5@TLOStCg@Z`v2VTcHH3p7l&Zk3MN! z8_!2PZ>&1E6NZVcLbiMwEr${R;u~Y7LZvS{GGA3s$$Lj$1h={7q`6L+`W z3B1b#*Tv9QhhvdPBuO3cu=#7g{=odNLX+4pTXJl|3V(txm1PSbk)4L;Agc>ojh%!I zA_1g6NhXD#YNB1fY(Iod!c7TSp<))t1#f>uYb_)GKMI~xSk|NG*)kd1JhXfrOR)M7 zW%k7c?aOang&+lFLK+{2BPe!8wnhP2tx6)`B(%RVyigoD3xs(~;}&m=`L*Lm3nrYL zK+vpY>01zZa%BABY2ehy>^G$Js*U@GnKLaqm#OB^2VUPy^_aD{f&;xG1GU5wwKB_V)hcBV#gp_0|nsh~Gs`8Kg38UkrXXTmku?Mo0 znb-S#$KFSVXSmwHDKk&~whhzJ6i(iFB0`jcVVlwmYq&ehP?${EBKdI`$JH2iAOecF z$g`2{TA|DE#aCT2yK?98LP814ftMbIA~{R9W%Z48!Ybba9dlRpC!4b{>>H$+Y!xr^ z+V-geM|Smk513B9e&5MrygSf3rj??q*HZ9MT}mA=hq%aHMn%U7muZn(rr@cD{k!9#d zM@K783Yjm9;ETRzgE^4*if;54FCWLDfA#Sfh@v+Ptz(HqSO!W0h^tCr4wh_(0Dq1v zgG~gj+D$A{7H{~p2m?*#11drO!5KY{$4!u0FoWou4)(j}AQTJ_^-7x;=;1wav!S~e zCcNph0}beNU`x0(*YMwI4^pZso*r#8c&Ah$bSQ|2_54TjyBJbRvA4@TEPFo2{8W09 zT1T9e(4v{~*Z)i;UsKc?I!qg#I6T$mg60$j>k4l(wq&@aH-}$tyDVI09X5`9FExC_ z-stj^{7unI)r36DlnV@3Pp%e#V)?`#bFWR&pXB{9t8>kUr3W{DmNSOMJ#VFD{`{FX zf}DuJZGK_)yrO4+fbvg85!Yjt?lzoma{UTz!Nr!u6}I_jY5uH}EwrzLeBjN1am5}v zW{L~3@>qN+o4Iiw(bfRSUrFtm!m$$mdZ=r5TN_Qpg>`Jh*Io?=PF=^&LO>^5DZ`f;X zA68@2k>3{kij=Ri3X}#&RJGuqd7~pl9Furt+b`?<{G}Jh2*MzoA7x3*=$J)Sb_WI~k-Y zk@m!Vf)nnFeLt@~0OunK`|wl)Qpg7+4p2rKQ4@RJx9yH}*0qxAyx@$ot98(J4x=6! z^5iUte;{!iZ7$@|n;}ehC;r!pwWECugI9+_D-6@SJu6buN2%^kbG94;dFmp!X7648 z5?C8>(GO20R{$oHv8PeVb6skEshgAnI+MM>vd(Ewr>N5k)WwaQL(Ge?-2b^GSiTos zaV)K}OKVd#x9VGFz7S?P^`mAwD)O9u4Ovx~TytbaHh!#TcW_dvE%qP1ba-jS!EL_k z{(W8v6T5&bE-atcW9p{9T~y~&5v*&md>#EF78O-OYq2mHQ-Fk}3~kk^c^0t4kD#$m zvwe@mJ>^6SQSJom&0yn?(kjeW%gen$#&|erp))8`Tq2^lY@^EkIAmN0C)l9+7^W(= zdDu{Il8|Flb8*V!bW=@NLEF+x)uc^Z3ZK^Dxl^qc+V|?;B}5*&S+{y-ym6ZRwoM?M zwVjdxF->KlgQ5aF zoT(%NF@${SRK}#yh53o@IC8I<#M?Dtr$@wvItW3eSbzZ!I4+wtOk+D$Htsl@p(PoO zh)mo{%zd<0i>(jXa``AL6UQz)S$=$_C!`J=!ZpUf+*?}t*fe~yO8XTu(|nw1wt|EFN?4dVvP)Ea?i!-`(T4+I@C z1ROJ!56fyCKrd!aqBXe6bM1+$=^VNTW&~na;HPkT*vN++;gxm z=Xh4s1P=oKy_yhaoC`TROa!CR9-(MXP7?^;6|y!Q1Z)^VpnIeApX#i63FA%uxq zUCnF3^$&mwYjW4?3gW%sXenZoxrA#i{3HI;GFRp=oG=0g!EgUaN1u-~9ujr3p_;{i z@0jC}dY*)cB3mxti~a*;Kh8??fD$2>pa#vYU>=-n=RY0fKF%oM4qrfbPY%*)_4I;||*9TNZdAz8qO30$O9>*-(N>ZlAuSP{pPv4rk0)uv7jGQcla z%C?@!5JR>bbZ4tB5Zx!4&Eq-FrJYw|lY5F;T8ogOfmA{$5j4M@9Kk=u1BJe9dfLjo z87|Pec8}|i9gQ#NIzRX3xJ;w<9oQ07NhrFb#POY@dwi=aH2gfnkPm#1wrB$j?X!Dg zY;tO!t=%ooBV}j+s?aKukPXBUEZy_39zCQ!G;n+bD~*N(-C1Pc6U~@-;H#S^5>yq+ zfnXRN*EA+{^-}kCvjS+Szu3!2jd)+oQ>b4I)7Sy9)m?ikawgZbbzmsuws?L|z&XvL zQ}C1MwP@;2 z9wp2KE^XtD6=`rzb4=`y>V-4Q#({+RG-I~T@{^mO6<1>KnoJ;>l?%!=QAqdqM5Wo) z2IyN~MX9y&an`Fo$b8(#11~v2eS=y*8wA9^M77rNhfH77N}}EUK3|bX4{f=OqKhuG&V2;p)@Nv?#Nhw_Ajm?7M@SB z;GV|Uk0Z!Du?Ho!3o9K8HxEB+pi7Pyb8jou4Ia77SMW9)fI?-b@Mo>+J!BJ%8{7l?*APU;%g|euGg}k4W8RGjiC1a#Pm^O14 zWyVQ9Xy4-_PM>8s$vvjIzHHt47U6v!75TrLU@L=-UGaqGq6H9OFt#{0bPVhSZcx2j zW3J+u(B8gyUb1d7!+7!og2vxs^1;S89c^ERkeUYDv)Eci^++`omB9xdUiJ?+<#LPZ zHJHX45PP+R7VQq7fU02^FhWMI?RooSzMHwfz(<+~I7!`frD`cu;o_0O7$gJu@05U3| z%gi?_O8j1r5Lo9%3I!?b&vA&+J%nHE$AIZOvK_7HR!mXIacl32w66J*964@Yr;X`u zR2e3v5PH10d8hrA7?w=sG0Vp!RIJu-W)N%)R&&fs3h({8s95e@6V+*-`uzR?S2M+C z6p7(is?dDO0KJ0$e*Gs)mRc1=05S~ZgSUAVf7}e`q>8L-)1+x8_OT5j+|XKjEH9FA z;>A2c*zMO%S=r0r%}X96-SmX2jhYt4swrkMLgg%7kk+5omjz3@mHXLfxDjtNxTaEt z18~qWb-)C?F8tpbwt%^HzQ&s9XI271$tPE)2PH(=nn2yQc(?NDUk)E3+wy>D%LzIa z`Hk;Vpiw#+Riz$BN77oYkt1coE_YBX?KwFG!?saP_~j4wgVVjyQYG_H=R2g5{L=g4v*-_R|CLpZV}JqJEz731h{rXufsb*6{+)7b(CJoj)^h zJ0=wO+>OBm4T63uBfsT`@>MtRKc_m2BLrSUGWn;ZJZkYVHbHH>}9BoarOfVgz|y zkM+!M1bLJZ&%nuoTl~K+D&Um<$NF2`02PYKD8*N%jWiir>t?}QNT$BxfoR!OL2J{c zT{^?(ZOLb;Q^6Ayyudk)THe-Hv!kqzQZq9RLT{JIjH-#$5IZYMP;#~g+9~n)%j&@# z{HZ%dFacNuWCrXFT3=1CF~0nbw11QvVm{;0r#gU24oTAZ`B|E(cWnF-Rc@$k2IaV7;uRk3mm8-OfS7eAF=2#(i8b zJHrtfZ~6yf|3P)E(zyD}5e7@b!lwJ$K99(o5KD4|S`sTyjeg47th!NA`tHS$fym>` z8>2Kb0oqNy_4H)2>hgzZ4)-}_^P{d+mOohk=wq->x?K;~IK!;%8!VERIPtqoNQhfs zuWXdEUhos>>CkXr&&Q~JvCjCH7dh`)3D-3-fofRKbh&66qZ$0ZS25ez37Ly_P&S&F z=t*6)0}andO}WJU`F@o51*8wpOMdITW$-=#!2k#lAV7w$pJjHh1$uw7VnV=YqC5z_S5*X*%nQ%AY=p*^=l-<4K_@WWwyzw4~f6n};az?yxk? zxuS14yu>F-*9(F88s^^;hKMRyzbtKiVH8@LH`Lr6QRyX3>O$t}x&meZz_9{Lh#<=S zbSEln8I#BzQu)K&KNz0hwZhs3$oYM4jQqJF5m>a=hibdWD-$EgIep7M8U4rojqlS> zG-=xqGS#7!MV54-o_&6i{b>@zBT5lhXAp-;>~PJdQq8DdU)WBAK2X}5-!y=`mmMjF zp0{8DPyg63%^OCR}?IH8C69`sZv3AXE>a)v8YBO46yl4McUq2q7qBS zA$1iI3#BmX>IqS|+Q8J8D1o(vTVe^0vpmbJAuf6fC29M5k? zq)3R!d(CpLD&0y06z4P7wDuU?cjceVYXEVMOXbM!c=VsdD}^flDyt?WMfl!z7%m#} zrPRx$!Q8noT&AC~xYD+D^?xkC!$zxIiKxk8Paa2AGcTW^T4yk(xvrwGTgj4u$JK$Q zrqdVF^!h>!`(Ae*az76^+f~M_{Jharpyr?P$d}^=(&%Jg-kX1>RQ087rT@cQ-AA#> zU)=if*Gz5Sjl>*X_@bQZn38rPmCCj1{8Uy~hi9+1ivVIMu{&X;qvTX^xJ&f#(Gtn0 z;$<3-Hz)9Um`ow|X()i+GHTYR*JI1E6eZYbMTAeBc%W+cs|p>1*Bv&I-=;b3s(k$e#>v> z16rC8Wa`H(|19JkNWa$V6^v(8a z^PI@M-E;MmBA-QLqtl4eT{J(&5;?xLAa{VlsGc!&7?B(vmtXZm&NkM_^1w-_JcCqO zZn8Xlq}Wb<t%rF+7}Yb~>sgz9hbVx)@bL_vRtNoK z3T#4@NUL=K<@j`GV{>tixFMU6IcmUnqt0`5=IfUW2ysPzvJ{kBaPY52AV$h~%t zZ>H#h@#G|j=hDAP)Fxe=8KA+d@mA}V(q^)ZCtjyum#HFq^a#WI^;-LmkYo<`6&~TD zmPI^698bu^1zRl`XE`!mCE(+)aytT(B`m@jsRGI}w{!^Xpo<%8d^UQ~*?$oCgVh1WT&k=*|pd9d<9H}Ip*H!vs0 z6gu1apbrQnHq4zF_quM)Y|>a1VO&?tMSboKEs>+h>5PH~7i zp?9%EZ%Gcb2Fl%HrPh*Mp3_yCm=^%(&0Q6+%(L#34Yl0KOFW=3vlM#Zo(h;O>!g!% zGa31?jP#y!LRZdjpQ7fvsV2%JZ$yc$b^X=&!hdlKHfSn;#l$x9d9I6hKbF$}kJ@4C zH7FUbs^K4hRns zP@$T_fxxL?BGJ{Yr8q7%v1s-!T+9!9x*UD6q;y-d6Hg=SNOzZI!!!-F`ldA~WS$|D z@LMuFm)m!Z4#JO-eCnocmB{TJ#;Ha>+NvWJ+!n=y#_iVAy&E}IYnc&O`679-DBK+L z922J7^iAz{Scf``_mo1l>meOecho~<$b?EHO~yQChr9by3cwwFFtP8|ExxvLqv4{N z{Gy{vGQdKdpkOWj6xa7Gn!D=E#bU&=QzHQSiNOob|H8{NiTB;Nq%BonVxM75grez& z_*_DEcU~L7p@x&`v@k$z_L$5CcX(*=-3V1}6$;{K1sO6=mj+LyF%ZHsfs1Q+IkAAT zQX;F~1Ps{MTZ4ZlbLw&_(15jHhEzmr6N z#bvCa6E&B@lu2ynaIKoi(bM(>Ba$xtVl*8yxF87gwA|`B$=g-l94a3?Qt7ub-UZgk zqM|z8@-=<;2!uEIcZCVi$C2n6tIAZl)`{R>PSOPbMHjRhYz2r385wrKheo zw8b$AvsPjfFLPhr>lZFC&$T{6Y&;>ihG#4tvmCG(k^_pT7OK*5J(_xW+=(;xV?@|G zyisP8S7z3V-ZcJyv-=et8(Dn1EXUmmv)+Dg`CWp&&t-`Oz@uWRc~sdU@g+C2Q=kb6 zaXLal7Jm|5${FfaKtQFwH zM+uqJq*WR8I>G-@f$n9XCRjVv#@}58@&l!8DP`jC%K(<`v1bp&SI@lm+VX9YR63@- zW_ll1bs(JSf!Ki6VT;LO@O2D}>7=ICPJR6NIlScj75yAO8Dn|YIUnM|-m7ca6xztK zBwlt>D&}7=>LBM`Deg%Fz(L6Ns8ERa>=?)=Z6ycsQ(Ci#^^sfs|60XX@Dw=~6@=-!#hvE~`O{XN~1kX2mqL1*NMMN_ybA3%J(Buh3JX5A~LC z{BEvRA`8tGC@&PeS~*+MMLHwvp^Y7d{4?E@T&_1E1k^U9AiJZ*`|%LeEsS``hV3|9 z*jIljz766qCHO*nHAB&7`kk?iay`6jBK`-$IzNlIJmk=GKp^tTTQm_6qN@9aVwlUW z&G68IB^tsmbFw(`valR5&IX&C{&LCxegD*DR}@XrJG>rchQuZ&OrXw(>PHNQHY4Z; zi9?N^)QxM*RU6l4$^SQSv62oPAH)^}b0&Mn+IPz%J!ELfB%lM@CP6RsJJ|@EQ+U7l z%08VU=#-jh{^Y9)Rtp8E;8FKGw=d8fY756g{_V;SonZLfa%Cr{3{YNr;do|R&YTb4 z1`TqtuU>>Zz0VWP{W>C4ux@vzn_pbMT-l-2?P*VTafrWa?ImH1JLp6(MSreqD0mM+ z0T&#G!XSXN|6~tun{C#w)3$cK(lnBvu(gM&Vw!#Aod6V5Si0-K(fW{FlJgDNz5;(J zfQv+^W`3VbK#tD6v@(0k6+>6PRSm)N7fL;+Cg+KTEvTEhS40r-(>yBTv1yg~O5EWg z2sioupNkO&s0OS>=7|2WroW1pwvyM$1emyELcsB{Iv@QukgjSmp>R-UH&!{rH+bs-chDY&8iDP*V~ zk1s?2s<`BQNsz^b_G&5HT8Qv%Uq5kO!l{KRs{aYX0Y;PlD{r@F>B&>58`LI(r{_`_ z6WZC9lH<#4-KoBQ5SGUrzDvAkZ{N0Rht=u6sO z`5eK2yA{pOj9*qY0v7UwOiqw%`9;Uj0eRO^*<*@WYxF+ebwZNL3fd&&<7Cv2I?C(72~0J)Q&l^mSIhpN}%>T8^> z6)e?`_wgA>cJ`#&zi#I5D+3UTS2S`gc&^-@%Db#kbInXSS4#3gF_R%l%* zS)%;p3RJx=bfu9B5$t$%I_yS=sT$t~F?MeHLt=^4O;eQ{dMovNU9axwC3f$BQEq?Vz zb1AKHgM$jvRuRa3Je;?EJJ+<&0NZcgGU67libhCcExzWv4W7vZwkt{hiqBd1#!X2; z%x^n_S?x0{vTWjN?_9DIADTu^owN4>Qi_}j)J&Fo zS)*;_MpBDWdE-OpP_^b7va1pR&>AZD%AqH68n-XAIVbEge3CE3IB^;mi9A~yHqcjG zw|>YL!FuIS?QQZ-kHj;2C_u=)@!~E>PjD=aGtsQ_eyl5#*WK4vw;F7Y&i2e9ygq5C zJrP_YuqO(VwdTAvf5TYe5T05CS0cB!?PAW<(%1G4295yK=u!nc_;&6KShJ8sU~Z6l z3M&$)$N;ZQ8H?e+M}}7vtS+2K6Q3Ed?|c-hcw_Ou>#uv#NBvc4{f;`}@O$MCcy5P{4IP$o zc|QEw|6ldXB6pCzNrBsdw?k+tUm;;cohB*gcQ0gDIV4Gmj%S0~=+0rN!2FUja`aY+ zAh)iF{wjvzu2usLfaurL*8gTcyXq%a(1amvZO|lB&?(B3rCc*@@DV*B)gP}dkCO$^ zCIKm1@6vx)QYr(*@#NkZ8IbYDzO=&sr7FfGq)ZnRc?`BXLPlOUx*!}{uS5WpZX?ZG zP446~FwAWZE5xZ}@BwiHAf~M9a%d__!8P{2p~jlT3}KWkX5O-?MwTaqAtNgDdM?pm zS4&N%Ajl<0|AhUysJ528<*8e617QCiTN&dCVZv9Yu-hb?mM zV|j%|U`Dr#4ih5r36F0yrV0FibHub`O|Ut^1#H~h3ahkp2d%b61G_wLagMl_Y1zzY z%Nx&hc+)NaF3+y>2|b+ujyeq&hzC3$EFm^6yB1i>Wlc;cpeN5-3`Gxodu$Pg;Ae7o z-6uGxE49kOmwhOz1L93c`hm={5tN8+Y}}pUfY4Gq>H<)T{GY%l6HN%2YoF!Atv4fn zs@9S`vIXvyLTC`U_DiHH%Pv(+hSuGq_pHJ+BMk! z66ccFpq_<=CC^#v=RbBbvlJ;Ouhz1uO{DT4_n%!|#Wz%ELrEiULDY|(1DMnQ*ZlVl zv{kRN9lYv*`|zF?-(*=}_p-zIN&pM4(KyE#*;9Q1N#XV)pA-!Ddw|&!i7Gl}ABj$a z@=45Aie*y))n-jUHKPSf;jUzS1334&WH^6E1UMi-fbRfTEb*5DgnwCVWt*#6yTyY1 z;N9S{w3!qfH@tRN0ztoTyiiiQ@CKZ?6@x6-pR;0c2l!`f* zCpO~1?K^_0MTaQ6fHv7KGM-vR$bI}CYW+_DcCS+KTKtYSbw|xjW&6uhZ#ugFFY?kG zUsy(JJqytZ9_o~_z-LekZ((&?4I?5L13PiKRA>^^wN(RPyYT1!`5umHevYHX`Y{kK z`5@APqY{yn{49!;WQVpZaq+XqZY|XAYo8fNw~?us0001i0tGMGJk>dcvUkfqNC*R$ zIG{DrsXgbBz$n*!eTMUfe+kq(uc-zESBDDu{){Mf5e}d>oB1tq@vV6dxdPLRU7aE% zT9s^M?U(C2uQ0*Nna}gW^|=}K2QsXCvQu>1TSd31oc|H=xnSbGHTlxLmTD7{;|~@5 zk5GXwxVJD+X(pIGhf)bvS}vq%CS#P(B5`!P6D8)BYdIfG>LKFdmftjg zw!P+dCxKITjfW49YT_Sh3;4p8U(Ye46phzYYj za=O$r2Tsw^#GA8?o>v--onQc{cr*r1e!vAs8a>XoxX-Q%Q#U(sY@kAz@ZsX4X>v-yF1!i7d1^5>~m7IU2XZ z{T4)qEA1p&Qkcx(yx&%cIS)f|pkeEtP&#kj7orFCkuw=XhBu@DC7h0d zRIBT4ezzk=^+yP2ONqj}wx&bQr^ls#FE_t)x=@KT>)lmbip;XQb?b4wQokIgt}w)` zn_L$p*j~vPYyhWXTpq)|TjqqO46MVj_&zwX<}Ctp=?kIWvoWUg)~Us4$K`fELv2(ddm#|pN;is z<(eYBgsRnXQ-qJqTwuiH(uMM6QCSMY>GcPDKOP*QMjvS!a^0K0+(LJgWQ_Ww74GV+ zuJ@NdP{G$O&(=UZyKuY982=pBKV@~4ySauD?!;=~%d|A=ZwCrl53yI%QWbh~`Sy;N zu;A$E67k|XLldB2`DL0b`ZD)gM<}D9Bc<3ZY6Ypa&W&0s>+?H5XAddO!=st5Vg-UI za87)OtrP98Nh5kT$TYDaJI@?J*zPdZYU3$7&6PAtuMD#Aso>Ie)tP54m76gge%DIR zh6c2l&{+C4Lm_&+v)6Ke)+q@9Fx32q&A=qXW8-d=Df82AZQW6zpMM~q7qJ4?ok*hA zh2B9kR!$iQ;fbzAb_)CjZ#TyDf;frDHwT@y&@F=NI#xhuu;KjJ6!Y5%L0TEt2!IQH zbumB6J4A=aRSft54tD%t^Iu+4-W^fSIy2D&%}m}h=Stb^h{c&`7e~Qyjm_tCFyL1a zf&F`{_+X|@bl8H6y3q@i=x94m9mLWuJ)0VdsM%)U!V63eB6T8cptp}sV=ax6@N`&7 z+TU=lQt;WKh>>Z%=BLh@+qB;fAoRmcn#x|}4oG{Wu1^0G!jYn(Z}Kox}m8(mO! zYV-Tt4jRXI?0{}^j-#dTJUX1*6Y=Db7#+iWJx1+4OqlJ>ECpfmEM|2|zvS-z z!QBtw!+eo@NOdOqX^4Hsj~z7e)xD*qe&ciIAqf}O%=KTT)4+*+??C?(-Pjqy9zi1KG=FQ7l5mvL+g-7pqbuPtvPXWCj3i0U_O?)#Uia1UAy#TXq>XYhNIJ zRht0?E2q?CX`!WtIdD?zADQ+g)Hk9+)BOP2G3wg?vE)`MslM6xDBwOzSG~&C-I_@8 zwE$7eqGn$Q9W8kEO`K4x=*!e(y0+*4-Q z=o=4Oi>47X3VAr1+I?ngFn`sx1CwTHQ>Uh~i7C+`wv`XV>5+h4h}Hs4qg-Fp2z7c# zFML>f=YK_4P0lN`5k_v=nLpFcOsDmC64Y{{-_u4d61O0*%BiY}31XWa;D(!DODJh? zwe$+Meg!_itom8!%t+8E$PrCbin!a}2O~kLBv`2-_k2oNSi61R*jUW~{Eaau55ap; zMw+byrw{ZqdkK~JqcEFk)*bKfP(%nMB%j!iQ0bXFx z-G0&^xtjEs-zMOVL9zz($UgpV@h$b!((-~Os7hng{ve%|VyhP%*Z}3%v`wy)NE5hs z&-&eEbqc-FGHR(De4x7vX}sEQcFzmVO=qoN7n{q(xOUMyK7aa*3W7ac)axZ9%L2u; zmyS-B7v?m~mlq9yoq&1jI*_wTfttl=xitk3p1x-5chIkOQ;^x>_*^?DidL5OyOWt6P&bJd2Y`QAN*{5I{bGqla08NTnR#Zvm=&1xx5A|k&R+L;zO z2jKpxUvM(F=v*0s8YkUu3@TcSL+p3zel7Lh(rqZhYsgG(PjRo4+Lwc(-Ae#lE0EzK zNf8mCZIceE7h1_}67=e*T=Kq8ojHqQZ|k@1UW9V>{XDc#&M+*}o{f6Yg+=!DqsX3?$g3apEB9Um(A zpjHH@HjG+3&Je(zGj?gjls1YunBhkjp!*zS9rALJ11`a7Uk%k%%ht2*Vcs33d!fwr z#F$srH^-7O%$kO)^PbZh#OAOEDl4{*%;hYFNNfv=D#}~i&3&FmC&?AvR|yf4dM>n5 z7oxjDHb1|5{Ok=o&XZBB#U>U93&J^eEitpyoHa)k4*j=5UPnJL)K4o#E0df{8$8#i z=-395O{xcysV3?wRJEL=*&rY{zMwAvIojiE`(A&;9@^J!b)nl8nu#GccFR_U|E`7k znSGicyf4};$zJ+CoM>iG!d-pO7R+|J1Jew9j@Ft5M1>wbU-!_+cd^nKYFbs8!Ufx_ ze6`KO0h}b(Dklk%TpvY3>HvWP1PBl#XIX>h$;lwV)^+>W0!dSU@4{rZj{BL`9$4GB z#n-P{2M!yBC`<>LVTT!cj6iV+`hCM*lhVtOEy6Ufk~6%PXQ*k%PfCs*#U@xhsKB;# zcHr6h$@A(bw2+=-vvZRfcNlFV3%fJpm?X{rdrjW%3}ZL}$Wl$D?%G)qW$=`A17Ly5 z#@CY}VaQ9A)>sxz;8+L|i8z06q8{Ctjq-n10c)_%d8E2-*sX3PeyJfEdV-jMaf)L| z=TY{j1&vwo`{)VyjfV)ir5&JZ^A$br7cJf#P?@vQc)NUv3E|J&_xS1bWLb^1dZQoF zGT?b)d&yL{PQAU1J!OfYeiW7uhv4vZPFJZSFc+~p3@BY#e+7o>A*SkD_ zHkw};T)O!G{Z}Fn{o}NJVt|umP_dtHEpZKF0o?b%+WB!R-QJB(I?N+Hn%E!qhB$~_ zu%qf}bo%1Xl8;1Xc5gwZy8B8l&(+IQtXBuXFpB|w16W#xsJtL@YN04}ndEZas`+kA zJX{zP*F?<3i~o;S00q{UbKYB-9m4xon{@9S%x!Q0Tp6j0r zKIw&zXEMUrXjP{%?uUs~_gj1EtJLzG#`RMFUR9W@mC6-JD%0K+R7kvNS{ z^FM*Ae7@7)D&9>uVM|3^dhcx`q?;GfI+1dv1UMc%Vij|9JB}q6b~u6U(cn zJH!o>Svo#SvV+Q{^uM;lUWZ`>2oNAZfmVK($@I1g01AQHixeeO*Q*t>!+N31Ph|Bx zGLu^twPg{Fy;*^cc@aY<@+%2+Ce_#)xgssQDo>BV1w*V- z(wleC=<-KC!5VanMvQ@OR<>*2EM8Ujq{5hxyKGwD1FFW0H&Jl=g$CH%z}mD7Yu(dv z?2E@s;|H(6(5tntnd#Vt4l&E-@91V4nYx=HX@fUWa_B8dBb<_h0YqKp8%M|n@%@39 zyo?$p`VvvxGRO||*!bi>pP6F<(jF~Um4}s~!)&&N$SS|I2TGiB*d|s;XxX|?GC`tM zuQb=MHv!)zuy1@-=uOje%Y8SQj+Vef1>*iy4`I@eqg&t@t9^lwQcCOk@{u z{|9a`=Z^5=1e2s2Z=h$d7CtpH6@G&+VZ9-md8{prEH*vI;M^y%)=gE4TPdt?&rcw4 zhX`ahwlX6J?J?&(xr{{0pmiA5mZPm?f&y8#5PE$*-PMGTvGoM@cQe|9a;L%MNxDGH zPItG>Np-(#^vbvO zYlkPNa&>c(=5}Q?L()|JtMBPnFnoSyD=J*2_@S`Qiju9l=kE+ZTVV+N^Z3Xx(^v)c z5WVqznmd(W+;tmQ62VK!J3k2qIZFsbAhIi~%bFMz5!+U`hjw<1C-76K!Acftr~L@= z_D5Y@Yn)aqA07kx#-{v8f~d&2n&!CuD)^OPgGu^YTVDori{ceo-4X2Yba!~bt<{<8 z>TMp5fJh4&P39#~7<(f<_lx%{Eh<;OYQM1@(<~|uA)4yU3Zl7ja{|%X%TtoMyYRpe z$(L@>daFcLhvldIS>r#cI06I!000000aUy?5J>D?Wl5R~m`n%HTUvVZJ{N*w8(Lh{ z^_#Ko~Xplcg?!2@UUQ z=1B;g&Dm9zy3=xYiF>)`U_;pefc-`yAIuk`d63N fn*wjQQLZMzvf=dzVx*Eu|Ha&qP81{s>dU*pKfkT7 diff --git a/ldb/testdata/blocks1-256.bz2 b/testdata/blocks1-256.bz2 similarity index 100% rename from ldb/testdata/blocks1-256.bz2 rename to testdata/blocks1-256.bz2 From 2ec951189199584c2f73a0bea12d02d6e11983eb Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sun, 13 Oct 2013 13:02:45 -0500 Subject: [PATCH 069/163] Go fmt. --- ldb/dbcache.go | 4 ++-- ldb/leveldb.go | 2 +- sqlite3/sqlitedbcache.go | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/ldb/dbcache.go b/ldb/dbcache.go index 0ac0506e..92b5fbc3 100644 --- a/ldb/dbcache.go +++ b/ldb/dbcache.go @@ -82,7 +82,7 @@ func (db *LevelDb) fetchTxDataBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, //log.Trace("transaction %v is at block %v %v txoff %v, txlen %v\n", // txsha, blksha, blkHeight, txOff, txLen) - rbuf := bytes.NewBuffer(blkbuf[txOff:txOff+txLen]) + rbuf := bytes.NewBuffer(blkbuf[txOff : txOff+txLen]) var tx btcwire.MsgTx err = tx.Deserialize(rbuf) @@ -102,7 +102,7 @@ func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) ([]*btcdb.TxListReply, e return []*btcdb.TxListReply{}, err } - replies := make ([]*btcdb.TxListReply, 1) + replies := make([]*btcdb.TxListReply, 1) btxspent := make([]bool, len(tx.TxOut), len(tx.TxOut)) for idx := range tx.TxOut { diff --git a/ldb/leveldb.go b/ldb/leveldb.go index a74b321a..a3d98d22 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -10,10 +10,10 @@ import ( "github.com/conformal/btcdb" "github.com/conformal/btcutil" "github.com/conformal/btcwire" - "github.com/conformal/seelog" "github.com/conformal/goleveldb/leveldb" "github.com/conformal/goleveldb/leveldb/cache" "github.com/conformal/goleveldb/leveldb/opt" + "github.com/conformal/seelog" "os" "runtime" "sync" diff --git a/sqlite3/sqlitedbcache.go b/sqlite3/sqlitedbcache.go index daac3ec2..3ec61a09 100644 --- a/sqlite3/sqlitedbcache.go +++ b/sqlite3/sqlitedbcache.go @@ -242,7 +242,7 @@ func (db *SqliteDb) FetchTxBySha(txsha *btcwire.ShaHash) ([]*btcdb.TxListReply, // Check Tx cache if txc, ok := db.fetchTxCache(txsha); ok { - replies := make ([]*btcdb.TxListReply, 1) + replies := make([]*btcdb.TxListReply, 1) tx := txc.tx btxspent := make([]bool, len(tx.TxOut), len(tx.TxOut)) @@ -308,7 +308,6 @@ func (db *SqliteDb) FetchTxBySha(txsha *btcwire.ShaHash) ([]*btcdb.TxListReply, txc.blksha = *blksha db.insertTxCache(&txc) - btxspent := make([]bool, len(tx.TxOut), len(tx.TxOut)) for idx := range tx.TxOut { byteidx := idx / 8 @@ -316,7 +315,7 @@ func (db *SqliteDb) FetchTxBySha(txsha *btcwire.ShaHash) ([]*btcdb.TxListReply, btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0 } - replies := make ([]*btcdb.TxListReply, 1) + replies := make([]*btcdb.TxListReply, 1) txlre := btcdb.TxListReply{Sha: txsha, Tx: &tx, BlkSha: blksha, Height: height, TxSpent: btxspent, Err: err} replies[0] = &txlre return replies, nil From 75896b63ec221388bba096f8235c6d08c8ca1f42 Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Sun, 13 Oct 2013 11:08:21 -0400 Subject: [PATCH 070/163] Add Api to fetch potentially fully spent tx (most recent only) Using FetchUnSpentTxByShaList only API required unexpected contortions in btcchain. --- db.go | 12 ++++++++++-- ldb/dbcache.go | 7 +++++++ sqlite3/sqlitedbcache.go | 7 +++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/db.go b/db.go index b15c95b6..3f6d418c 100644 --- a/db.go +++ b/db.go @@ -79,7 +79,15 @@ type Db interface { // FetchTxByShaList returns a TxListReply given an array of transaction // hashes. The implementation may cache the underlying data if desired. - FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) ([]*TxListReply) + // This differs from FetchUnSpentTxByShaList in that it will return + // the most recent known Tx, if it is fully spent or not. + FetchTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply + + // FetchUnSpentTxByShaList returns a TxListReply given an array of + // transaction hashes. The implementation may cache the underlying + // data if desired. Fully spent transactions will not normally not + // be returned in this operation. + FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply // InsertBlock inserts raw block and transaction data from a block // into the database. The first block inserted into the database @@ -145,7 +153,7 @@ type DriverDB struct { type TxListReply struct { Sha *btcwire.ShaHash Tx *btcwire.MsgTx - BlkSha *btcwire.ShaHash + BlkSha *btcwire.ShaHash Height int64 TxSpent []bool Err error diff --git a/ldb/dbcache.go b/ldb/dbcache.go index 92b5fbc3..e430819c 100644 --- a/ldb/dbcache.go +++ b/ldb/dbcache.go @@ -36,6 +36,13 @@ func (db *LevelDb) fetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, er return } +// FetchTxByShaList returns the most recent tx of the name fully spent or not +func (db *LevelDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { + // until the fully spent separation of tx is complete this is identical + // to FetchUnSpentTxByShaList + return db.FetchUnSpentTxByShaList(txShaList) +} + // FetchUnSpentTxByShaList given a array of ShaHash, look up the transactions // and return them in a TxListReply array. func (db *LevelDb) FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { diff --git a/sqlite3/sqlitedbcache.go b/sqlite3/sqlitedbcache.go index 3ec61a09..3caef680 100644 --- a/sqlite3/sqlitedbcache.go +++ b/sqlite3/sqlitedbcache.go @@ -131,6 +131,13 @@ func (db *SqliteDb) insertBlockCache(sha *btcwire.ShaHash, blk *btcutil.Block) { bc.blockMap[blkObj.sha] = &blkObj } +// FetchTxByShaList returns the most recent tx of the name fully spent or not +func (db *SqliteDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { + // until the fully spent separation of tx is complete this is identical + // to FetchUnSpentTxByShaList + return db.FetchUnSpentTxByShaList(txShaList) +} + // FetchUnSpentTxByShaList given a array of ShaHash, look up the transactions // and return them in a TxListReply array. func (db *SqliteDb) FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { From a27c37793bd4b829e5740ba2708d7288704cf6e3 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sun, 13 Oct 2013 18:16:11 -0500 Subject: [PATCH 071/163] Add basic infrastructure for interface tests. This is simply at start at providing generic interface tests. The only thing is tests so far is the empty database conditions on NewestSha, but it adds infrastructure for creating and opening databases with special type handling based on the database and necessary logic to teardown so multiple backends can be tested simultaneously. This and the next series of commits all discussed with drahn@. --- common_test.go | 131 ++++++++++++++++++++++++++++++++++++++++++++++ interface_test.go | 54 +++++++++++++++++++ 2 files changed, 185 insertions(+) create mode 100644 common_test.go create mode 100644 interface_test.go diff --git a/common_test.go b/common_test.go new file mode 100644 index 00000000..89f7dc69 --- /dev/null +++ b/common_test.go @@ -0,0 +1,131 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package btcdb_test + +import ( + "fmt" + "github.com/conformal/btcdb" + _ "github.com/conformal/btcdb/ldb" + _ "github.com/conformal/btcdb/sqlite3" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" + "os" + "path/filepath" +) + +var zeroHash = btcwire.ShaHash{} + +// testDbRoot is the root directory used to create all test databases. +const testDbRoot = "testdbs" + +// filesExists returns whether or not the named file or directory exists. +func fileExists(name string) bool { + if _, err := os.Stat(name); err != nil { + if os.IsNotExist(err) { + return false + } + } + return true +} + +// openDB is used to open an existing database based on the database type and +// name. +func openDB(dbType, dbName string) (btcdb.Db, error) { + // Handle memdb specially since it has no files on disk. + if dbType == "memdb" { + db, err := btcdb.OpenDB(dbType, "") + if err != nil { + return nil, fmt.Errorf("error opening db: %v", err) + } + return db, nil + } + + dbPath := filepath.Join(testDbRoot, dbName) + db, err := btcdb.OpenDB(dbType, dbPath) + if err != nil { + return nil, fmt.Errorf("error opening db: %v", err) + } + + return db, nil +} + +// createDB creates a new db instance and returns a teardown function the caller +// should invoke when done testing to clean up. The close flag indicates +// whether or not the teardown function should sync and close the database +// during teardown. +func createDB(dbType, dbName string, close bool) (btcdb.Db, func(), error) { + // Handle memory database specially since it doesn't need the disk + // specific handling. + if dbType == "memdb" { + db, err := btcdb.CreateDB(dbType, "") + if err != nil { + return nil, nil, fmt.Errorf("error creating db: %v", err) + } + + // Setup a teardown function for cleaning up. This function is + // returned to the caller to be invoked when it is done testing. + teardown := func() { + if close { + db.Close() + } + } + + return db, teardown, nil + } + + // Create the root directory for test databases. + if !fileExists(testDbRoot) { + if err := os.MkdirAll(testDbRoot, 0700); err != nil { + err := fmt.Errorf("unable to create test db "+ + "root: %v", err) + return nil, nil, err + } + } + + // Create a new database to store the accepted blocks into. + dbPath := filepath.Join(testDbRoot, dbName) + _ = os.RemoveAll(dbPath) + db, err := btcdb.CreateDB(dbType, dbPath) + if err != nil { + return nil, nil, fmt.Errorf("error creating db: %v", err) + } + + // Setup a teardown function for cleaning up. This function is + // returned to the caller to be invoked when it is done testing. + teardown := func() { + dbVersionPath := filepath.Join(testDbRoot, dbName+".ver") + if close { + db.Sync() + db.Close() + } + os.RemoveAll(dbPath) + os.Remove(dbVersionPath) + os.RemoveAll(testDbRoot) + } + + return db, teardown, nil +} + +// setupDB is used to create a new db instance with the genesis block already +// inserted. In addition to the new db instance, it returns a teardown function +// the caller should invoke when done testing to clean up. +func setupDB(dbType, dbName string) (btcdb.Db, func(), error) { + db, teardown, err := createDB(dbType, dbName, true) + if err != nil { + return nil, nil, err + } + + // Insert the main network genesis block. This is part of the initial + // database setup. + genesisBlock := btcutil.NewBlock(&btcwire.GenesisBlock) + _, err = db.InsertBlock(genesisBlock) + if err != nil { + teardown() + err := fmt.Errorf("failed to insert genesis block: %v", err) + return nil, nil, err + } + + return db, teardown, nil +} diff --git a/interface_test.go b/interface_test.go new file mode 100644 index 00000000..ce3c50a8 --- /dev/null +++ b/interface_test.go @@ -0,0 +1,54 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package btcdb_test + +import ( + "github.com/conformal/btcdb" + "testing" +) + +// testNewestShaEmpty ensures the NewestSha returns the values expected by +// the interface contract. +func testNewestShaEmpty(t *testing.T, db btcdb.Db) { + sha, height, err := db.NewestSha() + if err != nil { + t.Errorf("NewestSha error %v", err) + } + if !sha.IsEqual(&zeroHash) { + t.Errorf("NewestSha wrong hash got: %s, want %s", sha, &zeroHash) + + } + if height != -1 { + t.Errorf("NewestSha wrong height got: %s, want %s", height, -1) + } +} + +// TestEmptyDB tests that empty databases are handled properly. +func TestEmptyDB(t *testing.T) { + for _, dbType := range btcdb.SupportedDBs() { + // Ensure NewestSha returns expected values for a newly created + // db. + db, teardown, err := createDB(dbType, "emptydb", false) + if err != nil { + t.Errorf("Failed to create test database %v", err) + return + } + testNewestShaEmpty(t, db) + + // Ensure NewestSha still returns expected values for an empty + // database after reopen. + db.Close() + db, err = openDB(dbType, "emptydb") + if err != nil { + t.Errorf("Failed to open test database %v", err) + return + } + testNewestShaEmpty(t, db) + db.Close() + + // Clean up the old db. + teardown() + } +} From 562294f938606496448f0b966ee8a279efdc8a13 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sun, 13 Oct 2013 19:25:21 -0500 Subject: [PATCH 072/163] Add loadBlocks infrastructure to interface tests. --- common_test.go | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/common_test.go b/common_test.go index 89f7dc69..16e56561 100644 --- a/common_test.go +++ b/common_test.go @@ -5,14 +5,33 @@ package btcdb_test import ( + "compress/bzip2" + "encoding/binary" "fmt" "github.com/conformal/btcdb" _ "github.com/conformal/btcdb/ldb" _ "github.com/conformal/btcdb/sqlite3" "github.com/conformal/btcutil" "github.com/conformal/btcwire" + "io" "os" "path/filepath" + "strings" + "testing" +) + +var ( + // network is the expected bitcoin network in the test block data. + network = btcwire.MainNet + + // savedBlocks is used to store blocks loaded from the blockDataFile + // so multiple invocations to loadBlocks from the various test functions + // do not have to reload them from disk. + savedBlocks []*btcutil.Block + + // blockDataFile is the path to a file containing the first 256 blocks + // of the block chain. + blockDataFile = filepath.Join("testdata", "blocks1-256.bz2") ) var zeroHash = btcwire.ShaHash{} @@ -129,3 +148,72 @@ func setupDB(dbType, dbName string) (btcdb.Db, func(), error) { return db, teardown, nil } + +// loadBlocks loads the blocks contained in the testdata directory and returns +// a slice of them. +func loadBlocks(t *testing.T) ([]*btcutil.Block, error) { + if len(savedBlocks) != 0 { + return savedBlocks, nil + } + + var dr io.Reader + fi, err := os.Open(blockDataFile) + if err != nil { + t.Errorf("failed to open file %v, err %v", blockDataFile, err) + return nil, err + } + if strings.HasSuffix(blockDataFile, ".bz2") { + z := bzip2.NewReader(fi) + dr = z + } else { + dr = fi + } + + defer func() { + if err := fi.Close(); err != nil { + t.Errorf("failed to close file %v %v", blockDataFile, err) + } + }() + + // Set the first block as the genesis block. + blocks := make([]*btcutil.Block, 0, 256) + genesis := btcutil.NewBlock(&btcwire.GenesisBlock) + blocks = append(blocks, genesis) + + for height := int64(1); err == nil; height++ { + var rintbuf uint32 + err := binary.Read(dr, binary.LittleEndian, &rintbuf) + if err == io.EOF { + // hit end of file at expected offset: no warning + height-- + err = nil + break + } + if err != nil { + t.Errorf("failed to load network type, err %v", err) + break + } + if rintbuf != uint32(network) { + t.Errorf("Block doesn't match network: %v expects %v", + rintbuf, network) + break + } + err = binary.Read(dr, binary.LittleEndian, &rintbuf) + blocklen := rintbuf + + rbytes := make([]byte, blocklen) + + // read block + dr.Read(rbytes) + + block, err := btcutil.NewBlockFromBytes(rbytes) + if err != nil { + t.Errorf("failed to parse block %v", height) + return nil, err + } + blocks = append(blocks, block) + } + + savedBlocks = blocks + return blocks, nil +} From 53ea2cf0bad3c5d9a83a5321fcce2e0b78536f7e Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sun, 13 Oct 2013 19:28:40 -0500 Subject: [PATCH 073/163] Add interface test for adding a duplicate driver. This commit adds an interface test to ensure that attempting to add a duplicate driver for a given database type does not overwrite an existing one. --- interface_test.go | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/interface_test.go b/interface_test.go index ce3c50a8..fe383f55 100644 --- a/interface_test.go +++ b/interface_test.go @@ -5,6 +5,7 @@ package btcdb_test import ( + "fmt" "github.com/conformal/btcdb" "testing" ) @@ -52,3 +53,43 @@ func TestEmptyDB(t *testing.T) { teardown() } } + +// TestAddDuplicateDriver ensures that adding a duplicate driver does not +// overwrite an existing one. +func TestAddDuplicateDriver(t *testing.T) { + supportedDBs := btcdb.SupportedDBs() + if len(supportedDBs) == 0 { + t.Errorf("TestAddDuplicateDriver: No backends to test") + return + } + dbType := supportedDBs[0] + + // bogusCreateDB is a function which acts as a bogus create and open + // driver function and intentionally returns a failure that can be + // detected if the interface allows a duplicate driver to overwrite an + // existing one. + bogusCreateDB := func(string) (btcdb.Db, error) { + return nil, fmt.Errorf("duplicate driver allowed for database "+ + "type [%v]", dbType) + } + + // Create a driver that tries to replace an existing one. Set its + // create and open functions to a function that causes a test failure if + // they are invoked. + driver := btcdb.DriverDB{ + DbType: dbType, + Create: bogusCreateDB, + Open: bogusCreateDB, + } + btcdb.AddDBDriver(driver) + + // Ensure creating a database of the type that we tried to replace + // doesn't fail (if it does, it indicates the driver was erroneously + // replaced). + _, teardown, err := createDB(dbType, "dupdrivertest", true) + if err != nil { + t.Errorf("TestAddDuplicateDriver: %v", err) + return + } + teardown() +} From d4c12144969942712994ccccd1f082b8dcbc406f Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sun, 13 Oct 2013 19:44:15 -0500 Subject: [PATCH 074/163] Add interface test for create and open failures. This commit adds an interface test to ensure that failures which occur while creating or opening a database are properly handled. --- interface_test.go | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/interface_test.go b/interface_test.go index fe383f55..b28bd64e 100644 --- a/interface_test.go +++ b/interface_test.go @@ -93,3 +93,45 @@ func TestAddDuplicateDriver(t *testing.T) { } teardown() } + +// TestCreateOpenFail ensures that errors which occur while opening or closing +// a database are handled properly. +func TestCreateOpenFail(t *testing.T) { + // bogusCreateDB is a function which acts as a bogus create and open + // driver function that intentionally returns a failure which can be + // detected. + dbType := "createopenfail" + openError := fmt.Errorf("failed to create or open database for "+ + "database type [%v]", dbType) + bogusCreateDB := func(string) (btcdb.Db, error) { + return nil, openError + } + + // Create and add driver that intentionally fails when created or opened + // to ensure errors on database open and create are handled properly. + driver := btcdb.DriverDB{ + DbType: dbType, + Create: bogusCreateDB, + Open: bogusCreateDB, + } + btcdb.AddDBDriver(driver) + + // Ensure creating a database with the new type fails with the expected + // error. + _, err := btcdb.CreateDB(dbType, "createfailtest") + if err != openError { + t.Errorf("TestCreateOpenFail: expected error not received - "+ + "got: %v, want %v", err, openError) + return + } + + // Ensure opening a database with the new type fails with the expected + // error. + _, err = btcdb.OpenDB(dbType, "openfailtest") + if err != openError { + t.Errorf("TestCreateOpenFail: expected error not received - "+ + "got: %v, want %v", err, openError) + return + } + +} From 82d1898b12b2e32e4b0cb359a487f7feffc7d57b Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sun, 13 Oct 2013 20:03:11 -0500 Subject: [PATCH 075/163] Add interface test for unsupported dbtype failures. This commit adds an interface test to ensure that the interface returns the expected error when trying to open or create an unsupported database type. --- interface_test.go | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/interface_test.go b/interface_test.go index b28bd64e..66ea7926 100644 --- a/interface_test.go +++ b/interface_test.go @@ -133,5 +133,27 @@ func TestCreateOpenFail(t *testing.T) { "got: %v, want %v", err, openError) return } - +} + +// TestCreateOpenUnsupported ensures that attempting to create or open an +// unsupported database type is handled properly. +func TestCreateOpenUnsupported(t *testing.T) { + // Ensure creating a database with an unsupported type fails with the + // expected error. + dbType := "unsupported" + _, err := btcdb.CreateDB(dbType, "unsupportedcreatetest") + if err != btcdb.DbUnknownType { + t.Errorf("TestCreateOpenUnsupported: expected error not "+ + "received - got: %v, want %v", err, btcdb.DbUnknownType) + return + } + + // Ensure opening a database with the new type fails with the expected + // error. + _, err = btcdb.OpenDB(dbType, "unsupportedopentest") + if err != btcdb.DbUnknownType { + t.Errorf("TestCreateOpenUnsupported: expected error not "+ + "received - got: %v, want %v", err, btcdb.DbUnknownType) + return + } } From ff429203c429d4fa3ba82da64c4489e2772741ea Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sun, 13 Oct 2013 20:24:03 -0500 Subject: [PATCH 076/163] Start interface tests that require state. --- interface_test.go | 84 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/interface_test.go b/interface_test.go index 66ea7926..f223375d 100644 --- a/interface_test.go +++ b/interface_test.go @@ -10,6 +10,14 @@ import ( "testing" ) +var ( + // ignoreDbTypes are types which should be ignored when running tests + // that iterate all supported DB types. This allows some tests to add + // bogus drivers for testing purposes while still allowing other tests + // to easily iterate all supported drivers. + ignoreDbTypes = map[string]bool{"createopenfail": true} +) + // testNewestShaEmpty ensures the NewestSha returns the values expected by // the interface contract. func testNewestShaEmpty(t *testing.T, db btcdb.Db) { @@ -157,3 +165,79 @@ func TestCreateOpenUnsupported(t *testing.T) { return } } + +// testInterface tests performs tests for the various interfaces of btcdb which +// require state in the database for the given database type. +func testInterface(t *testing.T, dbType string) { + db, teardown, err := setupDB(dbType, "interface") + if err != nil { + t.Errorf("Failed to create test database %v", err) + return + } + defer teardown() + + // Load up a bunch of test blocks. + blocks, err := loadBlocks(t) + if err != nil { + t.Errorf("Unable to load blocks from test data %v: %v", + blockDataFile, err) + return + } + + t.Logf("Loaded %d blocks", len(blocks)) + for height := int64(1); height < int64(len(blocks)); height++ { + block := blocks[height] + + // Ensure there are no errors inserting each block into the + // database. + newHeight, err := db.InsertBlock(block) + if err != nil { + t.Errorf("InsertBlock: failed to insert block %v err %v", + height, err) + return + } + if newHeight != height { + t.Errorf("InsertBlock: height mismatch got: %v, want: %v", + newHeight, height) + return + } + } + + // TODO(davec): Need to figure out how to handle the special checks + // required for the duplicate transactions allowed by blocks 91842 and + // 91880 on the main network due to the old miner + Satoshi client bug. + + // TODO(davec): Add tests for the following functions: + /* + Close() + DropAfterBlockBySha(*btcwire.ShaHash) (err error) + ExistsSha(sha *btcwire.ShaHash) (exists bool) + FetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) + FetchBlockShaByHeight(height int64) (sha *btcwire.ShaHash, err error) + FetchHeightRange(startHeight, endHeight int64) (rshalist []btcwire.ShaHash, err error) + ExistsTxSha(sha *btcwire.ShaHash) (exists bool) + FetchTxBySha(txsha *btcwire.ShaHash) ([]*TxListReply, error) + FetchTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply + FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply + - InsertBlock(block *btcutil.Block) (height int64, err error) + InvalidateBlockCache() + InvalidateCache() + InvalidateTxCache() + NewIterateBlocks() (pbi BlockIterator, err error) + NewestSha() (sha *btcwire.ShaHash, height int64, err error) + RollbackClose() + SetDBInsertMode(InsertMode) + Sync() + */ +} + +// TestInterface performs tests for the various interfaces of btcdb which +// require state in the database for each supported database type (those loaded +// in common_test.go that is). +func TestInterface(t *testing.T) { + for _, dbType := range btcdb.SupportedDBs() { + if _, exists := ignoreDbTypes[dbType]; !exists { + testInterface(t, dbType) + } + } +} From 2d84623493410e6ea13953db13296041e9dfef40 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sun, 13 Oct 2013 20:30:38 -0500 Subject: [PATCH 077/163] Move non-interface specific tests into db_test.go. The idea here is that interface_test.go will be directly usable in each implementation to increase test coverage there as well, but also works at the top-most level to test arbitrary backends. --- db_test.go | 178 ++++++++++++++++++++++++++++++++++++++++++++++ interface_test.go | 169 ------------------------------------------- 2 files changed, 178 insertions(+), 169 deletions(-) create mode 100644 db_test.go diff --git a/db_test.go b/db_test.go new file mode 100644 index 00000000..32dcd3ae --- /dev/null +++ b/db_test.go @@ -0,0 +1,178 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package btcdb_test + +import ( + "fmt" + "github.com/conformal/btcdb" + "testing" +) + +var ( + // ignoreDbTypes are types which should be ignored when running tests + // that iterate all supported DB types. This allows some tests to add + // bogus drivers for testing purposes while still allowing other tests + // to easily iterate all supported drivers. + ignoreDbTypes = map[string]bool{"createopenfail": true} +) + +// testNewestShaEmpty ensures the NewestSha returns the values expected by +// the interface contract. +func testNewestShaEmpty(t *testing.T, db btcdb.Db) { + sha, height, err := db.NewestSha() + if err != nil { + t.Errorf("NewestSha error %v", err) + } + if !sha.IsEqual(&zeroHash) { + t.Errorf("NewestSha wrong hash got: %s, want %s", sha, &zeroHash) + + } + if height != -1 { + t.Errorf("NewestSha wrong height got: %s, want %s", height, -1) + } +} + +// TestEmptyDB tests that empty databases are handled properly. +func TestEmptyDB(t *testing.T) { + for _, dbType := range btcdb.SupportedDBs() { + // Ensure NewestSha returns expected values for a newly created + // db. + db, teardown, err := createDB(dbType, "emptydb", false) + if err != nil { + t.Errorf("Failed to create test database %v", err) + return + } + testNewestShaEmpty(t, db) + + // Ensure NewestSha still returns expected values for an empty + // database after reopen. + db.Close() + db, err = openDB(dbType, "emptydb") + if err != nil { + t.Errorf("Failed to open test database %v", err) + return + } + testNewestShaEmpty(t, db) + db.Close() + + // Clean up the old db. + teardown() + } +} + +// TestAddDuplicateDriver ensures that adding a duplicate driver does not +// overwrite an existing one. +func TestAddDuplicateDriver(t *testing.T) { + supportedDBs := btcdb.SupportedDBs() + if len(supportedDBs) == 0 { + t.Errorf("TestAddDuplicateDriver: No backends to test") + return + } + dbType := supportedDBs[0] + + // bogusCreateDB is a function which acts as a bogus create and open + // driver function and intentionally returns a failure that can be + // detected if the interface allows a duplicate driver to overwrite an + // existing one. + bogusCreateDB := func(string) (btcdb.Db, error) { + return nil, fmt.Errorf("duplicate driver allowed for database "+ + "type [%v]", dbType) + } + + // Create a driver that tries to replace an existing one. Set its + // create and open functions to a function that causes a test failure if + // they are invoked. + driver := btcdb.DriverDB{ + DbType: dbType, + Create: bogusCreateDB, + Open: bogusCreateDB, + } + btcdb.AddDBDriver(driver) + + // Ensure creating a database of the type that we tried to replace + // doesn't fail (if it does, it indicates the driver was erroneously + // replaced). + _, teardown, err := createDB(dbType, "dupdrivertest", true) + if err != nil { + t.Errorf("TestAddDuplicateDriver: %v", err) + return + } + teardown() +} + +// TestCreateOpenFail ensures that errors which occur while opening or closing +// a database are handled properly. +func TestCreateOpenFail(t *testing.T) { + // bogusCreateDB is a function which acts as a bogus create and open + // driver function that intentionally returns a failure which can be + // detected. + dbType := "createopenfail" + openError := fmt.Errorf("failed to create or open database for "+ + "database type [%v]", dbType) + bogusCreateDB := func(string) (btcdb.Db, error) { + return nil, openError + } + + // Create and add driver that intentionally fails when created or opened + // to ensure errors on database open and create are handled properly. + driver := btcdb.DriverDB{ + DbType: dbType, + Create: bogusCreateDB, + Open: bogusCreateDB, + } + btcdb.AddDBDriver(driver) + + // Ensure creating a database with the new type fails with the expected + // error. + _, err := btcdb.CreateDB(dbType, "createfailtest") + if err != openError { + t.Errorf("TestCreateOpenFail: expected error not received - "+ + "got: %v, want %v", err, openError) + return + } + + // Ensure opening a database with the new type fails with the expected + // error. + _, err = btcdb.OpenDB(dbType, "openfailtest") + if err != openError { + t.Errorf("TestCreateOpenFail: expected error not received - "+ + "got: %v, want %v", err, openError) + return + } +} + +// TestCreateOpenUnsupported ensures that attempting to create or open an +// unsupported database type is handled properly. +func TestCreateOpenUnsupported(t *testing.T) { + // Ensure creating a database with an unsupported type fails with the + // expected error. + dbType := "unsupported" + _, err := btcdb.CreateDB(dbType, "unsupportedcreatetest") + if err != btcdb.DbUnknownType { + t.Errorf("TestCreateOpenUnsupported: expected error not "+ + "received - got: %v, want %v", err, btcdb.DbUnknownType) + return + } + + // Ensure opening a database with the new type fails with the expected + // error. + _, err = btcdb.OpenDB(dbType, "unsupportedopentest") + if err != btcdb.DbUnknownType { + t.Errorf("TestCreateOpenUnsupported: expected error not "+ + "received - got: %v, want %v", err, btcdb.DbUnknownType) + return + } +} + +// TestInterface performs tests for the various interfaces of btcdb which +// require state in the database for each supported database type (those loaded +// in common_test.go that is). +func TestInterface(t *testing.T) { + for _, dbType := range btcdb.SupportedDBs() { + if _, exists := ignoreDbTypes[dbType]; !exists { + testInterface(t, dbType) + } + } +} diff --git a/interface_test.go b/interface_test.go index f223375d..9c84fd90 100644 --- a/interface_test.go +++ b/interface_test.go @@ -5,167 +5,9 @@ package btcdb_test import ( - "fmt" - "github.com/conformal/btcdb" "testing" ) -var ( - // ignoreDbTypes are types which should be ignored when running tests - // that iterate all supported DB types. This allows some tests to add - // bogus drivers for testing purposes while still allowing other tests - // to easily iterate all supported drivers. - ignoreDbTypes = map[string]bool{"createopenfail": true} -) - -// testNewestShaEmpty ensures the NewestSha returns the values expected by -// the interface contract. -func testNewestShaEmpty(t *testing.T, db btcdb.Db) { - sha, height, err := db.NewestSha() - if err != nil { - t.Errorf("NewestSha error %v", err) - } - if !sha.IsEqual(&zeroHash) { - t.Errorf("NewestSha wrong hash got: %s, want %s", sha, &zeroHash) - - } - if height != -1 { - t.Errorf("NewestSha wrong height got: %s, want %s", height, -1) - } -} - -// TestEmptyDB tests that empty databases are handled properly. -func TestEmptyDB(t *testing.T) { - for _, dbType := range btcdb.SupportedDBs() { - // Ensure NewestSha returns expected values for a newly created - // db. - db, teardown, err := createDB(dbType, "emptydb", false) - if err != nil { - t.Errorf("Failed to create test database %v", err) - return - } - testNewestShaEmpty(t, db) - - // Ensure NewestSha still returns expected values for an empty - // database after reopen. - db.Close() - db, err = openDB(dbType, "emptydb") - if err != nil { - t.Errorf("Failed to open test database %v", err) - return - } - testNewestShaEmpty(t, db) - db.Close() - - // Clean up the old db. - teardown() - } -} - -// TestAddDuplicateDriver ensures that adding a duplicate driver does not -// overwrite an existing one. -func TestAddDuplicateDriver(t *testing.T) { - supportedDBs := btcdb.SupportedDBs() - if len(supportedDBs) == 0 { - t.Errorf("TestAddDuplicateDriver: No backends to test") - return - } - dbType := supportedDBs[0] - - // bogusCreateDB is a function which acts as a bogus create and open - // driver function and intentionally returns a failure that can be - // detected if the interface allows a duplicate driver to overwrite an - // existing one. - bogusCreateDB := func(string) (btcdb.Db, error) { - return nil, fmt.Errorf("duplicate driver allowed for database "+ - "type [%v]", dbType) - } - - // Create a driver that tries to replace an existing one. Set its - // create and open functions to a function that causes a test failure if - // they are invoked. - driver := btcdb.DriverDB{ - DbType: dbType, - Create: bogusCreateDB, - Open: bogusCreateDB, - } - btcdb.AddDBDriver(driver) - - // Ensure creating a database of the type that we tried to replace - // doesn't fail (if it does, it indicates the driver was erroneously - // replaced). - _, teardown, err := createDB(dbType, "dupdrivertest", true) - if err != nil { - t.Errorf("TestAddDuplicateDriver: %v", err) - return - } - teardown() -} - -// TestCreateOpenFail ensures that errors which occur while opening or closing -// a database are handled properly. -func TestCreateOpenFail(t *testing.T) { - // bogusCreateDB is a function which acts as a bogus create and open - // driver function that intentionally returns a failure which can be - // detected. - dbType := "createopenfail" - openError := fmt.Errorf("failed to create or open database for "+ - "database type [%v]", dbType) - bogusCreateDB := func(string) (btcdb.Db, error) { - return nil, openError - } - - // Create and add driver that intentionally fails when created or opened - // to ensure errors on database open and create are handled properly. - driver := btcdb.DriverDB{ - DbType: dbType, - Create: bogusCreateDB, - Open: bogusCreateDB, - } - btcdb.AddDBDriver(driver) - - // Ensure creating a database with the new type fails with the expected - // error. - _, err := btcdb.CreateDB(dbType, "createfailtest") - if err != openError { - t.Errorf("TestCreateOpenFail: expected error not received - "+ - "got: %v, want %v", err, openError) - return - } - - // Ensure opening a database with the new type fails with the expected - // error. - _, err = btcdb.OpenDB(dbType, "openfailtest") - if err != openError { - t.Errorf("TestCreateOpenFail: expected error not received - "+ - "got: %v, want %v", err, openError) - return - } -} - -// TestCreateOpenUnsupported ensures that attempting to create or open an -// unsupported database type is handled properly. -func TestCreateOpenUnsupported(t *testing.T) { - // Ensure creating a database with an unsupported type fails with the - // expected error. - dbType := "unsupported" - _, err := btcdb.CreateDB(dbType, "unsupportedcreatetest") - if err != btcdb.DbUnknownType { - t.Errorf("TestCreateOpenUnsupported: expected error not "+ - "received - got: %v, want %v", err, btcdb.DbUnknownType) - return - } - - // Ensure opening a database with the new type fails with the expected - // error. - _, err = btcdb.OpenDB(dbType, "unsupportedopentest") - if err != btcdb.DbUnknownType { - t.Errorf("TestCreateOpenUnsupported: expected error not "+ - "received - got: %v, want %v", err, btcdb.DbUnknownType) - return - } -} - // testInterface tests performs tests for the various interfaces of btcdb which // require state in the database for the given database type. func testInterface(t *testing.T, dbType string) { @@ -230,14 +72,3 @@ func testInterface(t *testing.T, dbType string) { Sync() */ } - -// TestInterface performs tests for the various interfaces of btcdb which -// require state in the database for each supported database type (those loaded -// in common_test.go that is). -func TestInterface(t *testing.T) { - for _, dbType := range btcdb.SupportedDBs() { - if _, exists := ignoreDbTypes[dbType]; !exists { - testInterface(t, dbType) - } - } -} From 396d1b056ddf3c55a21afa09920732eab924ba83 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sun, 13 Oct 2013 21:30:19 -0500 Subject: [PATCH 078/163] Add interface test for FetchBlockBySha. This commit adds an interface test for FetchBlockBySha to ensure it returns the same MsgBlock and raw bytes that were inserted. --- interface_test.go | 50 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/interface_test.go b/interface_test.go index 9c84fd90..8ffa33d7 100644 --- a/interface_test.go +++ b/interface_test.go @@ -5,6 +5,8 @@ package btcdb_test import ( + "github.com/davecgh/go-spew/spew" + "reflect" "testing" ) @@ -43,6 +45,50 @@ func testInterface(t *testing.T, dbType string) { newHeight, height) return } + + // Ensure the block now exists in the database. + expectedHash, err := block.Sha() + if err != nil { + t.Errorf("block.Sha: %v", err) + return + } + if exists := db.ExistsSha(expectedHash); !exists { + t.Errorf("ExistsSha: block %v does not exist", + expectedHash) + return + } + + // Ensure loading the block back from the database gives back + // the same MsgBlock and raw bytes. + blockFromDb, err := db.FetchBlockBySha(expectedHash) + if err != nil { + t.Errorf("FetchBlockBySha: %v", err) + } + if !reflect.DeepEqual(block.MsgBlock(), blockFromDb.MsgBlock()) { + t.Errorf("Block from database does not match stored "+ + "block\ngot: %v\nwant: %v", + spew.Sdump(blockFromDb.MsgBlock()), + spew.Sdump(block.MsgBlock())) + return + } + blockBytes, err := block.Bytes() + if err != nil { + t.Errorf("block.Bytes: %v", err) + return + } + blockFromDbBytes, err := blockFromDb.Bytes() + if err != nil { + t.Errorf("blockFromDb.Bytes: %v", err) + return + } + if !reflect.DeepEqual(blockBytes, blockFromDbBytes) { + t.Errorf("Block bytes from database do not match "+ + "stored block bytes\ngot: %v\nwant: %v", + spew.Sdump(blockFromDbBytes), + spew.Sdump(blockBytes)) + return + } + } // TODO(davec): Need to figure out how to handle the special checks @@ -53,8 +99,8 @@ func testInterface(t *testing.T, dbType string) { /* Close() DropAfterBlockBySha(*btcwire.ShaHash) (err error) - ExistsSha(sha *btcwire.ShaHash) (exists bool) - FetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) + - ExistsSha(sha *btcwire.ShaHash) (exists bool) + - FetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) FetchBlockShaByHeight(height int64) (sha *btcwire.ShaHash, err error) FetchHeightRange(startHeight, endHeight int64) (rshalist []btcwire.ShaHash, err error) ExistsTxSha(sha *btcwire.ShaHash) (exists bool) From 670d83a74e66feb6118bc3ffd3c76695a535eaa7 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sun, 13 Oct 2013 21:37:17 -0500 Subject: [PATCH 079/163] Improve error message for FetchBlockBySha test.wq --- interface_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/interface_test.go b/interface_test.go index 8ffa33d7..0cdf0dec 100644 --- a/interface_test.go +++ b/interface_test.go @@ -65,8 +65,8 @@ func testInterface(t *testing.T, dbType string) { t.Errorf("FetchBlockBySha: %v", err) } if !reflect.DeepEqual(block.MsgBlock(), blockFromDb.MsgBlock()) { - t.Errorf("Block from database does not match stored "+ - "block\ngot: %v\nwant: %v", + t.Errorf("FetchBlockBySha: block from database does "+ + "not match stored block\ngot: %v\nwant: %v", spew.Sdump(blockFromDb.MsgBlock()), spew.Sdump(block.MsgBlock())) return @@ -82,9 +82,9 @@ func testInterface(t *testing.T, dbType string) { return } if !reflect.DeepEqual(blockBytes, blockFromDbBytes) { - t.Errorf("Block bytes from database do not match "+ - "stored block bytes\ngot: %v\nwant: %v", - spew.Sdump(blockFromDbBytes), + t.Errorf("FetchBlockBySha: block bytes from database "+ + "do not match stored block bytes\ngot: %v\n"+ + "want: %v", spew.Sdump(blockFromDbBytes), spew.Sdump(blockBytes)) return } From 6eebf0218347a52d042151595beddc6bf126cbd7 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sun, 13 Oct 2013 22:03:50 -0500 Subject: [PATCH 080/163] Add interface test for FetchBlockShaByHeight. This commit adds an interface test for FetchBlockShaByHeight to ensure it returns the same expected hash for each block inserted block height. --- interface_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/interface_test.go b/interface_test.go index 0cdf0dec..4d99fbb6 100644 --- a/interface_test.go +++ b/interface_test.go @@ -89,6 +89,19 @@ func testInterface(t *testing.T, dbType string) { return } + // Ensure the hash returned for the block by its height is the + // expected value. + hashFromDb, err := db.FetchBlockShaByHeight(height) + if err != nil { + t.Errorf("FetchBlockShaByHeight: %v", err) + return + } + if !hashFromDb.IsEqual(expectedHash) { + t.Errorf("FetchBlockShaByHeight: returned hash does "+ + "not match expected value - got: %v, want: %v", + hashFromDb, expectedHash) + return + } } // TODO(davec): Need to figure out how to handle the special checks From 06d6e5fce8fb28659964ab7d97d4fb2a8433a04b Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sun, 13 Oct 2013 22:11:12 -0500 Subject: [PATCH 081/163] Add db type to test error outputs. --- interface_test.go | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/interface_test.go b/interface_test.go index 4d99fbb6..510bacda 100644 --- a/interface_test.go +++ b/interface_test.go @@ -15,7 +15,7 @@ import ( func testInterface(t *testing.T, dbType string) { db, teardown, err := setupDB(dbType, "interface") if err != nil { - t.Errorf("Failed to create test database %v", err) + t.Errorf("Failed to create test database (%s) %v", dbType, err) return } defer teardown() @@ -36,13 +36,13 @@ func testInterface(t *testing.T, dbType string) { // database. newHeight, err := db.InsertBlock(block) if err != nil { - t.Errorf("InsertBlock: failed to insert block %v err %v", - height, err) + t.Errorf("InsertBlock (%s): failed to insert block %v "+ + "err %v", dbType, height, err) return } if newHeight != height { - t.Errorf("InsertBlock: height mismatch got: %v, want: %v", - newHeight, height) + t.Errorf("InsertBlock (%s): height mismatch got: %v, "+ + "want: %v", dbType, newHeight, height) return } @@ -53,8 +53,8 @@ func testInterface(t *testing.T, dbType string) { return } if exists := db.ExistsSha(expectedHash); !exists { - t.Errorf("ExistsSha: block %v does not exist", - expectedHash) + t.Errorf("ExistsSha (%s): block %v does not exist", + dbType, expectedHash) return } @@ -62,11 +62,13 @@ func testInterface(t *testing.T, dbType string) { // the same MsgBlock and raw bytes. blockFromDb, err := db.FetchBlockBySha(expectedHash) if err != nil { - t.Errorf("FetchBlockBySha: %v", err) + t.Errorf("FetchBlockBySha (%s): %v", dbType, err) + return } if !reflect.DeepEqual(block.MsgBlock(), blockFromDb.MsgBlock()) { - t.Errorf("FetchBlockBySha: block from database does "+ - "not match stored block\ngot: %v\nwant: %v", + t.Errorf("FetchBlockBySha (%s): block from database "+ + "does not match stored block\ngot: %v\n"+ + "want: %v", dbType, spew.Sdump(blockFromDb.MsgBlock()), spew.Sdump(block.MsgBlock())) return @@ -82,9 +84,10 @@ func testInterface(t *testing.T, dbType string) { return } if !reflect.DeepEqual(blockBytes, blockFromDbBytes) { - t.Errorf("FetchBlockBySha: block bytes from database "+ - "do not match stored block bytes\ngot: %v\n"+ - "want: %v", spew.Sdump(blockFromDbBytes), + t.Errorf("FetchBlockBySha (%s): block bytes from "+ + "database do not match stored block bytes\n"+ + "got: %v\nwant: %v", dbType, + spew.Sdump(blockFromDbBytes), spew.Sdump(blockBytes)) return } @@ -93,13 +96,13 @@ func testInterface(t *testing.T, dbType string) { // expected value. hashFromDb, err := db.FetchBlockShaByHeight(height) if err != nil { - t.Errorf("FetchBlockShaByHeight: %v", err) + t.Errorf("FetchBlockShaByHeight (%s): %v", dbType, err) return } if !hashFromDb.IsEqual(expectedHash) { - t.Errorf("FetchBlockShaByHeight: returned hash does "+ - "not match expected value - got: %v, want: %v", - hashFromDb, expectedHash) + t.Errorf("FetchBlockShaByHeight (%s): returned hash "+ + "does not match expected value - got: %v, "+ + "want: %v", dbType, hashFromDb, expectedHash) return } } From f37fabb855c1a0aeaf5e3a5054a8abdc950468c3 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sun, 13 Oct 2013 22:13:47 -0500 Subject: [PATCH 082/163] Add negative tests for FetchBlockShaByHeight. This commit adds tests to ensure FetchBlockShaByHeight returns expected errors when invalid heights are specified. --- interface_test.go | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/interface_test.go b/interface_test.go index 510bacda..8ae40d51 100644 --- a/interface_test.go +++ b/interface_test.go @@ -5,11 +5,35 @@ package btcdb_test import ( + "github.com/conformal/btcdb" "github.com/davecgh/go-spew/spew" "reflect" "testing" ) +// testFetchBlockShaByHeightErrors ensures FetchBlockShaByHeight handles invalid +// heights correctly. +func testFetchBlockShaByHeightErrors(t *testing.T, dbType string, db btcdb.Db, numBlocks int64) bool { + tests := []int64{-1, numBlocks, numBlocks + 1} + for i, wantHeight := range tests { + hashFromDb, err := db.FetchBlockShaByHeight(wantHeight) + if err == nil { + t.Errorf("FetchBlockShaByHeight #%d (%s): did not "+ + "return error on invalid index: %d - got: %v, "+ + "want: non-nil", i, dbType, wantHeight, err) + return false + } + if hashFromDb != nil { + t.Errorf("FetchBlockShaByHeight #%d (%s): returned "+ + "hash is not nil on invalid index: %d - got: "+ + "%v, want: nil", i, dbType, wantHeight, err) + return false + } + } + + return false +} + // testInterface tests performs tests for the various interfaces of btcdb which // require state in the database for the given database type. func testInterface(t *testing.T, dbType string) { @@ -107,6 +131,11 @@ func testInterface(t *testing.T, dbType string) { } } + // Ensure FetchBlockShaByHeight handles invalid heights properly. + if !testFetchBlockShaByHeightErrors(t, dbType, db, int64(len(blocks))) { + return + } + // TODO(davec): Need to figure out how to handle the special checks // required for the duplicate transactions allowed by blocks 91842 and // 91880 on the main network due to the old miner + Satoshi client bug. @@ -117,7 +146,7 @@ func testInterface(t *testing.T, dbType string) { DropAfterBlockBySha(*btcwire.ShaHash) (err error) - ExistsSha(sha *btcwire.ShaHash) (exists bool) - FetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) - FetchBlockShaByHeight(height int64) (sha *btcwire.ShaHash, err error) + - FetchBlockShaByHeight(height int64) (sha *btcwire.ShaHash, err error) FetchHeightRange(startHeight, endHeight int64) (rshalist []btcwire.ShaHash, err error) ExistsTxSha(sha *btcwire.ShaHash) (exists bool) FetchTxBySha(txsha *btcwire.ShaHash) ([]*TxListReply, error) From 6a3824b8aae0a4704bcda25c66491038e0c416d7 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Mon, 14 Oct 2013 00:27:11 -0500 Subject: [PATCH 083/163] Add interface test for ExistsTxSha. This commit adds an interface test for ExistsTxSha to ensure all transactions inserted as a part of a block are available. --- interface_test.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/interface_test.go b/interface_test.go index 8ae40d51..355abf95 100644 --- a/interface_test.go +++ b/interface_test.go @@ -129,6 +129,23 @@ func testInterface(t *testing.T, dbType string) { "want: %v", dbType, hashFromDb, expectedHash) return } + + // The following set of tests examine all of the transactions in + // the block. + txHashes, err := block.TxShas() + if err != nil { + t.Errorf("block.TxShas: %v", err) + return + } + for i, tx := range block.MsgBlock().Transactions { + // Ensure the transaction exists. + txHash := txHashes[i] + if exists := db.ExistsTxSha(txHash); !exists { + t.Errorf("ExistsTxSha (%s): tx %v does not exist", + dbType, txHash) + return + } + } } // Ensure FetchBlockShaByHeight handles invalid heights properly. @@ -148,7 +165,7 @@ func testInterface(t *testing.T, dbType string) { - FetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) - FetchBlockShaByHeight(height int64) (sha *btcwire.ShaHash, err error) FetchHeightRange(startHeight, endHeight int64) (rshalist []btcwire.ShaHash, err error) - ExistsTxSha(sha *btcwire.ShaHash) (exists bool) + - ExistsTxSha(sha *btcwire.ShaHash) (exists bool) FetchTxBySha(txsha *btcwire.ShaHash) ([]*TxListReply, error) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply From 06cb2ec817f8dfd375e228386ba17ec224c3f5a7 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Mon, 14 Oct 2013 00:42:58 -0500 Subject: [PATCH 084/163] Add interface test for FetchTxBySha. This commit adds an interface test for FetchTxBySha to ensure it returns the same MsgTx that was part of an inserted block. --- interface_test.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/interface_test.go b/interface_test.go index 355abf95..6a6d78ac 100644 --- a/interface_test.go +++ b/interface_test.go @@ -145,6 +145,27 @@ func testInterface(t *testing.T, dbType string) { dbType, txHash) return } + + // Ensure loading the transaction back from the database + // gives back the same MsgTx. + txReplyList, err := db.FetchTxBySha(txHash) + if err != nil { + t.Errorf("FetchTxBySha (%s): %v", dbType, err) + return + } + if len(txReplyList) == 0 { + t.Errorf("FetchTxBySha (%s): tx %v did not "+ + "return reply data", dbType, txHash) + return + } + txFromDb := txReplyList[len(txReplyList)-1].Tx + if !reflect.DeepEqual(tx, txFromDb) { + t.Errorf("FetchTxBySha (%s): tx %v from "+ + "database does not match stored tx\n"+ + "got: %v\nwant: %v", dbType, txHash, + spew.Sdump(txFromDb), spew.Sdump(tx)) + return + } } } @@ -166,7 +187,7 @@ func testInterface(t *testing.T, dbType string) { - FetchBlockShaByHeight(height int64) (sha *btcwire.ShaHash, err error) FetchHeightRange(startHeight, endHeight int64) (rshalist []btcwire.ShaHash, err error) - ExistsTxSha(sha *btcwire.ShaHash) (exists bool) - FetchTxBySha(txsha *btcwire.ShaHash) ([]*TxListReply, error) + - FetchTxBySha(txsha *btcwire.ShaHash) ([]*TxListReply, error) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply - InsertBlock(block *btcutil.Block) (height int64, err error) From fb5f9c0b5ac2b0aad36d4b7063864a202e43723a Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Mon, 14 Oct 2013 10:21:17 -0500 Subject: [PATCH 085/163] Separate interface tests and add test context. Rather than having one mammoth interface test function, create a test context and several helper functions that accept the context. --- interface_test.go | 304 +++++++++++++++++++++++++++++----------------- 1 file changed, 195 insertions(+), 109 deletions(-) diff --git a/interface_test.go b/interface_test.go index 6a6d78ac..24a8c7ad 100644 --- a/interface_test.go +++ b/interface_test.go @@ -6,32 +6,187 @@ package btcdb_test import ( "github.com/conformal/btcdb" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" "github.com/davecgh/go-spew/spew" "reflect" "testing" ) -// testFetchBlockShaByHeightErrors ensures FetchBlockShaByHeight handles invalid -// heights correctly. -func testFetchBlockShaByHeightErrors(t *testing.T, dbType string, db btcdb.Db, numBlocks int64) bool { - tests := []int64{-1, numBlocks, numBlocks + 1} +// testContext is used to store context information about a running test which +// is passed into helper functions. +type testContext struct { + t *testing.T + dbType string + db btcdb.Db + blockHeight int64 + blockHash *btcwire.ShaHash + block *btcutil.Block +} + +// testInsertBlock ensures InsertBlock conforms to the interface contract. +func testInsertBlock(tc *testContext) bool { + // The block must insert without any errors. + newHeight, err := tc.db.InsertBlock(tc.block) + if err != nil { + tc.t.Errorf("InsertBlock (%s): failed to insert block %v "+ + "err %v", tc.dbType, tc.blockHeight, err) + return false + } + + // The returned height must be the expected value. + if newHeight != tc.blockHeight { + tc.t.Errorf("InsertBlock (%s): height mismatch got: %v, "+ + "want: %v", tc.dbType, newHeight, tc.blockHeight) + return false + } + + return true +} + +// testExistsSha ensures ExistsSha conforms to the interface contract. +func testExistsSha(tc *testContext) bool { + // The block must exist in the database. + if exists := tc.db.ExistsSha(tc.blockHash); !exists { + tc.t.Errorf("ExistsSha (%s): block %v does not exist", + tc.dbType, tc.blockHash) + return false + } + + return true +} + +// testFetchBlockBySha ensures FetchBlockBySha conforms to the interface +// contract. +func testFetchBlockBySha(tc *testContext) bool { + // The block must be fetchable by its hash without any errors. + blockFromDb, err := tc.db.FetchBlockBySha(tc.blockHash) + if err != nil { + tc.t.Errorf("FetchBlockBySha (%s): %v", tc.dbType, err) + return false + } + + // The block fetched from the database must give back the same MsgBlock + // and raw bytes that were stored. + if !reflect.DeepEqual(tc.block.MsgBlock(), blockFromDb.MsgBlock()) { + tc.t.Errorf("FetchBlockBySha (%s): block from database "+ + "does not match stored block\ngot: %v\n"+ + "want: %v", tc.dbType, + spew.Sdump(blockFromDb.MsgBlock()), + spew.Sdump(tc.block.MsgBlock())) + return false + } + blockBytes, err := tc.block.Bytes() + if err != nil { + tc.t.Errorf("block.Bytes: %v", err) + return false + } + blockFromDbBytes, err := blockFromDb.Bytes() + if err != nil { + tc.t.Errorf("blockFromDb.Bytes: %v", err) + return false + } + if !reflect.DeepEqual(blockBytes, blockFromDbBytes) { + tc.t.Errorf("FetchBlockBySha (%s): block bytes from "+ + "database do not match stored block bytes\n"+ + "got: %v\nwant: %v", tc.dbType, + spew.Sdump(blockFromDbBytes), spew.Sdump(blockBytes)) + return false + } + + return true +} + +// testFetchBlockShaByHeight ensures FetchBlockShaByHeight conforms to the +// interface contract. +func testFetchBlockShaByHeight(tc *testContext) bool { + // The hash returned for the block by its height must be the expected + // value. + hashFromDb, err := tc.db.FetchBlockShaByHeight(tc.blockHeight) + if err != nil { + tc.t.Errorf("FetchBlockShaByHeight (%s): %v", tc.dbType, err) + return false + } + if !hashFromDb.IsEqual(tc.blockHash) { + tc.t.Errorf("FetchBlockShaByHeight (%s): returned hash "+ + "does not match expected value - got: %v, "+ + "want: %v", tc.dbType, hashFromDb, tc.blockHash) + return false + } + + // Invalid heights must error and return a nil hash. + tests := []int64{-1, tc.blockHeight + 1, tc.blockHeight + 2} for i, wantHeight := range tests { - hashFromDb, err := db.FetchBlockShaByHeight(wantHeight) + hashFromDb, err := tc.db.FetchBlockShaByHeight(wantHeight) if err == nil { - t.Errorf("FetchBlockShaByHeight #%d (%s): did not "+ + tc.t.Errorf("FetchBlockShaByHeight #%d (%s): did not "+ "return error on invalid index: %d - got: %v, "+ - "want: non-nil", i, dbType, wantHeight, err) + "want: non-nil", i, tc.dbType, wantHeight, err) return false } if hashFromDb != nil { - t.Errorf("FetchBlockShaByHeight #%d (%s): returned "+ + tc.t.Errorf("FetchBlockShaByHeight #%d (%s): returned "+ "hash is not nil on invalid index: %d - got: "+ - "%v, want: nil", i, dbType, wantHeight, err) + "%v, want: nil", i, tc.dbType, wantHeight, err) return false } } - return false + return true +} + +// testExistsTxSha ensures ExistsTxSha conforms to the interface contract. +func testExistsTxSha(tc *testContext) bool { + txHashes, err := tc.block.TxShas() + if err != nil { + tc.t.Errorf("block.TxShas: %v", err) + return false + } + + for i := range txHashes { + // The transaction must exist in the database. + txHash := txHashes[i] + if exists := tc.db.ExistsTxSha(txHash); !exists { + tc.t.Errorf("ExistsTxSha (%s): tx %v does not exist", + tc.dbType, txHash) + return false + } + } + + return true +} + +// testFetchTxBySha ensures FetchTxBySha conforms to the interface contract. +func testFetchTxBySha(tc *testContext) bool { + txHashes, err := tc.block.TxShas() + if err != nil { + tc.t.Errorf("block.TxShas: %v", err) + return false + } + + for i, tx := range tc.block.MsgBlock().Transactions { + txHash := txHashes[i] + txReplyList, err := tc.db.FetchTxBySha(txHash) + if err != nil { + tc.t.Errorf("FetchTxBySha (%s): %v", tc.dbType, err) + return false + } + if len(txReplyList) == 0 { + tc.t.Errorf("FetchTxBySha (%s): tx %v did not "+ + "return reply data", tc.dbType, txHash) + return false + } + txFromDb := txReplyList[len(txReplyList)-1].Tx + if !reflect.DeepEqual(tx, txFromDb) { + tc.t.Errorf("FetchTxBySha (%s): tx %v from "+ + "database does not match stored tx\n"+ + "got: %v\nwant: %v", tc.dbType, txHash, + spew.Sdump(txFromDb), spew.Sdump(tx)) + return false + } + } + + return true } // testInterface tests performs tests for the various interfaces of btcdb which @@ -52,128 +207,59 @@ func testInterface(t *testing.T, dbType string) { return } + // Create a test context to pass around. + context := testContext{t: t, dbType: dbType, db: db} + t.Logf("Loaded %d blocks", len(blocks)) for height := int64(1); height < int64(len(blocks)); height++ { + // Get the appropriate block and hash and update the test + // context accordingly. block := blocks[height] - - // Ensure there are no errors inserting each block into the - // database. - newHeight, err := db.InsertBlock(block) - if err != nil { - t.Errorf("InsertBlock (%s): failed to insert block %v "+ - "err %v", dbType, height, err) - return - } - if newHeight != height { - t.Errorf("InsertBlock (%s): height mismatch got: %v, "+ - "want: %v", dbType, newHeight, height) - return - } - - // Ensure the block now exists in the database. - expectedHash, err := block.Sha() + blockHash, err := block.Sha() if err != nil { t.Errorf("block.Sha: %v", err) return } - if exists := db.ExistsSha(expectedHash); !exists { - t.Errorf("ExistsSha (%s): block %v does not exist", - dbType, expectedHash) + context.blockHeight = height + context.blockHash = blockHash + context.block = block + + // The block must insert without any errors and return the + // expected height. + if !testInsertBlock(&context) { return } - // Ensure loading the block back from the database gives back - // the same MsgBlock and raw bytes. - blockFromDb, err := db.FetchBlockBySha(expectedHash) - if err != nil { - t.Errorf("FetchBlockBySha (%s): %v", dbType, err) - return - } - if !reflect.DeepEqual(block.MsgBlock(), blockFromDb.MsgBlock()) { - t.Errorf("FetchBlockBySha (%s): block from database "+ - "does not match stored block\ngot: %v\n"+ - "want: %v", dbType, - spew.Sdump(blockFromDb.MsgBlock()), - spew.Sdump(block.MsgBlock())) - return - } - blockBytes, err := block.Bytes() - if err != nil { - t.Errorf("block.Bytes: %v", err) - return - } - blockFromDbBytes, err := blockFromDb.Bytes() - if err != nil { - t.Errorf("blockFromDb.Bytes: %v", err) - return - } - if !reflect.DeepEqual(blockBytes, blockFromDbBytes) { - t.Errorf("FetchBlockBySha (%s): block bytes from "+ - "database do not match stored block bytes\n"+ - "got: %v\nwant: %v", dbType, - spew.Sdump(blockFromDbBytes), - spew.Sdump(blockBytes)) + // The block must now exist in the database. + if !testExistsSha(&context) { return } - // Ensure the hash returned for the block by its height is the + // Loading the block back from the database must give back + // the same MsgBlock and raw bytes that were stored. + if !testFetchBlockBySha(&context) { + return + } + + // The hash returned for the block by its height must be the // expected value. - hashFromDb, err := db.FetchBlockShaByHeight(height) - if err != nil { - t.Errorf("FetchBlockShaByHeight (%s): %v", dbType, err) - return - } - if !hashFromDb.IsEqual(expectedHash) { - t.Errorf("FetchBlockShaByHeight (%s): returned hash "+ - "does not match expected value - got: %v, "+ - "want: %v", dbType, hashFromDb, expectedHash) + if !testFetchBlockShaByHeight(&context) { return } - // The following set of tests examine all of the transactions in - // the block. - txHashes, err := block.TxShas() - if err != nil { - t.Errorf("block.TxShas: %v", err) + // All of the transactions in the block must now exist in the + // database. + if !testExistsTxSha(&context) { return } - for i, tx := range block.MsgBlock().Transactions { - // Ensure the transaction exists. - txHash := txHashes[i] - if exists := db.ExistsTxSha(txHash); !exists { - t.Errorf("ExistsTxSha (%s): tx %v does not exist", - dbType, txHash) - return - } - // Ensure loading the transaction back from the database - // gives back the same MsgTx. - txReplyList, err := db.FetchTxBySha(txHash) - if err != nil { - t.Errorf("FetchTxBySha (%s): %v", dbType, err) - return - } - if len(txReplyList) == 0 { - t.Errorf("FetchTxBySha (%s): tx %v did not "+ - "return reply data", dbType, txHash) - return - } - txFromDb := txReplyList[len(txReplyList)-1].Tx - if !reflect.DeepEqual(tx, txFromDb) { - t.Errorf("FetchTxBySha (%s): tx %v from "+ - "database does not match stored tx\n"+ - "got: %v\nwant: %v", dbType, txHash, - spew.Sdump(txFromDb), spew.Sdump(tx)) - return - } + // Loading all of the transactions in the block back from the + // database must give back the same MsgTx that was stored. + if !testFetchTxBySha(&context) { + return } } - // Ensure FetchBlockShaByHeight handles invalid heights properly. - if !testFetchBlockShaByHeightErrors(t, dbType, db, int64(len(blocks))) { - return - } - // TODO(davec): Need to figure out how to handle the special checks // required for the duplicate transactions allowed by blocks 91842 and // 91880 on the main network due to the old miner + Satoshi client bug. From 2ab3e0a382be727ce7fa430104a07f86c22c5ff6 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Mon, 14 Oct 2013 13:49:10 -0500 Subject: [PATCH 086/163] Add interface tests for FetchTxByShaList. --- interface_test.go | 102 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 101 insertions(+), 1 deletion(-) diff --git a/interface_test.go b/interface_test.go index 24a8c7ad..3905c3d3 100644 --- a/interface_test.go +++ b/interface_test.go @@ -189,6 +189,99 @@ func testFetchTxBySha(tc *testContext) bool { return true } +// testFetchTxByShaList ensures FetchTxByShaList conforms to the interface +// contract. +func testFetchTxByShaList(tc *testContext) bool { + txHashes, err := tc.block.TxShas() + if err != nil { + tc.t.Errorf("block.TxShas: %v", err) + return false + } + + txReplyList := tc.db.FetchTxByShaList(txHashes) + if len(txReplyList) != len(txHashes) { + tc.t.Errorf("FetchTxByShaList (%s): tx reply list for "+ + "block #%d (%v) does not match expected length "+ + "- got: %v, want: %v", tc.dbType, tc.blockHeight, + tc.blockHash, len(txReplyList), len(txHashes)) + return false + } + for i, tx := range tc.block.MsgBlock().Transactions { + txHash := txHashes[i] + txD := txReplyList[i] + + // The transaction hash in the reply must be the expected value. + if !txD.Sha.IsEqual(txHash) { + tc.t.Errorf("FetchTxByShaList (%s): tx #%d hash "+ + "does not match expected - got %v, "+ + "want %v", tc.dbType, i, txD.Sha, txHashes[i]) + return false + } + + // The reply must not indicate any errors. + if txD.Err != nil { + tc.t.Errorf("FetchTxByShaList (%s): tx #%d (%v) "+ + "returned unexpected error - got %v, "+ + "want nil", tc.dbType, i, txD.Sha, txD.Err) + return false + } + + // The transaction in the reply fetched from the database must + // be the same MsgTx that was stored. + if !reflect.DeepEqual(tx, txD.Tx) { + tc.t.Errorf("FetchTxByShaList (%s): tx #%d (%v) from "+ + "database does not match stored tx\n"+ + "got: %v\nwant: %v", tc.dbType, i, txHash, + spew.Sdump(txD.Tx), spew.Sdump(tx)) + return false + } + + // The block hash in the reply from the database must be the + // expected value. + if txD.BlkSha == nil { + tc.t.Errorf("FetchTxByShaList (%s): tx #%d (%v) "+ + "returned nil block hash", tc.dbType, i, txD.Sha) + return false + } + if !txD.BlkSha.IsEqual(tc.blockHash) { + tc.t.Errorf("FetchTxByShaList (%s): tx #%d (%v) "+ + "returned unexpected block hash - got %v, "+ + "want %v", tc.dbType, i, txD.Sha, txD.BlkSha, + tc.blockHash) + return false + } + + // The block height in the reply from the database must be the + // expected value. + if txD.Height != tc.blockHeight { + tc.t.Errorf("FetchTxByShaList (%s): tx #%d (%v) "+ + "returned unexpected block height - got %v, "+ + "want %v", tc.dbType, i, txD.Sha, txD.Height, + tc.blockHeight) + return false + } + + // The spend data in the reply from the database must not + // indicate any of the transactions that were just inserted are + // spent. + if txD.TxSpent == nil { + tc.t.Errorf("FetchTxByShaList (%s): tx #%d (%v) "+ + "returned nil spend data", tc.dbType, i, txD.Sha) + return false + } + noSpends := make([]bool, len(tx.TxOut)) + if !reflect.DeepEqual(txD.TxSpent, noSpends) { + tc.t.Errorf("FetchTxByShaList (%s): tx #%d (%v) "+ + "returned unexpected spend data - got %v, "+ + "want %v", tc.dbType, i, txD.Sha, txD.TxSpent, + noSpends) + return false + } + } + + return true +} + // testInterface tests performs tests for the various interfaces of btcdb which // require state in the database for the given database type. func testInterface(t *testing.T, dbType string) { @@ -258,6 +351,13 @@ func testInterface(t *testing.T, dbType string) { if !testFetchTxBySha(&context) { return } + + // All of the transactions in the block must be fetchable via + // FetchTxByShaList and all of the list replies must have the + // expected values. + if !testFetchTxByShaList(&context) { + return + } } // TODO(davec): Need to figure out how to handle the special checks @@ -274,7 +374,7 @@ func testInterface(t *testing.T, dbType string) { FetchHeightRange(startHeight, endHeight int64) (rshalist []btcwire.ShaHash, err error) - ExistsTxSha(sha *btcwire.ShaHash) (exists bool) - FetchTxBySha(txsha *btcwire.ShaHash) ([]*TxListReply, error) - FetchTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply + - FetchTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply - InsertBlock(block *btcutil.Block) (height int64, err error) InvalidateBlockCache() From 01af05c8ac2210e04ee7102c3f6e9f8f651aceb1 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Mon, 14 Oct 2013 13:56:21 -0500 Subject: [PATCH 087/163] Add interface tests for FetchUnSpentTxByShaList. --- interface_test.go | 102 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 101 insertions(+), 1 deletion(-) diff --git a/interface_test.go b/interface_test.go index 3905c3d3..46318c5b 100644 --- a/interface_test.go +++ b/interface_test.go @@ -282,6 +282,99 @@ func testFetchTxByShaList(tc *testContext) bool { return true } +// testFetchUnSpentTxByShaList ensures FetchUnSpentTxByShaList conforms to the +// interface contract. +func testFetchUnSpentTxByShaList(tc *testContext) bool { + txHashes, err := tc.block.TxShas() + if err != nil { + tc.t.Errorf("block.TxShas: %v", err) + return false + } + + txReplyList := tc.db.FetchUnSpentTxByShaList(txHashes) + if len(txReplyList) != len(txHashes) { + tc.t.Errorf("FetchUnSpentTxByShaList (%s): tx reply list for "+ + "block #%d (%v) does not match expected length "+ + "- got: %v, want: %v", tc.dbType, tc.blockHeight, + tc.blockHash, len(txReplyList), len(txHashes)) + return false + } + for i, tx := range tc.block.MsgBlock().Transactions { + txHash := txHashes[i] + txD := txReplyList[i] + + // The transaction hash in the reply must be the expected value. + if !txD.Sha.IsEqual(txHash) { + tc.t.Errorf("FetchUnSpentTxByShaList (%s): tx #%d hash "+ + "does not match expected - got %v, "+ + "want %v", tc.dbType, i, txD.Sha, txHashes[i]) + return false + } + + // The reply must not indicate any errors. + if txD.Err != nil { + tc.t.Errorf("FetchUnSpentTxByShaList (%s): tx #%d (%v) "+ + "returned unexpected error - got %v, "+ + "want nil", tc.dbType, i, txD.Sha, txD.Err) + return false + } + + // The transaction in the reply fetched from the database must + // be the same MsgTx that was stored. + if !reflect.DeepEqual(tx, txD.Tx) { + tc.t.Errorf("FetchUnSpentTxByShaList (%s): tx #%d (%v) "+ + "from database does not match stored tx\n"+ + "got: %v\nwant: %v", tc.dbType, i, txHash, + spew.Sdump(txD.Tx), spew.Sdump(tx)) + return false + } + + // The block hash in the reply from the database must be the + // expected value. + if txD.BlkSha == nil { + tc.t.Errorf("FetchUnSpentTxByShaList (%s): tx #%d (%v) "+ + "returned nil block hash", tc.dbType, i, txD.Sha) + return false + } + if !txD.BlkSha.IsEqual(tc.blockHash) { + tc.t.Errorf("FetchUnSpentTxByShaList (%s): tx #%d (%v) "+ + "returned unexpected block hash - got %v, "+ + "want %v", tc.dbType, i, txD.Sha, txD.BlkSha, + tc.blockHash) + return false + } + + // The block height in the reply from the database must be the + // expected value. + if txD.Height != tc.blockHeight { + tc.t.Errorf("FetchUnSpentTxByShaList (%s): tx #%d (%v) "+ + "returned unexpected block height - got %v, "+ + "want %v", tc.dbType, i, txD.Sha, txD.Height, + tc.blockHeight) + return false + } + + // The spend data in the reply from the database must not + // indicate any of the transactions that were just inserted are + // spent. + if txD.TxSpent == nil { + tc.t.Errorf("FetchUnSpentTxByShaList (%s): tx #%d (%v) "+ + "returned nil spend data", tc.dbType, i, txD.Sha) + return false + } + noSpends := make([]bool, len(tx.TxOut)) + if !reflect.DeepEqual(txD.TxSpent, noSpends) { + tc.t.Errorf("FetchUnSpentTxByShaList (%s): tx #%d (%v) "+ + "returned unexpected spend data - got %v, "+ + "want %v", tc.dbType, i, txD.Sha, txD.TxSpent, + noSpends) + return false + } + } + + return true +} + // testInterface tests performs tests for the various interfaces of btcdb which // require state in the database for the given database type. func testInterface(t *testing.T, dbType string) { @@ -358,6 +451,13 @@ func testInterface(t *testing.T, dbType string) { if !testFetchTxByShaList(&context) { return } + + // All of the transactions in the block must be fetchable via + // FetchUnSpentTxByShaList and all of the list replies must have + // the expected values. + if !testFetchUnSpentTxByShaList(&context) { + return + } } // TODO(davec): Need to figure out how to handle the special checks @@ -375,7 +475,7 @@ func testInterface(t *testing.T, dbType string) { - ExistsTxSha(sha *btcwire.ShaHash) (exists bool) - FetchTxBySha(txsha *btcwire.ShaHash) ([]*TxListReply, error) - FetchTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply - FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply + - FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply - InsertBlock(block *btcutil.Block) (height int64, err error) InvalidateBlockCache() InvalidateCache() From c99416c1217a66be44cee2383fc31d229b0393c7 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Mon, 14 Oct 2013 14:34:54 -0500 Subject: [PATCH 088/163] Include db type in interface tests loading message. --- interface_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface_test.go b/interface_test.go index 46318c5b..088c1eec 100644 --- a/interface_test.go +++ b/interface_test.go @@ -396,7 +396,7 @@ func testInterface(t *testing.T, dbType string) { // Create a test context to pass around. context := testContext{t: t, dbType: dbType, db: db} - t.Logf("Loaded %d blocks", len(blocks)) + t.Logf("Loaded %d blocks for testing %s", len(blocks), dbType) for height := int64(1); height < int64(len(blocks)); height++ { // Get the appropriate block and hash and update the test // context accordingly. From 9ec86518ac5655dae41b18c3062cb497aebdf277 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Mon, 14 Oct 2013 18:27:12 -0500 Subject: [PATCH 089/163] Improve interface test error messages. This commit changes the interface test errors to all report the block height and hash prior to the failure. Test failures that transactions also now report the transaction number and hash. The makes it much easier to track down where a test failure occurs. --- interface_test.go | 178 +++++++++++++++++++++++++--------------------- 1 file changed, 98 insertions(+), 80 deletions(-) diff --git a/interface_test.go b/interface_test.go index 088c1eec..5349a8f7 100644 --- a/interface_test.go +++ b/interface_test.go @@ -29,8 +29,8 @@ func testInsertBlock(tc *testContext) bool { // The block must insert without any errors. newHeight, err := tc.db.InsertBlock(tc.block) if err != nil { - tc.t.Errorf("InsertBlock (%s): failed to insert block %v "+ - "err %v", tc.dbType, tc.blockHeight, err) + tc.t.Errorf("InsertBlock (%s): failed to insert block #%d (%s) "+ + "err %v", tc.dbType, tc.blockHeight, tc.blockHash, err) return false } @@ -48,8 +48,8 @@ func testInsertBlock(tc *testContext) bool { func testExistsSha(tc *testContext) bool { // The block must exist in the database. if exists := tc.db.ExistsSha(tc.blockHash); !exists { - tc.t.Errorf("ExistsSha (%s): block %v does not exist", - tc.dbType, tc.blockHash) + tc.t.Errorf("ExistsSha (%s): block #%d (%s) does not exist", + tc.dbType, tc.blockHeight, tc.blockHash) return false } @@ -62,16 +62,17 @@ func testFetchBlockBySha(tc *testContext) bool { // The block must be fetchable by its hash without any errors. blockFromDb, err := tc.db.FetchBlockBySha(tc.blockHash) if err != nil { - tc.t.Errorf("FetchBlockBySha (%s): %v", tc.dbType, err) + tc.t.Errorf("FetchBlockBySha (%s): block #%d (%s) err: %v", + tc.dbType, tc.blockHeight, tc.blockHash, err) return false } // The block fetched from the database must give back the same MsgBlock // and raw bytes that were stored. if !reflect.DeepEqual(tc.block.MsgBlock(), blockFromDb.MsgBlock()) { - tc.t.Errorf("FetchBlockBySha (%s): block from database "+ - "does not match stored block\ngot: %v\n"+ - "want: %v", tc.dbType, + tc.t.Errorf("FetchBlockBySha (%s): block #%d (%s) does not "+ + "match stored block\ngot: %v\nwant: %v", tc.dbType, + tc.blockHeight, tc.blockHash, spew.Sdump(blockFromDb.MsgBlock()), spew.Sdump(tc.block.MsgBlock())) return false @@ -87,9 +88,9 @@ func testFetchBlockBySha(tc *testContext) bool { return false } if !reflect.DeepEqual(blockBytes, blockFromDbBytes) { - tc.t.Errorf("FetchBlockBySha (%s): block bytes from "+ - "database do not match stored block bytes\n"+ - "got: %v\nwant: %v", tc.dbType, + tc.t.Errorf("FetchBlockBySha (%s): block #%d (%s) bytes do "+ + "not match stored bytes\ngot: %v\nwant: %v", tc.dbType, + tc.blockHeight, tc.blockHash, spew.Sdump(blockFromDbBytes), spew.Sdump(blockBytes)) return false } @@ -104,13 +105,14 @@ func testFetchBlockShaByHeight(tc *testContext) bool { // value. hashFromDb, err := tc.db.FetchBlockShaByHeight(tc.blockHeight) if err != nil { - tc.t.Errorf("FetchBlockShaByHeight (%s): %v", tc.dbType, err) + tc.t.Errorf("FetchBlockShaByHeight (%s): block #%d (%s) err: %v", + tc.dbType, tc.blockHeight, tc.blockHash, err) return false } if !hashFromDb.IsEqual(tc.blockHash) { - tc.t.Errorf("FetchBlockShaByHeight (%s): returned hash "+ - "does not match expected value - got: %v, "+ - "want: %v", tc.dbType, hashFromDb, tc.blockHash) + tc.t.Errorf("FetchBlockShaByHeight (%s): block #%d (%s) hash "+ + "does not match expected value - got: %v", tc.dbType, + tc.blockHeight, tc.blockHash, hashFromDb) return false } @@ -147,8 +149,9 @@ func testExistsTxSha(tc *testContext) bool { // The transaction must exist in the database. txHash := txHashes[i] if exists := tc.db.ExistsTxSha(txHash); !exists { - tc.t.Errorf("ExistsTxSha (%s): tx %v does not exist", - tc.dbType, txHash) + tc.t.Errorf("ExistsTxSha (%s): block #%d (%s) "+ + "tx #%d (%s) does not exist", tc.dbType, + tc.blockHeight, tc.blockHash, i, txHash) return false } } @@ -168,20 +171,25 @@ func testFetchTxBySha(tc *testContext) bool { txHash := txHashes[i] txReplyList, err := tc.db.FetchTxBySha(txHash) if err != nil { - tc.t.Errorf("FetchTxBySha (%s): %v", tc.dbType, err) + tc.t.Errorf("FetchTxBySha (%s): block #%d (%s) "+ + "tx #%d (%s) err: %v", tc.dbType, tc.blockHeight, + tc.blockHash, i, txHash, err) return false } if len(txReplyList) == 0 { - tc.t.Errorf("FetchTxBySha (%s): tx %v did not "+ - "return reply data", tc.dbType, txHash) + tc.t.Errorf("FetchTxBySha (%s): block #%d (%s) "+ + "tx #%d (%s) did not return reply data", + tc.dbType, tc.blockHeight, tc.blockHash, i, + txHash) return false } txFromDb := txReplyList[len(txReplyList)-1].Tx if !reflect.DeepEqual(tx, txFromDb) { - tc.t.Errorf("FetchTxBySha (%s): tx %v from "+ - "database does not match stored tx\n"+ - "got: %v\nwant: %v", tc.dbType, txHash, - spew.Sdump(txFromDb), spew.Sdump(tx)) + tc.t.Errorf("FetchTxBySha (%s): block #%d (%s) "+ + "tx #%d (%s) does not match stored tx\n"+ + "got: %v\nwant: %v", tc.dbType, tc.blockHeight, + tc.blockHash, i, txHash, spew.Sdump(txFromDb), + spew.Sdump(tx)) return false } } @@ -200,8 +208,8 @@ func testFetchTxByShaList(tc *testContext) bool { txReplyList := tc.db.FetchTxByShaList(txHashes) if len(txReplyList) != len(txHashes) { - tc.t.Errorf("FetchTxByShaList (%s): tx reply list for "+ - "block #%d (%v) does not match expected length "+ + tc.t.Errorf("FetchTxByShaList (%s): block #%d (%s) "+ + "tx reply list does not match expected length "+ "- got: %v, want: %v", tc.dbType, tc.blockHeight, tc.blockHash, len(txReplyList), len(txHashes)) return false @@ -212,52 +220,56 @@ func testFetchTxByShaList(tc *testContext) bool { // The transaction hash in the reply must be the expected value. if !txD.Sha.IsEqual(txHash) { - tc.t.Errorf("FetchTxByShaList (%s): tx #%d hash "+ - "does not match expected - got %v, "+ - "want %v", tc.dbType, i, txD.Sha, txHashes[i]) + tc.t.Errorf("FetchTxByShaList (%s): block #%d (%s) "+ + "tx #%d (%s) hash does not match expected "+ + "value - got %v", tc.dbType, tc.blockHeight, + tc.blockHash, i, txHash, txD.Sha) return false } // The reply must not indicate any errors. if txD.Err != nil { - tc.t.Errorf("FetchTxByShaList (%s): tx #%d (%v) "+ - "returned unexpected error - got %v, "+ - "want nil", tc.dbType, i, txD.Sha, txD.Err) + tc.t.Errorf("FetchTxByShaList (%s): block #%d (%s) "+ + "tx #%d (%s) returned unexpected error - "+ + "got %v, want nil", tc.dbType, tc.blockHeight, + tc.blockHash, i, txHash, txD.Err) return false } // The transaction in the reply fetched from the database must // be the same MsgTx that was stored. if !reflect.DeepEqual(tx, txD.Tx) { - tc.t.Errorf("FetchTxByShaList (%s): tx #%d (%v) from "+ - "database does not match stored tx\n"+ - "got: %v\nwant: %v", tc.dbType, i, txHash, - spew.Sdump(txD.Tx), spew.Sdump(tx)) + tc.t.Errorf("FetchTxByShaList (%s): block #%d (%s) "+ + "tx #%d (%s) does not match stored tx\n"+ + "got: %v\nwant: %v", tc.dbType, tc.blockHeight, + tc.blockHash, i, txHash, spew.Sdump(txD.Tx), + spew.Sdump(tx)) return false } // The block hash in the reply from the database must be the // expected value. if txD.BlkSha == nil { - tc.t.Errorf("FetchTxByShaList (%s): tx #%d (%v) "+ - "returned nil block hash", tc.dbType, i, txD.Sha) + tc.t.Errorf("FetchTxByShaList (%s): block #%d (%s) "+ + "tx #%d (%s) returned nil block hash", tc.dbType, + tc.blockHeight, tc.blockHash, i, txHash) return false } if !txD.BlkSha.IsEqual(tc.blockHash) { - tc.t.Errorf("FetchTxByShaList (%s): tx #%d (%v) "+ - "returned unexpected block hash - got %v, "+ - "want %v", tc.dbType, i, txD.Sha, txD.BlkSha, - tc.blockHash) + tc.t.Errorf("FetchTxByShaList (%s): block #%d (%s) "+ + "tx #%d (%s) returned unexpected block hash - "+ + "got %v", tc.dbType, tc.blockHeight, + tc.blockHash, i, txHash, txD.BlkSha) return false } // The block height in the reply from the database must be the // expected value. if txD.Height != tc.blockHeight { - tc.t.Errorf("FetchTxByShaList (%s): tx #%d (%v) "+ - "returned unexpected block height - got %v, "+ - "want %v", tc.dbType, i, txD.Sha, txD.Height, - tc.blockHeight) + tc.t.Errorf("FetchTxByShaList (%s): block #%d (%s) "+ + "tx #%d (%s) returned unexpected block height "+ + "- got %v", tc.dbType, tc.blockHeight, + tc.blockHash, i, txHash, txD.Height) return false } @@ -265,16 +277,17 @@ func testFetchTxByShaList(tc *testContext) bool { // indicate any of the transactions that were just inserted are // spent. if txD.TxSpent == nil { - tc.t.Errorf("FetchTxByShaList (%s): tx #%d (%v) "+ - "returned nil spend data", tc.dbType, i, txD.Sha) + tc.t.Errorf("FetchTxByShaList (%s): block #%d (%s) "+ + "tx #%d (%s) returned nil spend data", tc.dbType, + tc.blockHeight, tc.blockHash, i, txHash) return false } noSpends := make([]bool, len(tx.TxOut)) if !reflect.DeepEqual(txD.TxSpent, noSpends) { - tc.t.Errorf("FetchTxByShaList (%s): tx #%d (%v) "+ - "returned unexpected spend data - got %v, "+ - "want %v", tc.dbType, i, txD.Sha, txD.TxSpent, - noSpends) + tc.t.Errorf("FetchTxByShaList (%s): block #%d (%s) "+ + "tx #%d (%s) returned unexpected spend data - "+ + "got %v, want %v", tc.dbType, tc.blockHeight, + tc.blockHash, i, txHash, txD.TxSpent, noSpends) return false } } @@ -293,8 +306,8 @@ func testFetchUnSpentTxByShaList(tc *testContext) bool { txReplyList := tc.db.FetchUnSpentTxByShaList(txHashes) if len(txReplyList) != len(txHashes) { - tc.t.Errorf("FetchUnSpentTxByShaList (%s): tx reply list for "+ - "block #%d (%v) does not match expected length "+ + tc.t.Errorf("FetchUnSpentTxByShaList (%s): block #%d (%s) "+ + "tx reply list does not match expected length "+ "- got: %v, want: %v", tc.dbType, tc.blockHeight, tc.blockHash, len(txReplyList), len(txHashes)) return false @@ -305,52 +318,56 @@ func testFetchUnSpentTxByShaList(tc *testContext) bool { // The transaction hash in the reply must be the expected value. if !txD.Sha.IsEqual(txHash) { - tc.t.Errorf("FetchUnSpentTxByShaList (%s): tx #%d hash "+ - "does not match expected - got %v, "+ - "want %v", tc.dbType, i, txD.Sha, txHashes[i]) + tc.t.Errorf("FetchUnSpentTxByShaList (%s): block #%d (%s) "+ + "tx #%d (%s) hash does not match expected "+ + "value - got %v", tc.dbType, tc.blockHeight, + tc.blockHash, i, txHash, txD.Sha) return false } // The reply must not indicate any errors. if txD.Err != nil { - tc.t.Errorf("FetchUnSpentTxByShaList (%s): tx #%d (%v) "+ - "returned unexpected error - got %v, "+ - "want nil", tc.dbType, i, txD.Sha, txD.Err) + tc.t.Errorf("FetchUnSpentTxByShaList (%s): block #%d (%s) "+ + "tx #%d (%s) returned unexpected error - "+ + "got %v, want nil", tc.dbType, tc.blockHeight, + tc.blockHash, i, txHash, txD.Err) return false } // The transaction in the reply fetched from the database must // be the same MsgTx that was stored. if !reflect.DeepEqual(tx, txD.Tx) { - tc.t.Errorf("FetchUnSpentTxByShaList (%s): tx #%d (%v) "+ - "from database does not match stored tx\n"+ - "got: %v\nwant: %v", tc.dbType, i, txHash, - spew.Sdump(txD.Tx), spew.Sdump(tx)) + tc.t.Errorf("FetchUnSpentTxByShaList (%s): block #%d (%s) "+ + "tx #%d (%s) does not match stored tx\n"+ + "got: %v\nwant: %v", tc.dbType, tc.blockHeight, + tc.blockHash, i, txHash, spew.Sdump(txD.Tx), + spew.Sdump(tx)) return false } // The block hash in the reply from the database must be the // expected value. if txD.BlkSha == nil { - tc.t.Errorf("FetchUnSpentTxByShaList (%s): tx #%d (%v) "+ - "returned nil block hash", tc.dbType, i, txD.Sha) + tc.t.Errorf("FetchUnSpentTxByShaList (%s): block #%d (%s) "+ + "tx #%d (%s) returned nil block hash", tc.dbType, + tc.blockHeight, tc.blockHash, i, txHash) return false } if !txD.BlkSha.IsEqual(tc.blockHash) { - tc.t.Errorf("FetchUnSpentTxByShaList (%s): tx #%d (%v) "+ - "returned unexpected block hash - got %v, "+ - "want %v", tc.dbType, i, txD.Sha, txD.BlkSha, - tc.blockHash) + tc.t.Errorf("FetchUnSpentTxByShaList (%s): block #%d (%s) "+ + "tx #%d (%s) returned unexpected block hash - "+ + "got %v", tc.dbType, tc.blockHeight, + tc.blockHash, i, txHash, txD.BlkSha) return false } // The block height in the reply from the database must be the // expected value. if txD.Height != tc.blockHeight { - tc.t.Errorf("FetchUnSpentTxByShaList (%s): tx #%d (%v) "+ - "returned unexpected block height - got %v, "+ - "want %v", tc.dbType, i, txD.Sha, txD.Height, - tc.blockHeight) + tc.t.Errorf("FetchUnSpentTxByShaList (%s): block #%d (%s) "+ + "tx #%d (%s) returned unexpected block height "+ + "- got %v", tc.dbType, tc.blockHeight, + tc.blockHash, i, txHash, txD.Height) return false } @@ -358,16 +375,17 @@ func testFetchUnSpentTxByShaList(tc *testContext) bool { // indicate any of the transactions that were just inserted are // spent. if txD.TxSpent == nil { - tc.t.Errorf("FetchUnSpentTxByShaList (%s): tx #%d (%v) "+ - "returned nil spend data", tc.dbType, i, txD.Sha) + tc.t.Errorf("FetchUnSpentTxByShaList (%s): block #%d (%s) "+ + "tx #%d (%s) returned nil spend data", tc.dbType, + tc.blockHeight, tc.blockHash, i, txHash) return false } noSpends := make([]bool, len(tx.TxOut)) if !reflect.DeepEqual(txD.TxSpent, noSpends) { - tc.t.Errorf("FetchUnSpentTxByShaList (%s): tx #%d (%v) "+ - "returned unexpected spend data - got %v, "+ - "want %v", tc.dbType, i, txD.Sha, txD.TxSpent, - noSpends) + tc.t.Errorf("FetchUnSpentTxByShaList (%s): block #%d (%s) "+ + "tx #%d (%s) returned unexpected spend data - "+ + "got %v, want %v", tc.dbType, tc.blockHeight, + tc.blockHash, i, txHash, txD.TxSpent, noSpends) return false } } From a9d719f0c4cca70486d47b5a6607767f81150131 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Mon, 14 Oct 2013 18:29:50 -0500 Subject: [PATCH 090/163] Separate interface integrity tests. This paves the way for executing the integrity tests multiple times. --- interface_test.go | 96 +++++++++++++++++++++++++++++------------------ 1 file changed, 59 insertions(+), 37 deletions(-) diff --git a/interface_test.go b/interface_test.go index 5349a8f7..83717cca 100644 --- a/interface_test.go +++ b/interface_test.go @@ -116,6 +116,10 @@ func testFetchBlockShaByHeight(tc *testContext) bool { return false } + return true +} + +func testFetchBlockShaByHeightErrors(tc *testContext) bool { // Invalid heights must error and return a nil hash. tests := []int64{-1, tc.blockHeight + 1, tc.blockHeight + 2} for i, wantHeight := range tests { @@ -393,6 +397,55 @@ func testFetchUnSpentTxByShaList(tc *testContext) bool { return true } +// testIntegrity performs a series of tests against the interface functions +// which fetch and check for data existence. +func testIntegrity(tc *testContext) bool { + // The block must now exist in the database. + if !testExistsSha(tc) { + return false + } + + // Loading the block back from the database must give back + // the same MsgBlock and raw bytes that were stored. + if !testFetchBlockBySha(tc) { + return false + } + + // The hash returned for the block by its height must be the + // expected value. + if !testFetchBlockShaByHeight(tc) { + return false + } + + // All of the transactions in the block must now exist in the + // database. + if !testExistsTxSha(tc) { + return false + } + + // Loading all of the transactions in the block back from the + // database must give back the same MsgTx that was stored. + if !testFetchTxBySha(tc) { + return false + } + + // All of the transactions in the block must be fetchable via + // FetchTxByShaList and all of the list replies must have the + // expected values. + if !testFetchTxByShaList(tc) { + return false + } + + // All of the transactions in the block must be fetchable via + // FetchUnSpentTxByShaList and all of the list replies must have + // the expected values. + if !testFetchUnSpentTxByShaList(tc) { + return false + } + + return true +} + // testInterface tests performs tests for the various interfaces of btcdb which // require state in the database for the given database type. func testInterface(t *testing.T, dbType string) { @@ -434,46 +487,15 @@ func testInterface(t *testing.T, dbType string) { return } - // The block must now exist in the database. - if !testExistsSha(&context) { + // The block must pass all data integrity tests which involve + // invoking all and testing the result of all interface + // functions which deal with fetch and checking for data + // existence. + if !testIntegrity(&context) { return } - // Loading the block back from the database must give back - // the same MsgBlock and raw bytes that were stored. - if !testFetchBlockBySha(&context) { - return - } - - // The hash returned for the block by its height must be the - // expected value. - if !testFetchBlockShaByHeight(&context) { - return - } - - // All of the transactions in the block must now exist in the - // database. - if !testExistsTxSha(&context) { - return - } - - // Loading all of the transactions in the block back from the - // database must give back the same MsgTx that was stored. - if !testFetchTxBySha(&context) { - return - } - - // All of the transactions in the block must be fetchable via - // FetchTxByShaList and all of the list replies must have the - // expected values. - if !testFetchTxByShaList(&context) { - return - } - - // All of the transactions in the block must be fetchable via - // FetchUnSpentTxByShaList and all of the list replies must have - // the expected values. - if !testFetchUnSpentTxByShaList(&context) { + if !testFetchBlockShaByHeightErrors(&context) { return } } From 54ad820133f32d1512d2a3010546551b35081c2a Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Mon, 14 Oct 2013 19:24:08 -0500 Subject: [PATCH 091/163] Combine two test functions used to fetch tx lists. The two functions are nearly identical, so rather than repeating the test code, refactor it into a separate function that takes a flag. --- interface_test.go | 165 ++++++++++++---------------------------------- 1 file changed, 42 insertions(+), 123 deletions(-) diff --git a/interface_test.go b/interface_test.go index 83717cca..288ca983 100644 --- a/interface_test.go +++ b/interface_test.go @@ -201,21 +201,26 @@ func testFetchTxBySha(tc *testContext) bool { return true } -// testFetchTxByShaList ensures FetchTxByShaList conforms to the interface -// contract. -func testFetchTxByShaList(tc *testContext) bool { +func testFetchTxByShaListCommon(tc *testContext, includeSpent bool) bool { + fetchFunc := tc.db.FetchUnSpentTxByShaList + funcName := "FetchUnSpentTxByShaList" + if includeSpent { + fetchFunc = tc.db.FetchTxByShaList + funcName = "FetchTxByShaList" + } + txHashes, err := tc.block.TxShas() if err != nil { tc.t.Errorf("block.TxShas: %v", err) return false } - txReplyList := tc.db.FetchTxByShaList(txHashes) + txReplyList := fetchFunc(txHashes) if len(txReplyList) != len(txHashes) { - tc.t.Errorf("FetchTxByShaList (%s): block #%d (%s) "+ - "tx reply list does not match expected length "+ - "- got: %v, want: %v", tc.dbType, tc.blockHeight, - tc.blockHash, len(txReplyList), len(txHashes)) + tc.t.Errorf("%s (%s): block #%d (%s) tx reply list does not "+ + " match expected length - got: %v, want: %v", funcName, + tc.dbType, tc.blockHeight, tc.blockHash, + len(txReplyList), len(txHashes)) return false } for i, tx := range tc.block.MsgBlock().Transactions { @@ -224,18 +229,18 @@ func testFetchTxByShaList(tc *testContext) bool { // The transaction hash in the reply must be the expected value. if !txD.Sha.IsEqual(txHash) { - tc.t.Errorf("FetchTxByShaList (%s): block #%d (%s) "+ - "tx #%d (%s) hash does not match expected "+ - "value - got %v", tc.dbType, tc.blockHeight, + tc.t.Errorf("%s (%s): block #%d (%s) tx #%d (%s) "+ + "hash does not match expected value - got %v", + funcName, tc.dbType, tc.blockHeight, tc.blockHash, i, txHash, txD.Sha) return false } // The reply must not indicate any errors. if txD.Err != nil { - tc.t.Errorf("FetchTxByShaList (%s): block #%d (%s) "+ - "tx #%d (%s) returned unexpected error - "+ - "got %v, want nil", tc.dbType, tc.blockHeight, + tc.t.Errorf("%s (%s): block #%d (%s) tx #%d (%s) "+ + "returned unexpected error - got %v, want nil", + funcName, tc.dbType, tc.blockHeight, tc.blockHash, i, txHash, txD.Err) return false } @@ -243,9 +248,9 @@ func testFetchTxByShaList(tc *testContext) bool { // The transaction in the reply fetched from the database must // be the same MsgTx that was stored. if !reflect.DeepEqual(tx, txD.Tx) { - tc.t.Errorf("FetchTxByShaList (%s): block #%d (%s) "+ - "tx #%d (%s) does not match stored tx\n"+ - "got: %v\nwant: %v", tc.dbType, tc.blockHeight, + tc.t.Errorf("%s (%s): block #%d (%s) tx #%d (%s) does "+ + "not match stored tx\ngot: %v\nwant: %v", + funcName, tc.dbType, tc.blockHeight, tc.blockHash, i, txHash, spew.Sdump(txD.Tx), spew.Sdump(tx)) return false @@ -254,15 +259,15 @@ func testFetchTxByShaList(tc *testContext) bool { // The block hash in the reply from the database must be the // expected value. if txD.BlkSha == nil { - tc.t.Errorf("FetchTxByShaList (%s): block #%d (%s) "+ - "tx #%d (%s) returned nil block hash", tc.dbType, + tc.t.Errorf("%s (%s): block #%d (%s) tx #%d (%s) "+ + "returned nil block hash", funcName, tc.dbType, tc.blockHeight, tc.blockHash, i, txHash) return false } if !txD.BlkSha.IsEqual(tc.blockHash) { - tc.t.Errorf("FetchTxByShaList (%s): block #%d (%s) "+ - "tx #%d (%s) returned unexpected block hash - "+ - "got %v", tc.dbType, tc.blockHeight, + tc.t.Errorf("%s (%s): block #%d (%s) tx #%d (%s)"+ + "returned unexpected block hash - got %v", + funcName, tc.dbType, tc.blockHeight, tc.blockHash, i, txHash, txD.BlkSha) return false } @@ -270,9 +275,9 @@ func testFetchTxByShaList(tc *testContext) bool { // The block height in the reply from the database must be the // expected value. if txD.Height != tc.blockHeight { - tc.t.Errorf("FetchTxByShaList (%s): block #%d (%s) "+ - "tx #%d (%s) returned unexpected block height "+ - "- got %v", tc.dbType, tc.blockHeight, + tc.t.Errorf("%s (%s): block #%d (%s) tx #%d (%s) "+ + "returned unexpected block height - got %v", + funcName, tc.dbType, tc.blockHeight, tc.blockHash, i, txHash, txD.Height) return false } @@ -281,16 +286,16 @@ func testFetchTxByShaList(tc *testContext) bool { // indicate any of the transactions that were just inserted are // spent. if txD.TxSpent == nil { - tc.t.Errorf("FetchTxByShaList (%s): block #%d (%s) "+ - "tx #%d (%s) returned nil spend data", tc.dbType, + tc.t.Errorf("%s (%s): block #%d (%s) tx #%d (%s) "+ + "returned nil spend data", funcName, tc.dbType, tc.blockHeight, tc.blockHash, i, txHash) return false } noSpends := make([]bool, len(tx.TxOut)) if !reflect.DeepEqual(txD.TxSpent, noSpends) { - tc.t.Errorf("FetchTxByShaList (%s): block #%d (%s) "+ - "tx #%d (%s) returned unexpected spend data - "+ - "got %v, want %v", tc.dbType, tc.blockHeight, + tc.t.Errorf("%s (%s): block #%d (%s) tx #%d (%s) "+ + "returned unexpected spend data - got %v, "+ + "want %v", funcName, tc.dbType, tc.blockHeight, tc.blockHash, i, txHash, txD.TxSpent, noSpends) return false } @@ -299,102 +304,16 @@ func testFetchTxByShaList(tc *testContext) bool { return true } +// testFetchTxByShaList ensures FetchTxByShaList conforms to the interface +// contract. +func testFetchTxByShaList(tc *testContext) bool { + return testFetchTxByShaListCommon(tc, true) +} + // testFetchUnSpentTxByShaList ensures FetchUnSpentTxByShaList conforms to the // interface contract. func testFetchUnSpentTxByShaList(tc *testContext) bool { - txHashes, err := tc.block.TxShas() - if err != nil { - tc.t.Errorf("block.TxShas: %v", err) - return false - } - - txReplyList := tc.db.FetchUnSpentTxByShaList(txHashes) - if len(txReplyList) != len(txHashes) { - tc.t.Errorf("FetchUnSpentTxByShaList (%s): block #%d (%s) "+ - "tx reply list does not match expected length "+ - "- got: %v, want: %v", tc.dbType, tc.blockHeight, - tc.blockHash, len(txReplyList), len(txHashes)) - return false - } - for i, tx := range tc.block.MsgBlock().Transactions { - txHash := txHashes[i] - txD := txReplyList[i] - - // The transaction hash in the reply must be the expected value. - if !txD.Sha.IsEqual(txHash) { - tc.t.Errorf("FetchUnSpentTxByShaList (%s): block #%d (%s) "+ - "tx #%d (%s) hash does not match expected "+ - "value - got %v", tc.dbType, tc.blockHeight, - tc.blockHash, i, txHash, txD.Sha) - return false - } - - // The reply must not indicate any errors. - if txD.Err != nil { - tc.t.Errorf("FetchUnSpentTxByShaList (%s): block #%d (%s) "+ - "tx #%d (%s) returned unexpected error - "+ - "got %v, want nil", tc.dbType, tc.blockHeight, - tc.blockHash, i, txHash, txD.Err) - return false - } - - // The transaction in the reply fetched from the database must - // be the same MsgTx that was stored. - if !reflect.DeepEqual(tx, txD.Tx) { - tc.t.Errorf("FetchUnSpentTxByShaList (%s): block #%d (%s) "+ - "tx #%d (%s) does not match stored tx\n"+ - "got: %v\nwant: %v", tc.dbType, tc.blockHeight, - tc.blockHash, i, txHash, spew.Sdump(txD.Tx), - spew.Sdump(tx)) - return false - } - - // The block hash in the reply from the database must be the - // expected value. - if txD.BlkSha == nil { - tc.t.Errorf("FetchUnSpentTxByShaList (%s): block #%d (%s) "+ - "tx #%d (%s) returned nil block hash", tc.dbType, - tc.blockHeight, tc.blockHash, i, txHash) - return false - } - if !txD.BlkSha.IsEqual(tc.blockHash) { - tc.t.Errorf("FetchUnSpentTxByShaList (%s): block #%d (%s) "+ - "tx #%d (%s) returned unexpected block hash - "+ - "got %v", tc.dbType, tc.blockHeight, - tc.blockHash, i, txHash, txD.BlkSha) - return false - } - - // The block height in the reply from the database must be the - // expected value. - if txD.Height != tc.blockHeight { - tc.t.Errorf("FetchUnSpentTxByShaList (%s): block #%d (%s) "+ - "tx #%d (%s) returned unexpected block height "+ - "- got %v", tc.dbType, tc.blockHeight, - tc.blockHash, i, txHash, txD.Height) - return false - } - - // The spend data in the reply from the database must not - // indicate any of the transactions that were just inserted are - // spent. - if txD.TxSpent == nil { - tc.t.Errorf("FetchUnSpentTxByShaList (%s): block #%d (%s) "+ - "tx #%d (%s) returned nil spend data", tc.dbType, - tc.blockHeight, tc.blockHash, i, txHash) - return false - } - noSpends := make([]bool, len(tx.TxOut)) - if !reflect.DeepEqual(txD.TxSpent, noSpends) { - tc.t.Errorf("FetchUnSpentTxByShaList (%s): block #%d (%s) "+ - "tx #%d (%s) returned unexpected spend data - "+ - "got %v, want %v", tc.dbType, tc.blockHeight, - tc.blockHash, i, txHash, txD.TxSpent, noSpends) - return false - } - } - - return true + return testFetchTxByShaListCommon(tc, false) } // testIntegrity performs a series of tests against the interface functions From 0c8a15a9d54d4f1025aeba490e54afa5ad672fc5 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Mon, 14 Oct 2013 20:37:58 -0500 Subject: [PATCH 092/163] Add tests to verify integrity after clearing cache. This commit adds the infrastructure needed to re-run the integrity tests after calling the InvalidBlockCache, InvalidateTxCache, and InvalidateCache functions. The tests intentially randomize the order the invalidate functions are called in to increase the likelyhood of finding in issues realted to ordering of the calls. --- interface_test.go | 86 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 82 insertions(+), 4 deletions(-) diff --git a/interface_test.go b/interface_test.go index 288ca983..7bbcd781 100644 --- a/interface_test.go +++ b/interface_test.go @@ -14,7 +14,13 @@ import ( ) // testContext is used to store context information about a running test which -// is passed into helper functions. +// is passed into helper functions. The useSpends field indicates whether or +// not the spend data should be empty or figure it out based on the specific +// test blocks provided. This is needed because the first loop where the blocks +// are inserted, the tests are running against the latest block and therefore +// none of the outputs can be spent yet. However, on subsequent runs, all +// blocks have been inserted and therefore some of the transaction outputs are +// spent. type testContext struct { t *testing.T dbType string @@ -22,6 +28,7 @@ type testContext struct { blockHeight int64 blockHash *btcwire.ShaHash block *btcutil.Block + useSpends bool } // testInsertBlock ensures InsertBlock conforms to the interface contract. @@ -201,6 +208,46 @@ func testFetchTxBySha(tc *testContext) bool { return true } +// expectedSpentBuf returns the expected transaction spend information depending +// on the block height and and transaction number. NOTE: These figures are +// only valid for the specific set of test data provided at the time these tests +// were written. In particular, this means the first 256 blocks of the mainnet +// block chain. +// +// The first run through while the blocks are still being inserted, the tests +// are running against the latest block and therefore none of the outputs can +// be spent yet. However, on subsequent runs, all blocks have been inserted and +// therefore some of the transaction outputs are spent. +func expectedSpentBuf(tc *testContext, txNum int) []bool { + numTxOut := len(tc.block.MsgBlock().Transactions[txNum].TxOut) + spentBuf := make([]bool, numTxOut) + if tc.useSpends { + if tc.blockHeight == 9 && txNum == 0 { + spentBuf[0] = true + } + + if tc.blockHeight == 170 && txNum == 1 { + spentBuf[1] = true + } + + if tc.blockHeight == 181 && txNum == 1 { + spentBuf[1] = true + } + + if tc.blockHeight == 182 && txNum == 1 { + spentBuf[0] = true + spentBuf[1] = true + } + + if tc.blockHeight == 183 && txNum == 1 { + spentBuf[0] = true + spentBuf[1] = true + } + } + + return spentBuf +} + func testFetchTxByShaListCommon(tc *testContext, includeSpent bool) bool { fetchFunc := tc.db.FetchUnSpentTxByShaList funcName := "FetchUnSpentTxByShaList" @@ -291,12 +338,12 @@ func testFetchTxByShaListCommon(tc *testContext, includeSpent bool) bool { tc.blockHeight, tc.blockHash, i, txHash) return false } - noSpends := make([]bool, len(tx.TxOut)) - if !reflect.DeepEqual(txD.TxSpent, noSpends) { + spentBuf := expectedSpentBuf(tc, i) + if !reflect.DeepEqual(txD.TxSpent, spentBuf) { tc.t.Errorf("%s (%s): block #%d (%s) tx #%d (%s) "+ "returned unexpected spend data - got %v, "+ "want %v", funcName, tc.dbType, tc.blockHeight, - tc.blockHash, i, txHash, txD.TxSpent, noSpends) + tc.blockHash, i, txHash, txD.TxSpent, spentBuf) return false } } @@ -419,6 +466,37 @@ func testInterface(t *testing.T, dbType string) { } } + // The data integrity tests must still pass after calling each of the + // invalidate cache functions. This intentionally uses a map since + // map iteration is not the same order every run. This helps catch + // issues that could be caused by calling one version before another. + context.useSpends = true + invalidateCacheFuncs := map[string]func(){ + "InvalidateBlockCache": db.InvalidateBlockCache, + "InvalidateTxCache": db.InvalidateTxCache, + "InvalidateCache": db.InvalidateCache, + } + for funcName, invalidateCacheFunc := range invalidateCacheFuncs { + t.Logf("Running integrity tests after calling %s", funcName) + invalidateCacheFunc() + + for height := int64(0); height < int64(len(blocks)); height++ { + // Get the appropriate block and hash and update the + // test context accordingly. + block := blocks[height] + blockHash, err := block.Sha() + if err != nil { + t.Errorf("block.Sha: %v", err) + return + } + context.blockHeight = height + context.blockHash = blockHash + context.block = block + + testIntegrity(&context) + } + } + // TODO(davec): Need to figure out how to handle the special checks // required for the duplicate transactions allowed by blocks 91842 and // 91880 on the main network due to the old miner + Satoshi client bug. From 3b56ccaff7748219a695a12570c34f57badd253b Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Tue, 15 Oct 2013 10:22:08 -0500 Subject: [PATCH 093/163] Add interface test for NewestSha. --- interface_test.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/interface_test.go b/interface_test.go index 7bbcd781..029c1604 100644 --- a/interface_test.go +++ b/interface_test.go @@ -51,6 +51,36 @@ func testInsertBlock(tc *testContext) bool { return true } +// testNewestSha ensures the NewestSha returns the values expected by the +// interface contract. +func testNewestSha(tc *testContext) bool { + // The news block hash and height must be returned without any errors. + sha, height, err := tc.db.NewestSha() + if err != nil { + tc.t.Errorf("NewestSha (%s): block #%d (%s) error %v", + tc.dbType, tc.blockHeight, tc.blockHash, err) + return false + } + + // The returned hash must be the expected value. + if !sha.IsEqual(tc.blockHash) { + tc.t.Errorf("NewestSha (%s): block #%d (%s) wrong hash got: %s", + tc.dbType, tc.blockHeight, tc.blockHash, sha) + return false + + } + + // The returned height must be the expected value. + if height != tc.blockHeight { + tc.t.Errorf("NewestSha (%s): block #%d (%s) wrong height "+ + "got: %d", tc.dbType, tc.blockHeight, tc.blockHash, + height) + return false + } + + return true +} + // testExistsSha ensures ExistsSha conforms to the interface contract. func testExistsSha(tc *testContext) bool { // The block must exist in the database. @@ -453,6 +483,12 @@ func testInterface(t *testing.T, dbType string) { return } + // The NewestSha function must return the correct information + // about the block that was just inserted. + if !testNewestSha(&context) { + return + } + // The block must pass all data integrity tests which involve // invoking all and testing the result of all interface // functions which deal with fetch and checking for data From 5eee027f92235abcce144f4590427686a9d9d90c Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Tue, 22 Oct 2013 15:25:33 -0500 Subject: [PATCH 094/163] Correct leveldb package documenation. The leveldb package documentation still referred to sqlite from original copy/paste. --- doc.go | 10 ++++------ ldb/doc.go | 12 ++---------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/doc.go b/doc.go index f3065900..a2c60e3e 100644 --- a/doc.go +++ b/doc.go @@ -9,8 +9,7 @@ As of May 2013, there are over 235,000 blocks in the Bitcoin block chain and and over 17 million transactions (which turns out to be over 11GB of data). btcdb provides a database layer to store and retrieve this data in a fairly simple and efficient manner. The use of this should not require specific -knowledge of the database backend used although currently only db_sqlite is -provided. +knowledge of the database backend. Basic Design @@ -20,8 +19,7 @@ increases monotonically. Each transaction belongs to a single block although a block can have a variable number of transactions. Along with these two items, several convenience functions for dealing with the database are provided as well as functions to query specific items -that may be present in a block or tx (although many of these are in -the sqlite3 subpackage). +that may be present in a block or tx. Usage @@ -34,14 +32,14 @@ referenced parent block to already exist. In a more concrete example: // Import packages. import ( "github.com/conformal/btcdb" - _ "github.com/conformal/btcdb/sqlite3" + _ "github.com/conformal/btcdb/ldb" "github.com/conformal/btcutil" "github.com/conformal/btcwire" ) // Create a database and schedule it to be closed on exit. dbName := "example.db" - db, err := btcdb.CreateDB("sqlite", dbName) + db, err := btcdb.CreateDB("leveldb", dbName) if err != nil { // Log and handle the error } diff --git a/ldb/doc.go b/ldb/doc.go index 9ee98f21..629d429a 100644 --- a/ldb/doc.go +++ b/ldb/doc.go @@ -3,20 +3,12 @@ // license that can be found in the LICENSE file. /* -Package sqlite3 implements a sqlite3 instance of btcdb. - -sqlite provides a zero setup, single file database. It requires cgo -and the presence of the sqlite library and headers, but nothing else. -The performance is generally high although it goes down with database -size. - -Many of the block or tx specific functions for btcdb are in this subpackage. +Package ldb implements a instance of btcdb backed by leveldb. Database version number is stored in a flat file .ver Currently a single (littlendian) integer in the file. If there is -additional data to save in the future, the presense of additional +additional data to save in the future, the presence of additional data can be indicated by changing the version number, then parsing the file differently. - */ package ldb From 998682ec8383a40351e772f65555a8079b78d520 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Tue, 22 Oct 2013 15:28:32 -0500 Subject: [PATCH 095/163] Remove sqlitedb from the common tests. The sqlite db backend is now deprecated. --- common_test.go | 1 - ldb/doc.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/common_test.go b/common_test.go index 16e56561..a0c771c4 100644 --- a/common_test.go +++ b/common_test.go @@ -10,7 +10,6 @@ import ( "fmt" "github.com/conformal/btcdb" _ "github.com/conformal/btcdb/ldb" - _ "github.com/conformal/btcdb/sqlite3" "github.com/conformal/btcutil" "github.com/conformal/btcwire" "io" diff --git a/ldb/doc.go b/ldb/doc.go index 629d429a..9899a4c6 100644 --- a/ldb/doc.go +++ b/ldb/doc.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. /* -Package ldb implements a instance of btcdb backed by leveldb. +Package ldb implements an instance of btcdb backed by leveldb. Database version number is stored in a flat file .ver Currently a single (littlendian) integer in the file. If there is From 19880b177e1ebb08ea7f98d22af3143234f44c6b Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sat, 26 Oct 2013 18:02:09 -0500 Subject: [PATCH 096/163] Optimize InsertBlock with cached tx hashes. This commit optimizes InsertBlock slightly by using the cached transaction hashes instead of recomputing them. Also, while here, use the ShaHash.IsEqual function to do comparisons for consistency. This is ongoing work towards conformal/btcd#25. ok @drahn. --- ldb/leveldb.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/ldb/leveldb.go b/ldb/leveldb.go index a3d98d22..f6352615 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -289,11 +289,11 @@ func (db *LevelDb) DropAfterBlockBySha(sha *btcwire.ShaHash) (rerr error) { } } // rather than iterate the list of tx backward, do it twice. - for _, tx := range blk.MsgBlock().Transactions { - txSha, _ := tx.TxSha() + txShas, _ := blk.TxShas() + for _, txSha := range txShas { var txUo txUpdateObj txUo.delete = true - db.txUpdateMap[txSha] = &txUo + db.txUpdateMap[*txSha] = &txUo } db.lBatch().Delete(shaBlkToKey(blksha)) db.lBatch().Delete(int64ToKey(height)) @@ -347,8 +347,7 @@ func (db *LevelDb) InsertBlock(block *btcutil.Block) (height int64, rerr error) // miners, the sha of the transaction exists in a previous block, // detect this condition and 'accept' the block. for txidx, tx := range mblock.Transactions { - var txsha btcwire.ShaHash - txsha, err = tx.TxSha() + txsha, err := block.TxSha(txidx) if err != nil { log.Warnf("failed to compute tx name block %v idx %v err %v", blocksha, txidx, err) return 0, err @@ -362,7 +361,7 @@ func (db *LevelDb) InsertBlock(block *btcutil.Block) (height int64, rerr error) if err != nil { panic("invalid sha string in source") } - if txsha == *dupsha { + if txsha.IsEqual(dupsha) { //log.Tracef("skipping sha %v %v", dupsha, newheight) continue } @@ -372,7 +371,7 @@ func (db *LevelDb) InsertBlock(block *btcutil.Block) (height int64, rerr error) if err != nil { panic("invalid sha string in source") } - if txsha == *dupsha { + if txsha.IsEqual(dupsha) { //log.Tracef("skipping sha %v %v", dupsha, newheight) continue } @@ -385,15 +384,15 @@ func (db *LevelDb) InsertBlock(block *btcutil.Block) (height int64, rerr error) } } - err = db.insertTx(&txsha, newheight, txloc[txidx].TxStart, txloc[txidx].TxLen, spentbuf) + err = db.insertTx(txsha, newheight, txloc[txidx].TxStart, txloc[txidx].TxLen, spentbuf) if err != nil { - log.Warnf("block %v idx %v failed to insert tx %v %v err %v", blocksha, newheight, &txsha, txidx, err) + log.Warnf("block %v idx %v failed to insert tx %v %v err %v", blocksha, newheight, txsha, txidx, err) return } err = db.doSpend(tx) if err != nil { - log.Warnf("block %v idx %v failed to spend tx %v %v err %v", blocksha, newheight, &txsha, txidx, err) + log.Warnf("block %v idx %v failed to spend tx %v %v err %v", blocksha, newheight, txsha, txidx, err) return } From 83b1e74d6c1fd157b1d76f74f5f19f2872c0e913 Mon Sep 17 00:00:00 2001 From: David Hill Date: Sat, 26 Oct 2013 11:08:23 -0400 Subject: [PATCH 097/163] Remove explicit garbage collection. --- ldb/leveldb.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/ldb/leveldb.go b/ldb/leveldb.go index f6352615..768b79d3 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -15,7 +15,6 @@ import ( "github.com/conformal/goleveldb/leveldb/opt" "github.com/conformal/seelog" "os" - "runtime" "sync" ) @@ -548,7 +547,6 @@ func (db *LevelDb) processBatches() error { } db.lbatch.Reset() db.txUpdateMap = map[btcwire.ShaHash]*txUpdateObj{} - runtime.GC() } return nil From 80981b469629c6800e4b3b2a60abcc5be92a8e8d Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Fri, 1 Nov 2013 21:17:06 -0500 Subject: [PATCH 098/163] Update to no longer use deprecated TxShas. The btcutil code was recently changed to provide a new Tx type which provides hash caching (among other things). As a result, the notion of obtaining a transaction hashes via TxShas was deprecated as well. This commit updates the tests and backend implementation code accordingly. --- interface_test.go | 40 ++++++++++++++-------------------------- ldb/leveldb.go | 5 ++--- 2 files changed, 16 insertions(+), 29 deletions(-) diff --git a/interface_test.go b/interface_test.go index 029c1604..31f4a2ba 100644 --- a/interface_test.go +++ b/interface_test.go @@ -180,15 +180,9 @@ func testFetchBlockShaByHeightErrors(tc *testContext) bool { // testExistsTxSha ensures ExistsTxSha conforms to the interface contract. func testExistsTxSha(tc *testContext) bool { - txHashes, err := tc.block.TxShas() - if err != nil { - tc.t.Errorf("block.TxShas: %v", err) - return false - } - - for i := range txHashes { + for i, tx := range tc.block.Transactions() { // The transaction must exist in the database. - txHash := txHashes[i] + txHash := tx.Sha() if exists := tc.db.ExistsTxSha(txHash); !exists { tc.t.Errorf("ExistsTxSha (%s): block #%d (%s) "+ "tx #%d (%s) does not exist", tc.dbType, @@ -202,14 +196,8 @@ func testExistsTxSha(tc *testContext) bool { // testFetchTxBySha ensures FetchTxBySha conforms to the interface contract. func testFetchTxBySha(tc *testContext) bool { - txHashes, err := tc.block.TxShas() - if err != nil { - tc.t.Errorf("block.TxShas: %v", err) - return false - } - - for i, tx := range tc.block.MsgBlock().Transactions { - txHash := txHashes[i] + for i, tx := range tc.block.Transactions() { + txHash := tx.Sha() txReplyList, err := tc.db.FetchTxBySha(txHash) if err != nil { tc.t.Errorf("FetchTxBySha (%s): block #%d (%s) "+ @@ -225,12 +213,12 @@ func testFetchTxBySha(tc *testContext) bool { return false } txFromDb := txReplyList[len(txReplyList)-1].Tx - if !reflect.DeepEqual(tx, txFromDb) { + if !reflect.DeepEqual(tx.MsgTx(), txFromDb) { tc.t.Errorf("FetchTxBySha (%s): block #%d (%s) "+ "tx #%d (%s) does not match stored tx\n"+ "got: %v\nwant: %v", tc.dbType, tc.blockHeight, tc.blockHash, i, txHash, spew.Sdump(txFromDb), - spew.Sdump(tx)) + spew.Sdump(tx.MsgTx())) return false } } @@ -286,10 +274,10 @@ func testFetchTxByShaListCommon(tc *testContext, includeSpent bool) bool { funcName = "FetchTxByShaList" } - txHashes, err := tc.block.TxShas() - if err != nil { - tc.t.Errorf("block.TxShas: %v", err) - return false + transactions := tc.block.Transactions() + txHashes := make([]*btcwire.ShaHash, len(transactions)) + for i, tx := range transactions { + txHashes[i] = tx.Sha() } txReplyList := fetchFunc(txHashes) @@ -300,8 +288,8 @@ func testFetchTxByShaListCommon(tc *testContext, includeSpent bool) bool { len(txReplyList), len(txHashes)) return false } - for i, tx := range tc.block.MsgBlock().Transactions { - txHash := txHashes[i] + for i, tx := range transactions { + txHash := tx.Sha() txD := txReplyList[i] // The transaction hash in the reply must be the expected value. @@ -324,12 +312,12 @@ func testFetchTxByShaListCommon(tc *testContext, includeSpent bool) bool { // The transaction in the reply fetched from the database must // be the same MsgTx that was stored. - if !reflect.DeepEqual(tx, txD.Tx) { + if !reflect.DeepEqual(tx.MsgTx(), txD.Tx) { tc.t.Errorf("%s (%s): block #%d (%s) tx #%d (%s) does "+ "not match stored tx\ngot: %v\nwant: %v", funcName, tc.dbType, tc.blockHeight, tc.blockHash, i, txHash, spew.Sdump(txD.Tx), - spew.Sdump(tx)) + spew.Sdump(tx.MsgTx())) return false } diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 768b79d3..bc6d0834 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -288,11 +288,10 @@ func (db *LevelDb) DropAfterBlockBySha(sha *btcwire.ShaHash) (rerr error) { } } // rather than iterate the list of tx backward, do it twice. - txShas, _ := blk.TxShas() - for _, txSha := range txShas { + for _, tx := range blk.Transactions() { var txUo txUpdateObj txUo.delete = true - db.txUpdateMap[*txSha] = &txUo + db.txUpdateMap[*tx.Sha()] = &txUo } db.lBatch().Delete(shaBlkToKey(blksha)) db.lBatch().Delete(int64ToKey(height)) From b406ca7ae7c66254d95313c80a8358cda9376967 Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Mon, 28 Oct 2013 23:01:52 -0400 Subject: [PATCH 099/163] If ExistsTx fails try looking up the tx, it may be fully spent. --- interface_test.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/interface_test.go b/interface_test.go index 31f4a2ba..907edf00 100644 --- a/interface_test.go +++ b/interface_test.go @@ -184,9 +184,12 @@ func testExistsTxSha(tc *testContext) bool { // The transaction must exist in the database. txHash := tx.Sha() if exists := tc.db.ExistsTxSha(txHash); !exists { - tc.t.Errorf("ExistsTxSha (%s): block #%d (%s) "+ - "tx #%d (%s) does not exist", tc.dbType, - tc.blockHeight, tc.blockHash, i, txHash) + _, err := tc.db.FetchTxBySha(txHash) + if err != nil { + tc.t.Errorf("ExistsTxSha (%s): block #%d (%s) "+ + "tx #%d (%s) does not exist", tc.dbType, + tc.blockHeight, tc.blockHash, i, txHash) + } return false } } From 078fb4d38e281d15c141f84de97d01e6691f1b84 Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Sat, 5 Oct 2013 21:05:29 -0400 Subject: [PATCH 100/163] Delete obsolete API functions. --- ldb/tx.go | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/ldb/tx.go b/ldb/tx.go index 7bc00300..a60fc427 100644 --- a/ldb/tx.go +++ b/ldb/tx.go @@ -143,28 +143,3 @@ func (db *LevelDb) existsTxSha(txSha *btcwire.ShaHash) (exists bool) { return false } - -// FetchLocationBySha looks up the Tx sha information by name. -func (db *LevelDb) FetchLocationBySha(txsha *btcwire.ShaHash) (blockidx int64, txoff int, txlen int, err error) { - db.dbLock.Lock() - defer db.dbLock.Unlock() - err = fmt.Errorf("obsolete function") - return -} - -// FetchTxUsedBySha returns the used/spent buffer for a given transaction. -func (db *LevelDb) FetchTxUsedBySha(txSha *btcwire.ShaHash) (spentbuf []byte, err error) { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - _, _, _, spentbuf, err = db.getTxData(txSha) - if err != nil { - return - } - return // spentbuf has the value already -} - -func (db *LevelDb) fetchLocationUsedBySha(txsha *btcwire.ShaHash) error { - // delete me - return fmt.Errorf("Deleted function") -} From 13ef0e2be973073be3c55813d22bbd60b6498c5a Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Mon, 30 Sep 2013 17:44:26 -0400 Subject: [PATCH 101/163] Rework leveldb support for duplicate tx. This separates fully spent tx from tx containing unspent TxOut This handles duplicate tx and Ideally should be faster. --- ldb/block.go | 26 ++++ ldb/dbcache.go | 117 +--------------- ldb/dup_test.go | 176 ++++++++++++++++++++++++ ldb/insertremove_test.go | 4 +- ldb/leveldb.go | 173 ++++++++++++++++++++---- ldb/tx.go | 280 ++++++++++++++++++++++++++++++++++++++- 6 files changed, 629 insertions(+), 147 deletions(-) create mode 100644 ldb/dup_test.go diff --git a/ldb/block.go b/ldb/block.go index 6423ac1c..b8b467c3 100644 --- a/ldb/block.go +++ b/ldb/block.go @@ -9,9 +9,35 @@ import ( "encoding/binary" "fmt" "github.com/conformal/btcdb" + "github.com/conformal/btcutil" "github.com/conformal/btcwire" ) +// FetchBlockBySha - return a btcutil Block +func (db *LevelDb) FetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + return db.fetchBlockBySha(sha) +} + +// fetchBlockBySha - return a btcutil Block +// Must be called with db lock held. +func (db *LevelDb) fetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) { + + buf, height, err := db.fetchSha(sha) + if err != nil { + return + } + + blk, err = btcutil.NewBlockFromBytes(buf) + if err != nil { + return + } + blk.SetHeight(height) + + return +} + func (db *LevelDb) getBlkLoc(sha *btcwire.ShaHash) (int64, error) { var blkHeight int64 diff --git a/ldb/dbcache.go b/ldb/dbcache.go index e430819c..6f9e3883 100644 --- a/ldb/dbcache.go +++ b/ldb/dbcache.go @@ -5,124 +5,9 @@ package ldb import ( - "bytes" - "github.com/conformal/btcdb" - "github.com/conformal/btcutil" - "github.com/conformal/btcwire" + //"fmt" ) -// FetchBlockBySha - return a btcutil Block, object may be a cached. -func (db *LevelDb) FetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) { - db.dbLock.Lock() - defer db.dbLock.Unlock() - return db.fetchBlockBySha(sha) -} - -// fetchBlockBySha - return a btcutil Block, object may be a cached. -// Must be called with db lock held. -func (db *LevelDb) fetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) { - - buf, height, err := db.fetchSha(sha) - if err != nil { - return - } - - blk, err = btcutil.NewBlockFromBytes(buf) - if err != nil { - return - } - blk.SetHeight(height) - - return -} - -// FetchTxByShaList returns the most recent tx of the name fully spent or not -func (db *LevelDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { - // until the fully spent separation of tx is complete this is identical - // to FetchUnSpentTxByShaList - return db.FetchUnSpentTxByShaList(txShaList) -} - -// FetchUnSpentTxByShaList given a array of ShaHash, look up the transactions -// and return them in a TxListReply array. -func (db *LevelDb) FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - replies := make([]*btcdb.TxListReply, len(txShaList)) - for i, txsha := range txShaList { - tx, blockSha, height, txspent, err := db.fetchTxDataBySha(txsha) - btxspent := []bool{} - if err == nil { - btxspent = make([]bool, len(tx.TxOut), len(tx.TxOut)) - for idx := range tx.TxOut { - byteidx := idx / 8 - byteoff := uint(idx % 8) - btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0 - } - } - txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, BlkSha: blockSha, Height: height, TxSpent: btxspent, Err: err} - replies[i] = &txlre - } - return replies -} - -// fetchTxDataBySha returns several pieces of data regarding the given sha. -func (db *LevelDb) fetchTxDataBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rblksha *btcwire.ShaHash, rheight int64, rtxspent []byte, err error) { - var blksha *btcwire.ShaHash - var blkHeight int64 - var txspent []byte - var txOff, txLen int - var blkbuf []byte - - blkHeight, txOff, txLen, txspent, err = db.getTxData(txsha) - if err != nil { - err = btcdb.TxShaMissing - return - } - - blksha, blkbuf, err = db.getBlkByHeight(blkHeight) - if err != nil { - return - } - - //log.Trace("transaction %v is at block %v %v txoff %v, txlen %v\n", - // txsha, blksha, blkHeight, txOff, txLen) - - rbuf := bytes.NewBuffer(blkbuf[txOff : txOff+txLen]) - - var tx btcwire.MsgTx - err = tx.Deserialize(rbuf) - if err != nil { - log.Warnf("unable to decode tx block %v %v txoff %v txlen %v", - blkHeight, blksha, txOff, txLen) - return - } - - return &tx, blksha, blkHeight, txspent, nil -} - -// FetchTxBySha returns some data for the given Tx Sha. -func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) ([]*btcdb.TxListReply, error) { - tx, blksha, height, txspent, err := db.fetchTxDataBySha(txsha) - if err != nil { - return []*btcdb.TxListReply{}, err - } - - replies := make([]*btcdb.TxListReply, 1) - - btxspent := make([]bool, len(tx.TxOut), len(tx.TxOut)) - for idx := range tx.TxOut { - byteidx := idx / 8 - byteoff := uint(idx % 8) - btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0 - } - - txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, BlkSha: blksha, Height: height, TxSpent: btxspent, Err: err} - replies[0] = &txlre - return replies, nil -} - // InvalidateTxCache clear/release all cached transactions. func (db *LevelDb) InvalidateTxCache() { } diff --git a/ldb/dup_test.go b/ldb/dup_test.go new file mode 100644 index 00000000..4fa8a6f4 --- /dev/null +++ b/ldb/dup_test.go @@ -0,0 +1,176 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package ldb_test + +import ( + "fmt" + "github.com/conformal/btcdb" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" + "os" + "path/filepath" + "testing" +) + +func Test_dupTx(t *testing.T) { + + // Ignore db remove errors since it means we didn't have an old one. + dbname := fmt.Sprintf("tstdbdup0") + dbnamever := dbname + ".ver" + _ = os.RemoveAll(dbname) + _ = os.RemoveAll(dbnamever) + db, err := btcdb.CreateDB("leveldb", dbname) + if err != nil { + t.Errorf("Failed to open test database %v", err) + return + } + defer os.RemoveAll(dbname) + defer os.RemoveAll(dbnamever) + defer db.Close() + + testdatafile := filepath.Join("testdata", "blocks1-256.bz2") + blocks, err := loadBlocks(t, testdatafile) + if err != nil { + t.Errorf("Unable to load blocks from test data for: %v", + err) + return + } + + var lastSha *btcwire.ShaHash + + // Populate with the fisrt 256 blocks, so we have blocks to 'mess with' + err = nil +out: + for height := int64(0); height < int64(len(blocks)); height++ { + block := blocks[height] + + // except for NoVerify which does not allow lookups check inputs + mblock := block.MsgBlock() + var txneededList []*btcwire.ShaHash + for _, tx := range mblock.Transactions { + for _, txin := range tx.TxIn { + if txin.PreviousOutpoint.Index == uint32(4294967295) { + continue + } + origintxsha := &txin.PreviousOutpoint.Hash + txneededList = append(txneededList, origintxsha) + + if !db.ExistsTxSha(origintxsha) { + t.Errorf("referenced tx not found %v ", origintxsha) + } + + _, err = db.FetchTxBySha(origintxsha) + if err != nil { + t.Errorf("referenced tx not found %v err %v ", origintxsha, err) + } + } + } + txlist := db.FetchUnSpentTxByShaList(txneededList) + for _, txe := range txlist { + if txe.Err != nil { + t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) + break out + } + } + + + newheight, err := db.InsertBlock(block) + if err != nil { + t.Errorf("failed to insert block %v err %v", height, err) + break out + } + if newheight != height { + t.Errorf("height mismatch expect %v returned %v", height, newheight) + break out + } + + newSha, blkid, err := db.NewestSha() + if err != nil { + t.Errorf("failed to obtain latest sha %v %v", height, err) + } + + if blkid != height { + t.Errorf("height doe not match latest block height %v %v %v", blkid, height, err) + } + + blkSha, _ := block.Sha() + if *newSha != *blkSha { + t.Errorf("Newest block sha does not match freshly inserted one %v %v %v ", newSha, blkSha, err) + } + lastSha = blkSha + } + + // genrate a new block based on the last sha + // these block are not verified, so there are a bunch of garbage fields + // in the 'generated' block. + + var bh btcwire.BlockHeader + + bh.Version = 2 + bh.PrevBlock = *lastSha + // Bits, Nonce are not filled in + + mblk := btcwire.NewMsgBlock(&bh) + + hash, _ := btcwire.NewShaHashFromStr("df2b060fa2e5e9c8ed5eaf6a45c13753ec8c63282b2688322eba40cd98ea067a") + + po := btcwire.NewOutPoint(hash, 0) + txI := btcwire.NewTxIn(po, []byte("garbage")) + txO := btcwire.NewTxOut(50000000, []byte("garbageout")) + + var tx btcwire.MsgTx + tx.AddTxIn(txI) + tx.AddTxOut(txO) + + mblk.AddTransaction(&tx) + + blk := btcutil.NewBlock(mblk) + + fetchList := []*btcwire.ShaHash{hash} + listReply := db.FetchUnSpentTxByShaList(fetchList) + for _, lr := range listReply { + if lr.Err != nil { + t.Errorf("sha %v spent %v err %v\n", lr.Sha, + lr.TxSpent, lr.Err) + } + } + + _, err = db.InsertBlock(blk) + if err != nil { + t.Errorf("failed to insert phony block %v", err) + } + + // ok, did it 'spend' the tx ? + + listReply = db.FetchUnSpentTxByShaList(fetchList) + for _, lr := range listReply { + if lr.Err != btcdb.TxShaMissing { + t.Errorf("sha %v spent %v err %v\n", lr.Sha, + lr.TxSpent, lr.Err) + } + } + + txshalist, _ := blk.TxShas() + for _, txsha := range txshalist { + txReply, err := db.FetchTxBySha(txsha) + if err != nil { + t.Errorf("fully spent lookup %v err %v\n", hash, err) + } else { + for _, lr := range txReply { + if lr.Err != nil { + fmt.Errorf("stx %v spent %v err %v\n", lr.Sha, + lr.TxSpent, lr.Err) + } + } + } + } + + t.Logf("Dropping block") + + err = db.DropAfterBlockBySha(lastSha) + if err != nil { + t.Errorf("failed to drop spending block %v", err) + } +} diff --git a/ldb/insertremove_test.go b/ldb/insertremove_test.go index b5ce2dc5..66cf48ce 100644 --- a/ldb/insertremove_test.go +++ b/ldb/insertremove_test.go @@ -130,7 +130,7 @@ endtest: } txlookupmap := map[btcwire.ShaHash]*btcdb.TxListReply{} - txlist = db.FetchUnSpentTxByShaList(txlookupList) + txlist = db.FetchTxByShaList(txlookupList) for _, txe := range txlist { if txe.Err != nil { t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) @@ -188,7 +188,7 @@ endtest: break endtest } txlookupmap = map[btcwire.ShaHash]*btcdb.TxListReply{} - txlist = db.FetchUnSpentTxByShaList(txlookupList) + txlist = db.FetchTxByShaList(txlookupList) for _, txe := range txlist { if txe.Err != nil { t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) diff --git a/ldb/leveldb.go b/ldb/leveldb.go index bc6d0834..063f115b 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -52,6 +52,7 @@ type LevelDb struct { lastBlkIdx int64 txUpdateMap map[btcwire.ShaHash]*txUpdateObj + txSpentUpdateMap map[btcwire.ShaHash]*spentTxUpdate } var self = btcdb.DriverDB{DbType: "leveldb", Create: CreateDB, Open: OpenDB} @@ -136,6 +137,7 @@ func openDB(dbpath string, flag opt.OptionsFlag) (pbdb btcdb.Db, err error) { db.lDb = tlDb db.txUpdateMap = map[btcwire.ShaHash]*txUpdateObj{} + db.txSpentUpdateMap = make(map[btcwire.ShaHash]*spentTxUpdate) pbdb = &db } @@ -350,30 +352,6 @@ func (db *LevelDb) InsertBlock(block *btcutil.Block) (height int64, rerr error) log.Warnf("failed to compute tx name block %v idx %v err %v", blocksha, txidx, err) return 0, err } - // Some old blocks contain duplicate transactions - // Attempt to cleanly bypass this problem - // http://blockexplorer.com/b/91842 - // http://blockexplorer.com/b/91880 - if newheight == 91842 { - dupsha, err := btcwire.NewShaHashFromStr("d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599") - if err != nil { - panic("invalid sha string in source") - } - if txsha.IsEqual(dupsha) { - //log.Tracef("skipping sha %v %v", dupsha, newheight) - continue - } - } - if newheight == 91880 { - dupsha, err := btcwire.NewShaHashFromStr("e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468") - if err != nil { - panic("invalid sha string in source") - } - if txsha.IsEqual(dupsha) { - //log.Tracef("skipping sha %v %v", dupsha, newheight) - continue - } - } spentbuflen := (len(tx.TxOut) + 7) / 8 spentbuf := make([]byte, spentbuflen, spentbuflen) if len(tx.TxOut)%8 != 0 { @@ -384,10 +362,53 @@ func (db *LevelDb) InsertBlock(block *btcutil.Block) (height int64, rerr error) err = db.insertTx(txsha, newheight, txloc[txidx].TxStart, txloc[txidx].TxLen, spentbuf) if err != nil { - log.Warnf("block %v idx %v failed to insert tx %v %v err %v", blocksha, newheight, txsha, txidx, err) + log.Warnf("block %v idx %v failed to insert tx %v %v err %v", blocksha, newheight, &txsha, txidx, err) return } + + // Some old blocks contain duplicate transactions + // Attempt to cleanly bypass this problem by marking the + // first as fully spent. + // http://blockexplorer.com/b/91812 dup in 91842 + // http://blockexplorer.com/b/91722 dup in 91880 + if newheight == 91812 { + dupsha, err := btcwire.NewShaHashFromStr("d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599") + if err != nil { + panic("invalid sha string in source") + } + if txsha.IsEqual(dupsha) { + // marking TxOut[0] as spent + po := btcwire.NewOutPoint(dupsha, 0) + txI := btcwire.NewTxIn(po, []byte("garbage")) + + var spendtx btcwire.MsgTx + spendtx.AddTxIn(txI) + err = db.doSpend(&spendtx) + if err != nil { + log.Warnf("block %v idx %v failed to spend tx %v %v err %v", blocksha, newheight, &txsha, txidx, err) + } + } + } + if newheight == 91722 { + dupsha, err := btcwire.NewShaHashFromStr("e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468") + if err != nil { + panic("invalid sha string in source") + } + if txsha.IsEqual(dupsha) { + // marking TxOut[0] as spent + po := btcwire.NewOutPoint(dupsha, 0) + txI := btcwire.NewTxIn(po, []byte("garbage")) + + var spendtx btcwire.MsgTx + spendtx.AddTxIn(txI) + err = db.doSpend(&spendtx) + if err != nil { + log.Warnf("block %v idx %v failed to spend tx %v %v err %v", blocksha, newheight, &txsha, txidx, err) + } + } + } + err = db.doSpend(tx) if err != nil { log.Warnf("block %v idx %v failed to spend tx %v %v err %v", blocksha, newheight, txsha, txidx, err) @@ -467,7 +488,38 @@ func (db *LevelDb) setclearSpentData(txsha *btcwire.ShaHash, idx uint32, set boo var txU txUpdateObj blkHeight, txOff, txLen, spentData, err := db.getTxData(txsha) if err != nil { - return err + // setting a fully spent tx is an error. + if set == true { + return err + } + // if we are clearing a tx and it wasn't found + // in the tx table, it could be in the fully spent + // (duplicates) table. + spentTxList, err := db.getTxFullySpent(txsha) + if err != nil { + return err + } + + // need to reslice the list to exclude the most recent. + sTx := spentTxList [len(spentTxList) -1] + spentTxList [len(spentTxList) -1] = nil + if len (spentTxList) == 1 { + // write entry to delete tx from spent pool + // XXX + } else { + spentTxList = spentTxList [:len(spentTxList)-1] + // XXX format sTxList and set update Table + } + + // Create 'new' Tx update data. + blkHeight = sTx.blkHeight + txOff = sTx.txoff + txLen = sTx.txlen + spentbuflen := (sTx.numTxO + 7) / 8 + spentData = make([]byte, spentbuflen, spentbuflen) + for i := range spentData { + spentData[i] = ^byte(0) + } } txU.txSha = txsha @@ -488,7 +540,51 @@ func (db *LevelDb) setclearSpentData(txsha *btcwire.ShaHash, idx uint32, set boo txUo.spentData[byteidx] &= ^(byte(1) << byteoff) } - db.txUpdateMap[*txsha] = txUo + // check for fully spent Tx + fullySpent := true + for _, val := range txUo.spentData { + if val != ^byte(0) { + fullySpent = false + break + } + } + if fullySpent { + var txSu *spentTxUpdate + // Look up Tx in fully spent table + if txSuOld, ok := db.txSpentUpdateMap[*txsha] ; ok { + txSu = txSuOld + } else { + var txSuStore spentTxUpdate + txSu = &txSuStore + + txSuOld, err := db.getTxFullySpent(txsha) + if err == nil { + txSu.txl = txSuOld + } + } + + // Fill in spentTx + var sTx spentTx + sTx.blkHeight = txUo.blkHeight + sTx.txoff = txUo.txoff + sTx.txlen = txUo.txlen + // XXX -- there is no way to comput the real TxOut + // from the spent array. + sTx.numTxO = 8 * len(txUo.spentData) + + // append this txdata to fully spent txlist + txSu.txl = append(txSu.txl, &sTx) + + // mark txsha as deleted in the txUpdateMap + log.Tracef("***tx %v is fully spent\n", txsha) + + db.txSpentUpdateMap[*txsha] = txSu + + txUo.delete = true + db.txUpdateMap[*txsha] = txUo + } else { + db.txUpdateMap[*txsha] = txUo + } return nil } @@ -509,6 +605,12 @@ func shaTxToKey(sha *btcwire.ShaHash) []byte { return shaB } +func shaSpentTxToKey(sha *btcwire.ShaHash) []byte { + shaB := sha.Bytes() + shaB = append(shaB, "sx"...) + return shaB +} + func (db *LevelDb) lBatch() *leveldb.Batch { if db.lbatch == nil { db.lbatch = new(leveldb.Batch) @@ -519,7 +621,7 @@ func (db *LevelDb) lBatch() *leveldb.Batch { func (db *LevelDb) processBatches() error { var err error - if len(db.txUpdateMap) != 0 || db.lbatch != nil { + if len(db.txUpdateMap) != 0 || len(db.txSpentUpdateMap) != 0 || db.lbatch != nil { if db.lbatch == nil { db.lbatch = new(leveldb.Batch) } @@ -538,6 +640,20 @@ func (db *LevelDb) processBatches() error { db.lbatch.Put(key, txdat) } } + for txSha, txSu := range db.txSpentUpdateMap { + key := shaSpentTxToKey(&txSha) + if txSu.delete { + //log.Infof("deleting tx %v", txSha) + db.lbatch.Delete(key) + } else { + //log.Infof("inserting tx %v", txSha) + txdat, err := db.formatTxFullySpent(txSu.txl) + if err != nil { + return err + } + db.lbatch.Put(key, txdat) + } + } err = db.lDb.Write(db.lbatch, db.wo) if err != nil { @@ -546,6 +662,7 @@ func (db *LevelDb) processBatches() error { } db.lbatch.Reset() db.txUpdateMap = map[btcwire.ShaHash]*txUpdateObj{} + db.txSpentUpdateMap = make(map[btcwire.ShaHash]*spentTxUpdate) } return nil diff --git a/ldb/tx.go b/ldb/tx.go index a60fc427..06d30b1e 100644 --- a/ldb/tx.go +++ b/ldb/tx.go @@ -8,7 +8,8 @@ import ( "bytes" "encoding/binary" "fmt" - //"github.com/conformal/btcdb" + "github.com/conformal/btcdb" + "github.com/conformal/goleveldb/leveldb" "github.com/conformal/btcwire" ) @@ -17,10 +18,23 @@ type txUpdateObj struct { blkHeight int64 txoff int txlen int + ntxout int spentData []byte delete bool } +type spentTx struct { + blkHeight int64 + txoff int + txlen int + numTxO int + delete bool +} +type spentTxUpdate struct { + txl []*spentTx + delete bool +} + // InsertTx inserts a tx hash and its associated data into the database. func (db *LevelDb) InsertTx(txsha *btcwire.ShaHash, height int64, txoff int, txlen int, spentbuf []byte) (err error) { db.dbLock.Lock() @@ -123,6 +137,97 @@ func (db *LevelDb) getTxData(txsha *btcwire.ShaHash) (rblkHeight int64, return blkHeight, int(txOff), int(txLen), spentBuf, nil } +func (db *LevelDb) getTxFullySpent(txsha *btcwire.ShaHash) ([]*spentTx, error) { + + var badTxList, spentTxList []*spentTx + + key := shaSpentTxToKey(txsha) + buf, err := db.lDb.Get(key, db.ro) + if err == leveldb.ErrNotFound { + return badTxList, btcdb.TxShaMissing + } else if err != nil { + return badTxList, err + } + txListLen := len(buf) / 20 + txR := bytes.NewBuffer(buf) + spentTxList = make([]*spentTx, txListLen, txListLen) + + for i := range spentTxList { + var sTx spentTx + var blkHeight int64 + var txOff, txLen, numTxO int32 + + err := binary.Read(txR, binary.LittleEndian, &blkHeight) + if err != nil { + err = fmt.Errorf("sTx Read fail 0") + return nil, err + } + sTx.blkHeight = blkHeight + + err = binary.Read(txR, binary.LittleEndian, &txOff) + if err != nil { + err = fmt.Errorf("sTx Read fail 1") + return nil, err + } + sTx.txoff = int(txOff) + + err = binary.Read(txR, binary.LittleEndian, &txLen) + if err != nil { + err = fmt.Errorf("sTx Read fail 2") + return nil, err + } + sTx.txlen = int(txLen) + + err = binary.Read(txR, binary.LittleEndian, &numTxO) + if err != nil { + err = fmt.Errorf("sTx Read fail 3") + return nil, err + } + sTx.numTxO = int(numTxO) + + spentTxList[i] = &sTx + } + + return spentTxList, nil +} + +func (db *LevelDb) formatTxFullySpent(sTxList []*spentTx) ([]byte, error) { + var txW bytes.Buffer + + for _, sTx := range sTxList { + blkHeight := sTx.blkHeight + txOff := int32(sTx.txoff) + txLen := int32(sTx.txlen) + numTxO := int32(sTx.numTxO) + + err := binary.Write(&txW, binary.LittleEndian, blkHeight) + if err != nil { + err = fmt.Errorf("Write fail") + return nil, err + } + + err = binary.Write(&txW, binary.LittleEndian, txOff) + if err != nil { + err = fmt.Errorf("Write fail") + return nil, err + } + + err = binary.Write(&txW, binary.LittleEndian, txLen) + if err != nil { + err = fmt.Errorf("Write fail") + return nil, err + } + + err = binary.Write(&txW, binary.LittleEndian, numTxO) + if err != nil { + err = fmt.Errorf("Write fail") + return nil, err + } + } + + return txW.Bytes(), nil +} + // ExistsTxSha returns if the given tx sha exists in the database func (db *LevelDb) ExistsTxSha(txsha *btcwire.ShaHash) (exists bool) { db.dbLock.Lock() @@ -143,3 +248,176 @@ func (db *LevelDb) existsTxSha(txSha *btcwire.ShaHash) (exists bool) { return false } + +// FetchTxByShaList returns the most recent tx of the name fully spent or not +func (db *LevelDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { + // until the fully spent separation of tx is complete this is identical + // to FetchUnSpentTxByShaList + replies := make([]*btcdb.TxListReply, len(txShaList)) + for i, txsha := range txShaList { + tx, blockSha, height, txspent, err := db.fetchTxDataBySha(txsha) + btxspent := []bool{} + if err == nil { + btxspent = make([]bool, len(tx.TxOut), len(tx.TxOut)) + for idx := range tx.TxOut { + byteidx := idx / 8 + byteoff := uint(idx % 8) + btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0 + } + } + if err == btcdb.TxShaMissing { + // if the unspent pool did not have the tx, + // look in the fully spent pool (only last instance + + sTxList, fSerr := db.getTxFullySpent(txsha) + if fSerr == nil && len(sTxList) != 0 { + idx := len(sTxList) - 1 + stx := sTxList[idx] + + tx, blockSha, _, _, err = db.fetchTxDataByLoc( + stx.blkHeight, stx.txoff, stx.txlen, []byte{}) + if err == nil { + btxspent = make([]bool, len(tx.TxOut)) + for i := range btxspent { + btxspent[i] = true + } + } + } + } + txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, BlkSha: blockSha, Height: height, TxSpent: btxspent, Err: err} + replies[i] = &txlre + } + return replies +} + +// FetchUnSpentTxByShaList given a array of ShaHash, look up the transactions +// and return them in a TxListReply array. +func (db *LevelDb) FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + replies := make([]*btcdb.TxListReply, len(txShaList)) + for i, txsha := range txShaList { + tx, blockSha, height, txspent, err := db.fetchTxDataBySha(txsha) + btxspent := []bool{} + if err == nil { + btxspent = make([]bool, len(tx.TxOut), len(tx.TxOut)) + for idx := range tx.TxOut { + byteidx := idx / 8 + byteoff := uint(idx % 8) + btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0 + } + } + txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, BlkSha: blockSha, Height: height, TxSpent: btxspent, Err: err} + replies[i] = &txlre + } + return replies +} + +// fetchTxDataBySha returns several pieces of data regarding the given sha. +func (db *LevelDb) fetchTxDataBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rblksha *btcwire.ShaHash, rheight int64, rtxspent []byte, err error) { + var blkHeight int64 + var txspent []byte + var txOff, txLen int + + blkHeight, txOff, txLen, txspent, err = db.getTxData(txsha) + if err != nil { + if err == leveldb.ErrNotFound { + err = btcdb.TxShaMissing + } + return + } + return db.fetchTxDataByLoc(blkHeight, txOff, txLen, txspent) +} + +// fetchTxDataByLoc returns several pieces of data regarding the given tx +// located by the block/offset/size location +func (db *LevelDb) fetchTxDataByLoc(blkHeight int64, txOff int, txLen int, txspent []byte) (rtx *btcwire.MsgTx, rblksha *btcwire.ShaHash, rheight int64, rtxspent []byte, err error) { + var blksha *btcwire.ShaHash + var blkbuf []byte + + blksha, blkbuf, err = db.getBlkByHeight(blkHeight) + if err != nil { + if err == leveldb.ErrNotFound { + err = btcdb.TxShaMissing + } + return + } + + //log.Trace("transaction %v is at block %v %v txoff %v, txlen %v\n", + // txsha, blksha, blkHeight, txOff, txLen) + + rbuf := bytes.NewBuffer(blkbuf[txOff : txOff+txLen]) + + var tx btcwire.MsgTx + err = tx.Deserialize(rbuf) + if err != nil { + log.Warnf("unable to decode tx block %v %v txoff %v txlen %v", + blkHeight, blksha, txOff, txLen) + return + } + + return &tx, blksha, blkHeight, txspent, nil +} + +// FetchTxBySha returns some data for the given Tx Sha. +func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) ([]*btcdb.TxListReply, error) { + replylen := 0 + replycnt := 0 + + tx, blksha, height, txspent, txerr := db.fetchTxDataBySha(txsha) + if txerr == nil { + replylen++ + } else { + if txerr != btcdb.TxShaMissing { + return []*btcdb.TxListReply{}, txerr + } + replylen++ + } + + sTxList, fSerr := db.getTxFullySpent(txsha) + + if fSerr != nil { + if fSerr != btcdb.TxShaMissing { + return []*btcdb.TxListReply{}, fSerr + } + } else { + replylen += len(sTxList) + } + + replies := make ([]*btcdb.TxListReply, replylen) + + + if fSerr == nil { + for _, stx := range sTxList { + tx, blksha, _, _, err := db.fetchTxDataByLoc( + stx.blkHeight, stx.txoff, stx.txlen, []byte{}) + if err != nil { + if err != leveldb.ErrNotFound { + return []*btcdb.TxListReply{}, err + } + continue + } + btxspent := make([]bool, len(tx.TxOut), len(tx.TxOut)) + for i := range btxspent { + btxspent[i] = true + } + txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, BlkSha: blksha, Height: stx.blkHeight, TxSpent: btxspent, Err: nil} + replies[replycnt] = &txlre + replycnt++ + } + } + if txerr == nil { + btxspent := make([]bool, len(tx.TxOut), len(tx.TxOut)) + for idx := range tx.TxOut { + byteidx := idx / 8 + byteoff := uint(idx % 8) + btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0 + } + txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, BlkSha: blksha, Height: height, TxSpent: btxspent, Err: nil} + replies[replycnt] = &txlre + replycnt++ + } + return replies, nil +} + From e10cb732fd868c9c4528618d381999d0c6582c0d Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Thu, 7 Nov 2013 18:24:27 -0500 Subject: [PATCH 102/163] Fix. --- ldb/tx.go | 1 - 1 file changed, 1 deletion(-) diff --git a/ldb/tx.go b/ldb/tx.go index 06d30b1e..61883846 100644 --- a/ldb/tx.go +++ b/ldb/tx.go @@ -372,7 +372,6 @@ func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) ([]*btcdb.TxListReply, e if txerr != btcdb.TxShaMissing { return []*btcdb.TxListReply{}, txerr } - replylen++ } sTxList, fSerr := db.getTxFullySpent(txsha) From ec70b6d1b38186f6e43c41f03e546a6956f2ca81 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sat, 2 Nov 2013 02:52:57 -0500 Subject: [PATCH 103/163] Add comments which describe what is spent where. --- interface_test.go | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/interface_test.go b/interface_test.go index 907edf00..d11a3546 100644 --- a/interface_test.go +++ b/interface_test.go @@ -244,24 +244,40 @@ func expectedSpentBuf(tc *testContext, txNum int) []bool { spentBuf := make([]bool, numTxOut) if tc.useSpends { if tc.blockHeight == 9 && txNum == 0 { + // Spent by block 170, tx 1, input 0. + // tx f4184fc596403b9d638783cf57adfe4c75c605f6356fbc91338530e9831e9e16 spentBuf[0] = true } if tc.blockHeight == 170 && txNum == 1 { + // Spent by block 181, tx 1, input 0. + // tx a16f3ce4dd5deb92d98ef5cf8afeaf0775ebca408f708b2146c4fb42b41e14be spentBuf[1] = true } if tc.blockHeight == 181 && txNum == 1 { + // Spent by block 182, tx 1, input 0. + // tx 591e91f809d716912ca1d4a9295e70c3e78bab077683f79350f101da64588073 spentBuf[1] = true } if tc.blockHeight == 182 && txNum == 1 { + // Spent by block 221, tx 1, input 0. + // tx 298ca2045d174f8a158961806ffc4ef96fad02d71a6b84d9fa0491813a776160 spentBuf[0] = true + + // Spent by block 183, tx 1, input 0. + // tx 12b5633bad1f9c167d523ad1aa1947b2732a865bf5414eab2f9e5ae5d5c191ba spentBuf[1] = true } if tc.blockHeight == 183 && txNum == 1 { + // Spent by block 187, tx 1, input 0. + // tx 4385fcf8b14497d0659adccfe06ae7e38e0b5dc95ff8a13d7c62035994a0cd79 spentBuf[0] = true + + // Spent by block 248, tx 1, input 0. + // tx 828ef3b079f9c23829c56fe86e85b4a69d9e06e5b54ea597eef5fb3ffef509fe spentBuf[1] = true } } @@ -528,6 +544,15 @@ func testInterface(t *testing.T, dbType string) { // required for the duplicate transactions allowed by blocks 91842 and // 91880 on the main network due to the old miner + Satoshi client bug. + // TODO(davec): Add tests for error conditions: + // * Don't allow duplicate blocks + // * Don't allow insertion of block that contains a transaction that + // already exists unless the previous one is fully spent + // * Don't allow block that has a duplicate transaction in itself + // * Don't allow block which contains a tx that references a missing tx + // * Don't allow block which contains a tx that references another tx + // that comes after it in the same block + // TODO(davec): Add tests for the following functions: /* Close() @@ -541,9 +566,9 @@ func testInterface(t *testing.T, dbType string) { - FetchTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply - FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply - InsertBlock(block *btcutil.Block) (height int64, err error) - InvalidateBlockCache() - InvalidateCache() - InvalidateTxCache() + - InvalidateBlockCache() + - InvalidateCache() + - InvalidateTxCache() NewIterateBlocks() (pbi BlockIterator, err error) NewestSha() (sha *btcwire.ShaHash, height int64, err error) RollbackClose() From d2269684720afa0fa79f7b5f4c65e398304274c2 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sun, 13 Oct 2013 01:59:55 -0500 Subject: [PATCH 104/163] Add a new memory database backend named memdb. This commit adds a new backend driver which conforms to the btcdb interface to provide a memory only database. This is primarily useful for testing purposes as normal operations require a persistent block storage mechanism. --- common_test.go | 1 + memdb/doc.go | 12 + memdb/driver.go | 29 ++ memdb/memdb.go | 757 ++++++++++++++++++++++++++++++++++++++++++++ memdb/memdb_test.go | 112 +++++++ 5 files changed, 911 insertions(+) create mode 100644 memdb/doc.go create mode 100644 memdb/driver.go create mode 100644 memdb/memdb.go create mode 100644 memdb/memdb_test.go diff --git a/common_test.go b/common_test.go index a0c771c4..08330903 100644 --- a/common_test.go +++ b/common_test.go @@ -10,6 +10,7 @@ import ( "fmt" "github.com/conformal/btcdb" _ "github.com/conformal/btcdb/ldb" + _ "github.com/conformal/btcdb/memdb" "github.com/conformal/btcutil" "github.com/conformal/btcwire" "io" diff --git a/memdb/doc.go b/memdb/doc.go new file mode 100644 index 00000000..c2afb20e --- /dev/null +++ b/memdb/doc.go @@ -0,0 +1,12 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +/* +Package memdb implements an instance of btcdb that uses memory for the block +storage. + +This is primary used for testing purposes as normal operations require a +persistent block storage mechanism which this is not. +*/ +package memdb diff --git a/memdb/driver.go b/memdb/driver.go new file mode 100644 index 00000000..1304dc77 --- /dev/null +++ b/memdb/driver.go @@ -0,0 +1,29 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package memdb + +import ( + "github.com/conformal/btcdb" + "github.com/conformal/seelog" +) + +var log = seelog.Disabled + +func init() { + driver := btcdb.DriverDB{DbType: "memdb", Create: CreateDB, Open: OpenDB} + btcdb.AddDBDriver(driver) +} + +// OpenDB opens an existing database for use. +func OpenDB(dbpath string) (btcdb.Db, error) { + // A memory database is not persistent, so let CreateDB handle it. + return CreateDB(dbpath) +} + +// CreateDB creates, initializes, and opens a database for use. +func CreateDB(dbpath string) (btcdb.Db, error) { + log = btcdb.GetLog() + return newMemDb(), nil +} diff --git a/memdb/memdb.go b/memdb/memdb.go new file mode 100644 index 00000000..7f23e714 --- /dev/null +++ b/memdb/memdb.go @@ -0,0 +1,757 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package memdb + +import ( + "errors" + "fmt" + "github.com/conformal/btcdb" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" + "math" + "sync" +) + +// Errors that the various database functions may return. +var ( + ErrDbClosed = errors.New("Database is closed") +) + +var ( + zeroHash = btcwire.ShaHash{} + + // The following two hashes are ones that must be specially handled. + // See the comments where they're used for more details. + dupTxHash91842 = newShaHashFromStr("d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599") + dupTxHash91880 = newShaHashFromStr("e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468") +) + +// tTxInsertData holds information about the location and spent status of +// a transaction. +type tTxInsertData struct { + blockHeight int64 + offset int + spentBuf []bool +} + +// newShaHashFromStr converts the passed big-endian hex string into a +// btcwire.ShaHash. It only differs from the one available in btcwire in that +// it ignores the error since it will only (and must only) be called with +// hard-coded, and therefore known good, hashes. +func newShaHashFromStr(hexStr string) *btcwire.ShaHash { + sha, _ := btcwire.NewShaHashFromStr(hexStr) + return sha +} + +// isCoinbaseInput returns whether or not the passed transaction input is a +// coinbase input. A coinbase is a special transaction created by miners that +// has no inputs. This is represented in the block chain by a transaction with +// a single input that has a previous output transaction index set to the +// maximum value along with a zero hash. +func isCoinbaseInput(txIn *btcwire.TxIn) bool { + prevOut := &txIn.PreviousOutpoint + if prevOut.Index == math.MaxUint32 && prevOut.Hash.IsEqual(&zeroHash) { + return true + } + + return false +} + +// isFullySpent returns whether or not a transaction represented by the passed +// transaction insert data is fully spent. A fully spent transaction is one +// where all outputs are spent. +func isFullySpent(txD *tTxInsertData) bool { + for _, spent := range txD.spentBuf { + if !spent { + return false + } + } + + return true +} + +// MemDb is a concrete implementation of the btcdb.Db interface which provides +// a memory-only database. Since it is memory-only, it is obviously not +// persistent and is mostly only useful for testing purposes. +type MemDb struct { + // Embed a mutex for safe concurrent access. + sync.Mutex + + // blocks holds all of the bitcoin blocks that will be in the memory + // database. + blocks []*btcwire.MsgBlock + + // blocksBySha keeps track of block heights by hash. The height can + // be used as an index into the blocks slice. + blocksBySha map[btcwire.ShaHash]int64 + + // txns holds information about transactions such as which their + // block height and spent status of all their outputs. + txns map[btcwire.ShaHash][]*tTxInsertData + + // closed indicates whether or not the database has been closed and is + // therefore invalidated. + closed bool +} + +// removeTx removes the passed transaction including unspending it. +func (db *MemDb) removeTx(msgTx *btcwire.MsgTx, txHash *btcwire.ShaHash) { + // Undo all of the spends for the transaction. + for _, txIn := range msgTx.TxIn { + if isCoinbaseInput(txIn) { + continue + } + + prevOut := &txIn.PreviousOutpoint + originTxns, exists := db.txns[prevOut.Hash] + if !exists { + log.Warnf("Unable to find input transaction %s to "+ + "unspend %s index %d", prevOut.Hash, txHash, + prevOut.Index) + continue + } + + originTxD := originTxns[len(originTxns)-1] + originTxD.spentBuf[prevOut.Index] = false + } + + // Remove the info for the most recent version of the transaction. + txns := db.txns[*txHash] + lastIndex := len(txns) - 1 + txns[lastIndex] = nil + txns = txns[:lastIndex] + db.txns[*txHash] = txns + + // Remove the info entry from the map altogether if there not any older + // versions of the transaction. + if len(txns) == 0 { + delete(db.txns, *txHash) + } + +} + +// Close cleanly shuts down database. This is part of the btcdb.Db interface +// implementation. +// +// All data is purged upon close with this implementation since it is a +// memory-only database. +func (db *MemDb) Close() { + db.Lock() + defer db.Unlock() + + db.blocks = nil + db.blocksBySha = nil + db.txns = nil + db.closed = true +} + +// DropAfterBlockBySha removes any blocks from the database after the given +// block. This is different than a simple truncate since the spend information +// for each block must also be unwound. This is part of the btcdb.Db interface +// implementation. +func (db *MemDb) DropAfterBlockBySha(sha *btcwire.ShaHash) error { + db.Lock() + defer db.Unlock() + + if db.closed { + return ErrDbClosed + } + + // Begin by attempting to find the height associated with the passed + // hash. + height, exists := db.blocksBySha[*sha] + if !exists { + return fmt.Errorf("block %v does not exist in the database", + sha) + } + + // The spend information has to be undone in reverse order, so loop + // backwards from the last block through the block just after the passed + // block. While doing this unspend all transactions in each block and + // remove the block. + endHeight := int64(len(db.blocks) - 1) + for i := endHeight; i > height; i-- { + // Unspend and remove each transaction in reverse order because + // later transactions in a block can reference earlier ones. + transactions := db.blocks[i].Transactions + for j := len(transactions) - 1; j >= 0; j-- { + tx := transactions[j] + txHash, _ := tx.TxSha() + db.removeTx(tx, &txHash) + } + + db.blocks[i] = nil + db.blocks = db.blocks[:i] + } + + return nil +} + +// ExistsSha returns whether or not the given block hash is present in the +// database. This is part of the btcdb.Db interface implementation. +func (db *MemDb) ExistsSha(sha *btcwire.ShaHash) bool { + db.Lock() + defer db.Unlock() + + if db.closed { + log.Warnf("ExistsSha called after db close.") + return false + } + + if _, exists := db.blocksBySha[*sha]; exists { + return true + } + + return false +} + +// FetchBlockBySha returns a btcutil.Block. The implementation may cache the +// underlying data if desired. This is part of the btcdb.Db interface +// implementation. +// +// This implementation does not use any additional cache since the entire +// database is already in memory. +func (db *MemDb) FetchBlockBySha(sha *btcwire.ShaHash) (*btcutil.Block, error) { + db.Lock() + defer db.Unlock() + + if db.closed { + return nil, ErrDbClosed + } + + if blockHeight, exists := db.blocksBySha[*sha]; exists { + block := btcutil.NewBlock(db.blocks[int(blockHeight)]) + block.SetHeight(blockHeight) + return block, nil + } + + return nil, fmt.Errorf("block %v is not in database", sha) +} + +// FetchBlockShaByHeight returns a block hash based on its height in the block +// chain. This is part of the btcdb.Db interface implementation. +func (db *MemDb) FetchBlockShaByHeight(height int64) (*btcwire.ShaHash, error) { + db.Lock() + defer db.Unlock() + + if db.closed { + return nil, ErrDbClosed + } + + numBlocks := int64(len(db.blocks)) + if height < 0 || height > numBlocks-1 { + return nil, fmt.Errorf("unable to fetch block height %d since "+ + "it is not within the valid range (%d-%d)", height, 0, + numBlocks-1) + } + + msgBlock := db.blocks[height] + blockHash, err := msgBlock.BlockSha() + if err != nil { + return nil, err + } + + return &blockHash, nil +} + +// FetchHeightRange looks up a range of blocks by the start and ending heights. +// Fetch is inclusive of the start height and exclusive of the ending height. +// To fetch all hashes from the start height until no more are present, use the +// special id `AllShas'. This is part of the btcdb.Db interface implementation. +func (db *MemDb) FetchHeightRange(startHeight, endHeight int64) ([]btcwire.ShaHash, error) { + db.Lock() + defer db.Unlock() + + if db.closed { + return nil, ErrDbClosed + } + + // When the user passes the special AllShas id, adjust the end height + // accordingly. + if endHeight == btcdb.AllShas { + endHeight = int64(len(db.blocks)) + } + + // Ensure requested heights are sane. + if startHeight < 0 { + return nil, fmt.Errorf("start height of fetch range must not "+ + "be less than zero - got %d", startHeight) + } + if endHeight < startHeight { + return nil, fmt.Errorf("end height of fetch range must not "+ + "be less than the start height - got start %d, end %d", + startHeight, endHeight) + } + + // Fetch as many as are availalbe within the specified range. + lastBlockIndex := int64(len(db.blocks) - 1) + hashList := make([]btcwire.ShaHash, 0, endHeight-startHeight) + for i := startHeight; i < endHeight; i++ { + if i > lastBlockIndex { + break + } + + msgBlock := db.blocks[i] + blockHash, err := msgBlock.BlockSha() + if err != nil { + return nil, err + } + hashList = append(hashList, blockHash) + } + + return hashList, nil +} + +// ExistsTxSha returns whether or not the given transaction hash is present in +// the database and is not fully spent. This is part of the btcdb.Db interface +// implementation. +func (db *MemDb) ExistsTxSha(sha *btcwire.ShaHash) bool { + db.Lock() + defer db.Unlock() + + if db.closed { + log.Warnf("ExistsTxSha called after db close.") + return false + } + + if txns, exists := db.txns[*sha]; exists { + return !isFullySpent(txns[len(txns)-1]) + } + + return false +} + +// FetchTxBySha returns some data for the given transaction hash. The +// implementation may cache the underlying data if desired. This is part of the +// btcdb.Db interface implementation. +// +// This implementation does not use any additional cache since the entire +// database is already in memory. +func (db *MemDb) FetchTxBySha(txHash *btcwire.ShaHash) ([]*btcdb.TxListReply, error) { + db.Lock() + defer db.Unlock() + + if db.closed { + return nil, ErrDbClosed + } + + txns, exists := db.txns[*txHash] + if !exists { + log.Warnf("FetchTxBySha: requested hash of %s does not exist", + txHash) + return nil, btcdb.TxShaMissing + } + + txHashCopy := *txHash + replyList := make([]*btcdb.TxListReply, len(txns)) + for i, txD := range txns { + msgBlock := db.blocks[txD.blockHeight] + blockSha, err := msgBlock.BlockSha() + if err != nil { + return nil, err + } + + spentBuf := make([]bool, len(txD.spentBuf)) + copy(spentBuf, txD.spentBuf) + reply := btcdb.TxListReply{ + Sha: &txHashCopy, + Tx: msgBlock.Transactions[txD.offset], + BlkSha: &blockSha, + Height: txD.blockHeight, + TxSpent: spentBuf, + Err: nil, + } + replyList[i] = &reply + } + + return replyList, nil +} + +// fetchTxByShaList fetches transactions and information about them given an +// array of transaction hashes. The result is a slice of of TxListReply objects +// which contain the transaction and information about it such as what block and +// block height it's contained in and which outputs are spent. +// +// The includeSpent flag indicates whether or not information about transactions +// which are fully spent should be returned. When the flag is not set, the +// corresponding entry in the TxListReply slice for fully spent transactions +// will indicate the transaction does not exist. +// +// This function must be called with the db lock held. +func (db *MemDb) fetchTxByShaList(txShaList []*btcwire.ShaHash, includeSpent bool) []*btcdb.TxListReply { + replyList := make([]*btcdb.TxListReply, 0, len(txShaList)) + for i, hash := range txShaList { + // Every requested entry needs a response, so start with nothing + // more than a response with the requested hash marked missing. + // The reply will be updated below with the appropriate + // information if the transaction exists. + reply := btcdb.TxListReply{ + Sha: txShaList[i], + Err: btcdb.TxShaMissing, + } + replyList = append(replyList, &reply) + + if db.closed { + reply.Err = ErrDbClosed + continue + } + + if txns, exists := db.txns[*hash]; exists { + // A given transaction may have duplicates so long as the + // previous one is fully spent. We are only interested + // in the most recent version of the transaction for + // this function. The FetchTxBySha function can be + // used to get all versions of a transaction. + txD := txns[len(txns)-1] + if !includeSpent && isFullySpent(txD) { + continue + } + + // Look up the referenced block and get its hash. Set + // the reply error appropriately and go to the next + // requested transaction if anything goes wrong. + msgBlock := db.blocks[txD.blockHeight] + blockSha, err := msgBlock.BlockSha() + if err != nil { + reply.Err = err + continue + } + + // Make a copy of the spent buf to return so the caller + // can't accidentally modify it. + spentBuf := make([]bool, len(txD.spentBuf)) + copy(spentBuf, txD.spentBuf) + + // Populate the reply. + reply.Tx = msgBlock.Transactions[txD.offset] + reply.BlkSha = &blockSha + reply.Height = txD.blockHeight + reply.TxSpent = spentBuf + reply.Err = nil + } + } + + return replyList +} + +// FetchTxByShaList returns a TxListReply given an array of transaction hashes. +// The implementation may cache the underlying data if desired. This is part of +// the btcdb.Db interface implementation. +// +// This implementation does not use any additional cache since the entire +// database is already in memory. + +// FetchTxByShaList returns a TxListReply given an array of transaction +// hashes. This function differs from FetchUnSpentTxByShaList in that it +// returns the most recent version of fully spent transactions. Due to the +// increased number of transaction fetches, this function is typically more +// expensive than the unspent counterpart, however the specific performance +// details depend on the concrete implementation. The implementation may cache +// the underlying data if desired. This is part of the btcdb.Db interface +// implementation. +// +// To fetch all versions of a specific transaction, call FetchTxBySha. +// +// This implementation does not use any additional cache since the entire +// database is already in memory. +func (db *MemDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { + db.Lock() + defer db.Unlock() + + return db.fetchTxByShaList(txShaList, true) +} + +// FetchUnSpentTxByShaList returns a TxListReply given an array of transaction +// hashes. Any transactions which are fully spent will indicate they do not +// exist by setting the Err field to TxShaMissing. The implementation may cache +// the underlying data if desired. This is part of the btcdb.Db interface +// implementation. +// +// To obtain results which do contain the most recent version of a fully spent +// transactions, call FetchTxByShaList. To fetch all versions of a specific +// transaction, call FetchTxBySha. +// +// This implementation does not use any additional cache since the entire +// database is already in memory. +func (db *MemDb) FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { + db.Lock() + defer db.Unlock() + + return db.fetchTxByShaList(txShaList, false) +} + +// InsertBlock inserts raw block and transaction data from a block into the +// database. The first block inserted into the database will be treated as the +// genesis block. Every subsequent block insert requires the referenced parent +// block to already exist. This is part of the btcdb.Db interface +// implementation. +func (db *MemDb) InsertBlock(block *btcutil.Block) (int64, error) { + db.Lock() + defer db.Unlock() + + if db.closed { + return 0, ErrDbClosed + } + + blockHash, err := block.Sha() + if err != nil { + return 0, err + } + + // Reject the insert if the previously reference block does not exist + // except in the case there are no blocks inserted yet where the first + // inserted block is assumed to be a genesis block. + msgBlock := block.MsgBlock() + if _, exists := db.blocksBySha[msgBlock.Header.PrevBlock]; !exists { + if len(db.blocks) > 0 { + return 0, btcdb.PrevShaMissing + } + } + + // Build a map of in-flight transactions because some of the inputs in + // this block could be referencing other transactions earlier in this + // block which are not yet in the chain. + txInFlight := map[btcwire.ShaHash]int{} + transactions := block.Transactions() + for i, tx := range transactions { + txInFlight[*tx.Sha()] = i + } + + // Loop through all transactions and inputs to ensure there are no error + // conditions that would prevent them from be inserted into the db. + // Although these checks could could be done in the loop below, checking + // for error conditions up front means the code below doesn't have to + // deal with rollback on errors. + newHeight := int64(len(db.blocks)) + for i, tx := range transactions { + // Two old blocks contain duplicate transactions due to being + // mined by faulty miners and accepted by the origin Satoshi + // client. Rules have since been added to the ensure this + // problem can no longer happen, but the two duplicate + // transactions which were originally accepted are forever in + // the block chain history and must be dealth with specially. + // http://blockexplorer.com/b/91842 + // http://blockexplorer.com/b/91880 + if newHeight == 91842 && tx.Sha().IsEqual(dupTxHash91842) { + continue + } + + if newHeight == 91880 && tx.Sha().IsEqual(dupTxHash91880) { + continue + } + + for _, txIn := range tx.MsgTx().TxIn { + if isCoinbaseInput(txIn) { + continue + } + + // It is acceptable for a transaction input to reference + // the output of another transaction in this block only + // if the referenced transaction comes before the + // current one in this block. + prevOut := &txIn.PreviousOutpoint + if inFlightIndex, ok := txInFlight[prevOut.Hash]; ok { + if i <= inFlightIndex { + log.Warnf("InsertBlock: requested hash "+ + " of %s does not exist in-flight", + tx.Sha()) + return 0, btcdb.TxShaMissing + } + } else { + originTxns, exists := db.txns[prevOut.Hash] + if !exists { + log.Warnf("InsertBlock: requested hash "+ + "of %s by %s does not exist", + prevOut.Hash, tx.Sha()) + return 0, btcdb.TxShaMissing + } + originTxD := originTxns[len(originTxns)-1] + if prevOut.Index > uint32(len(originTxD.spentBuf)) { + log.Warnf("InsertBlock: requested hash "+ + "of %s with index %d does not "+ + "exist", tx.Sha(), prevOut.Index) + return 0, btcdb.TxShaMissing + } + } + } + + // Prevent duplicate transactions in the same block. + if inFlightIndex, exists := txInFlight[*tx.Sha()]; exists && + inFlightIndex < i { + log.Warnf("Block contains duplicate transaction %s", + tx.Sha()) + return 0, btcdb.DuplicateSha + } + + // Prevent duplicate transactions unless the old one is fully + // spent. + if txns, exists := db.txns[*tx.Sha()]; exists { + txD := txns[len(txns)-1] + if !isFullySpent(txD) { + log.Warnf("Attempt to insert duplicate "+ + "transaction %s", tx.Sha()) + return 0, btcdb.DuplicateSha + } + } + } + + db.blocks = append(db.blocks, msgBlock) + db.blocksBySha[*blockHash] = newHeight + + // Insert information about eacj transaction and spend all of the + // outputs referenced by the inputs to the transactions. + for i, tx := range block.Transactions() { + // Insert the transaction data. + txD := tTxInsertData{ + blockHeight: newHeight, + offset: i, + spentBuf: make([]bool, len(tx.MsgTx().TxOut)), + } + db.txns[*tx.Sha()] = append(db.txns[*tx.Sha()], &txD) + + // Spend all of the inputs. + for _, txIn := range tx.MsgTx().TxIn { + // Coinbase transaction has no inputs. + if isCoinbaseInput(txIn) { + continue + } + + // Already checked for existing and valid ranges above. + prevOut := &txIn.PreviousOutpoint + originTxns := db.txns[prevOut.Hash] + originTxD := originTxns[len(originTxns)-1] + originTxD.spentBuf[prevOut.Index] = true + } + } + + return newHeight, nil +} + +// InvalidateBlockCache releases all cached blocks. This is part of the +// btcdb.Db interface implementation. +// +// There is no need for a cache with this implementation since the entire +// database is already in memory As a result, this function doesn't do anything +// useful and is only provided to conform to the interface. +func (db *MemDb) InvalidateBlockCache() { + if db.closed { + log.Warnf("InvalidateBlockCache called after db close.") + } +} + +// InvalidateCache releases all cached blocks and transactions. This is part of +// the btcdb.Db interface implementation. +// +// There is no need for a cache with this implementation since the entire +// database is already in memory As a result, this function doesn't do anything +// useful and is only provided to conform to the interface. +func (db *MemDb) InvalidateCache() { + if db.closed { + log.Warnf("InvalidateCache called after db close.") + } +} + +// InvalidateTxCache releases all cached transactions. This is part of the +// btcdb.Db interface implementation. +// +// There is no need for a cache with this implementation since the entire +// database is already in memory As a result, this function doesn't do anything +// useful and is only provided to conform to the interface. +func (db *MemDb) InvalidateTxCache() { + if db.closed { + log.Warnf("InvalidateTxCache called after db close.") + } +} + +// NewIterateBlocks returns an iterator for all blocks in database. This is +// part of the btcdb.Db interface implementation. +// +// This implmentation does not implement an iterator, so an error is returned +// if this function is called. +func (db *MemDb) NewIterateBlocks() (btcdb.BlockIterator, error) { + return nil, fmt.Errorf("Not implemented") +} + +// NewestSha returns the hash and block height of the most recent (end) block of +// the block chain. It will return the zero hash, -1 for the block height, and +// no error (nil) if there are not any blocks in the database yet. This is part +// of the btcdb.Db interface implementation. +func (db *MemDb) NewestSha() (*btcwire.ShaHash, int64, error) { + db.Lock() + defer db.Unlock() + + if db.closed { + return nil, 0, ErrDbClosed + } + + // When the database has not had a genesis block inserted yet, return + // values specified by interface contract. + numBlocks := len(db.blocks) + if numBlocks == 0 { + return &zeroHash, -1, nil + } + + blockSha, err := db.blocks[numBlocks-1].BlockSha() + if err != nil { + return nil, -1, err + } + + return &blockSha, int64(numBlocks - 1), nil +} + +// RollbackClose discards the recent database changes to the previously saved +// data at last Sync and closes the database. This is part of the btcdb.Db +// interface implementation. +// +// The database is completely purged on close with this implementation since the +// entire database is only in memory. As a result, this function behaves no +// differently than Close. +func (db *MemDb) RollbackClose() { + // Rollback doesn't apply to a memory database, so just call Close. + // Close handles the mutex locks. + db.Close() +} + +// SetDBInsertMode provides hints to the database about how the application is +// running. This allows the database to work in optimized modes when the +// database may be very busy. This is part of the btcdb.Db interface +// implementation. +// +// No special mode handling is performed for this implementation. +func (db *MemDb) SetDBInsertMode(newmode btcdb.InsertMode) { + if db.closed { + log.Warnf("SetDBInsertMode called after db close.") + } +} + +// Sync verifies that the database is coherent on disk and no outstanding +// transactions are in flight. This is part of the btcdb.Db interface +// implementation. +// +// This implementation does not write any data to disk, so this function only +// grabs a lock to ensure it doesn't return until other operations are complete. +func (db *MemDb) Sync() { + db.Lock() + defer db.Unlock() + + if db.closed { + log.Warnf("Sync called after db close.") + } + + // There is nothing extra to do to sync the memory database. However, + // the lock is still grabbed to ensure the function does not return + // until other operations are complete. + return +} + +// newMemDb returns a new memory-only database ready for block inserts. +func newMemDb() *MemDb { + db := MemDb{ + blocks: make([]*btcwire.MsgBlock, 0, 200000), + blocksBySha: make(map[btcwire.ShaHash]int64), + txns: make(map[btcwire.ShaHash][]*tTxInsertData), + } + return &db +} diff --git a/memdb/memdb_test.go b/memdb/memdb_test.go new file mode 100644 index 00000000..ff4615c7 --- /dev/null +++ b/memdb/memdb_test.go @@ -0,0 +1,112 @@ +// Copyright (c) 2013 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package memdb_test + +import ( + "github.com/conformal/btcdb" + "github.com/conformal/btcdb/memdb" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" + "reflect" + "testing" +) + +// TestClosed ensure calling the interface functions on a closed database +// returns appropriate errors for the interface functions that return errors +// and does not panic or otherwise misbehave for functions which do not return +// errors. +func TestClosed(t *testing.T) { + db, err := btcdb.CreateDB("memdb", "") + if err != nil { + t.Errorf("Failed to open test database %v", err) + return + } + _, err = db.InsertBlock(btcutil.NewBlock(&btcwire.GenesisBlock)) + if err != nil { + t.Errorf("InsertBlock: %v", err) + } + db.Close() + + genesisHash := &btcwire.GenesisHash + if err := db.DropAfterBlockBySha(genesisHash); err != memdb.ErrDbClosed { + t.Errorf("DropAfterBlockBySha: unexpected error %v", err) + } + + if exists := db.ExistsSha(genesisHash); exists != false { + t.Errorf("ExistsSha: genesis hash exists after close") + } + + if _, err := db.FetchBlockBySha(genesisHash); err != memdb.ErrDbClosed { + t.Errorf("FetchBlockBySha: unexpected error %v", err) + } + + if _, err := db.FetchBlockShaByHeight(0); err != memdb.ErrDbClosed { + t.Errorf("FetchBlockShaByHeight: unexpected error %v", err) + } + + if _, err := db.FetchHeightRange(0, 1); err != memdb.ErrDbClosed { + t.Errorf("FetchHeightRange: unexpected error %v", err) + } + + genesisMerkleRoot := &btcwire.GenesisMerkleRoot + if exists := db.ExistsTxSha(genesisMerkleRoot); exists != false { + t.Errorf("ExistsTxSha: hash %v exists when it shouldn't", + genesisMerkleRoot) + } + + if _, err := db.FetchTxBySha(genesisHash); err != memdb.ErrDbClosed { + t.Errorf("FetchTxBySha: unexpected error %v", err) + } + + requestHashes := []*btcwire.ShaHash{genesisHash} + reply := db.FetchTxByShaList(requestHashes) + if len(reply) != len(requestHashes) { + t.Errorf("FetchUnSpentTxByShaList unexpected number of replies "+ + "got: %d, want: %d", len(reply), len(requestHashes)) + } + for i, txLR := range reply { + wantReply := &btcdb.TxListReply{ + Sha: requestHashes[i], + Err: memdb.ErrDbClosed, + } + if !reflect.DeepEqual(wantReply, txLR) { + t.Errorf("FetchTxByShaList unexpected reply\ngot: %v\n"+ + "want: %v", txLR, wantReply) + } + } + + reply = db.FetchUnSpentTxByShaList(requestHashes) + if len(reply) != len(requestHashes) { + t.Errorf("FetchUnSpentTxByShaList unexpected number of replies "+ + "got: %d, want: %d", len(reply), len(requestHashes)) + } + for i, txLR := range reply { + wantReply := &btcdb.TxListReply{ + Sha: requestHashes[i], + Err: memdb.ErrDbClosed, + } + if !reflect.DeepEqual(wantReply, txLR) { + t.Errorf("FetchUnSpentTxByShaList unexpected reply\n"+ + "got: %v\nwant: %v", txLR, wantReply) + } + } + + if _, _, err := db.NewestSha(); err != memdb.ErrDbClosed { + t.Errorf("NewestSha: unexpected error %v", err) + } + + // The following calls don't return errors from the interface to be able + // to detect a closed database, so just call them to ensure there are no + // panics. + db.InvalidateBlockCache() + db.InvalidateCache() + db.InvalidateTxCache() + db.NewIterateBlocks() + db.SetDBInsertMode(btcdb.InsertNormal) + db.SetDBInsertMode(btcdb.InsertFast) + db.SetDBInsertMode(btcdb.InsertValidatedInput) + db.Sync() + db.RollbackClose() +} From 3bc401fed1524baea7022ee4184eba9831e3dea1 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Thu, 21 Nov 2013 09:48:16 -0600 Subject: [PATCH 105/163] Use btclog for logging. --- ldb/leveldb.go | 4 ++-- log.go | 25 +++++++++++++++---------- memdb/driver.go | 4 ++-- sqlite3/sqlite.go | 4 ++-- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 063f115b..0f88ed5a 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -8,12 +8,12 @@ import ( "encoding/binary" "fmt" "github.com/conformal/btcdb" + "github.com/conformal/btclog" "github.com/conformal/btcutil" "github.com/conformal/btcwire" "github.com/conformal/goleveldb/leveldb" "github.com/conformal/goleveldb/leveldb/cache" "github.com/conformal/goleveldb/leveldb/opt" - "github.com/conformal/seelog" "os" "sync" ) @@ -24,7 +24,7 @@ const ( dbMaxTransMem = 64 * 1024 * 1024 // 64 MB ) -var log seelog.LoggerInterface = seelog.Disabled +var log = btclog.Disabled type tTxInsertData struct { txsha *btcwire.ShaHash diff --git a/log.go b/log.go index 667e0bcb..b6baf052 100644 --- a/log.go +++ b/log.go @@ -6,14 +6,14 @@ package btcdb import ( "errors" - "github.com/conformal/seelog" + "github.com/conformal/btclog" "io" ) // log is a logger that is initialized with no output filters. This // means the package will not perform any logging by default until the caller // requests it. -var log seelog.LoggerInterface +var log btclog.Logger // The default amount of logging is none. func init() { @@ -21,28 +21,33 @@ func init() { } // DisableLog disables all library log output. Logging output is disabled -// by default until either UserLogger or SetLogWriter are called. +// by default until either UseLogger or SetLogWriter are called. func DisableLog() { - log = seelog.Disabled + log = btclog.Disabled } // UseLogger uses a specified Logger to output package logging info. // This should be used in preference to SetLogWriter if the caller is also -// using seelog. -func UseLogger(logger seelog.LoggerInterface) { +// using btclog. +func UseLogger(logger btclog.Logger) { log = logger } // SetLogWriter uses a specified io.Writer to output package logging info. // This allows a caller to direct package logging output without needing a -// dependency on seelog. If the caller is also using seelog, UseLogger should +// dependency on seelog. If the caller is also using btclog, UseLogger should // be used instead. -func SetLogWriter(w io.Writer) error { +func SetLogWriter(w io.Writer, level string) error { if w == nil { return errors.New("nil writer") } - l, err := seelog.LoggerFromWriterWithMinLevel(w, seelog.TraceLvl) + lvl, ok := btclog.LogLevelFromString(level) + if !ok { + return errors.New("invalid log level") + } + + l, err := btclog.NewLoggerFromWriter(w, lvl) if err != nil { return err } @@ -52,6 +57,6 @@ func SetLogWriter(w io.Writer) error { } // GetLog returns the currently active logger. -func GetLog() seelog.LoggerInterface { +func GetLog() btclog.Logger { return log } diff --git a/memdb/driver.go b/memdb/driver.go index 1304dc77..e225e687 100644 --- a/memdb/driver.go +++ b/memdb/driver.go @@ -6,10 +6,10 @@ package memdb import ( "github.com/conformal/btcdb" - "github.com/conformal/seelog" + "github.com/conformal/btclog" ) -var log = seelog.Disabled +var log = btclog.Disabled func init() { driver := btcdb.DriverDB{DbType: "memdb", Create: CreateDB, Open: OpenDB} diff --git a/sqlite3/sqlite.go b/sqlite3/sqlite.go index 364c98d4..0cb3fc61 100644 --- a/sqlite3/sqlite.go +++ b/sqlite3/sqlite.go @@ -8,9 +8,9 @@ import ( "database/sql" "fmt" "github.com/conformal/btcdb" + "github.com/conformal/btclog" "github.com/conformal/btcutil" "github.com/conformal/btcwire" - "github.com/conformal/seelog" _ "github.com/mattn/go-sqlite3" "os" "sync" @@ -88,7 +88,7 @@ var txqueries []string = []string{ txtmpExistsShaStmt: "SELECT blockid FROM txtmp WHERE key = ?;", } -var log seelog.LoggerInterface = seelog.Disabled +var log = btclog.Disabled type tBlockInsertData struct { sha btcwire.ShaHash From fc11da9ca07e9e80e33191e1b95e241402b59669 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Thu, 21 Nov 2013 09:48:57 -0600 Subject: [PATCH 106/163] Gofmt. --- ldb/dbcache.go | 2 +- ldb/dup_test.go | 13 ++++++------- ldb/leveldb.go | 30 +++++++++++++++--------------- ldb/tx.go | 14 ++++++-------- 4 files changed, 28 insertions(+), 31 deletions(-) diff --git a/ldb/dbcache.go b/ldb/dbcache.go index 6f9e3883..24fb2856 100644 --- a/ldb/dbcache.go +++ b/ldb/dbcache.go @@ -5,7 +5,7 @@ package ldb import ( - //"fmt" +//"fmt" ) // InvalidateTxCache clear/release all cached transactions. diff --git a/ldb/dup_test.go b/ldb/dup_test.go index 4fa8a6f4..e1621408 100644 --- a/ldb/dup_test.go +++ b/ldb/dup_test.go @@ -23,7 +23,7 @@ func Test_dupTx(t *testing.T) { _ = os.RemoveAll(dbnamever) db, err := btcdb.CreateDB("leveldb", dbname) if err != nil { - t.Errorf("Failed to open test database %v", err) + t.Errorf("Failed to open test database %v", err) return } defer os.RemoveAll(dbname) @@ -75,7 +75,6 @@ out: } } - newheight, err := db.InsertBlock(block) if err != nil { t.Errorf("failed to insert block %v err %v", height, err) @@ -116,7 +115,7 @@ out: hash, _ := btcwire.NewShaHashFromStr("df2b060fa2e5e9c8ed5eaf6a45c13753ec8c63282b2688322eba40cd98ea067a") - po := btcwire.NewOutPoint(hash, 0) + po := btcwire.NewOutPoint(hash, 0) txI := btcwire.NewTxIn(po, []byte("garbage")) txO := btcwire.NewTxOut(50000000, []byte("garbageout")) @@ -133,7 +132,7 @@ out: for _, lr := range listReply { if lr.Err != nil { t.Errorf("sha %v spent %v err %v\n", lr.Sha, - lr.TxSpent, lr.Err) + lr.TxSpent, lr.Err) } } @@ -148,7 +147,7 @@ out: for _, lr := range listReply { if lr.Err != btcdb.TxShaMissing { t.Errorf("sha %v spent %v err %v\n", lr.Sha, - lr.TxSpent, lr.Err) + lr.TxSpent, lr.Err) } } @@ -160,8 +159,8 @@ out: } else { for _, lr := range txReply { if lr.Err != nil { - fmt.Errorf("stx %v spent %v err %v\n", lr.Sha, - lr.TxSpent, lr.Err) + fmt.Errorf("stx %v spent %v err %v\n", lr.Sha, + lr.TxSpent, lr.Err) } } } diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 0f88ed5a..ea7f882a 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -51,7 +51,7 @@ type LevelDb struct { lastBlkSha btcwire.ShaHash lastBlkIdx int64 - txUpdateMap map[btcwire.ShaHash]*txUpdateObj + txUpdateMap map[btcwire.ShaHash]*txUpdateObj txSpentUpdateMap map[btcwire.ShaHash]*spentTxUpdate } @@ -379,10 +379,10 @@ func (db *LevelDb) InsertBlock(block *btcutil.Block) (height int64, rerr error) } if txsha.IsEqual(dupsha) { // marking TxOut[0] as spent - po := btcwire.NewOutPoint(dupsha, 0) + po := btcwire.NewOutPoint(dupsha, 0) txI := btcwire.NewTxIn(po, []byte("garbage")) - var spendtx btcwire.MsgTx + var spendtx btcwire.MsgTx spendtx.AddTxIn(txI) err = db.doSpend(&spendtx) if err != nil { @@ -397,10 +397,10 @@ func (db *LevelDb) InsertBlock(block *btcutil.Block) (height int64, rerr error) } if txsha.IsEqual(dupsha) { // marking TxOut[0] as spent - po := btcwire.NewOutPoint(dupsha, 0) + po := btcwire.NewOutPoint(dupsha, 0) txI := btcwire.NewTxIn(po, []byte("garbage")) - var spendtx btcwire.MsgTx + var spendtx btcwire.MsgTx spendtx.AddTxIn(txI) err = db.doSpend(&spendtx) if err != nil { @@ -501,20 +501,20 @@ func (db *LevelDb) setclearSpentData(txsha *btcwire.ShaHash, idx uint32, set boo } // need to reslice the list to exclude the most recent. - sTx := spentTxList [len(spentTxList) -1] - spentTxList [len(spentTxList) -1] = nil - if len (spentTxList) == 1 { + sTx := spentTxList[len(spentTxList)-1] + spentTxList[len(spentTxList)-1] = nil + if len(spentTxList) == 1 { // write entry to delete tx from spent pool // XXX } else { - spentTxList = spentTxList [:len(spentTxList)-1] + spentTxList = spentTxList[:len(spentTxList)-1] // XXX format sTxList and set update Table } // Create 'new' Tx update data. blkHeight = sTx.blkHeight txOff = sTx.txoff - txLen = sTx.txlen + txLen = sTx.txlen spentbuflen := (sTx.numTxO + 7) / 8 spentData = make([]byte, spentbuflen, spentbuflen) for i := range spentData { @@ -542,7 +542,7 @@ func (db *LevelDb) setclearSpentData(txsha *btcwire.ShaHash, idx uint32, set boo // check for fully spent Tx fullySpent := true - for _, val := range txUo.spentData { + for _, val := range txUo.spentData { if val != ^byte(0) { fullySpent = false break @@ -551,11 +551,11 @@ func (db *LevelDb) setclearSpentData(txsha *btcwire.ShaHash, idx uint32, set boo if fullySpent { var txSu *spentTxUpdate // Look up Tx in fully spent table - if txSuOld, ok := db.txSpentUpdateMap[*txsha] ; ok { + if txSuOld, ok := db.txSpentUpdateMap[*txsha]; ok { txSu = txSuOld } else { var txSuStore spentTxUpdate - txSu = &txSuStore + txSu = &txSuStore txSuOld, err := db.getTxFullySpent(txsha) if err == nil { @@ -568,8 +568,8 @@ func (db *LevelDb) setclearSpentData(txsha *btcwire.ShaHash, idx uint32, set boo sTx.blkHeight = txUo.blkHeight sTx.txoff = txUo.txoff sTx.txlen = txUo.txlen - // XXX -- there is no way to comput the real TxOut - // from the spent array. + // XXX -- there is no way to comput the real TxOut + // from the spent array. sTx.numTxO = 8 * len(txUo.spentData) // append this txdata to fully spent txlist diff --git a/ldb/tx.go b/ldb/tx.go index 61883846..cf99a6a5 100644 --- a/ldb/tx.go +++ b/ldb/tx.go @@ -9,8 +9,8 @@ import ( "encoding/binary" "fmt" "github.com/conformal/btcdb" - "github.com/conformal/goleveldb/leveldb" "github.com/conformal/btcwire" + "github.com/conformal/goleveldb/leveldb" ) type txUpdateObj struct { @@ -31,7 +31,7 @@ type spentTx struct { delete bool } type spentTxUpdate struct { - txl []*spentTx + txl []*spentTx delete bool } @@ -154,7 +154,7 @@ func (db *LevelDb) getTxFullySpent(txsha *btcwire.ShaHash) ([]*spentTx, error) { for i := range spentTxList { var sTx spentTx - var blkHeight int64 + var blkHeight int64 var txOff, txLen, numTxO int32 err := binary.Read(txR, binary.LittleEndian, &blkHeight) @@ -275,7 +275,7 @@ func (db *LevelDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxLis stx := sTxList[idx] tx, blockSha, _, _, err = db.fetchTxDataByLoc( - stx.blkHeight, stx.txoff, stx.txlen, []byte{}) + stx.blkHeight, stx.txoff, stx.txlen, []byte{}) if err == nil { btxspent = make([]bool, len(tx.TxOut)) for i := range btxspent { @@ -384,13 +384,12 @@ func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) ([]*btcdb.TxListReply, e replylen += len(sTxList) } - replies := make ([]*btcdb.TxListReply, replylen) - + replies := make([]*btcdb.TxListReply, replylen) if fSerr == nil { for _, stx := range sTxList { tx, blksha, _, _, err := db.fetchTxDataByLoc( - stx.blkHeight, stx.txoff, stx.txlen, []byte{}) + stx.blkHeight, stx.txoff, stx.txlen, []byte{}) if err != nil { if err != leveldb.ErrNotFound { return []*btcdb.TxListReply{}, err @@ -419,4 +418,3 @@ func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) ([]*btcdb.TxListReply, e } return replies, nil } - From ca502abbf25575873c751b1434060c052b049b99 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 26 Nov 2013 15:58:13 -0500 Subject: [PATCH 107/163] Two fixes fix returning error on insertTx and doSpend. clear the batch on failures as well. --- ldb/leveldb.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ldb/leveldb.go b/ldb/leveldb.go index ea7f882a..442b4770 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -314,6 +314,8 @@ func (db *LevelDb) InsertBlock(block *btcutil.Block) (height int64, rerr error) defer func() { if rerr == nil { rerr = db.processBatches() + } else { + db.lBatch().Reset() } }() @@ -363,8 +365,7 @@ func (db *LevelDb) InsertBlock(block *btcutil.Block) (height int64, rerr error) err = db.insertTx(txsha, newheight, txloc[txidx].TxStart, txloc[txidx].TxLen, spentbuf) if err != nil { log.Warnf("block %v idx %v failed to insert tx %v %v err %v", blocksha, newheight, &txsha, txidx, err) - - return + return 0, err } // Some old blocks contain duplicate transactions @@ -412,8 +413,7 @@ func (db *LevelDb) InsertBlock(block *btcutil.Block) (height int64, rerr error) err = db.doSpend(tx) if err != nil { log.Warnf("block %v idx %v failed to spend tx %v %v err %v", blocksha, newheight, txsha, txidx, err) - - return + return 0, err } } return newheight, nil @@ -626,6 +626,8 @@ func (db *LevelDb) processBatches() error { db.lbatch = new(leveldb.Batch) } + defer db.lbatch.Reset() + for txSha, txU := range db.txUpdateMap { key := shaTxToKey(&txSha) if txU.delete { @@ -660,7 +662,6 @@ func (db *LevelDb) processBatches() error { log.Tracef("batch failed %v\n", err) return err } - db.lbatch.Reset() db.txUpdateMap = map[btcwire.ShaHash]*txUpdateObj{} db.txSpentUpdateMap = make(map[btcwire.ShaHash]*spentTxUpdate) } From 01b6ad719640c31b7231b5a698838c263be96622 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 26 Nov 2013 16:42:55 -0500 Subject: [PATCH 108/163] Reset batch on error --- ldb/leveldb.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 442b4770..4acecd17 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -259,7 +259,8 @@ func (db *LevelDb) DropAfterBlockBySha(sha *btcwire.ShaHash) (rerr error) { defer func() { if rerr == nil { rerr = db.processBatches() - + } else { + db.lBatch().Reset() } }() From 472c998c0d761fa29ef2e05ddfa2460e943033a6 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Tue, 10 Dec 2013 19:17:44 -0600 Subject: [PATCH 109/163] Add support for TravisCI. Also add TravisCI build status badge to README.md. --- .travis.yml | 3 +++ README.md | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..ae71c02f --- /dev/null +++ b/.travis.yml @@ -0,0 +1,3 @@ +language: go +go: release +install: go get -d -t -v ./... diff --git a/README.md b/README.md index 565f3f18..987e41ec 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ btcdb ===== +[![Build Status](https://travis-ci.org/conformal/btcdb.png?branch=master)] +(https://travis-ci.org/conformal/btcdb) + Package btcdb provides a database interface for the bitcoin block chain and transactions. There is a test suite with a high percentage of code coverage. See `test_coverage.txt` for the current coverage (using gocov). Alternatively, From 6b18796af7499df3237ab561bf878994aa655016 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Wed, 8 Jan 2014 23:54:52 -0600 Subject: [PATCH 110/163] Add 2014 to copyright dates. --- LICENSE | 2 +- common_test.go | 2 +- db.go | 2 +- db_test.go | 2 +- doc.go | 2 +- interface_test.go | 2 +- ldb/block.go | 2 +- ldb/boundary_test.go | 2 +- ldb/dbcache.go | 2 +- ldb/doc.go | 2 +- ldb/dup_test.go | 2 +- ldb/insertremove_test.go | 2 +- ldb/internal_test.go | 2 +- ldb/leveldb.go | 2 +- ldb/operational_test.go | 2 +- ldb/tx.go | 2 +- log.go | 2 +- memdb/doc.go | 2 +- memdb/driver.go | 2 +- memdb/memdb.go | 2 +- memdb/memdb_test.go | 2 +- sqlite3/doc.go | 2 +- sqlite3/insertfail_test.go | 2 +- sqlite3/insertremove_test.go | 2 +- sqlite3/internal_test.go | 2 +- sqlite3/operational_test.go | 2 +- sqlite3/sqlite.go | 2 +- sqlite3/sqliteblock.go | 2 +- sqlite3/sqlitedbcache.go | 2 +- sqlite3/sqlitetx.go | 2 +- 30 files changed, 30 insertions(+), 30 deletions(-) diff --git a/LICENSE b/LICENSE index 0d760cbb..992dd50d 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2013 Conformal Systems LLC. +Copyright (c) 2013-2014 Conformal Systems LLC. Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above diff --git a/common_test.go b/common_test.go index 08330903..31653445 100644 --- a/common_test.go +++ b/common_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/db.go b/db.go index 3f6d418c..a1144ec0 100644 --- a/db.go +++ b/db.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/db_test.go b/db_test.go index 32dcd3ae..74a3b89b 100644 --- a/db_test.go +++ b/db_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/doc.go b/doc.go index a2c60e3e..eb5e426d 100644 --- a/doc.go +++ b/doc.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/interface_test.go b/interface_test.go index d11a3546..a19e9978 100644 --- a/interface_test.go +++ b/interface_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/ldb/block.go b/ldb/block.go index b8b467c3..bcfa0696 100644 --- a/ldb/block.go +++ b/ldb/block.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/ldb/boundary_test.go b/ldb/boundary_test.go index 9d5c15f3..5a722441 100644 --- a/ldb/boundary_test.go +++ b/ldb/boundary_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/ldb/dbcache.go b/ldb/dbcache.go index 24fb2856..80298ee9 100644 --- a/ldb/dbcache.go +++ b/ldb/dbcache.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/ldb/doc.go b/ldb/doc.go index 9899a4c6..df99938b 100644 --- a/ldb/doc.go +++ b/ldb/doc.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/ldb/dup_test.go b/ldb/dup_test.go index e1621408..e22fc7f3 100644 --- a/ldb/dup_test.go +++ b/ldb/dup_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/ldb/insertremove_test.go b/ldb/insertremove_test.go index 66cf48ce..20a2b2d0 100644 --- a/ldb/insertremove_test.go +++ b/ldb/insertremove_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/ldb/internal_test.go b/ldb/internal_test.go index 3d6cabf3..12c61fb2 100644 --- a/ldb/internal_test.go +++ b/ldb/internal_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 4acecd17..fce55b2b 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/ldb/operational_test.go b/ldb/operational_test.go index 89fb1425..8db333e7 100644 --- a/ldb/operational_test.go +++ b/ldb/operational_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/ldb/tx.go b/ldb/tx.go index cf99a6a5..d9ff9a60 100644 --- a/ldb/tx.go +++ b/ldb/tx.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/log.go b/log.go index b6baf052..8aa02ec4 100644 --- a/log.go +++ b/log.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/memdb/doc.go b/memdb/doc.go index c2afb20e..985aeb60 100644 --- a/memdb/doc.go +++ b/memdb/doc.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/memdb/driver.go b/memdb/driver.go index e225e687..53e72c26 100644 --- a/memdb/driver.go +++ b/memdb/driver.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/memdb/memdb.go b/memdb/memdb.go index 7f23e714..4b805627 100644 --- a/memdb/memdb.go +++ b/memdb/memdb.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/memdb/memdb_test.go b/memdb/memdb_test.go index ff4615c7..f078bbb5 100644 --- a/memdb/memdb_test.go +++ b/memdb/memdb_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/sqlite3/doc.go b/sqlite3/doc.go index f9ade3b6..30d2dd90 100644 --- a/sqlite3/doc.go +++ b/sqlite3/doc.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/sqlite3/insertfail_test.go b/sqlite3/insertfail_test.go index 391c3146..6fe54153 100644 --- a/sqlite3/insertfail_test.go +++ b/sqlite3/insertfail_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/sqlite3/insertremove_test.go b/sqlite3/insertremove_test.go index 4b0f586e..34d564bf 100644 --- a/sqlite3/insertremove_test.go +++ b/sqlite3/insertremove_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/sqlite3/internal_test.go b/sqlite3/internal_test.go index 64e9f0dd..81728223 100644 --- a/sqlite3/internal_test.go +++ b/sqlite3/internal_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/sqlite3/operational_test.go b/sqlite3/operational_test.go index da73a962..e2f35b83 100644 --- a/sqlite3/operational_test.go +++ b/sqlite3/operational_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/sqlite3/sqlite.go b/sqlite3/sqlite.go index 0cb3fc61..813be68f 100644 --- a/sqlite3/sqlite.go +++ b/sqlite3/sqlite.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/sqlite3/sqliteblock.go b/sqlite3/sqliteblock.go index 0be4bcd9..f228dc70 100644 --- a/sqlite3/sqliteblock.go +++ b/sqlite3/sqliteblock.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/sqlite3/sqlitedbcache.go b/sqlite3/sqlitedbcache.go index 3caef680..9b87b498 100644 --- a/sqlite3/sqlitedbcache.go +++ b/sqlite3/sqlitedbcache.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. diff --git a/sqlite3/sqlitetx.go b/sqlite3/sqlitetx.go index 2e4ccaa3..d5963e19 100644 --- a/sqlite3/sqlitetx.go +++ b/sqlite3/sqlitetx.go @@ -1,4 +1,4 @@ -// Copyright (c) 2013 Conformal Systems LLC. +// Copyright (c) 2013-2014 Conformal Systems LLC. // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. From 6578e7345f7dafe775b56f0d2db1e08f5cd0e328 Mon Sep 17 00:00:00 2001 From: Marco Peereboom Date: Mon, 6 Jan 2014 16:02:26 -0600 Subject: [PATCH 111/163] catch up to new goleveldb --- ldb/leveldb.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ldb/leveldb.go b/ldb/leveldb.go index fce55b2b..813adc89 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -65,7 +65,7 @@ func init() { func OpenDB(dbpath string) (btcdb.Db, error) { log = btcdb.GetLog() - db, err := openDB(dbpath, 0) + db, err := openDB(dbpath, false) if err != nil { return nil, err } @@ -127,7 +127,7 @@ blocknarrow: var CurrentDBVersion int32 = 1 -func openDB(dbpath string, flag opt.OptionsFlag) (pbdb btcdb.Db, err error) { +func openDB(dbpath string, create bool) (pbdb btcdb.Db, err error) { var db LevelDb var tlDb *leveldb.DB var dbversion int32 @@ -143,7 +143,7 @@ func openDB(dbpath string, flag opt.OptionsFlag) (pbdb btcdb.Db, err error) { } }() - if flag&opt.OFCreateIfMissing == opt.OFCreateIfMissing { + if create == true { err = os.Mkdir(dbpath, 0750) if err != nil { log.Errorf("mkdir failed %v %v", dbpath, err) @@ -168,22 +168,22 @@ func openDB(dbpath string, flag opt.OptionsFlag) (pbdb btcdb.Db, err error) { dbversion = ^0 } } else { - if flag&opt.OFCreateIfMissing != 0 { + if create == true { needVersionFile = true dbversion = CurrentDBVersion } } myCache := cache.NewEmptyCache() - opts := &opt.Options{Flag: flag, - BlockCache: myCache, - MaxOpenFiles: 256, - CompressionType: opt.NoCompression, + opts := &opt.Options{ + BlockCache: myCache, + MaxOpenFiles: 256, + Compression: opt.NoCompression, } switch dbversion { case 0: - opts = &opt.Options{Flag: flag} + opts = &opt.Options{} case 1: // uses defaults from above default: @@ -220,7 +220,7 @@ func CreateDB(dbpath string) (btcdb.Db, error) { log = btcdb.GetLog() // No special setup needed, just OpenBB - db, err := openDB(dbpath, opt.OFCreateIfMissing) + db, err := openDB(dbpath, true) if err == nil { ldb := db.(*LevelDb) ldb.lastBlkIdx = -1 From 0696c757fa68f0426fcfec9b285adc5066672dbe Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sat, 18 Jan 2014 21:06:10 -0600 Subject: [PATCH 112/163] Update ldb/dbtest to latest goleveldb API. --- ldb/dbtest/dbtst.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ldb/dbtest/dbtst.go b/ldb/dbtest/dbtst.go index 7fccb0cd..11a34c0b 100644 --- a/ldb/dbtest/dbtst.go +++ b/ldb/dbtest/dbtst.go @@ -25,8 +25,9 @@ func main() { ro := &opt.ReadOptions{} wo := &opt.WriteOptions{} + opts := &opt.Options{} - ldb, err := leveldb.OpenFile("dbfile", &opt.Options{Flag: opt.OFCreateIfMissing}) + ldb, err := leveldb.OpenFile("dbfile", opts) if err != nil { fmt.Printf("db open failed %v\n", err) return From 27bc18ba2efc9d331c446c05d372511897cc9bc5 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sun, 19 Jan 2014 02:35:41 -0600 Subject: [PATCH 113/163] Switch test coverage tool to new 'go tool cover'. Also, make the coverage report test ldb and memdb instead of just sqlite3. --- cov_report.sh | 23 +++---- test_coverage.txt | 157 ++++++++++++++++++++++++++++++---------------- 2 files changed, 112 insertions(+), 68 deletions(-) diff --git a/cov_report.sh b/cov_report.sh index 860fbc0d..f384b207 100644 --- a/cov_report.sh +++ b/cov_report.sh @@ -1,17 +1,10 @@ #!/bin/sh -# This script uses gocov to generate a test coverage report. -# The gocov tool my be obtained with the following command: -# go get github.com/axw/gocov/gocov -# -# It will be installed to $GOPATH/bin, so ensure that location is in your $PATH. - -# Check for gocov. -type gocov >/dev/null 2>&1 -if [ $? -ne 0 ]; then - echo >&2 "This script requires the gocov tool." - echo >&2 "You may obtain it with the following command:" - echo >&2 "go get github.com/axw/gocov/gocov" - exit 1 -fi -(cd sqlite3 && gocov test | gocov report) +# This script uses go tool cover to generate a test coverage report. +go test -coverprofile=cov.out && go tool cover -func=cov.out && rm -f cov.out +echo "============================================================" +(cd ldb && go test -coverprofile=cov.out && go tool cover -func=cov.out && \ + rm -f cov.out) +echo "============================================================" +(cd memdb && go test -coverprofile=cov.out && go tool cover -func=cov.out && \ + rm -f cov.out) diff --git a/test_coverage.txt b/test_coverage.txt index fb0a17c5..d6ad3ca1 100644 --- a/test_coverage.txt +++ b/test_coverage.txt @@ -1,53 +1,104 @@ - -github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.NewestSha 100.00% (25/25) -github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.endTx 100.00% (20/20) -github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.close 100.00% (9/9) -github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.txop 100.00% (7/7) -github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteBlockIterator.Row 100.00% (7/7) -github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.fetchTxCache 100.00% (7/7) -github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.FetchTxByShaList 100.00% (6/6) -github.com/conformal/btcdb/sqlite3/sqlitetx.go SqliteDb.ExistsTxSha 100.00% (5/5) -github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.InvalidateBlockCache 100.00% (5/5) -github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.InvalidateTxCache 100.00% (5/5) -github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.InsertBlockData 100.00% (3/3) -github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.Close 100.00% (3/3) -github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.Sync 100.00% (3/3) -github.com/conformal/btcdb/sqlite3/sqlitetx.go SqliteDb.InsertTx 100.00% (3/3) -github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteBlockIterator.Close 100.00% (2/2) -github.com/conformal/btcdb/sqlite3/sqlite.go CreateSqliteDB 100.00% (2/2) -github.com/conformal/btcdb/sqlite3/sqlite.go OpenSqliteDB 100.00% (2/2) -github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.InvalidateCache 100.00% (2/2) -github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.FetchTxBufBySha 100.00% (2/2) -github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.FetchTxBySha 100.00% (2/2) -github.com/conformal/btcdb/sqlite3/sqlite.go init 100.00% (1/1) -github.com/conformal/btcdb/sqlite3/sqlitetx.go SqliteDb.FetchLocationBySha 100.00% (1/1) -github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteBlockIterator.NextRow 100.00% (1/1) -github.com/conformal/btcdb/sqlite3/sqlitetx.go SqliteDb.insertTx 96.77% (30/31) -github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.insertBlockCache 92.31% (12/13) -github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.FetchBlockShaByHeight 91.67% (11/12) -github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.insertTxCache 91.67% (11/12) -github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.rePlayTransaction 90.00% (18/20) -github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.NewIterateBlocks 88.24% (15/17) -github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.RollbackClose 87.50% (7/8) -github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.fetchSha 85.71% (12/14) -github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.ExistsSha 85.71% (6/7) -github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.FetchHeightRange 84.62% (22/26) -github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.insertBlockData 84.00% (21/25) -github.com/conformal/btcdb/sqlite3/sqlitetx.go SqliteDb.fetchLocationBySha 84.00% (21/25) -github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.FetchBlockBySha 83.33% (10/12) -github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.fetchBlockCache 83.33% (5/6) -github.com/conformal/btcdb/sqlite3/sqlitetx.go SqliteDb.FetchTxUsedBySha 82.61% (19/23) -github.com/conformal/btcdb/sqlite3/sqliteblock.go insertGenesisBlock 81.82% (9/11) -github.com/conformal/btcdb/sqlite3/sqlite.go newOrCreateSqliteDB 79.25% (42/53) -github.com/conformal/btcdb/sqlite3/sqlitetx.go SqliteDb.existsTxSha 78.95% (15/19) -github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.startTx 78.57% (11/14) -github.com/conformal/btcdb/sqlite3/sqliteblock.go SqliteDb.blkExistsSha 77.78% (7/9) -github.com/conformal/btcdb/sqlite3/sqlitedbcache.go SqliteDb.FetchTxAllBySha 77.14% (27/35) -github.com/conformal/btcdb/sqlite3/sqlitetx.go SqliteDb.migrateTmpTable 76.67% (46/60) -github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.SetDBInsertMode 74.07% (20/27) -github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.syncPoint 71.43% (5/7) -github.com/conformal/btcdb/sqlite3/sqlite.go createDB 70.59% (12/17) -github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.DropAfterBlockBySha 68.57% (24/35) -github.com/conformal/btcdb/sqlite3/sqlite.go SqliteDb.InsertBlock 46.00% (23/50) -github.com/conformal/btcdb/sqlite3 ------------------------------ 82.14% (584/711) - +PASS +coverage: 63.3% of statements +ok github.com/conformal/btcdb 0.781s +github.com\conformal\btcdb\db.go: AddDBDriver 100.0% +github.com\conformal\btcdb\db.go: CreateDB 100.0% +github.com\conformal\btcdb\db.go: OpenDB 100.0% +github.com\conformal\btcdb\db.go: SupportedDBs 100.0% +github.com\conformal\btcdb\log.go: init 100.0% +github.com\conformal\btcdb\log.go: DisableLog 100.0% +github.com\conformal\btcdb\log.go: UseLogger 0.0% +github.com\conformal\btcdb\log.go: SetLogWriter 0.0% +github.com\conformal\btcdb\log.go: GetLog 100.0% +total: (statements) 63.3% +============================================================ +PASS +coverage: 77.3% of statements +ok github.com/conformal/btcdb/ldb 0.581s +github.com\conformal\btcdb\ldb\block.go: FetchBlockBySha 100.0% +github.com\conformal\btcdb\ldb\block.go: fetchBlockBySha 75.0% +github.com\conformal\btcdb\ldb\block.go: getBlkLoc 75.0% +github.com\conformal\btcdb\ldb\block.go: getBlkByHeight 100.0% +github.com\conformal\btcdb\ldb\block.go: getBlk 77.8% +github.com\conformal\btcdb\ldb\block.go: setBlk 85.7% +github.com\conformal\btcdb\ldb\block.go: insertBlockData 86.7% +github.com\conformal\btcdb\ldb\block.go: fetchSha 83.3% +github.com\conformal\btcdb\ldb\block.go: ExistsSha 100.0% +github.com\conformal\btcdb\ldb\block.go: blkExistsSha 75.0% +github.com\conformal\btcdb\ldb\block.go: FetchBlockShaByHeight 0.0% +github.com\conformal\btcdb\ldb\block.go: fetchBlockShaByHeight 0.0% +github.com\conformal\btcdb\ldb\block.go: FetchHeightRange 94.4% +github.com\conformal\btcdb\ldb\block.go: NewestSha 100.0% +github.com\conformal\btcdb\ldb\block.go: NewIterateBlocks 0.0% +github.com\conformal\btcdb\ldb\dbcache.go: InvalidateTxCache 0.0% +github.com\conformal\btcdb\ldb\dbcache.go: InvalidateBlockCache 0.0% +github.com\conformal\btcdb\ldb\dbcache.go: InvalidateCache 0.0% +github.com\conformal\btcdb\ldb\leveldb.go: init 100.0% +github.com\conformal\btcdb\ldb\leveldb.go: OpenDB 97.1% +github.com\conformal\btcdb\ldb\leveldb.go: openDB 75.0% +github.com\conformal\btcdb\ldb\leveldb.go: CreateDB 100.0% +github.com\conformal\btcdb\ldb\leveldb.go: close 100.0% +github.com\conformal\btcdb\ldb\leveldb.go: Sync 100.0% +github.com\conformal\btcdb\ldb\leveldb.go: Close 100.0% +github.com\conformal\btcdb\ldb\leveldb.go: DropAfterBlockBySha 80.6% +github.com\conformal\btcdb\ldb\leveldb.go: InsertBlock 43.9% +github.com\conformal\btcdb\ldb\leveldb.go: SetDBInsertMode 0.0% +github.com\conformal\btcdb\ldb\leveldb.go: doSpend 90.0% +github.com\conformal\btcdb\ldb\leveldb.go: unSpend 90.0% +github.com\conformal\btcdb\ldb\leveldb.go: setSpentData 100.0% +github.com\conformal\btcdb\ldb\leveldb.go: clearSpentData 100.0% +github.com\conformal\btcdb\ldb\leveldb.go: setclearSpentData 93.2% +github.com\conformal\btcdb\ldb\leveldb.go: int64ToKey 100.0% +github.com\conformal\btcdb\ldb\leveldb.go: shaBlkToKey 100.0% +github.com\conformal\btcdb\ldb\leveldb.go: shaTxToKey 100.0% +github.com\conformal\btcdb\ldb\leveldb.go: shaSpentTxToKey 100.0% +github.com\conformal\btcdb\ldb\leveldb.go: lBatch 100.0% +github.com\conformal\btcdb\ldb\leveldb.go: processBatches 78.6% +github.com\conformal\btcdb\ldb\leveldb.go: RollbackClose 100.0% +github.com\conformal\btcdb\ldb\tx.go: InsertTx 0.0% +github.com\conformal\btcdb\ldb\tx.go: insertTx 100.0% +github.com\conformal\btcdb\ldb\tx.go: formatTx 66.7% +github.com\conformal\btcdb\ldb\tx.go: getTxData 69.2% +github.com\conformal\btcdb\ldb\tx.go: getTxFullySpent 75.0% +github.com\conformal\btcdb\ldb\tx.go: formatTxFullySpent 65.2% +github.com\conformal\btcdb\ldb\tx.go: ExistsTxSha 100.0% +github.com\conformal\btcdb\ldb\tx.go: existsTxSha 75.0% +github.com\conformal\btcdb\ldb\tx.go: FetchTxByShaList 100.0% +github.com\conformal\btcdb\ldb\tx.go: FetchUnSpentTxByShaList 100.0% +github.com\conformal\btcdb\ldb\tx.go: fetchTxDataBySha 100.0% +github.com\conformal\btcdb\ldb\tx.go: fetchTxDataByLoc 64.3% +github.com\conformal\btcdb\ldb\tx.go: FetchTxBySha 55.6% +total: (statements) 76.9% +============================================================ +PASS +coverage: 44.2% of statements +ok github.com/conformal/btcdb/memdb 0.454s +github.com\conformal\btcdb\memdb\driver.go: init 100.0% +github.com\conformal\btcdb\memdb\driver.go: OpenDB 0.0% +github.com\conformal\btcdb\memdb\driver.go: CreateDB 100.0% +github.com\conformal\btcdb\memdb\memdb.go: newShaHashFromStr 100.0% +github.com\conformal\btcdb\memdb\memdb.go: isCoinbaseInput 75.0% +github.com\conformal\btcdb\memdb\memdb.go: isFullySpent 0.0% +github.com\conformal\btcdb\memdb\memdb.go: removeTx 0.0% +github.com\conformal\btcdb\memdb\memdb.go: Close 100.0% +github.com\conformal\btcdb\memdb\memdb.go: DropAfterBlockBySha 23.5% +github.com\conformal\btcdb\memdb\memdb.go: ExistsSha 62.5% +github.com\conformal\btcdb\memdb\memdb.go: FetchBlockBySha 44.4% +github.com\conformal\btcdb\memdb\memdb.go: FetchBlockShaByHeight 33.3% +github.com\conformal\btcdb\memdb\memdb.go: FetchHeightRange 19.0% +github.com\conformal\btcdb\memdb\memdb.go: ExistsTxSha 62.5% +github.com\conformal\btcdb\memdb\memdb.go: FetchTxBySha 20.0% +github.com\conformal\btcdb\memdb\memdb.go: fetchTxByShaList 33.3% +github.com\conformal\btcdb\memdb\memdb.go: FetchTxByShaList 100.0% +github.com\conformal\btcdb\memdb\memdb.go: FetchUnSpentTxByShaList 100.0% +github.com\conformal\btcdb\memdb\memdb.go: InsertBlock 51.7% +github.com\conformal\btcdb\memdb\memdb.go: InvalidateBlockCache 100.0% +github.com\conformal\btcdb\memdb\memdb.go: InvalidateCache 100.0% +github.com\conformal\btcdb\memdb\memdb.go: InvalidateTxCache 100.0% +github.com\conformal\btcdb\memdb\memdb.go: NewIterateBlocks 100.0% +github.com\conformal\btcdb\memdb\memdb.go: NewestSha 36.4% +github.com\conformal\btcdb\memdb\memdb.go: RollbackClose 100.0% +github.com\conformal\btcdb\memdb\memdb.go: SetDBInsertMode 100.0% +github.com\conformal\btcdb\memdb\memdb.go: Sync 100.0% +github.com\conformal\btcdb\memdb\memdb.go: newMemDb 100.0% +total: (statements) 44.2% From 845aedf103dbe093a4287d3169cc060d27a9feeb Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sun, 19 Jan 2014 02:36:54 -0600 Subject: [PATCH 114/163] Remove deprecated sqlite3. The sqlite3 backend has been deprecated for quite some time. As a result, it has not been updated with many of the more recent changes which means the behavior no longer conforms to the interface contract. --- sqlite3/doc.go | 15 - sqlite3/insertfail_test.go | 140 ------ sqlite3/insertremove_test.go | 209 --------- sqlite3/internal_test.go | 84 ---- sqlite3/operational_test.go | 387 --------------- sqlite3/sqlite.go | 881 ----------------------------------- sqlite3/sqliteblock.go | 293 ------------ sqlite3/sqlitedbcache.go | 394 ---------------- sqlite3/sqlitetx.go | 327 ------------- 9 files changed, 2730 deletions(-) delete mode 100644 sqlite3/doc.go delete mode 100644 sqlite3/insertfail_test.go delete mode 100644 sqlite3/insertremove_test.go delete mode 100644 sqlite3/internal_test.go delete mode 100644 sqlite3/operational_test.go delete mode 100644 sqlite3/sqlite.go delete mode 100644 sqlite3/sqliteblock.go delete mode 100644 sqlite3/sqlitedbcache.go delete mode 100644 sqlite3/sqlitetx.go diff --git a/sqlite3/doc.go b/sqlite3/doc.go deleted file mode 100644 index 30d2dd90..00000000 --- a/sqlite3/doc.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) 2013-2014 Conformal Systems LLC. -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -/* -Package sqlite3 implements a sqlite3 instance of btcdb. - -sqlite provides a zero setup, single file database. It requires cgo -and the presence of the sqlite library and headers, but nothing else. -The performance is generally high although it goes down with database -size. - -Many of the block or tx specific functions for btcdb are in this subpackage. -*/ -package sqlite3 diff --git a/sqlite3/insertfail_test.go b/sqlite3/insertfail_test.go deleted file mode 100644 index 6fe54153..00000000 --- a/sqlite3/insertfail_test.go +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright (c) 2013-2014 Conformal Systems LLC. -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package sqlite3_test - -import ( - "github.com/conformal/btcdb" - "github.com/conformal/btcdb/sqlite3" - "os" - "path/filepath" - "testing" -) - -func TestFailOperational(t *testing.T) { - sqlite3.SetTestingT(t) - failtestOperationalMode(t, dbTmDefault) - failtestOperationalMode(t, dbTmNormal) - failtestOperationalMode(t, dbTmFast) - failtestOperationalMode(t, dbTmNoVerify) -} - -func failtestOperationalMode(t *testing.T, mode int) { - // simplified basic operation is: - // 1) fetch block from remote server - // 2) look up all txin (except coinbase in db) - // 3) insert block - - // Ignore db remove errors since it means we didn't have an old one. - dbname := "tstdbop1" - _ = os.Remove(dbname) - db, err := btcdb.CreateDB("sqlite", dbname) - if err != nil { - t.Errorf("Failed to open test database %v", err) - return - } - defer os.Remove(dbname) - defer db.Close() - - switch mode { - case dbTmDefault: // default - // no setup - case dbTmNormal: // explicit normal - db.SetDBInsertMode(btcdb.InsertNormal) - case dbTmFast: // fast mode - db.SetDBInsertMode(btcdb.InsertFast) - if sqldb, ok := db.(*sqlite3.SqliteDb); ok { - sqldb.TempTblMax = 100 - } else { - t.Errorf("not right type") - } - case dbTmNoVerify: // validated block - // no point in testing this - return - } - - // Since we are dealing with small dataset, reduce cache size - sqlite3.SetBlockCacheSize(db, 2) - sqlite3.SetTxCacheSize(db, 3) - - testdatafile := filepath.Join("..", "testdata", "blocks1-256.bz2") - blocks, err := loadBlocks(t, testdatafile) - if err != nil { - t.Errorf("Unable to load blocks from test data for mode %v: %v", - mode, err) - return - } - - err = nil -out: - for height := int64(0); height < int64(len(blocks)); height++ { - block := blocks[height] - - mblock := block.MsgBlock() - blockname, _ := block.Sha() - - if height == 248 { - // time to corrupt the datbase, to see if it leaves the block or tx in the db - if len(mblock.Transactions) != 2 { - t.Errorf("transaction #248 should have two transactions txid %v ?= 828ef3b079f9c23829c56fe86e85b4a69d9e06e5b54ea597eef5fb3ffef509fe", blockname) - return - } - tx := mblock.Transactions[1] - txin := tx.TxIn[0] - origintxsha := &txin.PreviousOutpoint.Hash - sqlite3.KillTx(db, origintxsha) - _, err = db.FetchTxBySha(origintxsha) - if err == nil { - t.Errorf("deleted tx found %v", origintxsha) - } - } - - if height == 248 { - } - newheight, err := db.InsertBlock(block) - if err != nil { - if height != 248 { - t.Errorf("failed to insert block %v err %v", height, err) - break out - } - } else { - if height == 248 { - t.Errorf("block insert with missing input tx succeeded block %v err %v", height, err) - break out - } - } - if height == 248 { - for _, tx := range mblock.Transactions { - txsha, err := tx.TxSha() - _, err = db.FetchTxBySha(&txsha) - if err == nil { - t.Errorf("referenced tx found, should not have been %v, ", txsha) - } - } - } - if height == 248 { - exists := db.ExistsSha(blockname) - if exists == true { - t.Errorf("block still present after failed insert") - } - // if we got here with no error, testing was successful - break out - } - if newheight != height { - t.Errorf("height mismatch expect %v returned %v", height, newheight) - break out - } - } - - switch mode { - case dbTmDefault: // default - // no cleanup - case dbTmNormal: // explicit normal - // no cleanup - case dbTmFast: // fast mode - db.SetDBInsertMode(btcdb.InsertNormal) - case dbTmNoVerify: // validated block - db.SetDBInsertMode(btcdb.InsertNormal) - } -} diff --git a/sqlite3/insertremove_test.go b/sqlite3/insertremove_test.go deleted file mode 100644 index 34d564bf..00000000 --- a/sqlite3/insertremove_test.go +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright (c) 2013-2014 Conformal Systems LLC. -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package sqlite3_test - -import ( - "github.com/conformal/btcdb" - "github.com/conformal/btcdb/sqlite3" - "github.com/conformal/btcutil" - "github.com/conformal/btcwire" - "os" - "path/filepath" - "testing" -) - -var tstBlocks []*btcutil.Block - -func loadblocks(t *testing.T) []*btcutil.Block { - if len(tstBlocks) != 0 { - return tstBlocks - } - - testdatafile := filepath.Join("..", "testdata", "blocks1-256.bz2") - blocks, err := loadBlocks(t, testdatafile) - if err != nil { - t.Errorf("Unable to load blocks from test data: %v", err) - return nil - } - tstBlocks = blocks - return blocks -} - -func TestUnspentInsert(t *testing.T) { - testUnspentInsert(t, dbTmDefault) - testUnspentInsert(t, dbTmNormal) - testUnspentInsert(t, dbTmFast) -} - -// insert every block in the test chain -// after each insert, fetch all the tx affected by the latest -// block and verify that the the tx is spent/unspent -// new tx should be fully unspent, referenced tx should have -// the associated txout set to spent. -func testUnspentInsert(t *testing.T, mode int) { - // Ignore db remove errors since it means we didn't have an old one. - dbname := "tstdbuspnt1" - _ = os.Remove(dbname) - db, err := btcdb.CreateDB("sqlite", dbname) - if err != nil { - t.Errorf("Failed to open test database %v", err) - return - } - defer os.Remove(dbname) - defer db.Close() - - switch mode { - case dbTmDefault: // default - // no setup - case dbTmNormal: // explicit normal - db.SetDBInsertMode(btcdb.InsertNormal) - case dbTmFast: // fast mode - db.SetDBInsertMode(btcdb.InsertFast) - if sqldb, ok := db.(*sqlite3.SqliteDb); ok { - sqldb.TempTblMax = 100 - } else { - t.Errorf("not right type") - } - case dbTmNoVerify: // validated block - t.Errorf("UnspentInsert test is not valid in NoVerify mode") - } - - // Since we are dealing with small dataset, reduce cache size - sqlite3.SetBlockCacheSize(db, 2) - sqlite3.SetTxCacheSize(db, 3) - - blocks := loadblocks(t) -endtest: - for height := int64(0); height < int64(len(blocks)); height++ { - - block := blocks[height] - // look up inputs to this x - mblock := block.MsgBlock() - var txneededList []*btcwire.ShaHash - var txlookupList []*btcwire.ShaHash - var txOutList []*btcwire.ShaHash - var txInList []*btcwire.OutPoint - for _, tx := range mblock.Transactions { - for _, txin := range tx.TxIn { - if txin.PreviousOutpoint.Index == uint32(4294967295) { - continue - } - origintxsha := &txin.PreviousOutpoint.Hash - - txInList = append(txInList, &txin.PreviousOutpoint) - txneededList = append(txneededList, origintxsha) - txlookupList = append(txlookupList, origintxsha) - - if !db.ExistsTxSha(origintxsha) { - t.Errorf("referenced tx not found %v ", origintxsha) - } - - } - txshaname, _ := tx.TxSha() - txlookupList = append(txlookupList, &txshaname) - txOutList = append(txOutList, &txshaname) - } - - txneededmap := map[btcwire.ShaHash]*btcdb.TxListReply{} - txlist := db.FetchUnSpentTxByShaList(txneededList) - for _, txe := range txlist { - if txe.Err != nil { - t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) - break endtest - } - txneededmap[*txe.Sha] = txe - } - for _, spend := range txInList { - itxe := txneededmap[spend.Hash] - if itxe.TxSpent[spend.Index] == true { - t.Errorf("txin %v:%v is already spent", spend.Hash, spend.Index) - } - } - - newheight, err := db.InsertBlock(block) - if err != nil { - t.Errorf("failed to insert block %v err %v", height, err) - break endtest - } - if newheight != height { - t.Errorf("height mismatch expect %v returned %v", height, newheight) - break endtest - } - - txlookupmap := map[btcwire.ShaHash]*btcdb.TxListReply{} - txlist = db.FetchUnSpentTxByShaList(txlookupList) - for _, txe := range txlist { - if txe.Err != nil { - t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) - break endtest - } - txlookupmap[*txe.Sha] = txe - } - for _, spend := range txInList { - itxe := txlookupmap[spend.Hash] - if itxe.TxSpent[spend.Index] == false { - t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent) - } - } - for _, txo := range txOutList { - itxe := txlookupmap[*txo] - for i, spent := range itxe.TxSpent { - if spent == true { - t.Errorf("freshly inserted tx %v already spent %v", txo, i) - } - } - - } - if len(txInList) == 0 { - continue - } - dropblock := blocks[height-1] - dropsha, _ := dropblock.Sha() - - err = db.DropAfterBlockBySha(dropsha) - if err != nil { - t.Errorf("failed to drop block %v err %v", height, err) - break endtest - } - - txlookupmap = map[btcwire.ShaHash]*btcdb.TxListReply{} - txlist = db.FetchUnSpentTxByShaList(txlookupList) - for _, txe := range txlist { - if txe.Err != nil { - if _, ok := txneededmap[*txe.Sha]; ok { - t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) - break endtest - } - } - txlookupmap[*txe.Sha] = txe - } - for _, spend := range txInList { - itxe := txlookupmap[spend.Hash] - if itxe.TxSpent[spend.Index] == true { - t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent) - } - } - newheight, err = db.InsertBlock(block) - if err != nil { - t.Errorf("failed to insert block %v err %v", height, err) - break endtest - } - txlookupmap = map[btcwire.ShaHash]*btcdb.TxListReply{} - txlist = db.FetchUnSpentTxByShaList(txlookupList) - for _, txe := range txlist { - if txe.Err != nil { - t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) - break endtest - } - txlookupmap[*txe.Sha] = txe - } - for _, spend := range txInList { - itxe := txlookupmap[spend.Hash] - if itxe.TxSpent[spend.Index] == false { - t.Errorf("txin %v:%v is unspent %v", spend.Hash, spend.Index, itxe.TxSpent) - } - } - } -} diff --git a/sqlite3/internal_test.go b/sqlite3/internal_test.go deleted file mode 100644 index 81728223..00000000 --- a/sqlite3/internal_test.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright (c) 2013-2014 Conformal Systems LLC. -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package sqlite3 - -import ( - "fmt" - "github.com/conformal/btcdb" - "github.com/conformal/btcwire" - "testing" -) - -var t *testing.T - -func SetTestingT(t_arg *testing.T) { - t = t_arg -} - -// FetchSha returns the datablock and pver for the given ShaHash. -// This is a testing only interface. -func FetchSha(db btcdb.Db, sha *btcwire.ShaHash) (buf []byte, pver uint32, - blkid int64, err error) { - sqldb, ok := db.(*SqliteDb) - if !ok { - err = fmt.Errorf("Invalid data type") - return - } - buf, pver, blkid, err = sqldb.fetchSha(*sha) - return -} - -// SetBlockCacheSize configures the maximum number of blocks in the cache to -// be the given size should be made before any fetching. -// This is a testing only interface. -func SetBlockCacheSize(db btcdb.Db, newsize int) { - sqldb, ok := db.(*SqliteDb) - if !ok { - return - } - bc := &sqldb.blockCache - bc.maxcount = newsize -} - -// SetTxCacheSize configures the maximum number of tx in the cache to -// be the given size should be made before any fetching. -// This is a testing only interface. -func SetTxCacheSize(db btcdb.Db, newsize int) { - sqldb, ok := db.(*SqliteDb) - if !ok { - return - } - tc := &sqldb.txCache - tc.maxcount = newsize -} - -// KillTx is a function that deletes a transaction from the database -// this should only be used for testing purposes to valiate error paths -// in the database. This is _expected_ to leave the database in an -// inconsistant state. -func KillTx(dbarg btcdb.Db, txsha *btcwire.ShaHash) { - db, ok := dbarg.(*SqliteDb) - if !ok { - return - } - db.endTx(false) - db.startTx() - tx := &db.txState - key := txsha.String() - _, err := tx.tx.Exec("DELETE FROM txtmp WHERE key == ?", key) - if err != nil { - log.Warnf("error deleting tx %v from txtmp", txsha) - } - _, err = tx.tx.Exec("DELETE FROM tx WHERE key == ?", key) - if err != nil { - log.Warnf("error deleting tx %v from tx (%v)", txsha, key) - } - err = db.endTx(true) - if err != nil { - // XXX - db.endTx(false) - } - db.InvalidateCache() -} diff --git a/sqlite3/operational_test.go b/sqlite3/operational_test.go deleted file mode 100644 index e2f35b83..00000000 --- a/sqlite3/operational_test.go +++ /dev/null @@ -1,387 +0,0 @@ -// Copyright (c) 2013-2014 Conformal Systems LLC. -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package sqlite3_test - -import ( - "compress/bzip2" - "encoding/binary" - "github.com/conformal/btcdb" - "github.com/conformal/btcdb/sqlite3" - "github.com/conformal/btcutil" - "github.com/conformal/btcwire" - "io" - "os" - "path/filepath" - "strings" - "testing" -) - -var network = btcwire.MainNet - -const ( - dbTmDefault = iota - dbTmNormal - dbTmFast - dbTmNoVerify -) - -func TestOperational(t *testing.T) { - testOperationalMode(t, dbTmDefault) - testOperationalMode(t, dbTmNormal) - testOperationalMode(t, dbTmFast) - testOperationalMode(t, dbTmNoVerify) -} - -func testOperationalMode(t *testing.T, mode int) { - // simplified basic operation is: - // 1) fetch block from remote server - // 2) look up all txin (except coinbase in db) - // 3) insert block - - // Ignore db remove errors since it means we didn't have an old one. - dbname := "tstdbop1" - _ = os.Remove(dbname) - db, err := btcdb.CreateDB("sqlite", dbname) - if err != nil { - t.Errorf("Failed to open test database %v", err) - return - } - defer os.Remove(dbname) - defer db.Close() - - switch mode { - case dbTmDefault: // default - // no setup - case dbTmNormal: // explicit normal - db.SetDBInsertMode(btcdb.InsertNormal) - case dbTmFast: // fast mode - db.SetDBInsertMode(btcdb.InsertFast) - if sqldb, ok := db.(*sqlite3.SqliteDb); ok { - sqldb.TempTblMax = 100 - } else { - t.Errorf("not right type") - } - case dbTmNoVerify: // validated block - db.SetDBInsertMode(btcdb.InsertValidatedInput) - } - - // Since we are dealing with small dataset, reduce cache size - sqlite3.SetBlockCacheSize(db, 2) - sqlite3.SetTxCacheSize(db, 3) - - testdatafile := filepath.Join("..", "testdata", "blocks1-256.bz2") - blocks, err := loadBlocks(t, testdatafile) - if err != nil { - t.Errorf("Unable to load blocks from test data for mode %v: %v", - mode, err) - return - } - - err = nil -out: - for height := int64(0); height < int64(len(blocks)); height++ { - block := blocks[height] - if mode != dbTmNoVerify { - // except for NoVerify which does not allow lookups check inputs - mblock := block.MsgBlock() - var txneededList []*btcwire.ShaHash - for _, tx := range mblock.Transactions { - for _, txin := range tx.TxIn { - if txin.PreviousOutpoint.Index == uint32(4294967295) { - continue - } - origintxsha := &txin.PreviousOutpoint.Hash - txneededList = append(txneededList, origintxsha) - - if !db.ExistsTxSha(origintxsha) { - t.Errorf("referenced tx not found %v ", origintxsha) - } - - _, err = db.FetchTxBySha(origintxsha) - if err != nil { - t.Errorf("referenced tx not found %v err %v ", origintxsha, err) - } - } - } - txlist := db.FetchUnSpentTxByShaList(txneededList) - for _, txe := range txlist { - if txe.Err != nil { - t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) - break out - } - } - - } - - newheight, err := db.InsertBlock(block) - if err != nil { - t.Errorf("failed to insert block %v err %v", height, err) - break out - } - if newheight != height { - t.Errorf("height mismatch expect %v returned %v", height, newheight) - break out - } - - newSha, blkid, err := db.NewestSha() - if err != nil { - t.Errorf("failed to obtain latest sha %v %v", height, err) - } - - if blkid != height { - t.Errorf("height does not match latest block height %v %v", blkid, height) - } - - blkSha, _ := block.Sha() - if *newSha != *blkSha { - t.Errorf("Newest block sha does not match freshly inserted one %v %v ", newSha, blkSha) - } - } - - // now that db is populated, do some additional test - testFetchRangeHeight(t, db, blocks) - - switch mode { - case dbTmDefault: // default - // no cleanup - case dbTmNormal: // explicit normal - // no cleanup - case dbTmFast: // fast mode - db.SetDBInsertMode(btcdb.InsertNormal) - case dbTmNoVerify: // validated block - db.SetDBInsertMode(btcdb.InsertNormal) - } -} - -func TestBackout(t *testing.T) { - testBackout(t, dbTmDefault) - testBackout(t, dbTmNormal) - testBackout(t, dbTmFast) -} - -func testBackout(t *testing.T, mode int) { - // simplified basic operation is: - // 1) fetch block from remote server - // 2) look up all txin (except coinbase in db) - // 3) insert block - - // Ignore db remove errors since it means we didn't have an old one. - dbname := "tstdbop2" - _ = os.Remove(dbname) - db, err := btcdb.CreateDB("sqlite", dbname) - if err != nil { - t.Errorf("Failed to open test database %v", err) - return - } - defer os.Remove(dbname) - defer db.Close() - - switch mode { - case dbTmDefault: // default - // no setup - case dbTmNormal: // explicit normal - db.SetDBInsertMode(btcdb.InsertNormal) - case dbTmFast: // fast mode - db.SetDBInsertMode(btcdb.InsertFast) - if sqldb, ok := db.(*sqlite3.SqliteDb); ok { - sqldb.TempTblMax = 100 - } else { - t.Errorf("not right type") - } - } - - // Since we are dealing with small dataset, reduce cache size - sqlite3.SetBlockCacheSize(db, 2) - sqlite3.SetTxCacheSize(db, 3) - - testdatafile := filepath.Join("..", "testdata", "blocks1-256.bz2") - blocks, err := loadBlocks(t, testdatafile) - if len(blocks) < 120 { - t.Errorf("test data too small") - return - } - - err = nil - for height := int64(0); height < int64(len(blocks)); height++ { - if height == 100 { - t.Logf("Syncing at block height 100") - db.Sync() - } - if height == 120 { - t.Logf("Simulating unexpected application quit") - // Simulate unexpected application quit - db.RollbackClose() - break - } - - block := blocks[height] - - newheight, err := db.InsertBlock(block) - if err != nil { - t.Errorf("failed to insert block %v err %v", height, err) - break - } - if newheight != height { - t.Errorf("height mismatch expect %v returned %v", height, newheight) - break - } - } - - // db was closed at height 120, so no cleanup is possible. - - // reopen db - db, err = btcdb.OpenDB("sqlite", dbname) - if err != nil { - t.Errorf("Failed to open test database %v", err) - return - } - defer db.Close() - - sha, err := blocks[99].Sha() - if err != nil { - t.Errorf("failed to get block 99 sha err %v", err) - return - } - _ = db.ExistsSha(sha) - _, err = db.FetchBlockBySha(sha) - if err != nil { - t.Errorf("failed to load block 99 from db %v", err) - } - - sha, err = blocks[110].Sha() - if err != nil { - t.Errorf("failed to get block 110 sha err %v", err) - return - } - _ = db.ExistsSha(sha) - _, err = db.FetchBlockBySha(sha) - if err == nil { - t.Errorf("loaded block 110 from db, failure expected") - return - } - - block := blocks[110] - mblock := block.MsgBlock() - txsha, err := mblock.Transactions[0].TxSha() - exists := db.ExistsTxSha(&txsha) - if exists { - t.Errorf("tx %v exists in db, failure expected", txsha) - } - - _, err = db.FetchTxBySha(&txsha) - _, err = db.FetchTxBySha(&txsha) -} - -func loadBlocks(t *testing.T, file string) (blocks []*btcutil.Block, err error) { - testdatafile := filepath.Join("..", "testdata", "blocks1-256.bz2") - var dr io.Reader - var fi io.ReadCloser - fi, err = os.Open(testdatafile) - if err != nil { - t.Errorf("failed to open file %v, err %v", testdatafile, err) - return - } - if strings.HasSuffix(testdatafile, ".bz2") { - z := bzip2.NewReader(fi) - dr = z - } else { - dr = fi - } - - defer func() { - if err := fi.Close(); err != nil { - t.Errorf("failed to close file %v %v", testdatafile, err) - } - }() - - // Set the first block as the genesis block. - genesis := btcutil.NewBlock(&btcwire.GenesisBlock) - blocks = append(blocks, genesis) - - var block *btcutil.Block - err = nil - for height := int64(1); err == nil; height++ { - var rintbuf uint32 - err = binary.Read(dr, binary.LittleEndian, &rintbuf) - if err == io.EOF { - // hit end of file at expected offset: no warning - height-- - err = nil - break - } - if err != nil { - t.Errorf("failed to load network type, err %v", err) - break - } - if rintbuf != uint32(network) { - t.Errorf("Block doesn't match network: %v expects %v", - rintbuf, network) - break - } - err = binary.Read(dr, binary.LittleEndian, &rintbuf) - blocklen := rintbuf - - rbytes := make([]byte, blocklen) - - // read block - dr.Read(rbytes) - - block, err = btcutil.NewBlockFromBytes(rbytes) - if err != nil { - t.Errorf("failed to parse block %v", height) - return - } - blocks = append(blocks, block) - } - return -} - -func testFetchRangeHeight(t *testing.T, db btcdb.Db, blocks []*btcutil.Block) { - - var testincrement int64 = 50 - var testcnt int64 = 100 - - shanames := make([]*btcwire.ShaHash, len(blocks)) - - nBlocks := int64(len(blocks)) - - for i := range blocks { - blockSha, err := blocks[i].Sha() - if err != nil { - t.Errorf("FetchRangeHeight: unexpected failure computing block sah %v", err) - } - shanames[i] = blockSha - } - - for startheight := int64(0); startheight < nBlocks; startheight += testincrement { - endheight := startheight + testcnt - - if endheight > nBlocks { - endheight = btcdb.AllShas - } - - shalist, err := db.FetchHeightRange(startheight, endheight) - if err != nil { - t.Errorf("FetchRangeHeight: unexpected failure looking up shas %v", err) - } - - if endheight == btcdb.AllShas { - if int64(len(shalist)) != nBlocks-startheight { - t.Errorf("FetchRangeHeight: expected A %v shas, got %v", nBlocks-startheight, len(shalist)) - } - } else { - if int64(len(shalist)) != testcnt { - t.Errorf("FetchRangeHeight: expected %v shas, got %v", testcnt, len(shalist)) - } - } - - for i := range shalist { - if *shanames[int64(i)+startheight] != shalist[i] { - t.Errorf("FetchRangeHeight: mismatch sha at %v requested range %v %v ", int64(i)+startheight, startheight, endheight) - } - } - } - -} diff --git a/sqlite3/sqlite.go b/sqlite3/sqlite.go deleted file mode 100644 index 813be68f..00000000 --- a/sqlite3/sqlite.go +++ /dev/null @@ -1,881 +0,0 @@ -// Copyright (c) 2013-2014 Conformal Systems LLC. -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package sqlite3 - -import ( - "database/sql" - "fmt" - "github.com/conformal/btcdb" - "github.com/conformal/btclog" - "github.com/conformal/btcutil" - "github.com/conformal/btcwire" - _ "github.com/mattn/go-sqlite3" - "os" - "sync" -) - -const ( - dbVersion int = 2 - dbMaxTransCnt = 20000 - dbMaxTransMem = 64 * 1024 * 1024 // 64 MB -) - -const ( - blkInsertSha = iota - blkFetchSha - blkExistsSha - blkFetchIdx - blkFetchIdxList -) - -const ( - txInsertStmt = iota - txFetchUsedByShaStmt - txFetchLocationByShaStmt - txFetchLocUsedByShaStmt - txUpdateUsedByShaStmt - - txtmpInsertStmt - txtmpFetchUsedByShaStmt - txtmpFetchLocationByShaStmt - txtmpFetchLocUsedByShaStmt - txtmpUpdateUsedByShaStmt - - txMigrateCopy - txMigrateClear - txMigratePrep - txMigrateFinish - txMigrateCount - txPragmaVacuumOn - txPragmaVacuumOff - txVacuum - txExistsShaStmt - txtmpExistsShaStmt -) - -var blkqueries []string = []string{ - blkInsertSha: "INSERT INTO block (key, pver, data) VALUES(?, ?, ?);", - blkFetchSha: "SELECT pver, data, blockid FROM block WHERE key = ?;", - blkExistsSha: "SELECT pver FROM block WHERE key = ?;", - blkFetchIdx: "SELECT key FROM block WHERE blockid = ?;", - blkFetchIdxList: "SELECT key FROM block WHERE blockid >= ? AND blockid < ? ORDER BY blockid ASC LIMIT 500;", -} - -var txqueries []string = []string{ - txInsertStmt: "INSERT INTO tx (key, blockid, txoff, txlen, data) VALUES(?, ?, ?, ?, ?);", - txFetchUsedByShaStmt: "SELECT data FROM tx WHERE key = ?;", - txFetchLocationByShaStmt: "SELECT blockid, txoff, txlen FROM tx WHERE key = ?;", - txFetchLocUsedByShaStmt: "SELECT blockid, txoff, txlen, data FROM tx WHERE key = ?;", - txUpdateUsedByShaStmt: "UPDATE tx SET data = ? WHERE key = ?;", - - txtmpInsertStmt: "INSERT INTO txtmp (key, blockid, txoff, txlen, data) VALUES(?, ?, ?, ?, ?);", - txtmpFetchUsedByShaStmt: "SELECT data FROM txtmp WHERE key = ?;", - txtmpFetchLocationByShaStmt: "SELECT blockid, txoff, txlen FROM txtmp WHERE key = ?;", - txtmpFetchLocUsedByShaStmt: "SELECT blockid, txoff, txlen, data FROM txtmp WHERE key = ?;", - txtmpUpdateUsedByShaStmt: "UPDATE txtmp SET data = ? WHERE key = ?;", - - txMigrateCopy: "INSERT INTO tx (key, blockid, txoff, txlen, data) SELECT key, blockid, txoff, txlen, data FROM txtmp;", - txMigrateClear: "DELETE from txtmp;", - txMigratePrep: "DROP index IF EXISTS uniquetx;", - txMigrateFinish: "CREATE UNIQUE INDEX IF NOT EXISTS uniquetx ON tx (key);", - txMigrateCount: "SELECT COUNT(*) FROM txtmp;", - txPragmaVacuumOn: "PRAGMA auto_vacuum = FULL;", - txPragmaVacuumOff: "PRAGMA auto_vacuum = NONE;", - txVacuum: "VACUUM;", - txExistsShaStmt: "SELECT blockid FROM tx WHERE key = ?;", - txtmpExistsShaStmt: "SELECT blockid FROM txtmp WHERE key = ?;", -} - -var log = btclog.Disabled - -type tBlockInsertData struct { - sha btcwire.ShaHash - pver uint32 - buf []byte -} -type tTxInsertData struct { - txsha *btcwire.ShaHash - blockid int64 - txoff int - txlen int - usedbuf []byte -} - -type txState struct { - tx *sql.Tx - writeCount int - txDataSz int - txInsertList []interface{} -} -type SqliteDb struct { - sqldb *sql.DB - blkStmts []*sql.Stmt - blkBaseStmts []*sql.Stmt - txStmts []*sql.Stmt - txBaseStmts []*sql.Stmt - txState txState - dbLock sync.Mutex - - lastBlkShaCached bool - lastBlkSha btcwire.ShaHash - lastBlkIdx int64 - txCache txCache - blockCache blockCache - - UseTempTX bool - TempTblSz int - TempTblMax int - - dbInsertMode btcdb.InsertMode -} - -var self = btcdb.DriverDB{DbType: "sqlite", Create: CreateSqliteDB, Open: OpenSqliteDB} - -func init() { - btcdb.AddDBDriver(self) -} - -// createDB configure the database, setting up all tables to initial state. -func createDB(db *sql.DB) error { - log.Infof("Initializing new block database") - - // XXX check for old tables - buildTables := []string{ - "CREATE TABLE dbversion (version integer);", - "CREATE TABLE block ( blockid INTEGER PRIMARY KEY, key BLOB UNIQUE, " + - "pver INTEGER NOT NULL, data BLOB NOT NULL);", - "INSERT INTO dbversion (version) VALUES (" + fmt.Sprintf("%d", dbVersion) + - ");", - } - buildtxTables := []string{ - "CREATE TABLE tx (txidx INTEGER PRIMARY KEY, " + - "key TEXT, " + - "blockid INTEGER NOT NULL, " + - "txoff INTEGER NOT NULL, txlen INTEGER NOT NULL, " + - "data BLOB NOT NULL, " + - "FOREIGN KEY(blockid) REFERENCES block(blockid));", - "CREATE TABLE txtmp (key TEXT PRIMARY KEY, " + - "blockid INTEGER NOT NULL, " + - "txoff INTEGER NOT NULL, txlen INTEGER NOT NULL, " + - "data BLOB NOT NULL, " + - "FOREIGN KEY(blockid) REFERENCES block(blockid));", - "CREATE UNIQUE INDEX uniquetx ON tx (key);", - } - for _, sql := range buildTables { - _, err := db.Exec(sql) - if err != nil { - log.Warnf("sql table op failed %v [%v]", err, sql) - return err - } - } - for _, sql := range buildtxTables { - _, err := db.Exec(sql) - if err != nil { - log.Warnf("sql table op failed %v [%v]", err, sql) - return err - } - } - - return nil -} - -// OpenSqliteDB opens an existing database for use. -func OpenSqliteDB(filepath string) (pbdb btcdb.Db, err error) { - log = btcdb.GetLog() - return newOrCreateSqliteDB(filepath, false) -} - -// CreateSqliteDB creates, initializes and opens a database for use. -func CreateSqliteDB(filepath string) (pbdb btcdb.Db, err error) { - log = btcdb.GetLog() - return newOrCreateSqliteDB(filepath, true) -} - -// newOrCreateSqliteDB opens a database, either creating it or opens -// existing database based on flag. -func newOrCreateSqliteDB(filepath string, create bool) (pbdb btcdb.Db, err error) { - var bdb SqliteDb - if create == false { - _, err = os.Stat(filepath) - if err != nil { - return nil, btcdb.DbDoesNotExist - } - } - - db, err := sql.Open("sqlite3", filepath) - if err != nil { - log.Warnf("db open failed %v\n", err) - return nil, err - } - - db.Exec("PRAGMA page_size=4096;") - db.Exec("PRAGMA foreign_keys=ON;") - db.Exec("PRAGMA journal_mode=WAL;") - - dbverstmt, err := db.Prepare("SELECT version FROM dbversion;") - if err != nil { - // about the only reason this would fail is that the database - // is not initialized - if create == false { - return nil, btcdb.DbDoesNotExist - } - err = createDB(db) - if err != nil { - // already warned in the called function - return nil, err - } - dbverstmt, err = db.Prepare("SELECT version FROM dbversion;") - if err != nil { - // if it failed this a second time, fail. - return nil, err - } - - } - row := dbverstmt.QueryRow() - var version int - err = row.Scan(&version) - if err != nil { - log.Warnf("unable to find db version: no row\n", err) - } - switch version { - case dbVersion: - // all good - default: - log.Warnf("mismatch db version: %v expected %v\n", version, dbVersion) - return nil, fmt.Errorf("Invalid version in database") - } - bdb.sqldb = db - - bdb.blkStmts = make([]*sql.Stmt, len(blkqueries)) - bdb.blkBaseStmts = make([]*sql.Stmt, len(blkqueries)) - for i := range blkqueries { - stmt, err := db.Prepare(blkqueries[i]) - if err != nil { - // XXX log/ - return nil, err - } - bdb.blkBaseStmts[i] = stmt - } - for i := range bdb.blkBaseStmts { - bdb.blkStmts[i] = bdb.blkBaseStmts[i] - } - - bdb.txBaseStmts = make([]*sql.Stmt, len(txqueries)) - for i := range txqueries { - stmt, err := db.Prepare(txqueries[i]) - if err != nil { - // XXX log/ - return nil, err - } - bdb.txBaseStmts[i] = stmt - } - // NOTE: all array entries in txStmts remain nil'ed - // tx statements are lazy bound - bdb.txStmts = make([]*sql.Stmt, len(txqueries)) - - bdb.blockCache.maxcount = 150 - bdb.blockCache.blockMap = map[btcwire.ShaHash]*blockCacheObj{} - bdb.blockCache.blockMap = map[btcwire.ShaHash]*blockCacheObj{} - bdb.blockCache.blockHeightMap = map[int64]*blockCacheObj{} - bdb.txCache.maxcount = 2000 - bdb.txCache.txMap = map[btcwire.ShaHash]*txCacheObj{} - - bdb.UseTempTX = true - bdb.TempTblMax = 1000000 - - return &bdb, nil -} - -// Sync verifies that the database is coherent on disk, -// and no outstanding transactions are in flight. -func (db *SqliteDb) Sync() { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - db.endTx(true) -} - -// syncPoint notifies the db that this is a safe time to sync the database, -// if there are many outstanding transactions. -// Must be called with db lock held. -func (db *SqliteDb) syncPoint() { - - tx := &db.txState - - if db.TempTblSz > db.TempTblMax { - err := db.migrateTmpTable() - if err != nil { - return - } - } else { - if len(tx.txInsertList) > dbMaxTransCnt || tx.txDataSz > dbMaxTransMem { - db.endTx(true) - } - } -} - -// Close cleanly shuts down database, syncing all data. -func (db *SqliteDb) Close() { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - db.close() -} - -// RollbackClose discards the recent database changes to the previously -// saved data at last Sync. -func (db *SqliteDb) RollbackClose() { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - tx := &db.txState - if tx.tx != nil { - err := tx.tx.Rollback() - if err != nil { - log.Debugf("Rollback failed: %v", err) - } else { - tx.tx = nil - } - } - db.close() -} - -// close performs the internal shutdown/close operation. -func (db *SqliteDb) close() { - db.endTx(true) - - db.InvalidateCache() - - for i := range db.blkBaseStmts { - db.blkBaseStmts[i].Close() - } - for i := range db.txBaseStmts { - if db.txBaseStmts[i] != nil { - db.txBaseStmts[i].Close() - db.txBaseStmts[i] = nil - } - } - db.sqldb.Close() -} - -// txop returns the appropriately prepared statement, based on -// transaction state of the database. -func (db *SqliteDb) txop(op int) *sql.Stmt { - if db.txStmts[op] != nil { - return db.txStmts[op] - } - if db.txState.tx == nil { - // we are not in a transaction, return the base statement - return db.txBaseStmts[op] - } - - if db.txStmts[op] == nil { - db.txStmts[op] = db.txState.tx.Stmt(db.txBaseStmts[op]) - } - - return db.txStmts[op] -} - -// startTx starts a transaction, preparing or scrubbing statements -// for proper operation inside a transaction. -func (db *SqliteDb) startTx() (err error) { - tx := &db.txState - if tx.tx != nil { - // this shouldn't happen... - log.Warnf("Db startTx called while in a transaction") - return - } - tx.tx, err = db.sqldb.Begin() - if err != nil { - log.Warnf("Db startTx: begin failed %v", err) - tx.tx = nil - return - } - for i := range db.blkBaseStmts { - db.blkStmts[i] = tx.tx.Stmt(db.blkBaseStmts[i]) - } - for i := range db.txBaseStmts { - db.txStmts[i] = nil // these are lazily prepared - } - return -} - -// endTx commits the current active transaction, it zaps all of the prepared -// statements associated with the transaction. -func (db *SqliteDb) endTx(recover bool) (err error) { - tx := &db.txState - - if tx.tx == nil { - return - } - - err = tx.tx.Commit() - if err != nil && recover { - // XXX - double check that the tx is dead after - // commit failure (rollback?) - - log.Warnf("Db endTx: commit failed %v", err) - err = db.rePlayTransaction() - if err != nil { - // We tried, return failure (after zeroing state) - // so the upper level can notice and restart - } - } - for i := range db.blkBaseStmts { - db.blkStmts[i].Close() - db.blkStmts[i] = db.blkBaseStmts[i] - } - for i := range db.txStmts { - if db.txStmts[i] != nil { - db.txStmts[i].Close() - db.txStmts[i] = nil - } - } - tx.tx = nil - var emptyTxList []interface{} - tx.txInsertList = emptyTxList - tx.txDataSz = 0 - return -} - -// rePlayTransaction will attempt to re-execute inserts performed -// sync the beginning of a transaction. This is to be used after -// a sql Commit operation fails to keep the database from losing data. -func (db *SqliteDb) rePlayTransaction() (err error) { - err = db.startTx() - if err != nil { - return - } - tx := &db.txState - for _, ins := range tx.txInsertList { - switch v := ins.(type) { - case tBlockInsertData: - block := v - _, err = db.blkStmts[blkInsertSha].Exec(block.sha.Bytes(), - block.pver, block.buf) - if err != nil { - break - } - case tTxInsertData: - txd := v - txnamebytes := txd.txsha.Bytes() - txop := db.txop(txInsertStmt) - _, err = txop.Exec(txd.blockid, txnamebytes, txd.txoff, - txd.txlen, txd.usedbuf) - if err != nil { - break - } - } - } - // This function is called even if we have failed. - // We need to clean up so the database can be used again. - // However we want the original error not any new error, - // unless there was no original error but the commit fails. - err2 := db.endTx(false) - if err == nil && err2 != nil { - err = err2 - } - - return -} - -// DropAfterBlockBySha will remove any blocks from the database after the given block. -// It terminates any existing transaction and performs its operations in an -// atomic transaction, it is terminated (committed) before exit. -func (db *SqliteDb) DropAfterBlockBySha(sha *btcwire.ShaHash) (err error) { - var row *sql.Row - db.dbLock.Lock() - defer db.dbLock.Unlock() - - // This is a destructive operation and involves multiple requests - // so requires a transaction, terminate any transaction to date - // and start a new transaction - err = db.endTx(true) - if err != nil { - return err - } - err = db.startTx() - if err != nil { - return err - } - - var startheight int64 - - if db.lastBlkShaCached { - startheight = db.lastBlkIdx - } else { - querystr := "SELECT blockid FROM block ORDER BY blockid DESC;" - - tx := &db.txState - if tx.tx != nil { - row = tx.tx.QueryRow(querystr) - } else { - row = db.sqldb.QueryRow(querystr) - } - var startblkidx int64 - err = row.Scan(&startblkidx) - if err != nil { - log.Warnf("DropAfterBlockBySha:unable to fetch blockheight %v", err) - return err - } - startheight = startblkidx - } - // also drop any cached sha data - db.lastBlkShaCached = false - - querystr := "SELECT blockid FROM block WHERE key = ?;" - - tx := &db.txState - row = tx.tx.QueryRow(querystr, sha.Bytes()) - - var keepidx int64 - err = row.Scan(&keepidx) - if err != nil { - // XXX - db.endTx(false) - return err - } - - for height := startheight; height > keepidx; height = height - 1 { - var blk *btcutil.Block - blkc, ok := db.fetchBlockHeightCache(height) - - if ok { - blk = blkc.blk - } else { - // must load the block from the db - sha, err = db.fetchBlockShaByHeight(height - 1) - if err != nil { - return - } - - var buf []byte - buf, _, _, err = db.fetchSha(*sha) - if err != nil { - return - } - - blk, err = btcutil.NewBlockFromBytes(buf) - if err != nil { - return - } - } - - for _, tx := range blk.MsgBlock().Transactions { - err = db.unSpend(tx) - if err != nil { - return - } - } - } - - // invalidate the cache after possibly using cached entries for block - // lookup to unspend coins in them - db.InvalidateCache() - - return db.delFromDB(keepidx) -} - -func (db *SqliteDb) delFromDB(keepidx int64) error { - tx := &db.txState - _, err := tx.tx.Exec("DELETE FROM txtmp WHERE blockid > ?", keepidx) - if err != nil { - // XXX - db.endTx(false) - return err - } - - _, err = tx.tx.Exec("DELETE FROM tx WHERE blockid > ?", keepidx) - if err != nil { - // XXX - db.endTx(false) - return err - } - - // delete from block last in case of foreign keys - _, err = tx.tx.Exec("DELETE FROM block WHERE blockid > ?", keepidx) - if err != nil { - // XXX - db.endTx(false) - return err - } - - err = db.endTx(true) - if err != nil { - return err - } - return err -} - -// InsertBlock inserts raw block and transaction data from a block into the -// database. The first block inserted into the database will be treated as the -// genesis block. Every subsequent block insert requires the referenced parent -// block to already exist. -func (db *SqliteDb) InsertBlock(block *btcutil.Block) (int64, error) { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - blocksha, err := block.Sha() - if err != nil { - log.Warnf("Failed to compute block sha %v", blocksha) - return -1, err - } - - mblock := block.MsgBlock() - rawMsg, err := block.Bytes() - if err != nil { - log.Warnf("Failed to obtain raw block sha %v", blocksha) - return -1, err - } - txloc, err := block.TxLoc() - if err != nil { - log.Warnf("Failed to obtain raw block sha %v", blocksha) - return -1, err - } - - // Insert block into database - newheight, err := db.insertBlockData(blocksha, &mblock.Header.PrevBlock, - 0, rawMsg) - if err != nil { - log.Warnf("Failed to insert block %v %v %v", blocksha, - &mblock.Header.PrevBlock, err) - return -1, err - } - - txinsertidx := -1 - success := false - - defer func() { - if success { - return - } - - for txidx := 0; txidx <= txinsertidx; txidx++ { - tx := mblock.Transactions[txidx] - - err = db.unSpend(tx) - if err != nil { - log.Warnf("unSpend error during block insert unwind %v %v %v", blocksha, txidx, err) - } - } - - err = db.delFromDB(newheight - 1) - if err != nil { - log.Warnf("Error during block insert unwind %v %v", blocksha, err) - } - }() - - // At least two blocks in the long past were generated by faulty - // miners, the sha of the transaction exists in a previous block, - // detect this condition and 'accept' the block. - for txidx, tx := range mblock.Transactions { - var txsha btcwire.ShaHash - txsha, err = tx.TxSha() - if err != nil { - log.Warnf("failed to compute tx name block %v idx %v err %v", blocksha, txidx, err) - return -1, err - } - - // num tx inserted, thus would need unwind if failure occurs - txinsertidx = txidx - - // Some old blocks contain duplicate transactions - // Attempt to cleanly bypass this problem - // http://blockexplorer.com/b/91842 - // http://blockexplorer.com/b/91880 - if newheight == 91842 { - dupsha, err := btcwire.NewShaHashFromStr("d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599") - if err != nil { - panic("invalid sha string in source") - } - if txsha == *dupsha { - log.Tracef("skipping sha %v %v", dupsha, newheight) - continue - } - } - if newheight == 91880 { - dupsha, err := btcwire.NewShaHashFromStr("e3bf3d07d4b0375638d5f1db5255fe07ba2c4cb067cd81b84ee974b6585fb468") - if err != nil { - panic("invalid sha string in source") - } - if txsha == *dupsha { - log.Tracef("skipping sha %v %v", dupsha, newheight) - continue - } - } - spentbuflen := (len(tx.TxOut) + 7) / 8 - spentbuf := make([]byte, spentbuflen, spentbuflen) - if len(tx.TxOut)%8 != 0 { - for i := uint(len(tx.TxOut) % 8); i < 8; i++ { - spentbuf[spentbuflen-1] |= (byte(1) << i) - } - } - - err = db.insertTx(&txsha, newheight, txloc[txidx].TxStart, txloc[txidx].TxLen, spentbuf) - if err != nil { - log.Warnf("block %v idx %v failed to insert tx %v %v err %v", blocksha, newheight, &txsha, txidx, err) - var oBlkIdx int64 - oBlkIdx, _, _, err = db.fetchLocationBySha(&txsha) - log.Warnf("oblkidx %v err %v", oBlkIdx, err) - - return -1, err - } - err = db.doSpend(tx) - if err != nil { - log.Warnf("block %v idx %v failed to spend tx %v %v err %v", blocksha, newheight, &txsha, txidx, err) - - return -1, err - } - } - success = true - db.syncPoint() - return newheight, nil -} - -// SetDBInsertMode provides hints to the database to how the application -// is running this allows the database to work in optimized modes when the -// database may be very busy. -func (db *SqliteDb) SetDBInsertMode(newmode btcdb.InsertMode) { - - oldMode := db.dbInsertMode - switch newmode { - case btcdb.InsertNormal: - // Normal mode inserts tx directly into the tx table - db.UseTempTX = false - db.dbInsertMode = newmode - switch oldMode { - case btcdb.InsertFast: - if db.TempTblSz != 0 { - err := db.migrateTmpTable() - if err != nil { - return - } - } - case btcdb.InsertValidatedInput: - // generate tx indexes - txop := db.txop(txMigrateFinish) - _, err := txop.Exec() - if err != nil { - log.Warnf("Failed to create tx table index - %v", err) - } - } - case btcdb.InsertFast: - // Fast mode inserts tx into txtmp with validation, - // then dumps to tx then rebuilds indexes at thresholds - db.UseTempTX = true - if oldMode != btcdb.InsertNormal { - log.Warnf("switching between invalid DB modes") - break - } - db.dbInsertMode = newmode - case btcdb.InsertValidatedInput: - // ValidatedInput mode inserts into tx table with - // no duplicate checks, then builds index on exit from - // ValidatedInput mode - if oldMode != btcdb.InsertNormal { - log.Warnf("switching between invalid DB modes") - break - } - // remove tx table index - txop := db.txop(txMigratePrep) - _, err := txop.Exec() - if err != nil { - log.Warnf("Failed to clear tx table index - %v", err) - } - db.dbInsertMode = newmode - - // XXX - db.UseTempTX = false - } -} -func (db *SqliteDb) doSpend(tx *btcwire.MsgTx) error { - for txinidx := range tx.TxIn { - txin := tx.TxIn[txinidx] - - inTxSha := txin.PreviousOutpoint.Hash - inTxidx := txin.PreviousOutpoint.Index - - if inTxidx == ^uint32(0) { - continue - } - - //log.Infof("spending %v %v", &inTxSha, inTxidx) - - err := db.setSpentData(&inTxSha, inTxidx) - if err != nil { - return err - } - } - return nil -} - -func (db *SqliteDb) unSpend(tx *btcwire.MsgTx) error { - for txinidx := range tx.TxIn { - txin := tx.TxIn[txinidx] - - inTxSha := txin.PreviousOutpoint.Hash - inTxidx := txin.PreviousOutpoint.Index - - if inTxidx == ^uint32(0) { - continue - } - - err := db.clearSpentData(&inTxSha, inTxidx) - if err != nil { - return err - } - } - return nil -} - -func (db *SqliteDb) setSpentData(sha *btcwire.ShaHash, idx uint32) error { - return db.setclearSpentData(sha, idx, true) -} - -func (db *SqliteDb) clearSpentData(sha *btcwire.ShaHash, idx uint32) error { - return db.setclearSpentData(sha, idx, false) -} - -func (db *SqliteDb) setclearSpentData(txsha *btcwire.ShaHash, idx uint32, set bool) error { - var spentdata []byte - usingtmp := false - txop := db.txop(txFetchUsedByShaStmt) - row := txop.QueryRow(txsha.String()) - err := row.Scan(&spentdata) - if err != nil { - // if the error is simply didn't fine continue otherwise - // retun failure - - usingtmp = true - txop = db.txop(txtmpFetchUsedByShaStmt) - row := txop.QueryRow(txsha.String()) - err := row.Scan(&spentdata) - if err != nil { - log.Warnf("Failed to locate spent data - %v %v", txsha, err) - return err - } - } - byteidx := idx / 8 - byteoff := idx % 8 - - if set { - spentdata[byteidx] |= (byte(1) << byteoff) - } else { - spentdata[byteidx] &= ^(byte(1) << byteoff) - } - txc, cached := db.fetchTxCache(txsha) - if cached { - txc.spent = spentdata - } - - if usingtmp { - txop = db.txop(txtmpUpdateUsedByShaStmt) - } else { - txop = db.txop(txUpdateUsedByShaStmt) - } - _, err = txop.Exec(spentdata, txsha.String()) - - return err -} diff --git a/sqlite3/sqliteblock.go b/sqlite3/sqliteblock.go deleted file mode 100644 index f228dc70..00000000 --- a/sqlite3/sqliteblock.go +++ /dev/null @@ -1,293 +0,0 @@ -// Copyright (c) 2013-2014 Conformal Systems LLC. -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package sqlite3 - -import ( - "database/sql" - "github.com/conformal/btcdb" - "github.com/conformal/btcwire" - _ "github.com/mattn/go-sqlite3" -) - -// insertSha stores a block hash and its associated data block with a -// previous sha of `prevSha' and a version of `pver'. -// insertSha shall be called with db lock held -func (db *SqliteDb) insertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHash, pver uint32, buf []byte) (int64, error) { - tx := &db.txState - if tx.tx == nil { - err := db.startTx() - if err != nil { - return 0, err - } - } - - // It is an error if the previous block does not already exist in the - // database, unless there are no blocks at all. - if prevOk := db.blkExistsSha(prevSha); !prevOk { - var numBlocks uint64 - querystr := "SELECT COUNT(blockid) FROM block;" - err := tx.tx.QueryRow(querystr).Scan(&numBlocks) - if err != nil { - return 0, err - } - if numBlocks != 0 { - return 0, btcdb.PrevShaMissing - } - } - - result, err := db.blkStmts[blkInsertSha].Exec(sha.Bytes(), pver, buf) - if err != nil { - return 0, err - } - - blkid, err := result.LastInsertId() - if err != nil { - return 0, err - } - blkid -= 1 // skew between btc blockid and sql - - // Because we don't know know what the last idx is, we don't - // cache unless already cached - if db.lastBlkShaCached == true { - db.lastBlkSha = *sha - db.lastBlkIdx++ - } - - bid := tBlockInsertData{*sha, pver, buf} - tx.txInsertList = append(tx.txInsertList, bid) - tx.txDataSz += len(buf) - - return blkid, nil -} - -// fetchSha returns the datablock and pver for the given ShaHash. -func (db *SqliteDb) fetchSha(sha btcwire.ShaHash) (buf []byte, pver uint32, - blkid int64, err error) { - - row := db.blkStmts[blkFetchSha].QueryRow(sha.Bytes()) - - var blockidx int64 - var databytes []byte - err = row.Scan(&pver, &databytes, &blockidx) - if err == sql.ErrNoRows { - return // no warning - } - if err != nil { - log.Warnf("fail 2 %v", err) - return - } - buf = databytes - blkid = blockidx - 1 // skew between btc blockid and sql - return -} - -// ExistsSha looks up the given block hash -// returns true if it is present in the database. -func (db *SqliteDb) ExistsSha(sha *btcwire.ShaHash) (exists bool) { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - _, exists = db.fetchBlockCache(sha) - if exists { - return - } - - // not in cache, try database - exists = db.blkExistsSha(sha) - return -} - -// blkExistsSha looks up the given block hash -// returns true if it is present in the database. -// CALLED WITH LOCK HELD -func (db *SqliteDb) blkExistsSha(sha *btcwire.ShaHash) bool { - var pver uint32 - - row := db.blkStmts[blkExistsSha].QueryRow(sha.Bytes()) - err := row.Scan(&pver) - - if err == sql.ErrNoRows { - return false - } - - if err != nil { - // ignore real errors? - log.Warnf("blkExistsSha: fail %v", err) - return false - } - return true -} - -// FetchBlockShaByHeight returns a block hash based on its height in the -// block chain. -func (db *SqliteDb) FetchBlockShaByHeight(height int64) (sha *btcwire.ShaHash, err error) { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - return db.fetchBlockShaByHeight(height) -} - -// fetchBlockShaByHeight returns a block hash based on its height in the -// block chain. -func (db *SqliteDb) fetchBlockShaByHeight(height int64) (sha *btcwire.ShaHash, err error) { - var row *sql.Row - - blockidx := height + 1 // skew between btc blockid and sql - - row = db.blkStmts[blkFetchIdx].QueryRow(blockidx) - - var shabytes []byte - err = row.Scan(&shabytes) - if err != nil { - return - } - var shaval btcwire.ShaHash - shaval.SetBytes(shabytes) - return &shaval, nil -} - -// FetchHeightRange looks up a range of blocks by the start and ending -// heights. Fetch is inclusive of the start height and exclusive of the -// ending height. To fetch all hashes from the start height until no -// more are present, use the special id `AllShas'. -func (db *SqliteDb) FetchHeightRange(startHeight, endHeight int64) (rshalist []btcwire.ShaHash, err error) { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - startidx := startHeight + 1 // skew between btc block height and sql - - var endidx int64 - if endHeight == btcdb.AllShas { - endidx = btcdb.AllShas // no skew if asking for all - } else { - endidx = endHeight + 1 // skew between btc block height and sql - } - rows, err := db.blkStmts[blkFetchIdxList].Query(startidx, endidx) - if err != nil { - log.Warnf("query failed %v", err) - return - } - - var shalist []btcwire.ShaHash - for rows.Next() { - var sha btcwire.ShaHash - var shabytes []byte - err = rows.Scan(&shabytes) - if err != nil { - log.Warnf("wtf? %v", err) - break - } - sha.SetBytes(shabytes) - shalist = append(shalist, sha) - } - rows.Close() - if err == nil { - rshalist = shalist - } - log.Tracef("FetchIdxRange idx %v %v returned %v shas err %v", startHeight, endHeight, len(shalist), err) - return -} - -// NewestSha returns the hash and block height of the most recent (end) block of -// the block chain. It will return the zero hash, -1 for the block height, and -// no error (nil) if there are not any blocks in the database yet. -func (db *SqliteDb) NewestSha() (sha *btcwire.ShaHash, blkid int64, err error) { - var row *sql.Row - var blockidx int64 - db.dbLock.Lock() - defer db.dbLock.Unlock() - - // answer may be cached - if db.lastBlkShaCached == true { - shacopy := db.lastBlkSha - sha = &shacopy - blkid = db.lastBlkIdx - 1 // skew between btc blockid and sql - return - } - - querystr := "SELECT key, blockid FROM block ORDER BY blockid DESC;" - - tx := &db.txState - if tx.tx != nil { - row = tx.tx.QueryRow(querystr) - } else { - row = db.sqldb.QueryRow(querystr) - } - - var shabytes []byte - err = row.Scan(&shabytes, &blockidx) - if err == sql.ErrNoRows { - return &btcwire.ShaHash{}, -1, nil - } - if err == nil { - var retsha btcwire.ShaHash - retsha.SetBytes(shabytes) - sha = &retsha - blkid = blockidx - 1 // skew between btc blockid and sql - - db.lastBlkSha = retsha - db.lastBlkIdx = blockidx - db.lastBlkShaCached = true - } - return -} - -type SqliteBlockIterator struct { - rows *sql.Rows - stmt *sql.Stmt - db *SqliteDb -} - -// NextRow iterates thru all blocks in database. -func (bi *SqliteBlockIterator) NextRow() bool { - return bi.rows.Next() -} - -// Row returns row data for block iterator. -func (bi *SqliteBlockIterator) Row() (key *btcwire.ShaHash, pver uint32, - buf []byte, err error) { - var keybytes []byte - - err = bi.rows.Scan(&keybytes, &pver, &buf) - if err == nil { - var retkey btcwire.ShaHash - retkey.SetBytes(keybytes) - key = &retkey - } - return -} - -// Close shuts down the iterator when done walking blocks in the database. -func (bi *SqliteBlockIterator) Close() { - bi.rows.Close() - bi.stmt.Close() -} - -// NewIterateBlocks prepares iterator for all blocks in database. -func (db *SqliteDb) NewIterateBlocks() (btcdb.BlockIterator, error) { - var bi SqliteBlockIterator - db.dbLock.Lock() - defer db.dbLock.Unlock() - - stmt, err := db.sqldb.Prepare("SELECT key, pver, data FROM block ORDER BY blockid;") - if err != nil { - return nil, err - } - tx := &db.txState - if tx.tx != nil { - txstmt := tx.tx.Stmt(stmt) - stmt.Close() - stmt = txstmt - } - bi.stmt = stmt - - bi.rows, err = bi.stmt.Query() - if err != nil { - return nil, err - } - bi.db = db - - return &bi, nil -} diff --git a/sqlite3/sqlitedbcache.go b/sqlite3/sqlitedbcache.go deleted file mode 100644 index 9b87b498..00000000 --- a/sqlite3/sqlitedbcache.go +++ /dev/null @@ -1,394 +0,0 @@ -// Copyright (c) 2013-2014 Conformal Systems LLC. -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package sqlite3 - -import ( - "bytes" - "container/list" - "github.com/conformal/btcdb" - "github.com/conformal/btcutil" - "github.com/conformal/btcwire" - "sync" -) - -type txCache struct { - maxcount int - fifo list.List - // NOTE: the key is specifically ShaHash, not *ShaHash - txMap map[btcwire.ShaHash]*txCacheObj - cacheLock sync.RWMutex -} - -type txCacheObj struct { - next *txCacheObj - sha btcwire.ShaHash - blksha btcwire.ShaHash - pver uint32 - tx *btcwire.MsgTx - height int64 - spent []byte - txbuf []byte -} - -type blockCache struct { - maxcount int - fifo list.List - blockMap map[btcwire.ShaHash]*blockCacheObj - blockHeightMap map[int64]*blockCacheObj - cacheLock sync.RWMutex -} - -type blockCacheObj struct { - next *blockCacheObj - sha btcwire.ShaHash - blk *btcutil.Block -} - -// FetchBlockBySha - return a btcutil Block, object may be a cached. -func (db *SqliteDb) FetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) { - db.dbLock.Lock() - defer db.dbLock.Unlock() - return db.fetchBlockBySha(sha) -} - -// fetchBlockBySha - return a btcutil Block, object may be a cached. -// Must be called with db lock held. -func (db *SqliteDb) fetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) { - - blkcache, ok := db.fetchBlockCache(sha) - if ok { - return blkcache.blk, nil - } - - buf, _, height, err := db.fetchSha(*sha) - if err != nil { - return nil, err - } - - blk, err = btcutil.NewBlockFromBytes(buf) - if err != nil { - return - } - blk.SetHeight(height) - db.insertBlockCache(sha, blk) - - return -} - -// fetchBlockCache check if a block is in the block cache, if so return it. -func (db *SqliteDb) fetchBlockCache(sha *btcwire.ShaHash) (*blockCacheObj, bool) { - - db.blockCache.cacheLock.RLock() - defer db.blockCache.cacheLock.RUnlock() - - blkobj, ok := db.blockCache.blockMap[*sha] - if !ok { // could this just return the map deref? - return nil, false - } - return blkobj, true -} - -// fetchBlockHeightCache check if a block is in the block cache, if so return it. -func (db *SqliteDb) fetchBlockHeightCache(height int64) (*blockCacheObj, bool) { - - db.blockCache.cacheLock.RLock() - defer db.blockCache.cacheLock.RUnlock() - - blkobj, ok := db.blockCache.blockHeightMap[height] - if !ok { // could this just return the map deref? - return nil, false - } - return blkobj, true -} - -// insertBlockCache insert the given sha/block into the cache map. -// If the block cache is determined to be full, it will release -// an old entry in FIFO order. -func (db *SqliteDb) insertBlockCache(sha *btcwire.ShaHash, blk *btcutil.Block) { - bc := &db.blockCache - - bc.cacheLock.Lock() - defer bc.cacheLock.Unlock() - - blkObj := blockCacheObj{sha: *sha, blk: blk} - bc.fifo.PushBack(&blkObj) - - if bc.fifo.Len() > bc.maxcount { - listobj := bc.fifo.Front() - bc.fifo.Remove(listobj) - tailObj, ok := listobj.Value.(*blockCacheObj) - if ok { - delete(bc.blockMap, tailObj.sha) - delete(bc.blockHeightMap, tailObj.blk.Height()) - } else { - panic("invalid type pushed on blockCache list") - } - } - - bc.blockHeightMap[blk.Height()] = &blkObj - bc.blockMap[blkObj.sha] = &blkObj -} - -// FetchTxByShaList returns the most recent tx of the name fully spent or not -func (db *SqliteDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { - // until the fully spent separation of tx is complete this is identical - // to FetchUnSpentTxByShaList - return db.FetchUnSpentTxByShaList(txShaList) -} - -// FetchUnSpentTxByShaList given a array of ShaHash, look up the transactions -// and return them in a TxListReply array. -func (db *SqliteDb) FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - var replies []*btcdb.TxListReply - for _, txsha := range txShaList { - tx, _, _, _, height, txspent, err := db.fetchTxDataBySha(txsha) - btxspent := []bool{} - if err == nil { - btxspent = make([]bool, len(tx.TxOut), len(tx.TxOut)) - for idx := range tx.TxOut { - byteidx := idx / 8 - byteoff := uint(idx % 8) - btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0 - } - } - txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, Height: height, TxSpent: btxspent, Err: err} - replies = append(replies, &txlre) - } - return replies -} - -// fetchTxDataBySha returns several pieces of data regarding the given sha. -func (db *SqliteDb) fetchTxDataBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, rtxbuf []byte, rpver uint32, rblksha *btcwire.ShaHash, rheight int64, rtxspent []byte, err error) { - - var pver uint32 - var blksha *btcwire.ShaHash - var height int64 - var txspent []byte - var toff int - var tlen int - var blk *btcutil.Block - var blkbuf []byte - - // Check Tx cache - if txc, ok := db.fetchTxCache(txsha); ok { - if txc.spent != nil { - return txc.tx, txc.txbuf, txc.pver, &txc.blksha, txc.height, txc.spent, nil - } - } - - // If not cached load it - height, toff, tlen, txspent, err = db.fetchLocationUsedBySha(txsha) - if err != nil { - return - } - - blksha, err = db.fetchBlockShaByHeight(height) - if err != nil { - log.Warnf("block idx lookup %v to %v", height, err) - return - } - log.Tracef("transaction %v is at block %v %v tx %v", - txsha, blksha, height, toff) - - blk, err = db.fetchBlockBySha(blksha) - if err != nil { - log.Warnf("unable to fetch block %v %v ", - height, &blksha) - return - } - - blkbuf, err = blk.Bytes() - if err != nil { - log.Warnf("unable to decode block %v %v", height, &blksha) - return - } - - txbuf := make([]byte, tlen) - copy(txbuf[:], blkbuf[toff:toff+tlen]) - rbuf := bytes.NewBuffer(txbuf) - - var tx btcwire.MsgTx - err = tx.BtcDecode(rbuf, pver) - if err != nil { - log.Warnf("unable to decode tx block %v %v txoff %v txlen %v", - height, &blksha, toff, tlen) - return - } - - // Shove data into TxCache - // XXX - - var txc txCacheObj - txc.sha = *txsha - txc.tx = &tx - txc.txbuf = txbuf - txc.pver = pver - txc.height = height - txc.spent = txspent - txc.blksha = *blksha - db.insertTxCache(&txc) - - return &tx, txbuf, pver, blksha, height, txspent, nil -} - -// FetchTxBySha returns several pieces of data regarding the given sha. -func (db *SqliteDb) FetchTxBySha(txsha *btcwire.ShaHash) ([]*btcdb.TxListReply, error) { - var pver uint32 - var blksha *btcwire.ShaHash - var height int64 - var toff int - var tlen int - var txspent []byte - var blk *btcutil.Block - var blkbuf []byte - var err error - - // Check Tx cache - if txc, ok := db.fetchTxCache(txsha); ok { - replies := make([]*btcdb.TxListReply, 1) - - tx := txc.tx - btxspent := make([]bool, len(tx.TxOut), len(tx.TxOut)) - for idx := range tx.TxOut { - byteidx := idx / 8 - byteoff := uint(idx % 8) - btxspent[idx] = (txc.spent[byteidx] & (byte(1) << byteoff)) != 0 - } - - txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, BlkSha: &txc.blksha, Height: txc.height, TxSpent: btxspent, Err: nil} - replies[0] = &txlre - return replies, nil - } - - // If not cached load it - height, toff, tlen, txspent, err = db.fetchLocationUsedBySha(txsha) - if err != nil { - return []*btcdb.TxListReply{}, err - } - - blksha, err = db.FetchBlockShaByHeight(height) - if err != nil { - log.Warnf("block idx lookup %v to %v", height, err) - return []*btcdb.TxListReply{}, err - } - log.Tracef("transaction %v is at block %v %v tx %v", - txsha, blksha, height, toff) - - blk, err = db.FetchBlockBySha(blksha) - if err != nil { - log.Warnf("unable to fetch block %v %v ", - height, &blksha) - return []*btcdb.TxListReply{}, err - } - - blkbuf, err = blk.Bytes() - if err != nil { - log.Warnf("unable to decode block %v %v", height, &blksha) - return []*btcdb.TxListReply{}, err - } - - txbuf := make([]byte, tlen) - copy(txbuf[:], blkbuf[toff:toff+tlen]) - rbuf := bytes.NewBuffer(txbuf) - - var tx btcwire.MsgTx - err = tx.BtcDecode(rbuf, pver) - if err != nil { - log.Warnf("unable to decode tx block %v %v txoff %v txlen %v", - height, &blksha, toff, tlen) - return []*btcdb.TxListReply{}, err - } - - // Shove data into TxCache - // XXX - - var txc txCacheObj - txc.sha = *txsha - txc.tx = &tx - txc.txbuf = txbuf - txc.pver = pver - txc.height = height - txc.spent = txspent - txc.blksha = *blksha - db.insertTxCache(&txc) - - btxspent := make([]bool, len(tx.TxOut), len(tx.TxOut)) - for idx := range tx.TxOut { - byteidx := idx / 8 - byteoff := uint(idx % 8) - btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0 - } - - replies := make([]*btcdb.TxListReply, 1) - txlre := btcdb.TxListReply{Sha: txsha, Tx: &tx, BlkSha: blksha, Height: height, TxSpent: btxspent, Err: err} - replies[0] = &txlre - return replies, nil -} - -// fetchTxCache look up the given transaction in the Tx cache. -func (db *SqliteDb) fetchTxCache(sha *btcwire.ShaHash) (*txCacheObj, bool) { - tc := &db.txCache - - tc.cacheLock.RLock() - defer tc.cacheLock.RUnlock() - - txObj, ok := tc.txMap[*sha] - if !ok { // could this just return the map deref? - return nil, false - } - return txObj, true -} - -// insertTxCache, insert the given txobj into the cache. -// if the tx cache is determined to be full, it will release -// an old entry in FIFO order. -func (db *SqliteDb) insertTxCache(txObj *txCacheObj) { - tc := &db.txCache - - tc.cacheLock.Lock() - defer tc.cacheLock.Unlock() - - tc.fifo.PushBack(txObj) - - if tc.fifo.Len() >= tc.maxcount { - listobj := tc.fifo.Front() - tc.fifo.Remove(listobj) - tailObj, ok := listobj.Value.(*txCacheObj) - if ok { - delete(tc.txMap, tailObj.sha) - } else { - panic("invalid type pushed on tx list") - } - - } - - tc.txMap[txObj.sha] = txObj -} - -// InvalidateTxCache clear/release all cached transactions. -func (db *SqliteDb) InvalidateTxCache() { - tc := &db.txCache - tc.cacheLock.Lock() - defer tc.cacheLock.Unlock() - tc.txMap = map[btcwire.ShaHash]*txCacheObj{} - tc.fifo = list.List{} -} - -// InvalidateTxCache clear/release all cached blocks. -func (db *SqliteDb) InvalidateBlockCache() { - bc := &db.blockCache - bc.cacheLock.Lock() - defer bc.cacheLock.Unlock() - bc.blockMap = map[btcwire.ShaHash]*blockCacheObj{} - bc.blockHeightMap = map[int64]*blockCacheObj{} - bc.fifo = list.List{} -} - -// InvalidateCache clear/release all cached blocks and transactions. -func (db *SqliteDb) InvalidateCache() { - db.InvalidateTxCache() - db.InvalidateBlockCache() -} diff --git a/sqlite3/sqlitetx.go b/sqlite3/sqlitetx.go deleted file mode 100644 index d5963e19..00000000 --- a/sqlite3/sqlitetx.go +++ /dev/null @@ -1,327 +0,0 @@ -// Copyright (c) 2013-2014 Conformal Systems LLC. -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package sqlite3 - -import ( - "database/sql" - "github.com/conformal/btcdb" - "github.com/conformal/btcwire" - _ "github.com/mattn/go-sqlite3" -) - -// insertTx inserts a tx hash and its associated data into the database. -// Must be called with db lock held. -func (db *SqliteDb) insertTx(txsha *btcwire.ShaHash, height int64, txoff int, txlen int, usedbuf []byte) (err error) { - - tx := &db.txState - if tx.tx == nil { - err = db.startTx() - if err != nil { - return - } - } - blockid := height + 1 - txd := tTxInsertData{txsha: txsha, blockid: blockid, txoff: txoff, txlen: txlen, usedbuf: usedbuf} - - log.Tracef("inserting tx %v for block %v off %v len %v", - txsha, blockid, txoff, txlen) - - rowBytes := txsha.String() - - var op int // which table to insert data into. - if db.UseTempTX { - var tblockid int64 - var ttxoff int - var ttxlen int - txop := db.txop(txFetchLocationByShaStmt) - row := txop.QueryRow(rowBytes) - err = row.Scan(&tblockid, &ttxoff, &ttxlen) - if err != sql.ErrNoRows { - // sha already present - err = btcdb.DuplicateSha - return - } - op = txtmpInsertStmt - } else { - op = txInsertStmt - } - - txop := db.txop(op) - _, err = txop.Exec(rowBytes, blockid, txoff, txlen, usedbuf) - if err != nil { - log.Warnf("failed to insert %v %v %v", txsha, blockid, err) - return - } - if db.UseTempTX { - db.TempTblSz++ - } - - // put in insert list for replay - tx.txInsertList = append(tx.txInsertList, txd) - - return -} - -// ExistsTxSha returns if the given tx sha exists in the database -func (db *SqliteDb) ExistsTxSha(txsha *btcwire.ShaHash) (exists bool) { - db.dbLock.Lock() - defer db.dbLock.Unlock() - - if _, ok := db.fetchTxCache(txsha); ok { - return true - } - - return db.existsTxSha(txsha) -} - -// existsTxSha returns if the given tx sha exists in the database.o -// Must be called with the db lock held. -func (db *SqliteDb) existsTxSha(txsha *btcwire.ShaHash) (exists bool) { - var blockid uint32 - - txop := db.txop(txExistsShaStmt) - row := txop.QueryRow(txsha.String()) - err := row.Scan(&blockid) - - if err == sql.ErrNoRows { - txop = db.txop(txtmpExistsShaStmt) - row = txop.QueryRow(txsha.String()) - err := row.Scan(&blockid) - - if err == sql.ErrNoRows { - return false - } - if err != nil { - log.Warnf("txTmpExistsTxSha: fail %v", err) - return false - } - log.Warnf("txtmpExistsTxSha: success") - return true - } - - if err != nil { - // ignore real errors? - log.Warnf("existsTxSha: fail %v", err) - return false - } - - return true -} - -// FetchLocationBySha looks up the Tx sha information by name. -func (db *SqliteDb) FetchLocationBySha(txsha *btcwire.ShaHash) (blockidx int64, txoff int, txlen int, err error) { - db.dbLock.Lock() - defer db.dbLock.Unlock() - return db.fetchLocationBySha(txsha) -} - -// fetchLocationBySha look up the Tx sha information by name. -// Must be called with db lock held. -func (db *SqliteDb) fetchLocationBySha(txsha *btcwire.ShaHash) (height int64, txoff int, txlen int, err error) { - var row *sql.Row - var blockid int64 - var ttxoff int - var ttxlen int - - rowBytes := txsha.String() - txop := db.txop(txFetchLocationByShaStmt) - row = txop.QueryRow(rowBytes) - - err = row.Scan(&blockid, &ttxoff, &ttxlen) - if err == sql.ErrNoRows { - txop = db.txop(txtmpFetchLocationByShaStmt) - row = txop.QueryRow(rowBytes) - - err = row.Scan(&blockid, &ttxoff, &ttxlen) - if err == sql.ErrNoRows { - err = btcdb.TxShaMissing - return - } - if err != nil { - log.Warnf("txtmp FetchLocationBySha: fail %v", - err) - return - } - } - if err != nil { - log.Warnf("FetchLocationBySha: fail %v", err) - return - } - height = blockid - 1 - txoff = ttxoff - txlen = ttxlen - return -} - -// fetchLocationUsedBySha look up the Tx sha information by name. -// Must be called with db lock held. -func (db *SqliteDb) fetchLocationUsedBySha(txsha *btcwire.ShaHash) (rheight int64, rtxoff int, rtxlen int, rspentbuf []byte, err error) { - var row *sql.Row - var blockid int64 - var txoff int - var txlen int - var txspent []byte - - rowBytes := txsha.String() - txop := db.txop(txFetchLocUsedByShaStmt) - row = txop.QueryRow(rowBytes) - - err = row.Scan(&blockid, &txoff, &txlen, &txspent) - if err == sql.ErrNoRows { - txop = db.txop(txtmpFetchLocUsedByShaStmt) - row = txop.QueryRow(rowBytes) - - err = row.Scan(&blockid, &txoff, &txlen, &txspent) - if err == sql.ErrNoRows { - err = btcdb.TxShaMissing - return - } - if err != nil { - log.Warnf("txtmp FetchLocationBySha: fail %v", - err) - return - } - } - if err != nil { - log.Warnf("FetchLocationBySha: fail %v", err) - return - } - height := blockid - 1 - return height, txoff, txlen, txspent, nil -} - -// FetchTxUsedBySha returns the used/spent buffer for a given transaction. -func (db *SqliteDb) FetchTxUsedBySha(txsha *btcwire.ShaHash) (spentbuf []byte, err error) { - var row *sql.Row - db.dbLock.Lock() - defer db.dbLock.Unlock() - - rowBytes := txsha.String() - txop := db.txop(txFetchUsedByShaStmt) - row = txop.QueryRow(rowBytes) - - var databytes []byte - err = row.Scan(&databytes) - if err == sql.ErrNoRows { - txop := db.txop(txtmpFetchUsedByShaStmt) - row = txop.QueryRow(rowBytes) - - err = row.Scan(&databytes) - if err == sql.ErrNoRows { - err = btcdb.TxShaMissing - return - } - if err != nil { - log.Warnf("txtmp FetchLocationBySha: fail %v", - err) - return - } - } - - if err != nil { - log.Warnf("FetchUsedBySha: fail %v", err) - return - } - spentbuf = databytes - return -} - -var vaccumDbNextMigrate bool - -// migrateTmpTable functions to perform internal db optimization when -// performing large numbers of database inserts. When in Fast operation -// mode, it inserts into txtmp, then when that table reaches a certain -// size limit it moves all tx in the txtmp table into the primary tx -// table and recomputes the index on the primary tx table. -func (db *SqliteDb) migrateTmpTable() error { - db.endTx(true) - db.startTx() // ??? - - db.UseTempTX = false - db.TempTblSz = 0 - - var doVacuum bool - var nsteps int - if vaccumDbNextMigrate { - nsteps = 6 - vaccumDbNextMigrate = false - doVacuum = true - } else { - nsteps = 5 - vaccumDbNextMigrate = true - } - - log.Infof("db compaction Stage 1/%v: Preparing", nsteps) - txop := db.txop(txMigratePrep) - _, err := txop.Exec() - if err != nil { - log.Warnf("Failed to prepare migrate - %v", err) - return err - } - - log.Infof("db compaction Stage 2/%v: Copying", nsteps) - txop = db.txop(txMigrateCopy) - _, err = txop.Exec() - if err != nil { - log.Warnf("Migrate read failed - %v", err) - return err - } - - log.Tracef("db compaction Stage 2a/%v: Enable db vacuum", nsteps) - txop = db.txop(txPragmaVacuumOn) - _, err = txop.Exec() - if err != nil { - log.Warnf("Migrate error trying to enable vacuum on "+ - "temporary transaction table - %v", err) - return err - } - - log.Infof("db compaction Stage 3/%v: Clearing old data", nsteps) - txop = db.txop(txMigrateClear) - _, err = txop.Exec() - if err != nil { - log.Warnf("Migrate error trying to clear temporary "+ - "transaction table - %v", err) - return err - } - - log.Tracef("db compaction Stage 3a/%v: Disable db vacuum", nsteps) - txop = db.txop(txPragmaVacuumOff) - _, err = txop.Exec() - if err != nil { - log.Warnf("Migrate error trying to disable vacuum on "+ - "temporary transaction table - %v", err) - return err - } - - log.Infof("db compaction Stage 4/%v: Rebuilding index", nsteps) - txop = db.txop(txMigrateFinish) - _, err = txop.Exec() - if err != nil { - log.Warnf("Migrate error trying to clear temporary "+ - "transaction table - %v", err) - return err - } - - log.Infof("db compaction Stage 5/%v: Finalizing transaction", nsteps) - db.endTx(true) // ??? - - if doVacuum { - log.Infof("db compaction Stage 6/%v: Optimizing database", nsteps) - txop = db.txop(txVacuum) - _, err = txop.Exec() - if err != nil { - log.Warnf("migrate error trying to clear txtmp tbl %v", err) - return err - } - } - - log.Infof("db compaction: Complete") - - // TODO(drahn) - determine if this should be turned back on or not - db.UseTempTX = true - - return nil -} From d6c2492c7f5a6df1f5bc3ed0b2058fbc7bd6b37a Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sat, 18 Jan 2014 20:59:53 -0600 Subject: [PATCH 115/163] Add FetchBlockHeightBySha/FetchBlockHeaderBySha. This commit introduces two new functions to the btcdb.Db interface named FetchBlockHeightBySha and FetchBlockHeaderBySha. The FetchBlockHeightBySha function is useful since previously it was only possible to get the height of block by fetching the entire block with FetchBlockBySha and pulling the height out of the returned btcutil.Block. The FetchBlockHeaderBySha function will ultimately make it much more efficient to fetch block headers. Currently, due to the database design in the ldb backend, the entire block has to be loaded anyways, so the only current benefit is to avoid the deserialize on all of the transactions. However, ultimately btcdb will gain a more efficient backend which can also avoid reading all of the extra transaction data altogether. --- db.go | 7 +++++++ ldb/block.go | 32 ++++++++++++++++++++++++++++++++ memdb/memdb.go | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+) diff --git a/db.go b/db.go index a1144ec0..68853f68 100644 --- a/db.go +++ b/db.go @@ -59,6 +59,13 @@ type Db interface { // cache the underlying data if desired. FetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) + // FetchBlockHeightBySha returns the block height for the given hash. + FetchBlockHeightBySha(sha *btcwire.ShaHash) (height int64, err error) + + // FetchBlockHeaderBySha returns a btcwire.BlockHeader for the given + // sha. The implementation may cache the underlying data if desired. + FetchBlockHeaderBySha(sha *btcwire.ShaHash) (bh *btcwire.BlockHeader, err error) + // FetchBlockShaByHeight returns a block hash based on its height in the // block chain. FetchBlockShaByHeight(height int64) (sha *btcwire.ShaHash, err error) diff --git a/ldb/block.go b/ldb/block.go index bcfa0696..64f2053c 100644 --- a/ldb/block.go +++ b/ldb/block.go @@ -38,6 +38,38 @@ func (db *LevelDb) fetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, er return } +// FetchBlockHeightBySha returns the block height for the given hash. This is +// part of the btcdb.Db interface implementation. +func (db *LevelDb) FetchBlockHeightBySha(sha *btcwire.ShaHash) (int64, error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + return db.getBlkLoc(sha) +} + +// FetchBlockHeaderBySha - return a btcwire ShaHash +func (db *LevelDb) FetchBlockHeaderBySha(sha *btcwire.ShaHash) (bh *btcwire.BlockHeader, err error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + // Read the raw block from the database. + buf, _, err := db.fetchSha(sha) + if err != nil { + return nil, err + } + + // Only deserialize the header portion and ensure the transaction count + // is zero since this is a standalone header. + var blockHeader btcwire.BlockHeader + err = blockHeader.Deserialize(bytes.NewBuffer(buf)) + if err != nil { + return nil, err + } + bh = &blockHeader + + return bh, err +} + func (db *LevelDb) getBlkLoc(sha *btcwire.ShaHash) (int64, error) { var blkHeight int64 diff --git a/memdb/memdb.go b/memdb/memdb.go index 4b805627..e9bd795a 100644 --- a/memdb/memdb.go +++ b/memdb/memdb.go @@ -230,6 +230,44 @@ func (db *MemDb) FetchBlockBySha(sha *btcwire.ShaHash) (*btcutil.Block, error) { return nil, fmt.Errorf("block %v is not in database", sha) } +// FetchBlockHeightBySha returns the block height for the given hash. This is +// part of the btcdb.Db interface implementation. +func (db *MemDb) FetchBlockHeightBySha(sha *btcwire.ShaHash) (int64, error) { + db.Lock() + defer db.Unlock() + + if db.closed { + return 0, ErrDbClosed + } + + if blockHeight, exists := db.blocksBySha[*sha]; exists { + return blockHeight, nil + } + + return 0, fmt.Errorf("block %v is not in database", sha) +} + +// FetchBlockHeaderBySha returns a btcwire.BlockHeader for the given sha. The +// implementation may cache the underlying data if desired. This is part of the +// btcdb.Db interface implementation. +// +// This implementation does not use any additional cache since the entire +// database is already in memory. +func (db *MemDb) FetchBlockHeaderBySha(sha *btcwire.ShaHash) (*btcwire.BlockHeader, error) { + db.Lock() + defer db.Unlock() + + if db.closed { + return nil, ErrDbClosed + } + + if blockHeight, exists := db.blocksBySha[*sha]; exists { + return &db.blocks[int(blockHeight)].Header, nil + } + + return nil, fmt.Errorf("block header %v is not in database", sha) +} + // FetchBlockShaByHeight returns a block hash based on its height in the block // chain. This is part of the btcdb.Db interface implementation. func (db *MemDb) FetchBlockShaByHeight(height int64) (*btcwire.ShaHash, error) { From 8c34084e24be362a296e3065c909d289953a7c21 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sun, 19 Jan 2014 01:37:25 -0600 Subject: [PATCH 116/163] Add interface tests for new Fetch functions. --- interface_test.go | 60 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/interface_test.go b/interface_test.go index a19e9978..19f56c79 100644 --- a/interface_test.go +++ b/interface_test.go @@ -135,6 +135,54 @@ func testFetchBlockBySha(tc *testContext) bool { return true } +// testFetchBlockHeightBySha ensures FetchBlockHeightBySha conforms to the +// interface contract. +func testFetchBlockHeightBySha(tc *testContext) bool { + // The block height must be fetchable by its hash without any errors. + blockHeight, err := tc.db.FetchBlockHeightBySha(tc.blockHash) + if err != nil { + tc.t.Errorf("FetchBlockHeightBySha (%s): block #%d (%s) err: %v", + tc.dbType, tc.blockHeight, tc.blockHash, err) + return false + } + + // The block height fetched from the database must match the expected + // height. + if blockHeight != tc.blockHeight { + tc.t.Errorf("FetchBlockHeightBySha (%s): block #%d (%s) height "+ + "does not match expected value - got: %v", tc.dbType, + tc.blockHeight, tc.blockHash, blockHeight) + return false + } + + return true +} + +// testFetchBlockHeaderBySha ensures FetchBlockHeaderBySha conforms to the +// interface contract. +func testFetchBlockHeaderBySha(tc *testContext) bool { + // The block header must be fetchable by its hash without any errors. + blockHeader, err := tc.db.FetchBlockHeaderBySha(tc.blockHash) + if err != nil { + tc.t.Errorf("FetchBlockHeaderBySha (%s): block #%d (%s) err: %v", + tc.dbType, tc.blockHeight, tc.blockHash, err) + return false + } + + // The block header fetched from the database must give back the same + // BlockHeader that was stored. + if !reflect.DeepEqual(&tc.block.MsgBlock().Header, blockHeader) { + tc.t.Errorf("FetchBlockHeaderBySha (%s): block header #%d (%s) "+ + " does not match stored block\ngot: %v\nwant: %v", + tc.dbType, tc.blockHeight, tc.blockHash, + spew.Sdump(blockHeader), + spew.Sdump(&tc.block.MsgBlock().Header)) + return false + } + + return true +} + // testFetchBlockShaByHeight ensures FetchBlockShaByHeight conforms to the // interface contract. func testFetchBlockShaByHeight(tc *testContext) bool { @@ -414,6 +462,18 @@ func testIntegrity(tc *testContext) bool { return false } + // The height returned for the block given its hash must be the + // expected value + if !testFetchBlockHeightBySha(tc) { + return false + } + + // Loading the header back from the database must give back + // the same BlockHeader that was stored. + if !testFetchBlockHeaderBySha(tc) { + return false + } + // The hash returned for the block by its height must be the // expected value. if !testFetchBlockShaByHeight(tc) { From 2eea55ae1dd09fa39fd8e13957291791d5d88bae Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sun, 19 Jan 2014 17:34:54 -0600 Subject: [PATCH 117/163] Prune the btcddb.Db interface. This commit prunes several unused functions from the Db interface and the underlying implementations. For the most part these are holdovers from the original sqlite implementation. It also removes the types associated with those functions since they are no longer needed. The following functions and types have been removed: - InvalidateCache - InvalidateBlockCache - InvalidateTxCache - SetDBInsertMode - InsertMode type and associated constants - NewIterateBlocks - BlockIterator interface The reasons for removing these are broken out below. - Neither of two current implementations implement these functions nor does any of the fully functional code using the interface invoke them. - After contemplating and testing caching of blocks and transactions at this layer, it doesn't seem to provide any real benefit unless very specific assumptions about the use case are made. Making those assumptions can make other use cases worse. For example, assuming a large cache is harmful to memory-constrained use cases. Leaving it up to the caller to choose when to cache block and transactions allows much greater flexibility. - The DB insert mode was an artifact of the original sqlite implementation and probably should have only been exposed specifically on the implementation as opposed through generic interface. If a specific implementation wishes to provide functionality such as special modes, that should be done through type assertions. --- db.go | 44 ---------------- interface_test.go | 91 ++++++++++++++------------------- ldb/block.go | 5 -- ldb/dbcache.go | 21 -------- ldb/insertremove_test.go | 21 ++------ ldb/leveldb.go | 8 --- ldb/operational_test.go | 106 +++++++++++---------------------------- memdb/memdb.go | 21 -------- memdb/memdb_test.go | 7 --- 9 files changed, 69 insertions(+), 255 deletions(-) delete mode 100644 ldb/dbcache.go diff --git a/db.go b/db.go index 68853f68..4e7f03e1 100644 --- a/db.go +++ b/db.go @@ -23,20 +23,6 @@ var ( // a range of shas by height to request them all. const AllShas = int64(^uint64(0) >> 1) -// InsertMode represents a hint to the database about how much data the -// application is expecting to send to the database in a short period of time. -// This in turn provides the database with the opportunity to work in optimized -// modes when it will be very busy such as during the initial block chain -// download. -type InsertMode int - -// Constants used to indicate the database insert mode hint. See InsertMode. -const ( - InsertNormal InsertMode = iota - InsertFast - InsertValidatedInput -) - // Db defines a generic interface that is used to request and insert data into // the bitcoin block chain. This interface is intended to be agnostic to actual // mechanism used for backend data storage. The AddDBDriver function can be @@ -102,18 +88,6 @@ type Db interface { // requires the referenced parent block to already exist. InsertBlock(block *btcutil.Block) (height int64, err error) - // InvalidateBlockCache releases all cached blocks. - InvalidateBlockCache() - - // InvalidateCache releases all cached blocks and transactions. - InvalidateCache() - - // InvalidateTxCache releases all cached transactions. - InvalidateTxCache() - - // NewIterateBlocks returns an iterator for all blocks in database. - NewIterateBlocks() (pbi BlockIterator, err error) - // NewestSha returns the hash and block height of the most recent (end) // block of the block chain. It will return the zero hash, -1 for // the block height, and no error (nil) if there are not any blocks in @@ -124,29 +98,11 @@ type Db interface { // saved data at last Sync and closes the database. RollbackClose() - // SetDBInsertMode provides hints to the database about how the - // application is running. This allows the database to work in - // optimized modes when the database may be very busy. - SetDBInsertMode(InsertMode) - // Sync verifies that the database is coherent on disk and no // outstanding transactions are in flight. Sync() } -// BlockIterator defines a generic interface for an iterator through the block -// chain. -type BlockIterator interface { - // Close shuts down the iterator when done walking blocks in the database. - Close() - - // NextRow iterates thru all blocks in database. - NextRow() bool - - // Row returns row data for block iterator. - Row() (key *btcwire.ShaHash, pver uint32, buf []byte, err error) -} - // DriverDB defines a structure for backend drivers to use when they registered // themselves as a backend which implements the Db interface. type DriverDB struct { diff --git a/interface_test.go b/interface_test.go index 19f56c79..be0e28f4 100644 --- a/interface_test.go +++ b/interface_test.go @@ -569,35 +569,23 @@ func testInterface(t *testing.T, dbType string) { } } - // The data integrity tests must still pass after calling each of the - // invalidate cache functions. This intentionally uses a map since - // map iteration is not the same order every run. This helps catch - // issues that could be caused by calling one version before another. + // Run the data integrity tests again after all blocks have been + // inserted to ensure the spend tracking is working properly. context.useSpends = true - invalidateCacheFuncs := map[string]func(){ - "InvalidateBlockCache": db.InvalidateBlockCache, - "InvalidateTxCache": db.InvalidateTxCache, - "InvalidateCache": db.InvalidateCache, - } - for funcName, invalidateCacheFunc := range invalidateCacheFuncs { - t.Logf("Running integrity tests after calling %s", funcName) - invalidateCacheFunc() - - for height := int64(0); height < int64(len(blocks)); height++ { - // Get the appropriate block and hash and update the - // test context accordingly. - block := blocks[height] - blockHash, err := block.Sha() - if err != nil { - t.Errorf("block.Sha: %v", err) - return - } - context.blockHeight = height - context.blockHash = blockHash - context.block = block - - testIntegrity(&context) + for height := int64(0); height < int64(len(blocks)); height++ { + // Get the appropriate block and hash and update the + // test context accordingly. + block := blocks[height] + blockHash, err := block.Sha() + if err != nil { + t.Errorf("block.Sha: %v", err) + return } + context.blockHeight = height + context.blockHash = blockHash + context.block = block + + testIntegrity(&context) } // TODO(davec): Need to figure out how to handle the special checks @@ -605,34 +593,31 @@ func testInterface(t *testing.T, dbType string) { // 91880 on the main network due to the old miner + Satoshi client bug. // TODO(davec): Add tests for error conditions: - // * Don't allow duplicate blocks - // * Don't allow insertion of block that contains a transaction that - // already exists unless the previous one is fully spent - // * Don't allow block that has a duplicate transaction in itself - // * Don't allow block which contains a tx that references a missing tx - // * Don't allow block which contains a tx that references another tx - // that comes after it in the same block + /* + - Don't allow duplicate blocks + - Don't allow insertion of block that contains a transaction that + already exists unless the previous one is fully spent + - Don't allow block that has a duplicate transaction in itself + - Don't allow block which contains a tx that references a missing tx + - Don't allow block which contains a tx that references another tx + that comes after it in the same block + */ // TODO(davec): Add tests for the following functions: /* - Close() - DropAfterBlockBySha(*btcwire.ShaHash) (err error) - - ExistsSha(sha *btcwire.ShaHash) (exists bool) - - FetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) - - FetchBlockShaByHeight(height int64) (sha *btcwire.ShaHash, err error) - FetchHeightRange(startHeight, endHeight int64) (rshalist []btcwire.ShaHash, err error) - - ExistsTxSha(sha *btcwire.ShaHash) (exists bool) - - FetchTxBySha(txsha *btcwire.ShaHash) ([]*TxListReply, error) - - FetchTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply - - FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply - - InsertBlock(block *btcutil.Block) (height int64, err error) - - InvalidateBlockCache() - - InvalidateCache() - - InvalidateTxCache() - NewIterateBlocks() (pbi BlockIterator, err error) - NewestSha() (sha *btcwire.ShaHash, height int64, err error) - RollbackClose() - SetDBInsertMode(InsertMode) - Sync() + - Close() + - DropAfterBlockBySha(*btcwire.ShaHash) (err error) + x ExistsSha(sha *btcwire.ShaHash) (exists bool) + x FetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, err error) + x FetchBlockShaByHeight(height int64) (sha *btcwire.ShaHash, err error) + - FetchHeightRange(startHeight, endHeight int64) (rshalist []btcwire.ShaHash, err error) + x ExistsTxSha(sha *btcwire.ShaHash) (exists bool) + x FetchTxBySha(txsha *btcwire.ShaHash) ([]*TxListReply, error) + x FetchTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply + x FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply + x InsertBlock(block *btcutil.Block) (height int64, err error) + x NewestSha() (sha *btcwire.ShaHash, height int64, err error) + - RollbackClose() + - Sync() */ } diff --git a/ldb/block.go b/ldb/block.go index 64f2053c..6739114f 100644 --- a/ldb/block.go +++ b/ldb/block.go @@ -308,8 +308,3 @@ func (db *LevelDb) NewestSha() (rsha *btcwire.ShaHash, rblkid int64, err error) return &sha, db.lastBlkIdx, nil } - -func (db *LevelDb) NewIterateBlocks() (rbogus btcdb.BlockIterator, err error) { - err = fmt.Errorf("Not implemented") - return -} diff --git a/ldb/dbcache.go b/ldb/dbcache.go deleted file mode 100644 index 80298ee9..00000000 --- a/ldb/dbcache.go +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) 2013-2014 Conformal Systems LLC. -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package ldb - -import ( -//"fmt" -) - -// InvalidateTxCache clear/release all cached transactions. -func (db *LevelDb) InvalidateTxCache() { -} - -// InvalidateTxCache clear/release all cached blocks. -func (db *LevelDb) InvalidateBlockCache() { -} - -// InvalidateCache clear/release all cached blocks and transactions. -func (db *LevelDb) InvalidateCache() { -} diff --git a/ldb/insertremove_test.go b/ldb/insertremove_test.go index 20a2b2d0..b09efc7b 100644 --- a/ldb/insertremove_test.go +++ b/ldb/insertremove_test.go @@ -33,9 +33,7 @@ func loadblocks(t *testing.T) []*btcutil.Block { } func TestUnspentInsert(t *testing.T) { - testUnspentInsert(t, dbTmDefault) - //testUnspentInsert(t, dbTmNormal) - //testUnspentInsert(t, dbTmFast) + testUnspentInsert(t) } // insert every block in the test chain @@ -43,9 +41,9 @@ func TestUnspentInsert(t *testing.T) { // block and verify that the the tx is spent/unspent // new tx should be fully unspent, referenced tx should have // the associated txout set to spent. -func testUnspentInsert(t *testing.T, mode int) { +func testUnspentInsert(t *testing.T) { // Ignore db remove errors since it means we didn't have an old one. - dbname := fmt.Sprintf("tstdbuspnt1.%d", mode) + dbname := fmt.Sprintf("tstdbuspnt1") dbnamever := dbname + ".ver" _ = os.RemoveAll(dbname) _ = os.RemoveAll(dbnamever) @@ -58,19 +56,6 @@ func testUnspentInsert(t *testing.T, mode int) { defer os.RemoveAll(dbnamever) defer db.Close() - switch mode { - case dbTmDefault: // default - // no setup - case dbTmNormal: // explicit normal - db.SetDBInsertMode(btcdb.InsertNormal) - case dbTmFast: // fast mode - - case dbTmNoVerify: // validated block - t.Errorf("UnspentInsert test is not valid in NoVerify mode") - } - - // Since we are dealing with small dataset, reduce cache size - blocks := loadblocks(t) endtest: for height := int64(0); height < int64(len(blocks)); height++ { diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 813adc89..8c6b5d0e 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -420,14 +420,6 @@ func (db *LevelDb) InsertBlock(block *btcutil.Block) (height int64, rerr error) return newheight, nil } -// SetDBInsertMode provides hints to the database to how the application -// is running this allows the database to work in optimized modes when the -// database may be very busy. -func (db *LevelDb) SetDBInsertMode(newmode btcdb.InsertMode) { - - // special modes are not supported -} - // doSpend iterates all TxIn in a bitcoin transaction marking each associated // TxOut as spent. func (db *LevelDb) doSpend(tx *btcwire.MsgTx) error { diff --git a/ldb/operational_test.go b/ldb/operational_test.go index 8db333e7..d9a6b2ea 100644 --- a/ldb/operational_test.go +++ b/ldb/operational_test.go @@ -20,28 +20,18 @@ import ( var network = btcwire.MainNet -const ( - dbTmDefault = iota - dbTmNormal - dbTmFast - dbTmNoVerify -) - func TestOperational(t *testing.T) { - testOperationalMode(t, dbTmDefault) - //testOperationalMode(t, dbTmNormal) - //testOperationalMode(t, dbTmFast) - //testOperationalMode(t, dbTmNoVerify) + testOperationalMode(t) } -func testOperationalMode(t *testing.T, mode int) { +func testOperationalMode(t *testing.T) { // simplified basic operation is: // 1) fetch block from remote server // 2) look up all txin (except coinbase in db) // 3) insert block // Ignore db remove errors since it means we didn't have an old one. - dbname := fmt.Sprintf("tstdbop1.%d", mode) + dbname := fmt.Sprintf("tstdbop1") dbnamever := dbname + ".ver" _ = os.RemoveAll(dbname) _ = os.RemoveAll(dbnamever) @@ -54,22 +44,10 @@ func testOperationalMode(t *testing.T, mode int) { defer os.RemoveAll(dbnamever) defer db.Close() - switch mode { - case dbTmDefault: // default - // no setup - case dbTmNormal: // explicit normal - db.SetDBInsertMode(btcdb.InsertNormal) - case dbTmFast: // fast mode - - case dbTmNoVerify: // validated block - db.SetDBInsertMode(btcdb.InsertValidatedInput) - } - testdatafile := filepath.Join("..", "testdata", "blocks1-256.bz2") blocks, err := loadBlocks(t, testdatafile) if err != nil { - t.Errorf("Unable to load blocks from test data for mode %v: %v", - mode, err) + t.Errorf("Unable to load blocks from test data: %v", err) return } @@ -77,36 +55,31 @@ func testOperationalMode(t *testing.T, mode int) { out: for height := int64(0); height < int64(len(blocks)); height++ { block := blocks[height] - if mode != dbTmNoVerify { - // except for NoVerify which does not allow lookups check inputs - mblock := block.MsgBlock() - var txneededList []*btcwire.ShaHash - for _, tx := range mblock.Transactions { - for _, txin := range tx.TxIn { - if txin.PreviousOutpoint.Index == uint32(4294967295) { - continue - } - origintxsha := &txin.PreviousOutpoint.Hash - txneededList = append(txneededList, origintxsha) + mblock := block.MsgBlock() + var txneededList []*btcwire.ShaHash + for _, tx := range mblock.Transactions { + for _, txin := range tx.TxIn { + if txin.PreviousOutpoint.Index == uint32(4294967295) { + continue + } + origintxsha := &txin.PreviousOutpoint.Hash + txneededList = append(txneededList, origintxsha) - if !db.ExistsTxSha(origintxsha) { - t.Errorf("referenced tx not found %v ", origintxsha) - } - - _, err = db.FetchTxBySha(origintxsha) - if err != nil { - t.Errorf("referenced tx not found %v err %v ", origintxsha, err) - } + if !db.ExistsTxSha(origintxsha) { + t.Errorf("referenced tx not found %v ", origintxsha) + } + _, err = db.FetchTxBySha(origintxsha) + if err != nil { + t.Errorf("referenced tx not found %v err %v ", origintxsha, err) } } - txlist := db.FetchUnSpentTxByShaList(txneededList) - for _, txe := range txlist { - if txe.Err != nil { - t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) - break out - } + } + txlist := db.FetchUnSpentTxByShaList(txneededList) + for _, txe := range txlist { + if txe.Err != nil { + t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) + break out } - } newheight, err := db.InsertBlock(block) @@ -136,34 +109,20 @@ out: // now that db is populated, do some additional test testFetchRangeHeight(t, db, blocks) - - switch mode { - case dbTmDefault: // default - // no cleanup - case dbTmNormal: // explicit normal - // no cleanup - case dbTmFast: // fast mode - db.SetDBInsertMode(btcdb.InsertNormal) - case dbTmNoVerify: // validated block - db.SetDBInsertMode(btcdb.InsertNormal) - } } func TestBackout(t *testing.T) { - testBackout(t, dbTmDefault) - //testBackout(t, dbTmNormal) - //testBackout(t, dbTmFast) + testBackout(t) } -func testBackout(t *testing.T, mode int) { +func testBackout(t *testing.T) { // simplified basic operation is: // 1) fetch block from remote server // 2) look up all txin (except coinbase in db) // 3) insert block - t.Logf("mode %v", mode) // Ignore db remove errors since it means we didn't have an old one. - dbname := fmt.Sprintf("tstdbop2.%d", mode) + dbname := fmt.Sprintf("tstdbop2") dbnamever := dbname + ".ver" _ = os.RemoveAll(dbname) _ = os.RemoveAll(dbnamever) @@ -176,15 +135,6 @@ func testBackout(t *testing.T, mode int) { defer os.RemoveAll(dbnamever) defer db.Close() - switch mode { - case dbTmDefault: // default - // no setup - case dbTmNormal: // explicit normal - db.SetDBInsertMode(btcdb.InsertNormal) - case dbTmFast: // fast mode - - } - testdatafile := filepath.Join("..", "testdata", "blocks1-256.bz2") blocks, err := loadBlocks(t, testdatafile) if len(blocks) < 120 { diff --git a/memdb/memdb.go b/memdb/memdb.go index e9bd795a..16793b91 100644 --- a/memdb/memdb.go +++ b/memdb/memdb.go @@ -703,15 +703,6 @@ func (db *MemDb) InvalidateTxCache() { } } -// NewIterateBlocks returns an iterator for all blocks in database. This is -// part of the btcdb.Db interface implementation. -// -// This implmentation does not implement an iterator, so an error is returned -// if this function is called. -func (db *MemDb) NewIterateBlocks() (btcdb.BlockIterator, error) { - return nil, fmt.Errorf("Not implemented") -} - // NewestSha returns the hash and block height of the most recent (end) block of // the block chain. It will return the zero hash, -1 for the block height, and // no error (nil) if there are not any blocks in the database yet. This is part @@ -752,18 +743,6 @@ func (db *MemDb) RollbackClose() { db.Close() } -// SetDBInsertMode provides hints to the database about how the application is -// running. This allows the database to work in optimized modes when the -// database may be very busy. This is part of the btcdb.Db interface -// implementation. -// -// No special mode handling is performed for this implementation. -func (db *MemDb) SetDBInsertMode(newmode btcdb.InsertMode) { - if db.closed { - log.Warnf("SetDBInsertMode called after db close.") - } -} - // Sync verifies that the database is coherent on disk and no outstanding // transactions are in flight. This is part of the btcdb.Db interface // implementation. diff --git a/memdb/memdb_test.go b/memdb/memdb_test.go index f078bbb5..73a1d57b 100644 --- a/memdb/memdb_test.go +++ b/memdb/memdb_test.go @@ -100,13 +100,6 @@ func TestClosed(t *testing.T) { // The following calls don't return errors from the interface to be able // to detect a closed database, so just call them to ensure there are no // panics. - db.InvalidateBlockCache() - db.InvalidateCache() - db.InvalidateTxCache() - db.NewIterateBlocks() - db.SetDBInsertMode(btcdb.InsertNormal) - db.SetDBInsertMode(btcdb.InsertFast) - db.SetDBInsertMode(btcdb.InsertValidatedInput) db.Sync() db.RollbackClose() } From 2aca9245145a6c22fb02f6681201105f6eb581e4 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sun, 19 Jan 2014 20:01:31 -0600 Subject: [PATCH 118/163] Allow the Open/CreateDB funcs to take any params. The specific parameters required by a backend is better left up to the backend itself. For example memdb has no need for a database path, while ldb does. This commit modifies the OpenDB and CreateDB functions to take a arbitrary arguments which are passed along to the driver. The driver is expected to verify the correct type and number of arguments and error accordingly. The existing backends have been updated accordingly. --- common_test.go | 4 ++-- db.go | 14 +++++++------- db_test.go | 16 ++++++++-------- ldb/leveldb.go | 30 +++++++++++++++++++++++++++--- memdb/driver.go | 27 +++++++++++++++++++++++---- memdb/memdb_test.go | 2 +- 6 files changed, 68 insertions(+), 25 deletions(-) diff --git a/common_test.go b/common_test.go index 31653445..ffe2653f 100644 --- a/common_test.go +++ b/common_test.go @@ -54,7 +54,7 @@ func fileExists(name string) bool { func openDB(dbType, dbName string) (btcdb.Db, error) { // Handle memdb specially since it has no files on disk. if dbType == "memdb" { - db, err := btcdb.OpenDB(dbType, "") + db, err := btcdb.OpenDB(dbType) if err != nil { return nil, fmt.Errorf("error opening db: %v", err) } @@ -78,7 +78,7 @@ func createDB(dbType, dbName string, close bool) (btcdb.Db, func(), error) { // Handle memory database specially since it doesn't need the disk // specific handling. if dbType == "memdb" { - db, err := btcdb.CreateDB(dbType, "") + db, err := btcdb.CreateDB(dbType) if err != nil { return nil, nil, fmt.Errorf("error creating db: %v", err) } diff --git a/db.go b/db.go index 4e7f03e1..c5b392de 100644 --- a/db.go +++ b/db.go @@ -106,9 +106,9 @@ type Db interface { // DriverDB defines a structure for backend drivers to use when they registered // themselves as a backend which implements the Db interface. type DriverDB struct { - DbType string - Create func(argstr string) (pbdb Db, err error) - Open func(filepath string) (pbdb Db, err error) + DbType string + CreateDB func(args ...interface{}) (pbdb Db, err error) + OpenDB func(args ...interface{}) (pbdb Db, err error) } // TxListReply is used to return individual transaction information when @@ -138,20 +138,20 @@ func AddDBDriver(instance DriverDB) { } // CreateDB intializes and opens a database. -func CreateDB(dbtype string, argstr string) (pbdb Db, err error) { +func CreateDB(dbtype string, args ...interface{}) (pbdb Db, err error) { for _, drv := range driverList { if drv.DbType == dbtype { - return drv.Create(argstr) + return drv.CreateDB(args...) } } return nil, DbUnknownType } // OpenDB opens an existing database. -func OpenDB(dbtype string, argstr string) (pbdb Db, err error) { +func OpenDB(dbtype string, args ...interface{}) (pbdb Db, err error) { for _, drv := range driverList { if drv.DbType == dbtype { - return drv.Open(argstr) + return drv.OpenDB(args...) } } return nil, DbUnknownType diff --git a/db_test.go b/db_test.go index 74a3b89b..75fc6174 100644 --- a/db_test.go +++ b/db_test.go @@ -76,7 +76,7 @@ func TestAddDuplicateDriver(t *testing.T) { // driver function and intentionally returns a failure that can be // detected if the interface allows a duplicate driver to overwrite an // existing one. - bogusCreateDB := func(string) (btcdb.Db, error) { + bogusCreateDB := func(args ...interface{}) (btcdb.Db, error) { return nil, fmt.Errorf("duplicate driver allowed for database "+ "type [%v]", dbType) } @@ -85,9 +85,9 @@ func TestAddDuplicateDriver(t *testing.T) { // create and open functions to a function that causes a test failure if // they are invoked. driver := btcdb.DriverDB{ - DbType: dbType, - Create: bogusCreateDB, - Open: bogusCreateDB, + DbType: dbType, + CreateDB: bogusCreateDB, + OpenDB: bogusCreateDB, } btcdb.AddDBDriver(driver) @@ -111,16 +111,16 @@ func TestCreateOpenFail(t *testing.T) { dbType := "createopenfail" openError := fmt.Errorf("failed to create or open database for "+ "database type [%v]", dbType) - bogusCreateDB := func(string) (btcdb.Db, error) { + bogusCreateDB := func(args ...interface{}) (btcdb.Db, error) { return nil, openError } // Create and add driver that intentionally fails when created or opened // to ensure errors on database open and create are handled properly. driver := btcdb.DriverDB{ - DbType: dbType, - Create: bogusCreateDB, - Open: bogusCreateDB, + DbType: dbType, + CreateDB: bogusCreateDB, + OpenDB: bogusCreateDB, } btcdb.AddDBDriver(driver) diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 8c6b5d0e..2209d833 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -55,14 +55,33 @@ type LevelDb struct { txSpentUpdateMap map[btcwire.ShaHash]*spentTxUpdate } -var self = btcdb.DriverDB{DbType: "leveldb", Create: CreateDB, Open: OpenDB} +var self = btcdb.DriverDB{DbType: "leveldb", CreateDB: CreateDB, OpenDB: OpenDB} func init() { btcdb.AddDBDriver(self) } +// parseArgs parses the arguments from the btcdb Open/Create methods. +func parseArgs(funcName string, args ...interface{}) (string, error) { + if len(args) != 1 { + return "", fmt.Errorf("Invalid arguments to ldb.%s -- "+ + "expected database path string", funcName) + } + dbPath, ok := args[0].(string) + if !ok { + return "", fmt.Errorf("First argument to ldb.%s is invalid -- "+ + "expected database path string", funcName) + } + return dbPath, nil +} + // OpenDB opens an existing database for use. -func OpenDB(dbpath string) (btcdb.Db, error) { +func OpenDB(args ...interface{}) (btcdb.Db, error) { + dbpath, err := parseArgs("OpenDB", args...) + if err != nil { + return nil, err + } + log = btcdb.GetLog() db, err := openDB(dbpath, false) @@ -216,7 +235,12 @@ func openDB(dbpath string, create bool) (pbdb btcdb.Db, err error) { } // CreateDB creates, initializes and opens a database for use. -func CreateDB(dbpath string) (btcdb.Db, error) { +func CreateDB(args ...interface{}) (btcdb.Db, error) { + dbpath, err := parseArgs("Create", args...) + if err != nil { + return nil, err + } + log = btcdb.GetLog() // No special setup needed, just OpenBB diff --git a/memdb/driver.go b/memdb/driver.go index 53e72c26..581c0feb 100644 --- a/memdb/driver.go +++ b/memdb/driver.go @@ -5,6 +5,7 @@ package memdb import ( + "fmt" "github.com/conformal/btcdb" "github.com/conformal/btclog" ) @@ -12,18 +13,36 @@ import ( var log = btclog.Disabled func init() { - driver := btcdb.DriverDB{DbType: "memdb", Create: CreateDB, Open: OpenDB} + driver := btcdb.DriverDB{DbType: "memdb", CreateDB: CreateDB, OpenDB: OpenDB} btcdb.AddDBDriver(driver) } +// parseArgs parses the arguments from the btcdb Open/Create methods. +func parseArgs(funcName string, args ...interface{}) error { + if len(args) != 0 { + return fmt.Errorf("memdb.%s does not accept any arguments", + funcName) + } + + return nil +} + // OpenDB opens an existing database for use. -func OpenDB(dbpath string) (btcdb.Db, error) { +func OpenDB(args ...interface{}) (btcdb.Db, error) { + if err := parseArgs("OpenDB", args...); err != nil { + return nil, err + } + // A memory database is not persistent, so let CreateDB handle it. - return CreateDB(dbpath) + return CreateDB() } // CreateDB creates, initializes, and opens a database for use. -func CreateDB(dbpath string) (btcdb.Db, error) { +func CreateDB(args ...interface{}) (btcdb.Db, error) { + if err := parseArgs("CreateDB", args...); err != nil { + return nil, err + } + log = btcdb.GetLog() return newMemDb(), nil } diff --git a/memdb/memdb_test.go b/memdb/memdb_test.go index 73a1d57b..4d96b149 100644 --- a/memdb/memdb_test.go +++ b/memdb/memdb_test.go @@ -18,7 +18,7 @@ import ( // and does not panic or otherwise misbehave for functions which do not return // errors. func TestClosed(t *testing.T) { - db, err := btcdb.CreateDB("memdb", "") + db, err := btcdb.CreateDB("memdb") if err != nil { t.Errorf("Failed to open test database %v", err) return From 0a86df4a162ddd8311194d44231f69e94abd1d23 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sun, 19 Jan 2014 21:41:52 -0600 Subject: [PATCH 119/163] Correct test print found by go vet. --- db_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db_test.go b/db_test.go index 75fc6174..6a8b088f 100644 --- a/db_test.go +++ b/db_test.go @@ -30,7 +30,7 @@ func testNewestShaEmpty(t *testing.T, db btcdb.Db) { } if height != -1 { - t.Errorf("NewestSha wrong height got: %s, want %s", height, -1) + t.Errorf("NewestSha wrong height got: %d, want %d", height, -1) } } From c570830104055febdf5c41ed7a86a7a1daaf2c76 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Mon, 17 Mar 2014 12:58:22 -0500 Subject: [PATCH 120/163] Update README.md example to use leveldb. SQLite is now fully deprecated and has been removed. Spotted and reported by @jolan. Closes #3. --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 987e41ec..7a3722e1 100644 --- a/README.md +++ b/README.md @@ -19,14 +19,14 @@ change. // Import packages. import ( "github.com/conformal/btcdb" - _ "github.com/conformal/btcdb/sqlite3" + _ "github.com/conformal/btcdb/ldb" "github.com/conformal/btcutil" "github.com/conformal/btcwire" ) // Create a database and schedule it to be closed on exit. - dbName := "example.db" - db, err := btcdb.CreateDB("sqlite", dbName) + dbName := "example" + db, err := btcdb.CreateDB("leveldb", dbName) if err != nil { // Log and handle the error } From 1b51d58e556c384ac657d69b0c8100b942223ce6 Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 20 Mar 2014 11:07:40 -0400 Subject: [PATCH 121/163] In FetchHeightRange, preallocate the hash array. Change FetchBlockByHeight to do the database lookup itself instead of calling getBlkByHeight. This saves an allocation of the entire block, which it wasn't using. In OpenDB, use fetchBlockShaByHeight instead of getBlkByHeight since it only needs the sha. This saves memory on startup. ok drahn@ --- ldb/block.go | 15 ++++++++++----- ldb/leveldb.go | 4 ++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/ldb/block.go b/ldb/block.go index 6739114f..729ca943 100644 --- a/ldb/block.go +++ b/ldb/block.go @@ -247,13 +247,18 @@ func (db *LevelDb) FetchBlockShaByHeight(height int64) (sha *btcwire.ShaHash, er // fetchBlockShaByHeight returns a block hash based on its height in the // block chain. func (db *LevelDb) fetchBlockShaByHeight(height int64) (rsha *btcwire.ShaHash, err error) { - var sha *btcwire.ShaHash - sha, _, err = db.getBlkByHeight(height) + key := int64ToKey(height) + + blkVal, err := db.lDb.Get(key, db.ro) if err != nil { - return + log.Tracef("failed to find height %v", height) + return // exists ??? } - return sha, nil + var sha btcwire.ShaHash + sha.SetBytes(blkVal[0:32]) + + return &sha, nil } // FetchHeightRange looks up a range of blocks by the start and ending @@ -271,7 +276,7 @@ func (db *LevelDb) FetchHeightRange(startHeight, endHeight int64) (rshalist []bt endidx = endHeight } - var shalist []btcwire.ShaHash + shalist := make([]btcwire.ShaHash, 0, endidx-startHeight) for height := startHeight; height < endidx; height++ { // TODO(drahn) fix blkFile from height diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 2209d833..34671dcd 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -101,7 +101,7 @@ func OpenDB(args ...interface{}) (btcdb.Db, error) { blockforward: for { - sha, _, err := ldb.getBlkByHeight(testblock) + sha, err := ldb.fetchBlockShaByHeight(testblock) if err == nil { // block is found lastSha = sha @@ -125,7 +125,7 @@ blockforward: blocknarrow: for lastknownblock != -1 { testblock = (lastknownblock + nextunknownblock) / 2 - sha, _, err := ldb.getBlkByHeight(testblock) + sha, err := ldb.fetchBlockShaByHeight(testblock) if err == nil { lastknownblock = testblock lastSha = sha From 4d630765261ac7f219a289ae9018cc82f5f496b3 Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Tue, 15 Apr 2014 10:16:10 -0400 Subject: [PATCH 122/163] All external entry points into the database must be locked. --- ldb/tx.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ldb/tx.go b/ldb/tx.go index d9ff9a60..0ecdf413 100644 --- a/ldb/tx.go +++ b/ldb/tx.go @@ -251,6 +251,9 @@ func (db *LevelDb) existsTxSha(txSha *btcwire.ShaHash) (exists bool) { // FetchTxByShaList returns the most recent tx of the name fully spent or not func (db *LevelDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { + db.dbLock.Lock() + defer db.dbLock.Unlock() + // until the fully spent separation of tx is complete this is identical // to FetchUnSpentTxByShaList replies := make([]*btcdb.TxListReply, len(txShaList)) From dd41d71a31a58376f7af6888916e422f7055f769 Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Tue, 15 Apr 2014 10:17:16 -0400 Subject: [PATCH 123/163] If data for tx is not present, return error. --- ldb/tx.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ldb/tx.go b/ldb/tx.go index 0ecdf413..9ff7b603 100644 --- a/ldb/tx.go +++ b/ldb/tx.go @@ -350,6 +350,10 @@ func (db *LevelDb) fetchTxDataByLoc(blkHeight int64, txOff int, txLen int, txspe //log.Trace("transaction %v is at block %v %v txoff %v, txlen %v\n", // txsha, blksha, blkHeight, txOff, txLen) + if len(blkbuf) < txOff+txLen { + err = btcdb.TxShaMissing + return + } rbuf := bytes.NewBuffer(blkbuf[txOff : txOff+txLen]) var tx btcwire.MsgTx From ae25e28d7e5e815043e5422d106c7c7f36738dd8 Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Tue, 15 Apr 2014 10:43:12 -0400 Subject: [PATCH 124/163] Update test to new API. --- ldb/dup_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ldb/dup_test.go b/ldb/dup_test.go index e22fc7f3..802dc379 100644 --- a/ldb/dup_test.go +++ b/ldb/dup_test.go @@ -151,8 +151,9 @@ out: } } - txshalist, _ := blk.TxShas() - for _, txsha := range txshalist { + txlist := blk.Transactions() + for _, tx := range txlist { + txsha := tx.Sha() txReply, err := db.FetchTxBySha(txsha) if err != nil { t.Errorf("fully spent lookup %v err %v\n", hash, err) From 75e199ece8fc6061b1b2f2180d73d869b10e38d3 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Wed, 28 May 2014 01:29:51 -0500 Subject: [PATCH 125/163] Use btcnet genesis params for examples and tests. --- README.md | 4 ++-- common_test.go | 5 +++-- doc.go | 4 ++-- ldb/operational_test.go | 3 ++- memdb/memdb_test.go | 15 ++++++++++----- 5 files changed, 19 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 7a3722e1..97a25da7 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,8 @@ change. import ( "github.com/conformal/btcdb" _ "github.com/conformal/btcdb/ldb" + "github.com/conformal/btcnet" "github.com/conformal/btcutil" - "github.com/conformal/btcwire" ) // Create a database and schedule it to be closed on exit. @@ -34,7 +34,7 @@ change. // Insert the main network genesis block. - genesis := btcutil.NewBlock(&btcwire.GenesisBlock) + genesis := btcutil.NewBlock(btcnet.MainNetParams.GenesisBlock) newHeight, err := db.InsertBlock(genesis) if err != nil { // Log and handle the error diff --git a/common_test.go b/common_test.go index ffe2653f..2d283b1a 100644 --- a/common_test.go +++ b/common_test.go @@ -11,6 +11,7 @@ import ( "github.com/conformal/btcdb" _ "github.com/conformal/btcdb/ldb" _ "github.com/conformal/btcdb/memdb" + "github.com/conformal/btcnet" "github.com/conformal/btcutil" "github.com/conformal/btcwire" "io" @@ -138,7 +139,7 @@ func setupDB(dbType, dbName string) (btcdb.Db, func(), error) { // Insert the main network genesis block. This is part of the initial // database setup. - genesisBlock := btcutil.NewBlock(&btcwire.GenesisBlock) + genesisBlock := btcutil.NewBlock(btcnet.MainNetParams.GenesisBlock) _, err = db.InsertBlock(genesisBlock) if err != nil { teardown() @@ -177,7 +178,7 @@ func loadBlocks(t *testing.T) ([]*btcutil.Block, error) { // Set the first block as the genesis block. blocks := make([]*btcutil.Block, 0, 256) - genesis := btcutil.NewBlock(&btcwire.GenesisBlock) + genesis := btcutil.NewBlock(btcnet.MainNetParams.GenesisBlock) blocks = append(blocks, genesis) for height := int64(1); err == nil; height++ { diff --git a/doc.go b/doc.go index eb5e426d..5a89f75d 100644 --- a/doc.go +++ b/doc.go @@ -33,8 +33,8 @@ referenced parent block to already exist. In a more concrete example: import ( "github.com/conformal/btcdb" _ "github.com/conformal/btcdb/ldb" + "github.com/conformal/btcnet" "github.com/conformal/btcutil" - "github.com/conformal/btcwire" ) // Create a database and schedule it to be closed on exit. @@ -47,7 +47,7 @@ referenced parent block to already exist. In a more concrete example: // Insert the main network genesis block. - genesis := btcutil.NewBlock(&btcwire.GenesisBlock) + genesis := btcutil.NewBlock(btcnet.MainNetParams.GenesisBlock) newHeight, err := db.InsertBlock(genesis) if err != nil { // Log and handle the error diff --git a/ldb/operational_test.go b/ldb/operational_test.go index d9a6b2ea..2e8f82db 100644 --- a/ldb/operational_test.go +++ b/ldb/operational_test.go @@ -9,6 +9,7 @@ import ( "encoding/binary" "fmt" "github.com/conformal/btcdb" + "github.com/conformal/btcnet" "github.com/conformal/btcutil" "github.com/conformal/btcwire" "io" @@ -246,7 +247,7 @@ func loadBlocks(t *testing.T, file string) (blocks []*btcutil.Block, err error) }() // Set the first block as the genesis block. - genesis := btcutil.NewBlock(&btcwire.GenesisBlock) + genesis := btcutil.NewBlock(btcnet.MainNetParams.GenesisBlock) blocks = append(blocks, genesis) var block *btcutil.Block diff --git a/memdb/memdb_test.go b/memdb/memdb_test.go index 4d96b149..f5684483 100644 --- a/memdb/memdb_test.go +++ b/memdb/memdb_test.go @@ -7,6 +7,7 @@ package memdb_test import ( "github.com/conformal/btcdb" "github.com/conformal/btcdb/memdb" + "github.com/conformal/btcnet" "github.com/conformal/btcutil" "github.com/conformal/btcwire" "reflect" @@ -23,13 +24,13 @@ func TestClosed(t *testing.T) { t.Errorf("Failed to open test database %v", err) return } - _, err = db.InsertBlock(btcutil.NewBlock(&btcwire.GenesisBlock)) + _, err = db.InsertBlock(btcutil.NewBlock(btcnet.MainNetParams.GenesisBlock)) if err != nil { t.Errorf("InsertBlock: %v", err) } db.Close() - genesisHash := &btcwire.GenesisHash + genesisHash := btcnet.MainNetParams.GenesisHash if err := db.DropAfterBlockBySha(genesisHash); err != memdb.ErrDbClosed { t.Errorf("DropAfterBlockBySha: unexpected error %v", err) } @@ -50,10 +51,14 @@ func TestClosed(t *testing.T) { t.Errorf("FetchHeightRange: unexpected error %v", err) } - genesisMerkleRoot := &btcwire.GenesisMerkleRoot - if exists := db.ExistsTxSha(genesisMerkleRoot); exists != false { + genesisCoinbaseTx := btcnet.MainNetParams.GenesisBlock.Transactions[0] + coinbaseHash, err := genesisCoinbaseTx.TxSha() + if err != nil { + t.Errorf("TxSha: unexpected error %v", err) + } + if exists := db.ExistsTxSha(&coinbaseHash); exists != false { t.Errorf("ExistsTxSha: hash %v exists when it shouldn't", - genesisMerkleRoot) + &coinbaseHash) } if _, err := db.FetchTxBySha(genesisHash); err != memdb.ErrDbClosed { From b40f5a1b240a9696f9be63dbf182775cceb49a44 Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 30 May 2014 15:03:39 -0400 Subject: [PATCH 126/163] add missing Lock to FetchTxBySha --- ldb/tx.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ldb/tx.go b/ldb/tx.go index 9ff7b603..7e4fad81 100644 --- a/ldb/tx.go +++ b/ldb/tx.go @@ -369,6 +369,9 @@ func (db *LevelDb) fetchTxDataByLoc(blkHeight int64, txOff int, txLen int, txspe // FetchTxBySha returns some data for the given Tx Sha. func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) ([]*btcdb.TxListReply, error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + replylen := 0 replycnt := 0 From efb92fcc99e8ef808970164bdda9ab462a70dd5f Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Wed, 2 Jul 2014 19:47:24 -0500 Subject: [PATCH 127/163] goimports -w . --- common_test.go | 11 ++++++----- db.go | 1 + db_test.go | 3 ++- interface_test.go | 5 +++-- ldb/block.go | 1 + ldb/boundary_test.go | 5 +++-- ldb/dbtest/dbtst.go | 1 + ldb/dup_test.go | 7 ++++--- ldb/insertremove_test.go | 7 ++++--- ldb/internal_test.go | 1 + ldb/leveldb.go | 5 +++-- ldb/operational_test.go | 9 +++++---- ldb/tx.go | 1 + log.go | 3 ++- memdb/driver.go | 1 + memdb/memdb.go | 5 +++-- memdb/memdb_test.go | 5 +++-- 17 files changed, 44 insertions(+), 27 deletions(-) diff --git a/common_test.go b/common_test.go index 2d283b1a..56cb62e7 100644 --- a/common_test.go +++ b/common_test.go @@ -8,17 +8,18 @@ import ( "compress/bzip2" "encoding/binary" "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + "github.com/conformal/btcdb" _ "github.com/conformal/btcdb/ldb" _ "github.com/conformal/btcdb/memdb" "github.com/conformal/btcnet" "github.com/conformal/btcutil" "github.com/conformal/btcwire" - "io" - "os" - "path/filepath" - "strings" - "testing" ) var ( diff --git a/db.go b/db.go index c5b392de..adfbb405 100644 --- a/db.go +++ b/db.go @@ -6,6 +6,7 @@ package btcdb import ( "errors" + "github.com/conformal/btcutil" "github.com/conformal/btcwire" ) diff --git a/db_test.go b/db_test.go index 6a8b088f..9ec93cb6 100644 --- a/db_test.go +++ b/db_test.go @@ -6,8 +6,9 @@ package btcdb_test import ( "fmt" - "github.com/conformal/btcdb" "testing" + + "github.com/conformal/btcdb" ) var ( diff --git a/interface_test.go b/interface_test.go index be0e28f4..21500a67 100644 --- a/interface_test.go +++ b/interface_test.go @@ -5,12 +5,13 @@ package btcdb_test import ( + "reflect" + "testing" + "github.com/conformal/btcdb" "github.com/conformal/btcutil" "github.com/conformal/btcwire" "github.com/davecgh/go-spew/spew" - "reflect" - "testing" ) // testContext is used to store context information about a running test which diff --git a/ldb/block.go b/ldb/block.go index 729ca943..3d12360a 100644 --- a/ldb/block.go +++ b/ldb/block.go @@ -8,6 +8,7 @@ import ( "bytes" "encoding/binary" "fmt" + "github.com/conformal/btcdb" "github.com/conformal/btcutil" "github.com/conformal/btcwire" diff --git a/ldb/boundary_test.go b/ldb/boundary_test.go index 5a722441..af49a684 100644 --- a/ldb/boundary_test.go +++ b/ldb/boundary_test.go @@ -5,10 +5,11 @@ package ldb_test import ( - "github.com/conformal/btcdb" - "github.com/conformal/btcwire" "os" "testing" + + "github.com/conformal/btcdb" + "github.com/conformal/btcwire" ) // we need to test for empty databas and make certain it returns proper value diff --git a/ldb/dbtest/dbtst.go b/ldb/dbtest/dbtst.go index 11a34c0b..41e83cad 100644 --- a/ldb/dbtest/dbtst.go +++ b/ldb/dbtest/dbtst.go @@ -3,6 +3,7 @@ package main import ( "fmt" + "github.com/conformal/goleveldb/leveldb" "github.com/conformal/goleveldb/leveldb/opt" ) diff --git a/ldb/dup_test.go b/ldb/dup_test.go index 802dc379..8b638dfd 100644 --- a/ldb/dup_test.go +++ b/ldb/dup_test.go @@ -6,12 +6,13 @@ package ldb_test import ( "fmt" - "github.com/conformal/btcdb" - "github.com/conformal/btcutil" - "github.com/conformal/btcwire" "os" "path/filepath" "testing" + + "github.com/conformal/btcdb" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" ) func Test_dupTx(t *testing.T) { diff --git a/ldb/insertremove_test.go b/ldb/insertremove_test.go index b09efc7b..27fdc486 100644 --- a/ldb/insertremove_test.go +++ b/ldb/insertremove_test.go @@ -6,13 +6,14 @@ package ldb_test import ( "fmt" + "os" + "path/filepath" + "testing" + "github.com/conformal/btcdb" _ "github.com/conformal/btcdb/ldb" "github.com/conformal/btcutil" "github.com/conformal/btcwire" - "os" - "path/filepath" - "testing" ) var tstBlocks []*btcutil.Block diff --git a/ldb/internal_test.go b/ldb/internal_test.go index 12c61fb2..359a0e79 100644 --- a/ldb/internal_test.go +++ b/ldb/internal_test.go @@ -6,6 +6,7 @@ package ldb import ( "fmt" + "github.com/conformal/btcdb" "github.com/conformal/btcwire" ) diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 34671dcd..95f42bfb 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -7,6 +7,9 @@ package ldb import ( "encoding/binary" "fmt" + "os" + "sync" + "github.com/conformal/btcdb" "github.com/conformal/btclog" "github.com/conformal/btcutil" @@ -14,8 +17,6 @@ import ( "github.com/conformal/goleveldb/leveldb" "github.com/conformal/goleveldb/leveldb/cache" "github.com/conformal/goleveldb/leveldb/opt" - "os" - "sync" ) const ( diff --git a/ldb/operational_test.go b/ldb/operational_test.go index 2e8f82db..cd8d6395 100644 --- a/ldb/operational_test.go +++ b/ldb/operational_test.go @@ -8,15 +8,16 @@ import ( "compress/bzip2" "encoding/binary" "fmt" - "github.com/conformal/btcdb" - "github.com/conformal/btcnet" - "github.com/conformal/btcutil" - "github.com/conformal/btcwire" "io" "os" "path/filepath" "strings" "testing" + + "github.com/conformal/btcdb" + "github.com/conformal/btcnet" + "github.com/conformal/btcutil" + "github.com/conformal/btcwire" ) var network = btcwire.MainNet diff --git a/ldb/tx.go b/ldb/tx.go index 7e4fad81..1e2e7ce0 100644 --- a/ldb/tx.go +++ b/ldb/tx.go @@ -8,6 +8,7 @@ import ( "bytes" "encoding/binary" "fmt" + "github.com/conformal/btcdb" "github.com/conformal/btcwire" "github.com/conformal/goleveldb/leveldb" diff --git a/log.go b/log.go index 8aa02ec4..829dcaf1 100644 --- a/log.go +++ b/log.go @@ -6,8 +6,9 @@ package btcdb import ( "errors" - "github.com/conformal/btclog" "io" + + "github.com/conformal/btclog" ) // log is a logger that is initialized with no output filters. This diff --git a/memdb/driver.go b/memdb/driver.go index 581c0feb..49b5904a 100644 --- a/memdb/driver.go +++ b/memdb/driver.go @@ -6,6 +6,7 @@ package memdb import ( "fmt" + "github.com/conformal/btcdb" "github.com/conformal/btclog" ) diff --git a/memdb/memdb.go b/memdb/memdb.go index 16793b91..254ecadb 100644 --- a/memdb/memdb.go +++ b/memdb/memdb.go @@ -7,11 +7,12 @@ package memdb import ( "errors" "fmt" + "math" + "sync" + "github.com/conformal/btcdb" "github.com/conformal/btcutil" "github.com/conformal/btcwire" - "math" - "sync" ) // Errors that the various database functions may return. diff --git a/memdb/memdb_test.go b/memdb/memdb_test.go index f5684483..8e33de47 100644 --- a/memdb/memdb_test.go +++ b/memdb/memdb_test.go @@ -5,13 +5,14 @@ package memdb_test import ( + "reflect" + "testing" + "github.com/conformal/btcdb" "github.com/conformal/btcdb/memdb" "github.com/conformal/btcnet" "github.com/conformal/btcutil" "github.com/conformal/btcwire" - "reflect" - "testing" ) // TestClosed ensure calling the interface functions on a closed database From dd41a4a233a1e7aad028d7795f913ab062f71a1b Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sun, 6 Jul 2014 21:11:44 -0500 Subject: [PATCH 128/163] Remove unneeded functions from memdb. This commit removes some functions from memdb which only existed to satify the btcdb.Db interface, but have since been removed from the interface. --- memdb/memdb.go | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/memdb/memdb.go b/memdb/memdb.go index 254ecadb..bd2acb62 100644 --- a/memdb/memdb.go +++ b/memdb/memdb.go @@ -668,42 +668,6 @@ func (db *MemDb) InsertBlock(block *btcutil.Block) (int64, error) { return newHeight, nil } -// InvalidateBlockCache releases all cached blocks. This is part of the -// btcdb.Db interface implementation. -// -// There is no need for a cache with this implementation since the entire -// database is already in memory As a result, this function doesn't do anything -// useful and is only provided to conform to the interface. -func (db *MemDb) InvalidateBlockCache() { - if db.closed { - log.Warnf("InvalidateBlockCache called after db close.") - } -} - -// InvalidateCache releases all cached blocks and transactions. This is part of -// the btcdb.Db interface implementation. -// -// There is no need for a cache with this implementation since the entire -// database is already in memory As a result, this function doesn't do anything -// useful and is only provided to conform to the interface. -func (db *MemDb) InvalidateCache() { - if db.closed { - log.Warnf("InvalidateCache called after db close.") - } -} - -// InvalidateTxCache releases all cached transactions. This is part of the -// btcdb.Db interface implementation. -// -// There is no need for a cache with this implementation since the entire -// database is already in memory As a result, this function doesn't do anything -// useful and is only provided to conform to the interface. -func (db *MemDb) InvalidateTxCache() { - if db.closed { - log.Warnf("InvalidateTxCache called after db close.") - } -} - // NewestSha returns the hash and block height of the most recent (end) block of // the block chain. It will return the zero hash, -1 for the block height, and // no error (nil) if there are not any blocks in the database yet. This is part From fbb17fd35a9416ddba91d980a1a6d3b9fe9a4c8d Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Mon, 7 Jul 2014 09:33:47 -0500 Subject: [PATCH 129/163] Update TODO section of README.md. This ability for multiple database backends has been available for a long, long time. Looks like this was simply not updated when the interface was added. --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 97a25da7..6d6ada3e 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,6 @@ $ go get github.com/conformal/btcdb ## TODO - Increase test coverage to 100% -- Allow other database backends ## GPG Verification Key From e2628dfc0d4346bc4a4820357e4e837b250750aa Mon Sep 17 00:00:00 2001 From: David Hill Date: Wed, 4 Jun 2014 14:55:48 -0400 Subject: [PATCH 130/163] convert binary Read/Write to Uint/PutUint and copy --- ldb/block.go | 46 ++++--------- ldb/leveldb.go | 18 ++--- ldb/tx.go | 173 +++++++++++++------------------------------------ 3 files changed, 62 insertions(+), 175 deletions(-) diff --git a/ldb/block.go b/ldb/block.go index 3d12360a..5d4ff22f 100644 --- a/ldb/block.go +++ b/ldb/block.go @@ -7,7 +7,6 @@ package ldb import ( "bytes" "encoding/binary" - "fmt" "github.com/conformal/btcdb" "github.com/conformal/btcutil" @@ -72,25 +71,17 @@ func (db *LevelDb) FetchBlockHeaderBySha(sha *btcwire.ShaHash) (bh *btcwire.Bloc } func (db *LevelDb) getBlkLoc(sha *btcwire.ShaHash) (int64, error) { - var blkHeight int64 - key := shaBlkToKey(sha) data, err := db.lDb.Get(key, db.ro) - if err != nil { return 0, err } // deserialize - dr := bytes.NewBuffer(data) - err = binary.Read(dr, binary.LittleEndian, &blkHeight) - if err != nil { - log.Tracef("get getBlkLoc len %v\n", len(data)) - err = fmt.Errorf("Db Corrupt 0") - return 0, err - } - return blkHeight, nil + blkHeight := binary.LittleEndian.Uint64(data) + + return int64(blkHeight), nil } func (db *LevelDb) getBlkByHeight(blkHeight int64) (rsha *btcwire.ShaHash, rbuf []byte, err error) { @@ -131,17 +122,12 @@ func (db *LevelDb) getBlk(sha *btcwire.ShaHash) (rblkHeight int64, rbuf []byte, return blkHeight, buf, nil } -func (db *LevelDb) setBlk(sha *btcwire.ShaHash, blkHeight int64, buf []byte) error { - +func (db *LevelDb) setBlk(sha *btcwire.ShaHash, blkHeight int64, buf []byte) { // serialize - var lw bytes.Buffer - err := binary.Write(&lw, binary.LittleEndian, blkHeight) - if err != nil { - err = fmt.Errorf("Write Fail") - return err - } - shaKey := shaBlkToKey(sha) + var lw [8]byte + binary.LittleEndian.PutUint64(lw[:], uint64(blkHeight)) + shaKey := shaBlkToKey(sha) blkKey := int64ToKey(blkHeight) shaB := sha.Bytes() @@ -149,21 +135,15 @@ func (db *LevelDb) setBlk(sha *btcwire.ShaHash, blkHeight int64, buf []byte) err copy(blkVal[0:], shaB) copy(blkVal[len(shaB):], buf) - db.lBatch().Put(shaKey, lw.Bytes()) - + db.lBatch().Put(shaKey, lw[:]) db.lBatch().Put(blkKey, blkVal) - - return nil } // insertSha stores a block hash and its associated data block with a // previous sha of `prevSha'. // insertSha shall be called with db lock held -func (db *LevelDb) insertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHash, buf []byte) (blockid int64, err error) { - - var oBlkHeight int64 - oBlkHeight, err = db.getBlkLoc(prevSha) - +func (db *LevelDb) insertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHash, buf []byte) (int64, error) { + oBlkHeight, err := db.getBlkLoc(prevSha) if err != nil { // check current block count // if count != 0 { @@ -179,11 +159,7 @@ func (db *LevelDb) insertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHas // TODO(drahn) check curfile filesize, increment curfile if this puts it over blkHeight := oBlkHeight + 1 - err = db.setBlk(sha, blkHeight, buf) - - if err != nil { - return - } + db.setBlk(sha, blkHeight, buf) // update the last block cache db.lastBlkShaCached = true diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 95f42bfb..1e797a5a 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -649,28 +649,22 @@ func (db *LevelDb) processBatches() error { for txSha, txU := range db.txUpdateMap { key := shaTxToKey(&txSha) if txU.delete { - //log.Infof("deleting tx %v", txSha) + log.Tracef("deleting tx %v", txSha) db.lbatch.Delete(key) } else { - //log.Infof("inserting tx %v", txSha) - txdat, err := db.formatTx(txU) - if err != nil { - return err - } + log.Tracef("inserting tx %v", txSha) + txdat := db.formatTx(txU) db.lbatch.Put(key, txdat) } } for txSha, txSu := range db.txSpentUpdateMap { key := shaSpentTxToKey(&txSha) if txSu.delete { - //log.Infof("deleting tx %v", txSha) + log.Tracef("deleting tx %v", txSha) db.lbatch.Delete(key) } else { - //log.Infof("inserting tx %v", txSha) - txdat, err := db.formatTxFullySpent(txSu.txl) - if err != nil { - return err - } + log.Tracef("inserting tx %v", txSha) + txdat := db.formatTxFullySpent(txSu.txl) db.lbatch.Put(key, txdat) } } diff --git a/ldb/tx.go b/ldb/tx.go index 1e2e7ce0..c010998f 100644 --- a/ldb/tx.go +++ b/ldb/tx.go @@ -7,7 +7,6 @@ package ldb import ( "bytes" "encoding/binary" - "fmt" "github.com/conformal/btcdb" "github.com/conformal/btcwire" @@ -61,81 +60,36 @@ func (db *LevelDb) insertTx(txSha *btcwire.ShaHash, height int64, txoff int, txl } // formatTx generates the value buffer for the Tx db. -func (db *LevelDb) formatTx(txu *txUpdateObj) ([]byte, error) { - - blkHeight := txu.blkHeight - txoff := txu.txoff - txlen := txu.txlen +func (db *LevelDb) formatTx(txu *txUpdateObj) []byte { + blkHeight := uint64(txu.blkHeight) + txOff := uint32(txu.txoff) + txLen := uint32(txu.txlen) spentbuf := txu.spentData - txOff := int32(txoff) - txLen := int32(txlen) + txW := make([]byte, 16+len(spentbuf)) + binary.LittleEndian.PutUint64(txW[:], blkHeight) + binary.LittleEndian.PutUint32(txW[8:], txOff) + binary.LittleEndian.PutUint32(txW[12:], txLen) + copy(txW[16:], spentbuf) - var txW bytes.Buffer - - err := binary.Write(&txW, binary.LittleEndian, blkHeight) - if err != nil { - err = fmt.Errorf("Write fail") - return nil, err - } - - err = binary.Write(&txW, binary.LittleEndian, txOff) - if err != nil { - err = fmt.Errorf("Write fail") - return nil, err - } - - err = binary.Write(&txW, binary.LittleEndian, txLen) - if err != nil { - err = fmt.Errorf("Write fail") - return nil, err - } - - err = binary.Write(&txW, binary.LittleEndian, spentbuf) - if err != nil { - err = fmt.Errorf("Write fail") - return nil, err - } - - return txW.Bytes(), nil + return txW[:] } -func (db *LevelDb) getTxData(txsha *btcwire.ShaHash) (rblkHeight int64, - rtxOff int, rtxLen int, rspentBuf []byte, err error) { - var buf []byte - +func (db *LevelDb) getTxData(txsha *btcwire.ShaHash) (int64, int, int, []byte, error) { key := shaTxToKey(txsha) - buf, err = db.lDb.Get(key, db.ro) + buf, err := db.lDb.Get(key, db.ro) if err != nil { - return + return 0, 0, 0, nil, err } - var blkHeight int64 - var txOff, txLen int32 - dr := bytes.NewBuffer(buf) - err = binary.Read(dr, binary.LittleEndian, &blkHeight) - if err != nil { - err = fmt.Errorf("Db Corrupt 1") - return - } - err = binary.Read(dr, binary.LittleEndian, &txOff) - if err != nil { - err = fmt.Errorf("Db Corrupt 2") - return - } - err = binary.Read(dr, binary.LittleEndian, &txLen) - if err != nil { - err = fmt.Errorf("Db Corrupt 3") - return - } - // remainder of buffer is spentbuf - spentBuf := make([]byte, dr.Len()) - err = binary.Read(dr, binary.LittleEndian, spentBuf) - if err != nil { - err = fmt.Errorf("Db Corrupt 4") - return - } - return blkHeight, int(txOff), int(txLen), spentBuf, nil + blkHeight := binary.LittleEndian.Uint64(buf) + txOff := binary.LittleEndian.Uint32(buf[8:]) + txLen := binary.LittleEndian.Uint32(buf[12:]) + + spentBuf := make([]byte, len(buf)-16) + copy(spentBuf, buf[16:]) + + return int64(blkHeight), int(txOff), int(txLen), spentBuf, nil } func (db *LevelDb) getTxFullySpent(txsha *btcwire.ShaHash) ([]*spentTx, error) { @@ -150,41 +104,22 @@ func (db *LevelDb) getTxFullySpent(txsha *btcwire.ShaHash) ([]*spentTx, error) { return badTxList, err } txListLen := len(buf) / 20 - txR := bytes.NewBuffer(buf) + spentTxList = make([]*spentTx, txListLen, txListLen) - for i := range spentTxList { - var sTx spentTx - var blkHeight int64 - var txOff, txLen, numTxO int32 + offset := i * 20 - err := binary.Read(txR, binary.LittleEndian, &blkHeight) - if err != nil { - err = fmt.Errorf("sTx Read fail 0") - return nil, err - } - sTx.blkHeight = blkHeight + blkHeight := binary.LittleEndian.Uint64(buf[offset:]) + txOff := binary.LittleEndian.Uint32(buf[offset+8:]) + txLen := binary.LittleEndian.Uint32(buf[offset+12:]) + numTxO := binary.LittleEndian.Uint32(buf[offset+16:]) - err = binary.Read(txR, binary.LittleEndian, &txOff) - if err != nil { - err = fmt.Errorf("sTx Read fail 1") - return nil, err + sTx := spentTx{ + blkHeight: int64(blkHeight), + txoff: int(txOff), + txlen: int(txLen), + numTxO: int(numTxO), } - sTx.txoff = int(txOff) - - err = binary.Read(txR, binary.LittleEndian, &txLen) - if err != nil { - err = fmt.Errorf("sTx Read fail 2") - return nil, err - } - sTx.txlen = int(txLen) - - err = binary.Read(txR, binary.LittleEndian, &numTxO) - if err != nil { - err = fmt.Errorf("sTx Read fail 3") - return nil, err - } - sTx.numTxO = int(numTxO) spentTxList[i] = &sTx } @@ -192,41 +127,23 @@ func (db *LevelDb) getTxFullySpent(txsha *btcwire.ShaHash) ([]*spentTx, error) { return spentTxList, nil } -func (db *LevelDb) formatTxFullySpent(sTxList []*spentTx) ([]byte, error) { - var txW bytes.Buffer +func (db *LevelDb) formatTxFullySpent(sTxList []*spentTx) []byte { + txW := make([]byte, 20*len(sTxList)) - for _, sTx := range sTxList { - blkHeight := sTx.blkHeight - txOff := int32(sTx.txoff) - txLen := int32(sTx.txlen) - numTxO := int32(sTx.numTxO) + for i, sTx := range sTxList { + blkHeight := uint64(sTx.blkHeight) + txOff := uint32(sTx.txoff) + txLen := uint32(sTx.txlen) + numTxO := uint32(sTx.numTxO) + offset := i * 20 - err := binary.Write(&txW, binary.LittleEndian, blkHeight) - if err != nil { - err = fmt.Errorf("Write fail") - return nil, err - } - - err = binary.Write(&txW, binary.LittleEndian, txOff) - if err != nil { - err = fmt.Errorf("Write fail") - return nil, err - } - - err = binary.Write(&txW, binary.LittleEndian, txLen) - if err != nil { - err = fmt.Errorf("Write fail") - return nil, err - } - - err = binary.Write(&txW, binary.LittleEndian, numTxO) - if err != nil { - err = fmt.Errorf("Write fail") - return nil, err - } + binary.LittleEndian.PutUint64(txW[offset:], blkHeight) + binary.LittleEndian.PutUint32(txW[offset+8:], txOff) + binary.LittleEndian.PutUint32(txW[offset+12:], txLen) + binary.LittleEndian.PutUint32(txW[offset+16:], numTxO) } - return txW.Bytes(), nil + return txW } // ExistsTxSha returns if the given tx sha exists in the database From 0dd81ad003c805dde3470ed6424b0ba3f4dd5805 Mon Sep 17 00:00:00 2001 From: David Hill Date: Thu, 5 Jun 2014 13:45:43 -0400 Subject: [PATCH 131/163] Use NewReader instead of NewBuffer. NewReader is slightly faster than NewBuffer, but more importantly, ensures the data is read-only. --- ldb/block.go | 4 ++-- ldb/leveldb.go | 8 ++++---- ldb/tx.go | 30 +++++++++++++++--------------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/ldb/block.go b/ldb/block.go index 5d4ff22f..25e4c1af 100644 --- a/ldb/block.go +++ b/ldb/block.go @@ -61,7 +61,7 @@ func (db *LevelDb) FetchBlockHeaderBySha(sha *btcwire.ShaHash) (bh *btcwire.Bloc // Only deserialize the header portion and ensure the transaction count // is zero since this is a standalone header. var blockHeader btcwire.BlockHeader - err = blockHeader.Deserialize(bytes.NewBuffer(buf)) + err = blockHeader.Deserialize(bytes.NewReader(buf)) if err != nil { return nil, err } @@ -125,7 +125,7 @@ func (db *LevelDb) getBlk(sha *btcwire.ShaHash) (rblkHeight int64, rbuf []byte, func (db *LevelDb) setBlk(sha *btcwire.ShaHash, blkHeight int64, buf []byte) { // serialize var lw [8]byte - binary.LittleEndian.PutUint64(lw[:], uint64(blkHeight)) + binary.LittleEndian.PutUint64(lw[0:8], uint64(blkHeight)) shaKey := shaBlkToKey(sha) blkKey := int64ToKey(blkHeight) diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 1e797a5a..2d7a6e4a 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -649,10 +649,10 @@ func (db *LevelDb) processBatches() error { for txSha, txU := range db.txUpdateMap { key := shaTxToKey(&txSha) if txU.delete { - log.Tracef("deleting tx %v", txSha) + //log.Tracef("deleting tx %v", txSha) db.lbatch.Delete(key) } else { - log.Tracef("inserting tx %v", txSha) + //log.Tracef("inserting tx %v", txSha) txdat := db.formatTx(txU) db.lbatch.Put(key, txdat) } @@ -660,10 +660,10 @@ func (db *LevelDb) processBatches() error { for txSha, txSu := range db.txSpentUpdateMap { key := shaSpentTxToKey(&txSha) if txSu.delete { - log.Tracef("deleting tx %v", txSha) + //log.Tracef("deleting tx %v", txSha) db.lbatch.Delete(key) } else { - log.Tracef("inserting tx %v", txSha) + //log.Tracef("inserting tx %v", txSha) txdat := db.formatTxFullySpent(txSu.txl) db.lbatch.Put(key, txdat) } diff --git a/ldb/tx.go b/ldb/tx.go index c010998f..ef7ec77b 100644 --- a/ldb/tx.go +++ b/ldb/tx.go @@ -67,9 +67,9 @@ func (db *LevelDb) formatTx(txu *txUpdateObj) []byte { spentbuf := txu.spentData txW := make([]byte, 16+len(spentbuf)) - binary.LittleEndian.PutUint64(txW[:], blkHeight) - binary.LittleEndian.PutUint32(txW[8:], txOff) - binary.LittleEndian.PutUint32(txW[12:], txLen) + binary.LittleEndian.PutUint64(txW[0:8], blkHeight) + binary.LittleEndian.PutUint32(txW[8:12], txOff) + binary.LittleEndian.PutUint32(txW[12:16], txLen) copy(txW[16:], spentbuf) return txW[:] @@ -82,9 +82,9 @@ func (db *LevelDb) getTxData(txsha *btcwire.ShaHash) (int64, int, int, []byte, e return 0, 0, 0, nil, err } - blkHeight := binary.LittleEndian.Uint64(buf) - txOff := binary.LittleEndian.Uint32(buf[8:]) - txLen := binary.LittleEndian.Uint32(buf[12:]) + blkHeight := binary.LittleEndian.Uint64(buf[0:8]) + txOff := binary.LittleEndian.Uint32(buf[8:12]) + txLen := binary.LittleEndian.Uint32(buf[12:16]) spentBuf := make([]byte, len(buf)-16) copy(spentBuf, buf[16:]) @@ -109,10 +109,10 @@ func (db *LevelDb) getTxFullySpent(txsha *btcwire.ShaHash) ([]*spentTx, error) { for i := range spentTxList { offset := i * 20 - blkHeight := binary.LittleEndian.Uint64(buf[offset:]) - txOff := binary.LittleEndian.Uint32(buf[offset+8:]) - txLen := binary.LittleEndian.Uint32(buf[offset+12:]) - numTxO := binary.LittleEndian.Uint32(buf[offset+16:]) + blkHeight := binary.LittleEndian.Uint64(buf[offset:offset+8]) + txOff := binary.LittleEndian.Uint32(buf[offset+8:offset+12]) + txLen := binary.LittleEndian.Uint32(buf[offset+12:offset+16]) + numTxO := binary.LittleEndian.Uint32(buf[offset+16:offset+20]) sTx := spentTx{ blkHeight: int64(blkHeight), @@ -137,10 +137,10 @@ func (db *LevelDb) formatTxFullySpent(sTxList []*spentTx) []byte { numTxO := uint32(sTx.numTxO) offset := i * 20 - binary.LittleEndian.PutUint64(txW[offset:], blkHeight) - binary.LittleEndian.PutUint32(txW[offset+8:], txOff) - binary.LittleEndian.PutUint32(txW[offset+12:], txLen) - binary.LittleEndian.PutUint32(txW[offset+16:], numTxO) + binary.LittleEndian.PutUint64(txW[offset:offset+8], blkHeight) + binary.LittleEndian.PutUint32(txW[offset+8:offset+12], txOff) + binary.LittleEndian.PutUint32(txW[offset+12:offset+16], txLen) + binary.LittleEndian.PutUint32(txW[offset+16:offset+20], numTxO) } return txW @@ -272,7 +272,7 @@ func (db *LevelDb) fetchTxDataByLoc(blkHeight int64, txOff int, txLen int, txspe err = btcdb.TxShaMissing return } - rbuf := bytes.NewBuffer(blkbuf[txOff : txOff+txLen]) + rbuf := bytes.NewReader(blkbuf[txOff : txOff+txLen]) var tx btcwire.MsgTx err = tx.Deserialize(rbuf) From 879d69d0400b9a634bfefddb83e10b8f8c7b152b Mon Sep 17 00:00:00 2001 From: David Hill Date: Mon, 7 Jul 2014 11:07:07 -0400 Subject: [PATCH 132/163] use strconv.FormatInt which is much faster than fmt.Sprintf --- ldb/leveldb.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 2d7a6e4a..04ddb474 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -8,6 +8,7 @@ import ( "encoding/binary" "fmt" "os" + "strconv" "sync" "github.com/conformal/btcdb" @@ -608,7 +609,7 @@ func (db *LevelDb) setclearSpentData(txsha *btcwire.ShaHash, idx uint32, set boo } func int64ToKey(keyint int64) []byte { - key := fmt.Sprintf("%d", keyint) + key := strconv.FormatInt(keyint, 10) return []byte(key) } From d31183ff1966817133d9817cc33bf1c2771ac04c Mon Sep 17 00:00:00 2001 From: David Hill Date: Mon, 7 Jul 2014 11:07:30 -0400 Subject: [PATCH 133/163] go fmt --- ldb/tx.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ldb/tx.go b/ldb/tx.go index ef7ec77b..8f5c94c3 100644 --- a/ldb/tx.go +++ b/ldb/tx.go @@ -109,10 +109,10 @@ func (db *LevelDb) getTxFullySpent(txsha *btcwire.ShaHash) ([]*spentTx, error) { for i := range spentTxList { offset := i * 20 - blkHeight := binary.LittleEndian.Uint64(buf[offset:offset+8]) - txOff := binary.LittleEndian.Uint32(buf[offset+8:offset+12]) - txLen := binary.LittleEndian.Uint32(buf[offset+12:offset+16]) - numTxO := binary.LittleEndian.Uint32(buf[offset+16:offset+20]) + blkHeight := binary.LittleEndian.Uint64(buf[offset : offset+8]) + txOff := binary.LittleEndian.Uint32(buf[offset+8 : offset+12]) + txLen := binary.LittleEndian.Uint32(buf[offset+12 : offset+16]) + numTxO := binary.LittleEndian.Uint32(buf[offset+16 : offset+20]) sTx := spentTx{ blkHeight: int64(blkHeight), From 16dc2cf2d0231af4015546c6ef91d7ea3710a6f8 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Mon, 7 Jul 2014 09:50:50 -0500 Subject: [PATCH 134/163] Add errors to all interface methods. This commit adds error returns to all of the Db interface methods except for FetchTxByShaList and FetchUnSpentTxByShaList since they expose the errors on each individual transaction. It also updates all tests and code for both the leveldb and memdb drivers for the changes. Closes #5. ok @drahn --- db.go | 20 +++++++++++++++----- interface_test.go | 17 +++++++++++++++-- ldb/block.go | 23 ++++++++++------------- ldb/boundary_test.go | 10 ++++++++-- ldb/dup_test.go | 12 ++++++++++-- ldb/insertremove_test.go | 13 ++++++++++--- ldb/leveldb.go | 15 ++++++++------- ldb/operational_test.go | 32 ++++++++++++++++++++++++++------ ldb/tx.go | 16 ++++++++-------- memdb/memdb.go | 35 +++++++++++++++++++---------------- memdb/memdb_test.go | 29 ++++++++++++++++++----------- 11 files changed, 147 insertions(+), 75 deletions(-) diff --git a/db.go b/db.go index adfbb405..157f24ed 100644 --- a/db.go +++ b/db.go @@ -30,7 +30,7 @@ const AllShas = int64(^uint64(0) >> 1) // used to add a new backend data storage method. type Db interface { // Close cleanly shuts down the database and syncs all data. - Close() + Close() (err error) // DropAfterBlockBySha will remove any blocks from the database after // the given block. It terminates any existing transaction and performs @@ -40,7 +40,7 @@ type Db interface { // ExistsSha returns whether or not the given block hash is present in // the database. - ExistsSha(sha *btcwire.ShaHash) (exists bool) + ExistsSha(sha *btcwire.ShaHash) (exists bool, err error) // FetchBlockBySha returns a btcutil Block. The implementation may // cache the underlying data if desired. @@ -65,7 +65,7 @@ type Db interface { // ExistsTxSha returns whether or not the given tx hash is present in // the database - ExistsTxSha(sha *btcwire.ShaHash) (exists bool) + ExistsTxSha(sha *btcwire.ShaHash) (exists bool, err error) // FetchTxBySha returns some data for the given transaction hash. The // implementation may cache the underlying data if desired. @@ -75,12 +75,22 @@ type Db interface { // hashes. The implementation may cache the underlying data if desired. // This differs from FetchUnSpentTxByShaList in that it will return // the most recent known Tx, if it is fully spent or not. + // + // NOTE: This function does not return an error directly since it MUST + // return at least one TxListReply instance for each requested + // transaction. Each TxListReply instance then contains an Err field + // which can be used to detect errors. FetchTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply // FetchUnSpentTxByShaList returns a TxListReply given an array of // transaction hashes. The implementation may cache the underlying // data if desired. Fully spent transactions will not normally not // be returned in this operation. + // + // NOTE: This function does not return an error directly since it MUST + // return at least one TxListReply instance for each requested + // transaction. Each TxListReply instance then contains an Err field + // which can be used to detect errors. FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*TxListReply // InsertBlock inserts raw block and transaction data from a block @@ -97,11 +107,11 @@ type Db interface { // RollbackClose discards the recent database changes to the previously // saved data at last Sync and closes the database. - RollbackClose() + RollbackClose() (err error) // Sync verifies that the database is coherent on disk and no // outstanding transactions are in flight. - Sync() + Sync() (err error) } // DriverDB defines a structure for backend drivers to use when they registered diff --git a/interface_test.go b/interface_test.go index 21500a67..31383d56 100644 --- a/interface_test.go +++ b/interface_test.go @@ -85,7 +85,13 @@ func testNewestSha(tc *testContext) bool { // testExistsSha ensures ExistsSha conforms to the interface contract. func testExistsSha(tc *testContext) bool { // The block must exist in the database. - if exists := tc.db.ExistsSha(tc.blockHash); !exists { + exists, err := tc.db.ExistsSha(tc.blockHash) + if err != nil { + tc.t.Errorf("ExistsSha (%s): block #%d (%s) unexpected error: "+ + "%v", tc.dbType, tc.blockHeight, tc.blockHash, err) + return false + } + if !exists { tc.t.Errorf("ExistsSha (%s): block #%d (%s) does not exist", tc.dbType, tc.blockHeight, tc.blockHash) return false @@ -232,7 +238,14 @@ func testExistsTxSha(tc *testContext) bool { for i, tx := range tc.block.Transactions() { // The transaction must exist in the database. txHash := tx.Sha() - if exists := tc.db.ExistsTxSha(txHash); !exists { + exists, err := tc.db.ExistsTxSha(txHash) + if err != nil { + tc.t.Errorf("ExistsTxSha (%s): block #%d (%s) tx #%d "+ + "(%s) unexpected error: %v", tc.dbType, + tc.blockHeight, tc.blockHash, i, txHash, err) + return false + } + if !exists { _, err := tc.db.FetchTxBySha(txHash) if err != nil { tc.t.Errorf("ExistsTxSha (%s): block #%d (%s) "+ diff --git a/ldb/block.go b/ldb/block.go index 25e4c1af..5e5ad56d 100644 --- a/ldb/block.go +++ b/ldb/block.go @@ -11,6 +11,7 @@ import ( "github.com/conformal/btcdb" "github.com/conformal/btcutil" "github.com/conformal/btcwire" + "github.com/conformal/goleveldb/leveldb" ) // FetchBlockBySha - return a btcutil Block @@ -186,30 +187,26 @@ func (db *LevelDb) fetchSha(sha *btcwire.ShaHash) (rbuf []byte, // ExistsSha looks up the given block hash // returns true if it is present in the database. -func (db *LevelDb) ExistsSha(sha *btcwire.ShaHash) (exists bool) { +func (db *LevelDb) ExistsSha(sha *btcwire.ShaHash) (bool, error) { db.dbLock.Lock() defer db.dbLock.Unlock() // not in cache, try database - exists = db.blkExistsSha(sha) - return + return db.blkExistsSha(sha) } // blkExistsSha looks up the given block hash // returns true if it is present in the database. // CALLED WITH LOCK HELD -func (db *LevelDb) blkExistsSha(sha *btcwire.ShaHash) bool { - +func (db *LevelDb) blkExistsSha(sha *btcwire.ShaHash) (bool, error) { _, err := db.getBlkLoc(sha) - - if err != nil { - /* - should this warn if the failure is something besides does not exist ? - log.Warnf("blkExistsSha: fail %v", err) - */ - return false + switch err { + case nil: + return true, nil + case leveldb.ErrNotFound: + return false, nil } - return true + return false, err } // FetchBlockShaByHeight returns a block hash based on its height in the diff --git a/ldb/boundary_test.go b/ldb/boundary_test.go index af49a684..f9ff23c6 100644 --- a/ldb/boundary_test.go +++ b/ldb/boundary_test.go @@ -37,14 +37,20 @@ func TestEmptyDB(t *testing.T) { } // This is a reopen test - db.Close() + if err := db.Close(); err != nil { + t.Errorf("Close: unexpected error: %v", err) + } db, err = btcdb.OpenDB("leveldb", dbname) if err != nil { t.Errorf("Failed to open test database %v", err) return } - defer db.Close() + defer func() { + if err := db.Close(); err != nil { + t.Errorf("Close: unexpected error: %v", err) + } + }() sha, height, err = db.NewestSha() if !sha.IsEqual(&btcwire.ShaHash{}) { diff --git a/ldb/dup_test.go b/ldb/dup_test.go index 8b638dfd..dca2f1fa 100644 --- a/ldb/dup_test.go +++ b/ldb/dup_test.go @@ -29,7 +29,11 @@ func Test_dupTx(t *testing.T) { } defer os.RemoveAll(dbname) defer os.RemoveAll(dbnamever) - defer db.Close() + defer func() { + if err := db.Close(); err != nil { + t.Errorf("Close: unexpected error: %v", err) + } + }() testdatafile := filepath.Join("testdata", "blocks1-256.bz2") blocks, err := loadBlocks(t, testdatafile) @@ -58,7 +62,11 @@ out: origintxsha := &txin.PreviousOutpoint.Hash txneededList = append(txneededList, origintxsha) - if !db.ExistsTxSha(origintxsha) { + exists, err := db.ExistsTxSha(origintxsha) + if err != nil { + t.Errorf("ExistsTxSha: unexpected error %v ", err) + } + if !exists { t.Errorf("referenced tx not found %v ", origintxsha) } diff --git a/ldb/insertremove_test.go b/ldb/insertremove_test.go index 27fdc486..217502ef 100644 --- a/ldb/insertremove_test.go +++ b/ldb/insertremove_test.go @@ -55,7 +55,11 @@ func testUnspentInsert(t *testing.T) { } defer os.RemoveAll(dbname) defer os.RemoveAll(dbnamever) - defer db.Close() + defer func() { + if err := db.Close(); err != nil { + t.Errorf("Close: unexpected error: %v", err) + } + }() blocks := loadblocks(t) endtest: @@ -79,10 +83,13 @@ endtest: txneededList = append(txneededList, origintxsha) txlookupList = append(txlookupList, origintxsha) - if !db.ExistsTxSha(origintxsha) { + exists, err := db.ExistsTxSha(origintxsha) + if err != nil { + t.Errorf("ExistsTxSha: unexpected error %v ", err) + } + if !exists { t.Errorf("referenced tx not found %v ", origintxsha) } - } txshaname, _ := tx.TxSha() txlookupList = append(txlookupList, &txshaname) diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 04ddb474..08490df6 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -255,26 +255,27 @@ func CreateDB(args ...interface{}) (btcdb.Db, error) { return db, err } -func (db *LevelDb) close() { - db.lDb.Close() +func (db *LevelDb) close() error { + return db.lDb.Close() } // Sync verifies that the database is coherent on disk, // and no outstanding transactions are in flight. -func (db *LevelDb) Sync() { +func (db *LevelDb) Sync() error { db.dbLock.Lock() defer db.dbLock.Unlock() // while specified by the API, does nothing // however does grab lock to verify it does not return until other operations are complete. + return nil } // Close cleanly shuts down database, syncing all data. -func (db *LevelDb) Close() { +func (db *LevelDb) Close() error { db.dbLock.Lock() defer db.dbLock.Unlock() - db.close() + return db.close() } // DropAfterBlockBySha will remove any blocks from the database after @@ -682,9 +683,9 @@ func (db *LevelDb) processBatches() error { return nil } -func (db *LevelDb) RollbackClose() { +func (db *LevelDb) RollbackClose() error { db.dbLock.Lock() defer db.dbLock.Unlock() - db.close() + return db.close() } diff --git a/ldb/operational_test.go b/ldb/operational_test.go index cd8d6395..c0a3ca1e 100644 --- a/ldb/operational_test.go +++ b/ldb/operational_test.go @@ -44,7 +44,11 @@ func testOperationalMode(t *testing.T) { } defer os.RemoveAll(dbname) defer os.RemoveAll(dbnamever) - defer db.Close() + defer func() { + if err := db.Close(); err != nil { + t.Errorf("Close: unexpected error: %v", err) + } + }() testdatafile := filepath.Join("..", "testdata", "blocks1-256.bz2") blocks, err := loadBlocks(t, testdatafile) @@ -67,9 +71,14 @@ out: origintxsha := &txin.PreviousOutpoint.Hash txneededList = append(txneededList, origintxsha) - if !db.ExistsTxSha(origintxsha) { + exists, err := db.ExistsTxSha(origintxsha) + if err != nil { + t.Errorf("ExistsTxSha: unexpected error %v ", err) + } + if !exists { t.Errorf("referenced tx not found %v ", origintxsha) } + _, err = db.FetchTxBySha(origintxsha) if err != nil { t.Errorf("referenced tx not found %v err %v ", origintxsha, err) @@ -178,14 +187,20 @@ func testBackout(t *testing.T) { t.Errorf("Failed to open test database %v", err) return } - defer db.Close() + defer func() { + if err := db.Close(); err != nil { + t.Errorf("Close: unexpected error: %v", err) + } + }() sha, err := blocks[99].Sha() if err != nil { t.Errorf("failed to get block 99 sha err %v", err) return } - _ = db.ExistsSha(sha) + if _, err := db.ExistsSha(sha); err != nil { + t.Errorf("ExistsSha: unexpected error: %v") + } _, err = db.FetchBlockBySha(sha) if err != nil { t.Errorf("failed to load block 99 from db %v", err) @@ -197,7 +212,9 @@ func testBackout(t *testing.T) { t.Errorf("failed to get block 110 sha err %v", err) return } - _ = db.ExistsSha(sha) + if _, err := db.ExistsSha(sha); err != nil { + t.Errorf("ExistsSha: unexpected error: %v") + } _, err = db.FetchBlockBySha(sha) if err != nil { t.Errorf("loaded block 119 from db") @@ -207,7 +224,10 @@ func testBackout(t *testing.T) { block := blocks[119] mblock := block.MsgBlock() txsha, err := mblock.Transactions[0].TxSha() - exists := db.ExistsTxSha(&txsha) + exists, err := db.ExistsTxSha(&txsha) + if err != nil { + t.Errorf("ExistsTxSha: unexpected error %v ", err) + } if !exists { t.Errorf("tx %v not located db\n", txsha) } diff --git a/ldb/tx.go b/ldb/tx.go index 8f5c94c3..8201b1fb 100644 --- a/ldb/tx.go +++ b/ldb/tx.go @@ -147,7 +147,7 @@ func (db *LevelDb) formatTxFullySpent(sTxList []*spentTx) []byte { } // ExistsTxSha returns if the given tx sha exists in the database -func (db *LevelDb) ExistsTxSha(txsha *btcwire.ShaHash) (exists bool) { +func (db *LevelDb) ExistsTxSha(txsha *btcwire.ShaHash) (bool, error) { db.dbLock.Lock() defer db.dbLock.Unlock() @@ -156,15 +156,15 @@ func (db *LevelDb) ExistsTxSha(txsha *btcwire.ShaHash) (exists bool) { // existsTxSha returns if the given tx sha exists in the database.o // Must be called with the db lock held. -func (db *LevelDb) existsTxSha(txSha *btcwire.ShaHash) (exists bool) { +func (db *LevelDb) existsTxSha(txSha *btcwire.ShaHash) (bool, error) { _, _, _, _, err := db.getTxData(txSha) - if err == nil { - return true + switch err { + case nil: + return true, nil + case leveldb.ErrNotFound: + return false, nil } - - // BUG(drahn) If there was an error beside non-existant deal with it. - - return false + return false, err } // FetchTxByShaList returns the most recent tx of the name fully spent or not diff --git a/memdb/memdb.go b/memdb/memdb.go index bd2acb62..bbafd62f 100644 --- a/memdb/memdb.go +++ b/memdb/memdb.go @@ -138,14 +138,19 @@ func (db *MemDb) removeTx(msgTx *btcwire.MsgTx, txHash *btcwire.ShaHash) { // // All data is purged upon close with this implementation since it is a // memory-only database. -func (db *MemDb) Close() { +func (db *MemDb) Close() error { db.Lock() defer db.Unlock() + if db.closed { + return ErrDbClosed + } + db.blocks = nil db.blocksBySha = nil db.txns = nil db.closed = true + return nil } // DropAfterBlockBySha removes any blocks from the database after the given @@ -192,20 +197,19 @@ func (db *MemDb) DropAfterBlockBySha(sha *btcwire.ShaHash) error { // ExistsSha returns whether or not the given block hash is present in the // database. This is part of the btcdb.Db interface implementation. -func (db *MemDb) ExistsSha(sha *btcwire.ShaHash) bool { +func (db *MemDb) ExistsSha(sha *btcwire.ShaHash) (bool, error) { db.Lock() defer db.Unlock() if db.closed { - log.Warnf("ExistsSha called after db close.") - return false + return false, ErrDbClosed } if _, exists := db.blocksBySha[*sha]; exists { - return true + return true, nil } - return false + return false, nil } // FetchBlockBySha returns a btcutil.Block. The implementation may cache the @@ -346,20 +350,19 @@ func (db *MemDb) FetchHeightRange(startHeight, endHeight int64) ([]btcwire.ShaHa // ExistsTxSha returns whether or not the given transaction hash is present in // the database and is not fully spent. This is part of the btcdb.Db interface // implementation. -func (db *MemDb) ExistsTxSha(sha *btcwire.ShaHash) bool { +func (db *MemDb) ExistsTxSha(sha *btcwire.ShaHash) (bool, error) { db.Lock() defer db.Unlock() if db.closed { - log.Warnf("ExistsTxSha called after db close.") - return false + return false, ErrDbClosed } if txns, exists := db.txns[*sha]; exists { - return !isFullySpent(txns[len(txns)-1]) + return !isFullySpent(txns[len(txns)-1]), nil } - return false + return false, nil } // FetchTxBySha returns some data for the given transaction hash. The @@ -702,10 +705,10 @@ func (db *MemDb) NewestSha() (*btcwire.ShaHash, int64, error) { // The database is completely purged on close with this implementation since the // entire database is only in memory. As a result, this function behaves no // differently than Close. -func (db *MemDb) RollbackClose() { +func (db *MemDb) RollbackClose() error { // Rollback doesn't apply to a memory database, so just call Close. // Close handles the mutex locks. - db.Close() + return db.Close() } // Sync verifies that the database is coherent on disk and no outstanding @@ -714,18 +717,18 @@ func (db *MemDb) RollbackClose() { // // This implementation does not write any data to disk, so this function only // grabs a lock to ensure it doesn't return until other operations are complete. -func (db *MemDb) Sync() { +func (db *MemDb) Sync() error { db.Lock() defer db.Unlock() if db.closed { - log.Warnf("Sync called after db close.") + return ErrDbClosed } // There is nothing extra to do to sync the memory database. However, // the lock is still grabbed to ensure the function does not return // until other operations are complete. - return + return nil } // newMemDb returns a new memory-only database ready for block inserts. diff --git a/memdb/memdb_test.go b/memdb/memdb_test.go index 8e33de47..ddbe3717 100644 --- a/memdb/memdb_test.go +++ b/memdb/memdb_test.go @@ -29,15 +29,17 @@ func TestClosed(t *testing.T) { if err != nil { t.Errorf("InsertBlock: %v", err) } - db.Close() + if err := db.Close(); err != nil { + t.Errorf("Close: unexpected error %v", err) + } genesisHash := btcnet.MainNetParams.GenesisHash if err := db.DropAfterBlockBySha(genesisHash); err != memdb.ErrDbClosed { t.Errorf("DropAfterBlockBySha: unexpected error %v", err) } - if exists := db.ExistsSha(genesisHash); exists != false { - t.Errorf("ExistsSha: genesis hash exists after close") + if _, err := db.ExistsSha(genesisHash); err != memdb.ErrDbClosed { + t.Errorf("ExistsSha: Unexpected error: %v", err) } if _, err := db.FetchBlockBySha(genesisHash); err != memdb.ErrDbClosed { @@ -57,9 +59,8 @@ func TestClosed(t *testing.T) { if err != nil { t.Errorf("TxSha: unexpected error %v", err) } - if exists := db.ExistsTxSha(&coinbaseHash); exists != false { - t.Errorf("ExistsTxSha: hash %v exists when it shouldn't", - &coinbaseHash) + if _, err := db.ExistsTxSha(&coinbaseHash); err != memdb.ErrDbClosed { + t.Errorf("ExistsTxSha: unexpected error %v", err) } if _, err := db.FetchTxBySha(genesisHash); err != memdb.ErrDbClosed { @@ -103,9 +104,15 @@ func TestClosed(t *testing.T) { t.Errorf("NewestSha: unexpected error %v", err) } - // The following calls don't return errors from the interface to be able - // to detect a closed database, so just call them to ensure there are no - // panics. - db.Sync() - db.RollbackClose() + if err := db.Sync(); err != memdb.ErrDbClosed { + t.Errorf("Sync: unexpected error %v", err) + } + + if err := db.RollbackClose(); err != memdb.ErrDbClosed { + t.Errorf("RollbackClose: unexpected error %v", err) + } + + if err := db.Close(); err != memdb.ErrDbClosed { + t.Errorf("Close: unexpected error %v", err) + } } From 7354ecefe87b27827bf2eb9ad50bfc2b90de43de Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Tue, 8 Jul 2014 08:26:17 -0500 Subject: [PATCH 135/163] Add godoc reference badge to README.md. --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6d6ada3e..eb05c1ed 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,6 @@ See `test_coverage.txt` for the current coverage (using gocov). Alternatively, if you are running a POSIX OS, you can run the cov_report.sh script for a real-time report. Package btcdb is licensed under the liberal ISC license. -Note that the main `Db` interface is not yet finalized, so it is subject to -change. - ## Sample Use ```Go @@ -43,6 +40,9 @@ change. ## Documentation +[![GoDoc](https://godoc.org/github.com/conformal/btcdb?status.png)] +(http://godoc.org/github.com/conformal/btcdb) + Full `go doc` style documentation for the project can be viewed online without installing this package by using the GoDoc site [here](http://godoc.org/github.com/conformal/btcdb). From 9ed908f44ae91988a26ba7e9f1cee15ebe253a07 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Tue, 8 Jul 2014 08:45:37 -0500 Subject: [PATCH 136/163] Use testable example and update README.md. This commit moves the example to a test file so it integrates nicely with Go's example tooling. This allows the example output to be tested as a part of running the normal Go tests to help ensure it doesn't get out of date with the code. It is also nice to have the example in one place rather than repeating it in doc.go and README.md. Links and information about the example have been included in README.md in place of the example. --- README.md | 35 +++++++---------------------------- doc.go | 26 +------------------------- example_test.go | 42 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 53 deletions(-) create mode 100644 example_test.go diff --git a/README.md b/README.md index eb05c1ed..813eee02 100644 --- a/README.md +++ b/README.md @@ -10,34 +10,6 @@ See `test_coverage.txt` for the current coverage (using gocov). Alternatively, if you are running a POSIX OS, you can run the cov_report.sh script for a real-time report. Package btcdb is licensed under the liberal ISC license. -## Sample Use - -```Go - // Import packages. - import ( - "github.com/conformal/btcdb" - _ "github.com/conformal/btcdb/ldb" - "github.com/conformal/btcnet" - "github.com/conformal/btcutil" - ) - - // Create a database and schedule it to be closed on exit. - dbName := "example" - db, err := btcdb.CreateDB("leveldb", dbName) - if err != nil { - // Log and handle the error - } - defer db.Close() - - - // Insert the main network genesis block. - genesis := btcutil.NewBlock(btcnet.MainNetParams.GenesisBlock) - newHeight, err := db.InsertBlock(genesis) - if err != nil { - // Log and handle the error - } -``` - ## Documentation [![GoDoc](https://godoc.org/github.com/conformal/btcdb?status.png)] @@ -57,6 +29,13 @@ http://localhost:6060/pkg/github.com/conformal/btcdb $ go get github.com/conformal/btcdb ``` +## Examples + +* [CreateDB Example] + (http://godoc.org/github.com/conformal/btcdb#example-CreateDB) + Demonstrates creating a new database and inserting the genesis + block into it. + ## TODO - Increase test coverage to 100% diff --git a/doc.go b/doc.go index 5a89f75d..559428ff 100644 --- a/doc.go +++ b/doc.go @@ -27,30 +27,6 @@ At the highest level, the use of this packages just requires that you import it, setup a database, insert some data into it, and optionally, query the data back. The first block inserted into the database will be treated as the genesis block. Every subsequent block insert requires the -referenced parent block to already exist. In a more concrete example: - - // Import packages. - import ( - "github.com/conformal/btcdb" - _ "github.com/conformal/btcdb/ldb" - "github.com/conformal/btcnet" - "github.com/conformal/btcutil" - ) - - // Create a database and schedule it to be closed on exit. - dbName := "example.db" - db, err := btcdb.CreateDB("leveldb", dbName) - if err != nil { - // Log and handle the error - } - defer db.Close() - - - // Insert the main network genesis block. - genesis := btcutil.NewBlock(btcnet.MainNetParams.GenesisBlock) - newHeight, err := db.InsertBlock(genesis) - if err != nil { - // Log and handle the error - } +referenced parent block to already exist. */ package btcdb diff --git a/example_test.go b/example_test.go new file mode 100644 index 00000000..eda1c0d1 --- /dev/null +++ b/example_test.go @@ -0,0 +1,42 @@ +// Copyright (c) 2013-2014 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package btcdb_test + +import ( + "fmt" + "github.com/conformal/btcdb" + _ "github.com/conformal/btcdb/memdb" + "github.com/conformal/btcnet" + "github.com/conformal/btcutil" +) + +// This example demonstrates creating a new database and inserting the genesis +// block into it. +func ExampleCreateDB() { + // Create a database and schedule it to be closed on exit. This example + // uses a memory-only database to avoid needing to write anything to + // the disk. Typically, you would specify a persistent database driver + // such as "leveldb" and give it a database name as the second + // parameter. + db, err := btcdb.CreateDB("memdb") + if err != nil { + fmt.Println(err) + return + } + defer db.Close() + + // Insert the main network genesis block. + genesis := btcutil.NewBlock(btcnet.MainNetParams.GenesisBlock) + newHeight, err := db.InsertBlock(genesis) + if err != nil { + fmt.Println(err) + return + } + + fmt.Println("New height:", newHeight) + + // Output: + // New height: 0 +} From a12b62c24c7e3d7dac8f77ec5e398b880b7d770a Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Tue, 8 Jul 2014 08:50:12 -0500 Subject: [PATCH 137/163] Add commented imports to the example. --- example_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/example_test.go b/example_test.go index eda1c0d1..6c8e677f 100644 --- a/example_test.go +++ b/example_test.go @@ -15,6 +15,14 @@ import ( // This example demonstrates creating a new database and inserting the genesis // block into it. func ExampleCreateDB() { + // Notice in these example imports that the memdb driver is loaded. + // Ordinarily this would be whatever driver(s) your application + // requires. + // import ( + // "github.com/conformal/btcdb" + // _ "github.com/conformal/btcdb/memdb" + // ) + // Create a database and schedule it to be closed on exit. This example // uses a memory-only database to avoid needing to write anything to // the disk. Typically, you would specify a persistent database driver From 329c0bf094c8b9c4c4f7bda16ae5bc7a24ad95c7 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Tue, 8 Jul 2014 08:54:07 -0500 Subject: [PATCH 138/163] Update overview documentation to more recent stats. --- doc.go | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/doc.go b/doc.go index 559428ff..6e529ce4 100644 --- a/doc.go +++ b/doc.go @@ -5,28 +5,27 @@ /* Package btcdb provides a database interface for the Bitcoin block chain. -As of May 2013, there are over 235,000 blocks in the Bitcoin block chain and -and over 17 million transactions (which turns out to be over 11GB of data). +As of July 2014, there are over 309,000 blocks in the Bitcoin block chain and +and over 42 million transactions (which turns out to be over 21GB of data). btcdb provides a database layer to store and retrieve this data in a fairly simple and efficient manner. The use of this should not require specific knowledge of the database backend. Basic Design -The basic design of btcdb is to provide two classes of items in a -database; blocks and transactions (tx) where the block number -increases monotonically. Each transaction belongs to a single block -although a block can have a variable number of transactions. Along -with these two items, several convenience functions for dealing with -the database are provided as well as functions to query specific items -that may be present in a block or tx. +The basic design of btcdb is to provide two classes of items in a database; +blocks and transactions (tx) where the block number increases monotonically. +Each transaction belongs to a single block although a block can have a variable +number of transactions. Along with these two items, several convenience +functions for dealing with the database are provided as well as functions to +query specific items that may be present in a block or tx. Usage -At the highest level, the use of this packages just requires that you -import it, setup a database, insert some data into it, and optionally, -query the data back. The first block inserted into the database will be -treated as the genesis block. Every subsequent block insert requires the -referenced parent block to already exist. +At the highest level, the use of this packages just requires that you import it, +setup a database, insert some data into it, and optionally, query the data back. +The first block inserted into the database will be treated as the genesis block. +Every subsequent block insert requires the referenced parent block to already +exist. */ package btcdb From 05ff5bd8ec32c9fec8f8c923292c97603384b6a2 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Tue, 8 Jul 2014 09:10:14 -0500 Subject: [PATCH 139/163] Add NewestSha example. --- README.md | 8 ++++++-- example_test.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 813eee02..3ec1b0f0 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,12 @@ $ go get github.com/conformal/btcdb * [CreateDB Example] (http://godoc.org/github.com/conformal/btcdb#example-CreateDB) - Demonstrates creating a new database and inserting the genesis - block into it. + Demonstrates creating a new database and inserting the genesis block into it. + +* [NewestSha Example] + (http://godoc.org/github.com/conformal/btcdb#example-CreateDB) + Demonstrates querying the database for the most recent best block height and + hash. ## TODO - Increase test coverage to 100% diff --git a/example_test.go b/example_test.go index 6c8e677f..79219a28 100644 --- a/example_test.go +++ b/example_test.go @@ -48,3 +48,46 @@ func ExampleCreateDB() { // Output: // New height: 0 } + +// exampleLoadDb is used in the example to elide the setup code. +func exampleLoadDb() (btcdb.Db, error) { + db, err := btcdb.CreateDB("memdb") + if err != nil { + return nil, err + } + + // Insert the main network genesis block. + genesis := btcutil.NewBlock(btcnet.MainNetParams.GenesisBlock) + _, err = db.InsertBlock(genesis) + if err != nil { + return nil, err + } + + return db, err +} + +// This example demonstrates querying the database for the most recent best +// block height and hash. +func ExampleDb_NewestSha() { + // Load a database for the purposes of this example and schedule it to + // be closed on exit. See the CreateDB example for more details on what + // this step is doing. + db, err := exampleLoadDb() + if err != nil { + fmt.Println(err) + return + } + defer db.Close() + + latestHash, latestHeight, err := db.NewestSha() + if err != nil { + fmt.Println(err) + return + } + fmt.Println("Latest hash:", latestHash) + fmt.Println("Latest height:", latestHeight) + + // Output: + // Latest hash: 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f + // Latest height: 0 +} From 4422b14f63ff92faf5d4f7042a2ce6141ab9744d Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Tue, 8 Jul 2014 09:25:47 -0500 Subject: [PATCH 140/163] Categorize NewestSha example under Db interface. It seems the godoc example extraction does not recognize the same syntax used for functions with receivers when working with interfaces. So, instead, categorize the example on the interface. --- README.md | 2 +- example_test.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3ec1b0f0..2318a5b6 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ $ go get github.com/conformal/btcdb Demonstrates creating a new database and inserting the genesis block into it. * [NewestSha Example] - (http://godoc.org/github.com/conformal/btcdb#example-CreateDB) + (http://godoc.org/github.com/conformal/btcdb#example-Db) Demonstrates querying the database for the most recent best block height and hash. diff --git a/example_test.go b/example_test.go index 79219a28..c9e632fb 100644 --- a/example_test.go +++ b/example_test.go @@ -49,8 +49,8 @@ func ExampleCreateDB() { // New height: 0 } -// exampleLoadDb is used in the example to elide the setup code. -func exampleLoadDb() (btcdb.Db, error) { +// exampleLoadDB is used in the example to elide the setup code. +func exampleLoadDB() (btcdb.Db, error) { db, err := btcdb.CreateDB("memdb") if err != nil { return nil, err @@ -68,11 +68,11 @@ func exampleLoadDb() (btcdb.Db, error) { // This example demonstrates querying the database for the most recent best // block height and hash. -func ExampleDb_NewestSha() { +func ExampleDb() { // Load a database for the purposes of this example and schedule it to // be closed on exit. See the CreateDB example for more details on what // this step is doing. - db, err := exampleLoadDb() + db, err := exampleLoadDB() if err != nil { fmt.Println(err) return From 1c877b33ea2c897f649c90b0ea3220fbf234a86e Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Tue, 8 Jul 2014 10:04:03 -0500 Subject: [PATCH 141/163] Use magic syntax for godoc NewestSha example. --- README.md | 2 +- example_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2318a5b6..c5a91e52 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ $ go get github.com/conformal/btcdb Demonstrates creating a new database and inserting the genesis block into it. * [NewestSha Example] - (http://godoc.org/github.com/conformal/btcdb#example-Db) + (http://godoc.org/github.com/conformal/btcdb#example-Db_newestSha) Demonstrates querying the database for the most recent best block height and hash. diff --git a/example_test.go b/example_test.go index c9e632fb..80622e4b 100644 --- a/example_test.go +++ b/example_test.go @@ -68,7 +68,7 @@ func exampleLoadDB() (btcdb.Db, error) { // This example demonstrates querying the database for the most recent best // block height and hash. -func ExampleDb() { +func ExampleDb_newestSha() { // Load a database for the purposes of this example and schedule it to // be closed on exit. See the CreateDB example for more details on what // this step is doing. From d9ee066af6941375fa4e51b123786bd3f86f809b Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Tue, 8 Jul 2014 10:05:58 -0500 Subject: [PATCH 142/163] Example link godoc web don't match local godoc. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c5a91e52..2258af44 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ $ go get github.com/conformal/btcdb Demonstrates creating a new database and inserting the genesis block into it. * [NewestSha Example] - (http://godoc.org/github.com/conformal/btcdb#example-Db_newestSha) + (http://godoc.org/github.com/conformal/btcdb#example-Db--NewestSha) Demonstrates querying the database for the most recent best block height and hash. From f373ba3583122f07611fec4037caf9a2dc913876 Mon Sep 17 00:00:00 2001 From: Dale Rahn Date: Thu, 17 Jul 2014 10:48:03 -0400 Subject: [PATCH 143/163] Introduce new error to indicate that block is not found vs db error. ok @davecgh --- db.go | 11 ++++++----- ldb/block.go | 5 ++++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/db.go b/db.go index 157f24ed..489d4767 100644 --- a/db.go +++ b/db.go @@ -13,11 +13,12 @@ import ( // Errors that the various database functions may return. var ( - PrevShaMissing = errors.New("Previous sha missing from database") - TxShaMissing = errors.New("Requested transaction does not exist") - DuplicateSha = errors.New("Duplicate insert attempted") - DbDoesNotExist = errors.New("Non-existent database") - DbUnknownType = errors.New("Non-existent database type") + PrevShaMissing = errors.New("Previous sha missing from database") + TxShaMissing = errors.New("Requested transaction does not exist") + BlockShaMissing = errors.New("Requested block does not exist") + DuplicateSha = errors.New("Duplicate insert attempted") + DbDoesNotExist = errors.New("Non-existent database") + DbUnknownType = errors.New("Non-existent database type") ) // AllShas is a special value that can be used as the final sha when requesting diff --git a/ldb/block.go b/ldb/block.go index 5e5ad56d..85588d16 100644 --- a/ldb/block.go +++ b/ldb/block.go @@ -76,6 +76,9 @@ func (db *LevelDb) getBlkLoc(sha *btcwire.ShaHash) (int64, error) { data, err := db.lDb.Get(key, db.ro) if err != nil { + if err == leveldb.ErrNotFound { + err = btcdb.BlockShaMissing + } return 0, err } @@ -203,7 +206,7 @@ func (db *LevelDb) blkExistsSha(sha *btcwire.ShaHash) (bool, error) { switch err { case nil: return true, nil - case leveldb.ErrNotFound: + case leveldb.ErrNotFound, btcdb.BlockShaMissing: return false, nil } return false, err From 68d10f4dc8a75bc14b343758edb20258eb274c07 Mon Sep 17 00:00:00 2001 From: "John C. Vernaleo" Date: Tue, 16 Sep 2014 10:54:38 -0400 Subject: [PATCH 144/163] Add some missing comments. ok @davecgh --- ldb/leveldb.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 08490df6..c9e6317f 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -36,6 +36,7 @@ type tTxInsertData struct { usedbuf []byte } +// LevelDb holds internal state for databse. type LevelDb struct { // lock preventing multiple entry dbLock sync.Mutex @@ -146,6 +147,7 @@ blocknarrow: return db, nil } +// CurrentDBVersion is the database version. var CurrentDBVersion int32 = 1 func openDB(dbpath string, create bool) (pbdb btcdb.Db, err error) { @@ -683,6 +685,9 @@ func (db *LevelDb) processBatches() error { return nil } +// RollbackClose this is part of the btcdb.Db interface and should discard +// recent changes to the db and the close the db. This currently just does +// a clean shutdown. func (db *LevelDb) RollbackClose() error { db.dbLock.Lock() defer db.dbLock.Unlock() From 372bbe5c49fb54a7f7f51c67098918aca5ad552a Mon Sep 17 00:00:00 2001 From: "John C. Vernaleo" Date: Tue, 16 Sep 2014 11:31:51 -0400 Subject: [PATCH 145/163] Add vet and lint where possible to travis. --- .travis.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index ae71c02f..771b5207 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,16 @@ language: go -go: release -install: go get -d -t -v ./... +go: + - 1.2 + - release + - tip +install: + - go get -d -t -v ./... + - go get -v code.google.com/p/go.tools/cmd/vet + - go get -v github.com/GeertJohan/fgt + - go get -v github.com/golang/lint/golint +script: + - export PATH=$PATH:$HOME/gopath/bin + - go vet ./... + - fgt golint memdb + - fgt golint ldb + - go test -v From a7ac93f5b64f5fea083d1662c29fd8cb58dca5c6 Mon Sep 17 00:00:00 2001 From: "John C. Vernaleo" Date: Tue, 16 Sep 2014 14:43:09 -0400 Subject: [PATCH 146/163] Error improvements for lint. Rename errors to Err*. Lowercase error text. Add golint for top level package ok @davecgh --- .travis.yml | 1 + db.go | 16 ++++++++-------- ldb/block.go | 4 ++-- ldb/internal_test.go | 2 +- ldb/leveldb.go | 2 +- ldb/tx.go | 14 +++++++------- memdb/memdb.go | 18 +++++++++--------- 7 files changed, 29 insertions(+), 28 deletions(-) diff --git a/.travis.yml b/.travis.yml index 771b5207..aadc7570 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ install: script: - export PATH=$PATH:$HOME/gopath/bin - go vet ./... + - fgt golint . - fgt golint memdb - fgt golint ldb - go test -v diff --git a/db.go b/db.go index 489d4767..ea4bfdb5 100644 --- a/db.go +++ b/db.go @@ -13,12 +13,12 @@ import ( // Errors that the various database functions may return. var ( - PrevShaMissing = errors.New("Previous sha missing from database") - TxShaMissing = errors.New("Requested transaction does not exist") - BlockShaMissing = errors.New("Requested block does not exist") - DuplicateSha = errors.New("Duplicate insert attempted") - DbDoesNotExist = errors.New("Non-existent database") - DbUnknownType = errors.New("Non-existent database type") + ErrPrevShaMissing = errors.New("previous sha missing from database") + ErrTxShaMissing = errors.New("requested transaction does not exist") + ErrBlockShaMissing = errors.New("requested block does not exist") + ErrDuplicateSha = errors.New("duplicate insert attempted") + ErrDbDoesNotExist = errors.New("non-existent database") + ErrDbUnknownType = errors.New("non-existent database type") ) // AllShas is a special value that can be used as the final sha when requesting @@ -156,7 +156,7 @@ func CreateDB(dbtype string, args ...interface{}) (pbdb Db, err error) { return drv.CreateDB(args...) } } - return nil, DbUnknownType + return nil, ErrDbUnknownType } // OpenDB opens an existing database. @@ -166,7 +166,7 @@ func OpenDB(dbtype string, args ...interface{}) (pbdb Db, err error) { return drv.OpenDB(args...) } } - return nil, DbUnknownType + return nil, ErrDbUnknownType } // SupportedDBs returns a slice of strings that represent the database drivers diff --git a/ldb/block.go b/ldb/block.go index 85588d16..d8e8c2fa 100644 --- a/ldb/block.go +++ b/ldb/block.go @@ -77,7 +77,7 @@ func (db *LevelDb) getBlkLoc(sha *btcwire.ShaHash) (int64, error) { data, err := db.lDb.Get(key, db.ro) if err != nil { if err == leveldb.ErrNotFound { - err = btcdb.BlockShaMissing + err = btcdb.ErrBlockShaMissing } return 0, err } @@ -206,7 +206,7 @@ func (db *LevelDb) blkExistsSha(sha *btcwire.ShaHash) (bool, error) { switch err { case nil: return true, nil - case leveldb.ErrNotFound, btcdb.BlockShaMissing: + case leveldb.ErrNotFound, btcdb.ErrBlockShaMissing: return false, nil } return false, err diff --git a/ldb/internal_test.go b/ldb/internal_test.go index 359a0e79..a38bd644 100644 --- a/ldb/internal_test.go +++ b/ldb/internal_test.go @@ -17,7 +17,7 @@ func FetchSha(db btcdb.Db, sha *btcwire.ShaHash) (buf []byte, pver uint32, blkid int64, err error) { sqldb, ok := db.(*LevelDb) if !ok { - err = fmt.Errorf("Invalid data type") + err = fmt.Errorf("invalid data type") return } buf, blkid, err = sqldb.fetchSha(sha) diff --git a/ldb/leveldb.go b/ldb/leveldb.go index c9e6317f..ad1bb3c3 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -175,7 +175,7 @@ func openDB(dbpath string, create bool) (pbdb btcdb.Db, err error) { } else { _, err = os.Stat(dbpath) if err != nil { - err = btcdb.DbDoesNotExist + err = btcdb.ErrDbDoesNotExist return } } diff --git a/ldb/tx.go b/ldb/tx.go index 8201b1fb..5a433c1f 100644 --- a/ldb/tx.go +++ b/ldb/tx.go @@ -99,7 +99,7 @@ func (db *LevelDb) getTxFullySpent(txsha *btcwire.ShaHash) ([]*spentTx, error) { key := shaSpentTxToKey(txsha) buf, err := db.lDb.Get(key, db.ro) if err == leveldb.ErrNotFound { - return badTxList, btcdb.TxShaMissing + return badTxList, btcdb.ErrTxShaMissing } else if err != nil { return badTxList, err } @@ -186,7 +186,7 @@ func (db *LevelDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxLis btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0 } } - if err == btcdb.TxShaMissing { + if err == btcdb.ErrTxShaMissing { // if the unspent pool did not have the tx, // look in the fully spent pool (only last instance @@ -244,7 +244,7 @@ func (db *LevelDb) fetchTxDataBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, blkHeight, txOff, txLen, txspent, err = db.getTxData(txsha) if err != nil { if err == leveldb.ErrNotFound { - err = btcdb.TxShaMissing + err = btcdb.ErrTxShaMissing } return } @@ -260,7 +260,7 @@ func (db *LevelDb) fetchTxDataByLoc(blkHeight int64, txOff int, txLen int, txspe blksha, blkbuf, err = db.getBlkByHeight(blkHeight) if err != nil { if err == leveldb.ErrNotFound { - err = btcdb.TxShaMissing + err = btcdb.ErrTxShaMissing } return } @@ -269,7 +269,7 @@ func (db *LevelDb) fetchTxDataByLoc(blkHeight int64, txOff int, txLen int, txspe // txsha, blksha, blkHeight, txOff, txLen) if len(blkbuf) < txOff+txLen { - err = btcdb.TxShaMissing + err = btcdb.ErrTxShaMissing return } rbuf := bytes.NewReader(blkbuf[txOff : txOff+txLen]) @@ -297,7 +297,7 @@ func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) ([]*btcdb.TxListReply, e if txerr == nil { replylen++ } else { - if txerr != btcdb.TxShaMissing { + if txerr != btcdb.ErrTxShaMissing { return []*btcdb.TxListReply{}, txerr } } @@ -305,7 +305,7 @@ func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) ([]*btcdb.TxListReply, e sTxList, fSerr := db.getTxFullySpent(txsha) if fSerr != nil { - if fSerr != btcdb.TxShaMissing { + if fSerr != btcdb.ErrTxShaMissing { return []*btcdb.TxListReply{}, fSerr } } else { diff --git a/memdb/memdb.go b/memdb/memdb.go index bbafd62f..713a3c0a 100644 --- a/memdb/memdb.go +++ b/memdb/memdb.go @@ -17,7 +17,7 @@ import ( // Errors that the various database functions may return. var ( - ErrDbClosed = errors.New("Database is closed") + ErrDbClosed = errors.New("database is closed") ) var ( @@ -383,7 +383,7 @@ func (db *MemDb) FetchTxBySha(txHash *btcwire.ShaHash) ([]*btcdb.TxListReply, er if !exists { log.Warnf("FetchTxBySha: requested hash of %s does not exist", txHash) - return nil, btcdb.TxShaMissing + return nil, btcdb.ErrTxShaMissing } txHashCopy := *txHash @@ -431,7 +431,7 @@ func (db *MemDb) fetchTxByShaList(txShaList []*btcwire.ShaHash, includeSpent boo // information if the transaction exists. reply := btcdb.TxListReply{ Sha: txShaList[i], - Err: btcdb.TxShaMissing, + Err: btcdb.ErrTxShaMissing, } replyList = append(replyList, &reply) @@ -548,7 +548,7 @@ func (db *MemDb) InsertBlock(block *btcutil.Block) (int64, error) { msgBlock := block.MsgBlock() if _, exists := db.blocksBySha[msgBlock.Header.PrevBlock]; !exists { if len(db.blocks) > 0 { - return 0, btcdb.PrevShaMissing + return 0, btcdb.ErrPrevShaMissing } } @@ -599,7 +599,7 @@ func (db *MemDb) InsertBlock(block *btcutil.Block) (int64, error) { log.Warnf("InsertBlock: requested hash "+ " of %s does not exist in-flight", tx.Sha()) - return 0, btcdb.TxShaMissing + return 0, btcdb.ErrTxShaMissing } } else { originTxns, exists := db.txns[prevOut.Hash] @@ -607,14 +607,14 @@ func (db *MemDb) InsertBlock(block *btcutil.Block) (int64, error) { log.Warnf("InsertBlock: requested hash "+ "of %s by %s does not exist", prevOut.Hash, tx.Sha()) - return 0, btcdb.TxShaMissing + return 0, btcdb.ErrTxShaMissing } originTxD := originTxns[len(originTxns)-1] if prevOut.Index > uint32(len(originTxD.spentBuf)) { log.Warnf("InsertBlock: requested hash "+ "of %s with index %d does not "+ "exist", tx.Sha(), prevOut.Index) - return 0, btcdb.TxShaMissing + return 0, btcdb.ErrTxShaMissing } } } @@ -624,7 +624,7 @@ func (db *MemDb) InsertBlock(block *btcutil.Block) (int64, error) { inFlightIndex < i { log.Warnf("Block contains duplicate transaction %s", tx.Sha()) - return 0, btcdb.DuplicateSha + return 0, btcdb.ErrDuplicateSha } // Prevent duplicate transactions unless the old one is fully @@ -634,7 +634,7 @@ func (db *MemDb) InsertBlock(block *btcutil.Block) (int64, error) { if !isFullySpent(txD) { log.Warnf("Attempt to insert duplicate "+ "transaction %s", tx.Sha()) - return 0, btcdb.DuplicateSha + return 0, btcdb.ErrDuplicateSha } } } From d6945f94e6ffaf6a43f6b7e6c048ec3b6afd3457 Mon Sep 17 00:00:00 2001 From: "John C. Vernaleo" Date: Tue, 16 Sep 2014 15:13:31 -0400 Subject: [PATCH 147/163] Fix tests. --- db_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db_test.go b/db_test.go index 9ec93cb6..11baa7e6 100644 --- a/db_test.go +++ b/db_test.go @@ -151,18 +151,18 @@ func TestCreateOpenUnsupported(t *testing.T) { // expected error. dbType := "unsupported" _, err := btcdb.CreateDB(dbType, "unsupportedcreatetest") - if err != btcdb.DbUnknownType { + if err != btcdb.ErrDbUnknownType { t.Errorf("TestCreateOpenUnsupported: expected error not "+ - "received - got: %v, want %v", err, btcdb.DbUnknownType) + "received - got: %v, want %v", err, btcdb.ErrDbUnknownType) return } // Ensure opening a database with the new type fails with the expected // error. _, err = btcdb.OpenDB(dbType, "unsupportedopentest") - if err != btcdb.DbUnknownType { + if err != btcdb.ErrDbUnknownType { t.Errorf("TestCreateOpenUnsupported: expected error not "+ - "received - got: %v, want %v", err, btcdb.DbUnknownType) + "received - got: %v, want %v", err, btcdb.ErrDbUnknownType) return } } From 2a688a70b2aa4456fc8f01a03039a96fa4be2626 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Sat, 20 Sep 2014 00:47:20 -0500 Subject: [PATCH 148/163] Update ldb tests for latest API error changes. --- ldb/dup_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ldb/dup_test.go b/ldb/dup_test.go index dca2f1fa..b2a7ca3d 100644 --- a/ldb/dup_test.go +++ b/ldb/dup_test.go @@ -154,7 +154,7 @@ out: listReply = db.FetchUnSpentTxByShaList(fetchList) for _, lr := range listReply { - if lr.Err != btcdb.TxShaMissing { + if lr.Err != btcdb.ErrTxShaMissing { t.Errorf("sha %v spent %v err %v\n", lr.Sha, lr.TxSpent, lr.Err) } From 968b6da5dbf2d80cbae2d88389132c833590f5dc Mon Sep 17 00:00:00 2001 From: Jonathan Gillham Date: Wed, 1 Oct 2014 13:52:19 +0100 Subject: [PATCH 149/163] Changed TxIn.PreviousOutpoint to TxIn.PreviousOutPoint after btcwire API change. --- ldb/dup_test.go | 4 ++-- ldb/insertremove_test.go | 6 +++--- ldb/leveldb.go | 8 ++++---- ldb/operational_test.go | 4 ++-- memdb/memdb.go | 8 ++++---- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/ldb/dup_test.go b/ldb/dup_test.go index b2a7ca3d..1ce933bb 100644 --- a/ldb/dup_test.go +++ b/ldb/dup_test.go @@ -56,10 +56,10 @@ out: var txneededList []*btcwire.ShaHash for _, tx := range mblock.Transactions { for _, txin := range tx.TxIn { - if txin.PreviousOutpoint.Index == uint32(4294967295) { + if txin.PreviousOutPoint.Index == uint32(4294967295) { continue } - origintxsha := &txin.PreviousOutpoint.Hash + origintxsha := &txin.PreviousOutPoint.Hash txneededList = append(txneededList, origintxsha) exists, err := db.ExistsTxSha(origintxsha) diff --git a/ldb/insertremove_test.go b/ldb/insertremove_test.go index 217502ef..0481af8b 100644 --- a/ldb/insertremove_test.go +++ b/ldb/insertremove_test.go @@ -74,12 +74,12 @@ endtest: var txInList []*btcwire.OutPoint for _, tx := range mblock.Transactions { for _, txin := range tx.TxIn { - if txin.PreviousOutpoint.Index == uint32(4294967295) { + if txin.PreviousOutPoint.Index == uint32(4294967295) { continue } - origintxsha := &txin.PreviousOutpoint.Hash + origintxsha := &txin.PreviousOutPoint.Hash - txInList = append(txInList, &txin.PreviousOutpoint) + txInList = append(txInList, &txin.PreviousOutPoint) txneededList = append(txneededList, origintxsha) txlookupList = append(txlookupList, origintxsha) diff --git a/ldb/leveldb.go b/ldb/leveldb.go index ad1bb3c3..8c7ab826 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -455,8 +455,8 @@ func (db *LevelDb) doSpend(tx *btcwire.MsgTx) error { for txinidx := range tx.TxIn { txin := tx.TxIn[txinidx] - inTxSha := txin.PreviousOutpoint.Hash - inTxidx := txin.PreviousOutpoint.Index + inTxSha := txin.PreviousOutPoint.Hash + inTxidx := txin.PreviousOutPoint.Index if inTxidx == ^uint32(0) { continue @@ -478,8 +478,8 @@ func (db *LevelDb) unSpend(tx *btcwire.MsgTx) error { for txinidx := range tx.TxIn { txin := tx.TxIn[txinidx] - inTxSha := txin.PreviousOutpoint.Hash - inTxidx := txin.PreviousOutpoint.Index + inTxSha := txin.PreviousOutPoint.Hash + inTxidx := txin.PreviousOutPoint.Index if inTxidx == ^uint32(0) { continue diff --git a/ldb/operational_test.go b/ldb/operational_test.go index c0a3ca1e..a31564dc 100644 --- a/ldb/operational_test.go +++ b/ldb/operational_test.go @@ -65,10 +65,10 @@ out: var txneededList []*btcwire.ShaHash for _, tx := range mblock.Transactions { for _, txin := range tx.TxIn { - if txin.PreviousOutpoint.Index == uint32(4294967295) { + if txin.PreviousOutPoint.Index == uint32(4294967295) { continue } - origintxsha := &txin.PreviousOutpoint.Hash + origintxsha := &txin.PreviousOutPoint.Hash txneededList = append(txneededList, origintxsha) exists, err := db.ExistsTxSha(origintxsha) diff --git a/memdb/memdb.go b/memdb/memdb.go index 713a3c0a..20691b8c 100644 --- a/memdb/memdb.go +++ b/memdb/memdb.go @@ -52,7 +52,7 @@ func newShaHashFromStr(hexStr string) *btcwire.ShaHash { // a single input that has a previous output transaction index set to the // maximum value along with a zero hash. func isCoinbaseInput(txIn *btcwire.TxIn) bool { - prevOut := &txIn.PreviousOutpoint + prevOut := &txIn.PreviousOutPoint if prevOut.Index == math.MaxUint32 && prevOut.Hash.IsEqual(&zeroHash) { return true } @@ -105,7 +105,7 @@ func (db *MemDb) removeTx(msgTx *btcwire.MsgTx, txHash *btcwire.ShaHash) { continue } - prevOut := &txIn.PreviousOutpoint + prevOut := &txIn.PreviousOutPoint originTxns, exists := db.txns[prevOut.Hash] if !exists { log.Warnf("Unable to find input transaction %s to "+ @@ -593,7 +593,7 @@ func (db *MemDb) InsertBlock(block *btcutil.Block) (int64, error) { // the output of another transaction in this block only // if the referenced transaction comes before the // current one in this block. - prevOut := &txIn.PreviousOutpoint + prevOut := &txIn.PreviousOutPoint if inFlightIndex, ok := txInFlight[prevOut.Hash]; ok { if i <= inFlightIndex { log.Warnf("InsertBlock: requested hash "+ @@ -661,7 +661,7 @@ func (db *MemDb) InsertBlock(block *btcutil.Block) (int64, error) { } // Already checked for existing and valid ranges above. - prevOut := &txIn.PreviousOutpoint + prevOut := &txIn.PreviousOutPoint originTxns := db.txns[prevOut.Hash] originTxD := originTxns[len(originTxns)-1] originTxD.spentBuf[prevOut.Index] = true From 9c8ee93b5c74a626d68cc46dee9d2adda0238a19 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Mon, 22 Dec 2014 20:05:11 -0600 Subject: [PATCH 150/163] Update TravisCI to goclean script. Also, correct a couple of issues found by running the script. --- .travis.yml | 22 ++++++++++------------ example_test.go | 1 + ldb/operational_test.go | 4 ++-- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index aadc7570..6f025bf1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,17 +1,15 @@ language: go -go: - - 1.2 - - release - - tip +go: release +before_install: + - gocleandeps=c16c849abae90c23419d + - git clone https://gist.github.com/$gocleandeps.git + - goclean=71d0380287747d956a26 + - git clone https://gist.github.com/$goclean.git install: - go get -d -t -v ./... - - go get -v code.google.com/p/go.tools/cmd/vet - - go get -v github.com/GeertJohan/fgt - - go get -v github.com/golang/lint/golint + - bash $gocleandeps/gocleandeps.sh script: - export PATH=$PATH:$HOME/gopath/bin - - go vet ./... - - fgt golint . - - fgt golint memdb - - fgt golint ldb - - go test -v + - bash $goclean/goclean.sh +after_success: + - goveralls -coverprofile=profile.cov -service=travis-ci \ No newline at end of file diff --git a/example_test.go b/example_test.go index 80622e4b..8ca49c54 100644 --- a/example_test.go +++ b/example_test.go @@ -6,6 +6,7 @@ package btcdb_test import ( "fmt" + "github.com/conformal/btcdb" _ "github.com/conformal/btcdb/memdb" "github.com/conformal/btcnet" diff --git a/ldb/operational_test.go b/ldb/operational_test.go index a31564dc..6d4a4e93 100644 --- a/ldb/operational_test.go +++ b/ldb/operational_test.go @@ -199,7 +199,7 @@ func testBackout(t *testing.T) { return } if _, err := db.ExistsSha(sha); err != nil { - t.Errorf("ExistsSha: unexpected error: %v") + t.Errorf("ExistsSha: unexpected error: %v", err) } _, err = db.FetchBlockBySha(sha) if err != nil { @@ -213,7 +213,7 @@ func testBackout(t *testing.T) { return } if _, err := db.ExistsSha(sha); err != nil { - t.Errorf("ExistsSha: unexpected error: %v") + t.Errorf("ExistsSha: unexpected error: %v", err) } _, err = db.FetchBlockBySha(sha) if err != nil { From e6fe5877c4897b0d9c88116587ab941c8dc81f28 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Mon, 22 Dec 2014 20:11:08 -0600 Subject: [PATCH 151/163] Add test coverage badge to README.md. Also, remove the test_coverage.txt file since coverage is now reported by coveralls.io. --- README.md | 4 +- test_coverage.txt | 104 ---------------------------------------------- 2 files changed, 3 insertions(+), 105 deletions(-) delete mode 100644 test_coverage.txt diff --git a/README.md b/README.md index 2258af44..dbe2ba82 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,9 @@ btcdb ===== [![Build Status](https://travis-ci.org/conformal/btcdb.png?branch=master)] -(https://travis-ci.org/conformal/btcdb) +(https://travis-ci.org/conformal/btcdb)[![Coverage Status] +(https://img.shields.io/coveralls/conformal/btcdb.svg)] +(https://coveralls.io/r/conformal/btcdb) Package btcdb provides a database interface for the bitcoin block chain and transactions. There is a test suite with a high percentage of code coverage. diff --git a/test_coverage.txt b/test_coverage.txt deleted file mode 100644 index d6ad3ca1..00000000 --- a/test_coverage.txt +++ /dev/null @@ -1,104 +0,0 @@ -PASS -coverage: 63.3% of statements -ok github.com/conformal/btcdb 0.781s -github.com\conformal\btcdb\db.go: AddDBDriver 100.0% -github.com\conformal\btcdb\db.go: CreateDB 100.0% -github.com\conformal\btcdb\db.go: OpenDB 100.0% -github.com\conformal\btcdb\db.go: SupportedDBs 100.0% -github.com\conformal\btcdb\log.go: init 100.0% -github.com\conformal\btcdb\log.go: DisableLog 100.0% -github.com\conformal\btcdb\log.go: UseLogger 0.0% -github.com\conformal\btcdb\log.go: SetLogWriter 0.0% -github.com\conformal\btcdb\log.go: GetLog 100.0% -total: (statements) 63.3% -============================================================ -PASS -coverage: 77.3% of statements -ok github.com/conformal/btcdb/ldb 0.581s -github.com\conformal\btcdb\ldb\block.go: FetchBlockBySha 100.0% -github.com\conformal\btcdb\ldb\block.go: fetchBlockBySha 75.0% -github.com\conformal\btcdb\ldb\block.go: getBlkLoc 75.0% -github.com\conformal\btcdb\ldb\block.go: getBlkByHeight 100.0% -github.com\conformal\btcdb\ldb\block.go: getBlk 77.8% -github.com\conformal\btcdb\ldb\block.go: setBlk 85.7% -github.com\conformal\btcdb\ldb\block.go: insertBlockData 86.7% -github.com\conformal\btcdb\ldb\block.go: fetchSha 83.3% -github.com\conformal\btcdb\ldb\block.go: ExistsSha 100.0% -github.com\conformal\btcdb\ldb\block.go: blkExistsSha 75.0% -github.com\conformal\btcdb\ldb\block.go: FetchBlockShaByHeight 0.0% -github.com\conformal\btcdb\ldb\block.go: fetchBlockShaByHeight 0.0% -github.com\conformal\btcdb\ldb\block.go: FetchHeightRange 94.4% -github.com\conformal\btcdb\ldb\block.go: NewestSha 100.0% -github.com\conformal\btcdb\ldb\block.go: NewIterateBlocks 0.0% -github.com\conformal\btcdb\ldb\dbcache.go: InvalidateTxCache 0.0% -github.com\conformal\btcdb\ldb\dbcache.go: InvalidateBlockCache 0.0% -github.com\conformal\btcdb\ldb\dbcache.go: InvalidateCache 0.0% -github.com\conformal\btcdb\ldb\leveldb.go: init 100.0% -github.com\conformal\btcdb\ldb\leveldb.go: OpenDB 97.1% -github.com\conformal\btcdb\ldb\leveldb.go: openDB 75.0% -github.com\conformal\btcdb\ldb\leveldb.go: CreateDB 100.0% -github.com\conformal\btcdb\ldb\leveldb.go: close 100.0% -github.com\conformal\btcdb\ldb\leveldb.go: Sync 100.0% -github.com\conformal\btcdb\ldb\leveldb.go: Close 100.0% -github.com\conformal\btcdb\ldb\leveldb.go: DropAfterBlockBySha 80.6% -github.com\conformal\btcdb\ldb\leveldb.go: InsertBlock 43.9% -github.com\conformal\btcdb\ldb\leveldb.go: SetDBInsertMode 0.0% -github.com\conformal\btcdb\ldb\leveldb.go: doSpend 90.0% -github.com\conformal\btcdb\ldb\leveldb.go: unSpend 90.0% -github.com\conformal\btcdb\ldb\leveldb.go: setSpentData 100.0% -github.com\conformal\btcdb\ldb\leveldb.go: clearSpentData 100.0% -github.com\conformal\btcdb\ldb\leveldb.go: setclearSpentData 93.2% -github.com\conformal\btcdb\ldb\leveldb.go: int64ToKey 100.0% -github.com\conformal\btcdb\ldb\leveldb.go: shaBlkToKey 100.0% -github.com\conformal\btcdb\ldb\leveldb.go: shaTxToKey 100.0% -github.com\conformal\btcdb\ldb\leveldb.go: shaSpentTxToKey 100.0% -github.com\conformal\btcdb\ldb\leveldb.go: lBatch 100.0% -github.com\conformal\btcdb\ldb\leveldb.go: processBatches 78.6% -github.com\conformal\btcdb\ldb\leveldb.go: RollbackClose 100.0% -github.com\conformal\btcdb\ldb\tx.go: InsertTx 0.0% -github.com\conformal\btcdb\ldb\tx.go: insertTx 100.0% -github.com\conformal\btcdb\ldb\tx.go: formatTx 66.7% -github.com\conformal\btcdb\ldb\tx.go: getTxData 69.2% -github.com\conformal\btcdb\ldb\tx.go: getTxFullySpent 75.0% -github.com\conformal\btcdb\ldb\tx.go: formatTxFullySpent 65.2% -github.com\conformal\btcdb\ldb\tx.go: ExistsTxSha 100.0% -github.com\conformal\btcdb\ldb\tx.go: existsTxSha 75.0% -github.com\conformal\btcdb\ldb\tx.go: FetchTxByShaList 100.0% -github.com\conformal\btcdb\ldb\tx.go: FetchUnSpentTxByShaList 100.0% -github.com\conformal\btcdb\ldb\tx.go: fetchTxDataBySha 100.0% -github.com\conformal\btcdb\ldb\tx.go: fetchTxDataByLoc 64.3% -github.com\conformal\btcdb\ldb\tx.go: FetchTxBySha 55.6% -total: (statements) 76.9% -============================================================ -PASS -coverage: 44.2% of statements -ok github.com/conformal/btcdb/memdb 0.454s -github.com\conformal\btcdb\memdb\driver.go: init 100.0% -github.com\conformal\btcdb\memdb\driver.go: OpenDB 0.0% -github.com\conformal\btcdb\memdb\driver.go: CreateDB 100.0% -github.com\conformal\btcdb\memdb\memdb.go: newShaHashFromStr 100.0% -github.com\conformal\btcdb\memdb\memdb.go: isCoinbaseInput 75.0% -github.com\conformal\btcdb\memdb\memdb.go: isFullySpent 0.0% -github.com\conformal\btcdb\memdb\memdb.go: removeTx 0.0% -github.com\conformal\btcdb\memdb\memdb.go: Close 100.0% -github.com\conformal\btcdb\memdb\memdb.go: DropAfterBlockBySha 23.5% -github.com\conformal\btcdb\memdb\memdb.go: ExistsSha 62.5% -github.com\conformal\btcdb\memdb\memdb.go: FetchBlockBySha 44.4% -github.com\conformal\btcdb\memdb\memdb.go: FetchBlockShaByHeight 33.3% -github.com\conformal\btcdb\memdb\memdb.go: FetchHeightRange 19.0% -github.com\conformal\btcdb\memdb\memdb.go: ExistsTxSha 62.5% -github.com\conformal\btcdb\memdb\memdb.go: FetchTxBySha 20.0% -github.com\conformal\btcdb\memdb\memdb.go: fetchTxByShaList 33.3% -github.com\conformal\btcdb\memdb\memdb.go: FetchTxByShaList 100.0% -github.com\conformal\btcdb\memdb\memdb.go: FetchUnSpentTxByShaList 100.0% -github.com\conformal\btcdb\memdb\memdb.go: InsertBlock 51.7% -github.com\conformal\btcdb\memdb\memdb.go: InvalidateBlockCache 100.0% -github.com\conformal\btcdb\memdb\memdb.go: InvalidateCache 100.0% -github.com\conformal\btcdb\memdb\memdb.go: InvalidateTxCache 100.0% -github.com\conformal\btcdb\memdb\memdb.go: NewIterateBlocks 100.0% -github.com\conformal\btcdb\memdb\memdb.go: NewestSha 36.4% -github.com\conformal\btcdb\memdb\memdb.go: RollbackClose 100.0% -github.com\conformal\btcdb\memdb\memdb.go: SetDBInsertMode 100.0% -github.com\conformal\btcdb\memdb\memdb.go: Sync 100.0% -github.com\conformal\btcdb\memdb\memdb.go: newMemDb 100.0% -total: (statements) 44.2% From e49bf14993d2efac7e2687add05d4b8ae38149ab Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Mon, 22 Dec 2014 20:23:42 -0600 Subject: [PATCH 152/163] Update build status badge in README.md to SVG. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dbe2ba82..daf250a1 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ btcdb ===== -[![Build Status](https://travis-ci.org/conformal/btcdb.png?branch=master)] +[![Build Status](https://travis-ci.org/conformal/btcdb.svg?branch=master)] (https://travis-ci.org/conformal/btcdb)[![Coverage Status] (https://img.shields.io/coveralls/conformal/btcdb.svg)] (https://coveralls.io/r/conformal/btcdb) From f1f12a3f2f063cce3f68163321be038854f6931a Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Mon, 22 Dec 2014 22:46:39 -0600 Subject: [PATCH 153/163] Update all badges in README.md to SVG. Also, add a license badge. --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index daf250a1..e40fce6c 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,11 @@ btcdb ===== -[![Build Status](https://travis-ci.org/conformal/btcdb.svg?branch=master)] -(https://travis-ci.org/conformal/btcdb)[![Coverage Status] +[![Build Status](http://img.shields.io/travis/conformal/btcdb.svg)] +(https://travis-ci.org/conformal/btcdb) [![Coverage Status] (https://img.shields.io/coveralls/conformal/btcdb.svg)] -(https://coveralls.io/r/conformal/btcdb) +(https://coveralls.io/r/conformal/btcdb) [![ISC License] +(http://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org) Package btcdb provides a database interface for the bitcoin block chain and transactions. There is a test suite with a high percentage of code coverage. @@ -14,7 +15,7 @@ real-time report. Package btcdb is licensed under the liberal ISC license. ## Documentation -[![GoDoc](https://godoc.org/github.com/conformal/btcdb?status.png)] +[![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)] (http://godoc.org/github.com/conformal/btcdb) Full `go doc` style documentation for the project can be viewed online without From b2186ae5ec7910f29cbfd5ac14d27701aec26615 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Mon, 22 Dec 2014 22:50:17 -0600 Subject: [PATCH 154/163] Update README.md. --- README.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index e40fce6c..f4442229 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,7 @@ btcdb (http://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org) Package btcdb provides a database interface for the bitcoin block chain and -transactions. There is a test suite with a high percentage of code coverage. -See `test_coverage.txt` for the current coverage (using gocov). Alternatively, -if you are running a POSIX OS, you can run the cov_report.sh script for a -real-time report. Package btcdb is licensed under the liberal ISC license. +transactions. ## Documentation @@ -68,4 +65,4 @@ signature perform the following: ## License -Package btcdb is licensed under the liberal ISC License. +Package btcdb is licensed under the [copyfree](http://copyfree.org) ISC License. From 97f3917fcf6ff4f4d390533cb26f1fed55a5f518 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Thu, 15 Jan 2015 02:47:40 -0600 Subject: [PATCH 155/163] Update goleveldb import paths to new location. Also update for changes due to the upstream sync. --- ldb/block.go | 2 +- ldb/dbtest/dbtst.go | 4 ++-- ldb/leveldb.go | 12 +++++------- ldb/tx.go | 2 +- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/ldb/block.go b/ldb/block.go index d8e8c2fa..81980c7c 100644 --- a/ldb/block.go +++ b/ldb/block.go @@ -8,10 +8,10 @@ import ( "bytes" "encoding/binary" + "github.com/btcsuite/goleveldb/leveldb" "github.com/conformal/btcdb" "github.com/conformal/btcutil" "github.com/conformal/btcwire" - "github.com/conformal/goleveldb/leveldb" ) // FetchBlockBySha - return a btcutil Block diff --git a/ldb/dbtest/dbtst.go b/ldb/dbtest/dbtst.go index 41e83cad..582168cd 100644 --- a/ldb/dbtest/dbtst.go +++ b/ldb/dbtest/dbtst.go @@ -4,8 +4,8 @@ package main import ( "fmt" - "github.com/conformal/goleveldb/leveldb" - "github.com/conformal/goleveldb/leveldb/opt" + "github.com/btcsuite/goleveldb/leveldb" + "github.com/btcsuite/goleveldb/leveldb/opt" ) type tst struct { diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 8c7ab826..1e7fde4c 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -11,13 +11,12 @@ import ( "strconv" "sync" + "github.com/btcsuite/goleveldb/leveldb" + "github.com/btcsuite/goleveldb/leveldb/opt" "github.com/conformal/btcdb" "github.com/conformal/btclog" "github.com/conformal/btcutil" "github.com/conformal/btcwire" - "github.com/conformal/goleveldb/leveldb" - "github.com/conformal/goleveldb/leveldb/cache" - "github.com/conformal/goleveldb/leveldb/opt" ) const ( @@ -197,11 +196,10 @@ func openDB(dbpath string, create bool) (pbdb btcdb.Db, err error) { } } - myCache := cache.NewEmptyCache() opts := &opt.Options{ - BlockCache: myCache, - MaxOpenFiles: 256, - Compression: opt.NoCompression, + BlockCacher: opt.DefaultBlockCacher, + Compression: opt.NoCompression, + OpenFilesCacher: opt.DefaultOpenFilesCacher, } switch dbversion { diff --git a/ldb/tx.go b/ldb/tx.go index 5a433c1f..00bd838d 100644 --- a/ldb/tx.go +++ b/ldb/tx.go @@ -8,9 +8,9 @@ import ( "bytes" "encoding/binary" + "github.com/btcsuite/goleveldb/leveldb" "github.com/conformal/btcdb" "github.com/conformal/btcwire" - "github.com/conformal/goleveldb/leveldb" ) type txUpdateObj struct { From 0864e31a5480b2c78a11918e8a348cbb32a7f289 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Thu, 15 Jan 2015 10:33:06 -0600 Subject: [PATCH 156/163] Update btcutil import paths to new location. --- common_test.go | 2 +- db.go | 2 +- example_test.go | 2 +- interface_test.go | 2 +- ldb/block.go | 2 +- ldb/dup_test.go | 2 +- ldb/insertremove_test.go | 2 +- ldb/leveldb.go | 2 +- ldb/operational_test.go | 2 +- memdb/memdb.go | 2 +- memdb/memdb_test.go | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/common_test.go b/common_test.go index 56cb62e7..a1b28b56 100644 --- a/common_test.go +++ b/common_test.go @@ -14,11 +14,11 @@ import ( "strings" "testing" + "github.com/btcsuite/btcutil" "github.com/conformal/btcdb" _ "github.com/conformal/btcdb/ldb" _ "github.com/conformal/btcdb/memdb" "github.com/conformal/btcnet" - "github.com/conformal/btcutil" "github.com/conformal/btcwire" ) diff --git a/db.go b/db.go index ea4bfdb5..57a4a097 100644 --- a/db.go +++ b/db.go @@ -7,7 +7,7 @@ package btcdb import ( "errors" - "github.com/conformal/btcutil" + "github.com/btcsuite/btcutil" "github.com/conformal/btcwire" ) diff --git a/example_test.go b/example_test.go index 8ca49c54..e2951713 100644 --- a/example_test.go +++ b/example_test.go @@ -7,10 +7,10 @@ package btcdb_test import ( "fmt" + "github.com/btcsuite/btcutil" "github.com/conformal/btcdb" _ "github.com/conformal/btcdb/memdb" "github.com/conformal/btcnet" - "github.com/conformal/btcutil" ) // This example demonstrates creating a new database and inserting the genesis diff --git a/interface_test.go b/interface_test.go index 31383d56..5c2663fb 100644 --- a/interface_test.go +++ b/interface_test.go @@ -8,8 +8,8 @@ import ( "reflect" "testing" + "github.com/btcsuite/btcutil" "github.com/conformal/btcdb" - "github.com/conformal/btcutil" "github.com/conformal/btcwire" "github.com/davecgh/go-spew/spew" ) diff --git a/ldb/block.go b/ldb/block.go index 81980c7c..ee0d1970 100644 --- a/ldb/block.go +++ b/ldb/block.go @@ -8,9 +8,9 @@ import ( "bytes" "encoding/binary" + "github.com/btcsuite/btcutil" "github.com/btcsuite/goleveldb/leveldb" "github.com/conformal/btcdb" - "github.com/conformal/btcutil" "github.com/conformal/btcwire" ) diff --git a/ldb/dup_test.go b/ldb/dup_test.go index 1ce933bb..84d0452b 100644 --- a/ldb/dup_test.go +++ b/ldb/dup_test.go @@ -10,8 +10,8 @@ import ( "path/filepath" "testing" + "github.com/btcsuite/btcutil" "github.com/conformal/btcdb" - "github.com/conformal/btcutil" "github.com/conformal/btcwire" ) diff --git a/ldb/insertremove_test.go b/ldb/insertremove_test.go index 0481af8b..c980fdd5 100644 --- a/ldb/insertremove_test.go +++ b/ldb/insertremove_test.go @@ -10,9 +10,9 @@ import ( "path/filepath" "testing" + "github.com/btcsuite/btcutil" "github.com/conformal/btcdb" _ "github.com/conformal/btcdb/ldb" - "github.com/conformal/btcutil" "github.com/conformal/btcwire" ) diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 1e7fde4c..4025339a 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -11,11 +11,11 @@ import ( "strconv" "sync" + "github.com/btcsuite/btcutil" "github.com/btcsuite/goleveldb/leveldb" "github.com/btcsuite/goleveldb/leveldb/opt" "github.com/conformal/btcdb" "github.com/conformal/btclog" - "github.com/conformal/btcutil" "github.com/conformal/btcwire" ) diff --git a/ldb/operational_test.go b/ldb/operational_test.go index 6d4a4e93..4e1659ea 100644 --- a/ldb/operational_test.go +++ b/ldb/operational_test.go @@ -14,9 +14,9 @@ import ( "strings" "testing" + "github.com/btcsuite/btcutil" "github.com/conformal/btcdb" "github.com/conformal/btcnet" - "github.com/conformal/btcutil" "github.com/conformal/btcwire" ) diff --git a/memdb/memdb.go b/memdb/memdb.go index 20691b8c..b339d34b 100644 --- a/memdb/memdb.go +++ b/memdb/memdb.go @@ -10,8 +10,8 @@ import ( "math" "sync" + "github.com/btcsuite/btcutil" "github.com/conformal/btcdb" - "github.com/conformal/btcutil" "github.com/conformal/btcwire" ) diff --git a/memdb/memdb_test.go b/memdb/memdb_test.go index ddbe3717..af3aa53a 100644 --- a/memdb/memdb_test.go +++ b/memdb/memdb_test.go @@ -8,10 +8,10 @@ import ( "reflect" "testing" + "github.com/btcsuite/btcutil" "github.com/conformal/btcdb" "github.com/conformal/btcdb/memdb" "github.com/conformal/btcnet" - "github.com/conformal/btcutil" "github.com/conformal/btcwire" ) From e18b38318aad3ebd61a2bb317db8ce91476a9806 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Fri, 16 Jan 2015 11:15:32 -0600 Subject: [PATCH 157/163] Update btclog import paths to new location. --- ldb/leveldb.go | 2 +- log.go | 2 +- memdb/driver.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 4025339a..1b59d274 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -11,11 +11,11 @@ import ( "strconv" "sync" + "github.com/btcsuite/btclog" "github.com/btcsuite/btcutil" "github.com/btcsuite/goleveldb/leveldb" "github.com/btcsuite/goleveldb/leveldb/opt" "github.com/conformal/btcdb" - "github.com/conformal/btclog" "github.com/conformal/btcwire" ) diff --git a/log.go b/log.go index 829dcaf1..09a03d35 100644 --- a/log.go +++ b/log.go @@ -8,7 +8,7 @@ import ( "errors" "io" - "github.com/conformal/btclog" + "github.com/btcsuite/btclog" ) // log is a logger that is initialized with no output filters. This diff --git a/memdb/driver.go b/memdb/driver.go index 49b5904a..bbd31d66 100644 --- a/memdb/driver.go +++ b/memdb/driver.go @@ -7,8 +7,8 @@ package memdb import ( "fmt" + "github.com/btcsuite/btclog" "github.com/conformal/btcdb" - "github.com/conformal/btclog" ) var log = btclog.Disabled From 1b4d4995976d9d2c86ab40aea1a2c38760474b0e Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Fri, 16 Jan 2015 13:59:18 -0600 Subject: [PATCH 158/163] Update btcwire import paths to new location. --- common_test.go | 2 +- db.go | 2 +- interface_test.go | 2 +- ldb/block.go | 2 +- ldb/boundary_test.go | 2 +- ldb/dup_test.go | 2 +- ldb/insertremove_test.go | 2 +- ldb/internal_test.go | 2 +- ldb/leveldb.go | 2 +- ldb/operational_test.go | 2 +- ldb/tx.go | 2 +- memdb/memdb.go | 2 +- memdb/memdb_test.go | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/common_test.go b/common_test.go index a1b28b56..2322af64 100644 --- a/common_test.go +++ b/common_test.go @@ -15,11 +15,11 @@ import ( "testing" "github.com/btcsuite/btcutil" + "github.com/btcsuite/btcwire" "github.com/conformal/btcdb" _ "github.com/conformal/btcdb/ldb" _ "github.com/conformal/btcdb/memdb" "github.com/conformal/btcnet" - "github.com/conformal/btcwire" ) var ( diff --git a/db.go b/db.go index 57a4a097..9753efbb 100644 --- a/db.go +++ b/db.go @@ -8,7 +8,7 @@ import ( "errors" "github.com/btcsuite/btcutil" - "github.com/conformal/btcwire" + "github.com/btcsuite/btcwire" ) // Errors that the various database functions may return. diff --git a/interface_test.go b/interface_test.go index 5c2663fb..5a365fb5 100644 --- a/interface_test.go +++ b/interface_test.go @@ -9,8 +9,8 @@ import ( "testing" "github.com/btcsuite/btcutil" + "github.com/btcsuite/btcwire" "github.com/conformal/btcdb" - "github.com/conformal/btcwire" "github.com/davecgh/go-spew/spew" ) diff --git a/ldb/block.go b/ldb/block.go index ee0d1970..a5a9d640 100644 --- a/ldb/block.go +++ b/ldb/block.go @@ -9,9 +9,9 @@ import ( "encoding/binary" "github.com/btcsuite/btcutil" + "github.com/btcsuite/btcwire" "github.com/btcsuite/goleveldb/leveldb" "github.com/conformal/btcdb" - "github.com/conformal/btcwire" ) // FetchBlockBySha - return a btcutil Block diff --git a/ldb/boundary_test.go b/ldb/boundary_test.go index f9ff23c6..523267ea 100644 --- a/ldb/boundary_test.go +++ b/ldb/boundary_test.go @@ -8,8 +8,8 @@ import ( "os" "testing" + "github.com/btcsuite/btcwire" "github.com/conformal/btcdb" - "github.com/conformal/btcwire" ) // we need to test for empty databas and make certain it returns proper value diff --git a/ldb/dup_test.go b/ldb/dup_test.go index 84d0452b..b3256602 100644 --- a/ldb/dup_test.go +++ b/ldb/dup_test.go @@ -11,8 +11,8 @@ import ( "testing" "github.com/btcsuite/btcutil" + "github.com/btcsuite/btcwire" "github.com/conformal/btcdb" - "github.com/conformal/btcwire" ) func Test_dupTx(t *testing.T) { diff --git a/ldb/insertremove_test.go b/ldb/insertremove_test.go index c980fdd5..af83ba66 100644 --- a/ldb/insertremove_test.go +++ b/ldb/insertremove_test.go @@ -11,9 +11,9 @@ import ( "testing" "github.com/btcsuite/btcutil" + "github.com/btcsuite/btcwire" "github.com/conformal/btcdb" _ "github.com/conformal/btcdb/ldb" - "github.com/conformal/btcwire" ) var tstBlocks []*btcutil.Block diff --git a/ldb/internal_test.go b/ldb/internal_test.go index a38bd644..8fe64ff1 100644 --- a/ldb/internal_test.go +++ b/ldb/internal_test.go @@ -7,8 +7,8 @@ package ldb import ( "fmt" + "github.com/btcsuite/btcwire" "github.com/conformal/btcdb" - "github.com/conformal/btcwire" ) // FetchSha returns the datablock and pver for the given ShaHash. diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 1b59d274..20892c35 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -13,10 +13,10 @@ import ( "github.com/btcsuite/btclog" "github.com/btcsuite/btcutil" + "github.com/btcsuite/btcwire" "github.com/btcsuite/goleveldb/leveldb" "github.com/btcsuite/goleveldb/leveldb/opt" "github.com/conformal/btcdb" - "github.com/conformal/btcwire" ) const ( diff --git a/ldb/operational_test.go b/ldb/operational_test.go index 4e1659ea..fe8ce143 100644 --- a/ldb/operational_test.go +++ b/ldb/operational_test.go @@ -15,9 +15,9 @@ import ( "testing" "github.com/btcsuite/btcutil" + "github.com/btcsuite/btcwire" "github.com/conformal/btcdb" "github.com/conformal/btcnet" - "github.com/conformal/btcwire" ) var network = btcwire.MainNet diff --git a/ldb/tx.go b/ldb/tx.go index 00bd838d..afe06d57 100644 --- a/ldb/tx.go +++ b/ldb/tx.go @@ -8,9 +8,9 @@ import ( "bytes" "encoding/binary" + "github.com/btcsuite/btcwire" "github.com/btcsuite/goleveldb/leveldb" "github.com/conformal/btcdb" - "github.com/conformal/btcwire" ) type txUpdateObj struct { diff --git a/memdb/memdb.go b/memdb/memdb.go index b339d34b..173c403d 100644 --- a/memdb/memdb.go +++ b/memdb/memdb.go @@ -11,8 +11,8 @@ import ( "sync" "github.com/btcsuite/btcutil" + "github.com/btcsuite/btcwire" "github.com/conformal/btcdb" - "github.com/conformal/btcwire" ) // Errors that the various database functions may return. diff --git a/memdb/memdb_test.go b/memdb/memdb_test.go index af3aa53a..9420b61e 100644 --- a/memdb/memdb_test.go +++ b/memdb/memdb_test.go @@ -9,10 +9,10 @@ import ( "testing" "github.com/btcsuite/btcutil" + "github.com/btcsuite/btcwire" "github.com/conformal/btcdb" "github.com/conformal/btcdb/memdb" "github.com/conformal/btcnet" - "github.com/conformal/btcwire" ) // TestClosed ensure calling the interface functions on a closed database From 596fdf8327a586f36a56f47ba37374283e14cdb8 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Fri, 16 Jan 2015 17:27:16 -0600 Subject: [PATCH 159/163] Update btcnet import paths to new location. --- common_test.go | 2 +- example_test.go | 2 +- ldb/operational_test.go | 2 +- memdb/memdb_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/common_test.go b/common_test.go index 2322af64..24b18f0a 100644 --- a/common_test.go +++ b/common_test.go @@ -14,12 +14,12 @@ import ( "strings" "testing" + "github.com/btcsuite/btcnet" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcwire" "github.com/conformal/btcdb" _ "github.com/conformal/btcdb/ldb" _ "github.com/conformal/btcdb/memdb" - "github.com/conformal/btcnet" ) var ( diff --git a/example_test.go b/example_test.go index e2951713..f2942c17 100644 --- a/example_test.go +++ b/example_test.go @@ -7,10 +7,10 @@ package btcdb_test import ( "fmt" + "github.com/btcsuite/btcnet" "github.com/btcsuite/btcutil" "github.com/conformal/btcdb" _ "github.com/conformal/btcdb/memdb" - "github.com/conformal/btcnet" ) // This example demonstrates creating a new database and inserting the genesis diff --git a/ldb/operational_test.go b/ldb/operational_test.go index fe8ce143..2e47a10c 100644 --- a/ldb/operational_test.go +++ b/ldb/operational_test.go @@ -14,10 +14,10 @@ import ( "strings" "testing" + "github.com/btcsuite/btcnet" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcwire" "github.com/conformal/btcdb" - "github.com/conformal/btcnet" ) var network = btcwire.MainNet diff --git a/memdb/memdb_test.go b/memdb/memdb_test.go index 9420b61e..4879c5b9 100644 --- a/memdb/memdb_test.go +++ b/memdb/memdb_test.go @@ -8,11 +8,11 @@ import ( "reflect" "testing" + "github.com/btcsuite/btcnet" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcwire" "github.com/conformal/btcdb" "github.com/conformal/btcdb/memdb" - "github.com/conformal/btcnet" ) // TestClosed ensure calling the interface functions on a closed database From bad82efd8cff9cbd3b6b48ab12bf106b5b6745af Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Fri, 16 Jan 2015 18:26:00 -0600 Subject: [PATCH 160/163] Update btcdb import paths to new location. --- README.md | 12 ++++++------ common_test.go | 6 +++--- db_test.go | 2 +- example_test.go | 8 ++++---- interface_test.go | 2 +- ldb/block.go | 2 +- ldb/boundary_test.go | 2 +- ldb/dup_test.go | 2 +- ldb/insertremove_test.go | 4 ++-- ldb/internal_test.go | 2 +- ldb/leveldb.go | 2 +- ldb/operational_test.go | 2 +- ldb/tx.go | 2 +- memdb/driver.go | 2 +- memdb/memdb.go | 2 +- memdb/memdb_test.go | 4 ++-- 16 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index f4442229..432097da 100644 --- a/README.md +++ b/README.md @@ -13,30 +13,30 @@ transactions. ## Documentation [![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)] -(http://godoc.org/github.com/conformal/btcdb) +(http://godoc.org/github.com/btcsuite/btcdb) Full `go doc` style documentation for the project can be viewed online without installing this package by using the GoDoc site -[here](http://godoc.org/github.com/conformal/btcdb). +[here](http://godoc.org/github.com/btcsuite/btcdb). You can also view the documentation locally once the package is installed with the `godoc` tool by running `godoc -http=":6060"` and pointing your browser to -http://localhost:6060/pkg/github.com/conformal/btcdb +http://localhost:6060/pkg/github.com/btcsuite/btcdb ## Installation ```bash -$ go get github.com/conformal/btcdb +$ go get github.com/btcsuite/btcdb ``` ## Examples * [CreateDB Example] - (http://godoc.org/github.com/conformal/btcdb#example-CreateDB) + (http://godoc.org/github.com/btcsuite/btcdb#example-CreateDB) Demonstrates creating a new database and inserting the genesis block into it. * [NewestSha Example] - (http://godoc.org/github.com/conformal/btcdb#example-Db--NewestSha) + (http://godoc.org/github.com/btcsuite/btcdb#example-Db--NewestSha) Demonstrates querying the database for the most recent best block height and hash. diff --git a/common_test.go b/common_test.go index 24b18f0a..76762664 100644 --- a/common_test.go +++ b/common_test.go @@ -14,12 +14,12 @@ import ( "strings" "testing" + "github.com/btcsuite/btcdb" + _ "github.com/btcsuite/btcdb/ldb" + _ "github.com/btcsuite/btcdb/memdb" "github.com/btcsuite/btcnet" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcwire" - "github.com/conformal/btcdb" - _ "github.com/conformal/btcdb/ldb" - _ "github.com/conformal/btcdb/memdb" ) var ( diff --git a/db_test.go b/db_test.go index 11baa7e6..d96a7e8a 100644 --- a/db_test.go +++ b/db_test.go @@ -8,7 +8,7 @@ import ( "fmt" "testing" - "github.com/conformal/btcdb" + "github.com/btcsuite/btcdb" ) var ( diff --git a/example_test.go b/example_test.go index f2942c17..f50d7422 100644 --- a/example_test.go +++ b/example_test.go @@ -7,10 +7,10 @@ package btcdb_test import ( "fmt" + "github.com/btcsuite/btcdb" + _ "github.com/btcsuite/btcdb/memdb" "github.com/btcsuite/btcnet" "github.com/btcsuite/btcutil" - "github.com/conformal/btcdb" - _ "github.com/conformal/btcdb/memdb" ) // This example demonstrates creating a new database and inserting the genesis @@ -20,8 +20,8 @@ func ExampleCreateDB() { // Ordinarily this would be whatever driver(s) your application // requires. // import ( - // "github.com/conformal/btcdb" - // _ "github.com/conformal/btcdb/memdb" + // "github.com/btcsuite/btcdb" + // _ "github.com/btcsuite/btcdb/memdb" // ) // Create a database and schedule it to be closed on exit. This example diff --git a/interface_test.go b/interface_test.go index 5a365fb5..8127b971 100644 --- a/interface_test.go +++ b/interface_test.go @@ -8,9 +8,9 @@ import ( "reflect" "testing" + "github.com/btcsuite/btcdb" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcwire" - "github.com/conformal/btcdb" "github.com/davecgh/go-spew/spew" ) diff --git a/ldb/block.go b/ldb/block.go index a5a9d640..c5b4aa23 100644 --- a/ldb/block.go +++ b/ldb/block.go @@ -8,10 +8,10 @@ import ( "bytes" "encoding/binary" + "github.com/btcsuite/btcdb" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcwire" "github.com/btcsuite/goleveldb/leveldb" - "github.com/conformal/btcdb" ) // FetchBlockBySha - return a btcutil Block diff --git a/ldb/boundary_test.go b/ldb/boundary_test.go index 523267ea..d3f10893 100644 --- a/ldb/boundary_test.go +++ b/ldb/boundary_test.go @@ -8,8 +8,8 @@ import ( "os" "testing" + "github.com/btcsuite/btcdb" "github.com/btcsuite/btcwire" - "github.com/conformal/btcdb" ) // we need to test for empty databas and make certain it returns proper value diff --git a/ldb/dup_test.go b/ldb/dup_test.go index b3256602..10557203 100644 --- a/ldb/dup_test.go +++ b/ldb/dup_test.go @@ -10,9 +10,9 @@ import ( "path/filepath" "testing" + "github.com/btcsuite/btcdb" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcwire" - "github.com/conformal/btcdb" ) func Test_dupTx(t *testing.T) { diff --git a/ldb/insertremove_test.go b/ldb/insertremove_test.go index af83ba66..dfa68b92 100644 --- a/ldb/insertremove_test.go +++ b/ldb/insertremove_test.go @@ -10,10 +10,10 @@ import ( "path/filepath" "testing" + "github.com/btcsuite/btcdb" + _ "github.com/btcsuite/btcdb/ldb" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcwire" - "github.com/conformal/btcdb" - _ "github.com/conformal/btcdb/ldb" ) var tstBlocks []*btcutil.Block diff --git a/ldb/internal_test.go b/ldb/internal_test.go index 8fe64ff1..f0cf74bc 100644 --- a/ldb/internal_test.go +++ b/ldb/internal_test.go @@ -7,8 +7,8 @@ package ldb import ( "fmt" + "github.com/btcsuite/btcdb" "github.com/btcsuite/btcwire" - "github.com/conformal/btcdb" ) // FetchSha returns the datablock and pver for the given ShaHash. diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 20892c35..64b72e57 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -11,12 +11,12 @@ import ( "strconv" "sync" + "github.com/btcsuite/btcdb" "github.com/btcsuite/btclog" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcwire" "github.com/btcsuite/goleveldb/leveldb" "github.com/btcsuite/goleveldb/leveldb/opt" - "github.com/conformal/btcdb" ) const ( diff --git a/ldb/operational_test.go b/ldb/operational_test.go index 2e47a10c..bee33101 100644 --- a/ldb/operational_test.go +++ b/ldb/operational_test.go @@ -14,10 +14,10 @@ import ( "strings" "testing" + "github.com/btcsuite/btcdb" "github.com/btcsuite/btcnet" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcwire" - "github.com/conformal/btcdb" ) var network = btcwire.MainNet diff --git a/ldb/tx.go b/ldb/tx.go index afe06d57..8d28fed6 100644 --- a/ldb/tx.go +++ b/ldb/tx.go @@ -8,9 +8,9 @@ import ( "bytes" "encoding/binary" + "github.com/btcsuite/btcdb" "github.com/btcsuite/btcwire" "github.com/btcsuite/goleveldb/leveldb" - "github.com/conformal/btcdb" ) type txUpdateObj struct { diff --git a/memdb/driver.go b/memdb/driver.go index bbd31d66..cf4dddd6 100644 --- a/memdb/driver.go +++ b/memdb/driver.go @@ -7,8 +7,8 @@ package memdb import ( "fmt" + "github.com/btcsuite/btcdb" "github.com/btcsuite/btclog" - "github.com/conformal/btcdb" ) var log = btclog.Disabled diff --git a/memdb/memdb.go b/memdb/memdb.go index 173c403d..9294309d 100644 --- a/memdb/memdb.go +++ b/memdb/memdb.go @@ -10,9 +10,9 @@ import ( "math" "sync" + "github.com/btcsuite/btcdb" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcwire" - "github.com/conformal/btcdb" ) // Errors that the various database functions may return. diff --git a/memdb/memdb_test.go b/memdb/memdb_test.go index 4879c5b9..de8701b3 100644 --- a/memdb/memdb_test.go +++ b/memdb/memdb_test.go @@ -8,11 +8,11 @@ import ( "reflect" "testing" + "github.com/btcsuite/btcdb" + "github.com/btcsuite/btcdb/memdb" "github.com/btcsuite/btcnet" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcwire" - "github.com/conformal/btcdb" - "github.com/conformal/btcdb/memdb" ) // TestClosed ensure calling the interface functions on a closed database From c67a3059dfdbe72c647920cfed55ba873a2a52b4 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Fri, 16 Jan 2015 23:21:15 -0600 Subject: [PATCH 161/163] Update to new location in README.md too. --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 432097da..90f27b1a 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ btcdb ===== -[![Build Status](http://img.shields.io/travis/conformal/btcdb.svg)] -(https://travis-ci.org/conformal/btcdb) [![Coverage Status] -(https://img.shields.io/coveralls/conformal/btcdb.svg)] -(https://coveralls.io/r/conformal/btcdb) [![ISC License] +[![Build Status](http://img.shields.io/travis/btcsuite/btcdb.svg)] +(https://travis-ci.org/btcsuite/btcdb) [![Coverage Status] +(https://img.shields.io/coveralls/btcsuite/btcdb.svg)] +(https://coveralls.io/r/btcsuite/btcdb) [![ISC License] (http://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org) Package btcdb provides a database interface for the bitcoin block chain and From b284bf0f909aaea8fa8f6790200a09a756ee066c Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Wed, 24 Dec 2014 17:55:14 -0600 Subject: [PATCH 162/163] Implement ldb database functionality for optional addrindex. --- db.go | 41 +++++ db_test.go | 2 +- ldb/block.go | 38 +++++ ldb/boundary_test.go | 3 +- ldb/dup_test.go | 2 +- ldb/insertremove_test.go | 2 +- ldb/internal_test.go | 65 +++++-- ldb/leveldb.go | 13 +- ldb/operational_test.go | 360 ++++++++++++++++++++++++++++++++------- ldb/tx.go | 237 +++++++++++++++++++++++++- memdb/memdb.go | 25 +++ 11 files changed, 707 insertions(+), 81 deletions(-) diff --git a/db.go b/db.go index 9753efbb..93e5291f 100644 --- a/db.go +++ b/db.go @@ -9,16 +9,21 @@ import ( "github.com/btcsuite/btcutil" "github.com/btcsuite/btcwire" + "golang.org/x/crypto/ripemd160" ) // Errors that the various database functions may return. var ( + ErrAddrIndexDoesNotExist = errors.New("address index hasn't been built up yet") + ErrUnsupportedAddressType = errors.New("address type is not supported " + + "by the address-index") ErrPrevShaMissing = errors.New("previous sha missing from database") ErrTxShaMissing = errors.New("requested transaction does not exist") ErrBlockShaMissing = errors.New("requested block does not exist") ErrDuplicateSha = errors.New("duplicate insert attempted") ErrDbDoesNotExist = errors.New("non-existent database") ErrDbUnknownType = errors.New("non-existent database type") + ErrNotImplemented = errors.New("method has not yet been implemented") ) // AllShas is a special value that can be used as the final sha when requesting @@ -106,6 +111,34 @@ type Db interface { // the database yet. NewestSha() (sha *btcwire.ShaHash, height int64, err error) + // FetchAddrIndexTip returns the hash and block height of the most recent + // block which has had its address index populated. It will return + // ErrAddrIndexDoesNotExist along with a zero hash, and -1 if the + // addrindex hasn't yet been built up. + FetchAddrIndexTip() (sha *btcwire.ShaHash, height int64, err error) + + // UpdateAddrIndexForBlock updates the stored addrindex with passed + // index information for a particular block height. Additionally, it + // will update the stored meta-data related to the curent tip of the + // addr index. These two operations are performed in an atomic + // transaction which is commited before the function returns. + // Addresses are indexed by the raw bytes of their base58 decoded + // hash160. + UpdateAddrIndexForBlock(blkSha *btcwire.ShaHash, height int64, + addrIndex BlockAddrIndex) error + + // FetchTxsForAddr looks up and returns all transactions which either + // spend a previously created output of the passed address, or create + // a new output locked to the passed address. The, `limit` parameter + // should be the max number of transactions to be returned. + // Additionally, if the caller wishes to skip forward in the results + // some amount, the 'seek' represents how many results to skip. + // NOTE: Values for both `seek` and `limit` MUST be positive. + FetchTxsForAddr(addr btcutil.Address, skip int, limit int) ([]*TxListReply, error) + + // DeleteAddrIndex deletes the entire addrindex stored within the DB. + DeleteAddrIndex() error + // RollbackClose discards the recent database changes to the previously // saved data at last Sync and closes the database. RollbackClose() (err error) @@ -134,6 +167,14 @@ type TxListReply struct { Err error } +// AddrIndexKeySize is the number of bytes used by keys into the BlockAddrIndex. +const AddrIndexKeySize = ripemd160.Size + +// BlockAddrIndex represents the indexing structure for addresses. +// It maps a hash160 to a list of transaction locations within a block that +// either pays to or spends from the passed UTXO for the hash160. +type BlockAddrIndex map[[AddrIndexKeySize]byte][]*btcwire.TxLoc + // driverList holds all of the registered database backends. var driverList []DriverDB diff --git a/db_test.go b/db_test.go index d96a7e8a..b14804cd 100644 --- a/db_test.go +++ b/db_test.go @@ -19,7 +19,7 @@ var ( ignoreDbTypes = map[string]bool{"createopenfail": true} ) -// testNewestShaEmpty ensures the NewestSha returns the values expected by +// testNewestShaEmpty ensures that NewestSha returns the values expected by // the interface contract. func testNewestShaEmpty(t *testing.T, db btcdb.Db) { sha, height, err := db.NewestSha() diff --git a/ldb/block.go b/ldb/block.go index c5b4aa23..3e6c84f2 100644 --- a/ldb/block.go +++ b/ldb/block.go @@ -290,3 +290,41 @@ func (db *LevelDb) NewestSha() (rsha *btcwire.ShaHash, rblkid int64, err error) return &sha, db.lastBlkIdx, nil } + +// fetchAddrIndexTip returns the last block height and block sha to be indexed. +// Meta-data about the address tip is currently cached in memory, and will be +// updated accordingly by functions that modify the state. This function is +// used on start up to load the info into memory. Callers will use the public +// version of this function below, which returns our cached copy. +func (db *LevelDb) fetchAddrIndexTip() (*btcwire.ShaHash, int64, error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + data, err := db.lDb.Get(addrIndexMetaDataKey, db.ro) + if err != nil { + return &btcwire.ShaHash{}, -1, btcdb.ErrAddrIndexDoesNotExist + } + + var blkSha btcwire.ShaHash + blkSha.SetBytes(data[0:32]) + + blkHeight := binary.LittleEndian.Uint64(data[32:]) + + return &blkSha, int64(blkHeight), nil +} + +// FetchAddrIndexTip returns the hash and block height of the most recent +// block whose transactions have been indexed by address. It will return +// ErrAddrIndexDoesNotExist along with a zero hash, and -1 if the +// addrindex hasn't yet been built up. +func (db *LevelDb) FetchAddrIndexTip() (*btcwire.ShaHash, int64, error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + if db.lastAddrIndexBlkIdx == -1 { + return &btcwire.ShaHash{}, -1, btcdb.ErrAddrIndexDoesNotExist + } + sha := db.lastAddrIndexBlkSha + + return &sha, db.lastAddrIndexBlkIdx, nil +} diff --git a/ldb/boundary_test.go b/ldb/boundary_test.go index d3f10893..6d963d96 100644 --- a/ldb/boundary_test.go +++ b/ldb/boundary_test.go @@ -12,7 +12,8 @@ import ( "github.com/btcsuite/btcwire" ) -// we need to test for empty databas and make certain it returns proper value +// we need to test for an empty database and make certain it returns the proper +// values func TestEmptyDB(t *testing.T) { diff --git a/ldb/dup_test.go b/ldb/dup_test.go index 10557203..302aa05a 100644 --- a/ldb/dup_test.go +++ b/ldb/dup_test.go @@ -110,7 +110,7 @@ out: lastSha = blkSha } - // genrate a new block based on the last sha + // generate a new block based on the last sha // these block are not verified, so there are a bunch of garbage fields // in the 'generated' block. diff --git a/ldb/insertremove_test.go b/ldb/insertremove_test.go index dfa68b92..dd8ab524 100644 --- a/ldb/insertremove_test.go +++ b/ldb/insertremove_test.go @@ -66,7 +66,7 @@ endtest: for height := int64(0); height < int64(len(blocks)); height++ { block := blocks[height] - // look up inputs to this x + // look up inputs to this tx mblock := block.MsgBlock() var txneededList []*btcwire.ShaHash var txlookupList []*btcwire.ShaHash diff --git a/ldb/internal_test.go b/ldb/internal_test.go index f0cf74bc..d0fc00bd 100644 --- a/ldb/internal_test.go +++ b/ldb/internal_test.go @@ -5,21 +5,60 @@ package ldb import ( - "fmt" + "bytes" - "github.com/btcsuite/btcdb" - "github.com/btcsuite/btcwire" + "testing" + + "github.com/btcsuite/btcutil" + "golang.org/x/crypto/ripemd160" ) -// FetchSha returns the datablock and pver for the given ShaHash. -// This is a testing only interface. -func FetchSha(db btcdb.Db, sha *btcwire.ShaHash) (buf []byte, pver uint32, - blkid int64, err error) { - sqldb, ok := db.(*LevelDb) - if !ok { - err = fmt.Errorf("invalid data type") - return +func TestAddrIndexKeySerialization(t *testing.T) { + var hash160Bytes [ripemd160.Size]byte + + fakeHash160 := btcutil.Hash160([]byte("testing")) + copy(fakeHash160, hash160Bytes[:]) + + fakeIndex := txAddrIndex{ + hash160: hash160Bytes, + blkHeight: 1, + txoffset: 5, + txlen: 360, + } + + serializedKey := addrIndexToKey(&fakeIndex) + unpackedIndex := unpackTxIndex(serializedKey[22:]) + + if unpackedIndex.blkHeight != fakeIndex.blkHeight { + t.Errorf("Incorrect block height. Unpack addr index key"+ + "serialization failed. Expected %d, received %d", + 1, unpackedIndex.blkHeight) + } + + if unpackedIndex.txoffset != fakeIndex.txoffset { + t.Errorf("Incorrect tx offset. Unpack addr index key"+ + "serialization failed. Expected %d, received %d", + 5, unpackedIndex.txoffset) + } + + if unpackedIndex.txlen != fakeIndex.txlen { + t.Errorf("Incorrect tx len. Unpack addr index key"+ + "serialization failed. Expected %d, received %d", + 360, unpackedIndex.txlen) + } +} + +func TestBytesPrefix(t *testing.T) { + testKey := []byte("a") + + prefixRange := bytesPrefix(testKey) + if !bytes.Equal(prefixRange.Start, []byte("a")) { + t.Errorf("Wrong prefix start, got %d, expected %d", prefixRange.Start, + []byte("a")) + } + + if !bytes.Equal(prefixRange.Limit, []byte("b")) { + t.Errorf("Wrong prefix end, got %d, expected %d", prefixRange.Limit, + []byte("b")) } - buf, blkid, err = sqldb.fetchSha(sha) - return } diff --git a/ldb/leveldb.go b/ldb/leveldb.go index 64b72e57..cc946fbb 100644 --- a/ldb/leveldb.go +++ b/ldb/leveldb.go @@ -53,6 +53,9 @@ type LevelDb struct { lastBlkSha btcwire.ShaHash lastBlkIdx int64 + lastAddrIndexBlkSha btcwire.ShaHash + lastAddrIndexBlkIdx int64 + txUpdateMap map[btcwire.ShaHash]*txUpdateObj txSpentUpdateMap map[btcwire.ShaHash]*spentTxUpdate } @@ -92,7 +95,6 @@ func OpenDB(args ...interface{}) (btcdb.Db, error) { } // Need to find last block and tx - var lastknownblock, nextunknownblock, testblock int64 increment := int64(100000) @@ -139,6 +141,14 @@ blocknarrow: } } + // Load the last block whose transactions have been indexed by address. + if sha, idx, err := ldb.fetchAddrIndexTip(); err == nil { + ldb.lastAddrIndexBlkSha = *sha + ldb.lastAddrIndexBlkIdx = idx + } else { + ldb.lastAddrIndexBlkIdx = -1 + } + ldb.lastBlkSha = *lastSha ldb.lastBlkIdx = lastknownblock ldb.nextBlock = lastknownblock + 1 @@ -250,6 +260,7 @@ func CreateDB(args ...interface{}) (btcdb.Db, error) { if err == nil { ldb := db.(*LevelDb) ldb.lastBlkIdx = -1 + ldb.lastAddrIndexBlkIdx = -1 ldb.nextBlock = 0 } return db, err diff --git a/ldb/operational_test.go b/ldb/operational_test.go index bee33101..6cb16422 100644 --- a/ldb/operational_test.go +++ b/ldb/operational_test.go @@ -5,6 +5,7 @@ package ldb_test import ( + "bytes" "compress/bzip2" "encoding/binary" "fmt" @@ -16,22 +17,26 @@ import ( "github.com/btcsuite/btcdb" "github.com/btcsuite/btcnet" + "github.com/btcsuite/btcscript" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcwire" + "golang.org/x/crypto/ripemd160" ) var network = btcwire.MainNet -func TestOperational(t *testing.T) { - testOperationalMode(t) +// testDb is used to store db related context for a running test. +// the `cleanUpFunc` *must* be called after each test to maintain db +// consistency across tests. +type testDb struct { + db btcdb.Db + blocks []*btcutil.Block + dbName string + dbNameVer string + cleanUpFunc func() } -func testOperationalMode(t *testing.T) { - // simplified basic operation is: - // 1) fetch block from remote server - // 2) look up all txin (except coinbase in db) - // 3) insert block - +func setUpTestDb(t *testing.T) (*testDb, error) { // Ignore db remove errors since it means we didn't have an old one. dbname := fmt.Sprintf("tstdbop1") dbnamever := dbname + ".ver" @@ -39,28 +44,181 @@ func testOperationalMode(t *testing.T) { _ = os.RemoveAll(dbnamever) db, err := btcdb.CreateDB("leveldb", dbname) if err != nil { - t.Errorf("Failed to open test database %v", err) - return + return nil, err } - defer os.RemoveAll(dbname) - defer os.RemoveAll(dbnamever) - defer func() { - if err := db.Close(); err != nil { - t.Errorf("Close: unexpected error: %v", err) - } - }() testdatafile := filepath.Join("..", "testdata", "blocks1-256.bz2") blocks, err := loadBlocks(t, testdatafile) + if err != nil { + return nil, err + } + + cleanUp := func() { + db.Close() + os.RemoveAll(dbname) + os.RemoveAll(dbnamever) + } + + return &testDb{ + db: db, + blocks: blocks, + dbName: dbname, + dbNameVer: dbnamever, + cleanUpFunc: cleanUp, + }, nil +} + +func TestOperational(t *testing.T) { + testOperationalMode(t) +} + +// testAddrIndexOperations ensures that all normal operations concerning +// the optional address index function correctly. +func testAddrIndexOperations(t *testing.T, db btcdb.Db, newestBlock *btcutil.Block, newestSha *btcwire.ShaHash, newestBlockIdx int64) { + // Metadata about the current addr index state should be unset. + sha, height, err := db.FetchAddrIndexTip() + if err != btcdb.ErrAddrIndexDoesNotExist { + t.Fatalf("Address index metadata shouldn't be in db, hasn't been built up yet.") + } + + var zeroHash btcwire.ShaHash + if !sha.IsEqual(&zeroHash) { + t.Fatalf("AddrIndexTip wrong hash got: %s, want %s", sha, &zeroHash) + + } + + if height != -1 { + t.Fatalf("Addrindex not built up, yet a block index tip has been set to: %d.", height) + } + + // Test enforcement of constraints for "limit" and "skip" + var fakeAddr btcutil.Address + _, err = db.FetchTxsForAddr(fakeAddr, -1, 0) + if err == nil { + t.Fatalf("Negative value for skip passed, should return an error") + } + + _, err = db.FetchTxsForAddr(fakeAddr, 0, -1) + if err == nil { + t.Fatalf("Negative value for limit passed, should return an error") + } + + // Simple test to index outputs(s) of the first tx. + testIndex := make(btcdb.BlockAddrIndex) + testTx, err := newestBlock.Tx(0) + if err != nil { + t.Fatalf("Block has no transactions, unable to test addr "+ + "indexing, err %v", err) + } + + // Extract the dest addr from the tx. + _, testAddrs, _, err := btcscript.ExtractPkScriptAddrs(testTx.MsgTx().TxOut[0].PkScript, &btcnet.MainNetParams) + if err != nil { + t.Fatalf("Unable to decode tx output, err %v", err) + } + + // Extract the hash160 from the output script. + var hash160Bytes [ripemd160.Size]byte + testHash160 := testAddrs[0].(*btcutil.AddressPubKey).AddressPubKeyHash().ScriptAddress() + copy(hash160Bytes[:], testHash160[:]) + + // Create a fake index. + blktxLoc, _ := newestBlock.TxLoc() + testIndex[hash160Bytes] = []*btcwire.TxLoc{&blktxLoc[0]} + + // Insert our test addr index into the DB. + err = db.UpdateAddrIndexForBlock(newestSha, newestBlockIdx, testIndex) + if err != nil { + t.Fatalf("UpdateAddrIndexForBlock: failed to index"+ + " addrs for block #%d (%s) "+ + "err %v", newestBlockIdx, newestSha, err) + } + + // Chain Tip of address should've been updated. + assertAddrIndexTipIsUpdated(db, t, newestSha, newestBlockIdx) + + // Check index retrieval. + txReplies, err := db.FetchTxsForAddr(testAddrs[0], 0, 1000) + if err != nil { + t.Fatalf("FetchTxsForAddr failed to correctly fetch txs for an "+ + "address, err %v", err) + } + // Should have one reply. + if len(txReplies) != 1 { + t.Fatalf("Failed to properly index tx by address.") + } + + // Our test tx and indexed tx should have the same sha. + indexedTx := txReplies[0] + if !bytes.Equal(indexedTx.Sha.Bytes(), testTx.Sha().Bytes()) { + t.Fatalf("Failed to fetch proper indexed tx. Expected sha %v, "+ + "fetched %v", testTx.Sha(), indexedTx.Sha) + } + + // Shut down DB. + db.Sync() + db.Close() + + // Re-Open, tip still should be updated to current height and sha. + db, err = btcdb.OpenDB("leveldb", "tstdbop1") + if err != nil { + t.Fatalf("Unable to re-open created db, err %v", err) + } + assertAddrIndexTipIsUpdated(db, t, newestSha, newestBlockIdx) + + // Delete the entire index. + err = db.DeleteAddrIndex() + if err != nil { + t.Fatalf("Couldn't delete address index, err %v", err) + } + + // Former index should no longer exist. + txReplies, err = db.FetchTxsForAddr(testAddrs[0], 0, 1000) + if err != nil { + t.Fatalf("Unable to fetch transactions for address: %v", err) + } + if len(txReplies) != 0 { + t.Fatalf("Address index was not successfully deleted. "+ + "Should have 0 tx's indexed, %v were returned.", + len(txReplies)) + } + + // Tip should be blanked out. + if _, _, err := db.FetchAddrIndexTip(); err != btcdb.ErrAddrIndexDoesNotExist { + t.Fatalf("Address index was not fully deleted.") + } + +} + +func assertAddrIndexTipIsUpdated(db btcdb.Db, t *testing.T, newestSha *btcwire.ShaHash, newestBlockIdx int64) { + // Safe to ignore error, since height will be < 0 in "error" case. + sha, height, _ := db.FetchAddrIndexTip() + if newestBlockIdx != height { + t.Fatalf("Height of address index tip failed to update, "+ + "expected %v, got %v", newestBlockIdx, height) + } + if !bytes.Equal(newestSha.Bytes(), sha.Bytes()) { + t.Fatalf("Sha of address index tip failed to update, "+ + "expected %v, got %v", newestSha, sha) + } +} + +func testOperationalMode(t *testing.T) { + // simplified basic operation is: + // 1) fetch block from remote server + // 2) look up all txin (except coinbase in db) + // 3) insert block + // 4) exercise the optional addridex + testDb, err := setUpTestDb(t) + defer testDb.cleanUpFunc() if err != nil { t.Errorf("Unable to load blocks from test data: %v", err) return } - err = nil out: - for height := int64(0); height < int64(len(blocks)); height++ { - block := blocks[height] + for height := int64(0); height < int64(len(testDb.blocks)); height++ { + block := testDb.blocks[height] mblock := block.MsgBlock() var txneededList []*btcwire.ShaHash for _, tx := range mblock.Transactions { @@ -71,7 +229,7 @@ out: origintxsha := &txin.PreviousOutPoint.Hash txneededList = append(txneededList, origintxsha) - exists, err := db.ExistsTxSha(origintxsha) + exists, err := testDb.db.ExistsTxSha(origintxsha) if err != nil { t.Errorf("ExistsTxSha: unexpected error %v ", err) } @@ -79,13 +237,13 @@ out: t.Errorf("referenced tx not found %v ", origintxsha) } - _, err = db.FetchTxBySha(origintxsha) + _, err = testDb.db.FetchTxBySha(origintxsha) if err != nil { t.Errorf("referenced tx not found %v err %v ", origintxsha, err) } } } - txlist := db.FetchUnSpentTxByShaList(txneededList) + txlist := testDb.db.FetchUnSpentTxByShaList(txneededList) for _, txe := range txlist { if txe.Err != nil { t.Errorf("tx list fetch failed %v err %v ", txe.Sha, txe.Err) @@ -93,7 +251,7 @@ out: } } - newheight, err := db.InsertBlock(block) + newheight, err := testDb.db.InsertBlock(block) if err != nil { t.Errorf("failed to insert block %v err %v", height, err) break out @@ -103,13 +261,13 @@ out: break out } - newSha, blkid, err := db.NewestSha() + newSha, blkid, err := testDb.db.NewestSha() if err != nil { t.Errorf("failed to obtain latest sha %v %v", height, err) } if blkid != height { - t.Errorf("height doe not match latest block height %v %v %v", blkid, height, err) + t.Errorf("height does not match latest block height %v %v %v", blkid, height, err) } blkSha, _ := block.Sha() @@ -118,8 +276,13 @@ out: } } - // now that db is populated, do some additional test - testFetchRangeHeight(t, db, blocks) + // now that the db is populated, do some additional tests + testFetchHeightRange(t, testDb.db, testDb.blocks) + + // Ensure all operations dealing with the optional address index behave + // correctly. + newSha, blkid, err := testDb.db.NewestSha() + testAddrIndexOperations(t, testDb.db, testDb.blocks[len(testDb.blocks)-1], newSha, blkid) } func TestBackout(t *testing.T) { @@ -132,43 +295,35 @@ func testBackout(t *testing.T) { // 2) look up all txin (except coinbase in db) // 3) insert block - // Ignore db remove errors since it means we didn't have an old one. - dbname := fmt.Sprintf("tstdbop2") - dbnamever := dbname + ".ver" - _ = os.RemoveAll(dbname) - _ = os.RemoveAll(dbnamever) - db, err := btcdb.CreateDB("leveldb", dbname) + testDb, err := setUpTestDb(t) + defer testDb.cleanUpFunc() + if err != nil { t.Errorf("Failed to open test database %v", err) return } - defer os.RemoveAll(dbname) - defer os.RemoveAll(dbnamever) - defer db.Close() - testdatafile := filepath.Join("..", "testdata", "blocks1-256.bz2") - blocks, err := loadBlocks(t, testdatafile) - if len(blocks) < 120 { + if len(testDb.blocks) < 120 { t.Errorf("test data too small") return } err = nil - for height := int64(0); height < int64(len(blocks)); height++ { + for height := int64(0); height < int64(len(testDb.blocks)); height++ { if height == 100 { t.Logf("Syncing at block height 100") - db.Sync() + testDb.db.Sync() } if height == 120 { t.Logf("Simulating unexpected application quit") // Simulate unexpected application quit - db.RollbackClose() + testDb.db.RollbackClose() break } - block := blocks[height] + block := testDb.blocks[height] - newheight, err := db.InsertBlock(block) + newheight, err := testDb.db.InsertBlock(block) if err != nil { t.Errorf("failed to insert block %v err %v", height, err) return @@ -182,49 +337,49 @@ func testBackout(t *testing.T) { // db was closed at height 120, so no cleanup is possible. // reopen db - db, err = btcdb.OpenDB("leveldb", dbname) + testDb.db, err = btcdb.OpenDB("leveldb", testDb.dbName) if err != nil { t.Errorf("Failed to open test database %v", err) return } defer func() { - if err := db.Close(); err != nil { + if err := testDb.db.Close(); err != nil { t.Errorf("Close: unexpected error: %v", err) } }() - sha, err := blocks[99].Sha() + sha, err := testDb.blocks[99].Sha() if err != nil { t.Errorf("failed to get block 99 sha err %v", err) return } - if _, err := db.ExistsSha(sha); err != nil { + if _, err := testDb.db.ExistsSha(sha); err != nil { t.Errorf("ExistsSha: unexpected error: %v", err) } - _, err = db.FetchBlockBySha(sha) + _, err = testDb.db.FetchBlockBySha(sha) if err != nil { t.Errorf("failed to load block 99 from db %v", err) return } - sha, err = blocks[119].Sha() + sha, err = testDb.blocks[119].Sha() if err != nil { t.Errorf("failed to get block 110 sha err %v", err) return } - if _, err := db.ExistsSha(sha); err != nil { + if _, err := testDb.db.ExistsSha(sha); err != nil { t.Errorf("ExistsSha: unexpected error: %v", err) } - _, err = db.FetchBlockBySha(sha) + _, err = testDb.db.FetchBlockBySha(sha) if err != nil { t.Errorf("loaded block 119 from db") return } - block := blocks[119] + block := testDb.blocks[119] mblock := block.MsgBlock() txsha, err := mblock.Transactions[0].TxSha() - exists, err := db.ExistsTxSha(&txsha) + exists, err := testDb.db.ExistsTxSha(&txsha) if err != nil { t.Errorf("ExistsTxSha: unexpected error %v ", err) } @@ -232,7 +387,7 @@ func testBackout(t *testing.T) { t.Errorf("tx %v not located db\n", txsha) } - _, err = db.FetchTxBySha(&txsha) + _, err = testDb.db.FetchTxBySha(&txsha) if err != nil { t.Errorf("tx %v not located db\n", txsha) return @@ -310,7 +465,7 @@ func loadBlocks(t *testing.T, file string) (blocks []*btcutil.Block, err error) return } -func testFetchRangeHeight(t *testing.T, db btcdb.Db, blocks []*btcutil.Block) { +func testFetchHeightRange(t *testing.T, db btcdb.Db, blocks []*btcutil.Block) { var testincrement int64 = 50 var testcnt int64 = 100 @@ -322,7 +477,7 @@ func testFetchRangeHeight(t *testing.T, db btcdb.Db, blocks []*btcutil.Block) { for i := range blocks { blockSha, err := blocks[i].Sha() if err != nil { - t.Errorf("FetchRangeHeight: unexpected failure computing block sah %v", err) + t.Errorf("FetchHeightRange: unexpected failure computing block sah %v", err) } shanames[i] = blockSha } @@ -336,16 +491,16 @@ func testFetchRangeHeight(t *testing.T, db btcdb.Db, blocks []*btcutil.Block) { shalist, err := db.FetchHeightRange(startheight, endheight) if err != nil { - t.Errorf("FetchRangeHeight: unexpected failure looking up shas %v", err) + t.Errorf("FetchHeightRange: unexpected failure looking up shas %v", err) } if endheight == btcdb.AllShas { if int64(len(shalist)) != nBlocks-startheight { - t.Errorf("FetchRangeHeight: expected A %v shas, got %v", nBlocks-startheight, len(shalist)) + t.Errorf("FetchHeightRange: expected A %v shas, got %v", nBlocks-startheight, len(shalist)) } } else { if int64(len(shalist)) != testcnt { - t.Errorf("FetchRangeHeight: expected %v shas, got %v", testcnt, len(shalist)) + t.Errorf("FetchHeightRange: expected %v shas, got %v", testcnt, len(shalist)) } } @@ -353,9 +508,90 @@ func testFetchRangeHeight(t *testing.T, db btcdb.Db, blocks []*btcutil.Block) { sha0 := *shanames[int64(i)+startheight] sha1 := shalist[i] if sha0 != sha1 { - t.Errorf("FetchRangeHeight: mismatch sha at %v requested range %v %v: %v %v ", int64(i)+startheight, startheight, endheight, sha0, sha1) + t.Errorf("FetchHeightRange: mismatch sha at %v requested range %v %v: %v %v ", int64(i)+startheight, startheight, endheight, sha0, sha1) } } } } + +func TestLimitAndSkipFetchTxsForAddr(t *testing.T) { + testDb, err := setUpTestDb(t) + defer testDb.cleanUpFunc() + + // Insert a block with some fake test transactions. The block will have + // 10 copies of a fake transaction involving same address. + addrString := "1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa" + targetAddr, err := btcutil.DecodeAddress(addrString, &btcnet.MainNetParams) + if err != nil { + t.Fatalf("Unable to decode test address: %v", err) + } + outputScript, err := btcscript.PayToAddrScript(targetAddr) + if err != nil { + t.Fatalf("Unable make test pkScript %v", err) + } + fakeTxOut := btcwire.NewTxOut(10, outputScript) + var emptyHash btcwire.ShaHash + fakeHeader := btcwire.NewBlockHeader(&emptyHash, &emptyHash, 1, 1) + msgBlock := btcwire.NewMsgBlock(fakeHeader) + for i := 0; i < 10; i++ { + mtx := btcwire.NewMsgTx() + mtx.AddTxOut(fakeTxOut) + msgBlock.AddTransaction(mtx) + } + + // Insert the test block into the DB. + testBlock := btcutil.NewBlock(msgBlock) + newheight, err := testDb.db.InsertBlock(testBlock) + if err != nil { + t.Fatalf("Unable to insert block into db: %v", err) + } + + // Create and insert an address index for out test addr. + txLoc, _ := testBlock.TxLoc() + index := make(btcdb.BlockAddrIndex) + for i := range testBlock.Transactions() { + var hash160 [ripemd160.Size]byte + scriptAddr := targetAddr.ScriptAddress() + copy(hash160[:], scriptAddr[:]) + index[hash160] = append(index[hash160], &txLoc[i]) + } + blkSha, _ := testBlock.Sha() + err = testDb.db.UpdateAddrIndexForBlock(blkSha, newheight, index) + if err != nil { + t.Fatalf("UpdateAddrIndexForBlock: failed to index"+ + " addrs for block #%d (%s) "+ + "err %v", newheight, blkSha, err) + return + } + + // Try skipping the first 4 results, should get 6 in return. + txReply, err := testDb.db.FetchTxsForAddr(targetAddr, 4, 100000) + if err != nil { + t.Fatalf("Unable to fetch transactions for address: %v", err) + } + if len(txReply) != 6 { + t.Fatalf("Did not correctly skip forward in txs for address reply"+ + " got %v txs, expected %v", len(txReply), 6) + } + + // Limit the number of results to 3. + txReply, err = testDb.db.FetchTxsForAddr(targetAddr, 0, 3) + if err != nil { + t.Fatalf("Unable to fetch transactions for address: %v", err) + } + if len(txReply) != 3 { + t.Fatalf("Did not correctly limit in txs for address reply"+ + " got %v txs, expected %v", len(txReply), 3) + } + + // Skip 1, limit 5. + txReply, err = testDb.db.FetchTxsForAddr(targetAddr, 1, 5) + if err != nil { + t.Fatalf("Unable to fetch transactions for address: %v", err) + } + if len(txReply) != 5 { + t.Fatalf("Did not correctly limit in txs for address reply"+ + " got %v txs, expected %v", len(txReply), 5) + } +} diff --git a/ldb/tx.go b/ldb/tx.go index 8d28fed6..aff215a2 100644 --- a/ldb/tx.go +++ b/ldb/tx.go @@ -7,12 +7,35 @@ package ldb import ( "bytes" "encoding/binary" + "errors" "github.com/btcsuite/btcdb" + "github.com/btcsuite/btcutil" "github.com/btcsuite/btcwire" "github.com/btcsuite/goleveldb/leveldb" + "github.com/btcsuite/goleveldb/leveldb/util" + + "golang.org/x/crypto/ripemd160" ) +const ( + // Each address index is 34 bytes: + // -------------------------------------------------------- + // | Prefix | Hash160 | BlkHeight | Tx Offset | Tx Size | + // -------------------------------------------------------- + // | 2 bytes | 20 bytes | 4 bytes | 4 bytes | 4 bytes | + // -------------------------------------------------------- + addrIndexKeyLength = 2 + ripemd160.Size + 4 + 4 + 4 + + batchDeleteThreshold = 10000 +) + +var addrIndexMetaDataKey = []byte("addrindex") + +// All address index entries share this prefix to facilitate the use of +// iterators. +var addrIndexKeyPrefix = []byte("a-") + type txUpdateObj struct { txSha *btcwire.ShaHash blkHeight int64 @@ -35,6 +58,13 @@ type spentTxUpdate struct { delete bool } +type txAddrIndex struct { + hash160 [ripemd160.Size]byte + blkHeight int64 + txoffset int + txlen int +} + // InsertTx inserts a tx hash and its associated data into the database. func (db *LevelDb) InsertTx(txsha *btcwire.ShaHash, height int64, txoff int, txlen int, spentbuf []byte) (err error) { db.dbLock.Lock() @@ -188,7 +218,7 @@ func (db *LevelDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxLis } if err == btcdb.ErrTxShaMissing { // if the unspent pool did not have the tx, - // look in the fully spent pool (only last instance + // look in the fully spent pool (only last instance) sTxList, fSerr := db.getTxFullySpent(txsha) if fSerr == nil && len(sTxList) != 0 { @@ -346,3 +376,208 @@ func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) ([]*btcdb.TxListReply, e } return replies, nil } + +// addrIndexToKey serializes the passed txAddrIndex for storage within the DB. +func addrIndexToKey(index *txAddrIndex) []byte { + record := make([]byte, addrIndexKeyLength, addrIndexKeyLength) + copy(record[:2], addrIndexKeyPrefix) + copy(record[2:22], index.hash160[:]) + + // The index itself. + binary.LittleEndian.PutUint32(record[22:26], uint32(index.blkHeight)) + binary.LittleEndian.PutUint32(record[26:30], uint32(index.txoffset)) + binary.LittleEndian.PutUint32(record[30:34], uint32(index.txlen)) + + return record +} + +// unpackTxIndex deserializes the raw bytes of a address tx index. +func unpackTxIndex(rawIndex []byte) *txAddrIndex { + return &txAddrIndex{ + blkHeight: int64(binary.LittleEndian.Uint32(rawIndex[0:4])), + txoffset: int(binary.LittleEndian.Uint32(rawIndex[4:8])), + txlen: int(binary.LittleEndian.Uint32(rawIndex[8:12])), + } +} + +// bytesPrefix returns key range that satisfy the given prefix. +// This only applicable for the standard 'bytes comparer'. +func bytesPrefix(prefix []byte) *util.Range { + var limit []byte + for i := len(prefix) - 1; i >= 0; i-- { + c := prefix[i] + if c < 0xff { + limit = make([]byte, i+1) + copy(limit, prefix) + limit[i] = c + 1 + break + } + } + return &util.Range{Start: prefix, Limit: limit} +} + +// FetchTxsForAddr looks up and returns all transactions which either +// spend from a previously created output of the passed address, or +// create a new output locked to the passed address. The, `limit` parameter +// should be the max number of transactions to be returned. Additionally, if the +// caller wishes to seek forward in the results some amount, the 'seek' +// represents how many results to skip. +func (db *LevelDb) FetchTxsForAddr(addr btcutil.Address, skip int, + limit int) ([]*btcdb.TxListReply, error) { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + // Enforce constraints for skip and limit. + if skip < 0 { + return nil, errors.New("offset for skip must be positive") + } + if limit < 0 { + return nil, errors.New("value for limit must be positive") + } + + // Parse address type, bailing on an unknown type. + var addrKey []byte + switch addr := addr.(type) { + case *btcutil.AddressPubKeyHash: + hash160 := addr.Hash160() + addrKey = hash160[:] + case *btcutil.AddressScriptHash: + hash160 := addr.Hash160() + addrKey = hash160[:] + case *btcutil.AddressPubKey: + hash160 := addr.AddressPubKeyHash().Hash160() + addrKey = hash160[:] + default: + return nil, btcdb.ErrUnsupportedAddressType + } + + // Create the prefix for our search. + addrPrefix := make([]byte, 22, 22) + copy(addrPrefix[:2], addrIndexKeyPrefix) + copy(addrPrefix[2:], addrKey) + + var replies []*btcdb.TxListReply + iter := db.lDb.NewIterator(bytesPrefix(addrPrefix), nil) + + for skip != 0 && iter.Next() { + skip-- + } + + // Iterate through all address indexes that match the targeted prefix. + for iter.Next() && limit != 0 { + rawIndex := make([]byte, 22, 22) + copy(rawIndex, iter.Key()[22:]) + addrIndex := unpackTxIndex(rawIndex) + + tx, blkSha, blkHeight, _, err := db.fetchTxDataByLoc(addrIndex.blkHeight, + addrIndex.txoffset, addrIndex.txlen, []byte{}) + if err != nil { + // Eat a possible error due to a potential re-org. + continue + } + + txSha, _ := tx.TxSha() + txReply := &btcdb.TxListReply{Sha: &txSha, Tx: tx, + BlkSha: blkSha, Height: blkHeight, TxSpent: []bool{}, Err: err} + + replies = append(replies, txReply) + limit-- + } + iter.Release() + + return replies, nil +} + +// UpdateAddrIndexForBlock updates the stored addrindex with passed +// index information for a particular block height. Additionally, it +// will update the stored meta-data related to the curent tip of the +// addr index. These two operations are performed in an atomic +// transaction which is commited before the function returns. +// Transactions indexed by address are stored with the following format: +// * prefix || hash160 || blockHeight || txoffset || txlen +// Indexes are stored purely in the key, with blank data for the actual value +// in order to facilitate ease of iteration by their shared prefix and +// also to allow limiting the number of returned transactions (RPC). +// Alternatively, indexes for each address could be stored as an +// append-only list for the stored value. However, this add unnecessary +// overhead when storing and retrieving since the entire list must +// be fetched each time. +func (db *LevelDb) UpdateAddrIndexForBlock(blkSha *btcwire.ShaHash, blkHeight int64, addrIndex btcdb.BlockAddrIndex) error { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + var blankData []byte + batch := db.lBatch() + defer db.lbatch.Reset() + + // Write all data for the new address indexes in a single batch + // transaction. + for addrKey, indexes := range addrIndex { + for _, txLoc := range indexes { + index := &txAddrIndex{ + hash160: addrKey, + blkHeight: blkHeight, + txoffset: txLoc.TxStart, + txlen: txLoc.TxLen, + } + // The index is stored purely in the key. + packedIndex := addrIndexToKey(index) + batch.Put(packedIndex, blankData) + } + } + + // Update tip of addrindex. + newIndexTip := make([]byte, 40, 40) + copy(newIndexTip[:32], blkSha.Bytes()) + binary.LittleEndian.PutUint64(newIndexTip[32:], uint64(blkHeight)) + batch.Put(addrIndexMetaDataKey, newIndexTip) + + if err := db.lDb.Write(batch, db.wo); err != nil { + return err + } + + db.lastAddrIndexBlkIdx = blkHeight + db.lastAddrIndexBlkSha = *blkSha + + return nil +} + +// DeleteAddrIndex deletes the entire addrindex stored within the DB. +// It also resets the cached in-memory metadata about the addr index. +func (db *LevelDb) DeleteAddrIndex() error { + db.dbLock.Lock() + defer db.dbLock.Unlock() + + batch := db.lBatch() + defer batch.Reset() + + // Delete the entire index along with any metadata about it. + iter := db.lDb.NewIterator(bytesPrefix(addrIndexKeyPrefix), db.ro) + numInBatch := 0 + for iter.Next() { + key := iter.Key() + batch.Delete(key) + + numInBatch++ + + // Delete in chunks to potentially avoid very large batches. + if numInBatch >= batchDeleteThreshold { + if err := db.lDb.Write(batch, db.wo); err != nil { + return err + } + batch.Reset() + numInBatch = 0 + } + } + iter.Release() + + batch.Delete(addrIndexMetaDataKey) + if err := db.lDb.Write(batch, db.wo); err != nil { + return err + } + + db.lastAddrIndexBlkIdx = -1 + db.lastAddrIndexBlkSha = btcwire.ShaHash{} + + return nil +} diff --git a/memdb/memdb.go b/memdb/memdb.go index 9294309d..bacb9be6 100644 --- a/memdb/memdb.go +++ b/memdb/memdb.go @@ -698,6 +698,31 @@ func (db *MemDb) NewestSha() (*btcwire.ShaHash, int64, error) { return &blockSha, int64(numBlocks - 1), nil } +// FetchAddrIndexTip isn't currently implemented. This is a part of the +// btcdb.Db interface implementation. +func (db *MemDb) FetchAddrIndexTip() (*btcwire.ShaHash, int64, error) { + return nil, 0, btcdb.ErrNotImplemented +} + +// UpdateAddrIndexForBlock isn't currently implemented. This is a part of the +// btcdb.Db interface implementation. +func (db *MemDb) UpdateAddrIndexForBlock(*btcwire.ShaHash, int64, + btcdb.BlockAddrIndex) error { + return btcdb.ErrNotImplemented +} + +// FetchTxsForAddr isn't currently implemented. This is a part of the btcdb.Db +// interface implementation. +func (db *MemDb) FetchTxsForAddr(btcutil.Address, int, int) ([]*btcdb.TxListReply, error) { + return nil, btcdb.ErrNotImplemented +} + +// DeleteAddrIndex isn't currently implemented. This is a part of the btcdb.Db +// interface implementation. +func (db *MemDb) DeleteAddrIndex() error { + return btcdb.ErrNotImplemented +} + // RollbackClose discards the recent database changes to the previously saved // data at last Sync and closes the database. This is part of the btcdb.Db // interface implementation. From d574a3af6d40c494f5708f8b00c973ed8de78943 Mon Sep 17 00:00:00 2001 From: Dave Collins Date: Tue, 27 Jan 2015 11:45:10 -0600 Subject: [PATCH 163/163] Import btcdb repo into database directory. This commit contains the entire btcdb repository along with several changes needed to move all of the files into the database directory in order to prepare it for merging. This does NOT update btcd or any of the other packages to use the new location as that will be done separately. - All import paths in the old btcdb test files have been changed to the new location - All references to btcdb as the package name have been chagned to database - The coveralls badge has been removed since it unfortunately doesn't support coverage of sub-packages This is ongoing work toward #214. --- .gitignore | 28 ------ .travis.yml | 15 --- LICENSE | 13 --- README.md => database/README.md | 27 +++-- common_test.go => database/common_test.go | 22 ++--- cov_report.sh => database/cov_report.sh | 0 db.go => database/db.go | 2 +- db_test.go => database/db_test.go | 46 ++++----- database/doc.go | 31 ++++++ example_test.go => database/example_test.go | 16 +-- .../interface_test.go | 10 +- {ldb => database/ldb}/block.go | 16 +-- {ldb => database/ldb}/boundary_test.go | 6 +- {ldb => database/ldb}/dbtest/dbtst.go | 0 {ldb => database/ldb}/doc.go | 2 +- {ldb => database/ldb}/dup_test.go | 6 +- {ldb => database/ldb}/insertremove_test.go | 14 +-- {ldb => database/ldb}/internal_test.go | 0 {ldb => database/ldb}/leveldb.go | 22 ++--- {ldb => database/ldb}/operational_test.go | 28 +++--- {ldb => database/ldb}/tx.go | 52 +++++----- log.go => database/log.go | 2 +- {memdb => database/memdb}/doc.go | 4 +- {memdb => database/memdb}/driver.go | 14 +-- {memdb => database/memdb}/memdb.go | 92 +++++++++--------- {memdb => database/memdb}/memdb_test.go | 10 +- .../testdata}/blocks1-256.bz2 | Bin doc.go | 31 ------ 28 files changed, 226 insertions(+), 283 deletions(-) delete mode 100644 .gitignore delete mode 100644 .travis.yml delete mode 100644 LICENSE rename README.md => database/README.md (65%) rename common_test.go => database/common_test.go (91%) rename cov_report.sh => database/cov_report.sh (100%) rename db.go => database/db.go (99%) rename db_test.go => database/db_test.go (79%) create mode 100644 database/doc.go rename example_test.go => database/example_test.go (87%) rename interface_test.go => database/interface_test.go (99%) rename {ldb => database/ldb}/block.go (95%) rename {ldb => database/ldb}/boundary_test.go (90%) rename {ldb => database/ldb}/dbtest/dbtst.go (100%) rename {ldb => database/ldb}/doc.go (85%) rename {ldb => database/ldb}/dup_test.go (97%) rename {ldb => database/ldb}/insertremove_test.go (93%) rename {ldb => database/ldb}/internal_test.go (100%) rename {ldb => database/ldb}/leveldb.go (96%) rename {ldb => database/ldb}/operational_test.go (94%) rename {ldb => database/ldb}/tx.go (91%) rename log.go => database/log.go (98%) rename {memdb => database/memdb}/doc.go (74%) rename {memdb => database/memdb}/driver.go (68%) rename {memdb => database/memdb}/memdb.go (90%) rename {memdb => database/memdb}/memdb_test.go (94%) rename {testdata => database/testdata}/blocks1-256.bz2 (100%) delete mode 100644 doc.go diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 5b97dbba..00000000 --- a/.gitignore +++ /dev/null @@ -1,28 +0,0 @@ -# Temp files -*~ - -# Log files -*.log - -# Compiled Object files, Static and Dynamic libs (Shared Objects) -*.o -*.a -*.so - -# Folders -_obj -_test - -# Architecture specific extensions/prefixes -*.[568vq] -[568vq].out - -*.cgo1.go -*.cgo2.c -_cgo_defun.c -_cgo_gotypes.go -_cgo_export.* - -_testmain.go - -*.exe diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 6f025bf1..00000000 --- a/.travis.yml +++ /dev/null @@ -1,15 +0,0 @@ -language: go -go: release -before_install: - - gocleandeps=c16c849abae90c23419d - - git clone https://gist.github.com/$gocleandeps.git - - goclean=71d0380287747d956a26 - - git clone https://gist.github.com/$goclean.git -install: - - go get -d -t -v ./... - - bash $gocleandeps/gocleandeps.sh -script: - - export PATH=$PATH:$HOME/gopath/bin - - bash $goclean/goclean.sh -after_success: - - goveralls -coverprofile=profile.cov -service=travis-ci \ No newline at end of file diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 992dd50d..00000000 --- a/LICENSE +++ /dev/null @@ -1,13 +0,0 @@ -Copyright (c) 2013-2014 Conformal Systems LLC. - -Permission to use, copy, modify, and distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. \ No newline at end of file diff --git a/README.md b/database/README.md similarity index 65% rename from README.md rename to database/README.md index 90f27b1a..bac1f264 100644 --- a/README.md +++ b/database/README.md @@ -1,42 +1,40 @@ -btcdb -===== +database +======== -[![Build Status](http://img.shields.io/travis/btcsuite/btcdb.svg)] -(https://travis-ci.org/btcsuite/btcdb) [![Coverage Status] -(https://img.shields.io/coveralls/btcsuite/btcdb.svg)] -(https://coveralls.io/r/btcsuite/btcdb) [![ISC License] +[![Build Status](http://img.shields.io/travis/btcsuite/btcd.svg)] +(https://travis-ci.org/btcsuite/btcd) [![ISC License] (http://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org) -Package btcdb provides a database interface for the bitcoin block chain and +Package database provides a database interface for the bitcoin block chain and transactions. ## Documentation [![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)] -(http://godoc.org/github.com/btcsuite/btcdb) +(http://godoc.org/github.com/btcsuite/btcd/database) Full `go doc` style documentation for the project can be viewed online without installing this package by using the GoDoc site -[here](http://godoc.org/github.com/btcsuite/btcdb). +[here](http://godoc.org/github.com/btcsuite/btcd/database). You can also view the documentation locally once the package is installed with the `godoc` tool by running `godoc -http=":6060"` and pointing your browser to -http://localhost:6060/pkg/github.com/btcsuite/btcdb +http://localhost:6060/pkg/github.com/btcsuite/btcd/database ## Installation ```bash -$ go get github.com/btcsuite/btcdb +$ go get github.com/btcsuite/btcd/database ``` ## Examples * [CreateDB Example] - (http://godoc.org/github.com/btcsuite/btcdb#example-CreateDB) + (http://godoc.org/github.com/btcsuite/btcd/database#example-CreateDB) Demonstrates creating a new database and inserting the genesis block into it. * [NewestSha Example] - (http://godoc.org/github.com/btcsuite/btcdb#example-Db--NewestSha) + (http://godoc.org/github.com/btcsuite/btcd/database#example-Db--NewestSha) Demonstrates querying the database for the most recent best block height and hash. @@ -65,4 +63,5 @@ signature perform the following: ## License -Package btcdb is licensed under the [copyfree](http://copyfree.org) ISC License. +Package database is licensed under the [copyfree](http://copyfree.org) ISC +License. diff --git a/common_test.go b/database/common_test.go similarity index 91% rename from common_test.go rename to database/common_test.go index 76762664..eaca1a2f 100644 --- a/common_test.go +++ b/database/common_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package btcdb_test +package database_test import ( "compress/bzip2" @@ -14,9 +14,9 @@ import ( "strings" "testing" - "github.com/btcsuite/btcdb" - _ "github.com/btcsuite/btcdb/ldb" - _ "github.com/btcsuite/btcdb/memdb" + "github.com/btcsuite/btcd/database" + _ "github.com/btcsuite/btcd/database/ldb" + _ "github.com/btcsuite/btcd/database/memdb" "github.com/btcsuite/btcnet" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcwire" @@ -53,10 +53,10 @@ func fileExists(name string) bool { // openDB is used to open an existing database based on the database type and // name. -func openDB(dbType, dbName string) (btcdb.Db, error) { +func openDB(dbType, dbName string) (database.Db, error) { // Handle memdb specially since it has no files on disk. if dbType == "memdb" { - db, err := btcdb.OpenDB(dbType) + db, err := database.OpenDB(dbType) if err != nil { return nil, fmt.Errorf("error opening db: %v", err) } @@ -64,7 +64,7 @@ func openDB(dbType, dbName string) (btcdb.Db, error) { } dbPath := filepath.Join(testDbRoot, dbName) - db, err := btcdb.OpenDB(dbType, dbPath) + db, err := database.OpenDB(dbType, dbPath) if err != nil { return nil, fmt.Errorf("error opening db: %v", err) } @@ -76,11 +76,11 @@ func openDB(dbType, dbName string) (btcdb.Db, error) { // should invoke when done testing to clean up. The close flag indicates // whether or not the teardown function should sync and close the database // during teardown. -func createDB(dbType, dbName string, close bool) (btcdb.Db, func(), error) { +func createDB(dbType, dbName string, close bool) (database.Db, func(), error) { // Handle memory database specially since it doesn't need the disk // specific handling. if dbType == "memdb" { - db, err := btcdb.CreateDB(dbType) + db, err := database.CreateDB(dbType) if err != nil { return nil, nil, fmt.Errorf("error creating db: %v", err) } @@ -108,7 +108,7 @@ func createDB(dbType, dbName string, close bool) (btcdb.Db, func(), error) { // Create a new database to store the accepted blocks into. dbPath := filepath.Join(testDbRoot, dbName) _ = os.RemoveAll(dbPath) - db, err := btcdb.CreateDB(dbType, dbPath) + db, err := database.CreateDB(dbType, dbPath) if err != nil { return nil, nil, fmt.Errorf("error creating db: %v", err) } @@ -132,7 +132,7 @@ func createDB(dbType, dbName string, close bool) (btcdb.Db, func(), error) { // setupDB is used to create a new db instance with the genesis block already // inserted. In addition to the new db instance, it returns a teardown function // the caller should invoke when done testing to clean up. -func setupDB(dbType, dbName string) (btcdb.Db, func(), error) { +func setupDB(dbType, dbName string) (database.Db, func(), error) { db, teardown, err := createDB(dbType, dbName, true) if err != nil { return nil, nil, err diff --git a/cov_report.sh b/database/cov_report.sh similarity index 100% rename from cov_report.sh rename to database/cov_report.sh diff --git a/db.go b/database/db.go similarity index 99% rename from db.go rename to database/db.go index 93e5291f..56e74bc7 100644 --- a/db.go +++ b/database/db.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package btcdb +package database import ( "errors" diff --git a/db_test.go b/database/db_test.go similarity index 79% rename from db_test.go rename to database/db_test.go index b14804cd..801736d8 100644 --- a/db_test.go +++ b/database/db_test.go @@ -2,13 +2,13 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package btcdb_test +package database_test import ( "fmt" "testing" - "github.com/btcsuite/btcdb" + "github.com/btcsuite/btcd/database" ) var ( @@ -21,7 +21,7 @@ var ( // testNewestShaEmpty ensures that NewestSha returns the values expected by // the interface contract. -func testNewestShaEmpty(t *testing.T, db btcdb.Db) { +func testNewestShaEmpty(t *testing.T, db database.Db) { sha, height, err := db.NewestSha() if err != nil { t.Errorf("NewestSha error %v", err) @@ -37,7 +37,7 @@ func testNewestShaEmpty(t *testing.T, db btcdb.Db) { // TestEmptyDB tests that empty databases are handled properly. func TestEmptyDB(t *testing.T) { - for _, dbType := range btcdb.SupportedDBs() { + for _, dbType := range database.SupportedDBs() { // Ensure NewestSha returns expected values for a newly created // db. db, teardown, err := createDB(dbType, "emptydb", false) @@ -66,7 +66,7 @@ func TestEmptyDB(t *testing.T) { // TestAddDuplicateDriver ensures that adding a duplicate driver does not // overwrite an existing one. func TestAddDuplicateDriver(t *testing.T) { - supportedDBs := btcdb.SupportedDBs() + supportedDBs := database.SupportedDBs() if len(supportedDBs) == 0 { t.Errorf("TestAddDuplicateDriver: No backends to test") return @@ -77,7 +77,7 @@ func TestAddDuplicateDriver(t *testing.T) { // driver function and intentionally returns a failure that can be // detected if the interface allows a duplicate driver to overwrite an // existing one. - bogusCreateDB := func(args ...interface{}) (btcdb.Db, error) { + bogusCreateDB := func(args ...interface{}) (database.Db, error) { return nil, fmt.Errorf("duplicate driver allowed for database "+ "type [%v]", dbType) } @@ -85,12 +85,12 @@ func TestAddDuplicateDriver(t *testing.T) { // Create a driver that tries to replace an existing one. Set its // create and open functions to a function that causes a test failure if // they are invoked. - driver := btcdb.DriverDB{ + driver := database.DriverDB{ DbType: dbType, CreateDB: bogusCreateDB, OpenDB: bogusCreateDB, } - btcdb.AddDBDriver(driver) + database.AddDBDriver(driver) // Ensure creating a database of the type that we tried to replace // doesn't fail (if it does, it indicates the driver was erroneously @@ -112,22 +112,22 @@ func TestCreateOpenFail(t *testing.T) { dbType := "createopenfail" openError := fmt.Errorf("failed to create or open database for "+ "database type [%v]", dbType) - bogusCreateDB := func(args ...interface{}) (btcdb.Db, error) { + bogusCreateDB := func(args ...interface{}) (database.Db, error) { return nil, openError } // Create and add driver that intentionally fails when created or opened // to ensure errors on database open and create are handled properly. - driver := btcdb.DriverDB{ + driver := database.DriverDB{ DbType: dbType, CreateDB: bogusCreateDB, OpenDB: bogusCreateDB, } - btcdb.AddDBDriver(driver) + database.AddDBDriver(driver) // Ensure creating a database with the new type fails with the expected // error. - _, err := btcdb.CreateDB(dbType, "createfailtest") + _, err := database.CreateDB(dbType, "createfailtest") if err != openError { t.Errorf("TestCreateOpenFail: expected error not received - "+ "got: %v, want %v", err, openError) @@ -136,7 +136,7 @@ func TestCreateOpenFail(t *testing.T) { // Ensure opening a database with the new type fails with the expected // error. - _, err = btcdb.OpenDB(dbType, "openfailtest") + _, err = database.OpenDB(dbType, "openfailtest") if err != openError { t.Errorf("TestCreateOpenFail: expected error not received - "+ "got: %v, want %v", err, openError) @@ -150,28 +150,28 @@ func TestCreateOpenUnsupported(t *testing.T) { // Ensure creating a database with an unsupported type fails with the // expected error. dbType := "unsupported" - _, err := btcdb.CreateDB(dbType, "unsupportedcreatetest") - if err != btcdb.ErrDbUnknownType { + _, err := database.CreateDB(dbType, "unsupportedcreatetest") + if err != database.ErrDbUnknownType { t.Errorf("TestCreateOpenUnsupported: expected error not "+ - "received - got: %v, want %v", err, btcdb.ErrDbUnknownType) + "received - got: %v, want %v", err, database.ErrDbUnknownType) return } // Ensure opening a database with the new type fails with the expected // error. - _, err = btcdb.OpenDB(dbType, "unsupportedopentest") - if err != btcdb.ErrDbUnknownType { + _, err = database.OpenDB(dbType, "unsupportedopentest") + if err != database.ErrDbUnknownType { t.Errorf("TestCreateOpenUnsupported: expected error not "+ - "received - got: %v, want %v", err, btcdb.ErrDbUnknownType) + "received - got: %v, want %v", err, database.ErrDbUnknownType) return } } -// TestInterface performs tests for the various interfaces of btcdb which -// require state in the database for each supported database type (those loaded -// in common_test.go that is). +// TestInterface performs tests for the various interfaces of the database +// package which require state in the database for each supported database +// type (those loaded in common_test.go that is). func TestInterface(t *testing.T) { - for _, dbType := range btcdb.SupportedDBs() { + for _, dbType := range database.SupportedDBs() { if _, exists := ignoreDbTypes[dbType]; !exists { testInterface(t, dbType) } diff --git a/database/doc.go b/database/doc.go new file mode 100644 index 00000000..ec389016 --- /dev/null +++ b/database/doc.go @@ -0,0 +1,31 @@ +// Copyright (c) 2013-2014 Conformal Systems LLC. +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +/* +Package database provides a database interface for the Bitcoin block chain. + +As of July 2014, there are over 309,000 blocks in the Bitcoin block chain and +and over 42 million transactions (which turns out to be over 21GB of data). +This package provides a database layer to store and retrieve this data in a +fairly simple and efficient manner. The use of this should not require specific +knowledge of the database backend. + +Basic Design + +The basic design of this package is to provide two classes of items in a +database; blocks and transactions (tx) where the block number increases +monotonically. Each transaction belongs to a single block although a block can +have a variable number of transactions. Along with these two items, several +convenience functions for dealing with the database are provided as well as +functions to query specific items that may be present in a block or tx. + +Usage + +At the highest level, the use of this packages just requires that you import it, +setup a database, insert some data into it, and optionally, query the data back. +The first block inserted into the database will be treated as the genesis block. +Every subsequent block insert requires the referenced parent block to already +exist. +*/ +package database diff --git a/example_test.go b/database/example_test.go similarity index 87% rename from example_test.go rename to database/example_test.go index f50d7422..90da19a1 100644 --- a/example_test.go +++ b/database/example_test.go @@ -2,13 +2,13 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package btcdb_test +package database_test import ( "fmt" - "github.com/btcsuite/btcdb" - _ "github.com/btcsuite/btcdb/memdb" + "github.com/btcsuite/btcd/database" + _ "github.com/btcsuite/btcd/database/memdb" "github.com/btcsuite/btcnet" "github.com/btcsuite/btcutil" ) @@ -20,8 +20,8 @@ func ExampleCreateDB() { // Ordinarily this would be whatever driver(s) your application // requires. // import ( - // "github.com/btcsuite/btcdb" - // _ "github.com/btcsuite/btcdb/memdb" + // "github.com/btcsuite/btcd/database" + // _ "github.com/btcsuite/btcd/database/memdb" // ) // Create a database and schedule it to be closed on exit. This example @@ -29,7 +29,7 @@ func ExampleCreateDB() { // the disk. Typically, you would specify a persistent database driver // such as "leveldb" and give it a database name as the second // parameter. - db, err := btcdb.CreateDB("memdb") + db, err := database.CreateDB("memdb") if err != nil { fmt.Println(err) return @@ -51,8 +51,8 @@ func ExampleCreateDB() { } // exampleLoadDB is used in the example to elide the setup code. -func exampleLoadDB() (btcdb.Db, error) { - db, err := btcdb.CreateDB("memdb") +func exampleLoadDB() (database.Db, error) { + db, err := database.CreateDB("memdb") if err != nil { return nil, err } diff --git a/interface_test.go b/database/interface_test.go similarity index 99% rename from interface_test.go rename to database/interface_test.go index 8127b971..a7c5c432 100644 --- a/interface_test.go +++ b/database/interface_test.go @@ -2,13 +2,13 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package btcdb_test +package database_test import ( "reflect" "testing" - "github.com/btcsuite/btcdb" + "github.com/btcsuite/btcd/database" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcwire" "github.com/davecgh/go-spew/spew" @@ -25,7 +25,7 @@ import ( type testContext struct { t *testing.T dbType string - db btcdb.Db + db database.Db blockHeight int64 blockHash *btcwire.ShaHash block *btcutil.Block @@ -523,8 +523,8 @@ func testIntegrity(tc *testContext) bool { return true } -// testInterface tests performs tests for the various interfaces of btcdb which -// require state in the database for the given database type. +// testInterface tests performs tests for the various interfaces of the database +// package which require state in the database for the given database type. func testInterface(t *testing.T, dbType string) { db, teardown, err := setupDB(dbType, "interface") if err != nil { diff --git a/ldb/block.go b/database/ldb/block.go similarity index 95% rename from ldb/block.go rename to database/ldb/block.go index 3e6c84f2..8c5b5e10 100644 --- a/ldb/block.go +++ b/database/ldb/block.go @@ -8,7 +8,7 @@ import ( "bytes" "encoding/binary" - "github.com/btcsuite/btcdb" + "github.com/btcsuite/btcd/database" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcwire" "github.com/btcsuite/goleveldb/leveldb" @@ -40,7 +40,7 @@ func (db *LevelDb) fetchBlockBySha(sha *btcwire.ShaHash) (blk *btcutil.Block, er } // FetchBlockHeightBySha returns the block height for the given hash. This is -// part of the btcdb.Db interface implementation. +// part of the database.Db interface implementation. func (db *LevelDb) FetchBlockHeightBySha(sha *btcwire.ShaHash) (int64, error) { db.dbLock.Lock() defer db.dbLock.Unlock() @@ -77,7 +77,7 @@ func (db *LevelDb) getBlkLoc(sha *btcwire.ShaHash) (int64, error) { data, err := db.lDb.Get(key, db.ro) if err != nil { if err == leveldb.ErrNotFound { - err = btcdb.ErrBlockShaMissing + err = database.ErrBlockShaMissing } return 0, err } @@ -151,7 +151,7 @@ func (db *LevelDb) insertBlockData(sha *btcwire.ShaHash, prevSha *btcwire.ShaHas if err != nil { // check current block count // if count != 0 { - // err = btcdb.PrevShaMissing + // err = database.PrevShaMissing // return // } oBlkHeight = -1 @@ -206,7 +206,7 @@ func (db *LevelDb) blkExistsSha(sha *btcwire.ShaHash) (bool, error) { switch err { case nil: return true, nil - case leveldb.ErrNotFound, btcdb.ErrBlockShaMissing: + case leveldb.ErrNotFound, database.ErrBlockShaMissing: return false, nil } return false, err @@ -247,7 +247,7 @@ func (db *LevelDb) FetchHeightRange(startHeight, endHeight int64) (rshalist []bt defer db.dbLock.Unlock() var endidx int64 - if endHeight == btcdb.AllShas { + if endHeight == database.AllShas { endidx = startHeight + 500 } else { endidx = endHeight @@ -302,7 +302,7 @@ func (db *LevelDb) fetchAddrIndexTip() (*btcwire.ShaHash, int64, error) { data, err := db.lDb.Get(addrIndexMetaDataKey, db.ro) if err != nil { - return &btcwire.ShaHash{}, -1, btcdb.ErrAddrIndexDoesNotExist + return &btcwire.ShaHash{}, -1, database.ErrAddrIndexDoesNotExist } var blkSha btcwire.ShaHash @@ -322,7 +322,7 @@ func (db *LevelDb) FetchAddrIndexTip() (*btcwire.ShaHash, int64, error) { defer db.dbLock.Unlock() if db.lastAddrIndexBlkIdx == -1 { - return &btcwire.ShaHash{}, -1, btcdb.ErrAddrIndexDoesNotExist + return &btcwire.ShaHash{}, -1, database.ErrAddrIndexDoesNotExist } sha := db.lastAddrIndexBlkSha diff --git a/ldb/boundary_test.go b/database/ldb/boundary_test.go similarity index 90% rename from ldb/boundary_test.go rename to database/ldb/boundary_test.go index 6d963d96..986f6711 100644 --- a/ldb/boundary_test.go +++ b/database/ldb/boundary_test.go @@ -8,7 +8,7 @@ import ( "os" "testing" - "github.com/btcsuite/btcdb" + "github.com/btcsuite/btcd/database" "github.com/btcsuite/btcwire" ) @@ -21,7 +21,7 @@ func TestEmptyDB(t *testing.T) { dbnamever := dbname + ".ver" _ = os.RemoveAll(dbname) _ = os.RemoveAll(dbnamever) - db, err := btcdb.CreateDB("leveldb", dbname) + db, err := database.CreateDB("leveldb", dbname) if err != nil { t.Errorf("Failed to open test database %v", err) return @@ -42,7 +42,7 @@ func TestEmptyDB(t *testing.T) { t.Errorf("Close: unexpected error: %v", err) } - db, err = btcdb.OpenDB("leveldb", dbname) + db, err = database.OpenDB("leveldb", dbname) if err != nil { t.Errorf("Failed to open test database %v", err) return diff --git a/ldb/dbtest/dbtst.go b/database/ldb/dbtest/dbtst.go similarity index 100% rename from ldb/dbtest/dbtst.go rename to database/ldb/dbtest/dbtst.go diff --git a/ldb/doc.go b/database/ldb/doc.go similarity index 85% rename from ldb/doc.go rename to database/ldb/doc.go index df99938b..dff5445b 100644 --- a/ldb/doc.go +++ b/database/ldb/doc.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. /* -Package ldb implements an instance of btcdb backed by leveldb. +Package ldb implements an instance of the database package backed by leveldb. Database version number is stored in a flat file .ver Currently a single (littlendian) integer in the file. If there is diff --git a/ldb/dup_test.go b/database/ldb/dup_test.go similarity index 97% rename from ldb/dup_test.go rename to database/ldb/dup_test.go index 302aa05a..c8d12740 100644 --- a/ldb/dup_test.go +++ b/database/ldb/dup_test.go @@ -10,7 +10,7 @@ import ( "path/filepath" "testing" - "github.com/btcsuite/btcdb" + "github.com/btcsuite/btcd/database" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcwire" ) @@ -22,7 +22,7 @@ func Test_dupTx(t *testing.T) { dbnamever := dbname + ".ver" _ = os.RemoveAll(dbname) _ = os.RemoveAll(dbnamever) - db, err := btcdb.CreateDB("leveldb", dbname) + db, err := database.CreateDB("leveldb", dbname) if err != nil { t.Errorf("Failed to open test database %v", err) return @@ -154,7 +154,7 @@ out: listReply = db.FetchUnSpentTxByShaList(fetchList) for _, lr := range listReply { - if lr.Err != btcdb.ErrTxShaMissing { + if lr.Err != database.ErrTxShaMissing { t.Errorf("sha %v spent %v err %v\n", lr.Sha, lr.TxSpent, lr.Err) } diff --git a/ldb/insertremove_test.go b/database/ldb/insertremove_test.go similarity index 93% rename from ldb/insertremove_test.go rename to database/ldb/insertremove_test.go index dd8ab524..4e325f35 100644 --- a/ldb/insertremove_test.go +++ b/database/ldb/insertremove_test.go @@ -10,8 +10,8 @@ import ( "path/filepath" "testing" - "github.com/btcsuite/btcdb" - _ "github.com/btcsuite/btcdb/ldb" + "github.com/btcsuite/btcd/database" + _ "github.com/btcsuite/btcd/database/ldb" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcwire" ) @@ -48,7 +48,7 @@ func testUnspentInsert(t *testing.T) { dbnamever := dbname + ".ver" _ = os.RemoveAll(dbname) _ = os.RemoveAll(dbnamever) - db, err := btcdb.CreateDB("leveldb", dbname) + db, err := database.CreateDB("leveldb", dbname) if err != nil { t.Errorf("Failed to open test database %v", err) return @@ -96,7 +96,7 @@ endtest: txOutList = append(txOutList, &txshaname) } - txneededmap := map[btcwire.ShaHash]*btcdb.TxListReply{} + txneededmap := map[btcwire.ShaHash]*database.TxListReply{} txlist := db.FetchUnSpentTxByShaList(txneededList) for _, txe := range txlist { if txe.Err != nil { @@ -122,7 +122,7 @@ endtest: break endtest } - txlookupmap := map[btcwire.ShaHash]*btcdb.TxListReply{} + txlookupmap := map[btcwire.ShaHash]*database.TxListReply{} txlist = db.FetchTxByShaList(txlookupList) for _, txe := range txlist { if txe.Err != nil { @@ -158,7 +158,7 @@ endtest: break endtest } - txlookupmap = map[btcwire.ShaHash]*btcdb.TxListReply{} + txlookupmap = map[btcwire.ShaHash]*database.TxListReply{} txlist = db.FetchUnSpentTxByShaList(txlookupList) for _, txe := range txlist { if txe.Err != nil { @@ -180,7 +180,7 @@ endtest: t.Errorf("failed to insert block %v err %v", height, err) break endtest } - txlookupmap = map[btcwire.ShaHash]*btcdb.TxListReply{} + txlookupmap = map[btcwire.ShaHash]*database.TxListReply{} txlist = db.FetchTxByShaList(txlookupList) for _, txe := range txlist { if txe.Err != nil { diff --git a/ldb/internal_test.go b/database/ldb/internal_test.go similarity index 100% rename from ldb/internal_test.go rename to database/ldb/internal_test.go diff --git a/ldb/leveldb.go b/database/ldb/leveldb.go similarity index 96% rename from ldb/leveldb.go rename to database/ldb/leveldb.go index cc946fbb..42eda568 100644 --- a/ldb/leveldb.go +++ b/database/ldb/leveldb.go @@ -11,7 +11,7 @@ import ( "strconv" "sync" - "github.com/btcsuite/btcdb" + "github.com/btcsuite/btcd/database" "github.com/btcsuite/btclog" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcwire" @@ -60,13 +60,13 @@ type LevelDb struct { txSpentUpdateMap map[btcwire.ShaHash]*spentTxUpdate } -var self = btcdb.DriverDB{DbType: "leveldb", CreateDB: CreateDB, OpenDB: OpenDB} +var self = database.DriverDB{DbType: "leveldb", CreateDB: CreateDB, OpenDB: OpenDB} func init() { - btcdb.AddDBDriver(self) + database.AddDBDriver(self) } -// parseArgs parses the arguments from the btcdb Open/Create methods. +// parseArgs parses the arguments from the database package Open/Create methods. func parseArgs(funcName string, args ...interface{}) (string, error) { if len(args) != 1 { return "", fmt.Errorf("Invalid arguments to ldb.%s -- "+ @@ -81,13 +81,13 @@ func parseArgs(funcName string, args ...interface{}) (string, error) { } // OpenDB opens an existing database for use. -func OpenDB(args ...interface{}) (btcdb.Db, error) { +func OpenDB(args ...interface{}) (database.Db, error) { dbpath, err := parseArgs("OpenDB", args...) if err != nil { return nil, err } - log = btcdb.GetLog() + log = database.GetLog() db, err := openDB(dbpath, false) if err != nil { @@ -159,7 +159,7 @@ blocknarrow: // CurrentDBVersion is the database version. var CurrentDBVersion int32 = 1 -func openDB(dbpath string, create bool) (pbdb btcdb.Db, err error) { +func openDB(dbpath string, create bool) (pbdb database.Db, err error) { var db LevelDb var tlDb *leveldb.DB var dbversion int32 @@ -184,7 +184,7 @@ func openDB(dbpath string, create bool) (pbdb btcdb.Db, err error) { } else { _, err = os.Stat(dbpath) if err != nil { - err = btcdb.ErrDbDoesNotExist + err = database.ErrDbDoesNotExist return } } @@ -247,13 +247,13 @@ func openDB(dbpath string, create bool) (pbdb btcdb.Db, err error) { } // CreateDB creates, initializes and opens a database for use. -func CreateDB(args ...interface{}) (btcdb.Db, error) { +func CreateDB(args ...interface{}) (database.Db, error) { dbpath, err := parseArgs("Create", args...) if err != nil { return nil, err } - log = btcdb.GetLog() + log = database.GetLog() // No special setup needed, just OpenBB db, err := openDB(dbpath, true) @@ -694,7 +694,7 @@ func (db *LevelDb) processBatches() error { return nil } -// RollbackClose this is part of the btcdb.Db interface and should discard +// RollbackClose this is part of the database.Db interface and should discard // recent changes to the db and the close the db. This currently just does // a clean shutdown. func (db *LevelDb) RollbackClose() error { diff --git a/ldb/operational_test.go b/database/ldb/operational_test.go similarity index 94% rename from ldb/operational_test.go rename to database/ldb/operational_test.go index 6cb16422..349b615f 100644 --- a/ldb/operational_test.go +++ b/database/ldb/operational_test.go @@ -15,7 +15,7 @@ import ( "strings" "testing" - "github.com/btcsuite/btcdb" + "github.com/btcsuite/btcd/database" "github.com/btcsuite/btcnet" "github.com/btcsuite/btcscript" "github.com/btcsuite/btcutil" @@ -29,7 +29,7 @@ var network = btcwire.MainNet // the `cleanUpFunc` *must* be called after each test to maintain db // consistency across tests. type testDb struct { - db btcdb.Db + db database.Db blocks []*btcutil.Block dbName string dbNameVer string @@ -42,7 +42,7 @@ func setUpTestDb(t *testing.T) (*testDb, error) { dbnamever := dbname + ".ver" _ = os.RemoveAll(dbname) _ = os.RemoveAll(dbnamever) - db, err := btcdb.CreateDB("leveldb", dbname) + db, err := database.CreateDB("leveldb", dbname) if err != nil { return nil, err } @@ -74,10 +74,10 @@ func TestOperational(t *testing.T) { // testAddrIndexOperations ensures that all normal operations concerning // the optional address index function correctly. -func testAddrIndexOperations(t *testing.T, db btcdb.Db, newestBlock *btcutil.Block, newestSha *btcwire.ShaHash, newestBlockIdx int64) { +func testAddrIndexOperations(t *testing.T, db database.Db, newestBlock *btcutil.Block, newestSha *btcwire.ShaHash, newestBlockIdx int64) { // Metadata about the current addr index state should be unset. sha, height, err := db.FetchAddrIndexTip() - if err != btcdb.ErrAddrIndexDoesNotExist { + if err != database.ErrAddrIndexDoesNotExist { t.Fatalf("Address index metadata shouldn't be in db, hasn't been built up yet.") } @@ -104,7 +104,7 @@ func testAddrIndexOperations(t *testing.T, db btcdb.Db, newestBlock *btcutil.Blo } // Simple test to index outputs(s) of the first tx. - testIndex := make(btcdb.BlockAddrIndex) + testIndex := make(database.BlockAddrIndex) testTx, err := newestBlock.Tx(0) if err != nil { t.Fatalf("Block has no transactions, unable to test addr "+ @@ -160,7 +160,7 @@ func testAddrIndexOperations(t *testing.T, db btcdb.Db, newestBlock *btcutil.Blo db.Close() // Re-Open, tip still should be updated to current height and sha. - db, err = btcdb.OpenDB("leveldb", "tstdbop1") + db, err = database.OpenDB("leveldb", "tstdbop1") if err != nil { t.Fatalf("Unable to re-open created db, err %v", err) } @@ -184,13 +184,13 @@ func testAddrIndexOperations(t *testing.T, db btcdb.Db, newestBlock *btcutil.Blo } // Tip should be blanked out. - if _, _, err := db.FetchAddrIndexTip(); err != btcdb.ErrAddrIndexDoesNotExist { + if _, _, err := db.FetchAddrIndexTip(); err != database.ErrAddrIndexDoesNotExist { t.Fatalf("Address index was not fully deleted.") } } -func assertAddrIndexTipIsUpdated(db btcdb.Db, t *testing.T, newestSha *btcwire.ShaHash, newestBlockIdx int64) { +func assertAddrIndexTipIsUpdated(db database.Db, t *testing.T, newestSha *btcwire.ShaHash, newestBlockIdx int64) { // Safe to ignore error, since height will be < 0 in "error" case. sha, height, _ := db.FetchAddrIndexTip() if newestBlockIdx != height { @@ -337,7 +337,7 @@ func testBackout(t *testing.T) { // db was closed at height 120, so no cleanup is possible. // reopen db - testDb.db, err = btcdb.OpenDB("leveldb", testDb.dbName) + testDb.db, err = database.OpenDB("leveldb", testDb.dbName) if err != nil { t.Errorf("Failed to open test database %v", err) return @@ -465,7 +465,7 @@ func loadBlocks(t *testing.T, file string) (blocks []*btcutil.Block, err error) return } -func testFetchHeightRange(t *testing.T, db btcdb.Db, blocks []*btcutil.Block) { +func testFetchHeightRange(t *testing.T, db database.Db, blocks []*btcutil.Block) { var testincrement int64 = 50 var testcnt int64 = 100 @@ -486,7 +486,7 @@ func testFetchHeightRange(t *testing.T, db btcdb.Db, blocks []*btcutil.Block) { endheight := startheight + testcnt if endheight > nBlocks { - endheight = btcdb.AllShas + endheight = database.AllShas } shalist, err := db.FetchHeightRange(startheight, endheight) @@ -494,7 +494,7 @@ func testFetchHeightRange(t *testing.T, db btcdb.Db, blocks []*btcutil.Block) { t.Errorf("FetchHeightRange: unexpected failure looking up shas %v", err) } - if endheight == btcdb.AllShas { + if endheight == database.AllShas { if int64(len(shalist)) != nBlocks-startheight { t.Errorf("FetchHeightRange: expected A %v shas, got %v", nBlocks-startheight, len(shalist)) } @@ -549,7 +549,7 @@ func TestLimitAndSkipFetchTxsForAddr(t *testing.T) { // Create and insert an address index for out test addr. txLoc, _ := testBlock.TxLoc() - index := make(btcdb.BlockAddrIndex) + index := make(database.BlockAddrIndex) for i := range testBlock.Transactions() { var hash160 [ripemd160.Size]byte scriptAddr := targetAddr.ScriptAddress() diff --git a/ldb/tx.go b/database/ldb/tx.go similarity index 91% rename from ldb/tx.go rename to database/ldb/tx.go index aff215a2..4157b814 100644 --- a/ldb/tx.go +++ b/database/ldb/tx.go @@ -9,7 +9,7 @@ import ( "encoding/binary" "errors" - "github.com/btcsuite/btcdb" + "github.com/btcsuite/btcd/database" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcwire" "github.com/btcsuite/goleveldb/leveldb" @@ -129,7 +129,7 @@ func (db *LevelDb) getTxFullySpent(txsha *btcwire.ShaHash) ([]*spentTx, error) { key := shaSpentTxToKey(txsha) buf, err := db.lDb.Get(key, db.ro) if err == leveldb.ErrNotFound { - return badTxList, btcdb.ErrTxShaMissing + return badTxList, database.ErrTxShaMissing } else if err != nil { return badTxList, err } @@ -198,13 +198,13 @@ func (db *LevelDb) existsTxSha(txSha *btcwire.ShaHash) (bool, error) { } // FetchTxByShaList returns the most recent tx of the name fully spent or not -func (db *LevelDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { +func (db *LevelDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*database.TxListReply { db.dbLock.Lock() defer db.dbLock.Unlock() // until the fully spent separation of tx is complete this is identical // to FetchUnSpentTxByShaList - replies := make([]*btcdb.TxListReply, len(txShaList)) + replies := make([]*database.TxListReply, len(txShaList)) for i, txsha := range txShaList { tx, blockSha, height, txspent, err := db.fetchTxDataBySha(txsha) btxspent := []bool{} @@ -216,7 +216,7 @@ func (db *LevelDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxLis btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0 } } - if err == btcdb.ErrTxShaMissing { + if err == database.ErrTxShaMissing { // if the unspent pool did not have the tx, // look in the fully spent pool (only last instance) @@ -235,7 +235,7 @@ func (db *LevelDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxLis } } } - txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, BlkSha: blockSha, Height: height, TxSpent: btxspent, Err: err} + txlre := database.TxListReply{Sha: txsha, Tx: tx, BlkSha: blockSha, Height: height, TxSpent: btxspent, Err: err} replies[i] = &txlre } return replies @@ -243,11 +243,11 @@ func (db *LevelDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxLis // FetchUnSpentTxByShaList given a array of ShaHash, look up the transactions // and return them in a TxListReply array. -func (db *LevelDb) FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { +func (db *LevelDb) FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*database.TxListReply { db.dbLock.Lock() defer db.dbLock.Unlock() - replies := make([]*btcdb.TxListReply, len(txShaList)) + replies := make([]*database.TxListReply, len(txShaList)) for i, txsha := range txShaList { tx, blockSha, height, txspent, err := db.fetchTxDataBySha(txsha) btxspent := []bool{} @@ -259,7 +259,7 @@ func (db *LevelDb) FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*btcd btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0 } } - txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, BlkSha: blockSha, Height: height, TxSpent: btxspent, Err: err} + txlre := database.TxListReply{Sha: txsha, Tx: tx, BlkSha: blockSha, Height: height, TxSpent: btxspent, Err: err} replies[i] = &txlre } return replies @@ -274,7 +274,7 @@ func (db *LevelDb) fetchTxDataBySha(txsha *btcwire.ShaHash) (rtx *btcwire.MsgTx, blkHeight, txOff, txLen, txspent, err = db.getTxData(txsha) if err != nil { if err == leveldb.ErrNotFound { - err = btcdb.ErrTxShaMissing + err = database.ErrTxShaMissing } return } @@ -290,7 +290,7 @@ func (db *LevelDb) fetchTxDataByLoc(blkHeight int64, txOff int, txLen int, txspe blksha, blkbuf, err = db.getBlkByHeight(blkHeight) if err != nil { if err == leveldb.ErrNotFound { - err = btcdb.ErrTxShaMissing + err = database.ErrTxShaMissing } return } @@ -299,7 +299,7 @@ func (db *LevelDb) fetchTxDataByLoc(blkHeight int64, txOff int, txLen int, txspe // txsha, blksha, blkHeight, txOff, txLen) if len(blkbuf) < txOff+txLen { - err = btcdb.ErrTxShaMissing + err = database.ErrTxShaMissing return } rbuf := bytes.NewReader(blkbuf[txOff : txOff+txLen]) @@ -316,7 +316,7 @@ func (db *LevelDb) fetchTxDataByLoc(blkHeight int64, txOff int, txLen int, txspe } // FetchTxBySha returns some data for the given Tx Sha. -func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) ([]*btcdb.TxListReply, error) { +func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) ([]*database.TxListReply, error) { db.dbLock.Lock() defer db.dbLock.Unlock() @@ -327,22 +327,22 @@ func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) ([]*btcdb.TxListReply, e if txerr == nil { replylen++ } else { - if txerr != btcdb.ErrTxShaMissing { - return []*btcdb.TxListReply{}, txerr + if txerr != database.ErrTxShaMissing { + return []*database.TxListReply{}, txerr } } sTxList, fSerr := db.getTxFullySpent(txsha) if fSerr != nil { - if fSerr != btcdb.ErrTxShaMissing { - return []*btcdb.TxListReply{}, fSerr + if fSerr != database.ErrTxShaMissing { + return []*database.TxListReply{}, fSerr } } else { replylen += len(sTxList) } - replies := make([]*btcdb.TxListReply, replylen) + replies := make([]*database.TxListReply, replylen) if fSerr == nil { for _, stx := range sTxList { @@ -350,7 +350,7 @@ func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) ([]*btcdb.TxListReply, e stx.blkHeight, stx.txoff, stx.txlen, []byte{}) if err != nil { if err != leveldb.ErrNotFound { - return []*btcdb.TxListReply{}, err + return []*database.TxListReply{}, err } continue } @@ -358,7 +358,7 @@ func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) ([]*btcdb.TxListReply, e for i := range btxspent { btxspent[i] = true } - txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, BlkSha: blksha, Height: stx.blkHeight, TxSpent: btxspent, Err: nil} + txlre := database.TxListReply{Sha: txsha, Tx: tx, BlkSha: blksha, Height: stx.blkHeight, TxSpent: btxspent, Err: nil} replies[replycnt] = &txlre replycnt++ } @@ -370,7 +370,7 @@ func (db *LevelDb) FetchTxBySha(txsha *btcwire.ShaHash) ([]*btcdb.TxListReply, e byteoff := uint(idx % 8) btxspent[idx] = (txspent[byteidx] & (byte(1) << byteoff)) != 0 } - txlre := btcdb.TxListReply{Sha: txsha, Tx: tx, BlkSha: blksha, Height: height, TxSpent: btxspent, Err: nil} + txlre := database.TxListReply{Sha: txsha, Tx: tx, BlkSha: blksha, Height: height, TxSpent: btxspent, Err: nil} replies[replycnt] = &txlre replycnt++ } @@ -423,7 +423,7 @@ func bytesPrefix(prefix []byte) *util.Range { // caller wishes to seek forward in the results some amount, the 'seek' // represents how many results to skip. func (db *LevelDb) FetchTxsForAddr(addr btcutil.Address, skip int, - limit int) ([]*btcdb.TxListReply, error) { + limit int) ([]*database.TxListReply, error) { db.dbLock.Lock() defer db.dbLock.Unlock() @@ -448,7 +448,7 @@ func (db *LevelDb) FetchTxsForAddr(addr btcutil.Address, skip int, hash160 := addr.AddressPubKeyHash().Hash160() addrKey = hash160[:] default: - return nil, btcdb.ErrUnsupportedAddressType + return nil, database.ErrUnsupportedAddressType } // Create the prefix for our search. @@ -456,7 +456,7 @@ func (db *LevelDb) FetchTxsForAddr(addr btcutil.Address, skip int, copy(addrPrefix[:2], addrIndexKeyPrefix) copy(addrPrefix[2:], addrKey) - var replies []*btcdb.TxListReply + var replies []*database.TxListReply iter := db.lDb.NewIterator(bytesPrefix(addrPrefix), nil) for skip != 0 && iter.Next() { @@ -477,7 +477,7 @@ func (db *LevelDb) FetchTxsForAddr(addr btcutil.Address, skip int, } txSha, _ := tx.TxSha() - txReply := &btcdb.TxListReply{Sha: &txSha, Tx: tx, + txReply := &database.TxListReply{Sha: &txSha, Tx: tx, BlkSha: blkSha, Height: blkHeight, TxSpent: []bool{}, Err: err} replies = append(replies, txReply) @@ -502,7 +502,7 @@ func (db *LevelDb) FetchTxsForAddr(addr btcutil.Address, skip int, // append-only list for the stored value. However, this add unnecessary // overhead when storing and retrieving since the entire list must // be fetched each time. -func (db *LevelDb) UpdateAddrIndexForBlock(blkSha *btcwire.ShaHash, blkHeight int64, addrIndex btcdb.BlockAddrIndex) error { +func (db *LevelDb) UpdateAddrIndexForBlock(blkSha *btcwire.ShaHash, blkHeight int64, addrIndex database.BlockAddrIndex) error { db.dbLock.Lock() defer db.dbLock.Unlock() diff --git a/log.go b/database/log.go similarity index 98% rename from log.go rename to database/log.go index 09a03d35..b390bc40 100644 --- a/log.go +++ b/database/log.go @@ -2,7 +2,7 @@ // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. -package btcdb +package database import ( "errors" diff --git a/memdb/doc.go b/database/memdb/doc.go similarity index 74% rename from memdb/doc.go rename to database/memdb/doc.go index 985aeb60..67f240bd 100644 --- a/memdb/doc.go +++ b/database/memdb/doc.go @@ -3,8 +3,8 @@ // license that can be found in the LICENSE file. /* -Package memdb implements an instance of btcdb that uses memory for the block -storage. +Package memdb implements an instance of the database package that uses memory +for the block storage. This is primary used for testing purposes as normal operations require a persistent block storage mechanism which this is not. diff --git a/memdb/driver.go b/database/memdb/driver.go similarity index 68% rename from memdb/driver.go rename to database/memdb/driver.go index cf4dddd6..fc9c03b3 100644 --- a/memdb/driver.go +++ b/database/memdb/driver.go @@ -7,18 +7,18 @@ package memdb import ( "fmt" - "github.com/btcsuite/btcdb" + "github.com/btcsuite/btcd/database" "github.com/btcsuite/btclog" ) var log = btclog.Disabled func init() { - driver := btcdb.DriverDB{DbType: "memdb", CreateDB: CreateDB, OpenDB: OpenDB} - btcdb.AddDBDriver(driver) + driver := database.DriverDB{DbType: "memdb", CreateDB: CreateDB, OpenDB: OpenDB} + database.AddDBDriver(driver) } -// parseArgs parses the arguments from the btcdb Open/Create methods. +// parseArgs parses the arguments from the database package Open/Create methods. func parseArgs(funcName string, args ...interface{}) error { if len(args) != 0 { return fmt.Errorf("memdb.%s does not accept any arguments", @@ -29,7 +29,7 @@ func parseArgs(funcName string, args ...interface{}) error { } // OpenDB opens an existing database for use. -func OpenDB(args ...interface{}) (btcdb.Db, error) { +func OpenDB(args ...interface{}) (database.Db, error) { if err := parseArgs("OpenDB", args...); err != nil { return nil, err } @@ -39,11 +39,11 @@ func OpenDB(args ...interface{}) (btcdb.Db, error) { } // CreateDB creates, initializes, and opens a database for use. -func CreateDB(args ...interface{}) (btcdb.Db, error) { +func CreateDB(args ...interface{}) (database.Db, error) { if err := parseArgs("CreateDB", args...); err != nil { return nil, err } - log = btcdb.GetLog() + log = database.GetLog() return newMemDb(), nil } diff --git a/memdb/memdb.go b/database/memdb/memdb.go similarity index 90% rename from memdb/memdb.go rename to database/memdb/memdb.go index bacb9be6..e7c6d5f1 100644 --- a/memdb/memdb.go +++ b/database/memdb/memdb.go @@ -10,7 +10,7 @@ import ( "math" "sync" - "github.com/btcsuite/btcdb" + "github.com/btcsuite/btcd/database" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcwire" ) @@ -73,7 +73,7 @@ func isFullySpent(txD *tTxInsertData) bool { return true } -// MemDb is a concrete implementation of the btcdb.Db interface which provides +// MemDb is a concrete implementation of the database.Db interface which provides // a memory-only database. Since it is memory-only, it is obviously not // persistent and is mostly only useful for testing purposes. type MemDb struct { @@ -133,7 +133,7 @@ func (db *MemDb) removeTx(msgTx *btcwire.MsgTx, txHash *btcwire.ShaHash) { } -// Close cleanly shuts down database. This is part of the btcdb.Db interface +// Close cleanly shuts down database. This is part of the database.Db interface // implementation. // // All data is purged upon close with this implementation since it is a @@ -155,7 +155,7 @@ func (db *MemDb) Close() error { // DropAfterBlockBySha removes any blocks from the database after the given // block. This is different than a simple truncate since the spend information -// for each block must also be unwound. This is part of the btcdb.Db interface +// for each block must also be unwound. This is part of the database.Db interface // implementation. func (db *MemDb) DropAfterBlockBySha(sha *btcwire.ShaHash) error { db.Lock() @@ -196,7 +196,7 @@ func (db *MemDb) DropAfterBlockBySha(sha *btcwire.ShaHash) error { } // ExistsSha returns whether or not the given block hash is present in the -// database. This is part of the btcdb.Db interface implementation. +// database. This is part of the database.Db interface implementation. func (db *MemDb) ExistsSha(sha *btcwire.ShaHash) (bool, error) { db.Lock() defer db.Unlock() @@ -213,7 +213,7 @@ func (db *MemDb) ExistsSha(sha *btcwire.ShaHash) (bool, error) { } // FetchBlockBySha returns a btcutil.Block. The implementation may cache the -// underlying data if desired. This is part of the btcdb.Db interface +// underlying data if desired. This is part of the database.Db interface // implementation. // // This implementation does not use any additional cache since the entire @@ -236,7 +236,7 @@ func (db *MemDb) FetchBlockBySha(sha *btcwire.ShaHash) (*btcutil.Block, error) { } // FetchBlockHeightBySha returns the block height for the given hash. This is -// part of the btcdb.Db interface implementation. +// part of the database.Db interface implementation. func (db *MemDb) FetchBlockHeightBySha(sha *btcwire.ShaHash) (int64, error) { db.Lock() defer db.Unlock() @@ -254,7 +254,7 @@ func (db *MemDb) FetchBlockHeightBySha(sha *btcwire.ShaHash) (int64, error) { // FetchBlockHeaderBySha returns a btcwire.BlockHeader for the given sha. The // implementation may cache the underlying data if desired. This is part of the -// btcdb.Db interface implementation. +// database.Db interface implementation. // // This implementation does not use any additional cache since the entire // database is already in memory. @@ -274,7 +274,7 @@ func (db *MemDb) FetchBlockHeaderBySha(sha *btcwire.ShaHash) (*btcwire.BlockHead } // FetchBlockShaByHeight returns a block hash based on its height in the block -// chain. This is part of the btcdb.Db interface implementation. +// chain. This is part of the database.Db interface implementation. func (db *MemDb) FetchBlockShaByHeight(height int64) (*btcwire.ShaHash, error) { db.Lock() defer db.Unlock() @@ -302,7 +302,7 @@ func (db *MemDb) FetchBlockShaByHeight(height int64) (*btcwire.ShaHash, error) { // FetchHeightRange looks up a range of blocks by the start and ending heights. // Fetch is inclusive of the start height and exclusive of the ending height. // To fetch all hashes from the start height until no more are present, use the -// special id `AllShas'. This is part of the btcdb.Db interface implementation. +// special id `AllShas'. This is part of the database.Db interface implementation. func (db *MemDb) FetchHeightRange(startHeight, endHeight int64) ([]btcwire.ShaHash, error) { db.Lock() defer db.Unlock() @@ -313,7 +313,7 @@ func (db *MemDb) FetchHeightRange(startHeight, endHeight int64) ([]btcwire.ShaHa // When the user passes the special AllShas id, adjust the end height // accordingly. - if endHeight == btcdb.AllShas { + if endHeight == database.AllShas { endHeight = int64(len(db.blocks)) } @@ -348,7 +348,7 @@ func (db *MemDb) FetchHeightRange(startHeight, endHeight int64) ([]btcwire.ShaHa } // ExistsTxSha returns whether or not the given transaction hash is present in -// the database and is not fully spent. This is part of the btcdb.Db interface +// the database and is not fully spent. This is part of the database.Db interface // implementation. func (db *MemDb) ExistsTxSha(sha *btcwire.ShaHash) (bool, error) { db.Lock() @@ -367,11 +367,11 @@ func (db *MemDb) ExistsTxSha(sha *btcwire.ShaHash) (bool, error) { // FetchTxBySha returns some data for the given transaction hash. The // implementation may cache the underlying data if desired. This is part of the -// btcdb.Db interface implementation. +// database.Db interface implementation. // // This implementation does not use any additional cache since the entire // database is already in memory. -func (db *MemDb) FetchTxBySha(txHash *btcwire.ShaHash) ([]*btcdb.TxListReply, error) { +func (db *MemDb) FetchTxBySha(txHash *btcwire.ShaHash) ([]*database.TxListReply, error) { db.Lock() defer db.Unlock() @@ -383,11 +383,11 @@ func (db *MemDb) FetchTxBySha(txHash *btcwire.ShaHash) ([]*btcdb.TxListReply, er if !exists { log.Warnf("FetchTxBySha: requested hash of %s does not exist", txHash) - return nil, btcdb.ErrTxShaMissing + return nil, database.ErrTxShaMissing } txHashCopy := *txHash - replyList := make([]*btcdb.TxListReply, len(txns)) + replyList := make([]*database.TxListReply, len(txns)) for i, txD := range txns { msgBlock := db.blocks[txD.blockHeight] blockSha, err := msgBlock.BlockSha() @@ -397,7 +397,7 @@ func (db *MemDb) FetchTxBySha(txHash *btcwire.ShaHash) ([]*btcdb.TxListReply, er spentBuf := make([]bool, len(txD.spentBuf)) copy(spentBuf, txD.spentBuf) - reply := btcdb.TxListReply{ + reply := database.TxListReply{ Sha: &txHashCopy, Tx: msgBlock.Transactions[txD.offset], BlkSha: &blockSha, @@ -422,16 +422,16 @@ func (db *MemDb) FetchTxBySha(txHash *btcwire.ShaHash) ([]*btcdb.TxListReply, er // will indicate the transaction does not exist. // // This function must be called with the db lock held. -func (db *MemDb) fetchTxByShaList(txShaList []*btcwire.ShaHash, includeSpent bool) []*btcdb.TxListReply { - replyList := make([]*btcdb.TxListReply, 0, len(txShaList)) +func (db *MemDb) fetchTxByShaList(txShaList []*btcwire.ShaHash, includeSpent bool) []*database.TxListReply { + replyList := make([]*database.TxListReply, 0, len(txShaList)) for i, hash := range txShaList { // Every requested entry needs a response, so start with nothing // more than a response with the requested hash marked missing. // The reply will be updated below with the appropriate // information if the transaction exists. - reply := btcdb.TxListReply{ + reply := database.TxListReply{ Sha: txShaList[i], - Err: btcdb.ErrTxShaMissing, + Err: database.ErrTxShaMissing, } replyList = append(replyList, &reply) @@ -480,7 +480,7 @@ func (db *MemDb) fetchTxByShaList(txShaList []*btcwire.ShaHash, includeSpent boo // FetchTxByShaList returns a TxListReply given an array of transaction hashes. // The implementation may cache the underlying data if desired. This is part of -// the btcdb.Db interface implementation. +// the database.Db interface implementation. // // This implementation does not use any additional cache since the entire // database is already in memory. @@ -491,14 +491,14 @@ func (db *MemDb) fetchTxByShaList(txShaList []*btcwire.ShaHash, includeSpent boo // increased number of transaction fetches, this function is typically more // expensive than the unspent counterpart, however the specific performance // details depend on the concrete implementation. The implementation may cache -// the underlying data if desired. This is part of the btcdb.Db interface +// the underlying data if desired. This is part of the database.Db interface // implementation. // // To fetch all versions of a specific transaction, call FetchTxBySha. // // This implementation does not use any additional cache since the entire // database is already in memory. -func (db *MemDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { +func (db *MemDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*database.TxListReply { db.Lock() defer db.Unlock() @@ -508,7 +508,7 @@ func (db *MemDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListR // FetchUnSpentTxByShaList returns a TxListReply given an array of transaction // hashes. Any transactions which are fully spent will indicate they do not // exist by setting the Err field to TxShaMissing. The implementation may cache -// the underlying data if desired. This is part of the btcdb.Db interface +// the underlying data if desired. This is part of the database.Db interface // implementation. // // To obtain results which do contain the most recent version of a fully spent @@ -517,7 +517,7 @@ func (db *MemDb) FetchTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListR // // This implementation does not use any additional cache since the entire // database is already in memory. -func (db *MemDb) FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb.TxListReply { +func (db *MemDb) FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*database.TxListReply { db.Lock() defer db.Unlock() @@ -527,7 +527,7 @@ func (db *MemDb) FetchUnSpentTxByShaList(txShaList []*btcwire.ShaHash) []*btcdb. // InsertBlock inserts raw block and transaction data from a block into the // database. The first block inserted into the database will be treated as the // genesis block. Every subsequent block insert requires the referenced parent -// block to already exist. This is part of the btcdb.Db interface +// block to already exist. This is part of the database.Db interface // implementation. func (db *MemDb) InsertBlock(block *btcutil.Block) (int64, error) { db.Lock() @@ -548,7 +548,7 @@ func (db *MemDb) InsertBlock(block *btcutil.Block) (int64, error) { msgBlock := block.MsgBlock() if _, exists := db.blocksBySha[msgBlock.Header.PrevBlock]; !exists { if len(db.blocks) > 0 { - return 0, btcdb.ErrPrevShaMissing + return 0, database.ErrPrevShaMissing } } @@ -599,7 +599,7 @@ func (db *MemDb) InsertBlock(block *btcutil.Block) (int64, error) { log.Warnf("InsertBlock: requested hash "+ " of %s does not exist in-flight", tx.Sha()) - return 0, btcdb.ErrTxShaMissing + return 0, database.ErrTxShaMissing } } else { originTxns, exists := db.txns[prevOut.Hash] @@ -607,14 +607,14 @@ func (db *MemDb) InsertBlock(block *btcutil.Block) (int64, error) { log.Warnf("InsertBlock: requested hash "+ "of %s by %s does not exist", prevOut.Hash, tx.Sha()) - return 0, btcdb.ErrTxShaMissing + return 0, database.ErrTxShaMissing } originTxD := originTxns[len(originTxns)-1] if prevOut.Index > uint32(len(originTxD.spentBuf)) { log.Warnf("InsertBlock: requested hash "+ "of %s with index %d does not "+ "exist", tx.Sha(), prevOut.Index) - return 0, btcdb.ErrTxShaMissing + return 0, database.ErrTxShaMissing } } } @@ -624,7 +624,7 @@ func (db *MemDb) InsertBlock(block *btcutil.Block) (int64, error) { inFlightIndex < i { log.Warnf("Block contains duplicate transaction %s", tx.Sha()) - return 0, btcdb.ErrDuplicateSha + return 0, database.ErrDuplicateSha } // Prevent duplicate transactions unless the old one is fully @@ -634,7 +634,7 @@ func (db *MemDb) InsertBlock(block *btcutil.Block) (int64, error) { if !isFullySpent(txD) { log.Warnf("Attempt to insert duplicate "+ "transaction %s", tx.Sha()) - return 0, btcdb.ErrDuplicateSha + return 0, database.ErrDuplicateSha } } } @@ -674,7 +674,7 @@ func (db *MemDb) InsertBlock(block *btcutil.Block) (int64, error) { // NewestSha returns the hash and block height of the most recent (end) block of // the block chain. It will return the zero hash, -1 for the block height, and // no error (nil) if there are not any blocks in the database yet. This is part -// of the btcdb.Db interface implementation. +// of the database.Db interface implementation. func (db *MemDb) NewestSha() (*btcwire.ShaHash, int64, error) { db.Lock() defer db.Unlock() @@ -699,32 +699,32 @@ func (db *MemDb) NewestSha() (*btcwire.ShaHash, int64, error) { } // FetchAddrIndexTip isn't currently implemented. This is a part of the -// btcdb.Db interface implementation. +// database.Db interface implementation. func (db *MemDb) FetchAddrIndexTip() (*btcwire.ShaHash, int64, error) { - return nil, 0, btcdb.ErrNotImplemented + return nil, 0, database.ErrNotImplemented } // UpdateAddrIndexForBlock isn't currently implemented. This is a part of the -// btcdb.Db interface implementation. +// database.Db interface implementation. func (db *MemDb) UpdateAddrIndexForBlock(*btcwire.ShaHash, int64, - btcdb.BlockAddrIndex) error { - return btcdb.ErrNotImplemented + database.BlockAddrIndex) error { + return database.ErrNotImplemented } -// FetchTxsForAddr isn't currently implemented. This is a part of the btcdb.Db +// FetchTxsForAddr isn't currently implemented. This is a part of the database.Db // interface implementation. -func (db *MemDb) FetchTxsForAddr(btcutil.Address, int, int) ([]*btcdb.TxListReply, error) { - return nil, btcdb.ErrNotImplemented +func (db *MemDb) FetchTxsForAddr(btcutil.Address, int, int) ([]*database.TxListReply, error) { + return nil, database.ErrNotImplemented } -// DeleteAddrIndex isn't currently implemented. This is a part of the btcdb.Db +// DeleteAddrIndex isn't currently implemented. This is a part of the database.Db // interface implementation. func (db *MemDb) DeleteAddrIndex() error { - return btcdb.ErrNotImplemented + return database.ErrNotImplemented } // RollbackClose discards the recent database changes to the previously saved -// data at last Sync and closes the database. This is part of the btcdb.Db +// data at last Sync and closes the database. This is part of the database.Db // interface implementation. // // The database is completely purged on close with this implementation since the @@ -737,7 +737,7 @@ func (db *MemDb) RollbackClose() error { } // Sync verifies that the database is coherent on disk and no outstanding -// transactions are in flight. This is part of the btcdb.Db interface +// transactions are in flight. This is part of the database.Db interface // implementation. // // This implementation does not write any data to disk, so this function only diff --git a/memdb/memdb_test.go b/database/memdb/memdb_test.go similarity index 94% rename from memdb/memdb_test.go rename to database/memdb/memdb_test.go index de8701b3..f370fa55 100644 --- a/memdb/memdb_test.go +++ b/database/memdb/memdb_test.go @@ -8,8 +8,8 @@ import ( "reflect" "testing" - "github.com/btcsuite/btcdb" - "github.com/btcsuite/btcdb/memdb" + "github.com/btcsuite/btcd/database" + "github.com/btcsuite/btcd/database/memdb" "github.com/btcsuite/btcnet" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcwire" @@ -20,7 +20,7 @@ import ( // and does not panic or otherwise misbehave for functions which do not return // errors. func TestClosed(t *testing.T) { - db, err := btcdb.CreateDB("memdb") + db, err := database.CreateDB("memdb") if err != nil { t.Errorf("Failed to open test database %v", err) return @@ -74,7 +74,7 @@ func TestClosed(t *testing.T) { "got: %d, want: %d", len(reply), len(requestHashes)) } for i, txLR := range reply { - wantReply := &btcdb.TxListReply{ + wantReply := &database.TxListReply{ Sha: requestHashes[i], Err: memdb.ErrDbClosed, } @@ -90,7 +90,7 @@ func TestClosed(t *testing.T) { "got: %d, want: %d", len(reply), len(requestHashes)) } for i, txLR := range reply { - wantReply := &btcdb.TxListReply{ + wantReply := &database.TxListReply{ Sha: requestHashes[i], Err: memdb.ErrDbClosed, } diff --git a/testdata/blocks1-256.bz2 b/database/testdata/blocks1-256.bz2 similarity index 100% rename from testdata/blocks1-256.bz2 rename to database/testdata/blocks1-256.bz2 diff --git a/doc.go b/doc.go deleted file mode 100644 index 6e529ce4..00000000 --- a/doc.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) 2013-2014 Conformal Systems LLC. -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -/* -Package btcdb provides a database interface for the Bitcoin block chain. - -As of July 2014, there are over 309,000 blocks in the Bitcoin block chain and -and over 42 million transactions (which turns out to be over 21GB of data). -btcdb provides a database layer to store and retrieve this data in a fairly -simple and efficient manner. The use of this should not require specific -knowledge of the database backend. - -Basic Design - -The basic design of btcdb is to provide two classes of items in a database; -blocks and transactions (tx) where the block number increases monotonically. -Each transaction belongs to a single block although a block can have a variable -number of transactions. Along with these two items, several convenience -functions for dealing with the database are provided as well as functions to -query specific items that may be present in a block or tx. - -Usage - -At the highest level, the use of this packages just requires that you import it, -setup a database, insert some data into it, and optionally, query the data back. -The first block inserted into the database will be treated as the genesis block. -Every subsequent block insert requires the referenced parent block to already -exist. -*/ -package btcdb