From 28d4261c2d336ac097aaf25761bd35d6fad80b0f Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Fri, 6 Jul 2018 15:38:44 -0700 Subject: [PATCH 1/8] build: update glide for btcutil w/ latest BIP 158 --- glide.lock | 6 +++--- glide.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/glide.lock b/glide.lock index fb1bf674..c101cf19 100644 --- a/glide.lock +++ b/glide.lock @@ -1,12 +1,12 @@ -hash: c8cc08c3afbedc99feda4fed38a64bb1aa42da128520dcac028cfc7e7de06a6e -updated: 2018-05-23T20:46:41.891755246-07:00 +hash: 30562f0035d9bb4d4ee531762e4f7f762ede916255e9aa781df58537651f7382 +updated: 2018-07-06T16:07:52.877260592-07:00 imports: - name: github.com/aead/siphash version: e404fcfc888570cadd1610538e2dbc89f66af814 - name: github.com/btcsuite/btclog version: 84c8d2346e9fc8c7b947e243b9c24e6df9fd206a - name: github.com/btcsuite/btcutil - version: d4cc87b860166d00d6b5b9e0d3b3d71d6088d4d4 + version: ab6388e0c60ae4834a1f57511e20c17b5f78be4b subpackages: - base58 - bech32 diff --git a/glide.yaml b/glide.yaml index 59dd550e..55461004 100644 --- a/glide.yaml +++ b/glide.yaml @@ -2,7 +2,7 @@ package: github.com/btcsuite/btcd import: - package: github.com/btcsuite/btclog - package: github.com/btcsuite/btcutil - version: d4cc87b860166d00d6b5b9e0d3b3d71d6088d4d4 + version: ab6388e0c60ae4834a1f57511e20c17b5f78be4b subpackages: - bloom - hdkeychain From 66d632f32e76580a24bb0cd569fcebab44d45468 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Wed, 30 May 2018 20:54:23 -0700 Subject: [PATCH 2/8] wire: remove the extended filter --- wire/msgcfilter.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/wire/msgcfilter.go b/wire/msgcfilter.go index 7387587f..097590b2 100644 --- a/wire/msgcfilter.go +++ b/wire/msgcfilter.go @@ -17,9 +17,6 @@ type FilterType uint8 const ( // GCSFilterRegular is the regular filter type. GCSFilterRegular FilterType = iota - - // GCSFilterExtended is the extended filter type. - GCSFilterExtended ) const ( From 942116c5ae393582e56df918a23b76b7511ded71 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Wed, 30 May 2018 20:54:34 -0700 Subject: [PATCH 3/8] rpc: remove extended fitler from help --- rpcserverhelp.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rpcserverhelp.go b/rpcserverhelp.go index 489e9a87..c875d217 100644 --- a/rpcserverhelp.go +++ b/rpcserverhelp.go @@ -334,13 +334,13 @@ var helpDescsEnUS = map[string]string{ // GetCFilterCmd help. "getcfilter--synopsis": "Returns a block's committed filter given its hash.", - "getcfilter-filtertype": "The type of filter to return (0=regular, 1=extended)", + "getcfilter-filtertype": "The type of filter to return (0=regular)", "getcfilter-hash": "The hash of the block", "getcfilter--result0": "The block's committed filter", // GetCFilterHeaderCmd help. "getcfilterheader--synopsis": "Returns a block's compact filter header given its hash.", - "getcfilterheader-filtertype": "The type of filter header to return (0=regular, 1=extended)", + "getcfilterheader-filtertype": "The type of filter header to return (0=regular)", "getcfilterheader-hash": "The hash of the block", "getcfilterheader--result0": "The block's gcs filter header", From 576800a99ec8e1fdeca94bfece6a760a09fc87a7 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Wed, 30 May 2018 20:54:56 -0700 Subject: [PATCH 4/8] blockchain/indexer: remove extended filter --- blockchain/indexers/cfindex.go | 45 ++++++++++++---------------------- 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/blockchain/indexers/cfindex.go b/blockchain/indexers/cfindex.go index 5caee7fd..c16c3a08 100644 --- a/blockchain/indexers/cfindex.go +++ b/blockchain/indexers/cfindex.go @@ -22,39 +22,36 @@ const ( cfIndexName = "committed filter index" ) -// Committed filters come in two flavours: basic and extended. They are -// generated and dropped in pairs, and both are indexed by a block's hash. -// Besides holding different content, they also live in different buckets. +// Committed filters come in one flavor currently: basic. They are generated +// and dropped in pairs, and both are indexed by a block's hash. Besides +// holding different content, they also live in different buckets. var ( - // cfIndexParentBucketKey is the name of the parent bucket used to house - // the index. The rest of the buckets live below this bucket. + // cfIndexParentBucketKey is the name of the parent bucket used to + // house the index. The rest of the buckets live below this bucket. cfIndexParentBucketKey = []byte("cfindexparentbucket") // cfIndexKeys is an array of db bucket names used to house indexes of // block hashes to cfilters. cfIndexKeys = [][]byte{ []byte("cf0byhashidx"), - []byte("cf1byhashidx"), } // cfHeaderKeys is an array of db bucket names used to house indexes of // block hashes to cf headers. cfHeaderKeys = [][]byte{ []byte("cf0headerbyhashidx"), - []byte("cf1headerbyhashidx"), } // cfHashKeys is an array of db bucket names used to house indexes of // block hashes to cf hashes. cfHashKeys = [][]byte{ []byte("cf0hashbyhashidx"), - []byte("cf1hashbyhashidx"), } maxFilterType = uint8(len(cfHeaderKeys) - 1) - // zeroHash is the chainhash.Hash value of all zero bytes, defined here for - // convenience. + // zeroHash is the chainhash.Hash value of all zero bytes, defined here + // for convenience. zeroHash chainhash.Hash ) @@ -106,7 +103,7 @@ func (idx *CfIndex) Name() string { // Create is invoked when the indexer manager determines the index needs to // be created for the first time. It creates buckets for the two hash-based cf -// indexes (simple, extended). +// indexes (regular only currently). func (idx *CfIndex) Create(dbTx database.Tx) error { meta := dbTx.Metadata() @@ -209,17 +206,7 @@ func (idx *CfIndex) ConnectBlock(dbTx database.Tx, block *btcutil.Block, return err } - err = storeFilter(dbTx, block, f, wire.GCSFilterRegular) - if err != nil { - return err - } - - f, err = builder.BuildExtFilter(block.MsgBlock()) - if err != nil { - return err - } - - return storeFilter(dbTx, block, f, wire.GCSFilterExtended) + return storeFilter(dbTx, block, f, wire.GCSFilterRegular) } // DisconnectBlock is invoked by the index manager when a block has been @@ -296,42 +283,42 @@ func (idx *CfIndex) entriesByBlockHashes(filterTypeKeys [][]byte, } // FilterByBlockHash returns the serialized contents of a block's basic or -// extended committed filter. +// committed filter. func (idx *CfIndex) FilterByBlockHash(h *chainhash.Hash, filterType wire.FilterType) ([]byte, error) { return idx.entryByBlockHash(cfIndexKeys, filterType, h) } // FiltersByBlockHashes returns the serialized contents of a block's basic or -// extended committed filter for a set of blocks by hash. +// committed filter for a set of blocks by hash. func (idx *CfIndex) FiltersByBlockHashes(blockHashes []*chainhash.Hash, filterType wire.FilterType) ([][]byte, error) { return idx.entriesByBlockHashes(cfIndexKeys, filterType, blockHashes) } // FilterHeaderByBlockHash returns the serialized contents of a block's basic -// or extended committed filter header. +// committed filter header. func (idx *CfIndex) FilterHeaderByBlockHash(h *chainhash.Hash, filterType wire.FilterType) ([]byte, error) { return idx.entryByBlockHash(cfHeaderKeys, filterType, h) } -// FilterHeadersByBlockHashes returns the serialized contents of a block's basic -// or extended committed filter header for a set of blocks by hash. +// FilterHeadersByBlockHashes returns the serialized contents of a block's +// basic committed filter header for a set of blocks by hash. func (idx *CfIndex) FilterHeadersByBlockHashes(blockHashes []*chainhash.Hash, filterType wire.FilterType) ([][]byte, error) { return idx.entriesByBlockHashes(cfHeaderKeys, filterType, blockHashes) } // FilterHashByBlockHash returns the serialized contents of a block's basic -// or extended committed filter hash. +// committed filter hash. func (idx *CfIndex) FilterHashByBlockHash(h *chainhash.Hash, filterType wire.FilterType) ([]byte, error) { return idx.entryByBlockHash(cfHashKeys, filterType, h) } // FilterHashesByBlockHashes returns the serialized contents of a block's basic -// or extended committed filter hash for a set of blocks by hash. +// committed filter hash for a set of blocks by hash. func (idx *CfIndex) FilterHashesByBlockHashes(blockHashes []*chainhash.Hash, filterType wire.FilterType) ([][]byte, error) { return idx.entriesByBlockHashes(cfHashKeys, filterType, blockHashes) From 102ca293f6cbdbda19c21697c25b66febc1a1bb9 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Thu, 14 Jun 2018 17:50:39 -0700 Subject: [PATCH 5/8] blockchain: pass in prevScript for constructing the basic filter --- blockchain/indexers/cfindex.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/blockchain/indexers/cfindex.go b/blockchain/indexers/cfindex.go index c16c3a08..f8081be8 100644 --- a/blockchain/indexers/cfindex.go +++ b/blockchain/indexers/cfindex.go @@ -201,7 +201,12 @@ func storeFilter(dbTx database.Tx, block *btcutil.Block, f *gcs.Filter, func (idx *CfIndex) ConnectBlock(dbTx database.Tx, block *btcutil.Block, stxos []blockchain.SpentTxOut) error { - f, err := builder.BuildBasicFilter(block.MsgBlock()) + prevScripts := make([][]byte, len(stxos)) + for i, stxo := range stxos { + prevScripts[i] = stxo.PkScript + } + + f, err := builder.BuildBasicFilter(block.MsgBlock(), prevScripts) if err != nil { return err } From 298efd83591d109d672e11d93bc27b962a0e0a8b Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Fri, 6 Jul 2018 12:38:17 -0700 Subject: [PATCH 6/8] blockchain/indexers: ensure the cfindex gets inputs --- blockchain/indexers/cfindex.go | 11 +++++++++++ blockchain/indexers/manager.go | 36 ---------------------------------- 2 files changed, 11 insertions(+), 36 deletions(-) diff --git a/blockchain/indexers/cfindex.go b/blockchain/indexers/cfindex.go index f8081be8..8ea14728 100644 --- a/blockchain/indexers/cfindex.go +++ b/blockchain/indexers/cfindex.go @@ -83,6 +83,17 @@ type CfIndex struct { // Ensure the CfIndex type implements the Indexer interface. var _ Indexer = (*CfIndex)(nil) +// Ensure the CfIndex type implements the NeedsInputser interface. +var _ NeedsInputser = (*CfIndex)(nil) + +// NeedsInputs signals that the index requires the referenced inputs in order +// to properly create the index. +// +// This implements the NeedsInputser interface. +func (idx *CfIndex) NeedsInputs() bool { + return true +} + // Init initializes the hash-based cf index. This is part of the Indexer // interface. func (idx *CfIndex) Init() error { diff --git a/blockchain/indexers/manager.go b/blockchain/indexers/manager.go index cf6fbd96..09dae621 100644 --- a/blockchain/indexers/manager.go +++ b/blockchain/indexers/manager.go @@ -494,42 +494,6 @@ func dbFetchTx(dbTx database.Tx, hash *chainhash.Hash) (*wire.MsgTx, error) { return &msgTx, nil } -// makeUtxoView creates a mock unspent transaction output view by using the -// transaction index in order to look up all inputs referenced by the -// transactions in the block. This is sometimes needed when catching indexes up -// because many of the txouts could actually already be spent however the -// associated scripts are still required to index them. -func makeUtxoView(dbTx database.Tx, block *btcutil.Block, interrupt <-chan struct{}) (*blockchain.UtxoViewpoint, error) { - view := blockchain.NewUtxoViewpoint() - for txIdx, tx := range block.Transactions() { - // Coinbases do not reference any inputs. Since the block is - // required to have already gone through full validation, it has - // already been proven on the first transaction in the block is - // a coinbase. - if txIdx == 0 { - continue - } - - // Use the transaction index to load all of the referenced - // inputs and add their outputs to the view. - for _, txIn := range tx.MsgTx().TxIn { - originOut := &txIn.PreviousOutPoint - originTx, err := dbFetchTx(dbTx, &originOut.Hash) - if err != nil { - return nil, err - } - - view.AddTxOuts(btcutil.NewTx(originTx), 0) - } - - if interruptRequested(interrupt) { - return nil, errInterruptRequested - } - } - - return view, nil -} - // ConnectBlock must be invoked when a block is extending the main chain. It // keeps track of the state of each index it is managing, performs some sanity // checks, and invokes each indexer. From a05f62fabdec985789817f1c1be1223cd9195531 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Fri, 6 Jul 2018 15:47:02 -0700 Subject: [PATCH 7/8] btcjson: remove instances of the extended gcs filter --- btcjson/chainsvrcmds_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/btcjson/chainsvrcmds_test.go b/btcjson/chainsvrcmds_test.go index 0b3c7b7c..8cb4ee76 100644 --- a/btcjson/chainsvrcmds_test.go +++ b/btcjson/chainsvrcmds_test.go @@ -322,32 +322,32 @@ func TestChainSvrCmds(t *testing.T) { name: "getcfilter", newCmd: func() (interface{}, error) { return btcjson.NewCmd("getcfilter", "123", - wire.GCSFilterExtended) + wire.GCSFilterRegular) }, staticCmd: func() interface{} { return btcjson.NewGetCFilterCmd("123", - wire.GCSFilterExtended) + wire.GCSFilterRegular) }, - marshalled: `{"jsonrpc":"1.0","method":"getcfilter","params":["123",1],"id":1}`, + marshalled: `{"jsonrpc":"1.0","method":"getcfilter","params":["123",0],"id":1}`, unmarshalled: &btcjson.GetCFilterCmd{ Hash: "123", - FilterType: wire.GCSFilterExtended, + FilterType: wire.GCSFilterRegular, }, }, { name: "getcfilterheader", newCmd: func() (interface{}, error) { return btcjson.NewCmd("getcfilterheader", "123", - wire.GCSFilterExtended) + wire.GCSFilterRegular) }, staticCmd: func() interface{} { return btcjson.NewGetCFilterHeaderCmd("123", - wire.GCSFilterExtended) + wire.GCSFilterRegular) }, - marshalled: `{"jsonrpc":"1.0","method":"getcfilterheader","params":["123",1],"id":1}`, + marshalled: `{"jsonrpc":"1.0","method":"getcfilterheader","params":["123",0],"id":1}`, unmarshalled: &btcjson.GetCFilterHeaderCmd{ Hash: "123", - FilterType: wire.GCSFilterExtended, + FilterType: wire.GCSFilterRegular, }, }, { From 21b0303341865fea274e53557af44e2f21ec5f81 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Fri, 6 Jul 2018 15:47:15 -0700 Subject: [PATCH 8/8] wire: remove instances of the extended gcs filter --- wire/message_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/wire/message_test.go b/wire/message_test.go index a174c49a..3a422e66 100644 --- a/wire/message_test.go +++ b/wire/message_test.go @@ -69,13 +69,13 @@ func TestMessage(t *testing.T) { bh := NewBlockHeader(1, &chainhash.Hash{}, &chainhash.Hash{}, 0, 0) msgMerkleBlock := NewMsgMerkleBlock(bh) msgReject := NewMsgReject("block", RejectDuplicate, "duplicate block") - msgGetCFilters := NewMsgGetCFilters(GCSFilterExtended, 0, &chainhash.Hash{}) - msgGetCFHeaders := NewMsgGetCFHeaders(GCSFilterExtended, 0, &chainhash.Hash{}) - msgGetCFCheckpt := NewMsgGetCFCheckpt(GCSFilterExtended, &chainhash.Hash{}) - msgCFilter := NewMsgCFilter(GCSFilterExtended, &chainhash.Hash{}, + msgGetCFilters := NewMsgGetCFilters(GCSFilterRegular, 0, &chainhash.Hash{}) + msgGetCFHeaders := NewMsgGetCFHeaders(GCSFilterRegular, 0, &chainhash.Hash{}) + msgGetCFCheckpt := NewMsgGetCFCheckpt(GCSFilterRegular, &chainhash.Hash{}) + msgCFilter := NewMsgCFilter(GCSFilterRegular, &chainhash.Hash{}, []byte("payload")) msgCFHeaders := NewMsgCFHeaders() - msgCFCheckpt := NewMsgCFCheckpt(GCSFilterExtended, &chainhash.Hash{}, 0) + msgCFCheckpt := NewMsgCFCheckpt(GCSFilterRegular, &chainhash.Hash{}, 0) tests := []struct { in Message // Value to encode