From ae78470b839064734be67989c31f2c10d7a829bb Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 27 Oct 2020 09:58:56 -0700 Subject: [PATCH 01/18] First pass at tracking balance --- indexer/balance_storage_handler.go | 31 ++++++++++++++++++++ indexer/balance_storage_helper.go | 47 ++++++++++++++++++++++++++++++ indexer/indexer.go | 9 +++++- 3 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 indexer/balance_storage_handler.go create mode 100644 indexer/balance_storage_helper.go diff --git a/indexer/balance_storage_handler.go b/indexer/balance_storage_handler.go new file mode 100644 index 0000000..952753e --- /dev/null +++ b/indexer/balance_storage_handler.go @@ -0,0 +1,31 @@ +package indexer + +import ( + "context" + + "github.com/coinbase/rosetta-sdk-go/parser" + "github.com/coinbase/rosetta-sdk-go/storage" + "github.com/coinbase/rosetta-sdk-go/types" +) + +var _ storage.BalanceStorageHandler = (*BalanceStorageHandler)(nil) + +type BalanceStorageHandler struct{} + +// BlockAdded is called whenever a block is committed to BlockStorage. +func (h *BalanceStorageHandler) BlockAdded( + ctx context.Context, + block *types.Block, + changes []*parser.BalanceChange, +) error { + return nil +} + +// BlockRemoved is called whenever a block is removed from BlockStorage. +func (h *BalanceStorageHandler) BlockRemoved( + ctx context.Context, + block *types.Block, + changes []*parser.BalanceChange, +) error { + return nil +} diff --git a/indexer/balance_storage_helper.go b/indexer/balance_storage_helper.go new file mode 100644 index 0000000..fc1e0ca --- /dev/null +++ b/indexer/balance_storage_helper.go @@ -0,0 +1,47 @@ +package indexer + +import ( + "context" + + "github.com/coinbase/rosetta-sdk-go/asserter" + "github.com/coinbase/rosetta-sdk-go/parser" + "github.com/coinbase/rosetta-sdk-go/storage" + "github.com/coinbase/rosetta-sdk-go/types" +) + +var _ storage.BalanceStorageHelper = (*BalanceStorageHelper)(nil) + +type BalanceStorageHelper struct { + a *asserter.Asserter +} + +// AccountBalance attempts to fetch the balance +// for a missing account in storage. +func (h *BalanceStorageHelper) AccountBalance( + ctx context.Context, + account *types.AccountIdentifier, + currency *types.Currency, + block *types.BlockIdentifier, +) (*types.Amount, error) { + return &types.Amount{ + Value: "0", + Currency: currency, + }, nil +} + +// Asserter returns a *asserter.Asserter. +func (h *BalanceStorageHelper) Asserter() *asserter.Asserter { + return h.a +} + +// BalanceExemptions returns a list of *types.BalanceExemption. +func (h *BalanceStorageHelper) BalanceExemptions() []*types.BalanceExemption { + return []*types.BalanceExemption{} +} + +// ExemptFunc returns a parser.ExemptOperation. +func (h *BalanceStorageHelper) ExemptFunc() parser.ExemptOperation { + return func(op *types.Operation) bool { + return false + } +} diff --git a/indexer/indexer.go b/indexer/indexer.go index 3f33757..3c7d04b 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -199,7 +199,14 @@ func Initialize( coinStorage := storage.NewCoinStorage(localStore, i, asserter) i.coinStorage = coinStorage - i.workers = []storage.BlockWorker{coinStorage} + + balanceStorage := storage.NewBalanceStorage(localStore) + balanceStorage.Initialize( + &BalanceStorageHelper{asserter}, + &BalanceStorageHandler{}, + ) + + i.workers = []storage.BlockWorker{coinStorage, balanceStorage} return i, nil } From 272db602e455c96bae9aa5f22b10faf71253a138 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 27 Oct 2020 10:02:28 -0700 Subject: [PATCH 02/18] Break apart coin storage helper --- indexer/coin_storage_helper.go | 23 +++++++++++++++++++++++ indexer/indexer.go | 16 +++++----------- 2 files changed, 28 insertions(+), 11 deletions(-) create mode 100644 indexer/coin_storage_helper.go diff --git a/indexer/coin_storage_helper.go b/indexer/coin_storage_helper.go new file mode 100644 index 0000000..90f407e --- /dev/null +++ b/indexer/coin_storage_helper.go @@ -0,0 +1,23 @@ +package indexer + +import ( + "context" + + "github.com/coinbase/rosetta-sdk-go/storage" + "github.com/coinbase/rosetta-sdk-go/types" +) + +var _ storage.CoinStorageHelper = (*CoinStorageHelper)(nil) + +type CoinStorageHelper struct { + b *storage.BlockStorage +} + +// CurrentBlockIdentifier returns the current head block identifier +// and is used to comply with the CoinStorageHelper interface. +func (h *CoinStorageHelper) CurrentBlockIdentifier( + ctx context.Context, + transaction storage.DatabaseTransaction, +) (*types.BlockIdentifier, error) { + return h.b.GetHeadBlockIdentifierTransactional(ctx, transaction) +} diff --git a/indexer/indexer.go b/indexer/indexer.go index 3c7d04b..76fa0b1 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -74,7 +74,6 @@ type Client interface { var _ syncer.Handler = (*Indexer)(nil) var _ syncer.Helper = (*Indexer)(nil) var _ services.Indexer = (*Indexer)(nil) -var _ storage.CoinStorageHelper = (*Indexer)(nil) // Indexer caches blocks and provides balance query functionality. type Indexer struct { @@ -197,7 +196,11 @@ func Initialize( asserter: asserter, } - coinStorage := storage.NewCoinStorage(localStore, i, asserter) + coinStorage := storage.NewCoinStorage( + localStore, + &CoinStorageHelper{blockStorage}, + asserter, + ) i.coinStorage = coinStorage balanceStorage := storage.NewBalanceStorage(localStore) @@ -765,12 +768,3 @@ func (i *Indexer) GetCoins( ) ([]*types.Coin, *types.BlockIdentifier, error) { return i.coinStorage.GetCoins(ctx, accountIdentifier) } - -// CurrentBlockIdentifier returns the current head block identifier -// and is used to comply with the CoinStorageHelper interface. -func (i *Indexer) CurrentBlockIdentifier( - ctx context.Context, - transaction storage.DatabaseTransaction, -) (*types.BlockIdentifier, error) { - return i.blockStorage.GetHeadBlockIdentifierTransactional(ctx, transaction) -} From 2200f52687d47a5c5ebd7141525d7adace88ac71 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 27 Oct 2020 10:25:02 -0700 Subject: [PATCH 03/18] First pass at GetBalance --- indexer/indexer.go | 42 +++++++++++++++++++++++++----- main.go | 2 +- services/account_service.go | 52 +++++++++++++++++++++++++++---------- services/errors.go | 18 ++++++++++--- services/types.go | 11 +++++++- 5 files changed, 99 insertions(+), 26 deletions(-) diff --git a/indexer/indexer.go b/indexer/indexer.go index 76fa0b1..60289d2 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -84,11 +84,12 @@ type Indexer struct { client Client - asserter *asserter.Asserter - database storage.Database - blockStorage *storage.BlockStorage - coinStorage *storage.CoinStorage - workers []storage.BlockWorker + asserter *asserter.Asserter + database storage.Database + blockStorage *storage.BlockStorage + balanceStorage *storage.BalanceStorage + coinStorage *storage.CoinStorage + workers []storage.BlockWorker waiter *waitTable } @@ -208,6 +209,7 @@ func Initialize( &BalanceStorageHelper{asserter}, &BalanceStorageHandler{}, ) + i.balanceStorage = balanceStorage i.workers = []storage.BlockWorker{coinStorage, balanceStorage} @@ -758,7 +760,11 @@ func (i *Indexer) GetBlockTransaction( blockIdentifier *types.BlockIdentifier, transactionIdentifier *types.TransactionIdentifier, ) (*types.Transaction, error) { - return i.blockStorage.GetBlockTransaction(ctx, blockIdentifier, transactionIdentifier) + return i.blockStorage.GetBlockTransaction( + ctx, + blockIdentifier, + transactionIdentifier, + ) } // GetCoins returns all unspent coins for a particular *types.AccountIdentifier. @@ -768,3 +774,27 @@ func (i *Indexer) GetCoins( ) ([]*types.Coin, *types.BlockIdentifier, error) { return i.coinStorage.GetCoins(ctx, accountIdentifier) } + +func (i *Indexer) GetBalance( + ctx context.Context, + accountIdentifier *types.AccountIdentifier, + currency *types.Currency, + blockIdentifier *types.PartialBlockIdentifier, +) (*types.Amount, *types.BlockIdentifier, error) { + blockResponse, err := i.GetBlockLazy(ctx, blockIdentifier) + if err != nil { + return nil, nil, err + } + + amount, err := i.balanceStorage.GetBalance( + ctx, + accountIdentifier, + currency, + blockResponse.Block.BlockIdentifier, + ) + if err != nil { + return nil, nil, err + } + + return amount, blockResponse.Block.BlockIdentifier, nil +} diff --git a/main.go b/main.go index 6886f69..8271f45 100644 --- a/main.go +++ b/main.go @@ -154,7 +154,7 @@ func main() { // requests. asserter, err := asserter.NewServer( bitcoin.OperationTypes, - false, + true, []*types.NetworkIdentifier{cfg.Network}, nil, ) diff --git a/services/account_service.go b/services/account_service.go index 04a786f..ce7d72b 100644 --- a/services/account_service.go +++ b/services/account_service.go @@ -49,27 +49,51 @@ func (s *AccountAPIService) AccountBalance( return nil, wrapErr(ErrUnavailableOffline, nil) } - coins, block, err := s.i.GetCoins(ctx, request.AccountIdentifier) - if err != nil { - return nil, wrapErr(ErrUnableToGetCoins, err) + // If we are fetching the current balance, + // return all coins for an address and calculate + // the balance from those coins. + if request.BlockIdentifier == nil { + coins, block, err := s.i.GetCoins(ctx, request.AccountIdentifier) + if err != nil { + return nil, wrapErr(ErrUnableToGetCoins, err) + } + + balance := "0" + for _, coin := range coins { + balance, err = types.AddValues(balance, coin.Amount.Value) + if err != nil { + return nil, wrapErr(ErrUnableToParseIntermediateResult, err) + } + } + + return &types.AccountBalanceResponse{ + BlockIdentifier: block, + Coins: coins, + Balances: []*types.Amount{ + { + Value: balance, + Currency: s.config.Currency, + }, + }, + }, nil } - balance := "0" - for _, coin := range coins { - balance, err = types.AddValues(balance, coin.Amount.Value) - if err != nil { - return nil, wrapErr(ErrUnableToParseIntermediateResult, err) - } + // If we are fetching a historical balance, + // use balance storage and don't return coins. + amount, block, err := s.i.GetBalance( + ctx, + request.AccountIdentifier, + s.config.Currency, + request.BlockIdentifier, + ) + if err != nil { + return nil, wrapErr(ErrUnableToGetBalance, err) } return &types.AccountBalanceResponse{ BlockIdentifier: block, - Coins: coins, Balances: []*types.Amount{ - { - Value: balance, - Currency: s.config.Currency, - }, + amount, }, }, nil } diff --git a/services/errors.go b/services/errors.go index 43c6ad5..85ac75f 100644 --- a/services/errors.go +++ b/services/errors.go @@ -40,6 +40,7 @@ var ( ErrUnableToGetCoins, ErrTransactionNotFound, ErrCouldNotGetFeeRate, + ErrUnableToGetBalance, } // ErrUnimplemented is returned when an endpoint @@ -59,8 +60,9 @@ var ( // ErrNotReady is returned when bitcoind is not // yet ready to serve queries. ErrNotReady = &types.Error{ - Code: 2, //nolint - Message: "Bitcoind is not ready", + Code: 2, //nolint + Message: "Bitcoind is not ready", + Retriable: true, } // ErrBitcoind is returned when bitcoind @@ -173,6 +175,14 @@ var ( Code: 17, // nolint Message: "Could not get suggested fee rate", } + + // ErrUnableToGetBalance is returned by the indexer + // when it is not possible to get the balance + // of a *types.AccountIdentifier. + ErrUnableToGetBalance = &types.Error{ + Code: 18, //nolint + Message: "Unable to get balance", + } ) // wrapErr adds details to the types.Error provided. We use a function @@ -180,8 +190,8 @@ var ( // errors. func wrapErr(rErr *types.Error, err error) *types.Error { newErr := &types.Error{ - Code: rErr.Code, - Message: rErr.Message, + Code: rErr.Code, + Message: rErr.Message, Retriable: rErr.Retriable, } if err != nil { diff --git a/services/types.go b/services/types.go index f147d32..75c7620 100644 --- a/services/types.go +++ b/services/types.go @@ -48,7 +48,10 @@ type Client interface { // Indexer is used by the servicers to get block and account data. type Indexer interface { - GetBlockLazy(context.Context, *types.PartialBlockIdentifier) (*types.BlockResponse, error) + GetBlockLazy( + context.Context, + *types.PartialBlockIdentifier, + ) (*types.BlockResponse, error) GetBlockTransaction( context.Context, *types.BlockIdentifier, @@ -62,6 +65,12 @@ type Indexer interface { context.Context, []*types.Coin, ) ([]*bitcoin.ScriptPubKey, error) + GetBalance( + context.Context, + *types.AccountIdentifier, + *types.Currency, + *types.PartialBlockIdentifier, + ) (*types.Amount, *types.BlockIdentifier, error) } type unsignedTransaction struct { From c2b434e04a683b354b2f9521d310632b48cf1e0c Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 27 Oct 2020 10:25:57 -0700 Subject: [PATCH 04/18] nits --- indexer/indexer.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/indexer/indexer.go b/indexer/indexer.go index 60289d2..3be3b3f 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -775,6 +775,8 @@ func (i *Indexer) GetCoins( return i.coinStorage.GetCoins(ctx, accountIdentifier) } +// GetBalance returns the balance of an account +// at a particular *types.PartialBlockIdentifier. func (i *Indexer) GetBalance( ctx context.Context, accountIdentifier *types.AccountIdentifier, From 6bb3855c17734caf29ada5f3a759b6da5971c749 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 27 Oct 2020 10:26:31 -0700 Subject: [PATCH 05/18] Update mocks --- indexer/balance_storage_handler.go | 14 +++++++++++++ indexer/balance_storage_helper.go | 14 +++++++++++++ indexer/coin_storage_helper.go | 14 +++++++++++++ mocks/services/indexer.go | 32 ++++++++++++++++++++++++++++++ 4 files changed, 74 insertions(+) diff --git a/indexer/balance_storage_handler.go b/indexer/balance_storage_handler.go index 952753e..a7d768c 100644 --- a/indexer/balance_storage_handler.go +++ b/indexer/balance_storage_handler.go @@ -1,3 +1,17 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package indexer import ( diff --git a/indexer/balance_storage_helper.go b/indexer/balance_storage_helper.go index fc1e0ca..c0445d9 100644 --- a/indexer/balance_storage_helper.go +++ b/indexer/balance_storage_helper.go @@ -1,3 +1,17 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package indexer import ( diff --git a/indexer/coin_storage_helper.go b/indexer/coin_storage_helper.go index 90f407e..254e58b 100644 --- a/indexer/coin_storage_helper.go +++ b/indexer/coin_storage_helper.go @@ -1,3 +1,17 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package indexer import ( diff --git a/mocks/services/indexer.go b/mocks/services/indexer.go index a3c4711..749dbec 100644 --- a/mocks/services/indexer.go +++ b/mocks/services/indexer.go @@ -17,6 +17,38 @@ type Indexer struct { mock.Mock } +// GetBalance provides a mock function with given fields: _a0, _a1, _a2, _a3 +func (_m *Indexer) GetBalance(_a0 context.Context, _a1 *types.AccountIdentifier, _a2 *types.Currency, _a3 *types.PartialBlockIdentifier) (*types.Amount, *types.BlockIdentifier, error) { + ret := _m.Called(_a0, _a1, _a2, _a3) + + var r0 *types.Amount + if rf, ok := ret.Get(0).(func(context.Context, *types.AccountIdentifier, *types.Currency, *types.PartialBlockIdentifier) *types.Amount); ok { + r0 = rf(_a0, _a1, _a2, _a3) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.Amount) + } + } + + var r1 *types.BlockIdentifier + if rf, ok := ret.Get(1).(func(context.Context, *types.AccountIdentifier, *types.Currency, *types.PartialBlockIdentifier) *types.BlockIdentifier); ok { + r1 = rf(_a0, _a1, _a2, _a3) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(*types.BlockIdentifier) + } + } + + var r2 error + if rf, ok := ret.Get(2).(func(context.Context, *types.AccountIdentifier, *types.Currency, *types.PartialBlockIdentifier) error); ok { + r2 = rf(_a0, _a1, _a2, _a3) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + // GetBlockLazy provides a mock function with given fields: _a0, _a1 func (_m *Indexer) GetBlockLazy(_a0 context.Context, _a1 *types.PartialBlockIdentifier) (*types.BlockResponse, error) { ret := _m.Called(_a0, _a1) From a45bfe47c2167d2b8d6af6b2f6e68ff140c3632f Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 27 Oct 2020 10:29:15 -0700 Subject: [PATCH 06/18] Nits --- indexer/balance_storage_handler.go | 15 +-------------- indexer/balance_storage_helper.go | 1 + indexer/coin_storage_helper.go | 1 + 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/indexer/balance_storage_handler.go b/indexer/balance_storage_handler.go index a7d768c..da1b5dd 100644 --- a/indexer/balance_storage_handler.go +++ b/indexer/balance_storage_handler.go @@ -1,17 +1,3 @@ -// Copyright 2020 Coinbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package indexer import ( @@ -24,6 +10,7 @@ import ( var _ storage.BalanceStorageHandler = (*BalanceStorageHandler)(nil) +// BalanceStorageHandler implements storage.BalanceStorageHandler. type BalanceStorageHandler struct{} // BlockAdded is called whenever a block is committed to BlockStorage. diff --git a/indexer/balance_storage_helper.go b/indexer/balance_storage_helper.go index c0445d9..3600002 100644 --- a/indexer/balance_storage_helper.go +++ b/indexer/balance_storage_helper.go @@ -25,6 +25,7 @@ import ( var _ storage.BalanceStorageHelper = (*BalanceStorageHelper)(nil) +// BalanceStorageHelper implements storage.BalanceStorageHelper. type BalanceStorageHelper struct { a *asserter.Asserter } diff --git a/indexer/coin_storage_helper.go b/indexer/coin_storage_helper.go index 254e58b..36a7515 100644 --- a/indexer/coin_storage_helper.go +++ b/indexer/coin_storage_helper.go @@ -23,6 +23,7 @@ import ( var _ storage.CoinStorageHelper = (*CoinStorageHelper)(nil) +// CoinStorageHelper implements storage.CoinStorageHelper. type CoinStorageHelper struct { b *storage.BlockStorage } From fa7b34f90dfe6a1c03ec4e423199b1e93cd5fbec Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 27 Oct 2020 10:36:38 -0700 Subject: [PATCH 07/18] Add test for historical balances --- services/account_service_test.go | 47 +++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/services/account_service_test.go b/services/account_service_test.go index f50076f..ce94fb1 100644 --- a/services/account_service_test.go +++ b/services/account_service_test.go @@ -41,7 +41,7 @@ func TestAccountBalance_Offline(t *testing.T) { mockIndexer.AssertExpectations(t) } -func TestAccountBalance_Online(t *testing.T) { +func TestAccountBalance_Online_Current(t *testing.T) { cfg := &configuration.Configuration{ Mode: configuration.Online, Currency: bitcoin.MainnetCurrency, @@ -104,3 +104,48 @@ func TestAccountBalance_Online(t *testing.T) { mockIndexer.AssertExpectations(t) } + +func TestAccountBalance_Online_Historical(t *testing.T) { + cfg := &configuration.Configuration{ + Mode: configuration.Online, + Currency: bitcoin.MainnetCurrency, + } + mockIndexer := &mocks.Indexer{} + servicer := NewAccountAPIService(cfg, mockIndexer) + ctx := context.Background() + account := &types.AccountIdentifier{ + Address: "hello", + } + block := &types.BlockIdentifier{ + Index: 1000, + Hash: "block 1000", + } + partialBlock := &types.PartialBlockIdentifier{ + Index: &block.Index, + } + amount := &types.Amount{ + Value: "25", + Currency: bitcoin.MainnetCurrency, + } + + mockIndexer.On( + "GetBalance", + ctx, + account, + bitcoin.MainnetCurrency, + partialBlock, + ).Return(amount, block, nil).Once() + bal, err := servicer.AccountBalance(ctx, &types.AccountBalanceRequest{ + AccountIdentifier: account, + BlockIdentifier: partialBlock, + }) + assert.Nil(t, err) + assert.Equal(t, &types.AccountBalanceResponse{ + BlockIdentifier: block, + Balances: []*types.Amount{ + amount, + }, + }, bal) + + mockIndexer.AssertExpectations(t) +} From e64f4e0ee3437c03b3d7758faae83f0b9c97d19a Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 27 Oct 2020 10:37:47 -0700 Subject: [PATCH 08/18] Update version --- services/network_service_test.go | 2 +- services/types.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/network_service_test.go b/services/network_service_test.go index 11e0b78..4101b35 100644 --- a/services/network_service_test.go +++ b/services/network_service_test.go @@ -27,7 +27,7 @@ import ( ) var ( - middlewareVersion = "0.0.4" + middlewareVersion = "0.0.5" defaultNetworkOptions = &types.NetworkOptionsResponse{ Version: &types.Version{ RosettaVersion: types.RosettaAPIVersion, diff --git a/services/types.go b/services/types.go index 75c7620..e5b881e 100644 --- a/services/types.go +++ b/services/types.go @@ -34,7 +34,7 @@ var ( // variable instead of a constant because // we typically need the pointer of this // value. - MiddlewareVersion = "0.0.4" + MiddlewareVersion = "0.0.5" ) // Client is used by the servicers to get Peer information From 08d579b2c9f699391da50bba551484a028d5ab55 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 27 Oct 2020 10:39:45 -0700 Subject: [PATCH 09/18] Update dockerignore --- .dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.dockerignore b/.dockerignore index 2fab13c..5ded4fc 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,3 @@ rosetta-bitcoin bitcoin-data +cli-data From 58b468c901f032438bdc54476f3f741f225292d1 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 27 Oct 2020 10:40:20 -0700 Subject: [PATCH 10/18] Add license --- indexer/balance_storage_handler.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/indexer/balance_storage_handler.go b/indexer/balance_storage_handler.go index da1b5dd..52460a4 100644 --- a/indexer/balance_storage_handler.go +++ b/indexer/balance_storage_handler.go @@ -1,3 +1,17 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package indexer import ( From 5e9343a66233b3f1b14a5b5e8d38c94a917e7811 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 27 Oct 2020 10:44:57 -0700 Subject: [PATCH 11/18] Set historical balance lookup to true in services --- main.go | 2 +- services/network_service.go | 7 ++++--- services/network_service_test.go | 7 ++++--- services/types.go | 4 ++++ 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/main.go b/main.go index 8271f45..4b5d1c6 100644 --- a/main.go +++ b/main.go @@ -154,7 +154,7 @@ func main() { // requests. asserter, err := asserter.NewServer( bitcoin.OperationTypes, - true, + services.HistoricalBalanceLookup, []*types.NetworkIdentifier{cfg.Network}, nil, ) diff --git a/services/network_service.go b/services/network_service.go index 876c6d2..edeb721 100644 --- a/services/network_service.go +++ b/services/network_service.go @@ -95,9 +95,10 @@ func (s *NetworkAPIService) NetworkOptions( MiddlewareVersion: &MiddlewareVersion, }, Allow: &types.Allow{ - OperationStatuses: bitcoin.OperationStatuses, - OperationTypes: bitcoin.OperationTypes, - Errors: Errors, + OperationStatuses: bitcoin.OperationStatuses, + OperationTypes: bitcoin.OperationTypes, + Errors: Errors, + HistoricalBalanceLookup: HistoricalBalanceLookup, }, }, nil } diff --git a/services/network_service_test.go b/services/network_service_test.go index 4101b35..8854cbf 100644 --- a/services/network_service_test.go +++ b/services/network_service_test.go @@ -35,9 +35,10 @@ var ( MiddlewareVersion: &middlewareVersion, }, Allow: &types.Allow{ - OperationStatuses: bitcoin.OperationStatuses, - OperationTypes: bitcoin.OperationTypes, - Errors: Errors, + OperationStatuses: bitcoin.OperationStatuses, + OperationTypes: bitcoin.OperationTypes, + Errors: Errors, + HistoricalBalanceLookup: HistoricalBalanceLookup, }, } diff --git a/services/types.go b/services/types.go index e5b881e..b753d39 100644 --- a/services/types.go +++ b/services/types.go @@ -26,6 +26,10 @@ const ( // NodeVersion is the version of // bitcoin core we are using. NodeVersion = "0.20.1" + + // HistoricalBalanceLookup indicates + // that historical balance lookup is supported. + HistoricalBalanceLookup = true ) var ( From 9fa814a74af17a3d81b31c9ee1f463f84bec0047 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 27 Oct 2020 11:14:53 -0700 Subject: [PATCH 12/18] Return all transactions in a block --- rosetta-cli-conf/mainnet/config.json | 2 +- rosetta-cli-conf/testnet/config.json | 2 +- services/block_service.go | 33 +++++----- services/block_service_test.go | 90 ++++++++++++++++++---------- 4 files changed, 78 insertions(+), 49 deletions(-) diff --git a/rosetta-cli-conf/mainnet/config.json b/rosetta-cli-conf/mainnet/config.json index 2c0b8f5..69c25a1 100644 --- a/rosetta-cli-conf/mainnet/config.json +++ b/rosetta-cli-conf/mainnet/config.json @@ -7,7 +7,7 @@ "http_timeout": 300, "max_retries": 5, "retry_elapsed_time": 0, - "max_online_connections": 0, + "max_online_connections": 1000, "max_sync_concurrency": 0, "tip_delay": 1800, "log_configuration": false, diff --git a/rosetta-cli-conf/testnet/config.json b/rosetta-cli-conf/testnet/config.json index 7591bb5..ea1b75d 100644 --- a/rosetta-cli-conf/testnet/config.json +++ b/rosetta-cli-conf/testnet/config.json @@ -7,7 +7,7 @@ "http_timeout": 300, "max_retries": 5, "retry_elapsed_time": 0, - "max_online_connections": 0, + "max_online_connections": 1000, "max_sync_concurrency": 0, "tip_delay": 1800, "log_configuration": false, diff --git a/services/block_service.go b/services/block_service.go index 7eb76ed..b837765 100644 --- a/services/block_service.go +++ b/services/block_service.go @@ -54,6 +54,22 @@ func (s *BlockAPIService) Block( return nil, wrapErr(ErrBlockNotFound, err) } + txs := make([]*types.Transaction, len(blockResponse.OtherTransactions)) + for i, otherTx := range blockResponse.OtherTransactions { + transaction, err := s.i.GetBlockTransaction( + ctx, + blockResponse.Block.BlockIdentifier, + otherTx, + ) + if err != nil { + return nil, wrapErr(ErrTransactionNotFound, err) + } + + txs[i] = transaction + } + blockResponse.Block.Transactions = txs + + blockResponse.OtherTransactions = nil return blockResponse, nil } @@ -62,20 +78,5 @@ func (s *BlockAPIService) BlockTransaction( ctx context.Context, request *types.BlockTransactionRequest, ) (*types.BlockTransactionResponse, *types.Error) { - if s.config.Mode != configuration.Online { - return nil, wrapErr(ErrUnavailableOffline, nil) - } - - transaction, err := s.i.GetBlockTransaction( - ctx, - request.BlockIdentifier, - request.TransactionIdentifier, - ) - if err != nil { - return nil, wrapErr(ErrTransactionNotFound, err) - } - - return &types.BlockTransactionResponse{ - Transaction: transaction, - }, nil + return nil, ErrUnimplemented } diff --git a/services/block_service_test.go b/services/block_service_test.go index 9510e9f..8f7787b 100644 --- a/services/block_service_test.go +++ b/services/block_service_test.go @@ -40,8 +40,8 @@ func TestBlockService_Offline(t *testing.T) { blockTransaction, err := servicer.BlockTransaction(ctx, &types.BlockTransactionRequest{}) assert.Nil(t, blockTransaction) - assert.Equal(t, ErrUnavailableOffline.Code, err.Code) - assert.Equal(t, ErrUnavailableOffline.Message, err.Message) + assert.Equal(t, ErrUnimplemented.Code, err.Code) + assert.Equal(t, ErrUnimplemented.Message, err.Message) mockIndexer.AssertExpectations(t) } @@ -54,51 +54,49 @@ func TestBlockService_Online(t *testing.T) { servicer := NewBlockAPIService(cfg, mockIndexer) ctx := context.Background() - block := &types.Block{ + rawBlock := &types.Block{ BlockIdentifier: &types.BlockIdentifier{ Index: 100, Hash: "block 100", }, } - blockResponse := &types.BlockResponse{ - Block: block, - OtherTransactions: []*types.TransactionIdentifier{ - { - Hash: "tx1", - }, - }, - } - transaction := &types.Transaction{ TransactionIdentifier: &types.TransactionIdentifier{ Hash: "tx1", }, } + block := &types.Block{ + BlockIdentifier: &types.BlockIdentifier{ + Index: 100, + Hash: "block 100", + }, + Transactions: []*types.Transaction{ + transaction, + }, + } + + blockResponse := &types.BlockResponse{ + Block: block, + } + t.Run("nil identifier", func(t *testing.T) { mockIndexer.On( "GetBlockLazy", ctx, (*types.PartialBlockIdentifier)(nil), ).Return( - blockResponse, + &types.BlockResponse{ + Block: rawBlock, + OtherTransactions: []*types.TransactionIdentifier{ + { + Hash: "tx1", + }, + }, + }, nil, ).Once() - b, err := servicer.Block(ctx, &types.BlockRequest{}) - assert.Nil(t, err) - assert.Equal(t, blockResponse, b) - }) - - t.Run("populated identifier", func(t *testing.T) { - pbIdentifier := types.ConstructPartialBlockIdentifier(block.BlockIdentifier) - mockIndexer.On("GetBlockLazy", ctx, pbIdentifier).Return(blockResponse, nil).Once() - b, err := servicer.Block(ctx, &types.BlockRequest{ - BlockIdentifier: pbIdentifier, - }) - assert.Nil(t, err) - assert.Equal(t, blockResponse, b) - mockIndexer.On( "GetBlockTransaction", ctx, @@ -108,12 +106,42 @@ func TestBlockService_Online(t *testing.T) { transaction, nil, ).Once() - blockTransaction, err := servicer.BlockTransaction(ctx, &types.BlockTransactionRequest{ - BlockIdentifier: blockResponse.Block.BlockIdentifier, - TransactionIdentifier: transaction.TransactionIdentifier, + b, err := servicer.Block(ctx, &types.BlockRequest{}) + assert.Nil(t, err) + assert.Equal(t, blockResponse, b) + }) + + t.Run("populated identifier", func(t *testing.T) { + pbIdentifier := types.ConstructPartialBlockIdentifier(block.BlockIdentifier) + mockIndexer.On( + "GetBlockLazy", + ctx, + pbIdentifier, + ).Return( + &types.BlockResponse{ + Block: rawBlock, + OtherTransactions: []*types.TransactionIdentifier{ + { + Hash: "tx1", + }, + }, + }, + nil, + ).Once() + mockIndexer.On( + "GetBlockTransaction", + ctx, + blockResponse.Block.BlockIdentifier, + transaction.TransactionIdentifier, + ).Return( + transaction, + nil, + ).Once() + b, err := servicer.Block(ctx, &types.BlockRequest{ + BlockIdentifier: pbIdentifier, }) assert.Nil(t, err) - assert.Equal(t, transaction, blockTransaction.Transaction) + assert.Equal(t, blockResponse, b) }) mockIndexer.AssertExpectations(t) From e2443361781e5961b0764da5f7b5403b70da876a Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 27 Oct 2020 11:28:45 -0700 Subject: [PATCH 13/18] Scope future changes --- indexer/indexer.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/indexer/indexer.go b/indexer/indexer.go index 3be3b3f..0b6f68f 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -746,6 +746,7 @@ func (i *Indexer) GetScriptPubKeys( // GetBlockLazy returns a *types.BlockResponse from the indexer's block storage. // All transactions in a block must be fetched individually. +// TODO: replace with GetBlock func (i *Indexer) GetBlockLazy( ctx context.Context, blockIdentifier *types.PartialBlockIdentifier, @@ -755,6 +756,7 @@ func (i *Indexer) GetBlockLazy( // GetBlockTransaction returns a *types.Transaction if it is in the provided // *types.BlockIdentifier. +// TODO: remove method func (i *Indexer) GetBlockTransaction( ctx context.Context, blockIdentifier *types.BlockIdentifier, @@ -783,13 +785,20 @@ func (i *Indexer) GetBalance( currency *types.Currency, blockIdentifier *types.PartialBlockIdentifier, ) (*types.Amount, *types.BlockIdentifier, error) { + // TODO: add block lazy transactional blockResponse, err := i.GetBlockLazy(ctx, blockIdentifier) if err != nil { return nil, nil, err } - amount, err := i.balanceStorage.GetBalance( + // TODO: when false if we query unknown, this could cause issue + // TODO: add switch to not create unknown + dbTx := i.database.NewDatabaseTransaction(ctx, false) + defer dbTx.Discard(ctx) + + amount, err := i.balanceStorage.GetBalanceTransactional( ctx, + dbTx, accountIdentifier, currency, blockResponse.Block.BlockIdentifier, From dc10fea5f60538d71c80cd72aa29fbb69b550ce5 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 27 Oct 2020 12:42:58 -0700 Subject: [PATCH 14/18] Revert block change --- indexer/indexer.go | 2 - rosetta-cli-conf/mainnet/config.json | 2 + rosetta-cli-conf/testnet/config.json | 2 + services/block_service.go | 23 ++++++++- services/block_service_test.go | 72 ++++++++++++++++++++++++++-- services/types.go | 4 ++ 6 files changed, 99 insertions(+), 6 deletions(-) diff --git a/indexer/indexer.go b/indexer/indexer.go index 0b6f68f..15bdc02 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -746,7 +746,6 @@ func (i *Indexer) GetScriptPubKeys( // GetBlockLazy returns a *types.BlockResponse from the indexer's block storage. // All transactions in a block must be fetched individually. -// TODO: replace with GetBlock func (i *Indexer) GetBlockLazy( ctx context.Context, blockIdentifier *types.PartialBlockIdentifier, @@ -756,7 +755,6 @@ func (i *Indexer) GetBlockLazy( // GetBlockTransaction returns a *types.Transaction if it is in the provided // *types.BlockIdentifier. -// TODO: remove method func (i *Indexer) GetBlockTransaction( ctx context.Context, blockIdentifier *types.BlockIdentifier, diff --git a/rosetta-cli-conf/mainnet/config.json b/rosetta-cli-conf/mainnet/config.json index 69c25a1..e700c91 100644 --- a/rosetta-cli-conf/mainnet/config.json +++ b/rosetta-cli-conf/mainnet/config.json @@ -11,6 +11,8 @@ "max_sync_concurrency": 0, "tip_delay": 1800, "log_configuration": false, + "compression_disabled": true, + "memory_limit_disabled": true, "data": { "active_reconciliation_concurrency": 0, "inactive_reconciliation_concurrency": 0, diff --git a/rosetta-cli-conf/testnet/config.json b/rosetta-cli-conf/testnet/config.json index ea1b75d..cdd9b45 100644 --- a/rosetta-cli-conf/testnet/config.json +++ b/rosetta-cli-conf/testnet/config.json @@ -11,6 +11,8 @@ "max_sync_concurrency": 0, "tip_delay": 1800, "log_configuration": false, + "compression_disabled": true, + "memory_limit_disabled": true, "construction": { "max_offline_connections": 0, "stale_depth": 0, diff --git a/services/block_service.go b/services/block_service.go index b837765..489da48 100644 --- a/services/block_service.go +++ b/services/block_service.go @@ -54,6 +54,12 @@ func (s *BlockAPIService) Block( return nil, wrapErr(ErrBlockNotFound, err) } + // Direct client to fetch transactions individually if + // more than inlineFetchLimit. + if len(blockResponse.OtherTransactions) > inlineFetchLimit { + return blockResponse, nil + } + txs := make([]*types.Transaction, len(blockResponse.OtherTransactions)) for i, otherTx := range blockResponse.OtherTransactions { transaction, err := s.i.GetBlockTransaction( @@ -78,5 +84,20 @@ func (s *BlockAPIService) BlockTransaction( ctx context.Context, request *types.BlockTransactionRequest, ) (*types.BlockTransactionResponse, *types.Error) { - return nil, ErrUnimplemented + if s.config.Mode != configuration.Online { + return nil, wrapErr(ErrUnavailableOffline, nil) + } + + transaction, err := s.i.GetBlockTransaction( + ctx, + request.BlockIdentifier, + request.TransactionIdentifier, + ) + if err != nil { + return nil, wrapErr(ErrTransactionNotFound, err) + } + + return &types.BlockTransactionResponse{ + Transaction: transaction, + }, nil } diff --git a/services/block_service_test.go b/services/block_service_test.go index 8f7787b..bdbf297 100644 --- a/services/block_service_test.go +++ b/services/block_service_test.go @@ -16,6 +16,7 @@ package services import ( "context" + "fmt" "testing" "github.com/coinbase/rosetta-bitcoin/configuration" @@ -40,13 +41,13 @@ func TestBlockService_Offline(t *testing.T) { blockTransaction, err := servicer.BlockTransaction(ctx, &types.BlockTransactionRequest{}) assert.Nil(t, blockTransaction) - assert.Equal(t, ErrUnimplemented.Code, err.Code) - assert.Equal(t, ErrUnimplemented.Message, err.Message) + assert.Equal(t, ErrUnavailableOffline.Code, err.Code) + assert.Equal(t, ErrUnavailableOffline.Message, err.Message) mockIndexer.AssertExpectations(t) } -func TestBlockService_Online(t *testing.T) { +func TestBlockService_Online_Inline(t *testing.T) { cfg := &configuration.Configuration{ Mode: configuration.Online, } @@ -146,3 +147,68 @@ func TestBlockService_Online(t *testing.T) { mockIndexer.AssertExpectations(t) } + +func TestBlockService_Online_External(t *testing.T) { + cfg := &configuration.Configuration{ + Mode: configuration.Online, + } + mockIndexer := &mocks.Indexer{} + servicer := NewBlockAPIService(cfg, mockIndexer) + ctx := context.Background() + + blockResponse := &types.BlockResponse{ + Block: &types.Block{ + BlockIdentifier: &types.BlockIdentifier{ + Index: 100, + Hash: "block 100", + }, + }, + } + + otherTxs := []*types.TransactionIdentifier{} + for i := 0; i < 200; i++ { + otherTxs = append(otherTxs, &types.TransactionIdentifier{ + Hash: fmt.Sprintf("tx%d", i), + }) + } + blockResponse.OtherTransactions = otherTxs + + mockIndexer.On( + "GetBlockLazy", + ctx, + (*types.PartialBlockIdentifier)(nil), + ).Return( + blockResponse, + nil, + ).Once() + b, err := servicer.Block(ctx, &types.BlockRequest{}) + assert.Nil(t, err) + assert.Equal(t, blockResponse, b) + + for _, otherTx := range b.OtherTransactions { + tx := &types.Transaction{ + TransactionIdentifier: otherTx, + } + mockIndexer.On( + "GetBlockTransaction", + ctx, + blockResponse.Block.BlockIdentifier, + otherTx, + ).Return( + tx, + nil, + ).Once() + + bTx, err := servicer.BlockTransaction(ctx, &types.BlockTransactionRequest{ + BlockIdentifier: blockResponse.Block.BlockIdentifier, + TransactionIdentifier: otherTx, + }) + assert.Nil(t, err) + assert.Equal(t, &types.BlockTransactionResponse{ + Transaction: tx, + }, bTx) + + } + + mockIndexer.AssertExpectations(t) +} diff --git a/services/types.go b/services/types.go index b753d39..e039dc9 100644 --- a/services/types.go +++ b/services/types.go @@ -30,6 +30,10 @@ const ( // HistoricalBalanceLookup indicates // that historical balance lookup is supported. HistoricalBalanceLookup = true + + // inlineFetchLimit is the maximum number + // of transactions to fetch inline. + inlineFetchLimit = 100 ) var ( From 957c86c7f56536aaea4d63aa2575c6709c8616d3 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 27 Oct 2020 13:24:19 -0700 Subject: [PATCH 15/18] Update rosetta-sdk-go --- go.mod | 2 +- go.sum | 4 ++-- indexer/indexer.go | 16 +++++++++------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 983eb75..10a4f4c 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.13 require ( github.com/btcsuite/btcd v0.21.0-beta github.com/btcsuite/btcutil v1.0.2 - github.com/coinbase/rosetta-sdk-go v0.5.7 + github.com/coinbase/rosetta-sdk-go v0.5.8-0.20201027202152-ec894b6612f8 github.com/dgraph-io/badger/v2 v2.2007.2 github.com/grpc-ecosystem/go-grpc-middleware v1.2.2 github.com/stretchr/testify v1.6.1 diff --git a/go.sum b/go.sum index e461970..905b38a 100644 --- a/go.sum +++ b/go.sum @@ -61,8 +61,8 @@ github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/coinbase/rosetta-sdk-go v0.5.7 h1:BaR/+O3GzrsyunVNkVQHtjDCcId8G1Fh/RqEbeyExnk= -github.com/coinbase/rosetta-sdk-go v0.5.7/go.mod h1:l5aNeyeZKBkmWbVdkdLpWdToQ6hTwI7cZ1OU9cMbljY= +github.com/coinbase/rosetta-sdk-go v0.5.8-0.20201027202152-ec894b6612f8 h1:oOe+VuEj7mZdiAAr/r2Lmbdkl2mQoUhce74y0jReFtI= +github.com/coinbase/rosetta-sdk-go v0.5.8-0.20201027202152-ec894b6612f8/go.mod h1:l5aNeyeZKBkmWbVdkdLpWdToQ6hTwI7cZ1OU9cMbljY= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= diff --git a/indexer/indexer.go b/indexer/indexer.go index 15bdc02..1d1de87 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -783,23 +783,25 @@ func (i *Indexer) GetBalance( currency *types.Currency, blockIdentifier *types.PartialBlockIdentifier, ) (*types.Amount, *types.BlockIdentifier, error) { - // TODO: add block lazy transactional - blockResponse, err := i.GetBlockLazy(ctx, blockIdentifier) + dbTx := i.database.NewDatabaseTransaction(ctx, false) + defer dbTx.Discard(ctx) + + blockResponse, err := i.blockStorage.GetBlockLazyTransactional( + ctx, + blockIdentifier, + dbTx, + ) if err != nil { return nil, nil, err } - // TODO: when false if we query unknown, this could cause issue - // TODO: add switch to not create unknown - dbTx := i.database.NewDatabaseTransaction(ctx, false) - defer dbTx.Discard(ctx) - amount, err := i.balanceStorage.GetBalanceTransactional( ctx, dbTx, accountIdentifier, currency, blockResponse.Block.BlockIdentifier, + false, ) if err != nil { return nil, nil, err From 9a05090fe56ff452fece153d462a4d64bc35af09 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 27 Oct 2020 13:25:06 -0700 Subject: [PATCH 16/18] Tests --- services/block_service_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/services/block_service_test.go b/services/block_service_test.go index bdbf297..ff5ab08 100644 --- a/services/block_service_test.go +++ b/services/block_service_test.go @@ -207,7 +207,6 @@ func TestBlockService_Online_External(t *testing.T) { assert.Equal(t, &types.BlockTransactionResponse{ Transaction: tx, }, bTx) - } mockIndexer.AssertExpectations(t) From 165c6335755c29a51e3b9a7e842b1b5c1fc87c54 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 27 Oct 2020 15:24:05 -0700 Subject: [PATCH 17/18] Update rosetta-sdk-go --- go.mod | 2 +- go.sum | 4 ++-- indexer/indexer.go | 9 +++++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 10a4f4c..9e7b712 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.13 require ( github.com/btcsuite/btcd v0.21.0-beta github.com/btcsuite/btcutil v1.0.2 - github.com/coinbase/rosetta-sdk-go v0.5.8-0.20201027202152-ec894b6612f8 + github.com/coinbase/rosetta-sdk-go v0.5.8-0.20201027222031-dd9e29377d5f github.com/dgraph-io/badger/v2 v2.2007.2 github.com/grpc-ecosystem/go-grpc-middleware v1.2.2 github.com/stretchr/testify v1.6.1 diff --git a/go.sum b/go.sum index 905b38a..e86bb3d 100644 --- a/go.sum +++ b/go.sum @@ -61,8 +61,8 @@ github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/coinbase/rosetta-sdk-go v0.5.8-0.20201027202152-ec894b6612f8 h1:oOe+VuEj7mZdiAAr/r2Lmbdkl2mQoUhce74y0jReFtI= -github.com/coinbase/rosetta-sdk-go v0.5.8-0.20201027202152-ec894b6612f8/go.mod h1:l5aNeyeZKBkmWbVdkdLpWdToQ6hTwI7cZ1OU9cMbljY= +github.com/coinbase/rosetta-sdk-go v0.5.8-0.20201027222031-dd9e29377d5f h1:aWkN9dKMkMMpZKX5QycpePxH176Fj2fNNC7jESfLZw0= +github.com/coinbase/rosetta-sdk-go v0.5.8-0.20201027222031-dd9e29377d5f/go.mod h1:l5aNeyeZKBkmWbVdkdLpWdToQ6hTwI7cZ1OU9cMbljY= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= diff --git a/indexer/indexer.go b/indexer/indexer.go index 1d1de87..aaaeb31 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -800,9 +800,14 @@ func (i *Indexer) GetBalance( dbTx, accountIdentifier, currency, - blockResponse.Block.BlockIdentifier, - false, + blockResponse.Block.BlockIdentifier.Index, ) + if errors.Is(err, storage.ErrAccountMissing) { + return &types.Amount{ + Value: "0", + Currency: currency, + }, blockResponse.Block.BlockIdentifier, nil + } if err != nil { return nil, nil, err } From e1d4547c5be8ff40a98c1f5014e30feab74f567e Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 27 Oct 2020 15:25:08 -0700 Subject: [PATCH 18/18] nits --- indexer/balance_storage_helper.go | 2 +- indexer/indexer.go | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/indexer/balance_storage_helper.go b/indexer/balance_storage_helper.go index 3600002..c36699d 100644 --- a/indexer/balance_storage_helper.go +++ b/indexer/balance_storage_helper.go @@ -39,7 +39,7 @@ func (h *BalanceStorageHelper) AccountBalance( block *types.BlockIdentifier, ) (*types.Amount, error) { return &types.Amount{ - Value: "0", + Value: zeroValue, Currency: currency, }, nil } diff --git a/indexer/indexer.go b/indexer/indexer.go index aaaeb31..67514cf 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -53,6 +53,9 @@ const ( // this is the estimated memory overhead for each // block fetched by the indexer. sizeMultiplier = 15 + + // zeroValue is 0 as a string + zeroValue = "0" ) var ( @@ -804,7 +807,7 @@ func (i *Indexer) GetBalance( ) if errors.Is(err, storage.ErrAccountMissing) { return &types.Amount{ - Value: "0", + Value: zeroValue, Currency: currency, }, blockResponse.Block.BlockIdentifier, nil }