Merge pull request #30 from coinbase/patrick/historical-balance-lookup

[services] Add historical balance lookup

.dockerignore

@@ -1,2 +1,3 @@
 rosetta-bitcoin
 bitcoin-data
+cli-data

go.mod

@@ -5,7 +5,7 @@ go 1.13
 require (
 	github.com/btcsuite/btcd v0.21.0-beta
 	github.com/btcsuite/btcutil v1.0.2
-	github.com/coinbase/rosetta-sdk-go v0.5.7
+	github.com/coinbase/rosetta-sdk-go v0.5.8-0.20201027222031-dd9e29377d5f
 	github.com/dgraph-io/badger/v2 v2.2007.2
 	github.com/grpc-ecosystem/go-grpc-middleware v1.2.2
 	github.com/stretchr/testify v1.6.1

go.sum

@@ -61,8 +61,8 @@ github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJ
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/coinbase/rosetta-sdk-go v0.5.7 h1:BaR/+O3GzrsyunVNkVQHtjDCcId8G1Fh/RqEbeyExnk=
-github.com/coinbase/rosetta-sdk-go v0.5.7/go.mod h1:l5aNeyeZKBkmWbVdkdLpWdToQ6hTwI7cZ1OU9cMbljY=
+github.com/coinbase/rosetta-sdk-go v0.5.8-0.20201027222031-dd9e29377d5f h1:aWkN9dKMkMMpZKX5QycpePxH176Fj2fNNC7jESfLZw0=
+github.com/coinbase/rosetta-sdk-go v0.5.8-0.20201027222031-dd9e29377d5f/go.mod h1:l5aNeyeZKBkmWbVdkdLpWdToQ6hTwI7cZ1OU9cMbljY=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=

indexer/balance_storage_handler.go

@@ -0,0 +1,46 @@
+// Copyright 2020 Coinbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package indexer
+
+import (
+	"context"
+
+	"github.com/coinbase/rosetta-sdk-go/parser"
+	"github.com/coinbase/rosetta-sdk-go/storage"
+	"github.com/coinbase/rosetta-sdk-go/types"
+)
+
+var _ storage.BalanceStorageHandler = (*BalanceStorageHandler)(nil)
+
+// BalanceStorageHandler implements storage.BalanceStorageHandler.
+type BalanceStorageHandler struct{}
+
+// BlockAdded is called whenever a block is committed to BlockStorage.
+func (h *BalanceStorageHandler) BlockAdded(
+	ctx context.Context,
+	block *types.Block,
+	changes []*parser.BalanceChange,
+) error {
+	return nil
+}
+
+// BlockRemoved is called whenever a block is removed from BlockStorage.
+func (h *BalanceStorageHandler) BlockRemoved(
+	ctx context.Context,
+	block *types.Block,
+	changes []*parser.BalanceChange,
+) error {
+	return nil
+}

indexer/balance_storage_helper.go

@@ -0,0 +1,62 @@
+// Copyright 2020 Coinbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package indexer
+
+import (
+	"context"
+
+	"github.com/coinbase/rosetta-sdk-go/asserter"
+	"github.com/coinbase/rosetta-sdk-go/parser"
+	"github.com/coinbase/rosetta-sdk-go/storage"
+	"github.com/coinbase/rosetta-sdk-go/types"
+)
+
+var _ storage.BalanceStorageHelper = (*BalanceStorageHelper)(nil)
+
+// BalanceStorageHelper implements storage.BalanceStorageHelper.
+type BalanceStorageHelper struct {
+	a *asserter.Asserter
+}
+
+// AccountBalance attempts to fetch the balance
+// for a missing account in storage.
+func (h *BalanceStorageHelper) AccountBalance(
+	ctx context.Context,
+	account *types.AccountIdentifier,
+	currency *types.Currency,
+	block *types.BlockIdentifier,
+) (*types.Amount, error) {
+	return &types.Amount{
+		Value:    zeroValue,
+		Currency: currency,
+	}, nil
+}
+
+// Asserter returns a *asserter.Asserter.
+func (h *BalanceStorageHelper) Asserter() *asserter.Asserter {
+	return h.a
+}
+
+// BalanceExemptions returns a list of *types.BalanceExemption.
+func (h *BalanceStorageHelper) BalanceExemptions() []*types.BalanceExemption {
+	return []*types.BalanceExemption{}
+}
+
+// ExemptFunc returns a parser.ExemptOperation.
+func (h *BalanceStorageHelper) ExemptFunc() parser.ExemptOperation {
+	return func(op *types.Operation) bool {
+		return false
+	}
+}

indexer/coin_storage_helper.go

@@ -0,0 +1,38 @@
+// Copyright 2020 Coinbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package indexer
+
+import (
+	"context"
+
+	"github.com/coinbase/rosetta-sdk-go/storage"
+	"github.com/coinbase/rosetta-sdk-go/types"
+)
+
+var _ storage.CoinStorageHelper = (*CoinStorageHelper)(nil)
+
+// CoinStorageHelper implements storage.CoinStorageHelper.
+type CoinStorageHelper struct {
+	b *storage.BlockStorage
+}
+
+// CurrentBlockIdentifier returns the current head block identifier
+// and is used to comply with the CoinStorageHelper interface.
+func (h *CoinStorageHelper) CurrentBlockIdentifier(
+	ctx context.Context,
+	transaction storage.DatabaseTransaction,
+) (*types.BlockIdentifier, error) {
+	return h.b.GetHeadBlockIdentifierTransactional(ctx, transaction)
+}

indexer/indexer.go

@@ -53,6 +53,9 @@ const (
 	// this is the estimated memory overhead for each
 	// block fetched by the indexer.
 	sizeMultiplier = 15
+
+	// zeroValue is 0 as a string
+	zeroValue = "0"
 )
 
 var (
@@ -74,7 +77,6 @@ type Client interface {
 var _ syncer.Handler = (*Indexer)(nil)
 var _ syncer.Helper = (*Indexer)(nil)
 var _ services.Indexer = (*Indexer)(nil)
-var _ storage.CoinStorageHelper = (*Indexer)(nil)
 
 // Indexer caches blocks and provides balance query functionality.
 type Indexer struct {
@@ -85,11 +87,12 @@ type Indexer struct {
 
 	client Client
 
-	asserter     *asserter.Asserter
-	database     storage.Database
-	blockStorage *storage.BlockStorage
-	coinStorage  *storage.CoinStorage
-	workers      []storage.BlockWorker
+	asserter       *asserter.Asserter
+	database       storage.Database
+	blockStorage   *storage.BlockStorage
+	balanceStorage *storage.BalanceStorage
+	coinStorage    *storage.CoinStorage
+	workers        []storage.BlockWorker
 
 	waiter *waitTable
 }
@@ -197,9 +200,21 @@ func Initialize(
 		asserter:      asserter,
 	}
 
-	coinStorage := storage.NewCoinStorage(localStore, i, asserter)
+	coinStorage := storage.NewCoinStorage(
+		localStore,
+		&CoinStorageHelper{blockStorage},
+		asserter,
+	)
 	i.coinStorage = coinStorage
-	i.workers = []storage.BlockWorker{coinStorage}
+
+	balanceStorage := storage.NewBalanceStorage(localStore)
+	balanceStorage.Initialize(
+		&BalanceStorageHelper{asserter},
+		&BalanceStorageHandler{},
+	)
+	i.balanceStorage = balanceStorage
+
+	i.workers = []storage.BlockWorker{coinStorage, balanceStorage}
 
 	return i, nil
 }
@@ -748,7 +763,11 @@ func (i *Indexer) GetBlockTransaction(
 	blockIdentifier *types.BlockIdentifier,
 	transactionIdentifier *types.TransactionIdentifier,
 ) (*types.Transaction, error) {
-	return i.blockStorage.GetBlockTransaction(ctx, blockIdentifier, transactionIdentifier)
+	return i.blockStorage.GetBlockTransaction(
+		ctx,
+		blockIdentifier,
+		transactionIdentifier,
+	)
 }
 
 // GetCoins returns all unspent coins for a particular *types.AccountIdentifier.
@@ -759,11 +778,42 @@ func (i *Indexer) GetCoins(
 	return i.coinStorage.GetCoins(ctx, accountIdentifier)
 }
 
-// CurrentBlockIdentifier returns the current head block identifier
-// and is used to comply with the CoinStorageHelper interface.
-func (i *Indexer) CurrentBlockIdentifier(
+// GetBalance returns the balance of an account
+// at a particular *types.PartialBlockIdentifier.
+func (i *Indexer) GetBalance(
 	ctx context.Context,
-	transaction storage.DatabaseTransaction,
-) (*types.BlockIdentifier, error) {
-	return i.blockStorage.GetHeadBlockIdentifierTransactional(ctx, transaction)
+	accountIdentifier *types.AccountIdentifier,
+	currency *types.Currency,
+	blockIdentifier *types.PartialBlockIdentifier,
+) (*types.Amount, *types.BlockIdentifier, error) {
+	dbTx := i.database.NewDatabaseTransaction(ctx, false)
+	defer dbTx.Discard(ctx)
+
+	blockResponse, err := i.blockStorage.GetBlockLazyTransactional(
+		ctx,
+		blockIdentifier,
+		dbTx,
+	)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	amount, err := i.balanceStorage.GetBalanceTransactional(
+		ctx,
+		dbTx,
+		accountIdentifier,
+		currency,
+		blockResponse.Block.BlockIdentifier.Index,
+	)
+	if errors.Is(err, storage.ErrAccountMissing) {
+		return &types.Amount{
+			Value:    zeroValue,
+			Currency: currency,
+		}, blockResponse.Block.BlockIdentifier, nil
+	}
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return amount, blockResponse.Block.BlockIdentifier, nil
 }

main.go

@@ -154,7 +154,7 @@ func main() {
 	// requests.
 	asserter, err := asserter.NewServer(
 		bitcoin.OperationTypes,
-		false,
+		services.HistoricalBalanceLookup,
 		[]*types.NetworkIdentifier{cfg.Network},
 		nil,
 	)

mocks/services/indexer.go

@@ -17,6 +17,38 @@ type Indexer struct {
 	mock.Mock
 }
 
+// GetBalance provides a mock function with given fields: _a0, _a1, _a2, _a3
+func (_m *Indexer) GetBalance(_a0 context.Context, _a1 *types.AccountIdentifier, _a2 *types.Currency, _a3 *types.PartialBlockIdentifier) (*types.Amount, *types.BlockIdentifier, error) {
+	ret := _m.Called(_a0, _a1, _a2, _a3)
+
+	var r0 *types.Amount
+	if rf, ok := ret.Get(0).(func(context.Context, *types.AccountIdentifier, *types.Currency, *types.PartialBlockIdentifier) *types.Amount); ok {
+		r0 = rf(_a0, _a1, _a2, _a3)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*types.Amount)
+		}
+	}
+
+	var r1 *types.BlockIdentifier
+	if rf, ok := ret.Get(1).(func(context.Context, *types.AccountIdentifier, *types.Currency, *types.PartialBlockIdentifier) *types.BlockIdentifier); ok {
+		r1 = rf(_a0, _a1, _a2, _a3)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*types.BlockIdentifier)
+		}
+	}
+
+	var r2 error
+	if rf, ok := ret.Get(2).(func(context.Context, *types.AccountIdentifier, *types.Currency, *types.PartialBlockIdentifier) error); ok {
+		r2 = rf(_a0, _a1, _a2, _a3)
+	} else {
+		r2 = ret.Error(2)
+	}
+
+	return r0, r1, r2
+}
+
 // GetBlockLazy provides a mock function with given fields: _a0, _a1
 func (_m *Indexer) GetBlockLazy(_a0 context.Context, _a1 *types.PartialBlockIdentifier) (*types.BlockResponse, error) {
 	ret := _m.Called(_a0, _a1)

rosetta-cli-conf/mainnet/config.json

@@ -7,10 +7,12 @@
  "http_timeout": 300,
  "max_retries": 5,
  "retry_elapsed_time": 0,
- "max_online_connections": 0,
+ "max_online_connections": 1000,
  "max_sync_concurrency": 0,
  "tip_delay": 1800,
  "log_configuration": false,
+ "compression_disabled": true,
+ "memory_limit_disabled": true,
  "data": {
   "active_reconciliation_concurrency": 0,
   "inactive_reconciliation_concurrency": 0,

rosetta-cli-conf/testnet/config.json

@@ -7,10 +7,12 @@
  "http_timeout": 300,
  "max_retries": 5,
  "retry_elapsed_time": 0,
- "max_online_connections": 0,
+ "max_online_connections": 1000,
  "max_sync_concurrency": 0,
  "tip_delay": 1800,
  "log_configuration": false,
+ "compression_disabled": true,
+ "memory_limit_disabled": true,
  "construction": {
   "max_offline_connections": 0,
   "stale_depth": 0,

services/account_service.go

@@ -49,27 +49,51 @@ func (s *AccountAPIService) AccountBalance(
 		return nil, wrapErr(ErrUnavailableOffline, nil)
 	}
 
-	coins, block, err := s.i.GetCoins(ctx, request.AccountIdentifier)
-	if err != nil {
-		return nil, wrapErr(ErrUnableToGetCoins, err)
+	// If we are fetching the current balance,
+	// return all coins for an address and calculate
+	// the balance from those coins.
+	if request.BlockIdentifier == nil {
+		coins, block, err := s.i.GetCoins(ctx, request.AccountIdentifier)
+		if err != nil {
+			return nil, wrapErr(ErrUnableToGetCoins, err)
+		}
+
+		balance := "0"
+		for _, coin := range coins {
+			balance, err = types.AddValues(balance, coin.Amount.Value)
+			if err != nil {
+				return nil, wrapErr(ErrUnableToParseIntermediateResult, err)
+			}
+		}
+
+		return &types.AccountBalanceResponse{
+			BlockIdentifier: block,
+			Coins:           coins,
+			Balances: []*types.Amount{
+				{
+					Value:    balance,
+					Currency: s.config.Currency,
+				},
+			},
+		}, nil
 	}
 
-	balance := "0"
-	for _, coin := range coins {
-		balance, err = types.AddValues(balance, coin.Amount.Value)
-		if err != nil {
-			return nil, wrapErr(ErrUnableToParseIntermediateResult, err)
-		}
+	// If we are fetching a historical balance,
+	// use balance storage and don't return coins.
+	amount, block, err := s.i.GetBalance(
+		ctx,
+		request.AccountIdentifier,
+		s.config.Currency,
+		request.BlockIdentifier,
+	)
+	if err != nil {
+		return nil, wrapErr(ErrUnableToGetBalance, err)
 	}
 
 	return &types.AccountBalanceResponse{
 		BlockIdentifier: block,
-		Coins:           coins,
 		Balances: []*types.Amount{
-			{
-				Value:    balance,
-				Currency: s.config.Currency,
-			},
+			amount,
 		},
 	}, nil
 }

services/account_service_test.go

@@ -41,7 +41,7 @@ func TestAccountBalance_Offline(t *testing.T) {
 	mockIndexer.AssertExpectations(t)
 }
 
-func TestAccountBalance_Online(t *testing.T) {
+func TestAccountBalance_Online_Current(t *testing.T) {
 	cfg := &configuration.Configuration{
 		Mode:     configuration.Online,
 		Currency: bitcoin.MainnetCurrency,
@@ -104,3 +104,48 @@ func TestAccountBalance_Online(t *testing.T) {
 
 	mockIndexer.AssertExpectations(t)
 }
+
+func TestAccountBalance_Online_Historical(t *testing.T) {
+	cfg := &configuration.Configuration{
+		Mode:     configuration.Online,
+		Currency: bitcoin.MainnetCurrency,
+	}
+	mockIndexer := &mocks.Indexer{}
+	servicer := NewAccountAPIService(cfg, mockIndexer)
+	ctx := context.Background()
+	account := &types.AccountIdentifier{
+		Address: "hello",
+	}
+	block := &types.BlockIdentifier{
+		Index: 1000,
+		Hash:  "block 1000",
+	}
+	partialBlock := &types.PartialBlockIdentifier{
+		Index: &block.Index,
+	}
+	amount := &types.Amount{
+		Value:    "25",
+		Currency: bitcoin.MainnetCurrency,
+	}
+
+	mockIndexer.On(
+		"GetBalance",
+		ctx,
+		account,
+		bitcoin.MainnetCurrency,
+		partialBlock,
+	).Return(amount, block, nil).Once()
+	bal, err := servicer.AccountBalance(ctx, &types.AccountBalanceRequest{
+		AccountIdentifier: account,
+		BlockIdentifier:   partialBlock,
+	})
+	assert.Nil(t, err)
+	assert.Equal(t, &types.AccountBalanceResponse{
+		BlockIdentifier: block,
+		Balances: []*types.Amount{
+			amount,
+		},
+	}, bal)
+
+	mockIndexer.AssertExpectations(t)
+}

services/block_service.go

@@ -54,6 +54,28 @@ func (s *BlockAPIService) Block(
 		return nil, wrapErr(ErrBlockNotFound, err)
 	}
 
+	// Direct client to fetch transactions individually if
+	// more than inlineFetchLimit.
+	if len(blockResponse.OtherTransactions) > inlineFetchLimit {
+		return blockResponse, nil
+	}
+
+	txs := make([]*types.Transaction, len(blockResponse.OtherTransactions))
+	for i, otherTx := range blockResponse.OtherTransactions {
+		transaction, err := s.i.GetBlockTransaction(
+			ctx,
+			blockResponse.Block.BlockIdentifier,
+			otherTx,
+		)
+		if err != nil {
+			return nil, wrapErr(ErrTransactionNotFound, err)
+		}
+
+		txs[i] = transaction
+	}
+	blockResponse.Block.Transactions = txs
+
+	blockResponse.OtherTransactions = nil
 	return blockResponse, nil
 }
 

services/block_service_test.go

@@ -16,6 +16,7 @@ package services
 
 import (
 	"context"
+	"fmt"
 	"testing"
 
 	"github.com/coinbase/rosetta-bitcoin/configuration"
@@ -46,7 +47,7 @@ func TestBlockService_Offline(t *testing.T) {
 	mockIndexer.AssertExpectations(t)
 }
 
-func TestBlockService_Online(t *testing.T) {
+func TestBlockService_Online_Inline(t *testing.T) {
 	cfg := &configuration.Configuration{
 		Mode: configuration.Online,
 	}
@@ -54,51 +55,49 @@ func TestBlockService_Online(t *testing.T) {
 	servicer := NewBlockAPIService(cfg, mockIndexer)
 	ctx := context.Background()
 
-	block := &types.Block{
+	rawBlock := &types.Block{
 		BlockIdentifier: &types.BlockIdentifier{
 			Index: 100,
 			Hash:  "block 100",
 		},
 	}
 
-	blockResponse := &types.BlockResponse{
-		Block: block,
-		OtherTransactions: []*types.TransactionIdentifier{
-			{
-				Hash: "tx1",
-			},
-		},
-	}
-
 	transaction := &types.Transaction{
 		TransactionIdentifier: &types.TransactionIdentifier{
 			Hash: "tx1",
 		},
 	}
 
+	block := &types.Block{
+		BlockIdentifier: &types.BlockIdentifier{
+			Index: 100,
+			Hash:  "block 100",
+		},
+		Transactions: []*types.Transaction{
+			transaction,
+		},
+	}
+
+	blockResponse := &types.BlockResponse{
+		Block: block,
+	}
+
 	t.Run("nil identifier", func(t *testing.T) {
 		mockIndexer.On(
 			"GetBlockLazy",
 			ctx,
 			(*types.PartialBlockIdentifier)(nil),
 		).Return(
-			blockResponse,
+			&types.BlockResponse{
+				Block: rawBlock,
+				OtherTransactions: []*types.TransactionIdentifier{
+					{
+						Hash: "tx1",
+					},
+				},
+			},
 			nil,
 		).Once()
-		b, err := servicer.Block(ctx, &types.BlockRequest{})
-		assert.Nil(t, err)
-		assert.Equal(t, blockResponse, b)
-	})
-
-	t.Run("populated identifier", func(t *testing.T) {
-		pbIdentifier := types.ConstructPartialBlockIdentifier(block.BlockIdentifier)
-		mockIndexer.On("GetBlockLazy", ctx, pbIdentifier).Return(blockResponse, nil).Once()
-		b, err := servicer.Block(ctx, &types.BlockRequest{
-			BlockIdentifier: pbIdentifier,
-		})
-		assert.Nil(t, err)
-		assert.Equal(t, blockResponse, b)
-
 		mockIndexer.On(
 			"GetBlockTransaction",
 			ctx,
@@ -108,13 +107,107 @@ func TestBlockService_Online(t *testing.T) {
 			transaction,
 			nil,
 		).Once()
-		blockTransaction, err := servicer.BlockTransaction(ctx, &types.BlockTransactionRequest{
-			BlockIdentifier:       blockResponse.Block.BlockIdentifier,
-			TransactionIdentifier: transaction.TransactionIdentifier,
+		b, err := servicer.Block(ctx, &types.BlockRequest{})
+		assert.Nil(t, err)
+		assert.Equal(t, blockResponse, b)
+	})
+
+	t.Run("populated identifier", func(t *testing.T) {
+		pbIdentifier := types.ConstructPartialBlockIdentifier(block.BlockIdentifier)
+		mockIndexer.On(
+			"GetBlockLazy",
+			ctx,
+			pbIdentifier,
+		).Return(
+			&types.BlockResponse{
+				Block: rawBlock,
+				OtherTransactions: []*types.TransactionIdentifier{
+					{
+						Hash: "tx1",
+					},
+				},
+			},
+			nil,
+		).Once()
+		mockIndexer.On(
+			"GetBlockTransaction",
+			ctx,
+			blockResponse.Block.BlockIdentifier,
+			transaction.TransactionIdentifier,
+		).Return(
+			transaction,
+			nil,
+		).Once()
+		b, err := servicer.Block(ctx, &types.BlockRequest{
+			BlockIdentifier: pbIdentifier,
 		})
 		assert.Nil(t, err)
-		assert.Equal(t, transaction, blockTransaction.Transaction)
+		assert.Equal(t, blockResponse, b)
 	})
 
 	mockIndexer.AssertExpectations(t)
 }
+
+func TestBlockService_Online_External(t *testing.T) {
+	cfg := &configuration.Configuration{
+		Mode: configuration.Online,
+	}
+	mockIndexer := &mocks.Indexer{}
+	servicer := NewBlockAPIService(cfg, mockIndexer)
+	ctx := context.Background()
+
+	blockResponse := &types.BlockResponse{
+		Block: &types.Block{
+			BlockIdentifier: &types.BlockIdentifier{
+				Index: 100,
+				Hash:  "block 100",
+			},
+		},
+	}
+
+	otherTxs := []*types.TransactionIdentifier{}
+	for i := 0; i < 200; i++ {
+		otherTxs = append(otherTxs, &types.TransactionIdentifier{
+			Hash: fmt.Sprintf("tx%d", i),
+		})
+	}
+	blockResponse.OtherTransactions = otherTxs
+
+	mockIndexer.On(
+		"GetBlockLazy",
+		ctx,
+		(*types.PartialBlockIdentifier)(nil),
+	).Return(
+		blockResponse,
+		nil,
+	).Once()
+	b, err := servicer.Block(ctx, &types.BlockRequest{})
+	assert.Nil(t, err)
+	assert.Equal(t, blockResponse, b)
+
+	for _, otherTx := range b.OtherTransactions {
+		tx := &types.Transaction{
+			TransactionIdentifier: otherTx,
+		}
+		mockIndexer.On(
+			"GetBlockTransaction",
+			ctx,
+			blockResponse.Block.BlockIdentifier,
+			otherTx,
+		).Return(
+			tx,
+			nil,
+		).Once()
+
+		bTx, err := servicer.BlockTransaction(ctx, &types.BlockTransactionRequest{
+			BlockIdentifier:       blockResponse.Block.BlockIdentifier,
+			TransactionIdentifier: otherTx,
+		})
+		assert.Nil(t, err)
+		assert.Equal(t, &types.BlockTransactionResponse{
+			Transaction: tx,
+		}, bTx)
+	}
+
+	mockIndexer.AssertExpectations(t)
+}

services/errors.go

@@ -40,6 +40,7 @@ var (
 		ErrUnableToGetCoins,
 		ErrTransactionNotFound,
 		ErrCouldNotGetFeeRate,
+		ErrUnableToGetBalance,
 	}
 
 	// ErrUnimplemented is returned when an endpoint
@@ -59,8 +60,9 @@ var (
 	// ErrNotReady is returned when bitcoind is not
 	// yet ready to serve queries.
 	ErrNotReady = &types.Error{
-		Code:    2, //nolint
-		Message: "Bitcoind is not ready",
+		Code:      2, //nolint
+		Message:   "Bitcoind is not ready",
+		Retriable: true,
 	}
 
 	// ErrBitcoind is returned when bitcoind
@@ -173,6 +175,14 @@ var (
 		Code:    17, // nolint
 		Message: "Could not get suggested fee rate",
 	}
+
+	// ErrUnableToGetBalance is returned by the indexer
+	// when it is not possible to get the balance
+	// of a *types.AccountIdentifier.
+	ErrUnableToGetBalance = &types.Error{
+		Code:    18, //nolint
+		Message: "Unable to get balance",
+	}
 )
 
 // wrapErr adds details to the types.Error provided. We use a function
@@ -180,8 +190,8 @@ var (
 // errors.
 func wrapErr(rErr *types.Error, err error) *types.Error {
 	newErr := &types.Error{
-		Code:    rErr.Code,
-		Message: rErr.Message,
+		Code:      rErr.Code,
+		Message:   rErr.Message,
 		Retriable: rErr.Retriable,
 	}
 	if err != nil {

services/network_service.go

@@ -95,9 +95,10 @@ func (s *NetworkAPIService) NetworkOptions(
 			MiddlewareVersion: &MiddlewareVersion,
 		},
 		Allow: &types.Allow{
-			OperationStatuses: bitcoin.OperationStatuses,
-			OperationTypes:    bitcoin.OperationTypes,
-			Errors:            Errors,
+			OperationStatuses:       bitcoin.OperationStatuses,
+			OperationTypes:          bitcoin.OperationTypes,
+			Errors:                  Errors,
+			HistoricalBalanceLookup: HistoricalBalanceLookup,
 		},
 	}, nil
 }

services/network_service_test.go

@@ -27,7 +27,7 @@ import (
 )
 
 var (
-	middlewareVersion     = "0.0.4"
+	middlewareVersion     = "0.0.5"
 	defaultNetworkOptions = &types.NetworkOptionsResponse{
 		Version: &types.Version{
 			RosettaVersion:    types.RosettaAPIVersion,
@@ -35,9 +35,10 @@ var (
 			MiddlewareVersion: &middlewareVersion,
 		},
 		Allow: &types.Allow{
-			OperationStatuses: bitcoin.OperationStatuses,
-			OperationTypes:    bitcoin.OperationTypes,
-			Errors:            Errors,
+			OperationStatuses:       bitcoin.OperationStatuses,
+			OperationTypes:          bitcoin.OperationTypes,
+			Errors:                  Errors,
+			HistoricalBalanceLookup: HistoricalBalanceLookup,
 		},
 	}
 

services/types.go

@@ -26,6 +26,14 @@ const (
 	// NodeVersion is the version of
 	// bitcoin core we are using.
 	NodeVersion = "0.20.1"
+
+	// HistoricalBalanceLookup indicates
+	// that historical balance lookup is supported.
+	HistoricalBalanceLookup = true
+
+	// inlineFetchLimit is the maximum number
+	// of transactions to fetch inline.
+	inlineFetchLimit = 100
 )
 
 var (
@@ -34,7 +42,7 @@ var (
 	// variable instead of a constant because
 	// we typically need the pointer of this
 	// value.
-	MiddlewareVersion = "0.0.4"
+	MiddlewareVersion = "0.0.5"
 )
 
 // Client is used by the servicers to get Peer information
@@ -48,7 +56,10 @@ type Client interface {
 
 // Indexer is used by the servicers to get block and account data.
 type Indexer interface {
-	GetBlockLazy(context.Context, *types.PartialBlockIdentifier) (*types.BlockResponse, error)
+	GetBlockLazy(
+		context.Context,
+		*types.PartialBlockIdentifier,
+	) (*types.BlockResponse, error)
 	GetBlockTransaction(
 		context.Context,
 		*types.BlockIdentifier,
@@ -62,6 +73,12 @@ type Indexer interface {
 		context.Context,
 		[]*types.Coin,
 	) ([]*bitcoin.ScriptPubKey, error)
+	GetBalance(
+		context.Context,
+		*types.AccountIdentifier,
+		*types.Currency,
+		*types.PartialBlockIdentifier,
+	) (*types.Amount, *types.BlockIdentifier, error)
 }
 
 type unsignedTransaction struct {