From 699e689f2e93125a41e16f81d49c9589a3ca8646 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 18 Sep 2020 12:18:57 -0700 Subject: [PATCH 1/3] Use small index cache in tests --- indexer/indexer.go | 20 +++++++++++++++----- indexer/indexer_test.go | 8 ++++---- main.go | 8 +++++++- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/indexer/indexer.go b/indexer/indexer.go index 43e8d12..cfc37c0 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -33,6 +33,10 @@ import ( ) const ( + // DefaultIndexCacheSize is the default size of the indexer cache. The larger + // the index cache size, the better the performance. + DefaultIndexCacheSize = 5 << 30 // 5 GB + // indexPlaceholder is provided to the syncer // to indicate we should both start from the // last synced block and that we should sync @@ -52,10 +56,9 @@ const ( // block fetched by the indexer. sizeMultiplier = 15 - // BadgerDB options overrides + // Other BadgerDB options overrides defaultBlockSize = 1 << 20 // use large blocks so less table indexes (1 MB) defaultValueThreshold = 0 // put almost everything in value logs (only use table for key) - defaultIndexCacheSize = 5 << 30 // 5 GB ) var ( @@ -117,11 +120,14 @@ func (i *Indexer) CloseDatabase(ctx context.Context) { // the Badger default so we have a lot less indexes to store. We also // ensure all values are stored in value log files (as to minimize // table bloat at the cost of some performance). -func defaultBadgerOptions(path string) badger.Options { +func defaultBadgerOptions( + path string, + indexCacheSize int64, +) badger.Options { defaultOps := storage.DefaultBadgerOptions(path) defaultOps.BlockSize = defaultBlockSize defaultOps.ValueThreshold = defaultValueThreshold - defaultOps.IndexCacheSize = defaultIndexCacheSize + defaultOps.IndexCacheSize = indexCacheSize return defaultOps } @@ -132,12 +138,16 @@ func Initialize( cancel context.CancelFunc, config *configuration.Configuration, client Client, + indexCacheSize int64, ) (*Indexer, error) { localStore, err := storage.NewBadgerStorage( ctx, config.IndexerPath, storage.WithCompressorEntries(config.Compressors), - storage.WithCustomSettings(defaultBadgerOptions(config.IndexerPath)), + storage.WithCustomSettings(defaultBadgerOptions( + config.IndexerPath, + indexCacheSize, + )), ) if err != nil { return nil, fmt.Errorf("%w: unable to initialize storage", err) diff --git a/indexer/indexer_test.go b/indexer/indexer_test.go index 314a2f5..c42aa6a 100644 --- a/indexer/indexer_test.go +++ b/indexer/indexer_test.go @@ -72,7 +72,7 @@ func TestIndexer_Pruning(t *testing.T) { IndexerPath: newDir, } - i, err := Initialize(ctx, cancel, cfg, mockClient) + i, err := Initialize(ctx, cancel, cfg, mockClient, storage.TinyIndexCacheSize) assert.NoError(t, err) // Waiting for bitcoind... @@ -232,7 +232,7 @@ func TestIndexer_Transactions(t *testing.T) { IndexerPath: newDir, } - i, err := Initialize(ctx, cancel, cfg, mockClient) + i, err := Initialize(ctx, cancel, cfg, mockClient, storage.TinyIndexCacheSize) assert.NoError(t, err) // Sync to 1000 @@ -450,7 +450,7 @@ func TestIndexer_Reorg(t *testing.T) { IndexerPath: newDir, } - i, err := Initialize(ctx, cancel, cfg, mockClient) + i, err := Initialize(ctx, cancel, cfg, mockClient, storage.TinyIndexCacheSize) assert.NoError(t, err) // Sync to 1000 @@ -692,7 +692,7 @@ func TestIndexer_HeaderReorg(t *testing.T) { IndexerPath: newDir, } - i, err := Initialize(ctx, cancel, cfg, mockClient) + i, err := Initialize(ctx, cancel, cfg, mockClient, storage.TinyIndexCacheSize) assert.NoError(t, err) // Sync to 1000 diff --git a/main.go b/main.go index 0e8f30c..89a7f3d 100644 --- a/main.go +++ b/main.go @@ -94,7 +94,13 @@ func startOnlineDependencies( return bitcoin.StartBitcoind(ctx, cfg.ConfigPath, g) }) - i, err := indexer.Initialize(ctx, cancel, cfg, client) + i, err := indexer.Initialize( + ctx, + cancel, + cfg, + client, + indexer.DefaultIndexCacheSize, + ) if err != nil { return nil, nil, fmt.Errorf("%w: unable to initialize indexer", err) } From ee5b3c72d1b8d9c9538cd6dd9ea8df01de51cfe3 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 18 Sep 2020 12:26:51 -0700 Subject: [PATCH 2/3] Add goveralls --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 5e2f88a..9429579 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ ADDLICENCE_SCRIPT=${ADDLICENSE_CMD} -c "Coinbase, Inc." -l "apache" -v SPELLCHECK_CMD=go run github.com/client9/misspell/cmd/misspell GOLINES_CMD=go run github.com/segmentio/golines GOLINT_CMD=go run golang.org/x/lint/golint +GOVERALLS_CMD=go run github.com/mattn/goveralls GOIMPORTS_CMD=go run golang.org/x/tools/cmd/goimports GO_PACKAGES=./services/... ./indexer/... ./bitcoin/... ./configuration/... GO_FOLDERS=$(shell echo ${GO_PACKAGES} | sed -e "s/\.\///g" | sed -e "s/\/\.\.\.//g") From 0356f8efbb614ee22124e081a86ad10a0c4b8564 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 18 Sep 2020 12:33:55 -0700 Subject: [PATCH 3/3] Nits --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ae50fb8..a254b32 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,7 +18,7 @@ executors: docker: - image: circleci/golang:1.13 user: root # go directory is owned by root - working_directory: /go/src/github.com/coinbase/rosetta-sdk-go + working_directory: /go/src/github.com/coinbase/rosetta-bitcoin environment: - GO111MODULE: "on"