Merge pull request #25 from coinbase/patrick/upgrade-sdk-go

Update rosetta-sdk-go + mmap files

README.md

@@ -85,10 +85,9 @@ _If you cloned the repository, you can run `make run-testnet-offline`._
 
 ## System Requirements
 `rosetta-bitcoin` has been tested on an [AWS c5.2xlarge instance](https://aws.amazon.com/ec2/instance-types/c5).
-This instance type has 8 vCPU and 16 GB of RAM. If you use a computer with less than 16 GB of RAM,
+This instance type has 8 vCPU and 16 GB of RAM.
-it is possible that `rosetta-bitcoin` will exit with an OOM error.
 
-### Recommended OS Settings
+### Network Settings
 To increase the load `rosetta-bitcoin` can handle, it is recommended to tune your OS
 settings to allow for more connections. On a linux-based OS, you can run the following
 commands ([source](http://www.tweaked.io/guide/kernel)):
@@ -106,6 +105,14 @@ enabling it._
 You should also modify your open file settings to `100000`. This can be done on a linux-based OS
 with the command: `ulimit -n 100000`.
 
+### Memory-Mapped Files
+`rosetta-bitcoin` uses [memory-mapped files](https://en.wikipedia.org/wiki/Memory-mapped_file) to
+persist data in the `indexer`. As a result, you **must** run `rosetta-bitcoin` on a 64-bit
+architecture (the virtual address space easily exceeds 100s of GBs).
+
+If you receive a kernel OOM, you may need to increase the allocated size of swap space
+on your OS. There is a great tutorial for how to do this on Linux [here](https://linuxize.com/post/create-a-linux-swap-file/).
+
 ## Architecture
 `rosetta-bitcoin` uses the `syncer`, `storage`, `parser`, and `server` package
 from [`rosetta-sdk-go`](https://github.com/coinbase/rosetta-sdk-go) instead

go.mod

@@ -5,7 +5,7 @@ go 1.13
 require (
 	github.com/btcsuite/btcd v0.21.0-beta
 	github.com/btcsuite/btcutil v1.0.2
-	github.com/coinbase/rosetta-sdk-go v0.4.8
+	github.com/coinbase/rosetta-sdk-go v0.5.5
 	github.com/dgraph-io/badger/v2 v2.2007.2
 	github.com/grpc-ecosystem/go-grpc-middleware v1.2.2
 	github.com/stretchr/testify v1.6.1

go.sum

@@ -73,6 +73,10 @@ github.com/coinbase/rosetta-sdk-go v0.4.7 h1:5KFc0CgLMkKamX++hYUFvE58a5/tCn0wSqp
 github.com/coinbase/rosetta-sdk-go v0.4.7/go.mod h1:8d4iN4VSGvLUzl+jRQlvYSLyS9TeY0QZebneWouizqU=
 github.com/coinbase/rosetta-sdk-go v0.4.8 h1:+E1TM4q1c5/x/jE9FPI1IZIbNvUWy4tRRgDPLnKzUV4=
 github.com/coinbase/rosetta-sdk-go v0.4.8/go.mod h1:8d4iN4VSGvLUzl+jRQlvYSLyS9TeY0QZebneWouizqU=
+github.com/coinbase/rosetta-sdk-go v0.5.4 h1:pM18LK2ci8zZwIu+uETmP6BXHqZbQGXRulwQCjYudg4=
+github.com/coinbase/rosetta-sdk-go v0.5.4/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws=
+github.com/coinbase/rosetta-sdk-go v0.5.5 h1:Z61/VUO89BDVl1m6Zj0e6OWMl5GnVevj4z79Eh3sSL0=
+github.com/coinbase/rosetta-sdk-go v0.5.5/go.mod h1:JRO4BJjhWAI7nYGwzYZWFgYfCTsQOvdZB7tGyN6kEtE=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
@@ -116,6 +120,7 @@ github.com/ethereum/go-ethereum v1.9.21 h1:8qRlhzrItnmUGdVlBzZLI2Tb46S0RdSNjFwIC
 github.com/ethereum/go-ethereum v1.9.21/go.mod h1:RXAVzbGrSGmDkDnHymruTAIEjUR3E4TX0EOpaj702sI=
 github.com/ethereum/go-ethereum v1.9.22 h1:/Fea9n2EWJuNJ9oahMq9luqjRBcbW7QWdThbcJl13ek=
 github.com/ethereum/go-ethereum v1.9.22/go.mod h1:FQjK3ZwD8C5DYn7ukTmFee36rq1dOMESiUfXr5RUc1w=
+github.com/ethereum/go-ethereum v1.9.23/go.mod h1:JIfVb6esrqALTExdz9hRYvrP0xBDf6wCncIu1hNwHpM=
 github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
@@ -314,6 +319,8 @@ github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU=
 github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tidwall/sjson v1.1.1 h1:7h1vk049Jnd5EH9NyzNiEuwYW4b5qgreBbqRC19AS3U=
 github.com/tidwall/sjson v1.1.1/go.mod h1:yvVuSnpEQv5cYIrO+AT6kw4QVfd5SDZoGIS7/5+fZFs=
+github.com/tidwall/sjson v1.1.2 h1:NC5okI+tQ8OG/oyzchvwXXxRxCV/FVdhODbPKkQ25jQ=
+github.com/tidwall/sjson v1.1.2/go.mod h1:SEzaDwxiPzKzNfUEO4HbYF/m4UCSJDsGgNqsS1LvdoY=
 github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=

indexer/indexer.go

@@ -31,13 +31,10 @@ import (
 	"github.com/coinbase/rosetta-sdk-go/types"
 	sdkUtils "github.com/coinbase/rosetta-sdk-go/utils"
 	"github.com/dgraph-io/badger/v2"
+	"github.com/dgraph-io/badger/v2/options"
 )
 
 const (
-	// DefaultIndexCacheSize is the default size of the indexer cache. The larger
-	// the index cache size, the better the performance.
-	DefaultIndexCacheSize = 5 << 30 // 5 GB
-
 	// indexPlaceholder is provided to the syncer
 	// to indicate we should both start from the
 	// last synced block and that we should sync
@@ -56,10 +53,6 @@ const (
 	// this is the estimated memory overhead for each
 	// block fetched by the indexer.
 	sizeMultiplier = 15
-
-	// Other BadgerDB options overrides
-	defaultBlockSize      = 1 << 20 // use large blocks so less table indexes (1 MB)
-	defaultValueThreshold = 0       // put almost everything in value logs (only use table for key)
 )
 
 var (
@@ -114,23 +107,51 @@ func (i *Indexer) CloseDatabase(ctx context.Context) {
 }
 
 // defaultBadgerOptions returns a set of badger.Options optimized
-// for running a Rosetta implementation. After extensive research
+// for running a Rosetta implementation.
-// and profiling, we determined that the bottleneck to high RPC
-// load was fetching table indexes from disk on an index cache miss.
-// Thus, we increased the default block size to be much larger than
-// the Badger default so we have a lot less indexes to store. We also
-// ensure all values are stored in value log files (as to minimize
-// table bloat at the cost of some performance).
 func defaultBadgerOptions(
-	path string,
+	dir string,
-	indexCacheSize int64,
 ) badger.Options {
-	defaultOps := storage.DefaultBadgerOptions(path)
+	opts := badger.DefaultOptions(dir)
-	defaultOps.BlockSize = defaultBlockSize
-	defaultOps.ValueThreshold = defaultValueThreshold
-	defaultOps.IndexCacheSize = indexCacheSize
 
-	return defaultOps
+	// By default, we do not compress the table at all. Doing so can
+	// significantly increase memory usage.
+	opts.Compression = options.None
+
+	// Load tables into memory and memory map value logs.
+	opts.TableLoadingMode = options.MemoryMap
+	opts.ValueLogLoadingMode = options.MemoryMap
+
+	// Use an extended table size for larger commits.
+	opts.MaxTableSize = storage.DefaultMaxTableSize
+
+	// Smaller value log sizes means smaller contiguous memory allocations
+	// and less RAM usage on cleanup.
+	opts.ValueLogFileSize = storage.DefaultLogValueSize
+
+	// To allow writes at a faster speed, we create a new memtable as soon as
+	// an existing memtable is filled up. This option determines how many
+	// memtables should be kept in memory.
+	opts.NumMemtables = 1
+
+	// Don't keep multiple memtables in memory. With larger
+	// memtable size, this explodes memory usage.
+	opts.NumLevelZeroTables = 1
+	opts.NumLevelZeroTablesStall = 2
+
+	// This option will have a significant effect the memory. If the level is kept
+	// in-memory, read are faster but the tables will be kept in memory. By default,
+	// this is set to false.
+	opts.KeepL0InMemory = false
+
+	// We don't compact L0 on close as this can greatly delay shutdown time.
+	opts.CompactL0OnClose = false
+
+	// LoadBloomsOnOpen=false will improve the db startup speed. This is also
+	// a waste to enable with a limited index cache size (as many of the loaded bloom
+	// filters will be immediately discarded from the cache).
+	opts.LoadBloomsOnOpen = false
+
+	return opts
 }
 
 // Initialize returns a new Indexer.
@@ -139,7 +160,6 @@ func Initialize(
 	cancel context.CancelFunc,
 	config *configuration.Configuration,
 	client Client,
-	indexCacheSize int64,
 ) (*Indexer, error) {
 	localStore, err := storage.NewBadgerStorage(
 		ctx,
@@ -147,7 +167,6 @@ func Initialize(
 		storage.WithCompressorEntries(config.Compressors),
 		storage.WithCustomSettings(defaultBadgerOptions(
 			config.IndexerPath,
-			indexCacheSize,
 		)),
 	)
 	if err != nil {

indexer/indexer_test.go

@@ -72,7 +72,7 @@ func TestIndexer_Pruning(t *testing.T) {
 		IndexerPath: newDir,
 	}
 
-	i, err := Initialize(ctx, cancel, cfg, mockClient, storage.TinyIndexCacheSize)
+	i, err := Initialize(ctx, cancel, cfg, mockClient)
 	assert.NoError(t, err)
 
 	// Waiting for bitcoind...
@@ -232,7 +232,7 @@ func TestIndexer_Transactions(t *testing.T) {
 		IndexerPath:            newDir,
 	}
 
-	i, err := Initialize(ctx, cancel, cfg, mockClient, storage.TinyIndexCacheSize)
+	i, err := Initialize(ctx, cancel, cfg, mockClient)
 	assert.NoError(t, err)
 
 	// Sync to 1000
@@ -450,7 +450,7 @@ func TestIndexer_Reorg(t *testing.T) {
 		IndexerPath:            newDir,
 	}
 
-	i, err := Initialize(ctx, cancel, cfg, mockClient, storage.TinyIndexCacheSize)
+	i, err := Initialize(ctx, cancel, cfg, mockClient)
 	assert.NoError(t, err)
 
 	// Sync to 1000
@@ -692,7 +692,7 @@ func TestIndexer_HeaderReorg(t *testing.T) {
 		IndexerPath:            newDir,
 	}
 
-	i, err := Initialize(ctx, cancel, cfg, mockClient, storage.TinyIndexCacheSize)
+	i, err := Initialize(ctx, cancel, cfg, mockClient)
 	assert.NoError(t, err)
 
 	// Sync to 1000

main.go

@@ -51,10 +51,6 @@ const (
 	// idleTimeout is the maximum amount of time to wait for the
 	// next request when keep-alives are enabled.
 	idleTimeout = 30 * time.Second
-
-	// maxHeapUsage is the size of the heap in MB before we manually
-	// trigger garbage collection.
-	maxHeapUsage = 8500 // ~8.5 GB
 )
 
 var (
@@ -99,7 +95,6 @@ func startOnlineDependencies(
 		cancel,
 		cfg,
 		client,
-		indexer.DefaultIndexCacheSize,
 	)
 	if err != nil {
 		return nil, nil, fmt.Errorf("%w: unable to initialize indexer", err)
@@ -143,7 +138,7 @@ func main() {
 	g, ctx := errgroup.WithContext(ctx)
 
 	g.Go(func() error {
-		return utils.MonitorMemoryUsage(ctx, maxHeapUsage)
+		return utils.MonitorMemoryUsage(ctx, -1)
 	})
 
 	var i *indexer.Indexer
@@ -161,6 +156,7 @@ func main() {
 		bitcoin.OperationTypes,
 		false,
 		[]*types.NetworkIdentifier{cfg.Network},
+		nil,
 	)
 	if err != nil {
 		logger.Fatalw("unable to create new server asserter", "error", err)

services/network_service_test.go

@@ -27,10 +27,10 @@ import (
 )
 
 var (
-	middlewareVersion     = "0.0.3"
+	middlewareVersion     = "0.0.4"
 	defaultNetworkOptions = &types.NetworkOptionsResponse{
 		Version: &types.Version{
-			RosettaVersion:    "1.4.4",
+			RosettaVersion:    "1.4.5",
 			NodeVersion:       "0.20.1",
 			MiddlewareVersion: &middlewareVersion,
 		},

services/types.go

@@ -25,7 +25,7 @@ import (
 const (
 	// RosettaVersion is the version of the
 	// Rosetta Specification we are using.
-	RosettaVersion = "1.4.4"
+	RosettaVersion = "1.4.5"
 
 	// NodeVersion is the version of
 	// bitcoin core we are using.
@@ -38,7 +38,7 @@ var (
 	// variable instead of a constant because
 	// we typically need the pointer of this
 	// value.
-	MiddlewareVersion = "0.0.3"
+	MiddlewareVersion = "0.0.4"
 )
 
 // Client is used by the servicers to get Peer information