From c960310ed73e292a4f3adf9b4449572d1b586b4e Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 15 Oct 2020 15:03:50 -0700 Subject: [PATCH 01/10] Update rosetta-sdk-go --- go.mod | 2 +- go.sum | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 7241541..4427a53 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.13 require ( github.com/btcsuite/btcd v0.21.0-beta github.com/btcsuite/btcutil v1.0.2 - github.com/coinbase/rosetta-sdk-go v0.4.8 + github.com/coinbase/rosetta-sdk-go v0.5.4 github.com/dgraph-io/badger/v2 v2.2007.2 github.com/grpc-ecosystem/go-grpc-middleware v1.2.2 github.com/stretchr/testify v1.6.1 diff --git a/go.sum b/go.sum index 7e8bc51..b445c5c 100644 --- a/go.sum +++ b/go.sum @@ -73,6 +73,8 @@ github.com/coinbase/rosetta-sdk-go v0.4.7 h1:5KFc0CgLMkKamX++hYUFvE58a5/tCn0wSqp github.com/coinbase/rosetta-sdk-go v0.4.7/go.mod h1:8d4iN4VSGvLUzl+jRQlvYSLyS9TeY0QZebneWouizqU= github.com/coinbase/rosetta-sdk-go v0.4.8 h1:+E1TM4q1c5/x/jE9FPI1IZIbNvUWy4tRRgDPLnKzUV4= github.com/coinbase/rosetta-sdk-go v0.4.8/go.mod h1:8d4iN4VSGvLUzl+jRQlvYSLyS9TeY0QZebneWouizqU= +github.com/coinbase/rosetta-sdk-go v0.5.4 h1:pM18LK2ci8zZwIu+uETmP6BXHqZbQGXRulwQCjYudg4= +github.com/coinbase/rosetta-sdk-go v0.5.4/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -314,6 +316,7 @@ github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/sjson v1.1.1 h1:7h1vk049Jnd5EH9NyzNiEuwYW4b5qgreBbqRC19AS3U= github.com/tidwall/sjson v1.1.1/go.mod h1:yvVuSnpEQv5cYIrO+AT6kw4QVfd5SDZoGIS7/5+fZFs= +github.com/tidwall/sjson v1.1.2/go.mod h1:SEzaDwxiPzKzNfUEO4HbYF/m4UCSJDsGgNqsS1LvdoY= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= From 61ddb31e87cc036aa49ec9c242cc75dd77e300cd Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 15 Oct 2020 15:05:22 -0700 Subject: [PATCH 02/10] Remove max heap usage monitoring --- go.sum | 1 + main.go | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.sum b/go.sum index b445c5c..fb3ffec 100644 --- a/go.sum +++ b/go.sum @@ -316,6 +316,7 @@ github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/sjson v1.1.1 h1:7h1vk049Jnd5EH9NyzNiEuwYW4b5qgreBbqRC19AS3U= github.com/tidwall/sjson v1.1.1/go.mod h1:yvVuSnpEQv5cYIrO+AT6kw4QVfd5SDZoGIS7/5+fZFs= +github.com/tidwall/sjson v1.1.2 h1:NC5okI+tQ8OG/oyzchvwXXxRxCV/FVdhODbPKkQ25jQ= github.com/tidwall/sjson v1.1.2/go.mod h1:SEzaDwxiPzKzNfUEO4HbYF/m4UCSJDsGgNqsS1LvdoY= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= diff --git a/main.go b/main.go index 89a7f3d..d5f83f0 100644 --- a/main.go +++ b/main.go @@ -51,10 +51,6 @@ const ( // idleTimeout is the maximum amount of time to wait for the // next request when keep-alives are enabled. idleTimeout = 30 * time.Second - - // maxHeapUsage is the size of the heap in MB before we manually - // trigger garbage collection. - maxHeapUsage = 8500 // ~8.5 GB ) var ( @@ -143,7 +139,7 @@ func main() { g, ctx := errgroup.WithContext(ctx) g.Go(func() error { - return utils.MonitorMemoryUsage(ctx, maxHeapUsage) + return utils.MonitorMemoryUsage(ctx, -1) }) var i *indexer.Indexer @@ -161,6 +157,7 @@ func main() { bitcoin.OperationTypes, false, []*types.NetworkIdentifier{cfg.Network}, + nil, ) if err != nil { logger.Fatalw("unable to create new server asserter", "error", err) From cc0de8ce8ffd327f8b6c3d063364a8af82c0f714 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 15 Oct 2020 15:11:48 -0700 Subject: [PATCH 03/10] Customize storage settings --- indexer/indexer.go | 62 +++++++++++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/indexer/indexer.go b/indexer/indexer.go index b03f29c..1ff222c 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -31,13 +31,10 @@ import ( "github.com/coinbase/rosetta-sdk-go/types" sdkUtils "github.com/coinbase/rosetta-sdk-go/utils" "github.com/dgraph-io/badger/v2" + "github.com/dgraph-io/badger/v2/options" ) const ( - // DefaultIndexCacheSize is the default size of the indexer cache. The larger - // the index cache size, the better the performance. - DefaultIndexCacheSize = 5 << 30 // 5 GB - // indexPlaceholder is provided to the syncer // to indicate we should both start from the // last synced block and that we should sync @@ -56,10 +53,6 @@ const ( // this is the estimated memory overhead for each // block fetched by the indexer. sizeMultiplier = 15 - - // Other BadgerDB options overrides - defaultBlockSize = 1 << 20 // use large blocks so less table indexes (1 MB) - defaultValueThreshold = 0 // put almost everything in value logs (only use table for key) ) var ( @@ -114,23 +107,47 @@ func (i *Indexer) CloseDatabase(ctx context.Context) { } // defaultBadgerOptions returns a set of badger.Options optimized -// for running a Rosetta implementation. After extensive research -// and profiling, we determined that the bottleneck to high RPC -// load was fetching table indexes from disk on an index cache miss. -// Thus, we increased the default block size to be much larger than -// the Badger default so we have a lot less indexes to store. We also -// ensure all values are stored in value log files (as to minimize -// table bloat at the cost of some performance). +// for running a Rosetta implementation. func defaultBadgerOptions( - path string, - indexCacheSize int64, + dir string, ) badger.Options { - defaultOps := storage.DefaultBadgerOptions(path) - defaultOps.BlockSize = defaultBlockSize - defaultOps.ValueThreshold = defaultValueThreshold - defaultOps.IndexCacheSize = indexCacheSize + opts := badger.DefaultOptions(dir) - return defaultOps + // By default, we do not compress the table at all. Doing so can + // significantly increase memory usage. + opts.Compression = options.None + + // Load tables into memory and memory map value logs. + opts.TableLoadingMode = options.MemoryMap + opts.ValueLogLoadingMode = options.MemoryMap + + // Use an extended table size for larger commits. + opts.MaxTableSize = storage.DefaultMaxTableSize + + // To allow writes at a faster speed, we create a new memtable as soon as + // an existing memtable is filled up. This option determines how many + // memtables should be kept in memory. + opts.NumMemtables = 1 + + // Don't keep multiple memtables in memory. With larger + // memtable size, this explodes memory usage. + opts.NumLevelZeroTables = 1 + opts.NumLevelZeroTablesStall = 2 + + // This option will have a significant effect the memory. If the level is kept + // in-memory, read are faster but the tables will be kept in memory. By default, + // this is set to false. + opts.KeepL0InMemory = false + + // We don't compact L0 on close as this can greatly delay shutdown time. + opts.CompactL0OnClose = false + + // LoadBloomsOnOpen=false will improve the db startup speed. This is also + // a waste to enable with a limited index cache size (as many of the loaded bloom + // filters will be immediately discarded from the cache). + opts.LoadBloomsOnOpen = false + + return opts } // Initialize returns a new Indexer. @@ -147,7 +164,6 @@ func Initialize( storage.WithCompressorEntries(config.Compressors), storage.WithCustomSettings(defaultBadgerOptions( config.IndexerPath, - indexCacheSize, )), ) if err != nil { From 37264a71c7a4e6ce86c6898e04e2773e7c457ca3 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 15 Oct 2020 15:15:25 -0700 Subject: [PATCH 04/10] Add value log file size back --- indexer/indexer.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/indexer/indexer.go b/indexer/indexer.go index 1ff222c..e6aeba7 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -124,6 +124,10 @@ func defaultBadgerOptions( // Use an extended table size for larger commits. opts.MaxTableSize = storage.DefaultMaxTableSize + // Smaller value log sizes means smaller contiguous memory allocations + // and less RAM usage on cleanup. + opts.ValueLogFileSize = storage.DefaultLogValueSize + // To allow writes at a faster speed, we create a new memtable as soon as // an existing memtable is filled up. This option determines how many // memtables should be kept in memory. From c037fdefa2f28afed247457054b543760e348463 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 15 Oct 2020 15:16:46 -0700 Subject: [PATCH 05/10] Update bitcoin versions --- services/types.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/types.go b/services/types.go index 5d83625..706ee7b 100644 --- a/services/types.go +++ b/services/types.go @@ -25,7 +25,7 @@ import ( const ( // RosettaVersion is the version of the // Rosetta Specification we are using. - RosettaVersion = "1.4.4" + RosettaVersion = "1.4.5" // NodeVersion is the version of // bitcoin core we are using. @@ -38,7 +38,7 @@ var ( // variable instead of a constant because // we typically need the pointer of this // value. - MiddlewareVersion = "0.0.3" + MiddlewareVersion = "0.0.4" ) // Client is used by the servicers to get Peer information From f7fbba6e37c32a8ef84df30948a376cf9ae8ac0f Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 15 Oct 2020 15:39:43 -0700 Subject: [PATCH 06/10] Remove indexer cache size --- indexer/indexer.go | 1 - indexer/indexer_test.go | 8 ++++---- main.go | 1 - 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/indexer/indexer.go b/indexer/indexer.go index e6aeba7..e8b7edb 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -160,7 +160,6 @@ func Initialize( cancel context.CancelFunc, config *configuration.Configuration, client Client, - indexCacheSize int64, ) (*Indexer, error) { localStore, err := storage.NewBadgerStorage( ctx, diff --git a/indexer/indexer_test.go b/indexer/indexer_test.go index c42aa6a..314a2f5 100644 --- a/indexer/indexer_test.go +++ b/indexer/indexer_test.go @@ -72,7 +72,7 @@ func TestIndexer_Pruning(t *testing.T) { IndexerPath: newDir, } - i, err := Initialize(ctx, cancel, cfg, mockClient, storage.TinyIndexCacheSize) + i, err := Initialize(ctx, cancel, cfg, mockClient) assert.NoError(t, err) // Waiting for bitcoind... @@ -232,7 +232,7 @@ func TestIndexer_Transactions(t *testing.T) { IndexerPath: newDir, } - i, err := Initialize(ctx, cancel, cfg, mockClient, storage.TinyIndexCacheSize) + i, err := Initialize(ctx, cancel, cfg, mockClient) assert.NoError(t, err) // Sync to 1000 @@ -450,7 +450,7 @@ func TestIndexer_Reorg(t *testing.T) { IndexerPath: newDir, } - i, err := Initialize(ctx, cancel, cfg, mockClient, storage.TinyIndexCacheSize) + i, err := Initialize(ctx, cancel, cfg, mockClient) assert.NoError(t, err) // Sync to 1000 @@ -692,7 +692,7 @@ func TestIndexer_HeaderReorg(t *testing.T) { IndexerPath: newDir, } - i, err := Initialize(ctx, cancel, cfg, mockClient, storage.TinyIndexCacheSize) + i, err := Initialize(ctx, cancel, cfg, mockClient) assert.NoError(t, err) // Sync to 1000 diff --git a/main.go b/main.go index d5f83f0..6886f69 100644 --- a/main.go +++ b/main.go @@ -95,7 +95,6 @@ func startOnlineDependencies( cancel, cfg, client, - indexer.DefaultIndexCacheSize, ) if err != nil { return nil, nil, fmt.Errorf("%w: unable to initialize indexer", err) From 84abee7b2c81f8a0adca1eca639d46ad341e0783 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 16 Oct 2020 07:53:32 -0700 Subject: [PATCH 07/10] Add note about memory mapped files --- README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9d53b17..e804c0c 100644 --- a/README.md +++ b/README.md @@ -85,10 +85,9 @@ _If you cloned the repository, you can run `make run-testnet-offline`._ ## System Requirements `rosetta-bitcoin` has been tested on an [AWS c5.2xlarge instance](https://aws.amazon.com/ec2/instance-types/c5). -This instance type has 8 vCPU and 16 GB of RAM. If you use a computer with less than 16 GB of RAM, -it is possible that `rosetta-bitcoin` will exit with an OOM error. +This instance type has 8 vCPU and 16 GB of RAM. -### Recommended OS Settings +### Network Settings To increase the load `rosetta-bitcoin` can handle, it is recommended to tune your OS settings to allow for more connections. On a linux-based OS, you can run the following commands ([source](http://www.tweaked.io/guide/kernel)): @@ -106,6 +105,12 @@ enabling it._ You should also modify your open file settings to `100000`. This can be done on a linux-based OS with the command: `ulimit -n 100000`. +### Memory-Mapped Files +`rosetta-bitcoin` uses [memory-mapped files](https://en.wikipedia.org/wiki/Memory-mapped_file) to +persist data in the `indexer`. As a result, you **must** run `rosetta-bitcoin` on a 64-bit +architecture (the virtual address space easily exceeds 100s of GBs) and may need to increase the +allocated size of swap space to avoid a kernel OOM. + ## Architecture `rosetta-bitcoin` uses the `syncer`, `storage`, `parser`, and `server` package from [`rosetta-sdk-go`](https://github.com/coinbase/rosetta-sdk-go) instead From 4fa7f1f4590884ce59a6844475303799ac7839dc Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 16 Oct 2020 07:54:35 -0700 Subject: [PATCH 08/10] Fix tests --- services/network_service_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/network_service_test.go b/services/network_service_test.go index d60d6ed..363aff2 100644 --- a/services/network_service_test.go +++ b/services/network_service_test.go @@ -27,10 +27,10 @@ import ( ) var ( - middlewareVersion = "0.0.3" + middlewareVersion = "0.0.4" defaultNetworkOptions = &types.NetworkOptionsResponse{ Version: &types.Version{ - RosettaVersion: "1.4.4", + RosettaVersion: "1.4.5", NodeVersion: "0.20.1", MiddlewareVersion: &middlewareVersion, }, From 7487bf091fd5f5e855f298d51dde2d28f2ecb652 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 16 Oct 2020 08:06:13 -0700 Subject: [PATCH 09/10] Add instructions for increasing swap space --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e804c0c..5865e97 100644 --- a/README.md +++ b/README.md @@ -108,8 +108,10 @@ with the command: `ulimit -n 100000`. ### Memory-Mapped Files `rosetta-bitcoin` uses [memory-mapped files](https://en.wikipedia.org/wiki/Memory-mapped_file) to persist data in the `indexer`. As a result, you **must** run `rosetta-bitcoin` on a 64-bit -architecture (the virtual address space easily exceeds 100s of GBs) and may need to increase the -allocated size of swap space to avoid a kernel OOM. +architecture (the virtual address space easily exceeds 100s of GBs). + +If you receive a kernel OOM, you may need to increase the allocated size of swap space +on your OS. There is a great tutorial for how to do this on Linux [here](https://linuxize.com/post/create-a-linux-swap-file/). ## Architecture `rosetta-bitcoin` uses the `syncer`, `storage`, `parser`, and `server` package From 6d2c1ba29affcecbf06a40ab1c6b753f62f79b1e Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Fri, 16 Oct 2020 09:27:55 -0700 Subject: [PATCH 10/10] Update rosetta-sdk-go --- go.mod | 2 +- go.sum | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 4427a53..f591efb 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.13 require ( github.com/btcsuite/btcd v0.21.0-beta github.com/btcsuite/btcutil v1.0.2 - github.com/coinbase/rosetta-sdk-go v0.5.4 + github.com/coinbase/rosetta-sdk-go v0.5.5 github.com/dgraph-io/badger/v2 v2.2007.2 github.com/grpc-ecosystem/go-grpc-middleware v1.2.2 github.com/stretchr/testify v1.6.1 diff --git a/go.sum b/go.sum index fb3ffec..b0b409f 100644 --- a/go.sum +++ b/go.sum @@ -75,6 +75,8 @@ github.com/coinbase/rosetta-sdk-go v0.4.8 h1:+E1TM4q1c5/x/jE9FPI1IZIbNvUWy4tRRgD github.com/coinbase/rosetta-sdk-go v0.4.8/go.mod h1:8d4iN4VSGvLUzl+jRQlvYSLyS9TeY0QZebneWouizqU= github.com/coinbase/rosetta-sdk-go v0.5.4 h1:pM18LK2ci8zZwIu+uETmP6BXHqZbQGXRulwQCjYudg4= github.com/coinbase/rosetta-sdk-go v0.5.4/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws= +github.com/coinbase/rosetta-sdk-go v0.5.5 h1:Z61/VUO89BDVl1m6Zj0e6OWMl5GnVevj4z79Eh3sSL0= +github.com/coinbase/rosetta-sdk-go v0.5.5/go.mod h1:JRO4BJjhWAI7nYGwzYZWFgYfCTsQOvdZB7tGyN6kEtE= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -118,6 +120,7 @@ github.com/ethereum/go-ethereum v1.9.21 h1:8qRlhzrItnmUGdVlBzZLI2Tb46S0RdSNjFwIC github.com/ethereum/go-ethereum v1.9.21/go.mod h1:RXAVzbGrSGmDkDnHymruTAIEjUR3E4TX0EOpaj702sI= github.com/ethereum/go-ethereum v1.9.22 h1:/Fea9n2EWJuNJ9oahMq9luqjRBcbW7QWdThbcJl13ek= github.com/ethereum/go-ethereum v1.9.22/go.mod h1:FQjK3ZwD8C5DYn7ukTmFee36rq1dOMESiUfXr5RUc1w= +github.com/ethereum/go-ethereum v1.9.23/go.mod h1:JIfVb6esrqALTExdz9hRYvrP0xBDf6wCncIu1hNwHpM= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=