LBRYCommunity/rosetta-lbry
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity

Merge pull request #25 from coinbase/patrick/upgrade-sdk-go

Browse source 
Update rosetta-sdk-go + mmap files
This commit is contained in:
Patrick O'Grady 2020-10-16 09:44:21 -07:00 committed by GitHub
commit 2fc421ab6e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 71 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
README.md
View file

@ -85,10 +85,9 @@ _If you cloned the repository, you can run `make run-testnet-offline`._
## System Requirements
`rosetta-bitcoin` has been tested on an [AWS c5.2xlarge instance](https://aws.amazon.com/ec2/instance-types/c5).
This instance type has 8 vCPU and 16 GB of RAM. If you use a computer with less than 16 GB of RAM,
it is possible that `rosetta-bitcoin` will exit with an OOM error.
This instance type has 8 vCPU and 16 GB of RAM.
### Recommended OS Settings
### Network Settings
To increase the load `rosetta-bitcoin` can handle, it is recommended to tune your OS
settings to allow for more connections. On a linux-based OS, you can run the following
commands ([source](http://www.tweaked.io/guide/kernel)):
@ -106,6 +105,14 @@ enabling it._
You should also modify your open file settings to `100000`. This can be done on a linux-based OS
with the command: `ulimit -n 100000`.
### Memory-Mapped Files
`rosetta-bitcoin` uses [memory-mapped files](https://en.wikipedia.org/wiki/Memory-mapped_file) to
persist data in the `indexer`. As a result, you **must** run `rosetta-bitcoin` on a 64-bit
architecture (the virtual address space easily exceeds 100s of GBs).
If you receive a kernel OOM, you may need to increase the allocated size of swap space
on your OS. There is a great tutorial for how to do this on Linux [here](https://linuxize.com/post/create-a-linux-swap-file/).
## Architecture
`rosetta-bitcoin` uses the `syncer`, `storage`, `parser`, and `server` package
from [`rosetta-sdk-go`](https://github.com/coinbase/rosetta-sdk-go) instead

2
go.mod
View file

@ -5,7 +5,7 @@ go 1.13
require (
github.com/btcsuite/btcd v0.21.0-beta
github.com/btcsuite/btcutil v1.0.2
github.com/coinbase/rosetta-sdk-go v0.4.8
github.com/coinbase/rosetta-sdk-go v0.5.5
github.com/dgraph-io/badger/v2 v2.2007.2
github.com/grpc-ecosystem/go-grpc-middleware v1.2.2
github.com/stretchr/testify v1.6.1

7
go.sum
View file

@ -73,6 +73,10 @@ github.com/coinbase/rosetta-sdk-go v0.4.7 h1:5KFc0CgLMkKamX++hYUFvE58a5/tCn0wSqp
github.com/coinbase/rosetta-sdk-go v0.4.7/go.mod h1:8d4iN4VSGvLUzl+jRQlvYSLyS9TeY0QZebneWouizqU=
github.com/coinbase/rosetta-sdk-go v0.4.8 h1:+E1TM4q1c5/x/jE9FPI1IZIbNvUWy4tRRgDPLnKzUV4=
github.com/coinbase/rosetta-sdk-go v0.4.8/go.mod h1:8d4iN4VSGvLUzl+jRQlvYSLyS9TeY0QZebneWouizqU=
github.com/coinbase/rosetta-sdk-go v0.5.4 h1:pM18LK2ci8zZwIu+uETmP6BXHqZbQGXRulwQCjYudg4=
github.com/coinbase/rosetta-sdk-go v0.5.4/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws=
github.com/coinbase/rosetta-sdk-go v0.5.5 h1:Z61/VUO89BDVl1m6Zj0e6OWMl5GnVevj4z79Eh3sSL0=
github.com/coinbase/rosetta-sdk-go v0.5.5/go.mod h1:JRO4BJjhWAI7nYGwzYZWFgYfCTsQOvdZB7tGyN6kEtE=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
@ -116,6 +120,7 @@ github.com/ethereum/go-ethereum v1.9.21 h1:8qRlhzrItnmUGdVlBzZLI2Tb46S0RdSNjFwIC
github.com/ethereum/go-ethereum v1.9.21/go.mod h1:RXAVzbGrSGmDkDnHymruTAIEjUR3E4TX0EOpaj702sI=
github.com/ethereum/go-ethereum v1.9.22 h1:/Fea9n2EWJuNJ9oahMq9luqjRBcbW7QWdThbcJl13ek=
github.com/ethereum/go-ethereum v1.9.22/go.mod h1:FQjK3ZwD8C5DYn7ukTmFee36rq1dOMESiUfXr5RUc1w=
github.com/ethereum/go-ethereum v1.9.23/go.mod h1:JIfVb6esrqALTExdz9hRYvrP0xBDf6wCncIu1hNwHpM=
github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
@ -314,6 +319,8 @@ github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU=
github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tidwall/sjson v1.1.1 h1:7h1vk049Jnd5EH9NyzNiEuwYW4b5qgreBbqRC19AS3U=
github.com/tidwall/sjson v1.1.1/go.mod h1:yvVuSnpEQv5cYIrO+AT6kw4QVfd5SDZoGIS7/5+fZFs=
github.com/tidwall/sjson v1.1.2 h1:NC5okI+tQ8OG/oyzchvwXXxRxCV/FVdhODbPKkQ25jQ=
github.com/tidwall/sjson v1.1.2/go.mod h1:SEzaDwxiPzKzNfUEO4HbYF/m4UCSJDsGgNqsS1LvdoY=
github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=

67
indexer/indexer.go
View file

@ -31,13 +31,10 @@ import (
"github.com/coinbase/rosetta-sdk-go/types"
sdkUtils "github.com/coinbase/rosetta-sdk-go/utils"
"github.com/dgraph-io/badger/v2"
"github.com/dgraph-io/badger/v2/options"
)
const (
// DefaultIndexCacheSize is the default size of the indexer cache. The larger
// the index cache size, the better the performance.
DefaultIndexCacheSize = 5 << 30 // 5 GB
// indexPlaceholder is provided to the syncer
// to indicate we should both start from the
// last synced block and that we should sync
@ -56,10 +53,6 @@ const (
// this is the estimated memory overhead for each
// block fetched by the indexer.
sizeMultiplier = 15
// Other BadgerDB options overrides
defaultBlockSize = 1 << 20 // use large blocks so less table indexes (1 MB)
defaultValueThreshold = 0 // put almost everything in value logs (only use table for key)
)
var (
@ -114,23 +107,51 @@ func (i *Indexer) CloseDatabase(ctx context.Context) {
}
// defaultBadgerOptions returns a set of badger.Options optimized
// for running a Rosetta implementation. After extensive research
// and profiling, we determined that the bottleneck to high RPC
// load was fetching table indexes from disk on an index cache miss.
// Thus, we increased the default block size to be much larger than
// the Badger default so we have a lot less indexes to store. We also
// ensure all values are stored in value log files (as to minimize
// table bloat at the cost of some performance).
// for running a Rosetta implementation.
func defaultBadgerOptions(
path string,
indexCacheSize int64,
dir string,
) badger.Options {
defaultOps := storage.DefaultBadgerOptions(path)
defaultOps.BlockSize = defaultBlockSize
defaultOps.ValueThreshold = defaultValueThreshold
defaultOps.IndexCacheSize = indexCacheSize
opts := badger.DefaultOptions(dir)
return defaultOps
// By default, we do not compress the table at all. Doing so can
// significantly increase memory usage.
opts.Compression = options.None
// Load tables into memory and memory map value logs.
opts.TableLoadingMode = options.MemoryMap
opts.ValueLogLoadingMode = options.MemoryMap
// Use an extended table size for larger commits.
opts.MaxTableSize = storage.DefaultMaxTableSize
// Smaller value log sizes means smaller contiguous memory allocations
// and less RAM usage on cleanup.
opts.ValueLogFileSize = storage.DefaultLogValueSize
// To allow writes at a faster speed, we create a new memtable as soon as
// an existing memtable is filled up. This option determines how many
// memtables should be kept in memory.
opts.NumMemtables = 1
// Don't keep multiple memtables in memory. With larger
// memtable size, this explodes memory usage.
opts.NumLevelZeroTables = 1
opts.NumLevelZeroTablesStall = 2
// This option will have a significant effect the memory. If the level is kept
// in-memory, read are faster but the tables will be kept in memory. By default,
// this is set to false.
opts.KeepL0InMemory = false
// We don't compact L0 on close as this can greatly delay shutdown time.
opts.CompactL0OnClose = false
// LoadBloomsOnOpen=false will improve the db startup speed. This is also
// a waste to enable with a limited index cache size (as many of the loaded bloom
// filters will be immediately discarded from the cache).
opts.LoadBloomsOnOpen = false
return opts
}
// Initialize returns a new Indexer.
@ -139,7 +160,6 @@ func Initialize(
cancel context.CancelFunc,
config *configuration.Configuration,
client Client,
indexCacheSize int64,
) (*Indexer, error) {
localStore, err := storage.NewBadgerStorage(
ctx,
@ -147,7 +167,6 @@ func Initialize(
storage.WithCompressorEntries(config.Compressors),
storage.WithCustomSettings(defaultBadgerOptions(
config.IndexerPath,
indexCacheSize,
)),
)
if err != nil {

8
indexer/indexer_test.go
View file

@ -72,7 +72,7 @@ func TestIndexer_Pruning(t *testing.T) {
IndexerPath: newDir,
}
i, err := Initialize(ctx, cancel, cfg, mockClient, storage.TinyIndexCacheSize)
i, err := Initialize(ctx, cancel, cfg, mockClient)
assert.NoError(t, err)
// Waiting for bitcoind...
@ -232,7 +232,7 @@ func TestIndexer_Transactions(t *testing.T) {
IndexerPath: newDir,
}
i, err := Initialize(ctx, cancel, cfg, mockClient, storage.TinyIndexCacheSize)
i, err := Initialize(ctx, cancel, cfg, mockClient)
assert.NoError(t, err)
// Sync to 1000
@ -450,7 +450,7 @@ func TestIndexer_Reorg(t *testing.T) {
IndexerPath: newDir,
}
i, err := Initialize(ctx, cancel, cfg, mockClient, storage.TinyIndexCacheSize)
i, err := Initialize(ctx, cancel, cfg, mockClient)
assert.NoError(t, err)
// Sync to 1000
@ -692,7 +692,7 @@ func TestIndexer_HeaderReorg(t *testing.T) {
IndexerPath: newDir,
}
i, err := Initialize(ctx, cancel, cfg, mockClient, storage.TinyIndexCacheSize)
i, err := Initialize(ctx, cancel, cfg, mockClient)
assert.NoError(t, err)
// Sync to 1000

8
main.go
View file

@ -51,10 +51,6 @@ const (
// idleTimeout is the maximum amount of time to wait for the
// next request when keep-alives are enabled.
idleTimeout = 30 * time.Second
// maxHeapUsage is the size of the heap in MB before we manually
// trigger garbage collection.
maxHeapUsage = 8500 // ~8.5 GB
)
var (
@ -99,7 +95,6 @@ func startOnlineDependencies(
cancel,
cfg,
client,
indexer.DefaultIndexCacheSize,
)
if err != nil {
return nil, nil, fmt.Errorf("%w: unable to initialize indexer", err)
@ -143,7 +138,7 @@ func main() {
g, ctx := errgroup.WithContext(ctx)
g.Go(func() error {
return utils.MonitorMemoryUsage(ctx, maxHeapUsage)
return utils.MonitorMemoryUsage(ctx, -1)
})
var i *indexer.Indexer
@ -161,6 +156,7 @@ func main() {
bitcoin.OperationTypes,
false,
[]*types.NetworkIdentifier{cfg.Network},
nil,
)
if err != nil {
logger.Fatalw("unable to create new server asserter", "error", err)

4
services/network_service_test.go
View file

@ -27,10 +27,10 @@ import (
)
var (
middlewareVersion = "0.0.3"
middlewareVersion = "0.0.4"
defaultNetworkOptions = &types.NetworkOptionsResponse{
Version: &types.Version{
RosettaVersion: "1.4.4",
RosettaVersion: "1.4.5",
NodeVersion: "0.20.1",
MiddlewareVersion: &middlewareVersion,
},

4
services/types.go
View file

@ -25,7 +25,7 @@ import (
const (
// RosettaVersion is the version of the
// Rosetta Specification we are using.
RosettaVersion = "1.4.4"
RosettaVersion = "1.4.5"
// NodeVersion is the version of
// bitcoin core we are using.
@ -38,7 +38,7 @@ var (
// variable instead of a constant because
// we typically need the pointer of this
// value.
MiddlewareVersion = "0.0.3"
MiddlewareVersion = "0.0.4"
)
// Client is used by the servicers to get Peer information