Use small index cache in tests

This commit is contained in:
Patrick O'Grady 2020-09-18 12:18:57 -07:00
parent 6663cc070d
commit 699e689f2e
No known key found for this signature in database
GPG key ID: 8DE11C985C0C8D85
3 changed files with 26 additions and 10 deletions

View file

@ -33,6 +33,10 @@ import (
) )
const ( const (
// DefaultIndexCacheSize is the default size of the indexer cache. The larger
// the index cache size, the better the performance.
DefaultIndexCacheSize = 5 << 30 // 5 GB
// indexPlaceholder is provided to the syncer // indexPlaceholder is provided to the syncer
// to indicate we should both start from the // to indicate we should both start from the
// last synced block and that we should sync // last synced block and that we should sync
@ -52,10 +56,9 @@ const (
// block fetched by the indexer. // block fetched by the indexer.
sizeMultiplier = 15 sizeMultiplier = 15
// BadgerDB options overrides // Other BadgerDB options overrides
defaultBlockSize = 1 << 20 // use large blocks so less table indexes (1 MB) defaultBlockSize = 1 << 20 // use large blocks so less table indexes (1 MB)
defaultValueThreshold = 0 // put almost everything in value logs (only use table for key) defaultValueThreshold = 0 // put almost everything in value logs (only use table for key)
defaultIndexCacheSize = 5 << 30 // 5 GB
) )
var ( var (
@ -117,11 +120,14 @@ func (i *Indexer) CloseDatabase(ctx context.Context) {
// the Badger default so we have a lot less indexes to store. We also // the Badger default so we have a lot less indexes to store. We also
// ensure all values are stored in value log files (as to minimize // ensure all values are stored in value log files (as to minimize
// table bloat at the cost of some performance). // table bloat at the cost of some performance).
func defaultBadgerOptions(path string) badger.Options { func defaultBadgerOptions(
path string,
indexCacheSize int64,
) badger.Options {
defaultOps := storage.DefaultBadgerOptions(path) defaultOps := storage.DefaultBadgerOptions(path)
defaultOps.BlockSize = defaultBlockSize defaultOps.BlockSize = defaultBlockSize
defaultOps.ValueThreshold = defaultValueThreshold defaultOps.ValueThreshold = defaultValueThreshold
defaultOps.IndexCacheSize = defaultIndexCacheSize defaultOps.IndexCacheSize = indexCacheSize
return defaultOps return defaultOps
} }
@ -132,12 +138,16 @@ func Initialize(
cancel context.CancelFunc, cancel context.CancelFunc,
config *configuration.Configuration, config *configuration.Configuration,
client Client, client Client,
indexCacheSize int64,
) (*Indexer, error) { ) (*Indexer, error) {
localStore, err := storage.NewBadgerStorage( localStore, err := storage.NewBadgerStorage(
ctx, ctx,
config.IndexerPath, config.IndexerPath,
storage.WithCompressorEntries(config.Compressors), storage.WithCompressorEntries(config.Compressors),
storage.WithCustomSettings(defaultBadgerOptions(config.IndexerPath)), storage.WithCustomSettings(defaultBadgerOptions(
config.IndexerPath,
indexCacheSize,
)),
) )
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: unable to initialize storage", err) return nil, fmt.Errorf("%w: unable to initialize storage", err)

View file

@ -72,7 +72,7 @@ func TestIndexer_Pruning(t *testing.T) {
IndexerPath: newDir, IndexerPath: newDir,
} }
i, err := Initialize(ctx, cancel, cfg, mockClient) i, err := Initialize(ctx, cancel, cfg, mockClient, storage.TinyIndexCacheSize)
assert.NoError(t, err) assert.NoError(t, err)
// Waiting for bitcoind... // Waiting for bitcoind...
@ -232,7 +232,7 @@ func TestIndexer_Transactions(t *testing.T) {
IndexerPath: newDir, IndexerPath: newDir,
} }
i, err := Initialize(ctx, cancel, cfg, mockClient) i, err := Initialize(ctx, cancel, cfg, mockClient, storage.TinyIndexCacheSize)
assert.NoError(t, err) assert.NoError(t, err)
// Sync to 1000 // Sync to 1000
@ -450,7 +450,7 @@ func TestIndexer_Reorg(t *testing.T) {
IndexerPath: newDir, IndexerPath: newDir,
} }
i, err := Initialize(ctx, cancel, cfg, mockClient) i, err := Initialize(ctx, cancel, cfg, mockClient, storage.TinyIndexCacheSize)
assert.NoError(t, err) assert.NoError(t, err)
// Sync to 1000 // Sync to 1000
@ -692,7 +692,7 @@ func TestIndexer_HeaderReorg(t *testing.T) {
IndexerPath: newDir, IndexerPath: newDir,
} }
i, err := Initialize(ctx, cancel, cfg, mockClient) i, err := Initialize(ctx, cancel, cfg, mockClient, storage.TinyIndexCacheSize)
assert.NoError(t, err) assert.NoError(t, err)
// Sync to 1000 // Sync to 1000

View file

@ -94,7 +94,13 @@ func startOnlineDependencies(
return bitcoin.StartBitcoind(ctx, cfg.ConfigPath, g) return bitcoin.StartBitcoind(ctx, cfg.ConfigPath, g)
}) })
i, err := indexer.Initialize(ctx, cancel, cfg, client) i, err := indexer.Initialize(
ctx,
cancel,
cfg,
client,
indexer.DefaultIndexCacheSize,
)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("%w: unable to initialize indexer", err) return nil, nil, fmt.Errorf("%w: unable to initialize indexer", err)
} }