From 8e4e193a30f0560c60569bdf82a9f3bd110c5fec Mon Sep 17 00:00:00 2001 From: Jeffrey Picard Date: Sun, 12 Dec 2021 18:35:16 -0500 Subject: [PATCH] Fix rocksdb iterator and tests --- db/db.go | 165 ++++++++++++------ db/db_test.go | 11 +- main.go | 12 +- resources/asdf.db/000005.log | Bin 0 -> 576 bytes resources/asdf.db/CURRENT | 1 + resources/asdf.db/IDENTITY | 1 + resources/asdf.db/LOCK | 0 resources/asdf.db/LOG | 270 ++++++++++++++++++++++++++++++ resources/asdf.db/MANIFEST-000004 | Bin 0 -> 57 bytes resources/asdf.db/OPTIONS-000007 | 187 +++++++++++++++++++++ 10 files changed, 591 insertions(+), 56 deletions(-) create mode 100644 resources/asdf.db/000005.log create mode 100644 resources/asdf.db/CURRENT create mode 100644 resources/asdf.db/IDENTITY create mode 100644 resources/asdf.db/LOCK create mode 100644 resources/asdf.db/LOG create mode 100644 resources/asdf.db/MANIFEST-000004 create mode 100644 resources/asdf.db/OPTIONS-000007 diff --git a/db/db.go b/db/db.go index 1df02df..a9b4689 100644 --- a/db/db.go +++ b/db/db.go @@ -13,9 +13,13 @@ import ( ) type IterOptions struct { - FillCache bool - Start []byte //interface{} - Stop []byte //interface{} + FillCache bool + Start []byte //interface{} + Stop []byte //interface{} + IncludeStart bool + IncludeStop bool + IncludeKey bool + IncludeValue bool } type PrefixRow struct { @@ -45,11 +49,24 @@ type UTXOValue struct { Amount uint64 } +// NewIterateOptions creates a defualt options structure for a db iterator. +// Default values: +// FillCache: false, +// Start: nil, +// Stop: nil, +// IncludeStart: true, +// IncludeStop: false, +// IncludeKey: true, +// IncludeValue: false, func NewIterateOptions() *IterOptions { return &IterOptions{ - FillCache: false, - Start: nil, - Stop: nil, + FillCache: false, + Start: nil, + Stop: nil, + IncludeStart: true, + IncludeStop: false, + IncludeKey: true, + IncludeValue: false, } } @@ -68,6 +85,26 @@ func (o *IterOptions) WithStop(stop []byte) *IterOptions { return o } +func (o *IterOptions) WithIncludeStart(includeStart bool) *IterOptions { + o.IncludeStart = includeStart + return o +} + +func (o *IterOptions) WithIncludeStop(includeStop bool) *IterOptions { + o.IncludeStop = includeStop + return o +} + +func (o *IterOptions) WithIncludeKey(includeKey bool) *IterOptions { + o.IncludeKey = includeKey + return o +} + +func (o *IterOptions) WithIncludeValue(includeValue bool) *IterOptions { + o.IncludeValue = includeValue + return o +} + func (k *UTXOKey) String() string { return fmt.Sprintf( "%s(hashX=%s, tx_num=%d, nout=%d)", @@ -93,49 +130,72 @@ func (pr *PrefixRow) Iter(options *IterOptions) <-chan *PrefixRowKV { log.Println("Not seeking to start") } - /* - def _check_stop_iteration(self, key: bytes): - if self.stop is not None and (key.startswith(self.stop) or self.stop < key[:len(self.stop)]): - raise StopIteration - elif self.start is not None and self.start > key[:len(self.start)]: - raise StopIteration - elif self.prefix is not None and not key.startswith(self.prefix): - raise StopIteration - */ - terminateFunc := func(key []byte) bool { + stopIteration := func(key []byte) bool { if key == nil { - return true + return false } if options.Stop != nil && (bytes.HasPrefix(key, options.Stop) || bytes.Compare(options.Stop, key[:len(options.Stop)]) < 0) { - return false + return true } else if options.Start != nil && bytes.Compare(options.Start, key[:len(options.Start)]) > 0 { - return false + return true } else if pr.Prefix != nil && !bytes.HasPrefix(key, pr.Prefix) { - return false + return true } - return true + return false } - var prevKey []byte = nil go func() { defer it.Close() defer close(ch) - for ; terminateFunc(prevKey); it.Next() { - key := it.Key() - prevKey = key.Data() - value := it.Value() - ch <- &PrefixRowKV{ - Key: key.Data(), - Value: value.Data(), + if !options.IncludeStart { + it.Next() + } + var prevKey []byte = nil + for ; !stopIteration(prevKey); it.Next() { + key := it.Key() + keyData := key.Data() + keyLen := len(keyData) + value := it.Value() + valueData := value.Data() + valueLen := len(valueData) + + // We need to check the current key is we're not including the stop + // key. + if !options.IncludeStop && stopIteration(keyData) { + return + } + + var outputKeyData []byte = nil + // We have to copy the key no matter what because we need to check + // it on the next iterations to see if we're going to stop. + newKeyData := make([]byte, keyLen) + copy(newKeyData, keyData) + if options.IncludeKey { + outputKeyData = newKeyData + } + + var newValueData []byte = nil + // Value could be quite large, so this setting could be important + // for performance in some cases. + if options.IncludeValue { + newValueData = make([]byte, valueLen) + copy(newValueData, valueData) } key.Free() value.Free() + + ch <- &PrefixRowKV{ + Key: outputKeyData, + Value: newValueData, + } + prevKey = newKeyData + } }() @@ -230,7 +290,8 @@ func UTXOValueUnpack(value []byte) *UTXOValue { func GetDB(name string) (*grocksdb.DB, error) { opts := grocksdb.NewDefaultOptions() - db, err := grocksdb.OpenDb(opts, name) + // db, err := grocksdb.OpenDb(opts, name) + db, err := grocksdb.OpenDbAsSecondary(opts, name, "asdf") if err != nil { return nil, err } @@ -287,7 +348,6 @@ func OpenDB(name string, start string) int { var i = 0 it.Seek([]byte(start)) - // it.Seek([]byte{'u'}) for ; it.Valid(); it.Next() { key := it.Key() value := it.Value() @@ -319,34 +379,31 @@ func OpenAndWriteDB(prIn *PrefixRow, options *IterOptions, out string) { ch := prIn.Iter(options) var i = 0 - var prevKey []byte = nil for kv := range ch { - log.Println(kv) key := kv.Key value := kv.Value - unpackKeyFnValue := reflect.ValueOf(prIn.KeyUnpackFunc) - keyArgs := []reflect.Value{reflect.ValueOf(key)} - unpackKeyFnResult := unpackKeyFnValue.Call(keyArgs) - unpackedKey := unpackKeyFnResult[0].Interface() //.(reflect.TypeOf()) + var unpackedKey *UTXOKey = nil + var unpackedValue *UTXOValue = nil - unpackValueFnValue := reflect.ValueOf(prIn.ValueUnpackFunc) - valueArgs := []reflect.Value{reflect.ValueOf(value)} - unpackValueFnResult := unpackValueFnValue.Call(valueArgs) - unpackedValue := unpackValueFnResult[0].Interface() //.([]byte) - - log.Println(unpackedKey) - log.Println(unpackedValue) - - if bytes.Equal(prevKey, key) { - if err := db.Merge(wo, key, value); err != nil { - log.Println(err) - } - } else { - if err := db.Put(wo, key, value); err != nil { - log.Println(err) - } + if key != nil { + unpackKeyFnValue := reflect.ValueOf(prIn.KeyUnpackFunc) + keyArgs := []reflect.Value{reflect.ValueOf(key)} + unpackKeyFnResult := unpackKeyFnValue.Call(keyArgs) + unpackedKey = unpackKeyFnResult[0].Interface().(*UTXOKey) + } + + if value != nil { + unpackValueFnValue := reflect.ValueOf(prIn.ValueUnpackFunc) + valueArgs := []reflect.Value{reflect.ValueOf(value)} + unpackValueFnResult := unpackValueFnValue.Call(valueArgs) + unpackedValue = unpackValueFnResult[0].Interface().(*UTXOValue) + } + + log.Println(unpackedKey, unpackedValue) + + if err := db.Put(wo, key, value); err != nil { + log.Println(err) } - prevKey = key i++ } } diff --git a/db/db_test.go b/db/db_test.go index 758ac1a..4f56238 100644 --- a/db/db_test.go +++ b/db/db_test.go @@ -50,7 +50,16 @@ func TestReadUTXO2(t *testing.T) { } stop := UTXOKeyPackPartial(stopKey, 1) - options := NewIterateOptions().WithFillCache(false).WithStop(stop) + options := &IterOptions{ + FillCache: false, + Start: nil, + Stop: stop, + IncludeStart: true, + IncludeStop: false, + IncludeKey: true, + IncludeValue: true, + } + log.Println(options) ch := utxoRow.Iter(options) diff --git a/main.go b/main.go index a6d7e68..903283c 100644 --- a/main.go +++ b/main.go @@ -58,7 +58,17 @@ func main() { } stop := db.UTXOKeyPackPartial(stopKey, 1) - options := db.NewIterateOptions().WithFillCache(false).WithStop(stop) + log.Print(hex.EncodeToString(stop)) + + options := &db.IterOptions{ + FillCache: false, + Start: nil, + Stop: stop, + IncludeStart: true, + IncludeStop: false, + IncludeKey: true, + IncludeValue: true, + } db.OpenAndWriteDB(pr, options, "./resources/asdf.db") diff --git a/resources/asdf.db/000005.log b/resources/asdf.db/000005.log new file mode 100644 index 0000000000000000000000000000000000000000..fb1ba4de01bcbb459f84f923deaa27a1ceceb603 GIT binary patch literal 576 zcmZ3zg5i%Q10y2?1b}Epp;92pa%jzUwf>?njC&4BGca<1r4{v>g73Ph1JyG@lrtdI z|Irq^_2*s2LPqA$#|#W$_1v!yF?_3f8wpg;jIN#|Um2GRd4=3@UE2UL0K=% z{hX#{lm5i90@bsjtLJZ3s`}mgq-7Ihzm6DGJxfK#fqSRy?g7=aqpKI-o}j;IQH*{K zQzG{QB=?AJ{`3~89ui)_umtJ@ho?}m-=VX0oMs1@&n5!^I2$i)VkV literal 0 HcmV?d00001 diff --git a/resources/asdf.db/CURRENT b/resources/asdf.db/CURRENT new file mode 100644 index 0000000..cacca75 --- /dev/null +++ b/resources/asdf.db/CURRENT @@ -0,0 +1 @@ +MANIFEST-000004 diff --git a/resources/asdf.db/IDENTITY b/resources/asdf.db/IDENTITY new file mode 100644 index 0000000..e56c382 --- /dev/null +++ b/resources/asdf.db/IDENTITY @@ -0,0 +1 @@ +93687483-be4b-4c35-b561-8349e3fc224a \ No newline at end of file diff --git a/resources/asdf.db/LOCK b/resources/asdf.db/LOCK new file mode 100644 index 0000000..e69de29 diff --git a/resources/asdf.db/LOG b/resources/asdf.db/LOG new file mode 100644 index 0000000..40a8dca --- /dev/null +++ b/resources/asdf.db/LOG @@ -0,0 +1,270 @@ +2021/12/12-18:13:31.237469 7eff8ad55b00 RocksDB version: 6.26.1 +2021/12/12-18:13:31.237566 7eff8ad55b00 Git sha 4d57a393a8f1563c0dff1ff468756c23d89e3601 +2021/12/12-18:13:31.237568 7eff8ad55b00 Compile date 2021-11-18 14:47:16 +2021/12/12-18:13:31.237569 7eff8ad55b00 DB SUMMARY +2021/12/12-18:13:31.237570 7eff8ad55b00 DB Session ID: ID4IHW7V5HHX8XDOLN6H +2021/12/12-18:13:31.237577 7eff8ad55b00 SST files in ./resources/asdf.db dir, Total Num: 0, files: +2021/12/12-18:13:31.237578 7eff8ad55b00 Write Ahead Log file in ./resources/asdf.db: +2021/12/12-18:13:31.237579 7eff8ad55b00 Options.error_if_exists: 0 +2021/12/12-18:13:31.237579 7eff8ad55b00 Options.create_if_missing: 1 +2021/12/12-18:13:31.237580 7eff8ad55b00 Options.paranoid_checks: 1 +2021/12/12-18:13:31.237580 7eff8ad55b00 Options.flush_verify_memtable_count: 1 +2021/12/12-18:13:31.237581 7eff8ad55b00 Options.track_and_verify_wals_in_manifest: 0 +2021/12/12-18:13:31.237581 7eff8ad55b00 Options.env: 0x7eff8bc810c0 +2021/12/12-18:13:31.237582 7eff8ad55b00 Options.fs: Posix File System +2021/12/12-18:13:31.237582 7eff8ad55b00 Options.info_log: 0x1f834e0 +2021/12/12-18:13:31.237583 7eff8ad55b00 Options.max_file_opening_threads: 16 +2021/12/12-18:13:31.237583 7eff8ad55b00 Options.statistics: (nil) +2021/12/12-18:13:31.237584 7eff8ad55b00 Options.use_fsync: 0 +2021/12/12-18:13:31.237585 7eff8ad55b00 Options.max_log_file_size: 0 +2021/12/12-18:13:31.237585 7eff8ad55b00 Options.max_manifest_file_size: 1073741824 +2021/12/12-18:13:31.237586 7eff8ad55b00 Options.log_file_time_to_roll: 0 +2021/12/12-18:13:31.237586 7eff8ad55b00 Options.keep_log_file_num: 1000 +2021/12/12-18:13:31.237587 7eff8ad55b00 Options.recycle_log_file_num: 0 +2021/12/12-18:13:31.237587 7eff8ad55b00 Options.allow_fallocate: 1 +2021/12/12-18:13:31.237588 7eff8ad55b00 Options.allow_mmap_reads: 0 +2021/12/12-18:13:31.237588 7eff8ad55b00 Options.allow_mmap_writes: 0 +2021/12/12-18:13:31.237589 7eff8ad55b00 Options.use_direct_reads: 0 +2021/12/12-18:13:31.237589 7eff8ad55b00 Options.use_direct_io_for_flush_and_compaction: 0 +2021/12/12-18:13:31.237590 7eff8ad55b00 Options.create_missing_column_families: 0 +2021/12/12-18:13:31.237590 7eff8ad55b00 Options.db_log_dir: +2021/12/12-18:13:31.237591 7eff8ad55b00 Options.wal_dir: +2021/12/12-18:13:31.237591 7eff8ad55b00 Options.table_cache_numshardbits: 6 +2021/12/12-18:13:31.237592 7eff8ad55b00 Options.WAL_ttl_seconds: 0 +2021/12/12-18:13:31.237592 7eff8ad55b00 Options.WAL_size_limit_MB: 0 +2021/12/12-18:13:31.237593 7eff8ad55b00 Options.max_write_batch_group_size_bytes: 1048576 +2021/12/12-18:13:31.237593 7eff8ad55b00 Options.manifest_preallocation_size: 4194304 +2021/12/12-18:13:31.237594 7eff8ad55b00 Options.is_fd_close_on_exec: 1 +2021/12/12-18:13:31.237594 7eff8ad55b00 Options.advise_random_on_open: 1 +2021/12/12-18:13:31.237595 7eff8ad55b00 Options.experimental_mempurge_threshold: 0.000000 +2021/12/12-18:13:31.237597 7eff8ad55b00 Options.db_write_buffer_size: 0 +2021/12/12-18:13:31.237598 7eff8ad55b00 Options.write_buffer_manager: 0x1ae98a0 +2021/12/12-18:13:31.237598 7eff8ad55b00 Options.access_hint_on_compaction_start: 1 +2021/12/12-18:13:31.237599 7eff8ad55b00 Options.new_table_reader_for_compaction_inputs: 0 +2021/12/12-18:13:31.237599 7eff8ad55b00 Options.random_access_max_buffer_size: 1048576 +2021/12/12-18:13:31.237600 7eff8ad55b00 Options.use_adaptive_mutex: 0 +2021/12/12-18:13:31.237600 7eff8ad55b00 Options.rate_limiter: (nil) +2021/12/12-18:13:31.237620 7eff8ad55b00 Options.sst_file_manager.rate_bytes_per_sec: 0 +2021/12/12-18:13:31.237622 7eff8ad55b00 Options.wal_recovery_mode: 2 +2021/12/12-18:13:31.237623 7eff8ad55b00 Options.enable_thread_tracking: 0 +2021/12/12-18:13:31.237623 7eff8ad55b00 Options.enable_pipelined_write: 0 +2021/12/12-18:13:31.237624 7eff8ad55b00 Options.unordered_write: 0 +2021/12/12-18:13:31.237624 7eff8ad55b00 Options.allow_concurrent_memtable_write: 1 +2021/12/12-18:13:31.237625 7eff8ad55b00 Options.enable_write_thread_adaptive_yield: 1 +2021/12/12-18:13:31.237625 7eff8ad55b00 Options.write_thread_max_yield_usec: 100 +2021/12/12-18:13:31.237626 7eff8ad55b00 Options.write_thread_slow_yield_usec: 3 +2021/12/12-18:13:31.237626 7eff8ad55b00 Options.row_cache: None +2021/12/12-18:13:31.237627 7eff8ad55b00 Options.wal_filter: None +2021/12/12-18:13:31.237628 7eff8ad55b00 Options.avoid_flush_during_recovery: 0 +2021/12/12-18:13:31.237628 7eff8ad55b00 Options.allow_ingest_behind: 0 +2021/12/12-18:13:31.237629 7eff8ad55b00 Options.preserve_deletes: 0 +2021/12/12-18:13:31.237629 7eff8ad55b00 Options.two_write_queues: 0 +2021/12/12-18:13:31.237630 7eff8ad55b00 Options.manual_wal_flush: 0 +2021/12/12-18:13:31.237630 7eff8ad55b00 Options.atomic_flush: 0 +2021/12/12-18:13:31.237631 7eff8ad55b00 Options.avoid_unnecessary_blocking_io: 0 +2021/12/12-18:13:31.237631 7eff8ad55b00 Options.persist_stats_to_disk: 0 +2021/12/12-18:13:31.237632 7eff8ad55b00 Options.write_dbid_to_manifest: 0 +2021/12/12-18:13:31.237632 7eff8ad55b00 Options.log_readahead_size: 0 +2021/12/12-18:13:31.237633 7eff8ad55b00 Options.file_checksum_gen_factory: Unknown +2021/12/12-18:13:31.237633 7eff8ad55b00 Options.best_efforts_recovery: 0 +2021/12/12-18:13:31.237634 7eff8ad55b00 Options.max_bgerror_resume_count: 2147483647 +2021/12/12-18:13:31.237634 7eff8ad55b00 Options.bgerror_resume_retry_interval: 1000000 +2021/12/12-18:13:31.237635 7eff8ad55b00 Options.allow_data_in_errors: 0 +2021/12/12-18:13:31.237635 7eff8ad55b00 Options.db_host_id: __hostname__ +2021/12/12-18:13:31.237636 7eff8ad55b00 Options.max_background_jobs: 2 +2021/12/12-18:13:31.237636 7eff8ad55b00 Options.max_background_compactions: -1 +2021/12/12-18:13:31.237637 7eff8ad55b00 Options.max_subcompactions: 1 +2021/12/12-18:13:31.237637 7eff8ad55b00 Options.avoid_flush_during_shutdown: 0 +2021/12/12-18:13:31.237638 7eff8ad55b00 Options.writable_file_max_buffer_size: 1048576 +2021/12/12-18:13:31.237638 7eff8ad55b00 Options.delayed_write_rate : 16777216 +2021/12/12-18:13:31.237639 7eff8ad55b00 Options.max_total_wal_size: 0 +2021/12/12-18:13:31.237639 7eff8ad55b00 Options.delete_obsolete_files_period_micros: 21600000000 +2021/12/12-18:13:31.237640 7eff8ad55b00 Options.stats_dump_period_sec: 600 +2021/12/12-18:13:31.237640 7eff8ad55b00 Options.stats_persist_period_sec: 600 +2021/12/12-18:13:31.237641 7eff8ad55b00 Options.stats_history_buffer_size: 1048576 +2021/12/12-18:13:31.237641 7eff8ad55b00 Options.max_open_files: -1 +2021/12/12-18:13:31.237642 7eff8ad55b00 Options.bytes_per_sync: 0 +2021/12/12-18:13:31.237642 7eff8ad55b00 Options.wal_bytes_per_sync: 0 +2021/12/12-18:13:31.237643 7eff8ad55b00 Options.strict_bytes_per_sync: 0 +2021/12/12-18:13:31.237643 7eff8ad55b00 Options.compaction_readahead_size: 0 +2021/12/12-18:13:31.237644 7eff8ad55b00 Options.max_background_flushes: -1 +2021/12/12-18:13:31.237644 7eff8ad55b00 Compression algorithms supported: +2021/12/12-18:13:31.237664 7eff8ad55b00 kZSTDNotFinalCompression supported: 1 +2021/12/12-18:13:31.237666 7eff8ad55b00 kZSTD supported: 1 +2021/12/12-18:13:31.237667 7eff8ad55b00 kXpressCompression supported: 0 +2021/12/12-18:13:31.237668 7eff8ad55b00 kLZ4HCCompression supported: 1 +2021/12/12-18:13:31.237669 7eff8ad55b00 kLZ4Compression supported: 1 +2021/12/12-18:13:31.237669 7eff8ad55b00 kBZip2Compression supported: 0 +2021/12/12-18:13:31.237670 7eff8ad55b00 kZlibCompression supported: 1 +2021/12/12-18:13:31.237670 7eff8ad55b00 kSnappyCompression supported: 1 +2021/12/12-18:13:31.237673 7eff8ad55b00 Fast CRC32 supported: Supported on x86 +2021/12/12-18:13:31.240423 7eff8ad55b00 [/db_impl/db_impl_open.cc:300] Creating manifest 1 +2021/12/12-18:13:31.243984 7eff8ad55b00 [/version_set.cc:4847] Recovering from manifest file: ./resources/asdf.db/MANIFEST-000001 +2021/12/12-18:13:31.244057 7eff8ad55b00 [/column_family.cc:609] --------------- Options for column family [default]: +2021/12/12-18:13:31.244059 7eff8ad55b00 Options.comparator: leveldb.BytewiseComparator +2021/12/12-18:13:31.244060 7eff8ad55b00 Options.merge_operator: None +2021/12/12-18:13:31.244060 7eff8ad55b00 Options.compaction_filter: None +2021/12/12-18:13:31.244061 7eff8ad55b00 Options.compaction_filter_factory: None +2021/12/12-18:13:31.244061 7eff8ad55b00 Options.sst_partitioner_factory: None +2021/12/12-18:13:31.244062 7eff8ad55b00 Options.memtable_factory: SkipListFactory +2021/12/12-18:13:31.244062 7eff8ad55b00 Options.table_factory: BlockBasedTable +2021/12/12-18:13:31.244073 7eff8ad55b00 table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x28a8990) + cache_index_and_filter_blocks: 0 + cache_index_and_filter_blocks_with_high_priority: 1 + pin_l0_filter_and_index_blocks_in_cache: 0 + pin_top_level_index_and_filter: 1 + index_type: 0 + data_block_index_type: 0 + index_shortening: 1 + data_block_hash_table_util_ratio: 0.750000 + hash_index_allow_collision: 1 + checksum: 1 + no_block_cache: 0 + block_cache: 0x1f83a00 + block_cache_name: LRUCache + block_cache_options: + capacity : 8388608 + num_shard_bits : 4 + strict_capacity_limit : 0 + memory_allocator : None + high_pri_pool_ratio: 0.000 + block_cache_compressed: (nil) + persistent_cache: (nil) + block_size: 4096 + block_size_deviation: 10 + block_restart_interval: 16 + index_block_restart_interval: 1 + metadata_block_size: 4096 + partition_filters: 0 + use_delta_encoding: 1 + filter_policy: nullptr + whole_key_filtering: 1 + verify_compression: 0 + read_amp_bytes_per_bit: 0 + format_version: 5 + enable_index_compression: 1 + block_align: 0 + max_auto_readahead_size: 262144 + prepopulate_block_cache: 0 +2021/12/12-18:13:31.244074 7eff8ad55b00 Options.write_buffer_size: 67108864 +2021/12/12-18:13:31.244074 7eff8ad55b00 Options.max_write_buffer_number: 2 +2021/12/12-18:13:31.244075 7eff8ad55b00 Options.compression: Snappy +2021/12/12-18:13:31.244076 7eff8ad55b00 Options.bottommost_compression: Disabled +2021/12/12-18:13:31.244077 7eff8ad55b00 Options.prefix_extractor: nullptr +2021/12/12-18:13:31.244077 7eff8ad55b00 Options.memtable_insert_with_hint_prefix_extractor: nullptr +2021/12/12-18:13:31.244078 7eff8ad55b00 Options.num_levels: 7 +2021/12/12-18:13:31.244078 7eff8ad55b00 Options.min_write_buffer_number_to_merge: 1 +2021/12/12-18:13:31.244079 7eff8ad55b00 Options.max_write_buffer_number_to_maintain: 0 +2021/12/12-18:13:31.244079 7eff8ad55b00 Options.max_write_buffer_size_to_maintain: 0 +2021/12/12-18:13:31.244080 7eff8ad55b00 Options.bottommost_compression_opts.window_bits: -14 +2021/12/12-18:13:31.244080 7eff8ad55b00 Options.bottommost_compression_opts.level: 32767 +2021/12/12-18:13:31.244081 7eff8ad55b00 Options.bottommost_compression_opts.strategy: 0 +2021/12/12-18:13:31.244081 7eff8ad55b00 Options.bottommost_compression_opts.max_dict_bytes: 0 +2021/12/12-18:13:31.244082 7eff8ad55b00 Options.bottommost_compression_opts.zstd_max_train_bytes: 0 +2021/12/12-18:13:31.244105 7eff8ad55b00 Options.bottommost_compression_opts.parallel_threads: 1 +2021/12/12-18:13:31.244107 7eff8ad55b00 Options.bottommost_compression_opts.enabled: false +2021/12/12-18:13:31.244108 7eff8ad55b00 Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 +2021/12/12-18:13:31.244108 7eff8ad55b00 Options.compression_opts.window_bits: -14 +2021/12/12-18:13:31.244109 7eff8ad55b00 Options.compression_opts.level: 32767 +2021/12/12-18:13:31.244109 7eff8ad55b00 Options.compression_opts.strategy: 0 +2021/12/12-18:13:31.244110 7eff8ad55b00 Options.compression_opts.max_dict_bytes: 0 +2021/12/12-18:13:31.244110 7eff8ad55b00 Options.compression_opts.zstd_max_train_bytes: 0 +2021/12/12-18:13:31.244111 7eff8ad55b00 Options.compression_opts.parallel_threads: 1 +2021/12/12-18:13:31.244111 7eff8ad55b00 Options.compression_opts.enabled: false +2021/12/12-18:13:31.244112 7eff8ad55b00 Options.compression_opts.max_dict_buffer_bytes: 0 +2021/12/12-18:13:31.244112 7eff8ad55b00 Options.level0_file_num_compaction_trigger: 4 +2021/12/12-18:13:31.244113 7eff8ad55b00 Options.level0_slowdown_writes_trigger: 20 +2021/12/12-18:13:31.244113 7eff8ad55b00 Options.level0_stop_writes_trigger: 36 +2021/12/12-18:13:31.244114 7eff8ad55b00 Options.target_file_size_base: 67108864 +2021/12/12-18:13:31.244114 7eff8ad55b00 Options.target_file_size_multiplier: 1 +2021/12/12-18:13:31.244115 7eff8ad55b00 Options.max_bytes_for_level_base: 268435456 +2021/12/12-18:13:31.244115 7eff8ad55b00 Options.level_compaction_dynamic_level_bytes: 0 +2021/12/12-18:13:31.244116 7eff8ad55b00 Options.max_bytes_for_level_multiplier: 10.000000 +2021/12/12-18:13:31.244117 7eff8ad55b00 Options.max_bytes_for_level_multiplier_addtl[0]: 1 +2021/12/12-18:13:31.244118 7eff8ad55b00 Options.max_bytes_for_level_multiplier_addtl[1]: 1 +2021/12/12-18:13:31.244118 7eff8ad55b00 Options.max_bytes_for_level_multiplier_addtl[2]: 1 +2021/12/12-18:13:31.244119 7eff8ad55b00 Options.max_bytes_for_level_multiplier_addtl[3]: 1 +2021/12/12-18:13:31.244120 7eff8ad55b00 Options.max_bytes_for_level_multiplier_addtl[4]: 1 +2021/12/12-18:13:31.244120 7eff8ad55b00 Options.max_bytes_for_level_multiplier_addtl[5]: 1 +2021/12/12-18:13:31.244120 7eff8ad55b00 Options.max_bytes_for_level_multiplier_addtl[6]: 1 +2021/12/12-18:13:31.244121 7eff8ad55b00 Options.max_sequential_skip_in_iterations: 8 +2021/12/12-18:13:31.244121 7eff8ad55b00 Options.max_compaction_bytes: 1677721600 +2021/12/12-18:13:31.244122 7eff8ad55b00 Options.arena_block_size: 1048576 +2021/12/12-18:13:31.244123 7eff8ad55b00 Options.soft_pending_compaction_bytes_limit: 68719476736 +2021/12/12-18:13:31.244123 7eff8ad55b00 Options.hard_pending_compaction_bytes_limit: 274877906944 +2021/12/12-18:13:31.244124 7eff8ad55b00 Options.rate_limit_delay_max_milliseconds: 100 +2021/12/12-18:13:31.244124 7eff8ad55b00 Options.disable_auto_compactions: 0 +2021/12/12-18:13:31.244125 7eff8ad55b00 Options.compaction_style: kCompactionStyleLevel +2021/12/12-18:13:31.244126 7eff8ad55b00 Options.compaction_pri: kMinOverlappingRatio +2021/12/12-18:13:31.244127 7eff8ad55b00 Options.compaction_options_universal.size_ratio: 1 +2021/12/12-18:13:31.244127 7eff8ad55b00 Options.compaction_options_universal.min_merge_width: 2 +2021/12/12-18:13:31.244128 7eff8ad55b00 Options.compaction_options_universal.max_merge_width: 4294967295 +2021/12/12-18:13:31.244128 7eff8ad55b00 Options.compaction_options_universal.max_size_amplification_percent: 200 +2021/12/12-18:13:31.244129 7eff8ad55b00 Options.compaction_options_universal.compression_size_percent: -1 +2021/12/12-18:13:31.244130 7eff8ad55b00 Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize +2021/12/12-18:13:31.244130 7eff8ad55b00 Options.compaction_options_fifo.max_table_files_size: 1073741824 +2021/12/12-18:13:31.244150 7eff8ad55b00 Options.compaction_options_fifo.allow_compaction: 0 +2021/12/12-18:13:31.244155 7eff8ad55b00 Options.table_properties_collectors: +2021/12/12-18:13:31.244156 7eff8ad55b00 Options.inplace_update_support: 0 +2021/12/12-18:13:31.244157 7eff8ad55b00 Options.inplace_update_num_locks: 10000 +2021/12/12-18:13:31.244158 7eff8ad55b00 Options.memtable_prefix_bloom_size_ratio: 0.000000 +2021/12/12-18:13:31.244159 7eff8ad55b00 Options.memtable_whole_key_filtering: 0 +2021/12/12-18:13:31.244159 7eff8ad55b00 Options.memtable_huge_page_size: 0 +2021/12/12-18:13:31.244160 7eff8ad55b00 Options.bloom_locality: 0 +2021/12/12-18:13:31.244160 7eff8ad55b00 Options.max_successive_merges: 0 +2021/12/12-18:13:31.244161 7eff8ad55b00 Options.optimize_filters_for_hits: 0 +2021/12/12-18:13:31.244161 7eff8ad55b00 Options.paranoid_file_checks: 0 +2021/12/12-18:13:31.244162 7eff8ad55b00 Options.force_consistency_checks: 1 +2021/12/12-18:13:31.244162 7eff8ad55b00 Options.report_bg_io_stats: 0 +2021/12/12-18:13:31.244163 7eff8ad55b00 Options.ttl: 2592000 +2021/12/12-18:13:31.244163 7eff8ad55b00 Options.periodic_compaction_seconds: 0 +2021/12/12-18:13:31.244164 7eff8ad55b00 Options.enable_blob_files: false +2021/12/12-18:13:31.244164 7eff8ad55b00 Options.min_blob_size: 0 +2021/12/12-18:13:31.244165 7eff8ad55b00 Options.blob_file_size: 268435456 +2021/12/12-18:13:31.244165 7eff8ad55b00 Options.blob_compression_type: NoCompression +2021/12/12-18:13:31.244166 7eff8ad55b00 Options.enable_blob_garbage_collection: false +2021/12/12-18:13:31.244166 7eff8ad55b00 Options.blob_garbage_collection_age_cutoff: 0.250000 +2021/12/12-18:13:31.244167 7eff8ad55b00 Options.blob_garbage_collection_force_threshold: 1.000000 +2021/12/12-18:13:31.245521 7eff8ad55b00 [/version_set.cc:4887] Recovered from manifest file:./resources/asdf.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 +2021/12/12-18:13:31.245526 7eff8ad55b00 [/version_set.cc:4902] Column family [default] (ID 0), log number is 0 +2021/12/12-18:13:31.245575 7eff8ad55b00 [/version_set.cc:4385] Creating manifest 4 +2021/12/12-18:13:31.252666 7eff8ad55b00 [/db_impl/db_impl_open.cc:1786] SstFileManager instance 0x1c18110 +2021/12/12-18:13:31.252677 7eff8ad55b00 DB pointer 0x1f53cf0 +2021/12/12-18:13:31.252847 7eff3affd700 [/db_impl/db_impl.cc:1004] ------- DUMPING STATS ------- +2021/12/12-18:13:31.252864 7eff3affd700 [/db_impl/db_impl.cc:1006] +** DB Stats ** +Uptime(secs): 0.0 total, 0.0 interval +Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s +Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s +Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent +Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s +Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s +Interval stall: 00:00:0.000 H:M:S, 0.0 percent + +** Compaction Stats [default] ** +Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 + Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 + +** Compaction Stats [default] ** +Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + +Blob file count: 0, total size: 0.0 GB + +Uptime(secs): 0.0 total, 0.0 interval +Flush(GB): cumulative 0.000, interval 0.000 +AddFile(GB): cumulative 0.000, interval 0.000 +AddFile(Total Files): cumulative 0, interval 0 +AddFile(L0 Files): cumulative 0, interval 0 +AddFile(Keys): cumulative 0, interval 0 +Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds +Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds +Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count +Block cache LRUCache@0x1f83a00#530181 capacity: 8.00 MB collections: 1 last_copies: 0 last_secs: 2.9e-05 secs_since: 0 +Block cache entry stats(count,size,portion): Misc(1,0.00 KB,0%) + +** File Read Latency Histogram By Level [default] ** +2021/12/12-18:13:31.254311 7eff638bd700 [/db_impl/db_impl.cc:472] Shutdown: canceling all background work +2021/12/12-18:13:31.254489 7eff638bd700 [/db_impl/db_impl.cc:685] Shutdown complete diff --git a/resources/asdf.db/MANIFEST-000004 b/resources/asdf.db/MANIFEST-000004 new file mode 100644 index 0000000000000000000000000000000000000000..24ab5c10610d0de0dadb61f1e1b2b6980994b454 GIT binary patch literal 57 zcmWIhx#Ncn10$nUPHI_dPD+xVQ)NkNd1i5{bAE0?Vo_pAe$kBe5zCku7?~IX8;@78 NF)(s6Ftf5S003wp5+48n literal 0 HcmV?d00001 diff --git a/resources/asdf.db/OPTIONS-000007 b/resources/asdf.db/OPTIONS-000007 new file mode 100644 index 0000000..d716285 --- /dev/null +++ b/resources/asdf.db/OPTIONS-000007 @@ -0,0 +1,187 @@ +# This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +# in examples/rocksdb_option_file_example.ini +# + +[Version] + rocksdb_version=6.26.1 + options_file_version=1.1 + +[DBOptions] + max_open_files=-1 + stats_history_buffer_size=1048576 + stats_persist_period_sec=600 + max_background_flushes=-1 + stats_dump_period_sec=600 + compaction_readahead_size=0 + bytes_per_sync=0 + delete_obsolete_files_period_micros=21600000000 + max_total_wal_size=0 + delayed_write_rate=16777216 + wal_bytes_per_sync=0 + writable_file_max_buffer_size=1048576 + avoid_flush_during_shutdown=false + max_subcompactions=1 + strict_bytes_per_sync=false + max_background_compactions=-1 + base_background_compactions=-1 + max_background_jobs=2 + file_checksum_gen_factory=nullptr + db_host_id=__hostname__ + bgerror_resume_retry_interval=1000000 + best_efforts_recovery=false + avoid_unnecessary_blocking_io=false + two_write_queues=false + atomic_flush=false + preserve_deletes=false + allow_ingest_behind=false + lowest_used_cache_tier=kNonVolatileBlockTier + avoid_flush_during_recovery=false + info_log_level=INFO_LEVEL + access_hint_on_compaction_start=NORMAL + max_bgerror_resume_count=2147483647 + write_thread_slow_yield_usec=3 + allow_concurrent_memtable_write=true + WAL_ttl_seconds=0 + manual_wal_flush=false + dump_malloc_stats=false + wal_recovery_mode=kPointInTimeRecovery + log_file_time_to_roll=0 + enable_write_thread_adaptive_yield=true + recycle_log_file_num=0 + table_cache_numshardbits=6 + max_file_opening_threads=16 + allow_data_in_errors=false + use_fsync=false + unordered_write=false + fail_if_options_file_error=false + random_access_max_buffer_size=1048576 + new_table_reader_for_compaction_inputs=false + skip_checking_sst_file_sizes_on_db_open=false + skip_stats_update_on_db_open=false + persist_stats_to_disk=false + track_and_verify_wals_in_manifest=false + enable_pipelined_write=false + flush_verify_memtable_count=true + log_readahead_size=0 + is_fd_close_on_exec=true + WAL_size_limit_MB=0 + experimental_mempurge_threshold=0.000000 + write_dbid_to_manifest=false + use_adaptive_mutex=false + error_if_exists=false + write_thread_max_yield_usec=100 + enable_thread_tracking=false + db_write_buffer_size=0 + create_missing_column_families=false + paranoid_checks=true + create_if_missing=true + wal_filter=nullptr + max_manifest_file_size=1073741824 + allow_2pc=false + use_direct_io_for_flush_and_compaction=false + manifest_preallocation_size=4194304 + use_direct_reads=false + allow_fallocate=true + max_write_batch_group_size_bytes=1048576 + keep_log_file_num=1000 + allow_mmap_reads=false + max_log_file_size=0 + allow_mmap_writes=false + advise_random_on_open=true + + +[CFOptions "default"] + bottommost_compression=kDisableCompressionOption + bottommost_compression_opts={max_dict_buffer_bytes=0;enabled=false;parallel_threads=1;zstd_max_train_bytes=0;strategy=0;max_dict_bytes=0;level=32767;window_bits=-14;} + sample_for_compression=0 + blob_garbage_collection_force_threshold=1.000000 + blob_file_size=268435456 + compaction_options_universal={allow_trivial_move=false;incremental=false;stop_style=kCompactionStopStyleTotalSize;compression_size_percent=-1;max_size_amplification_percent=200;max_merge_width=4294967295;min_merge_width=2;size_ratio=1;} + compaction_options_fifo={allow_compaction=false;age_for_warm=0;max_table_files_size=1073741824;} + prefix_extractor=nullptr + max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1 + max_bytes_for_level_base=268435456 + memtable_whole_key_filtering=false + memtable_prefix_bloom_size_ratio=0.000000 + enable_blob_files=false + target_file_size_base=67108864 + memtable_huge_page_size=0 + max_successive_merges=0 + inplace_update_num_locks=10000 + max_sequential_skip_in_iterations=8 + arena_block_size=1048576 + target_file_size_multiplier=1 + max_write_buffer_number=2 + write_buffer_size=67108864 + blob_compression_type=kNoCompression + compression=kSnappyCompression + level0_stop_writes_trigger=36 + level0_slowdown_writes_trigger=20 + level0_file_num_compaction_trigger=4 + ttl=2592000 + max_compaction_bytes=1677721600 + blob_garbage_collection_age_cutoff=0.250000 + compression_opts={max_dict_buffer_bytes=0;enabled=false;parallel_threads=1;zstd_max_train_bytes=0;strategy=0;max_dict_bytes=0;level=32767;window_bits=-14;} + enable_blob_garbage_collection=false + soft_pending_compaction_bytes_limit=68719476736 + paranoid_file_checks=false + periodic_compaction_seconds=0 + check_flush_compaction_key_order=true + min_blob_size=0 + hard_pending_compaction_bytes_limit=274877906944 + disable_auto_compactions=false + max_bytes_for_level_multiplier=10.000000 + report_bg_io_stats=false + compaction_pri=kMinOverlappingRatio + compaction_filter_factory=nullptr + comparator=leveldb.BytewiseComparator + sst_partitioner_factory=nullptr + bloom_locality=0 + compaction_style=kCompactionStyleLevel + min_write_buffer_number_to_merge=1 + max_write_buffer_size_to_maintain=0 + max_write_buffer_number_to_maintain=0 + merge_operator=nullptr + memtable_factory=SkipListFactory + memtable_insert_with_hint_prefix_extractor=nullptr + num_levels=7 + force_consistency_checks=true + optimize_filters_for_hits=false + compaction_filter=nullptr + level_compaction_dynamic_level_bytes=false + inplace_update_support=false + table_factory=BlockBasedTable + +[TableOptions/BlockBasedTable "default"] + pin_top_level_index_and_filter=true + block_align=false + read_amp_bytes_per_bit=0 + verify_compression=false + enable_index_compression=true + whole_key_filtering=true + max_auto_readahead_size=262144 + optimize_filters_for_memory=false + index_block_restart_interval=1 + prepopulate_block_cache=kDisable + block_restart_interval=16 + block_size=4096 + format_version=5 + partition_filters=false + block_size_deviation=10 + no_block_cache=false + checksum=kCRC32c + data_block_hash_table_util_ratio=0.750000 + index_shortening=kShortenSeparators + data_block_index_type=kDataBlockBinarySearch + hash_index_allow_collision=true + filter_policy=nullptr + metadata_block_size=4096 + index_type=kBinarySearch + metadata_cache_options={unpartitioned_pinning=kFallback;partition_pinning=kFallback;top_level_index_pinning=kFallback;} + pin_l0_filter_and_index_blocks_in_cache=false + cache_index_and_filter_blocks_with_high_priority=true + cache_index_and_filter_blocks=false + flush_block_policy_factory=FlushBlockBySizePolicyFactory +