From 58ba17ac90d88cea99d4617afa5e13337f9cf825 Mon Sep 17 00:00:00 2001 From: twmht Date: Sat, 8 Apr 2017 00:18:38 +0800 Subject: [PATCH] Merge options source_compaction_factor, max_grandparent_overlap_bytes and expanded_compaction_factor into max_compaction_bytes --- README.rst | 2 +- rocksdb/_rocksdb.pyx | 34 ++++++++++++++++++++++++++++++++++ rocksdb/options.pxd | 17 ++++++++++++++++- rocksdb/tests/test_options.py | 7 ++++--- 4 files changed, 55 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index e07570b..b737b13 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,6 @@ Note ========= -The original pyrocksdb has not been updated for long time. I update the pyrocksdb to support the latest rocksdb. Please open the issues in github if you have any problem. +The original pyrocksdb (https://pypi.python.org/pypi/pyrocksdb/0.4) has not been updated for long time. I update pyrocksdb to support the latest rocksdb. Please open issues in github if you have any problem. pyrocksdb diff --git a/rocksdb/_rocksdb.pyx b/rocksdb/_rocksdb.pyx index 31ccaab..b39fd67 100644 --- a/rocksdb/_rocksdb.pyx +++ b/rocksdb/_rocksdb.pyx @@ -36,6 +36,8 @@ from universal_compaction cimport kCompactionStopStyleTotalSize from options cimport kCompactionStyleLevel from options cimport kCompactionStyleUniversal +from options cimport kCompactionStyleFIFO +from options cimport kCompactionStyleNone from slice_ cimport Slice from status cimport Status @@ -700,6 +702,10 @@ cdef class CompressionType(object): bzip2_compression = u'bzip2_compression' lz4_compression = u'lz4_compression' lz4hc_compression = u'lz4hc_compression' + xpress_compression = u'xpress_compression' + zstd_compression = u'zstd_compression' + zstdnotfinal_compression = u'zstdnotfinal_compression' + disable_compression = u'disable_compression' cdef class Options(object): cdef options.Options* opts @@ -763,6 +769,12 @@ cdef class Options(object): def __set__(self, value): self.opts.max_write_buffer_number = value + property max_compaction_bytes: + def __get__(self): + return self.opts.max_compaction_bytes + def __set__(self, value): + self.opts.max_compaction_bytes = value + property min_write_buffer_number_to_merge: def __get__(self): return self.opts.min_write_buffer_number_to_merge @@ -789,6 +801,14 @@ cdef class Options(object): return CompressionType.lz4_compression elif self.opts.compression == options.kLZ4HCCompression: return CompressionType.lz4hc_compression + elif self.opts.compression == options.kXpressCompression: + return CompressionType.xpress_compression + elif self.opts.compression == options.kZSTD: + return CompressionType.zstd_compression + elif self.opts.compression == options.kZSTDNotFinalCompression: + return CompressionType.zstdnotfinal_compression + elif self.opts.compression == options.kDisableCompressionOption: + return CompressionType.disable_compression else: raise Exception("Unknonw type: %s" % self.opts.compression) @@ -805,6 +825,12 @@ cdef class Options(object): self.opts.compression = options.kLZ4Compression elif value == CompressionType.lz4hc_compression: self.opts.compression = options.kLZ4HCCompression + elif value == CompressionType.zstd_compression: + self.opts.compression = options.kZSTD + elif value == CompressionType.zstdnotfinal_compression: + self.opts.compression = options.kZSTDNotFinalCompression + elif value == CompressionType.disable_compression: + self.opts.compression = options.kDisableCompressionOption else: raise TypeError("Unknown compression: %s" % value) @@ -1049,6 +1075,10 @@ cdef class Options(object): return 'level' if self.opts.compaction_style == kCompactionStyleUniversal: return 'universal' + if self.opts.compaction_style == kCompactionStyleFIFO: + return 'fifo' + if self.opts.compaction_style == kCompactionStyleNone: + return 'none' raise Exception("Unknown compaction_style") def __set__(self, str value): @@ -1056,6 +1086,10 @@ cdef class Options(object): self.opts.compaction_style = kCompactionStyleLevel elif value == 'universal': self.opts.compaction_style = kCompactionStyleUniversal + elif value == 'fifo': + self.opts.compaction_style = kCompactionStyleFIFO + elif value == 'none': + self.opts.compaction_style = kCompactionStyleNone else: raise Exception("Unknown compaction style") diff --git a/rocksdb/options.pxd b/rocksdb/options.pxd index aa404d1..93395ab 100644 --- a/rocksdb/options.pxd +++ b/rocksdb/options.pxd @@ -19,6 +19,8 @@ cdef extern from "rocksdb/options.h" namespace "rocksdb": ctypedef enum CompactionStyle: kCompactionStyleLevel kCompactionStyleUniversal + kCompactionStyleFIFO + kCompactionStyleNone ctypedef enum CompressionType: kNoCompression @@ -27,11 +29,21 @@ cdef extern from "rocksdb/options.h" namespace "rocksdb": kBZip2Compression kLZ4Compression kLZ4HCCompression + kXpressCompression + kZSTD + kZSTDNotFinalCompression + kDisableCompressionOption ctypedef enum ReadTier: kReadAllTier kBlockCacheTier + ctypedef enum CompactionPri: + kByCompensatedSize + kOldestLargestSeqFirst + kOldestSmallestSeqFirst + kMinOverlappingRatio + cdef cppclass Options: const Comparator* comparator shared_ptr[MergeOperator] merge_operator @@ -47,6 +59,7 @@ cdef extern from "rocksdb/options.h" namespace "rocksdb": int min_write_buffer_number_to_merge int max_open_files CompressionType compression + CompactionPri compaction_pri # TODO: compression_per_level # TODO: compression_opts shared_ptr[SliceTransform] prefix_extractor @@ -58,7 +71,7 @@ cdef extern from "rocksdb/options.h" namespace "rocksdb": uint64_t target_file_size_base int target_file_size_multiplier uint64_t max_bytes_for_level_base - int max_bytes_for_level_multiplier + double max_bytes_for_level_multiplier vector[int] max_bytes_for_level_multiplier_additional int expanded_compaction_factor int source_compaction_factor @@ -107,6 +120,8 @@ cdef extern from "rocksdb/options.h" namespace "rocksdb": cpp_bool inplace_update_support size_t inplace_update_num_locks shared_ptr[Cache] row_cache + # TODO: remove options source_compaction_factor, max_grandparent_overlap_bytes and expanded_compaction_factor from document + uint64_t max_compaction_bytes; cdef cppclass WriteOptions: cpp_bool sync diff --git a/rocksdb/tests/test_options.py b/rocksdb/tests/test_options.py index 0f2c6d8..09786a2 100644 --- a/rocksdb/tests/test_options.py +++ b/rocksdb/tests/test_options.py @@ -37,9 +37,10 @@ class TestOptions(unittest.TestCase): opts.comparator, rocksdb.BytewiseComparator) - self.assertEqual('snappy_compression', opts.compression) - opts.compression = rocksdb.CompressionType.no_compression - self.assertEqual('no_compression', opts.compression) + self.assertEqual(rocksdb.CompressionType.no_compression, opts.compression) + + opts.compression = rocksdb.CompressionType.zstd_compression + self.assertEqual(rocksdb.CompressionType.zstd_compression, opts.compression) def test_block_options(self): rocksdb.BlockBasedTableFactory(