From 6ceb6514b5c448ed22accce8beed6c33ee4049ca Mon Sep 17 00:00:00 2001 From: hofmockel Date: Tue, 29 Apr 2014 17:46:21 +0200 Subject: [PATCH] Allow configuration of universal style compaction --- docs/api/options.rst | 78 ++++++++++++++++++++++++++++++++ docs/tutorial/index.rst | 23 ++++++++-- rocksdb/_rocksdb.pyx | 74 ++++++++++++++++++++++++++++++ rocksdb/options.pxd | 9 +++- rocksdb/tests/test_options.py | 34 ++++++++++++++ rocksdb/universal_compaction.pxd | 15 ++++++ 6 files changed, 228 insertions(+), 5 deletions(-) create mode 100644 rocksdb/universal_compaction.pxd diff --git a/docs/api/options.rst b/docs/api/options.rst index ec04ad5..c66ee28 100644 --- a/docs/api/options.rst +++ b/docs/api/options.rst @@ -625,6 +625,84 @@ Options object | *Type:* ``bool`` | *Default:* ``True`` + .. py:attribute:: compaction_style + + The compaction style. Could be set to ``"level"`` to use level-style + compaction. For universal-style compaction use ``"universal"``. + + | *Type:* ``string`` + | *Default:* ``level`` + + .. py:attribute:: compaction_options_universal + + Options to use for universal-style compaction. They make only sense if + :py:attr:`rocksdb.Options.compaction_style` is set to ``"universal"``. + + It is a dict with the following keys. + + * ``size_ratio``: + Percentage flexibilty while comparing file size. + If the candidate file(s) size is 1% smaller than the next file's size, + then include next file into this candidate set. + Default: ``1`` + + * ``min_merge_width``: + The minimum number of files in a single compaction run. + Default: ``2`` + + * ``max_merge_width``: + The maximum number of files in a single compaction run. + Default: ``UINT_MAX`` + + * ``max_size_amplification_percent``: + The size amplification is defined as the amount (in percentage) of + additional storage needed to store a single byte of data in the database. + For example, a size amplification of 2% means that a database that + contains 100 bytes of user-data may occupy upto 102 bytes of + physical storage. By this definition, a fully compacted database has + a size amplification of 0%. Rocksdb uses the following heuristic + to calculate size amplification: it assumes that all files excluding + the earliest file contribute to the size amplification. + Default: ``200``, which means that a 100 byte database could require upto + 300 bytes of storage. + + * ``compression_size_percent``: + If this option is set to be -1 (the default value), all the output + files will follow compression type specified. + + If this option is not negative, we will try to make sure compressed + size is just above this value. In normal cases, at least this + percentage of data will be compressed. + + When we are compacting to a new file, here is the criteria whether + it needs to be compressed: assuming here are the list of files sorted + by generation time: ``A1...An B1...Bm C1...Ct`` + where ``A1`` is the newest and ``Ct`` is the oldest, and we are going + to compact ``B1...Bm``, we calculate the total size of all the files + as total_size, as well as the total size of ``C1...Ct`` as + ``total_C``, the compaction output file will be compressed if + ``total_C / total_size < this percentage``. + Default: -1 + + * ``stop_style``: + The algorithm used to stop picking files into a single compaction. + Can be either ``"similar_size"`` or ``"total_size"``. + + * ``similar_size``: Pick files of similar size. + * ``total_size``: Total size of picked files is greater than next file. + + Default: ``"total_size"`` + + For setting options, just assign a dict with the fields to set. + It is allowed to omit keys in this dict. Missing keys are just not set + to the underlying options object. + + This example just changes the stop_style and leaves the other options + untouched. :: + + opts = rocksdb.Options() + opts.compaction_options_universal = {'stop_style': 'similar_size'} + .. py:attribute:: filter_deletes Use KeyMayExist API to filter deletes when this is true. diff --git a/docs/tutorial/index.rst b/docs/tutorial/index.rst index 74c99ee..7a5fdea 100644 --- a/docs/tutorial/index.rst +++ b/docs/tutorial/index.rst @@ -29,7 +29,7 @@ A more production ready open can look like this :: It assings a cache of 2.5G, uses a bloom filter for faster lookups and keeps more data (64 MB) in memory before writting a .sst file. -About Bytes and Unicode +About Bytes And Unicode ======================== RocksDB stores all data as uninterpreted *byte strings*. @@ -272,12 +272,12 @@ The two arguments are the db_dir and wal_dir, which are mostly the same. :: backup.restore_latest_backup("test.db", "test.db") -Change Memtable or SST implementations +Change Memtable Or SST Implementations ====================================== As noted here :ref:`memtable_factories_label`, RocksDB offers different implementations for the memtable representation. Per default :py:class:`rocksdb.SkipListMemtableFactory` is used, -but changeing it to a different one is veary easy. +but changing it to a different one is veary easy. Here is an example for HashSkipList-MemtableFactory. Keep in mind: To use the hashed based MemtableFactories you must set @@ -325,3 +325,20 @@ Here is an example how to use one of the 'PlainTables'. :: opts.create_if_missing = True db = rocksdb.DB("test.db", opts) + +Change Compaction Style +======================= + +RocksDB has a compaction algorithm called *universal*. This style typically +results in lower write amplification but higher space amplification than +Level Style Compaction. See here for more details, +https://github.com/facebook/rocksdb/wiki/Rocksdb-Architecture-Guide#multi-threaded-compactions + +Here is an example to switch to *universal style compaction*. :: + + opts = rocksdb.Options() + opts.compaction_style = "universal" + opts.compaction_options_universal = {"min_merge_width": 3} + +See here for more options on *universal style compaction*, +:py:attr:`rocksdb.Options.compaction_options_universal` diff --git a/rocksdb/_rocksdb.pyx b/rocksdb/_rocksdb.pyx index 67eb39b..23b7f2c 100644 --- a/rocksdb/_rocksdb.pyx +++ b/rocksdb/_rocksdb.pyx @@ -26,6 +26,15 @@ cimport backup cimport env cimport table_factory cimport memtablerep +cimport universal_compaction + +# Enums are the only exception for direct imports +# Their name als already unique enough +from universal_compaction cimport kCompactionStopStyleSimilarSize +from universal_compaction cimport kCompactionStopStyleTotalSize + +from options cimport kCompactionStyleLevel +from options cimport kCompactionStyleUniversal from slice_ cimport Slice from status cimport Status @@ -1040,6 +1049,71 @@ cdef class Options(object): def __set__(self, value): self.opts.verify_checksums_in_compaction = value + property compaction_style: + def __get__(self): + if self.opts.compaction_style == kCompactionStyleLevel: + return 'level' + if self.opts.compaction_style == kCompactionStyleUniversal: + return 'universal' + raise Exception("Unknown compaction_style") + + def __set__(self, str value): + if value == 'level': + self.opts.compaction_style = kCompactionStyleLevel + elif value == 'universal': + self.opts.compaction_style = kCompactionStyleUniversal + else: + raise Exception("Unknown compaction style") + + property compaction_options_universal: + def __get__(self): + cdef universal_compaction.CompactionOptionsUniversal uopts + cdef dict ret_ob = {} + + uopts = self.opts.compaction_options_universal + + ret_ob['size_ratio'] = uopts.size_ratio + ret_ob['min_merge_width'] = uopts.min_merge_width + ret_ob['max_merge_width'] = uopts.max_merge_width + ret_ob['max_size_amplification_percent'] = uopts.max_size_amplification_percent + ret_ob['compression_size_percent'] = uopts.compression_size_percent + + if uopts.stop_style == kCompactionStopStyleSimilarSize: + ret_ob['stop_style'] = 'similar_size' + elif uopts.stop_style == kCompactionStopStyleTotalSize: + ret_ob['stop_style'] = 'total_size' + else: + raise Exception("Unknown compaction style") + + return ret_ob + + def __set__(self, dict value): + cdef universal_compaction.CompactionOptionsUniversal* uopts + uopts = cython.address(self.opts.compaction_options_universal) + + if 'size_ratio' in value: + uopts.size_ratio = value['size_ratio'] + + if 'min_merge_width' in value: + uopts.min_merge_width = value['min_merge_width'] + + if 'max_merge_width' in value: + uopts.max_merge_width = value['max_merge_width'] + + if 'max_size_amplification_percent' in value: + uopts.max_size_amplification_percent = value['max_size_amplification_percent'] + + if 'compression_size_percent' in value: + uopts.compression_size_percent = value['compression_size_percent'] + + if 'stop_style' in value: + if value['stop_style'] == 'similar_size': + uopts.stop_style = kCompactionStopStyleSimilarSize + elif value['stop_style'] == 'total_size': + uopts.stop_style = kCompactionStopStyleTotalSize + else: + raise Exception("Unknown compaction style") + property filter_deletes: def __get__(self): return self.opts.filter_deletes diff --git a/rocksdb/options.pxd b/rocksdb/options.pxd index 90575f4..0dd5e77 100644 --- a/rocksdb/options.pxd +++ b/rocksdb/options.pxd @@ -13,8 +13,13 @@ from snapshot cimport Snapshot from slice_transform cimport SliceTransform from table_factory cimport TableFactory from memtablerep cimport MemTableRepFactory +from universal_compaction cimport CompactionOptionsUniversal cdef extern from "rocksdb/options.h" namespace "rocksdb": + ctypedef enum CompactionStyle: + kCompactionStyleLevel + kCompactionStyleUniversal + ctypedef enum CompressionType: kNoCompression kSnappyCompression @@ -101,8 +106,8 @@ cdef extern from "rocksdb/options.h" namespace "rocksdb": cpp_bool use_adaptive_mutex uint64_t bytes_per_sync cpp_bool verify_checksums_in_compaction - # TODO: CompactionStyle compaction_style - # TODO: CompactionOptionsUniversal compaction_options_universal + CompactionStyle compaction_style + CompactionOptionsUniversal compaction_options_universal cpp_bool filter_deletes uint64_t max_sequential_skip_in_iterations shared_ptr[MemTableRepFactory] memtable_factory diff --git a/rocksdb/tests/test_options.py b/rocksdb/tests/test_options.py index bfd8751..ef36969 100644 --- a/rocksdb/tests/test_options.py +++ b/rocksdb/tests/test_options.py @@ -69,3 +69,37 @@ class TestOptions(unittest.TestCase): opts.table_factory = rocksdb.BlockBasedTableFactory() opts.table_factory = rocksdb.PlainTableFactory() opts.table_factory = rocksdb.TotalOrderPlainTableFactory() + + def test_compaction_style(self): + opts = rocksdb.Options() + self.assertEqual('level', opts.compaction_style) + + opts.compaction_style = 'universal' + self.assertEqual('universal', opts.compaction_style) + + opts.compaction_style = 'level' + self.assertEqual('level', opts.compaction_style) + + with self.assertRaisesRegexp(Exception, 'Unknown compaction style'): + opts.compaction_style = 'foo' + + def test_compaction_opts_universal(self): + opts = rocksdb.Options() + uopts = opts.compaction_options_universal + self.assertEqual(-1, uopts['compression_size_percent']) + self.assertEqual(200, uopts['max_size_amplification_percent']) + self.assertEqual('total_size', uopts['stop_style']) + self.assertEqual(1, uopts['size_ratio']) + self.assertEqual(2, uopts['min_merge_width']) + self.assertGreaterEqual(4294967295, uopts['max_merge_width']) + + new_opts = {'stop_style': 'similar_size', 'max_merge_width': 30} + opts.compaction_options_universal = new_opts + uopts = opts.compaction_options_universal + + self.assertEqual(-1, uopts['compression_size_percent']) + self.assertEqual(200, uopts['max_size_amplification_percent']) + self.assertEqual('similar_size', uopts['stop_style']) + self.assertEqual(1, uopts['size_ratio']) + self.assertEqual(2, uopts['min_merge_width']) + self.assertEqual(30, uopts['max_merge_width']) diff --git a/rocksdb/universal_compaction.pxd b/rocksdb/universal_compaction.pxd new file mode 100644 index 0000000..8622dba --- /dev/null +++ b/rocksdb/universal_compaction.pxd @@ -0,0 +1,15 @@ +cdef extern from "rocksdb/universal_compaction.h" namespace "rocksdb": + + ctypedef enum CompactionStopStyle: + kCompactionStopStyleSimilarSize + kCompactionStopStyleTotalSize + + cdef cppclass CompactionOptionsUniversal: + CompactionOptionsUniversal() + + unsigned int size_ratio + unsigned int min_merge_width + unsigned int max_merge_width + unsigned int max_size_amplification_percent + int compression_size_percent + CompactionStopStyle stop_style