Allow configuration of universal style compaction

docs/api/options.rst

@@ -625,6 +625,84 @@ Options object
         | *Type:* ``bool``
         | *Default:* ``True``
 
+    .. py:attribute:: compaction_style
+
+        The compaction style. Could be set to ``"level"`` to use level-style
+        compaction. For universal-style compaction use ``"universal"``.
+
+        | *Type:* ``string``
+        | *Default:* ``level``
+
+    .. py:attribute:: compaction_options_universal
+
+        Options to use for universal-style compaction. They make only sense if
+        :py:attr:`rocksdb.Options.compaction_style` is set to ``"universal"``.
+
+        It is a dict with the following keys.
+
+        * ``size_ratio``:
+            Percentage flexibilty while comparing file size.
+            If the candidate file(s) size is 1% smaller than the next file's size,
+            then include next file into this candidate set.
+            Default: ``1``
+
+        * ``min_merge_width``:
+            The minimum number of files in a single compaction run.
+            Default: ``2``
+
+        * ``max_merge_width``:
+            The maximum number of files in a single compaction run.
+            Default: ``UINT_MAX``
+
+        * ``max_size_amplification_percent``:
+            The size amplification is defined as the amount (in percentage) of
+            additional storage needed to store a single byte of data in the database.
+            For example, a size amplification of 2% means that a database that
+            contains 100 bytes of user-data may occupy upto 102 bytes of
+            physical storage. By this definition, a fully compacted database has
+            a size amplification of 0%. Rocksdb uses the following heuristic
+            to calculate size amplification: it assumes that all files excluding
+            the earliest file contribute to the size amplification.
+            Default: ``200``, which means that a 100 byte database could require upto
+            300 bytes of storage.
+
+        * ``compression_size_percent``:
+            If this option is set to be -1 (the default value), all the output
+            files will follow compression type specified.
+
+            If this option is not negative, we will try to make sure compressed
+            size is just above this value. In normal cases, at least this
+            percentage of data will be compressed.
+
+            When we are compacting to a new file, here is the criteria whether
+            it needs to be compressed: assuming here are the list of files sorted
+            by generation time: ``A1...An B1...Bm C1...Ct``
+            where ``A1`` is the newest and ``Ct`` is the oldest, and we are going
+            to compact ``B1...Bm``, we calculate the total size of all the files
+            as total_size, as well as the total size of ``C1...Ct`` as
+            ``total_C``, the compaction output file will be compressed if
+            ``total_C / total_size < this percentage``.
+            Default: -1
+
+        * ``stop_style``:
+            The algorithm used to stop picking files into a single compaction.
+            Can be either ``"similar_size"`` or ``"total_size"``.
+
+            * ``similar_size``: Pick files of similar size.
+            * ``total_size``: Total size of picked files is greater than next file.
+
+            Default: ``"total_size"``
+
+        For setting options, just assign a dict with the fields to set.
+        It is allowed to omit keys in this dict. Missing keys are just not set
+        to the underlying options object.
+
+        This example just changes the stop_style and leaves the other options
+        untouched. ::
+
+            opts = rocksdb.Options()
+            opts.compaction_options_universal = {'stop_style': 'similar_size'}
+
     .. py:attribute:: filter_deletes
 
         Use KeyMayExist API to filter deletes when this is true.

docs/tutorial/index.rst

@@ -29,7 +29,7 @@ A more production ready open can look like this ::
 It assings a cache of 2.5G, uses a bloom filter for faster lookups and keeps
 more data (64 MB) in memory before writting a .sst file.
 
-About Bytes and Unicode
+About Bytes And Unicode
 ========================
 
 RocksDB stores all data as uninterpreted *byte strings*.
@@ -272,12 +272,12 @@ The two arguments are the db_dir and wal_dir, which are mostly the same. ::
     backup.restore_latest_backup("test.db", "test.db")
 
 
-Change Memtable or SST implementations
+Change Memtable Or SST Implementations
 ======================================
 
 As noted here :ref:`memtable_factories_label`, RocksDB offers different implementations for the memtable
 representation. Per default :py:class:`rocksdb.SkipListMemtableFactory` is used,
-but changeing it to a different one is veary easy.
+but changing it to a different one is veary easy.
 
 Here is an example for HashSkipList-MemtableFactory.
 Keep in mind: To use the hashed based MemtableFactories you must set
@@ -325,3 +325,20 @@ Here is an example how to use one of the 'PlainTables'. ::
     opts.create_if_missing = True
 
     db = rocksdb.DB("test.db", opts)
+
+Change Compaction Style
+=======================
+
+RocksDB has a compaction algorithm called *universal*. This style typically
+results in lower write amplification but higher space amplification than
+Level Style Compaction. See here for more details,
+https://github.com/facebook/rocksdb/wiki/Rocksdb-Architecture-Guide#multi-threaded-compactions
+
+Here is an example to switch to *universal style compaction*. ::
+
+    opts = rocksdb.Options()
+    opts.compaction_style = "universal"
+    opts.compaction_options_universal = {"min_merge_width": 3}
+
+See here for more options on *universal style compaction*,
+:py:attr:`rocksdb.Options.compaction_options_universal`

rocksdb/_rocksdb.pyx

@@ -26,6 +26,15 @@ cimport backup
 cimport env
 cimport table_factory
 cimport memtablerep
+cimport universal_compaction
+
+# Enums are the only exception for direct imports
+# Their name als already unique enough
+from universal_compaction cimport kCompactionStopStyleSimilarSize
+from universal_compaction cimport kCompactionStopStyleTotalSize
+
+from options cimport kCompactionStyleLevel
+from options cimport kCompactionStyleUniversal
 
 from slice_ cimport Slice
 from status cimport Status
@@ -1040,6 +1049,71 @@ cdef class Options(object):
         def __set__(self, value):
             self.opts.verify_checksums_in_compaction = value
 
+    property compaction_style:
+        def __get__(self):
+            if self.opts.compaction_style == kCompactionStyleLevel:
+                return 'level'
+            if self.opts.compaction_style == kCompactionStyleUniversal:
+                return 'universal'
+            raise Exception("Unknown compaction_style")
+
+        def __set__(self, str value):
+            if value == 'level':
+                self.opts.compaction_style = kCompactionStyleLevel
+            elif value == 'universal':
+                self.opts.compaction_style = kCompactionStyleUniversal
+            else:
+                raise Exception("Unknown compaction style")
+
+    property compaction_options_universal:
+        def __get__(self):
+            cdef universal_compaction.CompactionOptionsUniversal uopts
+            cdef dict ret_ob = {}
+
+            uopts = self.opts.compaction_options_universal
+
+            ret_ob['size_ratio'] = uopts.size_ratio
+            ret_ob['min_merge_width'] = uopts.min_merge_width
+            ret_ob['max_merge_width'] = uopts.max_merge_width
+            ret_ob['max_size_amplification_percent'] = uopts.max_size_amplification_percent
+            ret_ob['compression_size_percent'] = uopts.compression_size_percent
+
+            if uopts.stop_style == kCompactionStopStyleSimilarSize:
+                ret_ob['stop_style'] = 'similar_size'
+            elif uopts.stop_style == kCompactionStopStyleTotalSize:
+                ret_ob['stop_style'] = 'total_size'
+            else:
+                raise Exception("Unknown compaction style")
+
+            return ret_ob
+
+        def __set__(self, dict value):
+            cdef universal_compaction.CompactionOptionsUniversal* uopts
+            uopts = cython.address(self.opts.compaction_options_universal)
+
+            if 'size_ratio' in value:
+                uopts.size_ratio  = value['size_ratio']
+
+            if 'min_merge_width' in value:
+                uopts.min_merge_width = value['min_merge_width']
+
+            if 'max_merge_width' in value:
+                uopts.max_merge_width = value['max_merge_width']
+
+            if 'max_size_amplification_percent' in value:
+                uopts.max_size_amplification_percent = value['max_size_amplification_percent']
+
+            if 'compression_size_percent' in value:
+                uopts.compression_size_percent = value['compression_size_percent']
+
+            if 'stop_style' in value:
+                if value['stop_style'] == 'similar_size':
+                    uopts.stop_style = kCompactionStopStyleSimilarSize
+                elif value['stop_style'] == 'total_size':
+                    uopts.stop_style = kCompactionStopStyleTotalSize
+                else:
+                    raise Exception("Unknown compaction style")
+
     property filter_deletes:
         def __get__(self):
             return self.opts.filter_deletes

rocksdb/options.pxd

@@ -13,8 +13,13 @@ from snapshot cimport Snapshot
 from slice_transform cimport SliceTransform
 from table_factory cimport TableFactory
 from memtablerep cimport MemTableRepFactory
+from universal_compaction cimport CompactionOptionsUniversal
 
 cdef extern from "rocksdb/options.h" namespace "rocksdb":
+    ctypedef enum CompactionStyle:
+        kCompactionStyleLevel
+        kCompactionStyleUniversal
+
     ctypedef enum CompressionType:
         kNoCompression
         kSnappyCompression
@@ -101,8 +106,8 @@ cdef extern from "rocksdb/options.h" namespace "rocksdb":
         cpp_bool use_adaptive_mutex
         uint64_t bytes_per_sync
         cpp_bool verify_checksums_in_compaction
-        # TODO: CompactionStyle compaction_style
+        CompactionStyle compaction_style
-        # TODO: CompactionOptionsUniversal compaction_options_universal
+        CompactionOptionsUniversal compaction_options_universal
         cpp_bool filter_deletes
         uint64_t max_sequential_skip_in_iterations
         shared_ptr[MemTableRepFactory] memtable_factory

rocksdb/tests/test_options.py

@@ -69,3 +69,37 @@ class TestOptions(unittest.TestCase):
         opts.table_factory = rocksdb.BlockBasedTableFactory()
         opts.table_factory = rocksdb.PlainTableFactory()
         opts.table_factory = rocksdb.TotalOrderPlainTableFactory()
+
+    def test_compaction_style(self):
+        opts = rocksdb.Options()
+        self.assertEqual('level', opts.compaction_style)
+
+        opts.compaction_style = 'universal'
+        self.assertEqual('universal', opts.compaction_style)
+
+        opts.compaction_style = 'level'
+        self.assertEqual('level', opts.compaction_style)
+
+        with self.assertRaisesRegexp(Exception, 'Unknown compaction style'):
+            opts.compaction_style = 'foo'
+
+    def test_compaction_opts_universal(self):
+        opts = rocksdb.Options()
+        uopts = opts.compaction_options_universal
+        self.assertEqual(-1, uopts['compression_size_percent'])
+        self.assertEqual(200, uopts['max_size_amplification_percent'])
+        self.assertEqual('total_size', uopts['stop_style'])
+        self.assertEqual(1, uopts['size_ratio'])
+        self.assertEqual(2, uopts['min_merge_width'])
+        self.assertGreaterEqual(4294967295, uopts['max_merge_width'])
+
+        new_opts = {'stop_style': 'similar_size', 'max_merge_width': 30}
+        opts.compaction_options_universal = new_opts
+        uopts = opts.compaction_options_universal
+
+        self.assertEqual(-1, uopts['compression_size_percent'])
+        self.assertEqual(200, uopts['max_size_amplification_percent'])
+        self.assertEqual('similar_size', uopts['stop_style'])
+        self.assertEqual(1, uopts['size_ratio'])
+        self.assertEqual(2, uopts['min_merge_width'])
+        self.assertEqual(30, uopts['max_merge_width'])

rocksdb/universal_compaction.pxd

@@ -0,0 +1,15 @@
+cdef extern from "rocksdb/universal_compaction.h" namespace "rocksdb":
+
+    ctypedef enum CompactionStopStyle:
+        kCompactionStopStyleSimilarSize
+        kCompactionStopStyleTotalSize
+
+    cdef cppclass CompactionOptionsUniversal:
+        CompactionOptionsUniversal()
+
+        unsigned int size_ratio
+        unsigned int min_merge_width
+        unsigned int max_merge_width
+        unsigned int max_size_amplification_percent
+        int compression_size_percent
+        CompactionStopStyle stop_style