From 5610d9474d0569b0772660e8a5332fe5e290726c Mon Sep 17 00:00:00 2001 From: twmht Date: Tue, 25 Apr 2017 08:25:02 +0800 Subject: [PATCH] update the usage of default operators --- docs/tutorial/index.rst | 22 ++++---- rocksdb/_rocksdb.pyx | 36 ++++++------ rocksdb/merge_operator.pxd | 20 ------- rocksdb/tests/test_db.py | 102 ++++++++++++++-------------------- rocksdb/tests/test_options.py | 22 ++++---- 5 files changed, 84 insertions(+), 118 deletions(-) diff --git a/docs/tutorial/index.rst b/docs/tutorial/index.rst index e804300..a77cdf5 100644 --- a/docs/tutorial/index.rst +++ b/docs/tutorial/index.rst @@ -216,20 +216,20 @@ The following example python merge operator implements a counter :: # prints b'2' print db.get(b"a") -We provide a set of default operators ``uintadd64``, ``put`` and ``stringappend`` +We provide a set of default operators ``uintadd64`` and ``stringappend``:: -The following example using ``uintadd64`` where each operand is ``uint64`` :: - - import rocksdb - import struct + from rocksdb.merge_operators import UintAddOperator, StringAppendOperator opts = rocksdb.Options() opts.create_if_missing = True - opts.merge_operator = 'uint64add' - db = rocksdb.DB("test.db", opts) - # since every operand is uint64, you need to pack it into string - db.put(b'a', struct.pack('Q', 1000)) - db.merge(b'a', struct.pack('Q', 2000)) - assert struct.unpack('Q', db.get(b'a'))[0] == 3000 + # you should also play with StringAppendOperator + opts.merge_operator = UintAddOperator() + db = rocksdb.DB('/tmp/test', opts) + self.db.put(b'a', struct.pack('Q', 5566)) + for x in range(1000): + self.db.merge(b"a", struct.pack('Q', x)) + self.assertEqual(5566 + sum(range(1000)), struct.unpack('Q', self.db.get(b'a'))[0]) + + PrefixExtractor =============== diff --git a/rocksdb/_rocksdb.pyx b/rocksdb/_rocksdb.pyx index 670faae..da216d3 100644 --- a/rocksdb/_rocksdb.pyx +++ b/rocksdb/_rocksdb.pyx @@ -181,6 +181,8 @@ cdef class PyBytewiseComparator(PyComparator): cdef const comparator.Comparator* get_comparator(self): return self.comparator_ptr + + cdef int compare_callback( void* ctx, logger.Logger* log, @@ -336,23 +338,23 @@ cdef class PyMergeOperator(object): ob, full_merge_callback, partial_merge_callback)) - elif isinstance(ob, str): - if ob == "put": - self.merge_op = merge_operator.MergeOperators.CreatePutOperator() - elif ob == "put_v1": - self.merge_op = merge_operator.MergeOperators.CreateDeprecatedPutOperator() - elif ob == "uint64add": - self.merge_op = merge_operator.MergeOperators.CreateUInt64AddOperator() - elif ob == "stringappend": - self.merge_op = merge_operator.MergeOperators.CreateStringAppendOperator() - #TODO: necessary? - # elif ob == "stringappendtest": - # self.merge_op = merge_operator.MergeOperators.CreateStringAppendTESTOperator() - elif ob == "max": - self.merge_op = merge_operator.MergeOperators.CreateMaxOperator() - else: - msg = "{0} is not the default type".format(ob) - raise TypeError(msg) + # elif isinstance(ob, str): + # if ob == "put": + # self.merge_op = merge_operator.MergeOperators.CreatePutOperator() + # elif ob == "put_v1": + # self.merge_op = merge_operator.MergeOperators.CreateDeprecatedPutOperator() + # elif ob == "uint64add": + # self.merge_op = merge_operator.MergeOperators.CreateUInt64AddOperator() + # elif ob == "stringappend": + # self.merge_op = merge_operator.MergeOperators.CreateStringAppendOperator() + # #TODO: necessary? + # # elif ob == "stringappendtest": + # # self.merge_op = merge_operator.MergeOperators.CreateStringAppendTESTOperator() + # elif ob == "max": + # self.merge_op = merge_operator.MergeOperators.CreateMaxOperator() + # else: + # msg = "{0} is not the default type".format(ob) + # raise TypeError(msg) else: msg = "%s is not of this types %s" msg %= (ob, (IAssociativeMergeOperator, IMergeOperator)) diff --git a/rocksdb/merge_operator.pxd b/rocksdb/merge_operator.pxd index 36ae98e..5db078b 100644 --- a/rocksdb/merge_operator.pxd +++ b/rocksdb/merge_operator.pxd @@ -9,26 +9,6 @@ cdef extern from "rocksdb/merge_operator.h" namespace "rocksdb": cdef cppclass MergeOperator: pass -# cdef extern from "utilities/merge_operators.h" namespace "rocksdb::MergeOperators": -cdef extern from "utilities/merge_operators.h" namespace "rocksdb": - cdef cppclass MergeOperators: - @staticmethod - shared_ptr[MergeOperator] CreatePutOperator() - @staticmethod - shared_ptr[MergeOperator] CreateDeprecatedPutOperator() - @staticmethod - shared_ptr[MergeOperator] CreateUInt64AddOperator() - @staticmethod - shared_ptr[MergeOperator] CreateStringAppendOperator() - @staticmethod - shared_ptr[MergeOperator] CreateStringAppendTESTOperator() - @staticmethod - shared_ptr[MergeOperator] CreateMaxOperator() - @staticmethod - shared_ptr[MergeOperator] CreateFromStringId(const string &) - - - ctypedef cpp_bool (*merge_func)( void*, const Slice&, diff --git a/rocksdb/tests/test_db.py b/rocksdb/tests/test_db.py index c180e4f..0cfb630 100644 --- a/rocksdb/tests/test_db.py +++ b/rocksdb/tests/test_db.py @@ -5,6 +5,7 @@ import unittest import rocksdb from itertools import takewhile import struct +from rocksdb.merge_operators import UintAddOperator, StringAppendOperator def int_to_bytes(ob): return str(ob).encode('ascii') @@ -277,11 +278,12 @@ class AssocCounter(rocksdb.interfaces.AssociativeMergeOperator): def name(self): return b'AssocCounter' + class TestUint64Merge(unittest.TestCase, TestHelper): def setUp(self): opts = rocksdb.Options() opts.create_if_missing = True - opts.merge_operator = "uint64add" + opts.merge_operator = UintAddOperator() self._clean() self.db = rocksdb.DB('/tmp/test', opts) @@ -292,64 +294,46 @@ class TestUint64Merge(unittest.TestCase, TestHelper): self.db.put(b'a', struct.pack('Q', 5566)) for x in range(1000): self.db.merge(b"a", struct.pack('Q', x)) - print ('value', struct.unpack('Q', self.db.get(b'a'))[0]) self.assertEqual(5566 + sum(range(1000)), struct.unpack('Q', self.db.get(b'a'))[0]) -class TestUint64Merge(unittest.TestCase, TestHelper): - def setUp(self): - opts = rocksdb.Options() - opts.create_if_missing = True - opts.merge_operator = "uint64add" - self._clean() - self.db = rocksdb.DB('/tmp/test', opts) - def tearDown(self): - self._close_db() +# class TestPutMerge(unittest.TestCase, TestHelper): + # def setUp(self): + # opts = rocksdb.Options() + # opts.create_if_missing = True + # opts.merge_operator = "put" + # self._clean() + # self.db = rocksdb.DB('/tmp/test', opts) - def test_merge(self): - self.db.put(b'a', struct.pack('Q', 5566)) - for x in range(1000): - self.db.merge(b"a", struct.pack('Q', x)) - # print ('value', struct.unpack('Q', self.db.get(b'a'))[0]) - self.assertEqual(5566 + sum(range(1000)), struct.unpack('Q', self.db.get(b'a'))[0]) + # def tearDown(self): + # self._close_db() -class TestPutMerge(unittest.TestCase, TestHelper): - def setUp(self): - opts = rocksdb.Options() - opts.create_if_missing = True - opts.merge_operator = "put" - self._clean() - self.db = rocksdb.DB('/tmp/test', opts) + # def test_merge(self): + # self.db.put(b'a', b'ccc') + # self.db.merge(b'a', b'ddd') + # self.assertEqual(self.db.get(b'a'), 'ddd') - def tearDown(self): - self._close_db() +# class TestPutV1Merge(unittest.TestCase, TestHelper): + # def setUp(self): + # opts = rocksdb.Options() + # opts.create_if_missing = True + # opts.merge_operator = "put_v1" + # self._clean() + # self.db = rocksdb.DB('/tmp/test', opts) - def test_merge(self): - self.db.put(b'a', b'ccc') - self.db.merge(b'a', b'ddd') - self.assertEqual(self.db.get(b'a'), 'ddd') + # def tearDown(self): + # self._close_db() -class TestPutV1Merge(unittest.TestCase, TestHelper): - def setUp(self): - opts = rocksdb.Options() - opts.create_if_missing = True - opts.merge_operator = "put_v1" - self._clean() - self.db = rocksdb.DB('/tmp/test', opts) - - def tearDown(self): - self._close_db() - - def test_merge(self): - self.db.put(b'a', b'ccc') - self.db.merge(b'a', b'ddd') - self.assertEqual(self.db.get(b'a'), 'ddd') + # def test_merge(self): + # self.db.put(b'a', b'ccc') + # self.db.merge(b'a', b'ddd') + # self.assertEqual(self.db.get(b'a'), 'ddd') class TestStringAppendOperatorMerge(unittest.TestCase, TestHelper): def setUp(self): opts = rocksdb.Options() opts.create_if_missing = True - opts.merge_operator = "stringappend" + opts.merge_operator = StringAppendOperator() self._clean() self.db = rocksdb.DB('/tmp/test', opts) @@ -361,21 +345,21 @@ class TestStringAppendOperatorMerge(unittest.TestCase, TestHelper): self.db.merge(b'a', b'ddd') self.assertEqual(self.db.get(b'a'), 'ccc,ddd') -class TestStringMaxOperatorMerge(unittest.TestCase, TestHelper): - def setUp(self): - opts = rocksdb.Options() - opts.create_if_missing = True - opts.merge_operator = "max" - self._clean() - self.db = rocksdb.DB('/tmp/test', opts) +# class TestStringMaxOperatorMerge(unittest.TestCase, TestHelper): + # def setUp(self): + # opts = rocksdb.Options() + # opts.create_if_missing = True + # opts.merge_operator = "max" + # self._clean() + # self.db = rocksdb.DB('/tmp/test', opts) - def tearDown(self): - self._close_db() + # def tearDown(self): + # self._close_db() - def test_merge(self): - self.db.put(b'a', int_to_bytes(55)) - self.db.merge(b'a', int_to_bytes(56)) - self.assertEqual(int(self.db.get(b'a')), 56) + # def test_merge(self): + # self.db.put(b'a', int_to_bytes(55)) + # self.db.merge(b'a', int_to_bytes(56)) + # self.assertEqual(int(self.db.get(b'a')), 56) class TestAssocMerge(unittest.TestCase, TestHelper): diff --git a/rocksdb/tests/test_options.py b/rocksdb/tests/test_options.py index fc9ec94..c585190 100644 --- a/rocksdb/tests/test_options.py +++ b/rocksdb/tests/test_options.py @@ -22,18 +22,18 @@ class TestMergeOperator(rocksdb.interfaces.MergeOperator): return b'testmergeop' class TestOptions(unittest.TestCase): - def test_default_merge_operator(self): - opts = rocksdb.Options() - self.assertEqual(True, opts.paranoid_checks) - opts.paranoid_checks = False - self.assertEqual(False, opts.paranoid_checks) + # def test_default_merge_operator(self): + # opts = rocksdb.Options() + # self.assertEqual(True, opts.paranoid_checks) + # opts.paranoid_checks = False + # self.assertEqual(False, opts.paranoid_checks) - self.assertIsNone(opts.merge_operator) - opts.merge_operator = "uint64add" - self.assertIsNotNone(opts.merge_operator) - self.assertEqual(opts.merge_operator, "uint64add") - with self.assertRaises(TypeError): - opts.merge_operator = "not an operator" + # self.assertIsNone(opts.merge_operator) + # opts.merge_operator = "uint64add" + # self.assertIsNotNone(opts.merge_operator) + # self.assertEqual(opts.merge_operator, "uint64add") + # with self.assertRaises(TypeError): + # opts.merge_operator = "not an operator" def test_compaction_pri(self): opts = rocksdb.Options()