LBRYCommunity/lbry-rocksdb
1
0
Fork 0
Code Issues 6 Pull requests 2 Projects Releases Packages Wiki Activity

update the usage of default operators

Browse source
This commit is contained in:
twmht 2017-04-25 08:25:02 +08:00
parent 4f9e341de8
commit 5610d9474d
5 changed files with 84 additions and 118 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
docs/tutorial/index.rst
View file

@ -216,20 +216,20 @@ The following example python merge operator implements a counter ::
# prints b'2' # prints b'2'
print db.get(b"a") print db.get(b"a")
We provide a set of default operators ``uintadd64``, ``put`` and ``stringappend`` We provide a set of default operators ``uintadd64`` and ``stringappend``::
The following example using ``uintadd64`` where each operand is ``uint64`` :: from rocksdb.merge_operators import UintAddOperator, StringAppendOperator
import rocksdb
import struct
opts = rocksdb.Options() opts = rocksdb.Options()
opts.create_if_missing = True opts.create_if_missing = True
opts.merge_operator = 'uint64add' # you should also play with StringAppendOperator
db = rocksdb.DB("test.db", opts) opts.merge_operator = UintAddOperator()
# since every operand is uint64, you need to pack it into string db = rocksdb.DB('/tmp/test', opts)
db.put(b'a', struct.pack('Q', 1000)) self.db.put(b'a', struct.pack('Q', 5566))
db.merge(b'a', struct.pack('Q', 2000)) for x in range(1000):
assert struct.unpack('Q', db.get(b'a'))[0] == 3000 self.db.merge(b"a", struct.pack('Q', x))
self.assertEqual(5566 + sum(range(1000)), struct.unpack('Q', self.db.get(b'a'))[0])
PrefixExtractor PrefixExtractor
=============== ===============

36
rocksdb/_rocksdb.pyx
View file

@ -181,6 +181,8 @@ cdef class PyBytewiseComparator(PyComparator):
cdef const comparator.Comparator* get_comparator(self): cdef const comparator.Comparator* get_comparator(self):
return self.comparator_ptr return self.comparator_ptr
cdef int compare_callback( cdef int compare_callback(
void* ctx, void* ctx,
logger.Logger* log, logger.Logger* log,
@ -336,23 +338,23 @@ cdef class PyMergeOperator(object):
<void*>ob, <void*>ob,
full_merge_callback, full_merge_callback,
partial_merge_callback)) partial_merge_callback))
elif isinstance(ob, str): # elif isinstance(ob, str):
if ob == "put": # if ob == "put":
self.merge_op = merge_operator.MergeOperators.CreatePutOperator() # self.merge_op = merge_operator.MergeOperators.CreatePutOperator()
elif ob == "put_v1": # elif ob == "put_v1":
self.merge_op = merge_operator.MergeOperators.CreateDeprecatedPutOperator() # self.merge_op = merge_operator.MergeOperators.CreateDeprecatedPutOperator()
elif ob == "uint64add": # elif ob == "uint64add":
self.merge_op = merge_operator.MergeOperators.CreateUInt64AddOperator() # self.merge_op = merge_operator.MergeOperators.CreateUInt64AddOperator()
elif ob == "stringappend": # elif ob == "stringappend":
self.merge_op = merge_operator.MergeOperators.CreateStringAppendOperator() # self.merge_op = merge_operator.MergeOperators.CreateStringAppendOperator()
#TODO: necessary? # #TODO: necessary?
# elif ob == "stringappendtest": # # elif ob == "stringappendtest":
# self.merge_op = merge_operator.MergeOperators.CreateStringAppendTESTOperator() # # self.merge_op = merge_operator.MergeOperators.CreateStringAppendTESTOperator()
elif ob == "max": # elif ob == "max":
self.merge_op = merge_operator.MergeOperators.CreateMaxOperator() # self.merge_op = merge_operator.MergeOperators.CreateMaxOperator()
else: # else:
msg = "{0} is not the default type".format(ob) # msg = "{0} is not the default type".format(ob)
raise TypeError(msg) # raise TypeError(msg)
else: else:
msg = "%s is not of this types %s" msg = "%s is not of this types %s"
msg %= (ob, (IAssociativeMergeOperator, IMergeOperator)) msg %= (ob, (IAssociativeMergeOperator, IMergeOperator))

20
rocksdb/merge_operator.pxd
View file

@ -9,26 +9,6 @@ cdef extern from "rocksdb/merge_operator.h" namespace "rocksdb":
cdef cppclass MergeOperator: cdef cppclass MergeOperator:
pass pass
# cdef extern from "utilities/merge_operators.h" namespace "rocksdb::MergeOperators":
cdef extern from "utilities/merge_operators.h" namespace "rocksdb":
cdef cppclass MergeOperators:
@staticmethod
shared_ptr[MergeOperator] CreatePutOperator()
@staticmethod
shared_ptr[MergeOperator] CreateDeprecatedPutOperator()
@staticmethod
shared_ptr[MergeOperator] CreateUInt64AddOperator()
@staticmethod
shared_ptr[MergeOperator] CreateStringAppendOperator()
@staticmethod
shared_ptr[MergeOperator] CreateStringAppendTESTOperator()
@staticmethod
shared_ptr[MergeOperator] CreateMaxOperator()
@staticmethod
shared_ptr[MergeOperator] CreateFromStringId(const string &)
ctypedef cpp_bool (*merge_func)( ctypedef cpp_bool (*merge_func)(
void*, void*,
const Slice&, const Slice&,

102
rocksdb/tests/test_db.py
View file

@ -5,6 +5,7 @@ import unittest
import rocksdb import rocksdb
from itertools import takewhile from itertools import takewhile
import struct import struct
from rocksdb.merge_operators import UintAddOperator, StringAppendOperator
def int_to_bytes(ob): def int_to_bytes(ob):
return str(ob).encode('ascii') return str(ob).encode('ascii')
@ -277,11 +278,12 @@ class AssocCounter(rocksdb.interfaces.AssociativeMergeOperator):
def name(self): def name(self):
return b'AssocCounter' return b'AssocCounter'
class TestUint64Merge(unittest.TestCase, TestHelper): class TestUint64Merge(unittest.TestCase, TestHelper):
def setUp(self): def setUp(self):
opts = rocksdb.Options() opts = rocksdb.Options()
opts.create_if_missing = True opts.create_if_missing = True
opts.merge_operator = "uint64add" opts.merge_operator = UintAddOperator()
self._clean() self._clean()
self.db = rocksdb.DB('/tmp/test', opts) self.db = rocksdb.DB('/tmp/test', opts)
@ -292,64 +294,46 @@ class TestUint64Merge(unittest.TestCase, TestHelper):
self.db.put(b'a', struct.pack('Q', 5566)) self.db.put(b'a', struct.pack('Q', 5566))
for x in range(1000): for x in range(1000):
self.db.merge(b"a", struct.pack('Q', x)) self.db.merge(b"a", struct.pack('Q', x))
print ('value', struct.unpack('Q', self.db.get(b'a'))[0])
self.assertEqual(5566 + sum(range(1000)), struct.unpack('Q', self.db.get(b'a'))[0]) self.assertEqual(5566 + sum(range(1000)), struct.unpack('Q', self.db.get(b'a'))[0])
class TestUint64Merge(unittest.TestCase, TestHelper):
def setUp(self):
opts = rocksdb.Options()
opts.create_if_missing = True
opts.merge_operator = "uint64add"
self._clean()
self.db = rocksdb.DB('/tmp/test', opts)
def tearDown(self): # class TestPutMerge(unittest.TestCase, TestHelper):
self._close_db() # def setUp(self):
# opts = rocksdb.Options()
# opts.create_if_missing = True
# opts.merge_operator = "put"
# self._clean()
# self.db = rocksdb.DB('/tmp/test', opts)
def test_merge(self): # def tearDown(self):
self.db.put(b'a', struct.pack('Q', 5566)) # self._close_db()
for x in range(1000):
self.db.merge(b"a", struct.pack('Q', x))
# print ('value', struct.unpack('Q', self.db.get(b'a'))[0])
self.assertEqual(5566 + sum(range(1000)), struct.unpack('Q', self.db.get(b'a'))[0])
class TestPutMerge(unittest.TestCase, TestHelper): # def test_merge(self):
def setUp(self): # self.db.put(b'a', b'ccc')
opts = rocksdb.Options() # self.db.merge(b'a', b'ddd')
opts.create_if_missing = True # self.assertEqual(self.db.get(b'a'), 'ddd')
opts.merge_operator = "put"
self._clean()
self.db = rocksdb.DB('/tmp/test', opts)
def tearDown(self): # class TestPutV1Merge(unittest.TestCase, TestHelper):
self._close_db() # def setUp(self):
# opts = rocksdb.Options()
# opts.create_if_missing = True
# opts.merge_operator = "put_v1"
# self._clean()
# self.db = rocksdb.DB('/tmp/test', opts)
def test_merge(self): # def tearDown(self):
self.db.put(b'a', b'ccc') # self._close_db()
self.db.merge(b'a', b'ddd')
self.assertEqual(self.db.get(b'a'), 'ddd')
class TestPutV1Merge(unittest.TestCase, TestHelper): # def test_merge(self):
def setUp(self): # self.db.put(b'a', b'ccc')
opts = rocksdb.Options() # self.db.merge(b'a', b'ddd')
opts.create_if_missing = True # self.assertEqual(self.db.get(b'a'), 'ddd')
opts.merge_operator = "put_v1"
self._clean()
self.db = rocksdb.DB('/tmp/test', opts)
def tearDown(self):
self._close_db()
def test_merge(self):
self.db.put(b'a', b'ccc')
self.db.merge(b'a', b'ddd')
self.assertEqual(self.db.get(b'a'), 'ddd')
class TestStringAppendOperatorMerge(unittest.TestCase, TestHelper): class TestStringAppendOperatorMerge(unittest.TestCase, TestHelper):
def setUp(self): def setUp(self):
opts = rocksdb.Options() opts = rocksdb.Options()
opts.create_if_missing = True opts.create_if_missing = True
opts.merge_operator = "stringappend" opts.merge_operator = StringAppendOperator()
self._clean() self._clean()
self.db = rocksdb.DB('/tmp/test', opts) self.db = rocksdb.DB('/tmp/test', opts)
@ -361,21 +345,21 @@ class TestStringAppendOperatorMerge(unittest.TestCase, TestHelper):
self.db.merge(b'a', b'ddd') self.db.merge(b'a', b'ddd')
self.assertEqual(self.db.get(b'a'), 'ccc,ddd') self.assertEqual(self.db.get(b'a'), 'ccc,ddd')
class TestStringMaxOperatorMerge(unittest.TestCase, TestHelper): # class TestStringMaxOperatorMerge(unittest.TestCase, TestHelper):
def setUp(self): # def setUp(self):
opts = rocksdb.Options() # opts = rocksdb.Options()
opts.create_if_missing = True # opts.create_if_missing = True
opts.merge_operator = "max" # opts.merge_operator = "max"
self._clean() # self._clean()
self.db = rocksdb.DB('/tmp/test', opts) # self.db = rocksdb.DB('/tmp/test', opts)
def tearDown(self): # def tearDown(self):
self._close_db() # self._close_db()
def test_merge(self): # def test_merge(self):
self.db.put(b'a', int_to_bytes(55)) # self.db.put(b'a', int_to_bytes(55))
self.db.merge(b'a', int_to_bytes(56)) # self.db.merge(b'a', int_to_bytes(56))
self.assertEqual(int(self.db.get(b'a')), 56) # self.assertEqual(int(self.db.get(b'a')), 56)
class TestAssocMerge(unittest.TestCase, TestHelper): class TestAssocMerge(unittest.TestCase, TestHelper):

22
rocksdb/tests/test_options.py
View file

@ -22,18 +22,18 @@ class TestMergeOperator(rocksdb.interfaces.MergeOperator):
return b'testmergeop' return b'testmergeop'
class TestOptions(unittest.TestCase): class TestOptions(unittest.TestCase):
def test_default_merge_operator(self): # def test_default_merge_operator(self):
opts = rocksdb.Options() # opts = rocksdb.Options()
self.assertEqual(True, opts.paranoid_checks) # self.assertEqual(True, opts.paranoid_checks)
opts.paranoid_checks = False # opts.paranoid_checks = False
self.assertEqual(False, opts.paranoid_checks) # self.assertEqual(False, opts.paranoid_checks)
self.assertIsNone(opts.merge_operator) # self.assertIsNone(opts.merge_operator)
opts.merge_operator = "uint64add" # opts.merge_operator = "uint64add"
self.assertIsNotNone(opts.merge_operator) # self.assertIsNotNone(opts.merge_operator)
self.assertEqual(opts.merge_operator, "uint64add") # self.assertEqual(opts.merge_operator, "uint64add")
with self.assertRaises(TypeError): # with self.assertRaises(TypeError):
opts.merge_operator = "not an operator" # opts.merge_operator = "not an operator"
def test_compaction_pri(self): def test_compaction_pri(self):
opts = rocksdb.Options() opts = rocksdb.Options()