LBRYCommunity/lbry-rocksdb
1
0
Fork 0
Code Issues 6 Pull requests 2 Projects Releases Packages Wiki Activity

update the usage of default operators

Browse source
This commit is contained in:
twmht 2017-04-25 08:25:02 +08:00
parent 4f9e341de8
commit 5610d9474d
5 changed files with 84 additions and 118 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
docs/tutorial/index.rst
View file

@ -216,20 +216,20 @@ The following example python merge operator implements a counter ::
# prints b'2'
print db.get(b"a")
We provide a set of default operators ``uintadd64``, ``put`` and ``stringappend``
We provide a set of default operators ``uintadd64`` and ``stringappend``::
The following example using ``uintadd64`` where each operand is ``uint64`` ::
import rocksdb
import struct
from rocksdb.merge_operators import UintAddOperator, StringAppendOperator
opts = rocksdb.Options()
opts.create_if_missing = True
opts.merge_operator = 'uint64add'
db = rocksdb.DB("test.db", opts)
# since every operand is uint64, you need to pack it into string
db.put(b'a', struct.pack('Q', 1000))
db.merge(b'a', struct.pack('Q', 2000))
assert struct.unpack('Q', db.get(b'a'))[0] == 3000
# you should also play with StringAppendOperator
opts.merge_operator = UintAddOperator()
db = rocksdb.DB('/tmp/test', opts)
self.db.put(b'a', struct.pack('Q', 5566))
for x in range(1000):
self.db.merge(b"a", struct.pack('Q', x))
self.assertEqual(5566 + sum(range(1000)), struct.unpack('Q', self.db.get(b'a'))[0])
PrefixExtractor
===============

36
rocksdb/_rocksdb.pyx
View file

@ -181,6 +181,8 @@ cdef class PyBytewiseComparator(PyComparator):
cdef const comparator.Comparator* get_comparator(self):
return self.comparator_ptr
cdef int compare_callback(
void* ctx,
logger.Logger* log,
@ -336,23 +338,23 @@ cdef class PyMergeOperator(object):
<void*>ob,
full_merge_callback,
partial_merge_callback))
elif isinstance(ob, str):
if ob == "put":
self.merge_op = merge_operator.MergeOperators.CreatePutOperator()
elif ob == "put_v1":
self.merge_op = merge_operator.MergeOperators.CreateDeprecatedPutOperator()
elif ob == "uint64add":
self.merge_op = merge_operator.MergeOperators.CreateUInt64AddOperator()
elif ob == "stringappend":
self.merge_op = merge_operator.MergeOperators.CreateStringAppendOperator()
#TODO: necessary?
# elif ob == "stringappendtest":
# self.merge_op = merge_operator.MergeOperators.CreateStringAppendTESTOperator()
elif ob == "max":
self.merge_op = merge_operator.MergeOperators.CreateMaxOperator()
else:
msg = "{0} is not the default type".format(ob)
raise TypeError(msg)
# elif isinstance(ob, str):
# if ob == "put":
# self.merge_op = merge_operator.MergeOperators.CreatePutOperator()
# elif ob == "put_v1":
# self.merge_op = merge_operator.MergeOperators.CreateDeprecatedPutOperator()
# elif ob == "uint64add":
# self.merge_op = merge_operator.MergeOperators.CreateUInt64AddOperator()
# elif ob == "stringappend":
# self.merge_op = merge_operator.MergeOperators.CreateStringAppendOperator()
# #TODO: necessary?
# # elif ob == "stringappendtest":
# # self.merge_op = merge_operator.MergeOperators.CreateStringAppendTESTOperator()
# elif ob == "max":
# self.merge_op = merge_operator.MergeOperators.CreateMaxOperator()
# else:
# msg = "{0} is not the default type".format(ob)
# raise TypeError(msg)
else:
msg = "%s is not of this types %s"
msg %= (ob, (IAssociativeMergeOperator, IMergeOperator))

20
rocksdb/merge_operator.pxd
View file

@ -9,26 +9,6 @@ cdef extern from "rocksdb/merge_operator.h" namespace "rocksdb":
cdef cppclass MergeOperator:
pass
# cdef extern from "utilities/merge_operators.h" namespace "rocksdb::MergeOperators":
cdef extern from "utilities/merge_operators.h" namespace "rocksdb":
cdef cppclass MergeOperators:
@staticmethod
shared_ptr[MergeOperator] CreatePutOperator()
@staticmethod
shared_ptr[MergeOperator] CreateDeprecatedPutOperator()
@staticmethod
shared_ptr[MergeOperator] CreateUInt64AddOperator()
@staticmethod
shared_ptr[MergeOperator] CreateStringAppendOperator()
@staticmethod
shared_ptr[MergeOperator] CreateStringAppendTESTOperator()
@staticmethod
shared_ptr[MergeOperator] CreateMaxOperator()
@staticmethod
shared_ptr[MergeOperator] CreateFromStringId(const string &)
ctypedef cpp_bool (*merge_func)(
void*,
const Slice&,

102
rocksdb/tests/test_db.py
View file

@ -5,6 +5,7 @@ import unittest
import rocksdb
from itertools import takewhile
import struct
from rocksdb.merge_operators import UintAddOperator, StringAppendOperator
def int_to_bytes(ob):
return str(ob).encode('ascii')
@ -277,11 +278,12 @@ class AssocCounter(rocksdb.interfaces.AssociativeMergeOperator):
def name(self):
return b'AssocCounter'
class TestUint64Merge(unittest.TestCase, TestHelper):
def setUp(self):
opts = rocksdb.Options()
opts.create_if_missing = True
opts.merge_operator = "uint64add"
opts.merge_operator = UintAddOperator()
self._clean()
self.db = rocksdb.DB('/tmp/test', opts)
@ -292,64 +294,46 @@ class TestUint64Merge(unittest.TestCase, TestHelper):
self.db.put(b'a', struct.pack('Q', 5566))
for x in range(1000):
self.db.merge(b"a", struct.pack('Q', x))
print ('value', struct.unpack('Q', self.db.get(b'a'))[0])
self.assertEqual(5566 + sum(range(1000)), struct.unpack('Q', self.db.get(b'a'))[0])
class TestUint64Merge(unittest.TestCase, TestHelper):
def setUp(self):
opts = rocksdb.Options()
opts.create_if_missing = True
opts.merge_operator = "uint64add"
self._clean()
self.db = rocksdb.DB('/tmp/test', opts)
def tearDown(self):
self._close_db()
# class TestPutMerge(unittest.TestCase, TestHelper):
# def setUp(self):
# opts = rocksdb.Options()
# opts.create_if_missing = True
# opts.merge_operator = "put"
# self._clean()
# self.db = rocksdb.DB('/tmp/test', opts)
def test_merge(self):
self.db.put(b'a', struct.pack('Q', 5566))
for x in range(1000):
self.db.merge(b"a", struct.pack('Q', x))
# print ('value', struct.unpack('Q', self.db.get(b'a'))[0])
self.assertEqual(5566 + sum(range(1000)), struct.unpack('Q', self.db.get(b'a'))[0])
# def tearDown(self):
# self._close_db()
class TestPutMerge(unittest.TestCase, TestHelper):
def setUp(self):
opts = rocksdb.Options()
opts.create_if_missing = True
opts.merge_operator = "put"
self._clean()
self.db = rocksdb.DB('/tmp/test', opts)
# def test_merge(self):
# self.db.put(b'a', b'ccc')
# self.db.merge(b'a', b'ddd')
# self.assertEqual(self.db.get(b'a'), 'ddd')
def tearDown(self):
self._close_db()
# class TestPutV1Merge(unittest.TestCase, TestHelper):
# def setUp(self):
# opts = rocksdb.Options()
# opts.create_if_missing = True
# opts.merge_operator = "put_v1"
# self._clean()
# self.db = rocksdb.DB('/tmp/test', opts)
def test_merge(self):
self.db.put(b'a', b'ccc')
self.db.merge(b'a', b'ddd')
self.assertEqual(self.db.get(b'a'), 'ddd')
# def tearDown(self):
# self._close_db()
class TestPutV1Merge(unittest.TestCase, TestHelper):
def setUp(self):
opts = rocksdb.Options()
opts.create_if_missing = True
opts.merge_operator = "put_v1"
self._clean()
self.db = rocksdb.DB('/tmp/test', opts)
def tearDown(self):
self._close_db()
def test_merge(self):
self.db.put(b'a', b'ccc')
self.db.merge(b'a', b'ddd')
self.assertEqual(self.db.get(b'a'), 'ddd')
# def test_merge(self):
# self.db.put(b'a', b'ccc')
# self.db.merge(b'a', b'ddd')
# self.assertEqual(self.db.get(b'a'), 'ddd')
class TestStringAppendOperatorMerge(unittest.TestCase, TestHelper):
def setUp(self):
opts = rocksdb.Options()
opts.create_if_missing = True
opts.merge_operator = "stringappend"
opts.merge_operator = StringAppendOperator()
self._clean()
self.db = rocksdb.DB('/tmp/test', opts)
@ -361,21 +345,21 @@ class TestStringAppendOperatorMerge(unittest.TestCase, TestHelper):
self.db.merge(b'a', b'ddd')
self.assertEqual(self.db.get(b'a'), 'ccc,ddd')
class TestStringMaxOperatorMerge(unittest.TestCase, TestHelper):
def setUp(self):
opts = rocksdb.Options()
opts.create_if_missing = True
opts.merge_operator = "max"
self._clean()
self.db = rocksdb.DB('/tmp/test', opts)
# class TestStringMaxOperatorMerge(unittest.TestCase, TestHelper):
# def setUp(self):
# opts = rocksdb.Options()
# opts.create_if_missing = True
# opts.merge_operator = "max"
# self._clean()
# self.db = rocksdb.DB('/tmp/test', opts)
def tearDown(self):
self._close_db()
# def tearDown(self):
# self._close_db()
def test_merge(self):
self.db.put(b'a', int_to_bytes(55))
self.db.merge(b'a', int_to_bytes(56))
self.assertEqual(int(self.db.get(b'a')), 56)
# def test_merge(self):
# self.db.put(b'a', int_to_bytes(55))
# self.db.merge(b'a', int_to_bytes(56))
# self.assertEqual(int(self.db.get(b'a')), 56)
class TestAssocMerge(unittest.TestCase, TestHelper):

22
rocksdb/tests/test_options.py
View file

@ -22,18 +22,18 @@ class TestMergeOperator(rocksdb.interfaces.MergeOperator):
return b'testmergeop'
class TestOptions(unittest.TestCase):
def test_default_merge_operator(self):
opts = rocksdb.Options()
self.assertEqual(True, opts.paranoid_checks)
opts.paranoid_checks = False
self.assertEqual(False, opts.paranoid_checks)
# def test_default_merge_operator(self):
# opts = rocksdb.Options()
# self.assertEqual(True, opts.paranoid_checks)
# opts.paranoid_checks = False
# self.assertEqual(False, opts.paranoid_checks)
self.assertIsNone(opts.merge_operator)
opts.merge_operator = "uint64add"
self.assertIsNotNone(opts.merge_operator)
self.assertEqual(opts.merge_operator, "uint64add")
with self.assertRaises(TypeError):
opts.merge_operator = "not an operator"
# self.assertIsNone(opts.merge_operator)
# opts.merge_operator = "uint64add"
# self.assertIsNotNone(opts.merge_operator)
# self.assertEqual(opts.merge_operator, "uint64add")
# with self.assertRaises(TypeError):
# opts.merge_operator = "not an operator"
def test_compaction_pri(self):
opts = rocksdb.Options()