From 828be96dba47ba17dd5da12efb829a8c2726cf9c Mon Sep 17 00:00:00 2001 From: hofmockel Date: Fri, 17 Jan 2014 07:42:48 +0100 Subject: [PATCH] Update docu about the bytes and unicode --- docs/api/database.rst | 52 +++++++++++++-------------- docs/api/interfaces.rst | 50 +++++++++++++------------- docs/index.rst | 16 +++------ docs/tutorial/index.rst | 80 +++++++++++++++++++++-------------------- 4 files changed, 96 insertions(+), 102 deletions(-) diff --git a/docs/api/database.rst b/docs/api/database.rst index 1e21ebf..c9944f8 100644 --- a/docs/api/database.rst +++ b/docs/api/database.rst @@ -20,8 +20,8 @@ Database object Set the database entry for "key" to "value". - :param string key: Name for this entry - :param string value: Data for this entry + :param bytes key: Name for this entry + :param bytes value: Data for this entry :param bool sync: If ``True``, the write will be flushed from the operating system buffer cache (by calling WritableFile::Sync()) before the write @@ -46,7 +46,7 @@ Database object Remove the database entry for "key". - :param string key: Name to delete + :param bytes key: Name to delete :param sync: See :py:meth:`rocksdb.DB.put` :param disable_wal: See :py:meth:`rocksdb.DB.put` :raises rocksdb.errors.NotFound: If the key did not exists @@ -74,7 +74,7 @@ Database object .. py:method:: get(key, verify_checksums=False, fill_cache=True, prefix_seek=False, snapshot=None, read_tier="all") - :param string key: Name to get + :param bytes key: Name to get :param bool verify_checksums: If ``True``, all data read from underlying storage will be @@ -110,12 +110,12 @@ Database object .. py:method:: multi_get(keys, verify_checksums=False, fill_cache=True, prefix_seek=False, snapshot=None, read_tier="all") :param keys: Keys to fetch - :type keys: list of strings + :type keys: list of bytes For the other params see :py:meth:`rocksdb.DB.get` :returns: - A ``dict`` where the value is either ``string`` or ``None`` if not found + A ``dict`` where the value is either ``bytes`` or ``None`` if not found :raises: If the fetch for a single key fails @@ -131,7 +131,7 @@ Database object This check is potentially lighter-weight than invoking DB::get(). One way to make this lighter weight is to avoid doing any IOs. - :param string key: Key to check + :param bytes key: Key to check :param bool fetch: Obtain also the value if found For the other params see :py:meth:`rocksdb.DB.get` @@ -146,7 +146,7 @@ Database object Iterate over the keys - :param string prefix: Not implemented yet + :param bytes prefix: Not implemented yet For other params see :py:meth:`rocksdb.DB.get` @@ -160,7 +160,7 @@ Database object Iterate over the values - :param string prefix: Not implemented yet + :param bytes prefix: Not implemented yet For other params see :py:meth:`rocksdb.DB.get` @@ -174,7 +174,7 @@ Database object Iterate over the items - :param string prefix: Not implemented yet + :param bytes prefix: Not implemented yet For other params see :py:meth:`rocksdb.DB.get` @@ -197,18 +197,18 @@ Database object DB implementations can export properties about their state via this method. If "property" is a valid property understood by this - DB implementation, a string with its value is returned. + DB implementation, a byte string with its value is returned. Otherwise ``None`` Valid property names include: - * ``"rocksdb.num-files-at-level"``: return the number of files at level , + * ``b"rocksdb.num-files-at-level"``: return the number of files at level , where is an ASCII representation of a level number (e.g. "0"). - * ``"rocksdb.stats"``: returns a multi-line string that describes statistics + * ``b"rocksdb.stats"``: returns a multi-line byte string that describes statistics about the internal operation of the DB. - * ``"rocksdb.sstables"``: returns a multi-line string that describes all + * ``b"rocksdb.sstables"``: returns a multi-line byte string that describes all of the sstables that make up the db contents. .. py:method:: get_live_files_metadata() @@ -265,7 +265,7 @@ Iterator .. py:method:: seek(key) - :param string key: Position at the first key in the source that at or past + :param bytes key: Position at the first key in the source that at or past Methods to support the python iterator protocol @@ -294,16 +294,16 @@ WriteBatch after the following batch is written:: batch = rocksdb.WriteBatch() - batch.put("key", "v1") - batch.delete("key") - batch.put("key", "v2") - batch.put("key", "v3") + batch.put(b"key", b"v1") + batch.delete(b"key") + batch.put(b"key", b"v2") + batch.put(b"key", b"v3") .. py:method:: __init__(data=None) Creates a WriteBatch. - :param string data: + :param bytes data: A serialized version of a previous WriteBatch. As retrieved from a previous .data() call. If ``None`` a empty WriteBatch is generated @@ -312,21 +312,21 @@ WriteBatch Store the mapping "key->value" in the database. - :param string key: Name of the entry to store - :param string value: Data of this entry + :param bytes key: Name of the entry to store + :param bytes value: Data of this entry .. py:method:: merge(key, value) Merge "value" with the existing value of "key" in the database. - :param string key: Name of the entry to merge - :param string value: Data to merge + :param bytes key: Name of the entry to merge + :param bytes value: Data to merge .. py:method:: delete(key) If the database contains a mapping for "key", erase it. Else do nothing. - :param string key: Key to erase + :param bytes key: Key to erase .. py:method:: clear() @@ -336,7 +336,7 @@ WriteBatch Retrieve the serialized version of this batch. - :rtype: string + :rtype: ``bytes`` .. py:method:: count() diff --git a/docs/api/interfaces.rst b/docs/api/interfaces.rst index 804b7af..f178ac6 100644 --- a/docs/api/interfaces.rst +++ b/docs/api/interfaces.rst @@ -15,8 +15,8 @@ Comparator Three-way comparison. - :param string a: First field to compare - :param string b: Second field to compare + :param bytes a: First field to compare + :param bytes b: Second field to compare :returns: * -1 if a < b * 0 if a == b * 1 if a > b @@ -35,7 +35,7 @@ Comparator Names starting with "rocksdb." are reserved and should not be used by any clients of this package. - :rtype: ``string`` + :rtype: ``bytes`` Merge Operator ============== @@ -75,11 +75,11 @@ AssociativeMergeOperator Gives the client a way to express the read -> modify -> write semantics - :param string key: The key that's associated with this merge operation - :param string existing_value: The current value in the db. + :param bytes key: The key that's associated with this merge operation + :param bytes existing_value: The current value in the db. ``None`` indicates the key does not exist before this op - :param string value: The value to update/merge the existing_value with + :param bytes value: The value to update/merge the existing_value with :returns: ``True`` and the new value on success. All values passed in will be client-specific values. @@ -88,7 +88,7 @@ AssociativeMergeOperator The client should assume that this will be treated as an error by the library. - :rtype: ``(bool, string)`` + :rtype: ``(bool, bytes)`` .. py:method:: name() @@ -96,7 +96,7 @@ AssociativeMergeOperator For example a DB created with one MergeOperator is accessed using a different MergeOperator. - :rtype: ``string`` + :rtype: ``bytes`` MergeOperator ------------- @@ -104,21 +104,21 @@ MergeOperator .. py:class:: rocksdb.interfaces.MergeOperator .. py:method:: full_merge(key, existing_value, operand_list) - + Gives the client a way to express the read -> modify -> write semantics - :param string key: The key that's associated with this merge operation. + :param bytes key: The key that's associated with this merge operation. Client could multiplex the merge operator based on it if the key space is partitioned and different subspaces refer to different types of data which have different merge operation semantics - :param string existing_value: The current value in the db. - ``None`` indicates the key does not exist - before this op + :param bytes existing_value: The current value in the db. + ``None`` indicates the key does not exist + before this op :param operand_list: The sequence of merge operations to apply. - :type operand_list: list of strings + :type operand_list: list of bytes :returns: ``True`` and the new value on success. All values passed in will be client-specific values. @@ -127,7 +127,7 @@ MergeOperator The client should assume that this will be treated as an error by the library. - :rtype: ``(bool, string)`` + :rtype: ``(bool, bytes)`` .. py:method:: partial_merge(key, left_operand, right_operand) @@ -147,10 +147,10 @@ MergeOperator operations, and apply them in the correct order once a base-value (a Put/Delete/End-of-Database) is seen. - :param string key: the key that is associated with this merge operation. - :param string left_operand: First operand to merge - :param string right_operand: Second operand to merge - :rtype: ``(bool, string)`` + :param bytes key: the key that is associated with this merge operation. + :param bytes left_operand: First operand to merge + :param bytes right_operand: Second operand to merge + :rtype: ``(bool, bytes)`` .. note:: @@ -166,7 +166,7 @@ MergeOperator For example a DB created with one MergeOperator is accessed using a different MergeOperator. - :rtype: ``string`` + :rtype: ``bytes`` FilterPolicy ============ @@ -180,17 +180,17 @@ FilterPolicy :param keys: list of keys (potentially with duplicates) that are ordered according to the user supplied comparator. - :type keys: list of strings + :type keys: list of bytes :returns: A filter that summarizes keys - :rtype: ``string`` + :rtype: ``bytes`` .. py:method:: key_may_match(key, filter) Check if the key is maybe in the filter. - :param string key: Key for a single entry inside the database - :param string filter: Contains the data returned by a preceding call + :param bytes key: Key for a single entry inside the database + :param bytes filter: Contains the data returned by a preceding call to create_filter on this class :returns: This method must return ``True`` if the key was in the list of keys passed to create_filter(). @@ -207,4 +207,4 @@ FilterPolicy must be changed. Otherwise, old incompatible filters may be passed to methods of this type. - :rtype: ``string`` + :rtype: ``bytes`` diff --git a/docs/index.rst b/docs/index.rst index 53add5a..1ece8ec 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,10 +7,10 @@ Python bindings to the C++ interface of http://rocksdb.org/ using cython:: import rocksdb db = rocksdb.DB("test.db", rocksdb.Options(create_if_missing=True)) - db.put("a", "b") - print db.get("a") + db.put(b"a", b"b") + print db.get(b"a") -Tested with python2.7 +Tested with python2.7 and python3.3 .. toctree:: :maxdepth: 2 @@ -23,16 +23,8 @@ Tested with python2.7 RoadMap/TODO ------------ -* Links from tutorial to API pages (for example merge operator) -* support python3.3. - Make it fix what kind of strings are allow. - - * Arbitrary ``unicode`` and then do some encoding/decoding, like - `redis-driver `_ - - * Or just ASCII ``bytes`` and let the user handle unicode. - * support prefix API +* Links from tutorial to API pages (for example merge operator) Indices and tables ================== diff --git a/docs/tutorial/index.rst b/docs/tutorial/index.rst index 95c0c4b..d6fa410 100644 --- a/docs/tutorial/index.rst +++ b/docs/tutorial/index.rst @@ -27,9 +27,9 @@ A more production ready open can look like this :: db = rocksdb.DB("test.db", opts) It assings a cache of 2.5G, uses a bloom filter for faster lookups and keeps -more data (64 MB) in memory before writting a .sst file +more data (64 MB) in memory before writting a .sst file. -About bytes and unicode +About Bytes and Unicode ======================== RocksDB stores all data as uninterpreted *byte strings*. @@ -49,8 +49,7 @@ The only place where you can pass unicode objects are filesytem paths like * :py:attr:`rocksdb.Options.db_log_dir` -To encode this unicode objects the `sys.getfilesystemencoding()` encoding is used - +To encode this path name, `sys.getfilesystemencoding()` encoding is used. Access ====== @@ -58,37 +57,37 @@ Access Store, Get, Delete is straight forward :: # Store - db.put("key", "value") + db.put(b"key", b"value") # Get - db.get("key") + db.get(b"key") # Delete - db.delete("key") + db.delete(b"key") It is also possible to gather modifications and apply them in a single operation :: batch = rocksdb.WriteBatch() - batch.put("key", "v1") - batch.delete("key") - batch.put("key", "v2") - batch.put("key", "v3") + batch.put(b"key", b"v1") + batch.delete(b"key") + batch.put(b"key", b"v2") + batch.put(b"key", b"v3") db.write(batch) Fetch of multiple values at once :: - db.put("key1", "v1") - db.put("key2", "v2") + db.put(b"key1", b"v1") + db.put(b"key2", b"v2") - ret = db.multi_get(["key1", "key2", "key3"]) + ret = db.multi_get([b"key1", b"key2", b"key3"]) - # prints "v1" - print ret["key1"] + # prints b"v1" + print ret[b"key1"] # prints None - print ret["key3"] + print ret[b"key3"] Iteration ========= @@ -96,22 +95,22 @@ Iteration Iterators behave slightly different than expected. Per default they are not valid. So you have to call one of its seek methods first :: - db.put("key1", "v1") - db.put("key2", "v2") - db.put("key3", "v3") + db.put(b"key1", b"v1") + db.put(b"key2", b"v2") + db.put(b"key3", b"v3") it = db.iterkeys() it.seek_to_first() - # prints ['key1', 'key2', 'key3'] + # prints [b'key1', b'key2', b'key3'] print list(it) it.seek_to_last() - # prints ['key3'] + # prints [b'key3'] print list(it) - it.seek('key2') - # prints ['key2', 'key3'] + it.seek(b'key2') + # prints [b'key2', b'key3'] print list(it) There are also methods to iterate over values/items :: @@ -119,13 +118,13 @@ There are also methods to iterate over values/items :: it = db.itervalues() it.seek_to_first() - # prints ['v1', 'v2', 'v3'] + # prints [b'v1', b'v2', b'v3'] print list(it) it = db.iteritems() it.seek_to_first() - # prints [('key1', 'v1'), ('key2, 'v2'), ('key3', 'v3')] + # prints [(b'key1', b'v1'), (b'key2, b'v2'), (b'key3', b'v3')] print list(it) Reversed iteration :: @@ -133,7 +132,7 @@ Reversed iteration :: it = db.iteritems() it.seek_to_last() - # prints [('key3', 'v3'), ('key2', 'v2'), ('key1', 'v1')] + # prints [(b'key3', b'v3'), (b'key2', b'v2'), (b'key1', b'v1')] print list(reversed(it)) @@ -142,23 +141,23 @@ Snapshots Snapshots are nice to get a consistent view on the database :: - self.db.put("a", "1") - self.db.put("b", "2") + self.db.put(b"a", b"1") + self.db.put(b"b", b"2") snapshot = self.db.snapshot() - self.db.put("a", "2") - self.db.delete("b") + self.db.put(b"a", b"2") + self.db.delete(b"b") it = self.db.iteritems() it.seek_to_first() - # prints {'a': '2'} + # prints {b'a': b'2'} print dict(it) it = self.db.iteritems(snapshot=snapshot) it.seek_to_first() - # prints {'a': '1', 'b': '2'} + # prints {b'a': b'1', b'b': b'2'} print dict(it) @@ -172,11 +171,12 @@ The simple Associative merge :: class AssocCounter(rocksdb.interfaces.AssociativeMergeOperator): def merge(self, key, existing_value, value): if existing_value: - return (True, str(int(existing_value) + int(value))) + s = int(existing_value) + int(value) + return (True, str(s).encode('ascii')) return (True, value) def name(self): - return 'AssocCounter' + return b'AssocCounter' opts = rocksdb.Options() @@ -184,8 +184,10 @@ The simple Associative merge :: opts.merge_operator = AssocCounter() db = rocksdb.DB('test.db', opts) - db.merge("a", "1") - db.merge("a", "1") + db.merge(b"a", b"1") + db.merge(b"a", b"1") + + # prints b'2' + print db.get(b"a") + - # prints '2' - print db.get("a")