300 lines
13 KiB
Python
300 lines
13 KiB
Python
import struct
|
|
import typing
|
|
import rocksdb
|
|
from typing import Optional
|
|
from scribe.db.common import DB_PREFIXES, COLUMN_SETTINGS
|
|
from scribe.db.revertable import RevertableOpStack, RevertablePut, RevertableDelete
|
|
|
|
|
|
ROW_TYPES = {}
|
|
|
|
|
|
class PrefixRowType(type):
|
|
def __new__(cls, name, bases, kwargs):
|
|
klass = super().__new__(cls, name, bases, kwargs)
|
|
if name != "PrefixRow":
|
|
ROW_TYPES[klass.prefix] = klass
|
|
cache_size = klass.cache_size
|
|
COLUMN_SETTINGS[klass.prefix] = {
|
|
'cache_size': cache_size,
|
|
}
|
|
return klass
|
|
|
|
|
|
class PrefixRow(metaclass=PrefixRowType):
|
|
prefix: bytes
|
|
key_struct: struct.Struct
|
|
value_struct: struct.Struct
|
|
key_part_lambdas = []
|
|
cache_size: int = 1024 * 1024 * 16
|
|
|
|
def __init__(self, db: 'rocksdb.DB', op_stack: RevertableOpStack):
|
|
self._db = db
|
|
self._op_stack = op_stack
|
|
self._column_family = self._db.get_column_family(self.prefix)
|
|
if not self._column_family.is_valid:
|
|
raise RuntimeError('column family is not valid')
|
|
|
|
def iterate(self, prefix=None, start=None, stop=None, reverse: bool = False, include_key: bool = True,
|
|
include_value: bool = True, fill_cache: bool = True, deserialize_key: bool = True,
|
|
deserialize_value: bool = True):
|
|
if not prefix and not start and not stop:
|
|
prefix = ()
|
|
if prefix is not None:
|
|
prefix = self.pack_partial_key(*prefix)
|
|
if stop is None:
|
|
try:
|
|
stop = (int.from_bytes(prefix, byteorder='big') + 1).to_bytes(len(prefix), byteorder='big')
|
|
except OverflowError:
|
|
stop = (int.from_bytes(prefix, byteorder='big') + 1).to_bytes(len(prefix) + 1, byteorder='big')
|
|
else:
|
|
stop = self.pack_partial_key(*stop)
|
|
else:
|
|
if start is not None:
|
|
start = self.pack_partial_key(*start)
|
|
if stop is not None:
|
|
stop = self.pack_partial_key(*stop)
|
|
|
|
if deserialize_key:
|
|
key_getter = lambda _k: self.unpack_key(_k)
|
|
else:
|
|
key_getter = lambda _k: _k
|
|
if deserialize_value:
|
|
value_getter = lambda _v: self.unpack_value(_v)
|
|
else:
|
|
value_getter = lambda _v: _v
|
|
|
|
it = self._db.iterator(
|
|
start or prefix, self._column_family, iterate_lower_bound=(start or prefix),
|
|
iterate_upper_bound=stop, reverse=reverse, include_key=include_key,
|
|
include_value=include_value, fill_cache=fill_cache, prefix_same_as_start=False
|
|
)
|
|
|
|
if include_key and include_value:
|
|
for k, v in it:
|
|
yield key_getter(k[1]), value_getter(v)
|
|
elif include_key:
|
|
for k in it:
|
|
yield key_getter(k[1])
|
|
elif include_value:
|
|
for v in it:
|
|
yield value_getter(v)
|
|
else:
|
|
for _ in it:
|
|
yield None
|
|
|
|
def get(self, *key_args, fill_cache=True, deserialize_value=True):
|
|
v = self._db.get((self._column_family, self.pack_key(*key_args)), fill_cache=fill_cache)
|
|
if v:
|
|
return v if not deserialize_value else self.unpack_value(v)
|
|
|
|
def multi_get(self, key_args: typing.List[typing.Tuple], fill_cache=True, deserialize_value=True):
|
|
packed_keys = {tuple(args): self.pack_key(*args) for args in key_args}
|
|
db_result = self._db.multi_get([(self._column_family, packed_keys[tuple(args)]) for args in key_args],
|
|
fill_cache=fill_cache)
|
|
result = {k[-1]: v for k, v in (db_result or {}).items()}
|
|
|
|
def handle_value(v):
|
|
return None if v is None else v if not deserialize_value else self.unpack_value(v)
|
|
|
|
return [
|
|
handle_value(result[packed_keys[tuple(k_args)]]) for k_args in key_args
|
|
]
|
|
|
|
def stage_multi_put(self, items):
|
|
self._op_stack.multi_put([RevertablePut(self.pack_key(*k), self.pack_value(*v)) for k, v in items])
|
|
|
|
def get_pending(self, *key_args, fill_cache=True, deserialize_value=True):
|
|
packed_key = self.pack_key(*key_args)
|
|
last_op = self._op_stack.get_last_op_for_key(packed_key)
|
|
if last_op:
|
|
if last_op.is_put:
|
|
return last_op.value if not deserialize_value else self.unpack_value(last_op.value)
|
|
else: # it's a delete
|
|
return
|
|
v = self._db.get((self._column_family, packed_key), fill_cache=fill_cache)
|
|
if v:
|
|
return v if not deserialize_value else self.unpack_value(v)
|
|
|
|
def stage_put(self, key_args=(), value_args=()):
|
|
self._op_stack.append_op(RevertablePut(self.pack_key(*key_args), self.pack_value(*value_args)))
|
|
|
|
def stage_delete(self, key_args=(), value_args=()):
|
|
self._op_stack.append_op(RevertableDelete(self.pack_key(*key_args), self.pack_value(*value_args)))
|
|
|
|
@classmethod
|
|
def pack_partial_key(cls, *args) -> bytes:
|
|
return cls.prefix + cls.key_part_lambdas[len(args)](*args)
|
|
|
|
@classmethod
|
|
def pack_key(cls, *args) -> bytes:
|
|
return cls.prefix + cls.key_struct.pack(*args)
|
|
|
|
@classmethod
|
|
def pack_value(cls, *args) -> bytes:
|
|
return cls.value_struct.pack(*args)
|
|
|
|
@classmethod
|
|
def unpack_key(cls, key: bytes):
|
|
assert key[:1] == cls.prefix, f"prefix should be {cls.prefix}, got {key[:1]}"
|
|
return cls.key_struct.unpack(key[1:])
|
|
|
|
@classmethod
|
|
def unpack_value(cls, data: bytes):
|
|
return cls.value_struct.unpack(data)
|
|
|
|
@classmethod
|
|
def unpack_item(cls, key: bytes, value: bytes):
|
|
return cls.unpack_key(key), cls.unpack_value(value)
|
|
|
|
def estimate_num_keys(self) -> int:
|
|
return int(self._db.get_property(b'rocksdb.estimate-num-keys', self._column_family).decode())
|
|
|
|
|
|
class BasePrefixDB:
|
|
"""
|
|
Base class for a revertable rocksdb database (a rocksdb db where each set of applied changes can be undone)
|
|
"""
|
|
UNDO_KEY_STRUCT = struct.Struct(b'>Q32s')
|
|
PARTIAL_UNDO_KEY_STRUCT = struct.Struct(b'>Q')
|
|
|
|
def __init__(self, path, max_open_files=64, secondary_path='', max_undo_depth: int = 200, unsafe_prefixes=None):
|
|
column_family_options = {}
|
|
for prefix in DB_PREFIXES:
|
|
settings = COLUMN_SETTINGS[prefix.value]
|
|
column_family_options[prefix.value] = rocksdb.ColumnFamilyOptions()
|
|
column_family_options[prefix.value].table_factory = rocksdb.BlockBasedTableFactory(
|
|
block_cache=rocksdb.LRUCache(settings['cache_size']),
|
|
)
|
|
self.column_families: typing.Dict[bytes, 'rocksdb.ColumnFamilyHandle'] = {}
|
|
options = rocksdb.Options(
|
|
create_if_missing=True, use_fsync=False, target_file_size_base=33554432,
|
|
max_open_files=max_open_files if not secondary_path else -1, create_missing_column_families=True
|
|
)
|
|
self._db = rocksdb.DB(
|
|
path, options, secondary_name=secondary_path, column_families=column_family_options
|
|
)
|
|
for prefix in DB_PREFIXES:
|
|
cf = self._db.get_column_family(prefix.value)
|
|
if cf is None and not secondary_path:
|
|
self._db.create_column_family(prefix.value, column_family_options[prefix.value])
|
|
cf = self._db.get_column_family(prefix.value)
|
|
self.column_families[prefix.value] = cf
|
|
|
|
self._op_stack = RevertableOpStack(self.get, self.multi_get, unsafe_prefixes=unsafe_prefixes)
|
|
self._max_undo_depth = max_undo_depth
|
|
|
|
def unsafe_commit(self):
|
|
"""
|
|
Write staged changes to the database without keeping undo information
|
|
Changes written cannot be undone
|
|
"""
|
|
try:
|
|
if not len(self._op_stack):
|
|
return
|
|
with self._db.write_batch(sync=True) as batch:
|
|
batch_put = batch.put
|
|
batch_delete = batch.delete
|
|
get_column_family = self.column_families.__getitem__
|
|
for staged_change in self._op_stack:
|
|
column_family = get_column_family(DB_PREFIXES(staged_change.key[:1]).value)
|
|
if staged_change.is_put:
|
|
batch_put((column_family, staged_change.key), staged_change.value)
|
|
else:
|
|
batch_delete((column_family, staged_change.key))
|
|
finally:
|
|
self._op_stack.clear()
|
|
|
|
def commit(self, height: int, block_hash: bytes):
|
|
"""
|
|
Write changes for a block height to the database and keep undo information so that the changes can be reverted
|
|
"""
|
|
undo_ops = self._op_stack.get_undo_ops()
|
|
delete_undos = []
|
|
if height > self._max_undo_depth:
|
|
delete_undos.extend(self._db.iterator(
|
|
start=DB_PREFIXES.undo.value + self.PARTIAL_UNDO_KEY_STRUCT.pack(0),
|
|
iterate_upper_bound=DB_PREFIXES.undo.value + self.PARTIAL_UNDO_KEY_STRUCT.pack(height - self._max_undo_depth),
|
|
include_value=False
|
|
))
|
|
try:
|
|
undo_c_f = self.column_families[DB_PREFIXES.undo.value]
|
|
with self._db.write_batch(sync=True) as batch:
|
|
batch_put = batch.put
|
|
batch_delete = batch.delete
|
|
get_column_family = self.column_families.__getitem__
|
|
for staged_change in self._op_stack:
|
|
column_family = get_column_family(DB_PREFIXES(staged_change.key[:1]).value)
|
|
if staged_change.is_put:
|
|
batch_put((column_family, staged_change.key), staged_change.value)
|
|
else:
|
|
batch_delete((column_family, staged_change.key))
|
|
for undo_to_delete in delete_undos:
|
|
batch_delete((undo_c_f, undo_to_delete))
|
|
batch_put((undo_c_f, DB_PREFIXES.undo.value + self.UNDO_KEY_STRUCT.pack(height, block_hash)), undo_ops)
|
|
finally:
|
|
self._op_stack.clear()
|
|
|
|
def rollback(self, height: int, block_hash: bytes):
|
|
"""
|
|
Revert changes for a block height
|
|
"""
|
|
undo_key = DB_PREFIXES.undo.value + self.UNDO_KEY_STRUCT.pack(height, block_hash)
|
|
undo_c_f = self.column_families[DB_PREFIXES.undo.value]
|
|
undo_info = self._db.get((undo_c_f, undo_key))
|
|
self._op_stack.apply_packed_undo_ops(undo_info)
|
|
try:
|
|
with self._db.write_batch(sync=True) as batch:
|
|
batch_put = batch.put
|
|
batch_delete = batch.delete
|
|
get_column_family = self.column_families.__getitem__
|
|
for staged_change in self._op_stack:
|
|
column_family = get_column_family(DB_PREFIXES(staged_change.key[:1]).value)
|
|
if staged_change.is_put:
|
|
batch_put((column_family, staged_change.key), staged_change.value)
|
|
else:
|
|
batch_delete((column_family, staged_change.key))
|
|
# batch_delete(undo_key)
|
|
finally:
|
|
self._op_stack.clear()
|
|
|
|
def get(self, key: bytes, fill_cache: bool = True) -> Optional[bytes]:
|
|
cf = self.column_families[key[:1]]
|
|
return self._db.get((cf, key), fill_cache=fill_cache)
|
|
|
|
def multi_get(self, keys: typing.List[bytes], fill_cache=True):
|
|
first_key = keys[0]
|
|
if not all(first_key[0] == key[0] for key in keys):
|
|
raise ValueError('cannot multi-delete across column families')
|
|
cf = self.column_families[first_key[:1]]
|
|
db_result = self._db.multi_get([(cf, k) for k in keys], fill_cache=fill_cache)
|
|
return list(db_result.values())
|
|
|
|
def multi_delete(self, items: typing.List[typing.Tuple[bytes, bytes]]):
|
|
self._op_stack.multi_delete([RevertableDelete(k, v) for k, v in items])
|
|
|
|
def iterator(self, start: bytes, column_family: 'rocksdb.ColumnFamilyHandle' = None,
|
|
iterate_lower_bound: bytes = None, iterate_upper_bound: bytes = None,
|
|
reverse: bool = False, include_key: bool = True, include_value: bool = True,
|
|
fill_cache: bool = True, prefix_same_as_start: bool = False, auto_prefix_mode: bool = True):
|
|
return self._db.iterator(
|
|
start=start, column_family=column_family, iterate_lower_bound=iterate_lower_bound,
|
|
iterate_upper_bound=iterate_upper_bound, reverse=reverse, include_key=include_key,
|
|
include_value=include_value, fill_cache=fill_cache, prefix_same_as_start=prefix_same_as_start,
|
|
auto_prefix_mode=auto_prefix_mode
|
|
)
|
|
|
|
def close(self):
|
|
self._db.close()
|
|
|
|
def try_catch_up_with_primary(self):
|
|
self._db.try_catch_up_with_primary()
|
|
|
|
def stage_raw_put(self, key: bytes, value: bytes):
|
|
self._op_stack.append_op(RevertablePut(key, value))
|
|
|
|
def stage_raw_delete(self, key: bytes, value: bytes):
|
|
self._op_stack.append_op(RevertableDelete(key, value))
|
|
|
|
def estimate_num_keys(self, column_family: 'rocksdb.ColumnFamilyHandle' = None):
|
|
return int(self._db.get_property(b'rocksdb.estimate-num-keys', column_family).decode())
|