diff --git a/convert_leveldb_rocksdb.py b/convert_leveldb_rocksdb.py new file mode 100644 index 000000000..1064c7c16 --- /dev/null +++ b/convert_leveldb_rocksdb.py @@ -0,0 +1,40 @@ +import os +import plyvel +from lbry.wallet.server.db.prefixes import HubDB, DB_PREFIXES + + +def main(db_dir: str): + old_path = os.path.join(db_dir, 'lbry-leveldb') + new_path = os.path.join(db_dir, 'lbry-rocksdb') + old_db = plyvel.DB( + old_path, create_if_missing=False, max_open_files=64, + ) + db = HubDB(new_path, max_open_files=64) + try: + + for prefix, cf in db.column_families.items(): + cnt = 0 + for shard_int in range(2**8): + shard_prefix = prefix + shard_int.to_bytes(1, byteorder='big') + with db._db.write_batch() as batch: + batch_put = batch.put + with old_db.iterator(prefix=shard_prefix, fill_cache=False) as it: + cnt_batch = 0 + for k, v in it: + batch_put((cf, k), v) + cnt += 1 + cnt_batch += 1 + if cnt % 100_000 == 0: + print(f"wrote {cnt} {DB_PREFIXES(prefix).name} items") + if cnt_batch % 1_000_000 == 0: + print(f"cnt_batch {cnt_batch} flushing 1m items") + db._db.write(batch) + batch.clear() + + finally: + old_db.close() + db.close() + + +if __name__ == "__main__": + main('/mnt/sdb/wallet_server/_data/') diff --git a/lbry/wallet/server/db/interface.py b/lbry/wallet/server/db/interface.py index 1452b16ca..f4fabac9d 100644 --- a/lbry/wallet/server/db/interface.py +++ b/lbry/wallet/server/db/interface.py @@ -1,5 +1,6 @@ import struct import typing +import os import rocksdb from typing import Optional @@ -15,15 +16,17 @@ class PrefixDB: PARTIAL_UNDO_KEY_STRUCT = struct.Struct(b'>Q') def __init__(self, path, max_open_files=64, secondary_path='', max_undo_depth: int = 200, unsafe_prefixes=None): + db_exists = os.path.exists(path) column_family_options = { prefix.value: rocksdb.ColumnFamilyOptions() for prefix in DB_PREFIXES - } if secondary_path else {} + } if secondary_path or db_exists else {} self.column_families: typing.Dict[bytes, 'rocksdb.ColumnFamilyHandle'] = {} - self._db = rocksdb.DB( - path, rocksdb.Options( + options = rocksdb.Options( create_if_missing=True, use_fsync=True, target_file_size_base=33554432, max_open_files=max_open_files if not secondary_path else -1 - ), secondary_name=secondary_path, column_families=column_family_options + ) + self._db = rocksdb.DB( + path, options, secondary_name='', column_families=column_family_options ) for prefix in DB_PREFIXES: cf = self._db.get_column_family(prefix.value)