convert script and interface changes

convert_leveldb_rocksdb.py

@@ -0,0 +1,40 @@
+import os
+import plyvel
+from lbry.wallet.server.db.prefixes import HubDB, DB_PREFIXES
+
+
+def main(db_dir: str):
+    old_path = os.path.join(db_dir, 'lbry-leveldb')
+    new_path = os.path.join(db_dir, 'lbry-rocksdb')
+    old_db = plyvel.DB(
+        old_path, create_if_missing=False, max_open_files=64,
+    )
+    db = HubDB(new_path, max_open_files=64)
+    try:
+
+        for prefix, cf in db.column_families.items():
+            cnt = 0
+            for shard_int in range(2**8):
+                shard_prefix = prefix + shard_int.to_bytes(1, byteorder='big')
+                with db._db.write_batch() as batch:
+                    batch_put = batch.put
+                    with old_db.iterator(prefix=shard_prefix, fill_cache=False) as it:
+                        cnt_batch = 0
+                        for k, v in it:
+                            batch_put((cf, k), v)
+                            cnt += 1
+                            cnt_batch += 1
+                            if cnt % 100_000 == 0:
+                                print(f"wrote {cnt} {DB_PREFIXES(prefix).name} items")
+                            if cnt_batch % 1_000_000 == 0:
+                                print(f"cnt_batch {cnt_batch} flushing 1m items")
+                                db._db.write(batch)
+                                batch.clear()
+
+    finally:
+        old_db.close()
+        db.close()
+
+
+if __name__ == "__main__":
+    main('/mnt/sdb/wallet_server/_data/')

lbry/wallet/server/db/interface.py

@@ -1,5 +1,6 @@
 import struct
 import typing
+import os
 
 import rocksdb
 from typing import Optional
@@ -15,15 +16,17 @@ class PrefixDB:
     PARTIAL_UNDO_KEY_STRUCT = struct.Struct(b'>Q')
 
     def __init__(self, path, max_open_files=64, secondary_path='', max_undo_depth: int = 200, unsafe_prefixes=None):
+        db_exists = os.path.exists(path)
         column_family_options = {
                 prefix.value: rocksdb.ColumnFamilyOptions() for prefix in DB_PREFIXES
-        } if secondary_path else {}
+        } if secondary_path or db_exists else {}
         self.column_families: typing.Dict[bytes, 'rocksdb.ColumnFamilyHandle'] = {}
-        self._db = rocksdb.DB(
-            path, rocksdb.Options(
+        options = rocksdb.Options(
                 create_if_missing=True, use_fsync=True, target_file_size_base=33554432,
                 max_open_files=max_open_files if not secondary_path else -1
-            ), secondary_name=secondary_path, column_families=column_family_options
+            )
+        self._db = rocksdb.DB(
+            path, options, secondary_name='', column_families=column_family_options
         )
         for prefix in DB_PREFIXES:
             cf = self._db.get_column_family(prefix.value)