From 305226a3bd2a9076916cc809a42a3d80d2540e87 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 25 Jun 2019 14:36:32 -0400 Subject: [PATCH] fix database migration bugs --- .../lbry/extras/daemon/migrator/dbmigrator.py | 21 ++++++++++++++++--- .../extras/daemon/migrator/migrate3to4.py | 2 +- .../extras/daemon/migrator/migrate4to5.py | 3 +-- .../extras/daemon/migrator/migrate8to9.py | 5 ++--- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/lbry/lbry/extras/daemon/migrator/dbmigrator.py b/lbry/lbry/extras/daemon/migrator/dbmigrator.py index 76d2264df..6f542448e 100644 --- a/lbry/lbry/extras/daemon/migrator/dbmigrator.py +++ b/lbry/lbry/extras/daemon/migrator/dbmigrator.py @@ -1,3 +1,4 @@ +import os import logging log = logging.getLogger(__name__) @@ -27,9 +28,23 @@ def migrate_db(conf, start, end): elif current == 10: from .migrate10to11 import do_migration else: - raise Exception("DB migration of version {} to {} is not available".format(current, - current+1)) - do_migration(conf) + raise Exception("DB migration of version {} to {} is not available".format(current, current+1)) + try: + do_migration(conf) + except Exception as err: + log.info("failed to migrate database: %s", str(err)) + if os.path.exists(os.path.join(conf.data_dir, "lbrynet.sqlite")): + backup_name = f"rev_{current}_unmigrated_database" + count = 0 + while os.path.exists(os.path.join(conf.data_dir, backup_name + ".sqlite")): + count += 1 + backup_name = f"rev_{current}_unmigrated_database_{count}" + backup_path = os.path.join(conf.data_dir, backup_name + ".sqlite") + os.rename(os.path.join(conf.data_dir, "lbrynet.sqlite"), backup_path) + log.info("made a backup of the unmigrated database: %s", backup_path) + if os.path.isfile(os.path.join(conf.data_dir, "db_revision")): + os.remove(os.path.join(conf.data_dir, "db_revision")) + return None current += 1 log.info("successfully migrated the database from revision %i to %i", current - 1, current) return None diff --git a/lbry/lbry/extras/daemon/migrator/migrate3to4.py b/lbry/lbry/extras/daemon/migrator/migrate3to4.py index 02e387e4e..ad8aec6bd 100644 --- a/lbry/lbry/extras/daemon/migrator/migrate3to4.py +++ b/lbry/lbry/extras/daemon/migrator/migrate3to4.py @@ -26,7 +26,7 @@ def migrate_blobs_db(db_dir): # if blobs.db doesn't exist, skip migration if not os.path.isfile(blobs_db): - log.error("blobs.db was not found but lbryfile_info.db was found, skipping migration") + log.info("blobs.db was not found but lbryfile_info.db was found, skipping migration") return blobs_db_file = sqlite3.connect(blobs_db) diff --git a/lbry/lbry/extras/daemon/migrator/migrate4to5.py b/lbry/lbry/extras/daemon/migrator/migrate4to5.py index c5a7feb1d..1990c54bc 100644 --- a/lbry/lbry/extras/daemon/migrator/migrate4to5.py +++ b/lbry/lbry/extras/daemon/migrator/migrate4to5.py @@ -24,8 +24,7 @@ def add_lbry_file_metadata(db_dir): return if not os.path.isfile(lbryfile_info_db): - log.error( - "blockchainname.db was not found but lbryfile_info.db was found, skipping migration") + log.info("blockchainname.db was not found but lbryfile_info.db was found, skipping migration") return name_metadata_db = sqlite3.connect(name_metadata) diff --git a/lbry/lbry/extras/daemon/migrator/migrate8to9.py b/lbry/lbry/extras/daemon/migrator/migrate8to9.py index ee3510dd9..c9aee0a37 100644 --- a/lbry/lbry/extras/daemon/migrator/migrate8to9.py +++ b/lbry/lbry/extras/daemon/migrator/migrate8to9.py @@ -1,7 +1,6 @@ import sqlite3 import logging import os -import asyncio from lbry.blob.blob_info import BlobInfo from lbry.stream.descriptor import StreamDescriptor @@ -24,10 +23,10 @@ def do_migration(conf): blobs_by_stream.setdefault(stream_hash, []).append(BlobInfo(position, blob_length or 0, iv, blob_hash)) for stream_name, stream_key, suggested_filename, sd_hash, stream_hash in streams: - sd = StreamDescriptor(asyncio.get_event_loop(), blob_dir, stream_name, stream_key, suggested_filename, + sd = StreamDescriptor(None, blob_dir, stream_name, stream_key, suggested_filename, blobs_by_stream[stream_hash], stream_hash, sd_hash) if sd_hash != sd.calculate_sd_hash(): - log.warning("Stream for descriptor %s is invalid, cleaning it up", sd_hash) + log.info("Stream for descriptor %s is invalid, cleaning it up", sd_hash) blob_hashes = [blob.blob_hash for blob in blobs_by_stream[stream_hash]] delete_stream(cursor, stream_hash, sd_hash, blob_hashes, blob_dir)