diff --git a/lbry/extras/daemon/storage.py b/lbry/extras/daemon/storage.py index 98698d276..cf5ae1a7b 100644 --- a/lbry/extras/daemon/storage.py +++ b/lbry/extras/daemon/storage.py @@ -539,22 +539,30 @@ class SQLiteStorage(SQLiteMixin): Update files that have been removed from the downloads directory since the last run """ def update_manually_removed_files(transaction: sqlite3.Connection): - removed = [] - for (stream_hash, download_directory, file_name) in transaction.execute( - "select stream_hash, download_directory, file_name from file where saved_file=1 " + files = {} + query = "select stream_hash, download_directory, file_name from file where saved_file=1 " \ "and stream_hash is not null" - ).fetchall(): - if download_directory and file_name and os.path.isfile( - os.path.join(binascii.unhexlify(download_directory).decode(), - binascii.unhexlify(file_name).decode())): - continue - removed.append((stream_hash,)) - if removed: - transaction.executemany( - "update file set file_name=null, download_directory=null, saved_file=0 where stream_hash=?", - removed - ).fetchall() - return await self.db.run(update_manually_removed_files) + for (stream_hash, download_directory, file_name) in transaction.execute(query).fetchall(): + if download_directory and file_name: + files[stream_hash] = download_directory, file_name + return files + + def detect_removed(files): + return [ + stream_hash for stream_hash, (download_directory, file_name) in files.items() + if not os.path.isfile(os.path.join(binascii.unhexlify(download_directory).decode(), + binascii.unhexlify(file_name).decode())) + ] + + def update_db_removed(transaction: sqlite3.Connection, removed): + query = "update file set file_name=null, download_directory=null, saved_file=0 where stream_hash in {}" + for cur in _batched_select(transaction, query, removed): + cur.fetchall() + + stream_and_file = await self.db.run(update_manually_removed_files) + removed = await self.loop.run_in_executor(None, detect_removed, stream_and_file) + if removed: + await self.db.run(update_db_removed, removed) def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]: return self.db.run(get_all_lbry_files)