From 9c8593d88e0f455129e6f08d9736653a8af734c0 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Thu, 14 Feb 2019 12:33:48 -0500 Subject: [PATCH] fix file_delete being slow --- lbrynet/extras/daemon/storage.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/lbrynet/extras/daemon/storage.py b/lbrynet/extras/daemon/storage.py index b37909240..6843b677b 100644 --- a/lbrynet/extras/daemon/storage.py +++ b/lbrynet/extras/daemon/storage.py @@ -105,6 +105,13 @@ def get_content_claim_from_outpoint(transaction: sqlite3.Connection, return StoredStreamClaim(*claim_fields) +def batched_operation(transaction, query, parameters, batch_size=900): + for start_index in range(0, len(parameters), batch_size): + current_batch = parameters[start_index:start_index+batch_size] + bind = "({})".format(','.join(['?'] * len(current_batch))) + transaction.execute(query.format(bind), current_batch) + + def _batched_select(transaction, query, parameters, batch_size=900): for start_index in range(0, len(parameters), batch_size): current_batch = parameters[start_index:start_index+batch_size] @@ -154,6 +161,16 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di return files +def delete_stream(transaction: sqlite3.Connection, descriptor: 'StreamDescriptor'): + blob_hashes = [blob.blob_hash for blob in descriptor.blobs[:-1]] + blob_hashes.append(descriptor.sd_hash) + transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,)) + transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash,)) + transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash,)) + transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash,)) + batched_operation(transaction, "delete from blob where blob_hash in {}", blob_hashes) + + class SQLiteStorage(SQLiteMixin): CREATE_TABLES_QUERY = """ pragma foreign_keys=on; @@ -425,15 +442,7 @@ class SQLiteStorage(SQLiteMixin): ) def delete_stream(self, descriptor: 'StreamDescriptor'): - def _delete_stream(transaction: sqlite3.Connection): - transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,)) - transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash, )) - transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash, )) - transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash, )) - transaction.execute("delete from blob where blob_hash=?", (descriptor.sd_hash, )) - transaction.executemany("delete from blob where blob_hash=?", - [(blob.blob_hash, ) for blob in descriptor.blobs[:-1]]) - return self.db.run(_delete_stream) + return self.db.run_with_foreign_keys_disabled(delete_stream, descriptor) # # # # # # # # # file stuff # # # # # # # # #