LBRYCommunity/lbry-sdk
1
0
Fork 1
Code Issues Pull requests Projects Releases 282 Packages Wiki Activity

fix file_delete being slow

Browse source
This commit is contained in:
Jack Robison 2019-02-14 12:33:48 -05:00
parent 85f41887fd
commit 9c8593d88e
No known key found for this signature in database
GPG key ID: DF25C68FE0239BB2
1 changed files with 18 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
lbrynet/extras/daemon/storage.py
View file

@ -105,6 +105,13 @@ def get_content_claim_from_outpoint(transaction: sqlite3.Connection,
return StoredStreamClaim(*claim_fields) return StoredStreamClaim(*claim_fields)
def batched_operation(transaction, query, parameters, batch_size=900):
for start_index in range(0, len(parameters), batch_size):
current_batch = parameters[start_index:start_index+batch_size]
bind = "({})".format(','.join(['?'] * len(current_batch)))
transaction.execute(query.format(bind), current_batch)
def _batched_select(transaction, query, parameters, batch_size=900): def _batched_select(transaction, query, parameters, batch_size=900):
for start_index in range(0, len(parameters), batch_size): for start_index in range(0, len(parameters), batch_size):
current_batch = parameters[start_index:start_index+batch_size] current_batch = parameters[start_index:start_index+batch_size]
@ -154,6 +161,16 @@ def get_all_lbry_files(transaction: sqlite3.Connection) -> typing.List[typing.Di
return files return files
def delete_stream(transaction: sqlite3.Connection, descriptor: 'StreamDescriptor'):
blob_hashes = [blob.blob_hash for blob in descriptor.blobs[:-1]]
blob_hashes.append(descriptor.sd_hash)
transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,))
transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash,))
transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash,))
transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash,))
batched_operation(transaction, "delete from blob where blob_hash in {}", blob_hashes)
class SQLiteStorage(SQLiteMixin): class SQLiteStorage(SQLiteMixin):
CREATE_TABLES_QUERY = """ CREATE_TABLES_QUERY = """
pragma foreign_keys=on; pragma foreign_keys=on;
@ -425,15 +442,7 @@ class SQLiteStorage(SQLiteMixin):
) )
def delete_stream(self, descriptor: 'StreamDescriptor'): def delete_stream(self, descriptor: 'StreamDescriptor'):
def _delete_stream(transaction: sqlite3.Connection): return self.db.run_with_foreign_keys_disabled(delete_stream, descriptor)
transaction.execute("delete from content_claim where stream_hash=? ", (descriptor.stream_hash,))
transaction.execute("delete from file where stream_hash=? ", (descriptor.stream_hash, ))
transaction.execute("delete from stream_blob where stream_hash=?", (descriptor.stream_hash, ))
transaction.execute("delete from stream where stream_hash=? ", (descriptor.stream_hash, ))
transaction.execute("delete from blob where blob_hash=?", (descriptor.sd_hash, ))
transaction.executemany("delete from blob where blob_hash=?",
[(blob.blob_hash, ) for blob in descriptor.blobs[:-1]])
return self.db.run(_delete_stream)
# # # # # # # # # file stuff # # # # # # # # # # # # # # # # # # file stuff # # # # # # # # #