LBRYCommunity/lbry-sdk
1
0
Fork 1
Code Issues Pull requests Packages Projects Releases 282 Wiki Activity

fix inefficient loop in update_manually_removed_files_since_last_run

Browse source
This commit is contained in:
Jack Robison 2020-01-13 12:46:00 -05:00
parent f0cfde36f2
commit 1d7b87b3a9
No known key found for this signature in database
GPG key ID: DF25C68FE0239BB2
1 changed files with 23 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
lbry/extras/daemon/storage.py
View file

@ -539,22 +539,30 @@ class SQLiteStorage(SQLiteMixin):
Update files that have been removed from the downloads directory since the last run
"""
def update_manually_removed_files(transaction: sqlite3.Connection):
removed = []
for (stream_hash, download_directory, file_name) in transaction.execute(
"select stream_hash, download_directory, file_name from file where saved_file=1 "
files = {}
query = "select stream_hash, download_directory, file_name from file where saved_file=1 " \
"and stream_hash is not null"
).fetchall():
if download_directory and file_name and os.path.isfile(
os.path.join(binascii.unhexlify(download_directory).decode(),
binascii.unhexlify(file_name).decode())):
continue
removed.append((stream_hash,))
if removed:
transaction.executemany(
"update file set file_name=null, download_directory=null, saved_file=0 where stream_hash=?",
removed
).fetchall()
return await self.db.run(update_manually_removed_files)
for (stream_hash, download_directory, file_name) in transaction.execute(query).fetchall():
if download_directory and file_name:
files[stream_hash] = download_directory, file_name
return files
def detect_removed(files):
return [
stream_hash for stream_hash, (download_directory, file_name) in files.items()
if not os.path.isfile(os.path.join(binascii.unhexlify(download_directory).decode(),
binascii.unhexlify(file_name).decode()))
]
def update_db_removed(transaction: sqlite3.Connection, removed):
query = "update file set file_name=null, download_directory=null, saved_file=0 where stream_hash in {}"
for cur in _batched_select(transaction, query, removed):
cur.fetchall()
stream_and_file = await self.db.run(update_manually_removed_files)
removed = await self.loop.run_in_executor(None, detect_removed, stream_and_file)
if removed:
await self.db.run(update_db_removed, removed)
def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]:
return self.db.run(get_all_lbry_files)