LBRYCommunity/lbry-sdk
1
0
Fork 1
Code Issues Pull requests Projects Releases 282 Packages Wiki Activity

fix inefficient loop in update_manually_removed_files_since_last_run

Browse source
This commit is contained in:
Jack Robison 2020-01-13 12:46:00 -05:00
parent f0cfde36f2
commit 1d7b87b3a9
No known key found for this signature in database
GPG key ID: DF25C68FE0239BB2
1 changed files with 23 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
lbry/extras/daemon/storage.py
View file

@ -539,22 +539,30 @@ class SQLiteStorage(SQLiteMixin):
Update files that have been removed from the downloads directory since the last run Update files that have been removed from the downloads directory since the last run
""" """
def update_manually_removed_files(transaction: sqlite3.Connection): def update_manually_removed_files(transaction: sqlite3.Connection):
removed = [] files = {}
for (stream_hash, download_directory, file_name) in transaction.execute( query = "select stream_hash, download_directory, file_name from file where saved_file=1 " \
"select stream_hash, download_directory, file_name from file where saved_file=1 "
"and stream_hash is not null" "and stream_hash is not null"
).fetchall(): for (stream_hash, download_directory, file_name) in transaction.execute(query).fetchall():
if download_directory and file_name and os.path.isfile( if download_directory and file_name:
os.path.join(binascii.unhexlify(download_directory).decode(), files[stream_hash] = download_directory, file_name
binascii.unhexlify(file_name).decode())): return files
continue
removed.append((stream_hash,)) def detect_removed(files):
if removed: return [
transaction.executemany( stream_hash for stream_hash, (download_directory, file_name) in files.items()
"update file set file_name=null, download_directory=null, saved_file=0 where stream_hash=?", if not os.path.isfile(os.path.join(binascii.unhexlify(download_directory).decode(),
removed binascii.unhexlify(file_name).decode()))
).fetchall() ]
return await self.db.run(update_manually_removed_files)
def update_db_removed(transaction: sqlite3.Connection, removed):
query = "update file set file_name=null, download_directory=null, saved_file=0 where stream_hash in {}"
for cur in _batched_select(transaction, query, removed):
cur.fetchall()
stream_and_file = await self.db.run(update_manually_removed_files)
removed = await self.loop.run_in_executor(None, detect_removed, stream_and_file)
if removed:
await self.db.run(update_db_removed, removed)
def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]: def get_all_lbry_files(self) -> typing.Awaitable[typing.List[typing.Dict]]:
return self.db.run(get_all_lbry_files) return self.db.run(get_all_lbry_files)