LBRYCommunity/lbry-sdk
1
0
Fork 1
Code Issues Pull requests Projects Releases 282 Packages Wiki Activity

cache space stats from running components so status is instant

Browse source
This commit is contained in:
Victor Shyba 2021-11-03 15:38:32 -03:00
parent 35905b99ff
commit b055c25156
3 changed files with 13 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
lbry/blob/disk_space_manager.py
View file

@ -14,6 +14,7 @@ class DiskSpaceManager:
self.running = False self.running = False
self.task = None self.task = None
self.analytics = analytics self.analytics = analytics
self._used_space_bytes = None
async def get_free_space_mb(self, is_network_blob=False): async def get_free_space_mb(self, is_network_blob=False):
limit_mb = self.config.network_storage_limit if is_network_blob else self.config.blob_storage_limit limit_mb = self.config.network_storage_limit if is_network_blob else self.config.blob_storage_limit
@ -22,10 +23,12 @@ class DiskSpaceManager:
return max(0, limit_mb - space_used_mb) return max(0, limit_mb - space_used_mb)
async def get_space_used_bytes(self): async def get_space_used_bytes(self):
return await self.db.get_stored_blob_disk_usage() self._used_space_bytes = await self.db.get_stored_blob_disk_usage()
return self._used_space_bytes
async def get_space_used_mb(self): async def get_space_used_mb(self, cached=True):
space_used_bytes = await self.get_space_used_bytes() cached = cached and self._used_space_bytes is not None
space_used_bytes = self._used_space_bytes if cached else await self.get_space_used_bytes()
return {key: int(value/1024.0/1024.0) for key, value in space_used_bytes.items()} return {key: int(value/1024.0/1024.0) for key, value in space_used_bytes.items()}
async def clean(self): async def clean(self):

8
lbry/extras/daemon/components.py
View file

@ -393,6 +393,7 @@ class BackgroundDownloaderComponent(Component):
self.blob_manager: typing.Optional[BlobManager] = None self.blob_manager: typing.Optional[BlobManager] = None
self.background_downloader: typing.Optional[BackgroundDownloader] = None self.background_downloader: typing.Optional[BackgroundDownloader] = None
self.dht_node: typing.Optional[Node] = None self.dht_node: typing.Optional[Node] = None
self.space_available: typing.Optional[int] = None
@property @property
def is_busy(self): def is_busy(self):
@ -404,12 +405,13 @@ class BackgroundDownloaderComponent(Component):
async def get_status(self): async def get_status(self):
return {'running': self.task is not None and not self.task.done(), return {'running': self.task is not None and not self.task.done(),
'available_free_space_mb': await self.space_manager.get_free_space_mb(True), 'available_free_space_mb': self.space_available,
'ongoing_download': self.is_busy} 'ongoing_download': self.is_busy}
async def loop(self): async def loop(self):
while True: while True:
if not self.is_busy and await self.space_manager.get_free_space_mb(True) > 10: self.space_available = await self.space_manager.get_free_space_mb(True)
if not self.is_busy and self.space_available > 10:
blob_hash = next((key.hex() for key in self.dht_node.stored_blob_hashes if blob_hash = next((key.hex() for key in self.dht_node.stored_blob_hashes if
key.hex() not in self.blob_manager.completed_blob_hashes), None) key.hex() not in self.blob_manager.completed_blob_hashes), None)
if blob_hash: if blob_hash:
@ -447,7 +449,7 @@ class DiskSpaceComponent(Component):
async def get_status(self): async def get_status(self):
if self.disk_space_manager: if self.disk_space_manager:
space_used = await self.disk_space_manager.get_space_used_mb() space_used = await self.disk_space_manager.get_space_used_mb(cached=True)
return { return {
'total_used_mb': space_used['total'], 'total_used_mb': space_used['total'],
'published_blobs_storage_used_mb': space_used['private_storage'], 'published_blobs_storage_used_mb': space_used['private_storage'],

2
tests/integration/datanetwork/test_file_commands.py
View file

@ -555,6 +555,7 @@ class DiskSpaceManagement(CommandTestCase):
await self.daemon.storage.update_blob_ownership(sd_hash1, False) await self.daemon.storage.update_blob_ownership(sd_hash1, False)
await self.daemon.storage.update_blob_ownership(sd_hash3, False) await self.daemon.storage.update_blob_ownership(sd_hash3, False)
await self.daemon.storage.update_blob_ownership(sd_hash4, False) await self.daemon.storage.update_blob_ownership(sd_hash4, False)
await self.blob_clean() # just to refresh caches, has no effect
self.assertEqual(7, (await self.status())['disk_space']['content_blobs_storage_used_mb']) self.assertEqual(7, (await self.status())['disk_space']['content_blobs_storage_used_mb'])
self.assertEqual(10, (await self.status())['disk_space']['total_used_mb']) self.assertEqual(10, (await self.status())['disk_space']['total_used_mb'])
@ -563,6 +564,7 @@ class DiskSpaceManagement(CommandTestCase):
await self.blob_clean() await self.blob_clean()
self.assertEqual(10, (await self.status())['disk_space']['total_used_mb']) self.assertEqual(10, (await self.status())['disk_space']['total_used_mb'])
self.assertEqual(7, (await self.status())['disk_space']['content_blobs_storage_used_mb'])
self.assertEqual(3, (await self.status())['disk_space']['published_blobs_storage_used_mb']) self.assertEqual(3, (await self.status())['disk_space']['published_blobs_storage_used_mb'])
self.assertEqual(blobs1 | blobs2 | blobs3 | blobs4, set(await self.blob_list())) self.assertEqual(blobs1 | blobs2 | blobs3 | blobs4, set(await self.blob_list()))