Madiator2011/lbry-sdk
1
0
Fork 0
forked from LBRYCommunity/lbry-sdk
Code Pull requests Activity

improve disk space manager status, include more info and unify space queries

Browse source
This commit is contained in:
Victor Shyba 2021-11-02 22:34:15 -03:00 committed by Jack Robison
parent d12a214c05
commit 5959b1be72
4 changed files with 59 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
lbry/blob/disk_space_manager.py
View file

@ -17,20 +17,27 @@ class DiskSpaceManager:
async def get_free_space_mb(self, is_network_blob=False):
limit_mb = self.config.network_storage_limit if is_network_blob else self.config.blob_storage_limit
return max(0, limit_mb - (await self.get_space_used_mb(is_network_blob)))
space_used_mb = await self.get_space_used_mb()
space_used_mb = space_used_mb['network_storage'] if is_network_blob else space_used_mb['content_storage']
return max(0, limit_mb - space_used_mb)
async def get_space_used_bytes(self, is_network_blob=False):
return await self.db.get_stored_blob_disk_usage(is_network_blob=is_network_blob)
async def get_space_used_bytes(self):
return await self.db.get_stored_blob_disk_usage()
async def get_space_used_mb(self, is_network_blob=False):
return int(await self.get_space_used_bytes(is_network_blob)/1024.0/1024.0)
async def get_space_used_mb(self):
space_used_bytes = await self.get_space_used_bytes()
return {key: int(value/1024.0/1024.0) for key, value in space_used_bytes.items()}
async def clean(self):
await self._clean(False)
await self._clean(True)
async def _clean(self, is_network_blob=False):
space_used_bytes = await self.get_space_used_bytes(is_network_blob)
space_used_bytes = await self.get_space_used_bytes()
if is_network_blob:
space_used_bytes = space_used_bytes['network_storage']
else:
space_used_bytes = space_used_bytes['content_storage'] + space_used_bytes['private_storage']
storage_limit_mb = self.config.network_storage_limit if is_network_blob else self.config.blob_storage_limit
storage_limit = storage_limit_mb*1024*1024 if storage_limit_mb else None
if self.analytics:

9
lbry/extras/daemon/components.py
View file

@ -403,7 +403,7 @@ class BackgroundDownloaderComponent(Component):
async def get_status(self):
return {'running': self.task is not None and not self.task.done(),
'available_free_space': await self.space_manager.get_free_space_mb(True),
'available_free_space_mb': await self.space_manager.get_free_space_mb(True),
'ongoing_download': self.is_busy}
async def loop(self):
@ -445,9 +445,12 @@ class DiskSpaceComponent(Component):
async def get_status(self):
if self.disk_space_manager:
space_used = await self.disk_space_manager.get_space_used_mb()
return {
'space_used': str(await self.disk_space_manager.get_space_used_mb()),
'network_seeding_space_used': str(await self.disk_space_manager.get_space_used_mb(True)),
'total_used_mb': space_used['total'],
'published_blobs_storage_used_mb': space_used['private_storage'],
'content_blobs_storage_used_mb': space_used['content_storage'],
'seed_blobs_storage_used_mb': space_used['network_storage'],
'running': self.disk_space_manager.running,
}
return {'space_used': '0', 'network_seeding_space_used': '0', 'running': False}

34
lbry/extras/daemon/storage.py
View file

@ -458,24 +458,34 @@ class SQLiteStorage(SQLiteMixin):
"where blob.is_mine=? order by blob.added_on asc",
(is_mine,)
)
stream_blobs = await self.db.execute_fetchall(
content_blobs = await self.db.execute_fetchall(
"select blob.blob_hash, blob.blob_length, blob.added_on "
"from blob join stream_blob using (blob_hash) cross join stream using (stream_hash)"
"cross join file using (stream_hash) where blob.is_mine=? order by blob.added_on asc",
(is_mine,)
)
return stream_blobs + sd_blobs
return content_blobs + sd_blobs
async def get_stored_blob_disk_usage(self, is_mine: Optional[bool] = None, is_network_blob: bool = False):
sql = "select coalesce(sum(blob_length), 0) "
if is_network_blob:
sql += "from blob left join stream_blob using (blob_hash) where stream_blob.stream_hash is null"
else:
sql += "from blob join stream_blob using (blob_hash)"
if is_mine is not None:
sql += f'{(" and " if is_network_blob else " where ")} is_mine=?'
args = (1 if is_mine else 0,) if is_mine is not None else ()
return (await self.db.execute_fetchone(sql, args))[0]
async def get_stored_blob_disk_usage(self):
total, network_size, content_size, private_size = await self.db.execute_fetchone("""
select coalesce(sum(blob_length), 0) as total,
coalesce(sum(case when
stream_blob.stream_hash is null
then blob_length else 0 end), 0) as network_storage,
coalesce(sum(case when
stream_blob.blob_hash is not null and is_mine=0
then blob_length else 0 end), 0) as content_storage,
coalesce(sum(case when
is_mine=1
then blob_length else 0 end), 0) as private_storage
from blob left join stream_blob using (blob_hash)
""")
return {
'network_storage': network_size,
'content_storage': content_size,
'private_storage': private_size,
'total': total
}
async def update_blob_ownership(self, sd_hash, is_mine: bool):
is_mine = 1 if is_mine else 0

32
tests/integration/datanetwork/test_file_commands.py
View file

@ -537,7 +537,7 @@ class DiskSpaceManagement(CommandTestCase):
async def test_file_management(self):
status = await self.status()
self.assertIn('disk_space', status)
self.assertEqual('0', status['disk_space']['space_used'])
self.assertEqual(0, status['disk_space']['total_used_mb'])
self.assertEqual(True, status['disk_space']['running'])
sd_hash1, blobs1 = await self.get_referenced_blobs(
await self.stream_create('foo1', '0.01', data=('0' * 2 * 1024 * 1024).encode())
@ -556,18 +556,22 @@ class DiskSpaceManagement(CommandTestCase):
await self.daemon.storage.update_blob_ownership(sd_hash3, False)
await self.daemon.storage.update_blob_ownership(sd_hash4, False)
self.assertEqual('10', (await self.status())['disk_space']['space_used'])
self.assertEqual(7, (await self.status())['disk_space']['content_blobs_storage_used_mb'])
self.assertEqual(10, (await self.status())['disk_space']['total_used_mb'])
self.assertEqual(blobs1 | blobs2 | blobs3 | blobs4, set(await self.blob_list()))
await self.blob_clean()
self.assertEqual('10', (await self.status())['disk_space']['space_used'])
self.assertEqual(10, (await self.status())['disk_space']['total_used_mb'])
self.assertEqual(3, (await self.status())['disk_space']['published_blobs_storage_used_mb'])
self.assertEqual(blobs1 | blobs2 | blobs3 | blobs4, set(await self.blob_list()))
self.daemon.conf.blob_storage_limit = 6
await self.blob_clean()
self.assertEqual('5', (await self.status())['disk_space']['space_used'])
self.assertEqual(5, (await self.status())['disk_space']['total_used_mb'])
self.assertEqual(2, (await self.status())['disk_space']['content_blobs_storage_used_mb'])
self.assertEqual(3, (await self.status())['disk_space']['published_blobs_storage_used_mb'])
blobs = set(await self.blob_list())
self.assertFalse(blobs1.issubset(blobs))
self.assertTrue(blobs2.issubset(blobs))
@ -610,27 +614,27 @@ class TestBackgroundDownloaderComponent(CommandTestCase):
content1 = content1['outputs'][0]['value']['source']['sd_hash']
content2 = await self.stream_create('content2', '0.01', data=bytes([0] * 16 * 1024 * 1024))
content2 = content2['outputs'][0]['value']['source']['sd_hash']
self.assertEqual('48', (await self.status())['disk_space']['space_used'])
self.assertEqual(48, (await self.status())['disk_space']['published_blobs_storage_used_mb'])
self.assertEqual(0, (await self.status())['disk_space']['content_blobs_storage_used_mb'])
background_downloader = BackgroundDownloader(self.daemon.conf, self.daemon.storage, self.daemon.blob_manager)
await self.clear()
self.assertEqual('0', (await self.status())['disk_space']['space_used'])
self.assertEqual('0', (await self.status())['disk_space']['network_seeding_space_used'])
self.assertEqual(0, (await self.status())['disk_space']['total_used_mb'])
await background_downloader.download_blobs(content1)
await self.assertBlobs(content1)
self.assertEqual('0', (await self.status())['disk_space']['space_used'])
self.assertEqual('32', (await self.status())['disk_space']['network_seeding_space_used'])
self.assertEqual(0, (await self.status())['disk_space']['content_blobs_storage_used_mb'])
self.assertEqual(32, (await self.status())['disk_space']['seed_blobs_storage_used_mb'])
await background_downloader.download_blobs(content2)
await self.assertBlobs(content1, content2)
self.assertEqual('0', (await self.status())['disk_space']['space_used'])
self.assertEqual('48', (await self.status())['disk_space']['network_seeding_space_used'])
self.assertEqual(0, (await self.status())['disk_space']['content_blobs_storage_used_mb'])
self.assertEqual(48, (await self.status())['disk_space']['seed_blobs_storage_used_mb'])
await self.clear()
await background_downloader.download_blobs(content2)
await self.assertBlobs(content2)
self.assertEqual('0', (await self.status())['disk_space']['space_used'])
self.assertEqual('16', (await self.status())['disk_space']['network_seeding_space_used'])
self.assertEqual(0, (await self.status())['disk_space']['content_blobs_storage_used_mb'])
self.assertEqual(16, (await self.status())['disk_space']['seed_blobs_storage_used_mb'])
self.daemon.conf.network_storage_limit = 100
self.assertEqual(84, (await self.status())['background_downloader']['available_free_space'])
self.assertEqual(84, (await self.status())['background_downloader']['available_free_space_mb'])
# tests that an attempt to download something that isn't a sd blob will download the single blob and stop
blobs = await self.get_blobs_from_sd_blob(self.reflector.blob_manager.get_blob(content1))