From 95703b3af53ea291e03004528074b52f73501af0 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 22 Oct 2021 00:32:04 -0300 Subject: [PATCH] separated network seeding space metrics --- lbry/blob/disk_space_manager.py | 8 ++++---- lbry/extras/daemon/components.py | 3 ++- lbry/extras/daemon/storage.py | 11 +++++++---- tests/integration/datanetwork/test_file_commands.py | 8 ++++++-- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/lbry/blob/disk_space_manager.py b/lbry/blob/disk_space_manager.py index 96053eca0..a0fb501ea 100644 --- a/lbry/blob/disk_space_manager.py +++ b/lbry/blob/disk_space_manager.py @@ -15,11 +15,11 @@ class DiskSpaceManager: self.task = None self.analytics = analytics - async def get_space_used_bytes(self): - return await self.db.get_stored_blob_disk_usage() + async def get_space_used_bytes(self, is_network_blob=False): + return await self.db.get_stored_blob_disk_usage(is_orphan_blob=is_network_blob) - async def get_space_used_mb(self): - return int(await self.get_space_used_bytes()/1024.0/1024.0) + async def get_space_used_mb(self, is_network_blob=False): + return int(await self.get_space_used_bytes(is_network_blob)/1024.0/1024.0) async def clean(self): space_used_bytes = await self.get_space_used_bytes() diff --git a/lbry/extras/daemon/components.py b/lbry/extras/daemon/components.py index 5371e74b5..ba8a35121 100644 --- a/lbry/extras/daemon/components.py +++ b/lbry/extras/daemon/components.py @@ -450,9 +450,10 @@ class DiskSpaceComponent(Component): if self.disk_space_manager: return { 'space_used': str(await self.disk_space_manager.get_space_used_mb()), + 'network_seeding_space_used': str(await self.disk_space_manager.get_space_used_mb(True)), 'running': self.disk_space_manager.running, } - return {'space_used': '0', 'running': False} + return {'space_used': '0', 'network_seeding_space_used': '0', 'running': False} async def start(self): db = self.component_manager.get_component(DATABASE_COMPONENT) diff --git a/lbry/extras/daemon/storage.py b/lbry/extras/daemon/storage.py index 832dab1c8..43ea9459b 100644 --- a/lbry/extras/daemon/storage.py +++ b/lbry/extras/daemon/storage.py @@ -453,11 +453,14 @@ class SQLiteStorage(SQLiteMixin): ) return normal_blobs + sd_blobs - async def get_stored_blob_disk_usage(self, is_mine: Optional[bool] = None): - if is_mine is None: - sql = "select coalesce(sum(blob_length), 0) from blob join stream_blob using (blob_hash)" + async def get_stored_blob_disk_usage(self, is_mine: Optional[bool] = None, is_orphan_blob: bool = False): + sql = "select coalesce(sum(blob_length), 0) " + if is_orphan_blob: + sql += "from blob left join stream_blob using (blob_hash) where stream_blob.stream_hash is null" else: - sql = "select coalesce(sum(blob_length), 0) from blob join stream_blob using (blob_hash) where is_mine=?" + sql += "from blob join stream_blob using (blob_hash)" + if is_mine is not None: + sql += f'{(" and " if is_orphan_blob else " where ")} is_mine=?' args = (1 if is_mine else 0,) if is_mine is not None else () return (await self.db.execute_fetchone(sql, args))[0] diff --git a/tests/integration/datanetwork/test_file_commands.py b/tests/integration/datanetwork/test_file_commands.py index c4766983c..2f06254ac 100644 --- a/tests/integration/datanetwork/test_file_commands.py +++ b/tests/integration/datanetwork/test_file_commands.py @@ -602,25 +602,29 @@ class TestProactiveDownloaderComponent(CommandTestCase): self.assertEqual(0, len((await self.daemon.jsonrpc_blob_list())['items'])) async def test_ensure_download(self): - content1 = await self.stream_create('content1', '0.01', data=bytes([0] * (2 << 24))) + content1 = await self.stream_create('content1', '0.01', data=bytes([0] * 32 * 1024 * 1024)) content1 = content1['outputs'][0]['value']['source']['sd_hash'] - content2 = await self.stream_create('content2', '0.01', data=bytes([0] * (2 << 23))) + content2 = await self.stream_create('content2', '0.01', data=bytes([0] * 16 * 1024 * 1024)) content2 = content2['outputs'][0]['value']['source']['sd_hash'] self.assertEqual('48', (await self.status())['disk_space']['space_used']) proactive_downloader = self.daemon.component_manager.get_component(BACKGROUND_DOWNLOADER_COMPONENT) await self.clear() self.assertEqual('0', (await self.status())['disk_space']['space_used']) + self.assertEqual('0', (await self.status())['disk_space']['network_seeding_space_used']) await proactive_downloader.download_blobs(content1) await self.assertBlobs(content1) self.assertEqual('0', (await self.status())['disk_space']['space_used']) + self.assertEqual('32', (await self.status())['disk_space']['network_seeding_space_used']) await proactive_downloader.download_blobs(content2) await self.assertBlobs(content1, content2) self.assertEqual('0', (await self.status())['disk_space']['space_used']) + self.assertEqual('48', (await self.status())['disk_space']['network_seeding_space_used']) await self.clear() await proactive_downloader.download_blobs(content2) await self.assertBlobs(content2) self.assertEqual('0', (await self.status())['disk_space']['space_used']) + self.assertEqual('16', (await self.status())['disk_space']['network_seeding_space_used']) # tests that an attempt to download something that isn't a sd blob will download the single blob and stop blobs = await self.get_blobs_from_sd_blob(self.reflector.blob_manager.get_blob(content1))