diff --git a/lbry/blob/disk_space_manager.py b/lbry/blob/disk_space_manager.py index d6af70239..ba1a20a3d 100644 --- a/lbry/blob/disk_space_manager.py +++ b/lbry/blob/disk_space_manager.py @@ -17,20 +17,27 @@ class DiskSpaceManager: async def get_free_space_mb(self, is_network_blob=False): limit_mb = self.config.network_storage_limit if is_network_blob else self.config.blob_storage_limit - return max(0, limit_mb - (await self.get_space_used_mb(is_network_blob))) + space_used_mb = await self.get_space_used_mb() + space_used_mb = space_used_mb['network_storage'] if is_network_blob else space_used_mb['content_storage'] + return max(0, limit_mb - space_used_mb) - async def get_space_used_bytes(self, is_network_blob=False): - return await self.db.get_stored_blob_disk_usage(is_network_blob=is_network_blob) + async def get_space_used_bytes(self): + return await self.db.get_stored_blob_disk_usage() - async def get_space_used_mb(self, is_network_blob=False): - return int(await self.get_space_used_bytes(is_network_blob)/1024.0/1024.0) + async def get_space_used_mb(self): + space_used_bytes = await self.get_space_used_bytes() + return {key: int(value/1024.0/1024.0) for key, value in space_used_bytes.items()} async def clean(self): await self._clean(False) await self._clean(True) async def _clean(self, is_network_blob=False): - space_used_bytes = await self.get_space_used_bytes(is_network_blob) + space_used_bytes = await self.get_space_used_bytes() + if is_network_blob: + space_used_bytes = space_used_bytes['network_storage'] + else: + space_used_bytes = space_used_bytes['content_storage'] + space_used_bytes['private_storage'] storage_limit_mb = self.config.network_storage_limit if is_network_blob else self.config.blob_storage_limit storage_limit = storage_limit_mb*1024*1024 if storage_limit_mb else None if self.analytics: diff --git a/lbry/extras/daemon/components.py b/lbry/extras/daemon/components.py index 61aaa9c51..9ab9388a9 100644 --- a/lbry/extras/daemon/components.py +++ b/lbry/extras/daemon/components.py @@ -403,7 +403,7 @@ class BackgroundDownloaderComponent(Component): async def get_status(self): return {'running': self.task is not None and not self.task.done(), - 'available_free_space': await self.space_manager.get_free_space_mb(True), + 'available_free_space_mb': await self.space_manager.get_free_space_mb(True), 'ongoing_download': self.is_busy} async def loop(self): @@ -445,9 +445,12 @@ class DiskSpaceComponent(Component): async def get_status(self): if self.disk_space_manager: + space_used = await self.disk_space_manager.get_space_used_mb() return { - 'space_used': str(await self.disk_space_manager.get_space_used_mb()), - 'network_seeding_space_used': str(await self.disk_space_manager.get_space_used_mb(True)), + 'total_used_mb': space_used['total'], + 'published_blobs_storage_used_mb': space_used['private_storage'], + 'content_blobs_storage_used_mb': space_used['content_storage'], + 'seed_blobs_storage_used_mb': space_used['network_storage'], 'running': self.disk_space_manager.running, } return {'space_used': '0', 'network_seeding_space_used': '0', 'running': False} diff --git a/lbry/extras/daemon/storage.py b/lbry/extras/daemon/storage.py index ee4ea2c93..efe791745 100644 --- a/lbry/extras/daemon/storage.py +++ b/lbry/extras/daemon/storage.py @@ -458,24 +458,34 @@ class SQLiteStorage(SQLiteMixin): "where blob.is_mine=? order by blob.added_on asc", (is_mine,) ) - stream_blobs = await self.db.execute_fetchall( + content_blobs = await self.db.execute_fetchall( "select blob.blob_hash, blob.blob_length, blob.added_on " "from blob join stream_blob using (blob_hash) cross join stream using (stream_hash)" "cross join file using (stream_hash) where blob.is_mine=? order by blob.added_on asc", (is_mine,) ) - return stream_blobs + sd_blobs + return content_blobs + sd_blobs - async def get_stored_blob_disk_usage(self, is_mine: Optional[bool] = None, is_network_blob: bool = False): - sql = "select coalesce(sum(blob_length), 0) " - if is_network_blob: - sql += "from blob left join stream_blob using (blob_hash) where stream_blob.stream_hash is null" - else: - sql += "from blob join stream_blob using (blob_hash)" - if is_mine is not None: - sql += f'{(" and " if is_network_blob else " where ")} is_mine=?' - args = (1 if is_mine else 0,) if is_mine is not None else () - return (await self.db.execute_fetchone(sql, args))[0] + async def get_stored_blob_disk_usage(self): + total, network_size, content_size, private_size = await self.db.execute_fetchone(""" + select coalesce(sum(blob_length), 0) as total, + coalesce(sum(case when + stream_blob.stream_hash is null + then blob_length else 0 end), 0) as network_storage, + coalesce(sum(case when + stream_blob.blob_hash is not null and is_mine=0 + then blob_length else 0 end), 0) as content_storage, + coalesce(sum(case when + is_mine=1 + then blob_length else 0 end), 0) as private_storage + from blob left join stream_blob using (blob_hash) + """) + return { + 'network_storage': network_size, + 'content_storage': content_size, + 'private_storage': private_size, + 'total': total + } async def update_blob_ownership(self, sd_hash, is_mine: bool): is_mine = 1 if is_mine else 0 diff --git a/tests/integration/datanetwork/test_file_commands.py b/tests/integration/datanetwork/test_file_commands.py index 716aa61ea..23cb8f1ef 100644 --- a/tests/integration/datanetwork/test_file_commands.py +++ b/tests/integration/datanetwork/test_file_commands.py @@ -537,7 +537,7 @@ class DiskSpaceManagement(CommandTestCase): async def test_file_management(self): status = await self.status() self.assertIn('disk_space', status) - self.assertEqual('0', status['disk_space']['space_used']) + self.assertEqual(0, status['disk_space']['total_used_mb']) self.assertEqual(True, status['disk_space']['running']) sd_hash1, blobs1 = await self.get_referenced_blobs( await self.stream_create('foo1', '0.01', data=('0' * 2 * 1024 * 1024).encode()) @@ -556,18 +556,22 @@ class DiskSpaceManagement(CommandTestCase): await self.daemon.storage.update_blob_ownership(sd_hash3, False) await self.daemon.storage.update_blob_ownership(sd_hash4, False) - self.assertEqual('10', (await self.status())['disk_space']['space_used']) + self.assertEqual(7, (await self.status())['disk_space']['content_blobs_storage_used_mb']) + self.assertEqual(10, (await self.status())['disk_space']['total_used_mb']) self.assertEqual(blobs1 | blobs2 | blobs3 | blobs4, set(await self.blob_list())) await self.blob_clean() - self.assertEqual('10', (await self.status())['disk_space']['space_used']) + self.assertEqual(10, (await self.status())['disk_space']['total_used_mb']) + self.assertEqual(3, (await self.status())['disk_space']['published_blobs_storage_used_mb']) self.assertEqual(blobs1 | blobs2 | blobs3 | blobs4, set(await self.blob_list())) self.daemon.conf.blob_storage_limit = 6 await self.blob_clean() - self.assertEqual('5', (await self.status())['disk_space']['space_used']) + self.assertEqual(5, (await self.status())['disk_space']['total_used_mb']) + self.assertEqual(2, (await self.status())['disk_space']['content_blobs_storage_used_mb']) + self.assertEqual(3, (await self.status())['disk_space']['published_blobs_storage_used_mb']) blobs = set(await self.blob_list()) self.assertFalse(blobs1.issubset(blobs)) self.assertTrue(blobs2.issubset(blobs)) @@ -610,27 +614,27 @@ class TestBackgroundDownloaderComponent(CommandTestCase): content1 = content1['outputs'][0]['value']['source']['sd_hash'] content2 = await self.stream_create('content2', '0.01', data=bytes([0] * 16 * 1024 * 1024)) content2 = content2['outputs'][0]['value']['source']['sd_hash'] - self.assertEqual('48', (await self.status())['disk_space']['space_used']) + self.assertEqual(48, (await self.status())['disk_space']['published_blobs_storage_used_mb']) + self.assertEqual(0, (await self.status())['disk_space']['content_blobs_storage_used_mb']) background_downloader = BackgroundDownloader(self.daemon.conf, self.daemon.storage, self.daemon.blob_manager) await self.clear() - self.assertEqual('0', (await self.status())['disk_space']['space_used']) - self.assertEqual('0', (await self.status())['disk_space']['network_seeding_space_used']) + self.assertEqual(0, (await self.status())['disk_space']['total_used_mb']) await background_downloader.download_blobs(content1) await self.assertBlobs(content1) - self.assertEqual('0', (await self.status())['disk_space']['space_used']) - self.assertEqual('32', (await self.status())['disk_space']['network_seeding_space_used']) + self.assertEqual(0, (await self.status())['disk_space']['content_blobs_storage_used_mb']) + self.assertEqual(32, (await self.status())['disk_space']['seed_blobs_storage_used_mb']) await background_downloader.download_blobs(content2) await self.assertBlobs(content1, content2) - self.assertEqual('0', (await self.status())['disk_space']['space_used']) - self.assertEqual('48', (await self.status())['disk_space']['network_seeding_space_used']) + self.assertEqual(0, (await self.status())['disk_space']['content_blobs_storage_used_mb']) + self.assertEqual(48, (await self.status())['disk_space']['seed_blobs_storage_used_mb']) await self.clear() await background_downloader.download_blobs(content2) await self.assertBlobs(content2) - self.assertEqual('0', (await self.status())['disk_space']['space_used']) - self.assertEqual('16', (await self.status())['disk_space']['network_seeding_space_used']) + self.assertEqual(0, (await self.status())['disk_space']['content_blobs_storage_used_mb']) + self.assertEqual(16, (await self.status())['disk_space']['seed_blobs_storage_used_mb']) self.daemon.conf.network_storage_limit = 100 - self.assertEqual(84, (await self.status())['background_downloader']['available_free_space']) + self.assertEqual(84, (await self.status())['background_downloader']['available_free_space_mb']) # tests that an attempt to download something that isn't a sd blob will download the single blob and stop blobs = await self.get_blobs_from_sd_blob(self.reflector.blob_manager.get_blob(content1))