improve disk space manager status, include more info and unify space queries

lbry/blob/disk_space_manager.py

@@ -17,20 +17,27 @@ class DiskSpaceManager:
 
     async def get_free_space_mb(self, is_network_blob=False):
         limit_mb = self.config.network_storage_limit if is_network_blob else self.config.blob_storage_limit
-        return max(0, limit_mb - (await self.get_space_used_mb(is_network_blob)))
+        space_used_mb = await self.get_space_used_mb()
+        space_used_mb = space_used_mb['network_storage'] if is_network_blob else space_used_mb['content_storage']
+        return max(0, limit_mb - space_used_mb)
 
-    async def get_space_used_bytes(self, is_network_blob=False):
+    async def get_space_used_bytes(self):
-        return await self.db.get_stored_blob_disk_usage(is_network_blob=is_network_blob)
+        return await self.db.get_stored_blob_disk_usage()
 
-    async def get_space_used_mb(self, is_network_blob=False):
+    async def get_space_used_mb(self):
-        return int(await self.get_space_used_bytes(is_network_blob)/1024.0/1024.0)
+        space_used_bytes = await self.get_space_used_bytes()
+        return {key: int(value/1024.0/1024.0) for key, value in space_used_bytes.items()}
 
     async def clean(self):
         await self._clean(False)
         await self._clean(True)
 
     async def _clean(self, is_network_blob=False):
-        space_used_bytes = await self.get_space_used_bytes(is_network_blob)
+        space_used_bytes = await self.get_space_used_bytes()
+        if is_network_blob:
+            space_used_bytes = space_used_bytes['network_storage']
+        else:
+            space_used_bytes = space_used_bytes['content_storage'] + space_used_bytes['private_storage']
         storage_limit_mb = self.config.network_storage_limit if is_network_blob else self.config.blob_storage_limit
         storage_limit = storage_limit_mb*1024*1024 if storage_limit_mb else None
         if self.analytics:

lbry/extras/daemon/components.py

@@ -403,7 +403,7 @@ class BackgroundDownloaderComponent(Component):
 
     async def get_status(self):
         return {'running': self.task is not None and not self.task.done(),
-                'available_free_space': await self.space_manager.get_free_space_mb(True),
+                'available_free_space_mb': await self.space_manager.get_free_space_mb(True),
                 'ongoing_download': self.is_busy}
 
     async def loop(self):
@@ -445,9 +445,12 @@ class DiskSpaceComponent(Component):
 
     async def get_status(self):
         if self.disk_space_manager:
+            space_used = await self.disk_space_manager.get_space_used_mb()
             return {
-                'space_used': str(await self.disk_space_manager.get_space_used_mb()),
+                'total_used_mb': space_used['total'],
-                'network_seeding_space_used': str(await self.disk_space_manager.get_space_used_mb(True)),
+                'published_blobs_storage_used_mb': space_used['private_storage'],
+                'content_blobs_storage_used_mb': space_used['content_storage'],
+                'seed_blobs_storage_used_mb': space_used['network_storage'],
                 'running': self.disk_space_manager.running,
             }
         return {'space_used': '0', 'network_seeding_space_used': '0', 'running': False}

lbry/extras/daemon/storage.py

@@ -458,24 +458,34 @@ class SQLiteStorage(SQLiteMixin):
             "where blob.is_mine=? order by blob.added_on asc",
             (is_mine,)
         )
-        stream_blobs = await self.db.execute_fetchall(
+        content_blobs = await self.db.execute_fetchall(
             "select blob.blob_hash, blob.blob_length, blob.added_on "
             "from blob join stream_blob using (blob_hash) cross join stream using (stream_hash)"
             "cross join file using (stream_hash) where blob.is_mine=? order by blob.added_on asc",
             (is_mine,)
         )
-        return stream_blobs + sd_blobs
+        return content_blobs + sd_blobs
 
-    async def get_stored_blob_disk_usage(self, is_mine: Optional[bool] = None, is_network_blob: bool = False):
+    async def get_stored_blob_disk_usage(self):
-        sql = "select coalesce(sum(blob_length), 0) "
+        total, network_size, content_size, private_size = await self.db.execute_fetchone("""
-        if is_network_blob:
+        select coalesce(sum(blob_length), 0) as total,
-            sql += "from blob left join stream_blob using (blob_hash) where stream_blob.stream_hash is null"
+               coalesce(sum(case when
-        else:
+                   stream_blob.stream_hash is null
-            sql += "from blob join stream_blob using (blob_hash)"
+               then blob_length else 0 end), 0) as network_storage,
-        if is_mine is not None:
+               coalesce(sum(case when
-            sql += f'{(" and " if is_network_blob else " where ")} is_mine=?'
+                   stream_blob.blob_hash is not null and is_mine=0
-        args = (1 if is_mine else 0,) if is_mine is not None else ()
+               then blob_length else 0 end), 0) as content_storage,
-        return (await self.db.execute_fetchone(sql, args))[0]
+               coalesce(sum(case when
+                   is_mine=1
+               then blob_length else 0 end), 0) as private_storage
+        from blob left join stream_blob using (blob_hash)
+        """)
+        return {
+            'network_storage': network_size,
+            'content_storage': content_size,
+            'private_storage': private_size,
+            'total': total
+        }
 
     async def update_blob_ownership(self, sd_hash, is_mine: bool):
         is_mine = 1 if is_mine else 0

tests/integration/datanetwork/test_file_commands.py

@@ -537,7 +537,7 @@ class DiskSpaceManagement(CommandTestCase):
     async def test_file_management(self):
         status = await self.status()
         self.assertIn('disk_space', status)
-        self.assertEqual('0', status['disk_space']['space_used'])
+        self.assertEqual(0, status['disk_space']['total_used_mb'])
         self.assertEqual(True, status['disk_space']['running'])
         sd_hash1, blobs1 = await self.get_referenced_blobs(
             await self.stream_create('foo1', '0.01', data=('0' * 2 * 1024 * 1024).encode())
@@ -556,18 +556,22 @@ class DiskSpaceManagement(CommandTestCase):
         await self.daemon.storage.update_blob_ownership(sd_hash3, False)
         await self.daemon.storage.update_blob_ownership(sd_hash4, False)
 
-        self.assertEqual('10', (await self.status())['disk_space']['space_used'])
+        self.assertEqual(7, (await self.status())['disk_space']['content_blobs_storage_used_mb'])
+        self.assertEqual(10, (await self.status())['disk_space']['total_used_mb'])
         self.assertEqual(blobs1 | blobs2 | blobs3 | blobs4, set(await self.blob_list()))
 
         await self.blob_clean()
 
-        self.assertEqual('10', (await self.status())['disk_space']['space_used'])
+        self.assertEqual(10, (await self.status())['disk_space']['total_used_mb'])
+        self.assertEqual(3, (await self.status())['disk_space']['published_blobs_storage_used_mb'])
         self.assertEqual(blobs1 | blobs2 | blobs3 | blobs4, set(await self.blob_list()))
 
         self.daemon.conf.blob_storage_limit = 6
         await self.blob_clean()
 
-        self.assertEqual('5', (await self.status())['disk_space']['space_used'])
+        self.assertEqual(5, (await self.status())['disk_space']['total_used_mb'])
+        self.assertEqual(2, (await self.status())['disk_space']['content_blobs_storage_used_mb'])
+        self.assertEqual(3, (await self.status())['disk_space']['published_blobs_storage_used_mb'])
         blobs = set(await self.blob_list())
         self.assertFalse(blobs1.issubset(blobs))
         self.assertTrue(blobs2.issubset(blobs))
@@ -610,27 +614,27 @@ class TestBackgroundDownloaderComponent(CommandTestCase):
         content1 = content1['outputs'][0]['value']['source']['sd_hash']
         content2 = await self.stream_create('content2', '0.01', data=bytes([0] * 16 * 1024 * 1024))
         content2 = content2['outputs'][0]['value']['source']['sd_hash']
-        self.assertEqual('48', (await self.status())['disk_space']['space_used'])
+        self.assertEqual(48, (await self.status())['disk_space']['published_blobs_storage_used_mb'])
+        self.assertEqual(0, (await self.status())['disk_space']['content_blobs_storage_used_mb'])
 
         background_downloader = BackgroundDownloader(self.daemon.conf, self.daemon.storage, self.daemon.blob_manager)
         await self.clear()
-        self.assertEqual('0', (await self.status())['disk_space']['space_used'])
+        self.assertEqual(0, (await self.status())['disk_space']['total_used_mb'])
-        self.assertEqual('0', (await self.status())['disk_space']['network_seeding_space_used'])
         await background_downloader.download_blobs(content1)
         await self.assertBlobs(content1)
-        self.assertEqual('0', (await self.status())['disk_space']['space_used'])
+        self.assertEqual(0, (await self.status())['disk_space']['content_blobs_storage_used_mb'])
-        self.assertEqual('32', (await self.status())['disk_space']['network_seeding_space_used'])
+        self.assertEqual(32, (await self.status())['disk_space']['seed_blobs_storage_used_mb'])
         await background_downloader.download_blobs(content2)
         await self.assertBlobs(content1, content2)
-        self.assertEqual('0', (await self.status())['disk_space']['space_used'])
+        self.assertEqual(0, (await self.status())['disk_space']['content_blobs_storage_used_mb'])
-        self.assertEqual('48', (await self.status())['disk_space']['network_seeding_space_used'])
+        self.assertEqual(48, (await self.status())['disk_space']['seed_blobs_storage_used_mb'])
         await self.clear()
         await background_downloader.download_blobs(content2)
         await self.assertBlobs(content2)
-        self.assertEqual('0', (await self.status())['disk_space']['space_used'])
+        self.assertEqual(0, (await self.status())['disk_space']['content_blobs_storage_used_mb'])
-        self.assertEqual('16', (await self.status())['disk_space']['network_seeding_space_used'])
+        self.assertEqual(16, (await self.status())['disk_space']['seed_blobs_storage_used_mb'])
         self.daemon.conf.network_storage_limit = 100
-        self.assertEqual(84, (await self.status())['background_downloader']['available_free_space'])
+        self.assertEqual(84, (await self.status())['background_downloader']['available_free_space_mb'])
 
         # tests that an attempt to download something that isn't a sd blob will download the single blob and stop
         blobs = await self.get_blobs_from_sd_blob(self.reflector.blob_manager.get_blob(content1))