From 5ab003534897c36bbcfeef0fcdf390957b5d276f Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Fri, 30 Jul 2021 10:21:52 -0400 Subject: [PATCH 01/10] run tests on windows and mac --- .github/workflows/main.yml | 16 ++++++++++++---- tests/unit/test_conf.py | 21 +++++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4105973f6..024bfe438 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,8 +31,8 @@ jobs: matrix: os: - ubuntu-latest - #- macos-latest - #- windows-latest + - macos-latest + - windows-latest runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 @@ -49,10 +49,18 @@ jobs: key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} restore-keys: ${{ runner.os }}-pip- - run: pip install --user --upgrade pip wheel - - run: pip install -e .[torrent,test] - - env: + - if: startsWith(runner.os, 'linux') + run: pip install -e .[torrent,test] + - if: startsWith(runner.os, 'linux') + env: HOME: /tmp run: make test-unit-coverage + - if: !startsWith(runner.os, 'linux') + run: pip install -e .[test] + - if: !startsWith(runner.os, 'linux') + env: + HOME: /tmp + run: python -m unittest tests/unit/test_conf.py tests-integration: name: "tests / integration" diff --git a/tests/unit/test_conf.py b/tests/unit/test_conf.py index e241dea5f..645406462 100644 --- a/tests/unit/test_conf.py +++ b/tests/unit/test_conf.py @@ -21,6 +21,27 @@ class TestConfig(BaseConfig): class ConfigurationTests(unittest.TestCase): + @unittest.skipIf('darwin' not in sys.platform, 'skipping mac only test') + def test_mac_defaults(self): + c = Config() + self.assertEqual(c.data_dir, os.path.expanduser("~/Library/Application Support/LBRY")) + self.assertEqual(c.wallet_dir, os.path.expanduser('~/.lbryum')) + self.assertEqual(c.download_dir, os.path.expanduser('~/Downloads')) + self.assertEqual(c.config, os.path.join(c.data_dir, 'daemon_settings.yml')) + self.assertEqual(c.api_connection_url, 'http://localhost:5279/lbryapi') + self.assertEqual(c.log_file_path, os.path.join(c.data_dir, 'lbrynet.log')) + + @unittest.skipIf('win32' not in sys.platform, 'skipping windows only test') + def test_windows_defaults(self): + c = Config() + prefix = os.path.join(r"C:\Users", os.getlogin(), r"AppData\Local\lbry") + self.assertEqual(c.data_dir, os.path.join(prefix, 'lbrynet')) + self.assertEqual(c.wallet_dir, os.path.join(prefix, 'lbryum')) + self.assertEqual(c.download_dir, os.path.join(r"C:\Users", os.getlogin(), "Downloads")) + self.assertEqual(c.config, os.path.join(c.data_dir, 'daemon_settings.yml')) + self.assertEqual(c.api_connection_url, 'http://localhost:5279/lbryapi') + self.assertEqual(c.log_file_path, os.path.join(c.data_dir, 'lbrynet.log')) + @unittest.skipIf('linux' not in sys.platform, 'skipping linux only test') def test_linux_defaults(self): c = Config() From e96875a4254e92cdb03b3d3b3e32b15262025191 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Fri, 30 Jul 2021 10:43:24 -0400 Subject: [PATCH 02/10] workflow syntax --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 024bfe438..2a8103924 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -55,9 +55,9 @@ jobs: env: HOME: /tmp run: make test-unit-coverage - - if: !startsWith(runner.os, 'linux') + - if: startsWith(runner.os, 'linux') != true run: pip install -e .[test] - - if: !startsWith(runner.os, 'linux') + - if: startsWith(runner.os, 'linux') != true env: HOME: /tmp run: python -m unittest tests/unit/test_conf.py From 4e8d10cb442723b569170e8595fba197cfad443d Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Fri, 6 Aug 2021 10:44:57 -0400 Subject: [PATCH 03/10] disk space manager and status API --- lbry/blob/disk_space_manager.py | 21 ++++++++++++++++ lbry/conf.py | 1 + lbry/extras/daemon/components.py | 25 +++++++++++++++++++ lbry/extras/daemon/daemon.py | 2 +- .../datanetwork/test_file_commands.py | 12 +++++++++ tests/unit/blob/test_disk_space_manager.py | 23 +++++++++++++++++ 6 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 lbry/blob/disk_space_manager.py create mode 100644 tests/unit/blob/test_disk_space_manager.py diff --git a/lbry/blob/disk_space_manager.py b/lbry/blob/disk_space_manager.py new file mode 100644 index 000000000..aadb78b79 --- /dev/null +++ b/lbry/blob/disk_space_manager.py @@ -0,0 +1,21 @@ +import os + + +class DiskSpaceManager: + + def __init__(self, config): + self.config = config + + @property + def space_used_bytes(self): + used = 0 + data_dir = self.config.data_dir + for item in os.listdir(data_dir): + blob_path = os.path.join(data_dir, item) + if os.path.isfile(blob_path): + used += os.path.getsize(blob_path) + return used + + @property + def space_used_mb(self): + return int(self.space_used_bytes/1024.0/1024.0) diff --git a/lbry/conf.py b/lbry/conf.py index 4250fe579..480be87ab 100644 --- a/lbry/conf.py +++ b/lbry/conf.py @@ -634,6 +634,7 @@ class Config(CLIConfig): # blob announcement and download save_blobs = Toggle("Save encrypted blob files for hosting, otherwise download blobs to memory only.", True) + blob_storage_limit = Integer("Disk space in MB to be allocated for blob storage. 0 = no limit", 0) blob_lru_cache_size = Integer( "LRU cache size for decrypted downloaded blobs used to minimize re-downloading the same blobs when " "replying to a range request. Set to 0 to disable.", 32 diff --git a/lbry/extras/daemon/components.py b/lbry/extras/daemon/components.py index 822022f32..fbaa598cb 100644 --- a/lbry/extras/daemon/components.py +++ b/lbry/extras/daemon/components.py @@ -15,6 +15,7 @@ from lbry.dht.node import Node from lbry.dht.peer import is_valid_public_ipv4 from lbry.dht.blob_announcer import BlobAnnouncer from lbry.blob.blob_manager import BlobManager +from lbry.blob.disk_space_manager import DiskSpaceManager from lbry.blob_exchange.server import BlobServer from lbry.stream.stream_manager import StreamManager from lbry.file.file_manager import FileManager @@ -40,6 +41,7 @@ WALLET_SERVER_PAYMENTS_COMPONENT = "wallet_server_payments" DHT_COMPONENT = "dht" HASH_ANNOUNCER_COMPONENT = "hash_announcer" FILE_MANAGER_COMPONENT = "file_manager" +DISK_SPACE_COMPONENT = "disk_space" PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server" UPNP_COMPONENT = "upnp" EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager" @@ -375,6 +377,29 @@ class FileManagerComponent(Component): self.file_manager.stop() +class DiskSpaceComponent(Component): + component_name = DISK_SPACE_COMPONENT + + def __init__(self, component_manager): + super().__init__(component_manager) + self.disk_space_manager = DiskSpaceManager(self.conf) + + @property + def component(self) -> typing.Optional[DiskSpaceManager]: + return self.disk_space_manager + + async def get_status(self): + return { + 'used': str(self.disk_space_manager.space_used_mb), + } + + async def start(self): + pass + + async def stop(self): + pass + + class TorrentComponent(Component): component_name = LIBTORRENT_COMPONENT diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index 4443c209f..bf33c0541 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -41,7 +41,7 @@ from lbry.error import ( from lbry.extras import system_info from lbry.extras.daemon import analytics from lbry.extras.daemon.components import WALLET_COMPONENT, DATABASE_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT -from lbry.extras.daemon.components import FILE_MANAGER_COMPONENT +from lbry.extras.daemon.components import FILE_MANAGER_COMPONENT, DISK_SPACE_COMPONENT from lbry.extras.daemon.components import EXCHANGE_RATE_MANAGER_COMPONENT, UPNP_COMPONENT from lbry.extras.daemon.componentmanager import RequiredCondition from lbry.extras.daemon.componentmanager import ComponentManager diff --git a/tests/integration/datanetwork/test_file_commands.py b/tests/integration/datanetwork/test_file_commands.py index ee65c755b..7d472f314 100644 --- a/tests/integration/datanetwork/test_file_commands.py +++ b/tests/integration/datanetwork/test_file_commands.py @@ -511,3 +511,15 @@ class FileCommands(CommandTestCase): await tx.sign([self.account]) await self.broadcast(tx) await self.confirm_tx(tx.id) + + +class DiskSpaceManagement(CommandTestCase): + + async def test_file_management(self): + status = await self.daemon.jsonrpc_status() + self.assertIn('disk_space', status) + self.assertEqual(status['disk_space']['used'], '0') + await self.stream_create('foo', '0.01', data=('0' * 3 * 1024 * 1024).encode()) + status = await self.daemon.jsonrpc_status() + self.assertIn('disk_space', status) + self.assertEqual(status['disk_space']['used'], '3') diff --git a/tests/unit/blob/test_disk_space_manager.py b/tests/unit/blob/test_disk_space_manager.py new file mode 100644 index 000000000..75aa07c4c --- /dev/null +++ b/tests/unit/blob/test_disk_space_manager.py @@ -0,0 +1,23 @@ +import os +import unittest +import tempfile + +import lbry.wallet +from lbry.conf import Config +from lbry.blob.disk_space_manager import DiskSpaceManager + + +class ConfigurationTests(unittest.TestCase): + + def test_space_calculation(self): + with tempfile.TemporaryDirectory() as temp_dir: + config = Config( + data_dir=temp_dir, + wallet_dir=temp_dir, + config=os.path.join(temp_dir, 'settings.yml') + ) + dsm = DiskSpaceManager(config) + self.assertEqual(0, dsm.space_used_mb) + with open(os.path.join(config.data_dir, '3mb-file'), 'w') as blob: + blob.write('0' * 3 * 1024 * 1024) + self.assertEqual(3, dsm.space_used_mb) From b4c3307cdf47958a3a249f070e5f540c1273e69c Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Fri, 6 Aug 2021 10:52:47 -0400 Subject: [PATCH 04/10] fixed tests --- lbry/blob/disk_space_manager.py | 2 +- tests/integration/datanetwork/test_file_commands.py | 3 ++- tests/unit/blob/test_disk_space_manager.py | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lbry/blob/disk_space_manager.py b/lbry/blob/disk_space_manager.py index aadb78b79..74463e4aa 100644 --- a/lbry/blob/disk_space_manager.py +++ b/lbry/blob/disk_space_manager.py @@ -9,7 +9,7 @@ class DiskSpaceManager: @property def space_used_bytes(self): used = 0 - data_dir = self.config.data_dir + data_dir = os.path.join(self.config.data_dir, 'blobfiles') for item in os.listdir(data_dir): blob_path = os.path.join(data_dir, item) if os.path.isfile(blob_path): diff --git a/tests/integration/datanetwork/test_file_commands.py b/tests/integration/datanetwork/test_file_commands.py index 7d472f314..39e9b315d 100644 --- a/tests/integration/datanetwork/test_file_commands.py +++ b/tests/integration/datanetwork/test_file_commands.py @@ -520,6 +520,7 @@ class DiskSpaceManagement(CommandTestCase): self.assertIn('disk_space', status) self.assertEqual(status['disk_space']['used'], '0') await self.stream_create('foo', '0.01', data=('0' * 3 * 1024 * 1024).encode()) + await self.stream_create('foo', '0.01', data=('0' * 2 * 1024 * 1024).encode()) status = await self.daemon.jsonrpc_status() self.assertIn('disk_space', status) - self.assertEqual(status['disk_space']['used'], '3') + self.assertEqual(status['disk_space']['used'], '5') diff --git a/tests/unit/blob/test_disk_space_manager.py b/tests/unit/blob/test_disk_space_manager.py index 75aa07c4c..95bd1960f 100644 --- a/tests/unit/blob/test_disk_space_manager.py +++ b/tests/unit/blob/test_disk_space_manager.py @@ -11,6 +11,7 @@ class ConfigurationTests(unittest.TestCase): def test_space_calculation(self): with tempfile.TemporaryDirectory() as temp_dir: + os.mkdir(os.path.join(temp_dir, 'blobfiles')) config = Config( data_dir=temp_dir, wallet_dir=temp_dir, @@ -18,6 +19,6 @@ class ConfigurationTests(unittest.TestCase): ) dsm = DiskSpaceManager(config) self.assertEqual(0, dsm.space_used_mb) - with open(os.path.join(config.data_dir, '3mb-file'), 'w') as blob: + with open(os.path.join(config.data_dir, 'blobfiles', '3mb-file'), 'w') as blob: blob.write('0' * 3 * 1024 * 1024) self.assertEqual(3, dsm.space_used_mb) From 51d21d8c86bbc750e94b3c97b47e883d718c145f Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Mon, 16 Aug 2021 14:15:12 -0400 Subject: [PATCH 05/10] working disk cleanup --- lbry/blob/disk_space_manager.py | 50 +++++++++++++++++-- lbry/extras/daemon/components.py | 7 +-- lbry/extras/daemon/daemon.py | 22 +++++++- lbry/testcase.py | 6 +++ .../datanetwork/test_file_commands.py | 18 ++++--- tests/unit/blob/test_disk_space_manager.py | 15 ++++-- 6 files changed, 97 insertions(+), 21 deletions(-) diff --git a/lbry/blob/disk_space_manager.py b/lbry/blob/disk_space_manager.py index 74463e4aa..e95a78cea 100644 --- a/lbry/blob/disk_space_manager.py +++ b/lbry/blob/disk_space_manager.py @@ -1,21 +1,61 @@ import os +import asyncio +import logging + +log = logging.getLogger(__name__) class DiskSpaceManager: - def __init__(self, config): + def __init__(self, config, cleaning_interval=30 * 60): self.config = config + self.cleaning_interval = cleaning_interval + self.running = False + self.task = None @property def space_used_bytes(self): used = 0 data_dir = os.path.join(self.config.data_dir, 'blobfiles') - for item in os.listdir(data_dir): - blob_path = os.path.join(data_dir, item) - if os.path.isfile(blob_path): - used += os.path.getsize(blob_path) + for item in os.scandir(data_dir): + if item.is_file: + used += item.stat().st_size return used @property def space_used_mb(self): return int(self.space_used_bytes/1024.0/1024.0) + + def clean(self): + if not self.config.blob_storage_limit: + return + used = 0 + files = [] + data_dir = os.path.join(self.config.data_dir, 'blobfiles') + for file in os.scandir(data_dir): + if file.is_file: + file_stats = file.stat() + used += file_stats.st_size + files.append((file_stats.st_mtime, file_stats.st_size, file)) + files.sort() + available = (self.config.blob_storage_limit*1024*1024) - used + for _, file_size, file in files: + available += file_size + if available > 0: + break + os.remove(file) + + async def cleaning_loop(self): + while self.running: + await asyncio.get_event_loop().run_in_executor(None, self.clean) + await asyncio.sleep(self.cleaning_interval) + + async def start(self): + self.running = True + self.task = asyncio.create_task(self.cleaning_loop()) + self.task.add_done_callback(lambda _: log.info("Stopping blob cleanup service.")) + + async def stop(self): + if self.running: + self.running = False + self.task.cancel() diff --git a/lbry/extras/daemon/components.py b/lbry/extras/daemon/components.py index fbaa598cb..a383b9ea5 100644 --- a/lbry/extras/daemon/components.py +++ b/lbry/extras/daemon/components.py @@ -390,14 +390,15 @@ class DiskSpaceComponent(Component): async def get_status(self): return { - 'used': str(self.disk_space_manager.space_used_mb), + 'space_used': str(self.disk_space_manager.space_used_mb), + 'running': self.disk_space_manager.running, } async def start(self): - pass + await self.disk_space_manager.start() async def stop(self): - pass + await self.disk_space_manager.stop() class TorrentComponent(Component): diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index bf33c0541..814f4e50e 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -56,7 +56,7 @@ from lbry.schema.url import URL if typing.TYPE_CHECKING: from lbry.blob.blob_manager import BlobManager from lbry.dht.node import Node - from lbry.extras.daemon.components import UPnPComponent + from lbry.extras.daemon.components import UPnPComponent, DiskSpaceManager from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager from lbry.extras.daemon.storage import SQLiteStorage from lbry.wallet import WalletManager, Ledger @@ -375,6 +375,10 @@ class Daemon(metaclass=JSONRPCServerType): def blob_manager(self) -> typing.Optional['BlobManager']: return self.component_manager.get_component(BLOB_COMPONENT) + @property + def disk_space_manager(self) -> typing.Optional['DiskSpaceManager']: + return self.component_manager.get_component(DISK_SPACE_COMPONENT) + @property def upnp(self) -> typing.Optional['UPnPComponent']: return self.component_manager.get_component(UPNP_COMPONENT) @@ -4916,6 +4920,22 @@ class Daemon(metaclass=JSONRPCServerType): raise NotImplementedError() + @requires(DISK_SPACE_COMPONENT) + async def jsonrpc_blob_clean(self): + """ + Deletes blobs to cleanup disk space + + Usage: + blob_clean + + Options: + None + + Returns: + (bool) true if successful + """ + return self.disk_space_manager.clean() + @requires(FILE_MANAGER_COMPONENT) async def jsonrpc_file_reflect(self, **kwargs): """ diff --git a/lbry/testcase.py b/lbry/testcase.py index 9b65e2fe8..210b99b3c 100644 --- a/lbry/testcase.py +++ b/lbry/testcase.py @@ -607,6 +607,12 @@ class CommandTestCase(IntegrationTestCase): await asyncio.wait([self.ledger.wait(tx, self.blockchain.block_expected) for tx in txs]) return self.sout(txs) + async def blob_clean(self): + return await self.out(self.daemon.jsonrpc_blob_clean()) + + async def status(self): + return await self.out(self.daemon.jsonrpc_status()) + async def resolve(self, uri, **kwargs): return (await self.out(self.daemon.jsonrpc_resolve(uri, **kwargs)))[uri] diff --git a/tests/integration/datanetwork/test_file_commands.py b/tests/integration/datanetwork/test_file_commands.py index 39e9b315d..dccaca324 100644 --- a/tests/integration/datanetwork/test_file_commands.py +++ b/tests/integration/datanetwork/test_file_commands.py @@ -516,11 +516,15 @@ class FileCommands(CommandTestCase): class DiskSpaceManagement(CommandTestCase): async def test_file_management(self): - status = await self.daemon.jsonrpc_status() + status = await self.status() self.assertIn('disk_space', status) - self.assertEqual(status['disk_space']['used'], '0') - await self.stream_create('foo', '0.01', data=('0' * 3 * 1024 * 1024).encode()) - await self.stream_create('foo', '0.01', data=('0' * 2 * 1024 * 1024).encode()) - status = await self.daemon.jsonrpc_status() - self.assertIn('disk_space', status) - self.assertEqual(status['disk_space']['used'], '5') + self.assertEqual('0', status['disk_space']['space_used']) + self.assertEqual(True, status['disk_space']['running']) + await self.stream_create('foo1', '0.01', data=('0' * 3 * 1024 * 1024).encode()) + await self.stream_create('foo2', '0.01', data=('0' * 2 * 1024 * 1024).encode()) + self.assertEqual('5', (await self.status())['disk_space']['space_used']) + await self.blob_clean() + self.assertEqual('5', (await self.status())['disk_space']['space_used']) + self.daemon.conf.blob_storage_limit = 3 + await self.blob_clean() + self.assertEqual('3', (await self.status())['disk_space']['space_used']) diff --git a/tests/unit/blob/test_disk_space_manager.py b/tests/unit/blob/test_disk_space_manager.py index 95bd1960f..35f11212c 100644 --- a/tests/unit/blob/test_disk_space_manager.py +++ b/tests/unit/blob/test_disk_space_manager.py @@ -9,16 +9,21 @@ from lbry.blob.disk_space_manager import DiskSpaceManager class ConfigurationTests(unittest.TestCase): - def test_space_calculation(self): + def test_space_management(self): with tempfile.TemporaryDirectory() as temp_dir: os.mkdir(os.path.join(temp_dir, 'blobfiles')) config = Config( + blob_storage_limit=5, data_dir=temp_dir, wallet_dir=temp_dir, - config=os.path.join(temp_dir, 'settings.yml') + config=os.path.join(temp_dir, 'settings.yml'), ) dsm = DiskSpaceManager(config) self.assertEqual(0, dsm.space_used_mb) - with open(os.path.join(config.data_dir, 'blobfiles', '3mb-file'), 'w') as blob: - blob.write('0' * 3 * 1024 * 1024) - self.assertEqual(3, dsm.space_used_mb) + for file_no in range(10): + with open(os.path.join(config.data_dir, 'blobfiles', f'3mb-{file_no}'), 'w') as blob: + blob.write('0' * 1 * 1024 * 1024) + self.assertEqual(10, dsm.space_used_mb) + dsm.clean() + self.assertEqual(4, dsm.space_used_mb) + From fd8658e31793bf41137ef77269739b0acd5b6801 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Mon, 16 Aug 2021 14:35:32 -0400 Subject: [PATCH 06/10] test component unit test --- tests/unit/components/test_component_manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/components/test_component_manager.py b/tests/unit/components/test_component_manager.py index 48d844168..f539f033e 100644 --- a/tests/unit/components/test_component_manager.py +++ b/tests/unit/components/test_component_manager.py @@ -15,6 +15,7 @@ class TestComponentManager(AsyncioTestCase): self.default_components_sort = [ [ components.DatabaseComponent, + components.DiskSpaceComponent, components.ExchangeRateManagerComponent, components.TorrentComponent, components.UPnPComponent From 4edab7bb7f9cb610145ce4341bf0a4ffd71e5a13 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Mon, 16 Aug 2021 14:41:16 -0400 Subject: [PATCH 07/10] fix sorting by DirEntry error --- lbry/blob/disk_space_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbry/blob/disk_space_manager.py b/lbry/blob/disk_space_manager.py index e95a78cea..7ef7a3889 100644 --- a/lbry/blob/disk_space_manager.py +++ b/lbry/blob/disk_space_manager.py @@ -36,7 +36,7 @@ class DiskSpaceManager: if file.is_file: file_stats = file.stat() used += file_stats.st_size - files.append((file_stats.st_mtime, file_stats.st_size, file)) + files.append((file_stats.st_mtime, file_stats.st_size, file.path)) files.sort() available = (self.config.blob_storage_limit*1024*1024) - used for _, file_size, file in files: From 2535b8adef951096b0e81672682675f345234cb3 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Mon, 16 Aug 2021 14:54:17 -0400 Subject: [PATCH 08/10] fix disk space unit test --- tests/unit/blob/test_disk_space_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/blob/test_disk_space_manager.py b/tests/unit/blob/test_disk_space_manager.py index 35f11212c..32237ff09 100644 --- a/tests/unit/blob/test_disk_space_manager.py +++ b/tests/unit/blob/test_disk_space_manager.py @@ -25,5 +25,5 @@ class ConfigurationTests(unittest.TestCase): blob.write('0' * 1 * 1024 * 1024) self.assertEqual(10, dsm.space_used_mb) dsm.clean() - self.assertEqual(4, dsm.space_used_mb) + self.assertEqual(5, dsm.space_used_mb) From 2cd5d75a2e913a0adf5c58e34d75a172a5e1f394 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Mon, 16 Aug 2021 17:02:13 -0400 Subject: [PATCH 09/10] return true/false if clean was performed --- lbry/blob/disk_space_manager.py | 5 ++++- tests/unit/blob/test_disk_space_manager.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lbry/blob/disk_space_manager.py b/lbry/blob/disk_space_manager.py index 7ef7a3889..67e8cda2b 100644 --- a/lbry/blob/disk_space_manager.py +++ b/lbry/blob/disk_space_manager.py @@ -28,7 +28,7 @@ class DiskSpaceManager: def clean(self): if not self.config.blob_storage_limit: - return + return False used = 0 files = [] data_dir = os.path.join(self.config.data_dir, 'blobfiles') @@ -39,11 +39,14 @@ class DiskSpaceManager: files.append((file_stats.st_mtime, file_stats.st_size, file.path)) files.sort() available = (self.config.blob_storage_limit*1024*1024) - used + cleaned = False for _, file_size, file in files: available += file_size if available > 0: break os.remove(file) + cleaned = True + return cleaned async def cleaning_loop(self): while self.running: diff --git a/tests/unit/blob/test_disk_space_manager.py b/tests/unit/blob/test_disk_space_manager.py index 32237ff09..54871f092 100644 --- a/tests/unit/blob/test_disk_space_manager.py +++ b/tests/unit/blob/test_disk_space_manager.py @@ -24,6 +24,7 @@ class ConfigurationTests(unittest.TestCase): with open(os.path.join(config.data_dir, 'blobfiles', f'3mb-{file_no}'), 'w') as blob: blob.write('0' * 1 * 1024 * 1024) self.assertEqual(10, dsm.space_used_mb) - dsm.clean() + self.assertTrue(dsm.clean()) self.assertEqual(5, dsm.space_used_mb) + self.assertFalse(dsm.clean()) From 3433c9e7084dab1d15a04f434badb2d51cc76b22 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Mon, 16 Aug 2021 17:03:40 -0400 Subject: [PATCH 10/10] return number of files deleted --- lbry/blob/disk_space_manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lbry/blob/disk_space_manager.py b/lbry/blob/disk_space_manager.py index 67e8cda2b..58e8b19bc 100644 --- a/lbry/blob/disk_space_manager.py +++ b/lbry/blob/disk_space_manager.py @@ -28,7 +28,7 @@ class DiskSpaceManager: def clean(self): if not self.config.blob_storage_limit: - return False + return 0 used = 0 files = [] data_dir = os.path.join(self.config.data_dir, 'blobfiles') @@ -39,13 +39,13 @@ class DiskSpaceManager: files.append((file_stats.st_mtime, file_stats.st_size, file.path)) files.sort() available = (self.config.blob_storage_limit*1024*1024) - used - cleaned = False + cleaned = 0 for _, file_size, file in files: available += file_size if available > 0: break os.remove(file) - cleaned = True + cleaned += 1 return cleaned async def cleaning_loop(self):