From 869be3e36194cab1a501420db2708adbf6fb30d1 Mon Sep 17 00:00:00 2001 From: Miroslav Kovar Date: Fri, 11 Oct 2019 17:54:12 +0200 Subject: [PATCH] Improve tests, update suggested file name when necessary --- lbry/lbry/extras/daemon/Daemon.py | 15 ++-- lbry/lbry/stream/descriptor.py | 21 ++--- lbry/lbry/testcase.py | 31 ++++---- lbry/tests/integration/test_file_commands.py | 81 +++++++++++++++----- 4 files changed, 93 insertions(+), 55 deletions(-) diff --git a/lbry/lbry/extras/daemon/Daemon.py b/lbry/lbry/extras/daemon/Daemon.py index bd32db9fe..8a1de8b85 100644 --- a/lbry/lbry/extras/daemon/Daemon.py +++ b/lbry/lbry/extras/daemon/Daemon.py @@ -2810,7 +2810,7 @@ class Daemon(metaclass=JSONRPCServerType): @requires(WALLET_COMPONENT, STREAM_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT) async def jsonrpc_stream_update( - self, claim_id, bid=None, file_path=None, + self, claim_id, bid=None, file_path=None, file_name=None, channel_id=None, channel_name=None, channel_account_id=None, clear_channel=False, account_id=None, wallet_id=None, claim_address=None, funding_account_ids=None, preview=False, blocking=False, replace=False, **kwargs): @@ -2979,7 +2979,7 @@ class Daemon(metaclass=JSONRPCServerType): claim.stream.update(file_path=file_path, **kwargs) else: claim = Claim.from_bytes(old_txo.claim.to_bytes()) - claim.stream.update(file_path=file_path, **kwargs) + claim.stream.update(file_path=file_path, file_name=file_name, **kwargs) tx = await Transaction.claim_update( old_txo, claim, amount, claim_address, funding_accounts, funding_accounts[0], channel ) @@ -2988,10 +2988,13 @@ class Daemon(metaclass=JSONRPCServerType): stream_hash = None if not preview: old_stream_hash = await self.storage.get_stream_hash_for_sd_hash(old_txo.claim.stream.source.sd_hash) - if file_path is not None: - if old_stream_hash: - stream_to_delete = self.stream_manager.get_stream_by_stream_hash(old_stream_hash) - await self.stream_manager.delete_stream(stream_to_delete, delete_file=False) + old_stream = self.stream_manager.get_stream_by_stream_hash(old_stream_hash) + if not file_path and file_name and old_stream: + old_path, _ = os.path.split(old_stream.full_path) + file_path = os.path.join(old_path, file_name) + if file_path: + if old_stream: + await self.stream_manager.delete_stream(old_stream, delete_file=False) file_stream = await self.stream_manager.create_stream(file_path) new_txo.claim.stream.source.sd_hash = file_stream.sd_hash new_txo.script.generate() diff --git a/lbry/lbry/stream/descriptor.py b/lbry/lbry/stream/descriptor.py index b965253cb..ec89478d1 100644 --- a/lbry/lbry/stream/descriptor.py +++ b/lbry/lbry/stream/descriptor.py @@ -15,6 +15,17 @@ from lbry.error import InvalidStreamDescriptorError log = logging.getLogger(__name__) +_exprs = [ + '[<>:"/\\\|\?\*]+', # Illegal characters + '[\\x00-\\x1F]+', # All characters in range 0-31 + '[ \t]*(\.)+[ \t]*$', # Dots at the end + '(^[ \t]+|[ \t]+$)', # Leading and trailing whitespace + '^CON$', '^PRN$', '^AUX$', # Illegal names + '^NUL$', '^COM[1-9]$', '^LPT[1-9]$' +] + +RES = re.compile('(' + '|'.join((expr for expr in _exprs)) + ')') + def format_sd_info(stream_name: str, key: str, suggested_file_name: str, stream_hash: str, blobs: typing.List[typing.Dict]) -> typing.Dict: @@ -48,16 +59,6 @@ def file_reader(file_path: str): def sanitize_file_name(dirty_name: str): - exprs = [ - '[<>:"/\\\|\?\*]+', # Illegal characters - '[\\x00-\\x1F]+', # All characters in range 0-31 - '[ \t]*(\.)+[ \t]*$', # Dots at the end - '(^[ \t]+|[ \t]+$)', # Leading and trailing whitespace - '^CON$', '^PRN$', '^AUX$', # Illegal names - '^NUL$', '^COM[1-9]$', '^LPT[1-9]$'] - - RES = re.compile('(' + '|'.join((expr for expr in exprs)) + ')') - file_name, ext = os.path.splitext(dirty_name) file_name = re.sub(RES, '', file_name) ext = re.sub(RES, '', ext) diff --git a/lbry/lbry/testcase.py b/lbry/lbry/testcase.py index 04fed3114..8c2c397d2 100644 --- a/lbry/lbry/testcase.py +++ b/lbry/lbry/testcase.py @@ -197,9 +197,11 @@ class CommandTestCase(IntegrationTestCase): """ Synchronous version of `out` method. """ return json.loads(jsonrpc_dumps_pretty(value, ledger=self.ledger))['result'] - async def stream_create(self, name='hovercraft', bid='1.0', data=b'hi!', confirm=True, - prefix=None, suffix=None, **kwargs): - file = tempfile.NamedTemporaryFile(prefix=prefix, suffix=suffix) + def create_tempfile(self, data=None, prefix=None, suffix=None, dir=None): + dir = dir or self.daemon.conf.download_dir + file = tempfile.NamedTemporaryFile(prefix=prefix, suffix=suffix, dir=dir) + file.write(data) + file.flush() def cleanup(): try: @@ -208,10 +210,13 @@ class CommandTestCase(IntegrationTestCase): pass self.addCleanup(cleanup) - file.write(data) - file.flush() + return file.name + + async def stream_create(self, name='hovercraft', bid='1.0', data=b'hi!', confirm=True, + prefix=None, suffix=None, **kwargs): + file_path = self.create_tempfile(data=data, prefix=prefix, suffix=suffix) claim = await self.out( - self.daemon.jsonrpc_stream_create(name, bid, file_path=file.name, **kwargs) + self.daemon.jsonrpc_stream_create(name, bid, file_path=file_path, **kwargs) ) self.assertEqual(claim['outputs'][0]['name'], name) if confirm: @@ -222,19 +227,9 @@ class CommandTestCase(IntegrationTestCase): async def stream_update(self, claim_id, data=None, confirm=True, **kwargs): if data: - file = tempfile.NamedTemporaryFile() - file.write(data) - file.flush() - - def cleanup(): - try: - file.close() - except FileNotFoundError: - pass - - self.addCleanup(cleanup) + file_path = self.create_tempfile(data) claim = await self.out( - self.daemon.jsonrpc_stream_update(claim_id, file_path=file.name, **kwargs) + self.daemon.jsonrpc_stream_update(claim_id, file_path=file_path, **kwargs) ) else: claim = await self.out(self.daemon.jsonrpc_stream_update(claim_id, **kwargs)) diff --git a/lbry/tests/integration/test_file_commands.py b/lbry/tests/integration/test_file_commands.py index 6881bce69..2e05b6d4b 100644 --- a/lbry/tests/integration/test_file_commands.py +++ b/lbry/tests/integration/test_file_commands.py @@ -45,40 +45,79 @@ class FileCommands(CommandTestCase): {stream.sd_hash, stream.descriptor.blobs[0].blob_hash} ) + async def _purge_file(self, claim_name, full_path): + self.assertTrue( + await self.daemon.jsonrpc_file_delete(claim_name=claim_name, delete_from_download_dir=True) + ) + self.assertEqual(len(self.daemon.jsonrpc_file_list()), 0) + self.assertFalse(os.path.isfile(full_path)) + async def test_publish_with_illegal_chars(self): + def check_prefix_suffix(name, prefix, suffix): + self.assertTrue(name.startswith(prefix)) + self.assertTrue(name.endswith(suffix)) + # Stream a file with file name containing invalid chars - prefix = '?an|t:z< m<' - suffix = '.ext.' - san_prefix = 'antz m' - san_suffix = '.ext' - tx = await self.stream_create('foo', '0.01', prefix=prefix, suffix=suffix) + claim_name = 'lolwindows' + prefix, suffix = 'derp?', '.ext.' + san_prefix, san_suffix = 'derp', '.ext' + tx = await self.stream_create(claim_name, '0.01', prefix=prefix, suffix=suffix) stream = self.daemon.jsonrpc_file_list()[0] + claim_id = self.get_claim_id(tx) # Assert that file list and source contains the local unsanitized name, but suggested name is sanitized + full_path = (await self.daemon.jsonrpc_get('lbry://' + claim_name)).full_path + stream_file_name = os.path.basename(full_path) source_file_name = tx['outputs'][0]['value']['source']['name'] file_list_name = stream.file_name suggested_file_name = stream.descriptor.suggested_file_name - self.assertTrue(source_file_name.startswith(prefix)) - self.assertTrue(source_file_name.endswith(suffix)) - self.assertEqual(file_list_name, source_file_name) - self.assertTrue(suggested_file_name.startswith(san_prefix)) - self.assertTrue(suggested_file_name.endswith(san_suffix)) - # Delete the file, re-download and assert that the file name is sanitized - self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='foo')) - self.assertEqual(len(self.daemon.jsonrpc_file_list()), 0) - full_path = (await self.daemon.jsonrpc_get('lbry://foo', save_file=True)).full_path - file_name = os.path.basename(full_path) self.assertTrue(os.path.isfile(full_path)) - self.assertTrue(file_name.startswith(san_prefix)) - self.assertTrue(file_name.endswith(san_suffix)) + check_prefix_suffix(stream_file_name, prefix, suffix) + self.assertEqual(stream_file_name, source_file_name) + self.assertEqual(stream_file_name, file_list_name) + check_prefix_suffix(suggested_file_name, san_prefix, san_suffix) + await self._purge_file(claim_name, full_path) + + # Re-download deleted file and assert that the file name is sanitized + full_path = (await self.daemon.jsonrpc_get('lbry://' + claim_name, save_file=True)).full_path + stream_file_name = os.path.basename(full_path) + stream = self.daemon.jsonrpc_file_list()[0] + file_list_name = stream.file_name + suggested_file_name = stream.descriptor.suggested_file_name + + self.assertTrue(os.path.isfile(full_path)) + check_prefix_suffix(stream_file_name, san_prefix, san_suffix) + self.assertEqual(stream_file_name, file_list_name) + self.assertEqual(stream_file_name, suggested_file_name) + await self._purge_file(claim_name, full_path) # Assert that the downloaded file name is not sanitized when user provides custom file name - self.assertTrue(await self.daemon.jsonrpc_file_delete(claim_name='foo')) - file_name = 'my