diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index ae913b182..8f4742b8c 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -9,6 +9,7 @@ import json import textwrap import signal from copy import deepcopy +from operator import attrgetter from twisted.web import server from twisted.internet import defer, threads, error, reactor from twisted.internet.task import LoopingCall @@ -97,10 +98,6 @@ CONNECTION_MESSAGES = { SHORT_ID_LEN = 20 MAX_UPDATE_FEE_ESTIMATE = 0.3 -FILE_SORT_FIELD_NAME = 'name' -FILE_SORT_FIELD_DATE = 'date' -FILE_SORT_FIELD_PRICE = 'price' - FILE_SORT_DIRECTION_ASCENDING = 'asc' FILE_SORT_DIRECTION_DESCENDING = 'desc' FILE_SORT_DIRECTIONS = ( @@ -935,13 +932,11 @@ class Daemon(AuthJSONRPCServer): defer.returnValue(lbry_file) @defer.inlineCallbacks - def _get_lbry_files(self, return_json=False, full_status=True, sort_by=None, **kwargs): + def _get_lbry_files(self, return_json=False, full_status=True, **kwargs): lbry_files = list(self.lbry_file_manager.lbry_files) if kwargs: for search_type, value in iter_lbry_file_search_values(kwargs): lbry_files = [l_f for l_f in lbry_files if l_f.__dict__[search_type] == value] - if sort_by: - lbry_files = self._sort_lbry_files(lbry_files, sort_by) if return_json: file_dicts = [] for lbry_file in lbry_files: @@ -954,25 +949,29 @@ class Daemon(AuthJSONRPCServer): def _sort_lbry_files(self, lbry_files, sort_by): for field, direction in sort_by: is_reverse = direction == FILE_SORT_DIRECTION_DESCENDING - if field == FILE_SORT_FIELD_NAME: - lbry_files = sorted(lbry_files, key=lambda f: f.file_name, reverse=is_reverse) - elif field == FILE_SORT_FIELD_DATE: - lbry_files = sorted(lbry_files, reverse=is_reverse) - elif field == FILE_SORT_FIELD_PRICE: - lbry_files = sorted(lbry_files, key=lambda f: f.points_paid, reverse=is_reverse) - else: - raise Exception('Unrecognized sort field "{}"'.format(field)) + key_getter = None + if field: + search_path = field.split('.') + def key_getter(value): + for key in search_path: + try: + value = value[key] + except KeyError as e: + errmsg = 'Failed to sort by "{}", key "{}" was not found.' + raise Exception(errmsg.format(field, e.message)) + return value + lbry_files = sorted(lbry_files, key=key_getter, reverse=is_reverse) return lbry_files def _parse_lbry_files_sort(self, sort): """ - Given a sort string like 'name, desc' or 'price', + Given a sort string like 'file_name, desc' or 'points_paid', parse the string into a tuple of (field, direction). Direction defaults to ascending. """ pieces = sort.rsplit(',', 1) - field = pieces[0].strip() + field = pieces[0].strip() or None direction = pieces[1].strip().lower() if len(pieces) > 1 else None if direction and direction not in FILE_SORT_DIRECTIONS: raise Exception('Sort direction must be one of {}'.format(FILE_SORT_DIRECTIONS)) @@ -1422,8 +1421,9 @@ class Daemon(AuthJSONRPCServer): --claim_name= : (str) get file with matching claim name --full_status : (bool) full status, populate the 'message' and 'size' fields - --sort= : (str) sort by any of 'name', 'date', or 'price' - to specify direction append ',asc' or ',desc' + --sort= : (str) sort by any property, like 'file_name' + or 'metadata.author'; to specify direction + append ',asc' or ',desc' Returns: (list) List of files @@ -1459,8 +1459,10 @@ class Daemon(AuthJSONRPCServer): ] """ - sort_by = [self._parse_lbry_files_sort(s) for s in sort] if sort else None - result = yield self._get_lbry_files(return_json=True, sort_by=sort_by, **kwargs) + result = yield self._get_lbry_files(return_json=True, **kwargs) + if sort: + sort_by = [self._parse_lbry_files_sort(s) for s in sort] + result = self._sort_lbry_files(result, sort_by) response = yield self._render_response(result) defer.returnValue(response) diff --git a/lbrynet/tests/unit/lbrynet_daemon/test_Daemon.py b/lbrynet/tests/unit/lbrynet_daemon/test_Daemon.py index 6d032ef78..af242af3f 100644 --- a/lbrynet/tests/unit/lbrynet_daemon/test_Daemon.py +++ b/lbrynet/tests/unit/lbrynet_daemon/test_Daemon.py @@ -1,6 +1,7 @@ import mock import json import unittest +import random from os import path from twisted.internet import defer @@ -146,88 +147,118 @@ class TestFileListSorting(trial.unittest.TestCase): # Pre-sorted lists of prices and file names in ascending order produced by # faker with seed 66410. This seed was chosen becacuse it produces 3 results # 'points_paid' at 6.0 and 2 results at 4.5 to test multiple sort criteria. - self.test_prices = [0.2, 2.9, 4.5, 4.5, 6.0, 6.0, 6.0, 6.8, 7.1, 9.2] + self.test_points_paid = [0.2, 2.9, 4.5, 4.5, 6.0, 6.0, 6.0, 6.8, 7.1, 9.2] self.test_file_names = ['also.mp3', 'better.css', 'call.mp3', 'pay.jpg', 'record.pages', 'sell.css', 'strategy.pages', 'thousand.pages', 'town.mov', 'vote.ppt'] + self.test_authors = ['angela41', 'edward70', 'fhart', 'johnrosales', + 'lucasfowler', 'peggytorres', 'qmitchell', + 'trevoranderson', 'xmitchell', 'zhangsusan'] - @defer.inlineCallbacks - def test_sort_by_price_no_direction_specified(self): - sort_options = ['price'] - file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options) - received = [f['points_paid'] for f in file_list] - self.assertEquals(self.test_prices, received) + def test_sort_by_points_paid_no_direction_specified(self): + sort_options = ['points_paid'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + file_list = self.successResultOf(deferred) + self.assertEquals(self.test_points_paid, [f['points_paid'] for f in file_list]) - @defer.inlineCallbacks - def test_sort_by_price_ascending(self): - sort_options = ['price,asc'] - file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options) - received = [f['points_paid'] for f in file_list] - self.assertEquals(self.test_prices, received) + def test_sort_by_points_paid_ascending(self): + sort_options = ['points_paid,asc'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + file_list = self.successResultOf(deferred) + self.assertEquals(self.test_points_paid, [f['points_paid'] for f in file_list]) - @defer.inlineCallbacks - def test_sort_by_price_descending(self): - sort_options = ['price, desc'] - file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options) - received = [f['points_paid'] for f in file_list] - expected = list(reversed(self.test_prices)) - self.assertEquals(expected, received) + def test_sort_by_points_paid_descending(self): + sort_options = ['points_paid, desc'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + file_list = self.successResultOf(deferred) + self.assertEquals(list(reversed(self.test_points_paid)), [f['points_paid'] for f in file_list]) - @defer.inlineCallbacks - def test_sort_by_name_no_direction_specified(self): - sort_options = ['name'] - file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options) - received = [f['file_name'] for f in file_list] - self.assertEquals(self.test_file_names, received) + def test_sort_by_file_name_no_direction_specified(self): + sort_options = ['file_name'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + file_list = self.successResultOf(deferred) + self.assertEquals(self.test_file_names, [f['file_name'] for f in file_list]) - @defer.inlineCallbacks - def test_sort_by_name_ascending(self): - sort_options = ['name,\nasc'] - file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options) - received = [f['file_name'] for f in file_list] - self.assertEquals(self.test_file_names, received) + def test_sort_by_file_name_ascending(self): + sort_options = ['file_name,\nasc'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + file_list = self.successResultOf(deferred) + self.assertEquals(self.test_file_names, [f['file_name'] for f in file_list]) - @defer.inlineCallbacks - def test_sort_by_name_descending(self): - sort_options = ['\tname,\n\tdesc'] - file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options) - received = [f['file_name'] for f in file_list] - expected = list(reversed(self.test_file_names)) - self.assertEquals(expected, received) + def test_sort_by_file_name_descending(self): + sort_options = ['\tfile_name,\n\tdesc'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + file_list = self.successResultOf(deferred) + self.assertEquals(list(reversed(self.test_file_names)), [f['file_name'] for f in file_list]) - @defer.inlineCallbacks def test_sort_by_multiple_criteria(self): - sort_options = ['name,asc', 'price,desc'] - file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options) - received = ['name={}, price={}'.format(f['file_name'], f['points_paid']) for f in file_list] - expected = ['name=record.pages, price=9.2', - 'name=vote.ppt, price=7.1', - 'name=strategy.pages, price=6.8', - 'name=also.mp3, price=6.0', - 'name=better.css, price=6.0', - 'name=town.mov, price=6.0', - 'name=sell.css, price=4.5', - 'name=thousand.pages, price=4.5', - 'name=call.mp3, price=2.9', - 'name=pay.jpg, price=0.2'] - self.assertEquals(expected, received) + expected = ['file_name=record.pages, points_paid=9.2', + 'file_name=vote.ppt, points_paid=7.1', + 'file_name=strategy.pages, points_paid=6.8', + 'file_name=also.mp3, points_paid=6.0', + 'file_name=better.css, points_paid=6.0', + 'file_name=town.mov, points_paid=6.0', + 'file_name=sell.css, points_paid=4.5', + 'file_name=thousand.pages, points_paid=4.5', + 'file_name=call.mp3, points_paid=2.9', + 'file_name=pay.jpg, points_paid=0.2'] + format_result = lambda f: 'file_name={}, points_paid={}'.format(f['file_name'], f['points_paid']) - # Check that the list is not sorted as expected when sorted only by name. - sort_options = ['name,asc'] - file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options) - received = ['name={}, price={}'.format(f['file_name'], f['points_paid']) for f in file_list] - self.assertNotEqual(expected, received) + sort_options = ['file_name,asc', 'points_paid,desc'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + file_list = self.successResultOf(deferred) + self.assertEquals(expected, map(format_result, file_list)) - # Check that the list is not sorted as expected when sorted only by price. - sort_options = ['price,desc'] - file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options) - received = ['name={}, price={}'.format(f['file_name'], f['points_paid']) for f in file_list] - self.assertNotEqual(expected, received) + # Check that the list is not sorted as expected when sorted only by file_name. + sort_options = ['file_name,asc'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + file_list = self.successResultOf(deferred) + self.assertNotEqual(expected, map(format_result, file_list)) + + # Check that the list is not sorted as expected when sorted only by points_paid. + sort_options = ['points_paid,desc'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + file_list = self.successResultOf(deferred) + self.assertNotEqual(expected, map(format_result, file_list)) # Check that the list is not sorted as expected when not sorted at all. - file_list = yield self.test_daemon.jsonrpc_file_list() - received = ['name={}, price={}'.format(f['file_name'], f['points_paid']) for f in file_list] - self.assertNotEqual(expected, received) + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list) + file_list = self.successResultOf(deferred) + self.assertNotEqual(expected, map(format_result, file_list)) + + def test_sort_by_nested_field(self): + extract_authors = lambda file_list: [f['metadata']['author'] for f in file_list] + + sort_options = ['metadata.author'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + file_list = self.successResultOf(deferred) + self.assertEquals(self.test_authors, extract_authors(file_list)) + + # Check that the list matches the expected in reverse when sorting in descending order. + sort_options = ['metadata.author,desc'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + file_list = self.successResultOf(deferred) + self.assertEquals(list(reversed(self.test_authors)), extract_authors(file_list)) + + # Check that the list is not sorted as expected when not sorted at all. + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list) + file_list = self.successResultOf(deferred) + self.assertNotEqual(self.test_authors, extract_authors(file_list)) + + def test_invalid_sort_produces_meaningful_errors(self): + sort_options = ['meta.author'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + failure_assertion = self.assertFailure(deferred, Exception) + exception = self.successResultOf(failure_assertion) + expected_message = 'Failed to sort by "meta.author", key "meta" was not found.' + self.assertEquals(expected_message, exception.message) + + sort_options = ['metadata.foo.bar'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + failure_assertion = self.assertFailure(deferred, Exception) + exception = self.successResultOf(failure_assertion) + expected_message = 'Failed to sort by "metadata.foo.bar", key "foo" was not found.' + self.assertEquals(expected_message, exception.message) def _get_fake_lbry_files(self): return [self._get_fake_lbry_file() for _ in range(10)] @@ -237,9 +268,11 @@ class TestFileListSorting(trial.unittest.TestCase): file_path = self.faker.file_path() stream_name = self.faker.file_name() + channel_claim_id = self.faker.sha1() + channel_name = self.faker.simple_profile()['username'] faked_attributes = { - 'channel_claim_id': self.faker.sha1(), - 'channel_name': '@' + self.faker.simple_profile()['username'], + 'channel_claim_id': channel_claim_id, + 'channel_name': '@' + channel_name, 'claim_id': self.faker.sha1(), 'claim_name': '-'.join(self.faker.words(4)), 'completed': self.faker.boolean(), @@ -247,7 +280,10 @@ class TestFileListSorting(trial.unittest.TestCase): 'download_path': file_path, 'file_name': path.basename(file_path), 'key': self.faker.md5(), - 'metadata': {}, + 'metadata': { + 'author': channel_name, + 'nsfw': random.randint(0, 1) == 1, + }, 'mime_type': self.faker.mime_type(), 'nout': abs(self.faker.pyint()), 'outpoint': self.faker.md5() + self.faker.md5(),