diff --git a/.travis.yml b/.travis.yml index 09e9e6a16..d5d248513 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,6 +35,7 @@ before_install: install: - pip install -U pip==9.0.3 - pip install -r requirements.txt + - pip install -r requirements_testing.txt - pip install . script: diff --git a/CHANGELOG.md b/CHANGELOG.md index 200c87d6a..469a76f95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,7 @@ at anytime. * integration tests for bootstrapping the dht * configurable `concurrent_announcers` and `s3_headers_depth` settings * `peer_ping` command + * `--sort` option in `file_list` * linux distro and desktop name added to analytics * certifi module for Twisted SSL verification on Windows diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index 0304e458a..94f317753 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -97,6 +97,9 @@ CONNECTION_MESSAGES = { SHORT_ID_LEN = 20 MAX_UPDATE_FEE_ESTIMATE = 0.3 +DIRECTION_ASCENDING = 'asc' +DIRECTION_DESCENDING = 'desc' +DIRECTIONS = DIRECTION_ASCENDING, DIRECTION_DESCENDING class IterableContainer(object): def __iter__(self): @@ -944,6 +947,28 @@ class Daemon(AuthJSONRPCServer): log.debug("Collected %i lbry files", len(lbry_files)) defer.returnValue(lbry_files) + def _sort_lbry_files(self, lbry_files, sort_by): + for field, direction in sort_by: + is_reverse = direction == DIRECTION_DESCENDING + key_getter = create_key_getter(field) if field else None + lbry_files = sorted(lbry_files, key=key_getter, reverse=is_reverse) + return lbry_files + + def _parse_lbry_files_sort(self, sort): + """ + Given a sort string like 'file_name, desc' or 'points_paid', + parse the string into a tuple of (field, direction). + Direction defaults to ascending. + """ + + pieces = [p.strip() for p in sort.split(',')] + field = pieces.pop(0) + direction = DIRECTION_ASCENDING + if pieces and pieces[0] in DIRECTIONS: + direction = pieces[0] + return field, direction + + def _get_single_peer_downloader(self): downloader = SinglePeerDownloader() downloader.setup(self.session.wallet) @@ -1363,7 +1388,7 @@ class Daemon(AuthJSONRPCServer): defer.returnValue(response) @defer.inlineCallbacks - def jsonrpc_file_list(self, **kwargs): + def jsonrpc_file_list(self, sort=None, **kwargs): """ List files limited by optional filters @@ -1371,7 +1396,7 @@ class Daemon(AuthJSONRPCServer): file_list [--sd_hash=] [--file_name=] [--stream_hash=] [--rowid=] [--claim_id=] [--outpoint=] [--txid=] [--nout=] [--channel_claim_id=] [--channel_name=] - [--claim_name=] [--full_status] + [--claim_name=] [--full_status] [--sort=...] Options: --sd_hash= : (str) get file with matching sd hash @@ -1388,6 +1413,9 @@ class Daemon(AuthJSONRPCServer): --claim_name= : (str) get file with matching claim name --full_status : (bool) full status, populate the 'message' and 'size' fields + --sort= : (str) sort by any property, like 'file_name' + or 'metadata.author'; to specify direction + append ',asc' or ',desc' Returns: (list) List of files @@ -1424,6 +1452,9 @@ class Daemon(AuthJSONRPCServer): """ result = yield self._get_lbry_files(return_json=True, **kwargs) + if sort: + sort_by = [self._parse_lbry_files_sort(s) for s in sort] + result = self._sort_lbry_files(result, sort_by) response = yield self._render_response(result) defer.returnValue(response) @@ -3398,3 +3429,16 @@ def get_blob_payment_rate_manager(session, payment_rate_manager=None): payment_rate_manager = rate_managers[payment_rate_manager] log.info("Downloading blob with rate manager: %s", payment_rate_manager) return payment_rate_manager or session.payment_rate_manager + + +def create_key_getter(field): + search_path = field.split('.') + def key_getter(value): + for key in search_path: + try: + value = value[key] + except KeyError as e: + errmsg = 'Failed to get "{}", key "{}" was not found.' + raise Exception(errmsg.format(field, e.message)) + return value + return key_getter diff --git a/lbrynet/tests/unit/lbrynet_daemon/test_Daemon.py b/lbrynet/tests/unit/lbrynet_daemon/test_Daemon.py index ae9451323..d47c36ba2 100644 --- a/lbrynet/tests/unit/lbrynet_daemon/test_Daemon.py +++ b/lbrynet/tests/unit/lbrynet_daemon/test_Daemon.py @@ -1,16 +1,22 @@ import mock import json import unittest +import random +from os import path from twisted.internet import defer from twisted import trial +from faker import Faker + from lbryschema.decode import smart_decode from lbryum.wallet import NewWallet from lbrynet import conf from lbrynet.core import Session, PaymentRateManager, Wallet from lbrynet.database.storage import SQLiteStorage from lbrynet.daemon.Daemon import Daemon as LBRYDaemon +from lbrynet.file_manager.EncryptedFileManager import EncryptedFileManager +from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloader from lbrynet.tests import util from lbrynet.tests.mocks import mock_conf_settings, FakeNetwork @@ -126,3 +132,172 @@ class TestJsonRpc(trial.unittest.TestCase): d = defer.maybeDeferred(self.test_daemon.jsonrpc_help, command='status') d.addCallback(lambda result: self.assertSubstring('daemon status', result['help'])) # self.assertSubstring('daemon status', d.result) + + +class TestFileListSorting(trial.unittest.TestCase): + def setUp(self): + mock_conf_settings(self) + util.resetTime(self) + self.faker = Faker('en_US') + self.faker.seed(66410) + self.test_daemon = get_test_daemon() + self.test_daemon.lbry_file_manager = mock.Mock(spec=EncryptedFileManager) + self.test_daemon.lbry_file_manager.lbry_files = self._get_fake_lbry_files() + + # Pre-sorted lists of prices and file names in ascending order produced by + # faker with seed 66410. This seed was chosen becacuse it produces 3 results + # 'points_paid' at 6.0 and 2 results at 4.5 to test multiple sort criteria. + self.test_points_paid = [0.2, 2.9, 4.5, 4.5, 6.0, 6.0, 6.0, 6.8, 7.1, 9.2] + self.test_file_names = ['also.mp3', 'better.css', 'call.mp3', 'pay.jpg', + 'record.pages', 'sell.css', 'strategy.pages', + 'thousand.pages', 'town.mov', 'vote.ppt'] + self.test_authors = ['angela41', 'edward70', 'fhart', 'johnrosales', + 'lucasfowler', 'peggytorres', 'qmitchell', + 'trevoranderson', 'xmitchell', 'zhangsusan'] + + def test_sort_by_points_paid_no_direction_specified(self): + sort_options = ['points_paid'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + file_list = self.successResultOf(deferred) + self.assertEquals(self.test_points_paid, [f['points_paid'] for f in file_list]) + + def test_sort_by_points_paid_ascending(self): + sort_options = ['points_paid,asc'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + file_list = self.successResultOf(deferred) + self.assertEquals(self.test_points_paid, [f['points_paid'] for f in file_list]) + + def test_sort_by_points_paid_descending(self): + sort_options = ['points_paid, desc'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + file_list = self.successResultOf(deferred) + self.assertEquals(list(reversed(self.test_points_paid)), [f['points_paid'] for f in file_list]) + + def test_sort_by_file_name_no_direction_specified(self): + sort_options = ['file_name'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + file_list = self.successResultOf(deferred) + self.assertEquals(self.test_file_names, [f['file_name'] for f in file_list]) + + def test_sort_by_file_name_ascending(self): + sort_options = ['file_name,\nasc'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + file_list = self.successResultOf(deferred) + self.assertEquals(self.test_file_names, [f['file_name'] for f in file_list]) + + def test_sort_by_file_name_descending(self): + sort_options = ['\tfile_name,\n\tdesc'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + file_list = self.successResultOf(deferred) + self.assertEquals(list(reversed(self.test_file_names)), [f['file_name'] for f in file_list]) + + def test_sort_by_multiple_criteria(self): + expected = ['file_name=record.pages, points_paid=9.2', + 'file_name=vote.ppt, points_paid=7.1', + 'file_name=strategy.pages, points_paid=6.8', + 'file_name=also.mp3, points_paid=6.0', + 'file_name=better.css, points_paid=6.0', + 'file_name=town.mov, points_paid=6.0', + 'file_name=sell.css, points_paid=4.5', + 'file_name=thousand.pages, points_paid=4.5', + 'file_name=call.mp3, points_paid=2.9', + 'file_name=pay.jpg, points_paid=0.2'] + format_result = lambda f: 'file_name={}, points_paid={}'.format(f['file_name'], f['points_paid']) + + sort_options = ['file_name,asc', 'points_paid,desc'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + file_list = self.successResultOf(deferred) + self.assertEquals(expected, map(format_result, file_list)) + + # Check that the list is not sorted as expected when sorted only by file_name. + sort_options = ['file_name,asc'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + file_list = self.successResultOf(deferred) + self.assertNotEqual(expected, map(format_result, file_list)) + + # Check that the list is not sorted as expected when sorted only by points_paid. + sort_options = ['points_paid,desc'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + file_list = self.successResultOf(deferred) + self.assertNotEqual(expected, map(format_result, file_list)) + + # Check that the list is not sorted as expected when not sorted at all. + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list) + file_list = self.successResultOf(deferred) + self.assertNotEqual(expected, map(format_result, file_list)) + + def test_sort_by_nested_field(self): + extract_authors = lambda file_list: [f['metadata']['author'] for f in file_list] + + sort_options = ['metadata.author'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + file_list = self.successResultOf(deferred) + self.assertEquals(self.test_authors, extract_authors(file_list)) + + # Check that the list matches the expected in reverse when sorting in descending order. + sort_options = ['metadata.author,desc'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + file_list = self.successResultOf(deferred) + self.assertEquals(list(reversed(self.test_authors)), extract_authors(file_list)) + + # Check that the list is not sorted as expected when not sorted at all. + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list) + file_list = self.successResultOf(deferred) + self.assertNotEqual(self.test_authors, extract_authors(file_list)) + + def test_invalid_sort_produces_meaningful_errors(self): + sort_options = ['meta.author'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + failure_assertion = self.assertFailure(deferred, Exception) + exception = self.successResultOf(failure_assertion) + expected_message = 'Failed to get "meta.author", key "meta" was not found.' + self.assertEquals(expected_message, exception.message) + + sort_options = ['metadata.foo.bar'] + deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options) + failure_assertion = self.assertFailure(deferred, Exception) + exception = self.successResultOf(failure_assertion) + expected_message = 'Failed to get "metadata.foo.bar", key "foo" was not found.' + self.assertEquals(expected_message, exception.message) + + def _get_fake_lbry_files(self): + return [self._get_fake_lbry_file() for _ in range(10)] + + def _get_fake_lbry_file(self): + lbry_file = mock.Mock(spec=ManagedEncryptedFileDownloader) + + file_path = self.faker.file_path() + stream_name = self.faker.file_name() + channel_claim_id = self.faker.sha1() + channel_name = self.faker.simple_profile()['username'] + faked_attributes = { + 'channel_claim_id': channel_claim_id, + 'channel_name': '@' + channel_name, + 'claim_id': self.faker.sha1(), + 'claim_name': '-'.join(self.faker.words(4)), + 'completed': self.faker.boolean(), + 'download_directory': path.dirname(file_path), + 'download_path': file_path, + 'file_name': path.basename(file_path), + 'key': self.faker.md5(), + 'metadata': { + 'author': channel_name, + 'nsfw': random.randint(0, 1) == 1, + }, + 'mime_type': self.faker.mime_type(), + 'nout': abs(self.faker.pyint()), + 'outpoint': self.faker.md5() + self.faker.md5(), + 'points_paid': self.faker.pyfloat(left_digits=1, right_digits=1, positive=True), + 'sd_hash': self.faker.md5() + self.faker.md5() + self.faker.md5(), + 'stopped': self.faker.boolean(), + 'stream_hash': self.faker.md5() + self.faker.md5() + self.faker.md5(), + 'stream_name': stream_name, + 'suggested_file_name': stream_name, + 'txid': self.faker.md5() + self.faker.md5(), + 'written_bytes': self.faker.pyint(), + } + + for key in faked_attributes: + setattr(lbry_file, key, faked_attributes[key]) + + return lbry_file diff --git a/requirements_testing.txt b/requirements_testing.txt index e69de29bb..59f64aa4a 100644 --- a/requirements_testing.txt +++ b/requirements_testing.txt @@ -0,0 +1,2 @@ +mock>=2.0,<3.0 +Faker>=0.8,<1.0