Merge pull request #1174 from kafene/feature/file_list_sorting

Implement sorting in file_list

.travis.yml

@@ -35,6 +35,7 @@ before_install:
 install:
   - pip install -U pip==9.0.3
   - pip install -r requirements.txt
+  - pip install -r requirements_testing.txt
   - pip install .
 
 script:

CHANGELOG.md

@@ -61,6 +61,7 @@ at anytime.
   * integration tests for bootstrapping the dht
   * configurable `concurrent_announcers` and `s3_headers_depth` settings
   * `peer_ping` command
+  * `--sort` option in `file_list`
   * linux distro and desktop name added to analytics
   * certifi module for Twisted SSL verification on Windows
 

lbrynet/daemon/Daemon.py

@@ -97,6 +97,9 @@ CONNECTION_MESSAGES = {
 SHORT_ID_LEN = 20
 MAX_UPDATE_FEE_ESTIMATE = 0.3
 
+DIRECTION_ASCENDING = 'asc'
+DIRECTION_DESCENDING = 'desc'
+DIRECTIONS = DIRECTION_ASCENDING, DIRECTION_DESCENDING
 
 class IterableContainer(object):
     def __iter__(self):
@@ -944,6 +947,28 @@ class Daemon(AuthJSONRPCServer):
         log.debug("Collected %i lbry files", len(lbry_files))
         defer.returnValue(lbry_files)
 
+    def _sort_lbry_files(self, lbry_files, sort_by):
+        for field, direction in sort_by:
+            is_reverse = direction == DIRECTION_DESCENDING
+            key_getter = create_key_getter(field) if field else None
+            lbry_files = sorted(lbry_files, key=key_getter, reverse=is_reverse)
+        return lbry_files
+
+    def _parse_lbry_files_sort(self, sort):
+        """
+        Given a sort string like 'file_name, desc' or 'points_paid',
+        parse the string into a tuple of (field, direction).
+        Direction defaults to ascending.
+        """
+
+        pieces = [p.strip() for p in sort.split(',')]
+        field = pieces.pop(0)
+        direction = DIRECTION_ASCENDING
+        if pieces and pieces[0] in DIRECTIONS:
+            direction = pieces[0]
+        return field, direction
+
+
     def _get_single_peer_downloader(self):
         downloader = SinglePeerDownloader()
         downloader.setup(self.session.wallet)
@@ -1363,7 +1388,7 @@ class Daemon(AuthJSONRPCServer):
         defer.returnValue(response)
 
     @defer.inlineCallbacks
-    def jsonrpc_file_list(self, **kwargs):
+    def jsonrpc_file_list(self, sort=None, **kwargs):
         """
         List files limited by optional filters
 
@@ -1371,7 +1396,7 @@ class Daemon(AuthJSONRPCServer):
             file_list [--sd_hash=<sd_hash>] [--file_name=<file_name>] [--stream_hash=<stream_hash>]
                       [--rowid=<rowid>] [--claim_id=<claim_id>] [--outpoint=<outpoint>] [--txid=<txid>] [--nout=<nout>]
                       [--channel_claim_id=<channel_claim_id>] [--channel_name=<channel_name>]
-                      [--claim_name=<claim_name>] [--full_status]
+                      [--claim_name=<claim_name>] [--full_status] [--sort=<sort_method>...]
 
         Options:
             --sd_hash=<sd_hash>                    : (str) get file with matching sd hash
@@ -1388,6 +1413,9 @@ class Daemon(AuthJSONRPCServer):
             --claim_name=<claim_name>              : (str) get file with matching claim name
             --full_status                          : (bool) full status, populate the
                                                      'message' and 'size' fields
+            --sort=<sort_method>                   : (str) sort by any property, like 'file_name'
+                                                     or 'metadata.author'; to specify direction
+                                                     append ',asc' or ',desc'
 
         Returns:
             (list) List of files
@@ -1424,6 +1452,9 @@ class Daemon(AuthJSONRPCServer):
         """
 
         result = yield self._get_lbry_files(return_json=True, **kwargs)
+        if sort:
+            sort_by = [self._parse_lbry_files_sort(s) for s in sort]
+            result = self._sort_lbry_files(result, sort_by)
         response = yield self._render_response(result)
         defer.returnValue(response)
 
@@ -3398,3 +3429,16 @@ def get_blob_payment_rate_manager(session, payment_rate_manager=None):
             payment_rate_manager = rate_managers[payment_rate_manager]
             log.info("Downloading blob with rate manager: %s", payment_rate_manager)
     return payment_rate_manager or session.payment_rate_manager
+
+
+def create_key_getter(field):
+    search_path = field.split('.')
+    def key_getter(value):
+        for key in search_path:
+            try:
+                value = value[key]
+            except KeyError as e:
+                errmsg = 'Failed to get "{}", key "{}" was not found.'
+                raise Exception(errmsg.format(field, e.message))
+        return value
+    return key_getter

lbrynet/tests/unit/lbrynet_daemon/test_Daemon.py

@@ -1,16 +1,22 @@
 import mock
 import json
 import unittest
+import random
+from os import path
 
 from twisted.internet import defer
 from twisted import trial
 
+from faker import Faker
+
 from lbryschema.decode import smart_decode
 from lbryum.wallet import NewWallet
 from lbrynet import conf
 from lbrynet.core import Session, PaymentRateManager, Wallet
 from lbrynet.database.storage import SQLiteStorage
 from lbrynet.daemon.Daemon import Daemon as LBRYDaemon
+from lbrynet.file_manager.EncryptedFileManager import EncryptedFileManager
+from lbrynet.file_manager.EncryptedFileDownloader import ManagedEncryptedFileDownloader
 
 from lbrynet.tests import util
 from lbrynet.tests.mocks import mock_conf_settings, FakeNetwork
@@ -126,3 +132,172 @@ class TestJsonRpc(trial.unittest.TestCase):
         d = defer.maybeDeferred(self.test_daemon.jsonrpc_help, command='status')
         d.addCallback(lambda result: self.assertSubstring('daemon status', result['help']))
         # self.assertSubstring('daemon status', d.result)
+
+
+class TestFileListSorting(trial.unittest.TestCase):
+    def setUp(self):
+        mock_conf_settings(self)
+        util.resetTime(self)
+        self.faker = Faker('en_US')
+        self.faker.seed(66410)
+        self.test_daemon = get_test_daemon()
+        self.test_daemon.lbry_file_manager = mock.Mock(spec=EncryptedFileManager)
+        self.test_daemon.lbry_file_manager.lbry_files = self._get_fake_lbry_files()
+
+        # Pre-sorted lists of prices and file names in ascending order produced by
+        # faker with seed 66410. This seed was chosen becacuse it produces 3 results
+        # 'points_paid' at 6.0 and 2 results at 4.5 to test multiple sort criteria.
+        self.test_points_paid = [0.2, 2.9, 4.5, 4.5, 6.0, 6.0, 6.0, 6.8, 7.1, 9.2]
+        self.test_file_names = ['also.mp3', 'better.css', 'call.mp3', 'pay.jpg',
+                                'record.pages', 'sell.css', 'strategy.pages',
+                                'thousand.pages', 'town.mov', 'vote.ppt']
+        self.test_authors = ['angela41', 'edward70', 'fhart', 'johnrosales',
+                             'lucasfowler', 'peggytorres', 'qmitchell',
+                             'trevoranderson', 'xmitchell', 'zhangsusan']
+
+    def test_sort_by_points_paid_no_direction_specified(self):
+        sort_options = ['points_paid']
+        deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
+        file_list = self.successResultOf(deferred)
+        self.assertEquals(self.test_points_paid, [f['points_paid'] for f in file_list])
+
+    def test_sort_by_points_paid_ascending(self):
+        sort_options = ['points_paid,asc']
+        deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
+        file_list = self.successResultOf(deferred)
+        self.assertEquals(self.test_points_paid, [f['points_paid'] for f in file_list])
+
+    def test_sort_by_points_paid_descending(self):
+        sort_options = ['points_paid, desc']
+        deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
+        file_list = self.successResultOf(deferred)
+        self.assertEquals(list(reversed(self.test_points_paid)), [f['points_paid'] for f in file_list])
+
+    def test_sort_by_file_name_no_direction_specified(self):
+        sort_options = ['file_name']
+        deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
+        file_list = self.successResultOf(deferred)
+        self.assertEquals(self.test_file_names, [f['file_name'] for f in file_list])
+
+    def test_sort_by_file_name_ascending(self):
+        sort_options = ['file_name,\nasc']
+        deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
+        file_list = self.successResultOf(deferred)
+        self.assertEquals(self.test_file_names, [f['file_name'] for f in file_list])
+
+    def test_sort_by_file_name_descending(self):
+        sort_options = ['\tfile_name,\n\tdesc']
+        deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
+        file_list = self.successResultOf(deferred)
+        self.assertEquals(list(reversed(self.test_file_names)), [f['file_name'] for f in file_list])
+
+    def test_sort_by_multiple_criteria(self):
+        expected = ['file_name=record.pages, points_paid=9.2',
+                     'file_name=vote.ppt, points_paid=7.1',
+                     'file_name=strategy.pages, points_paid=6.8',
+                     'file_name=also.mp3, points_paid=6.0',
+                     'file_name=better.css, points_paid=6.0',
+                     'file_name=town.mov, points_paid=6.0',
+                     'file_name=sell.css, points_paid=4.5',
+                     'file_name=thousand.pages, points_paid=4.5',
+                     'file_name=call.mp3, points_paid=2.9',
+                     'file_name=pay.jpg, points_paid=0.2']
+        format_result = lambda f: 'file_name={}, points_paid={}'.format(f['file_name'], f['points_paid'])
+
+        sort_options = ['file_name,asc', 'points_paid,desc']
+        deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
+        file_list = self.successResultOf(deferred)
+        self.assertEquals(expected, map(format_result, file_list))
+
+        # Check that the list is not sorted as expected when sorted only by file_name.
+        sort_options = ['file_name,asc']
+        deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
+        file_list = self.successResultOf(deferred)
+        self.assertNotEqual(expected, map(format_result, file_list))
+
+        # Check that the list is not sorted as expected when sorted only by points_paid.
+        sort_options = ['points_paid,desc']
+        deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
+        file_list = self.successResultOf(deferred)
+        self.assertNotEqual(expected, map(format_result, file_list))
+
+        # Check that the list is not sorted as expected when not sorted at all.
+        deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list)
+        file_list = self.successResultOf(deferred)
+        self.assertNotEqual(expected, map(format_result, file_list))
+
+    def test_sort_by_nested_field(self):
+        extract_authors = lambda file_list: [f['metadata']['author'] for f in file_list]
+
+        sort_options = ['metadata.author']
+        deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
+        file_list = self.successResultOf(deferred)
+        self.assertEquals(self.test_authors, extract_authors(file_list))
+
+        # Check that the list matches the expected in reverse when sorting in descending order.
+        sort_options = ['metadata.author,desc']
+        deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
+        file_list = self.successResultOf(deferred)
+        self.assertEquals(list(reversed(self.test_authors)), extract_authors(file_list))
+
+        # Check that the list is not sorted as expected when not sorted at all.
+        deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list)
+        file_list = self.successResultOf(deferred)
+        self.assertNotEqual(self.test_authors, extract_authors(file_list))
+
+    def test_invalid_sort_produces_meaningful_errors(self):
+        sort_options = ['meta.author']
+        deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
+        failure_assertion = self.assertFailure(deferred, Exception)
+        exception = self.successResultOf(failure_assertion)
+        expected_message = 'Failed to get "meta.author", key "meta" was not found.'
+        self.assertEquals(expected_message, exception.message)
+
+        sort_options = ['metadata.foo.bar']
+        deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
+        failure_assertion = self.assertFailure(deferred, Exception)
+        exception = self.successResultOf(failure_assertion)
+        expected_message = 'Failed to get "metadata.foo.bar", key "foo" was not found.'
+        self.assertEquals(expected_message, exception.message)
+
+    def _get_fake_lbry_files(self):
+        return [self._get_fake_lbry_file() for _ in range(10)]
+
+    def _get_fake_lbry_file(self):
+        lbry_file = mock.Mock(spec=ManagedEncryptedFileDownloader)
+
+        file_path = self.faker.file_path()
+        stream_name = self.faker.file_name()
+        channel_claim_id = self.faker.sha1()
+        channel_name = self.faker.simple_profile()['username']
+        faked_attributes = {
+            'channel_claim_id': channel_claim_id,
+            'channel_name': '@' + channel_name,
+            'claim_id': self.faker.sha1(),
+            'claim_name': '-'.join(self.faker.words(4)),
+            'completed': self.faker.boolean(),
+            'download_directory': path.dirname(file_path),
+            'download_path': file_path,
+            'file_name': path.basename(file_path),
+            'key': self.faker.md5(),
+            'metadata': {
+                'author': channel_name,
+                'nsfw': random.randint(0, 1) == 1,
+            },
+            'mime_type': self.faker.mime_type(),
+            'nout': abs(self.faker.pyint()),
+            'outpoint': self.faker.md5() + self.faker.md5(),
+            'points_paid': self.faker.pyfloat(left_digits=1, right_digits=1, positive=True),
+            'sd_hash': self.faker.md5() + self.faker.md5() + self.faker.md5(),
+            'stopped': self.faker.boolean(),
+            'stream_hash': self.faker.md5() + self.faker.md5() + self.faker.md5(),
+            'stream_name': stream_name,
+            'suggested_file_name': stream_name,
+            'txid': self.faker.md5() + self.faker.md5(),
+            'written_bytes': self.faker.pyint(),
+        }
+
+        for key in faked_attributes:
+            setattr(lbry_file, key, faked_attributes[key])
+
+        return lbry_file

requirements_testing.txt

@@ -0,0 +1,2 @@
+mock>=2.0,<3.0
+Faker>=0.8,<1.0