LBRYCommunity/lbry-sdk
1
0
Fork 1
Code Issues Pull requests Projects Releases 282 Packages Wiki Activity

refactor file_list sort to allow sorting by any field

Browse source
This commit is contained in:
kafene 2018-05-07 17:37:41 -04:00
parent 72bae90e06
commit 5069351287
2 changed files with 129 additions and 91 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
lbrynet/daemon/Daemon.py
View file

@ -9,6 +9,7 @@ import json
import textwrap
import signal
from copy import deepcopy
from operator import attrgetter
from twisted.web import server
from twisted.internet import defer, threads, error, reactor
from twisted.internet.task import LoopingCall
@ -97,10 +98,6 @@ CONNECTION_MESSAGES = {
SHORT_ID_LEN = 20
MAX_UPDATE_FEE_ESTIMATE = 0.3
FILE_SORT_FIELD_NAME = 'name'
FILE_SORT_FIELD_DATE = 'date'
FILE_SORT_FIELD_PRICE = 'price'
FILE_SORT_DIRECTION_ASCENDING = 'asc'
FILE_SORT_DIRECTION_DESCENDING = 'desc'
FILE_SORT_DIRECTIONS = (
@ -935,13 +932,11 @@ class Daemon(AuthJSONRPCServer):
defer.returnValue(lbry_file)
@defer.inlineCallbacks
def _get_lbry_files(self, return_json=False, full_status=True, sort_by=None, **kwargs):
def _get_lbry_files(self, return_json=False, full_status=True, **kwargs):
lbry_files = list(self.lbry_file_manager.lbry_files)
if kwargs:
for search_type, value in iter_lbry_file_search_values(kwargs):
lbry_files = [l_f for l_f in lbry_files if l_f.__dict__[search_type] == value]
if sort_by:
lbry_files = self._sort_lbry_files(lbry_files, sort_by)
if return_json:
file_dicts = []
for lbry_file in lbry_files:
@ -954,25 +949,29 @@ class Daemon(AuthJSONRPCServer):
def _sort_lbry_files(self, lbry_files, sort_by):
for field, direction in sort_by:
is_reverse = direction == FILE_SORT_DIRECTION_DESCENDING
if field == FILE_SORT_FIELD_NAME:
lbry_files = sorted(lbry_files, key=lambda f: f.file_name, reverse=is_reverse)
elif field == FILE_SORT_FIELD_DATE:
lbry_files = sorted(lbry_files, reverse=is_reverse)
elif field == FILE_SORT_FIELD_PRICE:
lbry_files = sorted(lbry_files, key=lambda f: f.points_paid, reverse=is_reverse)
else:
raise Exception('Unrecognized sort field "{}"'.format(field))
key_getter = None
if field:
search_path = field.split('.')
def key_getter(value):
for key in search_path:
try:
value = value[key]
except KeyError as e:
errmsg = 'Failed to sort by "{}", key "{}" was not found.'
raise Exception(errmsg.format(field, e.message))
return value
lbry_files = sorted(lbry_files, key=key_getter, reverse=is_reverse)
return lbry_files
def _parse_lbry_files_sort(self, sort):
"""
Given a sort string like 'name, desc' or 'price',
Given a sort string like 'file_name, desc' or 'points_paid',
parse the string into a tuple of (field, direction).
Direction defaults to ascending.
"""
pieces = sort.rsplit(',', 1)
field = pieces[0].strip()
field = pieces[0].strip() or None
direction = pieces[1].strip().lower() if len(pieces) > 1 else None
if direction and direction not in FILE_SORT_DIRECTIONS:
raise Exception('Sort direction must be one of {}'.format(FILE_SORT_DIRECTIONS))
@ -1422,8 +1421,9 @@ class Daemon(AuthJSONRPCServer):
--claim_name=<claim_name> : (str) get file with matching claim name
--full_status : (bool) full status, populate the
'message' and 'size' fields
--sort=<sort_method> : (str) sort by any of 'name', 'date', or 'price'
to specify direction append ',asc' or ',desc'
--sort=<sort_method> : (str) sort by any property, like 'file_name'
or 'metadata.author'; to specify direction
append ',asc' or ',desc'
Returns:
(list) List of files
@ -1459,8 +1459,10 @@ class Daemon(AuthJSONRPCServer):
]
"""
sort_by = [self._parse_lbry_files_sort(s) for s in sort] if sort else None
result = yield self._get_lbry_files(return_json=True, sort_by=sort_by, **kwargs)
result = yield self._get_lbry_files(return_json=True, **kwargs)
if sort:
sort_by = [self._parse_lbry_files_sort(s) for s in sort]
result = self._sort_lbry_files(result, sort_by)
response = yield self._render_response(result)
defer.returnValue(response)

176
lbrynet/tests/unit/lbrynet_daemon/test_Daemon.py
View file

@ -1,6 +1,7 @@
import mock
import json
import unittest
import random
from os import path
from twisted.internet import defer
@ -146,88 +147,118 @@ class TestFileListSorting(trial.unittest.TestCase):
# Pre-sorted lists of prices and file names in ascending order produced by
# faker with seed 66410. This seed was chosen becacuse it produces 3 results
# 'points_paid' at 6.0 and 2 results at 4.5 to test multiple sort criteria.
self.test_prices = [0.2, 2.9, 4.5, 4.5, 6.0, 6.0, 6.0, 6.8, 7.1, 9.2]
self.test_points_paid = [0.2, 2.9, 4.5, 4.5, 6.0, 6.0, 6.0, 6.8, 7.1, 9.2]
self.test_file_names = ['also.mp3', 'better.css', 'call.mp3', 'pay.jpg',
'record.pages', 'sell.css', 'strategy.pages',
'thousand.pages', 'town.mov', 'vote.ppt']
self.test_authors = ['angela41', 'edward70', 'fhart', 'johnrosales',
'lucasfowler', 'peggytorres', 'qmitchell',
'trevoranderson', 'xmitchell', 'zhangsusan']
@defer.inlineCallbacks
def test_sort_by_price_no_direction_specified(self):
sort_options = ['price']
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options)
received = [f['points_paid'] for f in file_list]
self.assertEquals(self.test_prices, received)
def test_sort_by_points_paid_no_direction_specified(self):
sort_options = ['points_paid']
deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
file_list = self.successResultOf(deferred)
self.assertEquals(self.test_points_paid, [f['points_paid'] for f in file_list])
@defer.inlineCallbacks
def test_sort_by_price_ascending(self):
sort_options = ['price,asc']
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options)
received = [f['points_paid'] for f in file_list]
self.assertEquals(self.test_prices, received)
def test_sort_by_points_paid_ascending(self):
sort_options = ['points_paid,asc']
deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
file_list = self.successResultOf(deferred)
self.assertEquals(self.test_points_paid, [f['points_paid'] for f in file_list])
@defer.inlineCallbacks
def test_sort_by_price_descending(self):
sort_options = ['price, desc']
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options)
received = [f['points_paid'] for f in file_list]
expected = list(reversed(self.test_prices))
self.assertEquals(expected, received)
def test_sort_by_points_paid_descending(self):
sort_options = ['points_paid, desc']
deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
file_list = self.successResultOf(deferred)
self.assertEquals(list(reversed(self.test_points_paid)), [f['points_paid'] for f in file_list])
@defer.inlineCallbacks
def test_sort_by_name_no_direction_specified(self):
sort_options = ['name']
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options)
received = [f['file_name'] for f in file_list]
self.assertEquals(self.test_file_names, received)
def test_sort_by_file_name_no_direction_specified(self):
sort_options = ['file_name']
deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
file_list = self.successResultOf(deferred)
self.assertEquals(self.test_file_names, [f['file_name'] for f in file_list])
@defer.inlineCallbacks
def test_sort_by_name_ascending(self):
sort_options = ['name,\nasc']
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options)
received = [f['file_name'] for f in file_list]
self.assertEquals(self.test_file_names, received)
def test_sort_by_file_name_ascending(self):
sort_options = ['file_name,\nasc']
deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
file_list = self.successResultOf(deferred)
self.assertEquals(self.test_file_names, [f['file_name'] for f in file_list])
@defer.inlineCallbacks
def test_sort_by_name_descending(self):
sort_options = ['\tname,\n\tdesc']
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options)
received = [f['file_name'] for f in file_list]
expected = list(reversed(self.test_file_names))
self.assertEquals(expected, received)
def test_sort_by_file_name_descending(self):
sort_options = ['\tfile_name,\n\tdesc']
deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
file_list = self.successResultOf(deferred)
self.assertEquals(list(reversed(self.test_file_names)), [f['file_name'] for f in file_list])
@defer.inlineCallbacks
def test_sort_by_multiple_criteria(self):
sort_options = ['name,asc', 'price,desc']
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options)
received = ['name={}, price={}'.format(f['file_name'], f['points_paid']) for f in file_list]
expected = ['name=record.pages, price=9.2',
'name=vote.ppt, price=7.1',
'name=strategy.pages, price=6.8',
'name=also.mp3, price=6.0',
'name=better.css, price=6.0',
'name=town.mov, price=6.0',
'name=sell.css, price=4.5',
'name=thousand.pages, price=4.5',
'name=call.mp3, price=2.9',
'name=pay.jpg, price=0.2']
self.assertEquals(expected, received)
expected = ['file_name=record.pages, points_paid=9.2',
'file_name=vote.ppt, points_paid=7.1',
'file_name=strategy.pages, points_paid=6.8',
'file_name=also.mp3, points_paid=6.0',
'file_name=better.css, points_paid=6.0',
'file_name=town.mov, points_paid=6.0',
'file_name=sell.css, points_paid=4.5',
'file_name=thousand.pages, points_paid=4.5',
'file_name=call.mp3, points_paid=2.9',
'file_name=pay.jpg, points_paid=0.2']
format_result = lambda f: 'file_name={}, points_paid={}'.format(f['file_name'], f['points_paid'])
# Check that the list is not sorted as expected when sorted only by name.
sort_options = ['name,asc']
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options)
received = ['name={}, price={}'.format(f['file_name'], f['points_paid']) for f in file_list]
self.assertNotEqual(expected, received)
sort_options = ['file_name,asc', 'points_paid,desc']
deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
file_list = self.successResultOf(deferred)
self.assertEquals(expected, map(format_result, file_list))
# Check that the list is not sorted as expected when sorted only by price.
sort_options = ['price,desc']
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options)
received = ['name={}, price={}'.format(f['file_name'], f['points_paid']) for f in file_list]
self.assertNotEqual(expected, received)
# Check that the list is not sorted as expected when sorted only by file_name.
sort_options = ['file_name,asc']
deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
file_list = self.successResultOf(deferred)
self.assertNotEqual(expected, map(format_result, file_list))
# Check that the list is not sorted as expected when sorted only by points_paid.
sort_options = ['points_paid,desc']
deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
file_list = self.successResultOf(deferred)
self.assertNotEqual(expected, map(format_result, file_list))
# Check that the list is not sorted as expected when not sorted at all.
file_list = yield self.test_daemon.jsonrpc_file_list()
received = ['name={}, price={}'.format(f['file_name'], f['points_paid']) for f in file_list]
self.assertNotEqual(expected, received)
deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list)
file_list = self.successResultOf(deferred)
self.assertNotEqual(expected, map(format_result, file_list))
def test_sort_by_nested_field(self):
extract_authors = lambda file_list: [f['metadata']['author'] for f in file_list]
sort_options = ['metadata.author']
deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
file_list = self.successResultOf(deferred)
self.assertEquals(self.test_authors, extract_authors(file_list))
# Check that the list matches the expected in reverse when sorting in descending order.
sort_options = ['metadata.author,desc']
deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
file_list = self.successResultOf(deferred)
self.assertEquals(list(reversed(self.test_authors)), extract_authors(file_list))
# Check that the list is not sorted as expected when not sorted at all.
deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list)
file_list = self.successResultOf(deferred)
self.assertNotEqual(self.test_authors, extract_authors(file_list))
def test_invalid_sort_produces_meaningful_errors(self):
sort_options = ['meta.author']
deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
failure_assertion = self.assertFailure(deferred, Exception)
exception = self.successResultOf(failure_assertion)
expected_message = 'Failed to sort by "meta.author", key "meta" was not found.'
self.assertEquals(expected_message, exception.message)
sort_options = ['metadata.foo.bar']
deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
failure_assertion = self.assertFailure(deferred, Exception)
exception = self.successResultOf(failure_assertion)
expected_message = 'Failed to sort by "metadata.foo.bar", key "foo" was not found.'
self.assertEquals(expected_message, exception.message)
def _get_fake_lbry_files(self):
return [self._get_fake_lbry_file() for _ in range(10)]
@ -237,9 +268,11 @@ class TestFileListSorting(trial.unittest.TestCase):
file_path = self.faker.file_path()
stream_name = self.faker.file_name()
channel_claim_id = self.faker.sha1()
channel_name = self.faker.simple_profile()['username']
faked_attributes = {
'channel_claim_id': self.faker.sha1(),
'channel_name': '@' + self.faker.simple_profile()['username'],
'channel_claim_id': channel_claim_id,
'channel_name': '@' + channel_name,
'claim_id': self.faker.sha1(),
'claim_name': '-'.join(self.faker.words(4)),
'completed': self.faker.boolean(),
@ -247,7 +280,10 @@ class TestFileListSorting(trial.unittest.TestCase):
'download_path': file_path,
'file_name': path.basename(file_path),
'key': self.faker.md5(),
'metadata': {},
'metadata': {
'author': channel_name,
'nsfw': random.randint(0, 1) == 1,
},
'mime_type': self.faker.mime_type(),
'nout': abs(self.faker.pyint()),
'outpoint': self.faker.md5() + self.faker.md5(),