Madiator2011/lbry-sdk
1
0
Fork 0
forked from LBRYCommunity/lbry-sdk
Code Pull requests Activity

refactor file_list sort to allow sorting by any field

Browse source
This commit is contained in:
kafene 2018-05-07 17:37:41 -04:00
parent 72bae90e06
commit 5069351287
2 changed files with 129 additions and 91 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
lbrynet/daemon/Daemon.py
View file

@ -9,6 +9,7 @@ import json
import textwrap import textwrap
import signal import signal
from copy import deepcopy from copy import deepcopy
from operator import attrgetter
from twisted.web import server from twisted.web import server
from twisted.internet import defer, threads, error, reactor from twisted.internet import defer, threads, error, reactor
from twisted.internet.task import LoopingCall from twisted.internet.task import LoopingCall
@ -97,10 +98,6 @@ CONNECTION_MESSAGES = {
SHORT_ID_LEN = 20 SHORT_ID_LEN = 20
MAX_UPDATE_FEE_ESTIMATE = 0.3 MAX_UPDATE_FEE_ESTIMATE = 0.3
FILE_SORT_FIELD_NAME = 'name'
FILE_SORT_FIELD_DATE = 'date'
FILE_SORT_FIELD_PRICE = 'price'
FILE_SORT_DIRECTION_ASCENDING = 'asc' FILE_SORT_DIRECTION_ASCENDING = 'asc'
FILE_SORT_DIRECTION_DESCENDING = 'desc' FILE_SORT_DIRECTION_DESCENDING = 'desc'
FILE_SORT_DIRECTIONS = ( FILE_SORT_DIRECTIONS = (
@ -935,13 +932,11 @@ class Daemon(AuthJSONRPCServer):
defer.returnValue(lbry_file) defer.returnValue(lbry_file)
@defer.inlineCallbacks @defer.inlineCallbacks
def _get_lbry_files(self, return_json=False, full_status=True, sort_by=None, **kwargs): def _get_lbry_files(self, return_json=False, full_status=True, **kwargs):
lbry_files = list(self.lbry_file_manager.lbry_files) lbry_files = list(self.lbry_file_manager.lbry_files)
if kwargs: if kwargs:
for search_type, value in iter_lbry_file_search_values(kwargs): for search_type, value in iter_lbry_file_search_values(kwargs):
lbry_files = [l_f for l_f in lbry_files if l_f.__dict__[search_type] == value] lbry_files = [l_f for l_f in lbry_files if l_f.__dict__[search_type] == value]
if sort_by:
lbry_files = self._sort_lbry_files(lbry_files, sort_by)
if return_json: if return_json:
file_dicts = [] file_dicts = []
for lbry_file in lbry_files: for lbry_file in lbry_files:
@ -954,25 +949,29 @@ class Daemon(AuthJSONRPCServer):
def _sort_lbry_files(self, lbry_files, sort_by): def _sort_lbry_files(self, lbry_files, sort_by):
for field, direction in sort_by: for field, direction in sort_by:
is_reverse = direction == FILE_SORT_DIRECTION_DESCENDING is_reverse = direction == FILE_SORT_DIRECTION_DESCENDING
if field == FILE_SORT_FIELD_NAME: key_getter = None
lbry_files = sorted(lbry_files, key=lambda f: f.file_name, reverse=is_reverse) if field:
elif field == FILE_SORT_FIELD_DATE: search_path = field.split('.')
lbry_files = sorted(lbry_files, reverse=is_reverse) def key_getter(value):
elif field == FILE_SORT_FIELD_PRICE: for key in search_path:
lbry_files = sorted(lbry_files, key=lambda f: f.points_paid, reverse=is_reverse) try:
else: value = value[key]
raise Exception('Unrecognized sort field "{}"'.format(field)) except KeyError as e:
errmsg = 'Failed to sort by "{}", key "{}" was not found.'
raise Exception(errmsg.format(field, e.message))
return value
lbry_files = sorted(lbry_files, key=key_getter, reverse=is_reverse)
return lbry_files return lbry_files
def _parse_lbry_files_sort(self, sort): def _parse_lbry_files_sort(self, sort):
""" """
Given a sort string like 'name, desc' or 'price', Given a sort string like 'file_name, desc' or 'points_paid',
parse the string into a tuple of (field, direction). parse the string into a tuple of (field, direction).
Direction defaults to ascending. Direction defaults to ascending.
""" """
pieces = sort.rsplit(',', 1) pieces = sort.rsplit(',', 1)
field = pieces[0].strip() field = pieces[0].strip() or None
direction = pieces[1].strip().lower() if len(pieces) > 1 else None direction = pieces[1].strip().lower() if len(pieces) > 1 else None
if direction and direction not in FILE_SORT_DIRECTIONS: if direction and direction not in FILE_SORT_DIRECTIONS:
raise Exception('Sort direction must be one of {}'.format(FILE_SORT_DIRECTIONS)) raise Exception('Sort direction must be one of {}'.format(FILE_SORT_DIRECTIONS))
@ -1422,8 +1421,9 @@ class Daemon(AuthJSONRPCServer):
--claim_name=<claim_name> : (str) get file with matching claim name --claim_name=<claim_name> : (str) get file with matching claim name
--full_status : (bool) full status, populate the --full_status : (bool) full status, populate the
'message' and 'size' fields 'message' and 'size' fields
--sort=<sort_method> : (str) sort by any of 'name', 'date', or 'price' --sort=<sort_method> : (str) sort by any property, like 'file_name'
to specify direction append ',asc' or ',desc' or 'metadata.author'; to specify direction
append ',asc' or ',desc'
Returns: Returns:
(list) List of files (list) List of files
@ -1459,8 +1459,10 @@ class Daemon(AuthJSONRPCServer):
] ]
""" """
sort_by = [self._parse_lbry_files_sort(s) for s in sort] if sort else None result = yield self._get_lbry_files(return_json=True, **kwargs)
result = yield self._get_lbry_files(return_json=True, sort_by=sort_by, **kwargs) if sort:
sort_by = [self._parse_lbry_files_sort(s) for s in sort]
result = self._sort_lbry_files(result, sort_by)
response = yield self._render_response(result) response = yield self._render_response(result)
defer.returnValue(response) defer.returnValue(response)

176
lbrynet/tests/unit/lbrynet_daemon/test_Daemon.py
View file

@ -1,6 +1,7 @@
import mock import mock
import json import json
import unittest import unittest
import random
from os import path from os import path
from twisted.internet import defer from twisted.internet import defer
@ -146,88 +147,118 @@ class TestFileListSorting(trial.unittest.TestCase):
# Pre-sorted lists of prices and file names in ascending order produced by # Pre-sorted lists of prices and file names in ascending order produced by
# faker with seed 66410. This seed was chosen becacuse it produces 3 results # faker with seed 66410. This seed was chosen becacuse it produces 3 results
# 'points_paid' at 6.0 and 2 results at 4.5 to test multiple sort criteria. # 'points_paid' at 6.0 and 2 results at 4.5 to test multiple sort criteria.
self.test_prices = [0.2, 2.9, 4.5, 4.5, 6.0, 6.0, 6.0, 6.8, 7.1, 9.2] self.test_points_paid = [0.2, 2.9, 4.5, 4.5, 6.0, 6.0, 6.0, 6.8, 7.1, 9.2]
self.test_file_names = ['also.mp3', 'better.css', 'call.mp3', 'pay.jpg', self.test_file_names = ['also.mp3', 'better.css', 'call.mp3', 'pay.jpg',
'record.pages', 'sell.css', 'strategy.pages', 'record.pages', 'sell.css', 'strategy.pages',
'thousand.pages', 'town.mov', 'vote.ppt'] 'thousand.pages', 'town.mov', 'vote.ppt']
self.test_authors = ['angela41', 'edward70', 'fhart', 'johnrosales',
'lucasfowler', 'peggytorres', 'qmitchell',
'trevoranderson', 'xmitchell', 'zhangsusan']
@defer.inlineCallbacks def test_sort_by_points_paid_no_direction_specified(self):
def test_sort_by_price_no_direction_specified(self): sort_options = ['points_paid']
sort_options = ['price'] deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options) file_list = self.successResultOf(deferred)
received = [f['points_paid'] for f in file_list] self.assertEquals(self.test_points_paid, [f['points_paid'] for f in file_list])
self.assertEquals(self.test_prices, received)
@defer.inlineCallbacks def test_sort_by_points_paid_ascending(self):
def test_sort_by_price_ascending(self): sort_options = ['points_paid,asc']
sort_options = ['price,asc'] deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options) file_list = self.successResultOf(deferred)
received = [f['points_paid'] for f in file_list] self.assertEquals(self.test_points_paid, [f['points_paid'] for f in file_list])
self.assertEquals(self.test_prices, received)
@defer.inlineCallbacks def test_sort_by_points_paid_descending(self):
def test_sort_by_price_descending(self): sort_options = ['points_paid, desc']
sort_options = ['price, desc'] deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options) file_list = self.successResultOf(deferred)
received = [f['points_paid'] for f in file_list] self.assertEquals(list(reversed(self.test_points_paid)), [f['points_paid'] for f in file_list])
expected = list(reversed(self.test_prices))
self.assertEquals(expected, received)
@defer.inlineCallbacks def test_sort_by_file_name_no_direction_specified(self):
def test_sort_by_name_no_direction_specified(self): sort_options = ['file_name']
sort_options = ['name'] deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options) file_list = self.successResultOf(deferred)
received = [f['file_name'] for f in file_list] self.assertEquals(self.test_file_names, [f['file_name'] for f in file_list])
self.assertEquals(self.test_file_names, received)
@defer.inlineCallbacks def test_sort_by_file_name_ascending(self):
def test_sort_by_name_ascending(self): sort_options = ['file_name,\nasc']
sort_options = ['name,\nasc'] deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options) file_list = self.successResultOf(deferred)
received = [f['file_name'] for f in file_list] self.assertEquals(self.test_file_names, [f['file_name'] for f in file_list])
self.assertEquals(self.test_file_names, received)
@defer.inlineCallbacks def test_sort_by_file_name_descending(self):
def test_sort_by_name_descending(self): sort_options = ['\tfile_name,\n\tdesc']
sort_options = ['\tname,\n\tdesc'] deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options) file_list = self.successResultOf(deferred)
received = [f['file_name'] for f in file_list] self.assertEquals(list(reversed(self.test_file_names)), [f['file_name'] for f in file_list])
expected = list(reversed(self.test_file_names))
self.assertEquals(expected, received)
@defer.inlineCallbacks
def test_sort_by_multiple_criteria(self): def test_sort_by_multiple_criteria(self):
sort_options = ['name,asc', 'price,desc'] expected = ['file_name=record.pages, points_paid=9.2',
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options) 'file_name=vote.ppt, points_paid=7.1',
received = ['name={}, price={}'.format(f['file_name'], f['points_paid']) for f in file_list] 'file_name=strategy.pages, points_paid=6.8',
expected = ['name=record.pages, price=9.2', 'file_name=also.mp3, points_paid=6.0',
'name=vote.ppt, price=7.1', 'file_name=better.css, points_paid=6.0',
'name=strategy.pages, price=6.8', 'file_name=town.mov, points_paid=6.0',
'name=also.mp3, price=6.0', 'file_name=sell.css, points_paid=4.5',
'name=better.css, price=6.0', 'file_name=thousand.pages, points_paid=4.5',
'name=town.mov, price=6.0', 'file_name=call.mp3, points_paid=2.9',
'name=sell.css, price=4.5', 'file_name=pay.jpg, points_paid=0.2']
'name=thousand.pages, price=4.5', format_result = lambda f: 'file_name={}, points_paid={}'.format(f['file_name'], f['points_paid'])
'name=call.mp3, price=2.9',
'name=pay.jpg, price=0.2']
self.assertEquals(expected, received)
# Check that the list is not sorted as expected when sorted only by name. sort_options = ['file_name,asc', 'points_paid,desc']
sort_options = ['name,asc'] deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options) file_list = self.successResultOf(deferred)
received = ['name={}, price={}'.format(f['file_name'], f['points_paid']) for f in file_list] self.assertEquals(expected, map(format_result, file_list))
self.assertNotEqual(expected, received)
# Check that the list is not sorted as expected when sorted only by price. # Check that the list is not sorted as expected when sorted only by file_name.
sort_options = ['price,desc'] sort_options = ['file_name,asc']
file_list = yield self.test_daemon.jsonrpc_file_list(sort=sort_options) deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
received = ['name={}, price={}'.format(f['file_name'], f['points_paid']) for f in file_list] file_list = self.successResultOf(deferred)
self.assertNotEqual(expected, received) self.assertNotEqual(expected, map(format_result, file_list))
# Check that the list is not sorted as expected when sorted only by points_paid.
sort_options = ['points_paid,desc']
deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
file_list = self.successResultOf(deferred)
self.assertNotEqual(expected, map(format_result, file_list))
# Check that the list is not sorted as expected when not sorted at all. # Check that the list is not sorted as expected when not sorted at all.
file_list = yield self.test_daemon.jsonrpc_file_list() deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list)
received = ['name={}, price={}'.format(f['file_name'], f['points_paid']) for f in file_list] file_list = self.successResultOf(deferred)
self.assertNotEqual(expected, received) self.assertNotEqual(expected, map(format_result, file_list))
def test_sort_by_nested_field(self):
extract_authors = lambda file_list: [f['metadata']['author'] for f in file_list]
sort_options = ['metadata.author']
deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
file_list = self.successResultOf(deferred)
self.assertEquals(self.test_authors, extract_authors(file_list))
# Check that the list matches the expected in reverse when sorting in descending order.
sort_options = ['metadata.author,desc']
deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
file_list = self.successResultOf(deferred)
self.assertEquals(list(reversed(self.test_authors)), extract_authors(file_list))
# Check that the list is not sorted as expected when not sorted at all.
deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list)
file_list = self.successResultOf(deferred)
self.assertNotEqual(self.test_authors, extract_authors(file_list))
def test_invalid_sort_produces_meaningful_errors(self):
sort_options = ['meta.author']
deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
failure_assertion = self.assertFailure(deferred, Exception)
exception = self.successResultOf(failure_assertion)
expected_message = 'Failed to sort by "meta.author", key "meta" was not found.'
self.assertEquals(expected_message, exception.message)
sort_options = ['metadata.foo.bar']
deferred = defer.maybeDeferred(self.test_daemon.jsonrpc_file_list, sort=sort_options)
failure_assertion = self.assertFailure(deferred, Exception)
exception = self.successResultOf(failure_assertion)
expected_message = 'Failed to sort by "metadata.foo.bar", key "foo" was not found.'
self.assertEquals(expected_message, exception.message)
def _get_fake_lbry_files(self): def _get_fake_lbry_files(self):
return [self._get_fake_lbry_file() for _ in range(10)] return [self._get_fake_lbry_file() for _ in range(10)]
@ -237,9 +268,11 @@ class TestFileListSorting(trial.unittest.TestCase):
file_path = self.faker.file_path() file_path = self.faker.file_path()
stream_name = self.faker.file_name() stream_name = self.faker.file_name()
channel_claim_id = self.faker.sha1()
channel_name = self.faker.simple_profile()['username']
faked_attributes = { faked_attributes = {
'channel_claim_id': self.faker.sha1(), 'channel_claim_id': channel_claim_id,
'channel_name': '@' + self.faker.simple_profile()['username'], 'channel_name': '@' + channel_name,
'claim_id': self.faker.sha1(), 'claim_id': self.faker.sha1(),
'claim_name': '-'.join(self.faker.words(4)), 'claim_name': '-'.join(self.faker.words(4)),
'completed': self.faker.boolean(), 'completed': self.faker.boolean(),
@ -247,7 +280,10 @@ class TestFileListSorting(trial.unittest.TestCase):
'download_path': file_path, 'download_path': file_path,
'file_name': path.basename(file_path), 'file_name': path.basename(file_path),
'key': self.faker.md5(), 'key': self.faker.md5(),
'metadata': {}, 'metadata': {
'author': channel_name,
'nsfw': random.randint(0, 1) == 1,
},
'mime_type': self.faker.mime_type(), 'mime_type': self.faker.mime_type(),
'nout': abs(self.faker.pyint()), 'nout': abs(self.faker.pyint()),
'outpoint': self.faker.md5() + self.faker.md5(), 'outpoint': self.faker.md5() + self.faker.md5(),