From b16402e094a4f99b2f41a57d05aeaf69d5c13999 Mon Sep 17 00:00:00 2001 From: jleute Date: Thu, 17 May 2018 23:41:23 +0200 Subject: [PATCH 01/29] fixes #1109 --- CHANGELOG.md | 1 + lbrynet/conf.py | 27 +++++++++++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b94d5fd44..f149688d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ at anytime. * `use_auth_http` in a config file being overridden by the default command line argument to `lbrynet-daemon`, now the command line value will only override the config file value if it is provided * `lbrynet-cli` not automatically switching to the authenticated client if the server is detected to be using authentication. This resulted in `lbrynet-cli` failing to run when `lbrynet-daemon` was run with the `--http-auth` flag * fixed error when using `claim_show` with `txid` and `nout` arguments + * fixed error when saving server list to conf file (issue #1109) ### Deprecated * diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 57dad2850..1d5d8972d 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -168,6 +168,8 @@ def server_port(server_and_port): def server_list(servers): return [server_port(server) for server in servers] +def server_list_reverse(servers): + return ["%s:%s" % (server, port) for server, port in servers] class Env(envparse.Env): """An Env parser that automatically namespaces the variables with LBRY""" @@ -267,7 +269,7 @@ ADJUSTABLE_SETTINGS = { 'is_generous_host': (bool, True), 'announce_head_blobs_only': (bool, True), 'concurrent_announcers': (int, DEFAULT_CONCURRENT_ANNOUNCERS), - 'known_dht_nodes': (list, DEFAULT_DHT_NODES, server_list), + 'known_dht_nodes': (list, DEFAULT_DHT_NODES, server_list, server_list_reverse), 'lbryum_wallet_dir': (str, default_lbryum_dir), 'max_connections_per_stream': (int, 5), 'seek_head_blob_first': (bool, True), @@ -285,7 +287,7 @@ ADJUSTABLE_SETTINGS = { # event the initial upload failed or was disconnected part way through, provided the auto_re_reflect_interval > 0) 'reflect_uploads': (bool, True), 'auto_re_reflect_interval': (int, 86400), # set to 0 to disable - 'reflector_servers': (list, [('reflector2.lbry.io', 5566)], server_list), + 'reflector_servers': (list, [('reflector2.lbry.io', 5566)], server_list, server_list_reverse), 'run_reflector_server': (bool, False), 'sd_download_timeout': (int, 3), 'share_usage_data': (bool, True), # whether to share usage stats and diagnostic info with LBRY @@ -295,7 +297,8 @@ ADJUSTABLE_SETTINGS = { 'use_keyring': (bool, False), 'wallet': (str, LBRYUM_WALLET), 'blockchain_name': (str, 'lbrycrd_main'), - 'lbryum_servers': (list, [('lbryum8.lbry.io', 50001), ('lbryum9.lbry.io', 50001)], server_list), + 'lbryum_servers': (list, [('lbryum8.lbry.io', 50001), ('lbryum9.lbry.io', + 50001)], server_list, server_list_reverse), 's3_headers_depth': (int, 96 * 10) # download headers from s3 when the local height is more than 10 chunks behind } @@ -497,18 +500,30 @@ class Config(object): path = conf_file else: path = self.get_conf_filename() - + # reverse the conversions done after loading the settings from the conf + # file + rev = self._convert_conf_file_lists_reverse(self._data[TYPE_PERSISTED]) ext = os.path.splitext(path)[1] encoder = settings_encoders.get(ext, False) assert encoder is not False, 'Unknown settings format %s' % ext with open(path, 'w') as settings_file: - settings_file.write(encoder(self._data[TYPE_PERSISTED])) + settings_file.write(encoder(rev)) + + @staticmethod + def _convert_conf_file_lists_reverse(converted): + rev = {} + for k in converted.iterkeys(): + if k in ADJUSTABLE_SETTINGS and len(ADJUSTABLE_SETTINGS[k]) == 4: + rev[k] = ADJUSTABLE_SETTINGS[k][3](converted[k]) + else: + rev[k] = converted[k] + return rev @staticmethod def _convert_conf_file_lists(decoded): converted = {} for k, v in decoded.iteritems(): - if k in ADJUSTABLE_SETTINGS and len(ADJUSTABLE_SETTINGS[k]) == 3: + if k in ADJUSTABLE_SETTINGS and len(ADJUSTABLE_SETTINGS[k]) >= 3: converted[k] = ADJUSTABLE_SETTINGS[k][2](v) else: converted[k] = v From 348544428fa42da2ebc00b5b7b51d24fd8534978 Mon Sep 17 00:00:00 2001 From: jleute Date: Fri, 18 May 2018 02:54:40 +0200 Subject: [PATCH 02/29] added unit test for reversal of conversion of config file entries --- lbrynet/tests/unit/test_conf.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lbrynet/tests/unit/test_conf.py b/lbrynet/tests/unit/test_conf.py index c18ed1f83..3baa5f7ab 100644 --- a/lbrynet/tests/unit/test_conf.py +++ b/lbrynet/tests/unit/test_conf.py @@ -77,3 +77,18 @@ class SettingsTest(unittest.TestCase): self.assertEqual(str, type(conf.default_download_dir)) self.assertEqual(str, type(conf.default_data_dir)) self.assertEqual(str, type(conf.default_lbryum_dir)) + + def test_conversion_reversal(self): + # simulate decoding, conversion, conversion reversal and encoding of + # server list settings from the config file. + settings = self.get_mock_config_instance() + encoder = conf.settings_encoders['.yml'] + decoder = conf.settings_decoders['.yml'] + conf_file_entry = "lbryum_servers: ['localhost:5001', 'localhost:5002']" + decoded = decoder(conf_file_entry) + converted = settings._convert_conf_file_lists(decoded) + converted_reversed = settings._convert_conf_file_lists_reverse(converted) + encoded = encoder(converted_reversed) + self.assertEqual(conf_file_entry, encoded.strip()) + self.assertEqual(decoded, converted_reversed) + From b543c0457236ea3f8d2292e30807acc31e4dc1ad Mon Sep 17 00:00:00 2001 From: jleute Date: Sat, 19 May 2018 01:32:51 +0200 Subject: [PATCH 03/29] Updated unittest for saving of server lists in the conf file --- lbrynet/conf.py | 3 ++- lbrynet/tests/unit/test_conf.py | 29 ++++++++++++++++------------- lbrynet/tests/util.py | 4 ++++ 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 1d5d8972d..e902192fd 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -538,7 +538,6 @@ class Config(object): path = conf_file else: path = self.get_conf_filename() - ext = os.path.splitext(path)[1] decoder = settings_decoders.get(ext, False) assert decoder is not False, 'Unknown settings format %s' % ext @@ -645,6 +644,8 @@ def get_default_env(): for k, v in ADJUSTABLE_SETTINGS.iteritems(): if len(v) == 3: env_defaults[k] = (v[0], None, v[2]) + elif len(v) == 4: + env_defaults[k] = (v[0], None, v[2], v[3]) else: env_defaults[k] = (v[0], None) return Env(**env_defaults) diff --git a/lbrynet/tests/unit/test_conf.py b/lbrynet/tests/unit/test_conf.py index 3baa5f7ab..87d5c03fc 100644 --- a/lbrynet/tests/unit/test_conf.py +++ b/lbrynet/tests/unit/test_conf.py @@ -4,6 +4,8 @@ import json from twisted.trial import unittest from lbrynet import conf from lbrynet.core.Error import InvalidCurrencyError +from lbrynet.tests import mocks +from lbrynet.tests.util import create_conf_file class SettingsTest(unittest.TestCase): def setUp(self): @@ -78,17 +80,18 @@ class SettingsTest(unittest.TestCase): self.assertEqual(str, type(conf.default_data_dir)) self.assertEqual(str, type(conf.default_lbryum_dir)) - def test_conversion_reversal(self): - # simulate decoding, conversion, conversion reversal and encoding of - # server list settings from the config file. - settings = self.get_mock_config_instance() - encoder = conf.settings_encoders['.yml'] - decoder = conf.settings_decoders['.yml'] - conf_file_entry = "lbryum_servers: ['localhost:5001', 'localhost:5002']" - decoded = decoder(conf_file_entry) - converted = settings._convert_conf_file_lists(decoded) - converted_reversed = settings._convert_conf_file_lists_reverse(converted) - encoded = encoder(converted_reversed) - self.assertEqual(conf_file_entry, encoded.strip()) - self.assertEqual(decoded, converted_reversed) + def test_load_save_load_config_file(self): + #settings = self.get_mock_config_instance() + conf_entry = 'lbryum_servers: ["localhost:50001", "localhost:50002"]\n' + conf_temp = create_conf_file(conf_entry) + conf.conf_file = conf_temp + adjustable_settings={'data_dir': (str, conf.default_data_dir), + 'lbryum_servers': (list, [('localhost', 5001)], + conf.server_list, conf.server_list_reverse)} + env = conf.Env(**adjustable_settings) + settings = conf.Config({}, adjustable_settings, environment=env) + conf.settings = settings + settings.load_conf_file_settings() + settings.save_conf_file_settings() + settings.load_conf_file_settings() diff --git a/lbrynet/tests/util.py b/lbrynet/tests/util.py index e6ad2005c..45e4f3a15 100644 --- a/lbrynet/tests/util.py +++ b/lbrynet/tests/util.py @@ -19,6 +19,10 @@ DEFAULT_ISO_TIME = time.mktime(DEFAULT_TIMESTAMP.timetuple()) log = logging.getLogger("lbrynet.tests.util") +def create_conf_file(entry): + with tempfile.NamedTemporaryFile(delete=False, suffix='.yml') as conf: + conf.write(entry) + return conf.name def mk_db_and_blob_dir(): db_dir = tempfile.mkdtemp() From 500998b8931b092f54237301fb1481650aec2cd0 Mon Sep 17 00:00:00 2001 From: jleute Date: Sat, 19 May 2018 02:15:37 +0200 Subject: [PATCH 04/29] add assert to test_load_save_load_config_file test --- lbrynet/tests/unit/test_conf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lbrynet/tests/unit/test_conf.py b/lbrynet/tests/unit/test_conf.py index 87d5c03fc..9159244ec 100644 --- a/lbrynet/tests/unit/test_conf.py +++ b/lbrynet/tests/unit/test_conf.py @@ -81,7 +81,6 @@ class SettingsTest(unittest.TestCase): self.assertEqual(str, type(conf.default_lbryum_dir)) def test_load_save_load_config_file(self): - #settings = self.get_mock_config_instance() conf_entry = 'lbryum_servers: ["localhost:50001", "localhost:50002"]\n' conf_temp = create_conf_file(conf_entry) conf.conf_file = conf_temp @@ -92,6 +91,8 @@ class SettingsTest(unittest.TestCase): settings = conf.Config({}, adjustable_settings, environment=env) conf.settings = settings settings.load_conf_file_settings() + first = settings.get('lbryum_servers', data_type=conf.TYPE_PERSISTED) settings.save_conf_file_settings() settings.load_conf_file_settings() - + second = settings.get('lbryum_servers', data_type=conf.TYPE_PERSISTED) + self.assertEqual(first, second) From 3add623ac9c42918693f3f48aa492285426703e4 Mon Sep 17 00:00:00 2001 From: jleute Date: Sat, 19 May 2018 02:42:45 +0200 Subject: [PATCH 05/29] clean up temporary conf file --- lbrynet/tests/unit/test_conf.py | 12 +++++++++--- lbrynet/tests/util.py | 5 ++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/lbrynet/tests/unit/test_conf.py b/lbrynet/tests/unit/test_conf.py index 9159244ec..aedf4f6cc 100644 --- a/lbrynet/tests/unit/test_conf.py +++ b/lbrynet/tests/unit/test_conf.py @@ -5,7 +5,7 @@ from twisted.trial import unittest from lbrynet import conf from lbrynet.core.Error import InvalidCurrencyError from lbrynet.tests import mocks -from lbrynet.tests.util import create_conf_file +from lbrynet.tests.util import create_conf_file, remove_conf_file class SettingsTest(unittest.TestCase): def setUp(self): @@ -84,7 +84,7 @@ class SettingsTest(unittest.TestCase): conf_entry = 'lbryum_servers: ["localhost:50001", "localhost:50002"]\n' conf_temp = create_conf_file(conf_entry) conf.conf_file = conf_temp - adjustable_settings={'data_dir': (str, conf.default_data_dir), + adjustable_settings = {'data_dir': (str, conf.default_data_dir), 'lbryum_servers': (list, [('localhost', 5001)], conf.server_list, conf.server_list_reverse)} env = conf.Env(**adjustable_settings) @@ -93,6 +93,12 @@ class SettingsTest(unittest.TestCase): settings.load_conf_file_settings() first = settings.get('lbryum_servers', data_type=conf.TYPE_PERSISTED) settings.save_conf_file_settings() - settings.load_conf_file_settings() + try: + settings.load_conf_file_settings() + except Exception, e: + remove_conf_file(conf_temp) + raise Exception(e) second = settings.get('lbryum_servers', data_type=conf.TYPE_PERSISTED) + remove_conf_file(conf_temp) self.assertEqual(first, second) + diff --git a/lbrynet/tests/util.py b/lbrynet/tests/util.py index 45e4f3a15..9edd70900 100644 --- a/lbrynet/tests/util.py +++ b/lbrynet/tests/util.py @@ -22,7 +22,10 @@ log = logging.getLogger("lbrynet.tests.util") def create_conf_file(entry): with tempfile.NamedTemporaryFile(delete=False, suffix='.yml') as conf: conf.write(entry) - return conf.name + return conf.name + +def remove_conf_file(filename): + os.remove(filename) def mk_db_and_blob_dir(): db_dir = tempfile.mkdtemp() From 93ee9fcff1fd7dc199011c83a0894ed4bc2c2aa2 Mon Sep 17 00:00:00 2001 From: jleute Date: Sat, 19 May 2018 09:44:15 +0200 Subject: [PATCH 06/29] remove unused import --- lbrynet/tests/unit/test_conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lbrynet/tests/unit/test_conf.py b/lbrynet/tests/unit/test_conf.py index aedf4f6cc..06be39600 100644 --- a/lbrynet/tests/unit/test_conf.py +++ b/lbrynet/tests/unit/test_conf.py @@ -4,7 +4,6 @@ import json from twisted.trial import unittest from lbrynet import conf from lbrynet.core.Error import InvalidCurrencyError -from lbrynet.tests import mocks from lbrynet.tests.util import create_conf_file, remove_conf_file class SettingsTest(unittest.TestCase): From 092581bf3256fa3d6130da6ee6183f8100bf7451 Mon Sep 17 00:00:00 2001 From: jleute Date: Mon, 21 May 2018 18:50:25 +0200 Subject: [PATCH 07/29] updated test to compare original and saved config file --- lbrynet/tests/unit/test_conf.py | 34 ++++++++++++++++++--------------- lbrynet/tests/util.py | 8 -------- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/lbrynet/tests/unit/test_conf.py b/lbrynet/tests/unit/test_conf.py index 06be39600..d605bf6dc 100644 --- a/lbrynet/tests/unit/test_conf.py +++ b/lbrynet/tests/unit/test_conf.py @@ -1,10 +1,10 @@ import os import json +import tempfile from twisted.trial import unittest from lbrynet import conf from lbrynet.core.Error import InvalidCurrencyError -from lbrynet.tests.util import create_conf_file, remove_conf_file class SettingsTest(unittest.TestCase): def setUp(self): @@ -79,25 +79,29 @@ class SettingsTest(unittest.TestCase): self.assertEqual(str, type(conf.default_data_dir)) self.assertEqual(str, type(conf.default_lbryum_dir)) - def test_load_save_load_config_file(self): - conf_entry = 'lbryum_servers: ["localhost:50001", "localhost:50002"]\n' - conf_temp = create_conf_file(conf_entry) - conf.conf_file = conf_temp + def test_load_save_config_file(self): + # setup settings adjustable_settings = {'data_dir': (str, conf.default_data_dir), 'lbryum_servers': (list, [('localhost', 5001)], conf.server_list, conf.server_list_reverse)} env = conf.Env(**adjustable_settings) settings = conf.Config({}, adjustable_settings, environment=env) conf.settings = settings - settings.load_conf_file_settings() - first = settings.get('lbryum_servers', data_type=conf.TYPE_PERSISTED) - settings.save_conf_file_settings() - try: + # setup tempfile + conf_entry = "lbryum_servers: ['localhost:50001', 'localhost:50002']\n" + with tempfile.NamedTemporaryFile(suffix='.yml') as conf_file: + conf_file.write(conf_entry) + conf_file.seek(0) + conf.conf_file = conf_file.name + # load and save settings from conf file settings.load_conf_file_settings() - except Exception, e: - remove_conf_file(conf_temp) - raise Exception(e) - second = settings.get('lbryum_servers', data_type=conf.TYPE_PERSISTED) - remove_conf_file(conf_temp) - self.assertEqual(first, second) + settings.save_conf_file_settings() + # test if overwritten entry equals original entry + # use decoded versions, because format might change without + # changing the interpretation + decoder = conf.settings_decoders['.yml'] + conf_decoded = decoder(conf_entry) + conf_entry_new = conf_file.read() + conf_decoded_new = decoder(conf_entry_new) + self.assertEqual(conf_decoded, conf_decoded_new) diff --git a/lbrynet/tests/util.py b/lbrynet/tests/util.py index 9edd70900..6b9780840 100644 --- a/lbrynet/tests/util.py +++ b/lbrynet/tests/util.py @@ -19,14 +19,6 @@ DEFAULT_ISO_TIME = time.mktime(DEFAULT_TIMESTAMP.timetuple()) log = logging.getLogger("lbrynet.tests.util") -def create_conf_file(entry): - with tempfile.NamedTemporaryFile(delete=False, suffix='.yml') as conf: - conf.write(entry) - return conf.name - -def remove_conf_file(filename): - os.remove(filename) - def mk_db_and_blob_dir(): db_dir = tempfile.mkdtemp() blob_dir = tempfile.mkdtemp() From 4aa503b863bb82f71b77a10632f4995bf6c2618e Mon Sep 17 00:00:00 2001 From: Sergey Rozhnov Date: Wed, 16 May 2018 18:29:44 +0400 Subject: [PATCH 08/29] claim_list and claim_list_mine in Daemon return sorted results --- lbrynet/daemon/Daemon.py | 5 +- lbrynet/daemon/claims_comparator.py | 36 +++++++++++++ lbrynet/tests/unit/daemon/__init__.py | 0 .../unit/daemon/test_claims_comparator.py | 52 +++++++++++++++++++ 4 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 lbrynet/daemon/claims_comparator.py create mode 100644 lbrynet/tests/unit/daemon/__init__.py create mode 100644 lbrynet/tests/unit/daemon/test_claims_comparator.py diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index 43c67e0d7..9da91e320 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -36,6 +36,7 @@ from lbrynet.daemon.Downloader import GetStream from lbrynet.daemon.Publisher import Publisher from lbrynet.daemon.ExchangeRateManager import ExchangeRateManager from lbrynet.daemon.auth.server import AuthJSONRPCServer +from lbrynet.daemon.claims_comparator import arrange_results from lbrynet.core.PaymentRateManager import OnlyFreePaymentsManager from lbrynet.core import utils, system_info from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob @@ -2293,6 +2294,7 @@ class Daemon(AuthJSONRPCServer): """ d = self.session.wallet.get_name_claims() + d.addCallback(arrange_results) d.addCallback(lambda claims: self._render_response(claims)) return d @@ -2331,7 +2333,8 @@ class Daemon(AuthJSONRPCServer): """ claims = yield self.session.wallet.get_claims_for_name(name) - defer.returnValue(claims) + result = arrange_results(claims) + defer.returnValue(result) @defer.inlineCallbacks def jsonrpc_claim_list_by_channel(self, page=0, page_size=10, uri=None, uris=[]): diff --git a/lbrynet/daemon/claims_comparator.py b/lbrynet/daemon/claims_comparator.py new file mode 100644 index 000000000..4a1ea9870 --- /dev/null +++ b/lbrynet/daemon/claims_comparator.py @@ -0,0 +1,36 @@ +_comparison_order = ['height', 'name', 'claim_id'] # TODO outpoint + + +def arrange_results(claims): + for claim in claims: + results = claim['result'] + sorted_results = sorted(results, cmp=_compare_results) + claim['result'] = sorted_results + return claims + + +def _compare_results(left, right): + """ + :type left: dict + :type right: dict + """ + result = 0 + + for attribute in _comparison_order: + left_value = left[attribute] + right_value = right[attribute] + sub_result = _cmp(left_value, right_value) + if sub_result is not 0: + result = sub_result + break + + return result + + +def _cmp(left, right): + if left == right: + return 0 + elif left < right: + return -1 + else: + return 1 diff --git a/lbrynet/tests/unit/daemon/__init__.py b/lbrynet/tests/unit/daemon/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lbrynet/tests/unit/daemon/test_claims_comparator.py b/lbrynet/tests/unit/daemon/test_claims_comparator.py new file mode 100644 index 000000000..9651c2137 --- /dev/null +++ b/lbrynet/tests/unit/daemon/test_claims_comparator.py @@ -0,0 +1,52 @@ +import unittest + +from lbrynet.daemon.claims_comparator import arrange_results + + +class ClaimsComparatorTest(unittest.TestCase): + def test_arrange_results(self): + results = [ + { + 'height': 1, + 'name': 'res', + 'claim_id': 'ccc' + }, + { + 'height': 1, + 'name': 'res', + 'claim_id': 'aaa' + }, + { + 'height': 1, + 'name': 'res', + 'claim_id': 'bbb' + } + ] + data = {'result': results} + + expected = [ + { + 'height': 1, + 'name': 'res', + 'claim_id': 'aaa' + }, + { + 'height': 1, + 'name': 'res', + 'claim_id': 'bbb' + }, + { + 'height': 1, + 'name': 'res', + 'claim_id': 'ccc' + } + ] + claims = arrange_results([data]) + claim = claims[0] + actual = claim['result'] + + self.assertEqual(expected, actual) + + +if __name__ == '__main__': + unittest.main() From 332b3e1fefaadb3e6b45ed77fcb5c99357aa66d3 Mon Sep 17 00:00:00 2001 From: Sergey Rozhnov Date: Thu, 17 May 2018 13:36:32 +0400 Subject: [PATCH 09/29] simplified claims comparison logic; refactored unit tests --- lbrynet/daemon/claims_comparator.py | 32 ++--------- .../unit/daemon/claims_comparator_cases.json | 53 +++++++++++++++++++ .../unit/daemon/test_claims_comparator.py | 53 +++++-------------- 3 files changed, 70 insertions(+), 68 deletions(-) create mode 100644 lbrynet/tests/unit/daemon/claims_comparator_cases.json diff --git a/lbrynet/daemon/claims_comparator.py b/lbrynet/daemon/claims_comparator.py index 4a1ea9870..44d27a955 100644 --- a/lbrynet/daemon/claims_comparator.py +++ b/lbrynet/daemon/claims_comparator.py @@ -1,36 +1,10 @@ -_comparison_order = ['height', 'name', 'claim_id'] # TODO outpoint - - def arrange_results(claims): for claim in claims: results = claim['result'] - sorted_results = sorted(results, cmp=_compare_results) + sorted_results = sorted(results, key=lambda d: (d['height'], d['name'], d['claim_id'], _outpoint(d))) claim['result'] = sorted_results return claims -def _compare_results(left, right): - """ - :type left: dict - :type right: dict - """ - result = 0 - - for attribute in _comparison_order: - left_value = left[attribute] - right_value = right[attribute] - sub_result = _cmp(left_value, right_value) - if sub_result is not 0: - result = sub_result - break - - return result - - -def _cmp(left, right): - if left == right: - return 0 - elif left < right: - return -1 - else: - return 1 +def _outpoint(claim): + return '{}:{}'.format(claim['txid'], claim['nout']) diff --git a/lbrynet/tests/unit/daemon/claims_comparator_cases.json b/lbrynet/tests/unit/daemon/claims_comparator_cases.json new file mode 100644 index 000000000..dbcb732b7 --- /dev/null +++ b/lbrynet/tests/unit/daemon/claims_comparator_cases.json @@ -0,0 +1,53 @@ +{ + "cases": [ + { + "description": "sort by claim_id", + "results": [ + { + "height": 1, + "name": "res", + "claim_id": "ccc", + "nout": 0, + "txid": "fdsafa" + }, + { + "height": 1, + "name": "res", + "claim_id": "aaa", + "nout": 0, + "txid": "w5tv8uorgt" + }, + { + "height": 1, + "name": "res", + "claim_id": "bbb", + "nout": 0, + "txid": "aecfaewcfa" + } + ], + "expected": [ + { + "height": 1, + "name": "res", + "claim_id": "aaa", + "nout": 0, + "txid": "w5tv8uorgt" + }, + { + "height": 1, + "name": "res", + "claim_id": "bbb", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res", + "claim_id": "ccc", + "nout": 0, + "txid": "fdsafa" + } + ] + } + ] +} \ No newline at end of file diff --git a/lbrynet/tests/unit/daemon/test_claims_comparator.py b/lbrynet/tests/unit/daemon/test_claims_comparator.py index 9651c2137..07fcbc503 100644 --- a/lbrynet/tests/unit/daemon/test_claims_comparator.py +++ b/lbrynet/tests/unit/daemon/test_claims_comparator.py @@ -1,51 +1,26 @@ +import json import unittest from lbrynet.daemon.claims_comparator import arrange_results class ClaimsComparatorTest(unittest.TestCase): + def setUp(self): + with open('claims_comparator_cases.json') as f: + document = json.load(f) + self.cases = document['cases'] + def test_arrange_results(self): - results = [ - { - 'height': 1, - 'name': 'res', - 'claim_id': 'ccc' - }, - { - 'height': 1, - 'name': 'res', - 'claim_id': 'aaa' - }, - { - 'height': 1, - 'name': 'res', - 'claim_id': 'bbb' - } - ] - data = {'result': results} + for case in self.cases: + results = case['results'] + data = {'result': results} + expected = case['expected'] - expected = [ - { - 'height': 1, - 'name': 'res', - 'claim_id': 'aaa' - }, - { - 'height': 1, - 'name': 'res', - 'claim_id': 'bbb' - }, - { - 'height': 1, - 'name': 'res', - 'claim_id': 'ccc' - } - ] - claims = arrange_results([data]) - claim = claims[0] - actual = claim['result'] + claims = arrange_results([data]) + claim = claims[0] + actual = claim['result'] - self.assertEqual(expected, actual) + self.assertEqual(expected, actual, case['description']) if __name__ == '__main__': From eb164d9ac7933cfe93f89d0c3903253f0974e744 Mon Sep 17 00:00:00 2001 From: Sergey Rozhnov Date: Thu, 17 May 2018 13:49:09 +0400 Subject: [PATCH 10/29] implemented additional test cases for claims sorting; updated changelog --- CHANGELOG.md | 1 + .../unit/daemon/claims_comparator_cases.json | 147 ++++++++++++++++++ 2 files changed, 148 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b94d5fd44..b95a4f1eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,7 @@ at anytime. * download blockchain headers from s3 before starting the wallet when the local height is more than `s3_headers_depth` (a config setting) blocks behind * track successful reflector uploads in sqlite to minimize how many streams are attempted by auto re-reflect * increase the default `auto_re_reflect_interval` to a day + * `claim_list` and `claim_list_mine` in Daemon `return` sorted results ### Added * virtual kademlia network and mock udp transport for dht integration tests diff --git a/lbrynet/tests/unit/daemon/claims_comparator_cases.json b/lbrynet/tests/unit/daemon/claims_comparator_cases.json index dbcb732b7..11592fbf1 100644 --- a/lbrynet/tests/unit/daemon/claims_comparator_cases.json +++ b/lbrynet/tests/unit/daemon/claims_comparator_cases.json @@ -48,6 +48,153 @@ "txid": "fdsafa" } ] + }, + { + "description": "sort by height", + "results": [ + { + "height": 1, + "name": "res", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 3, + "name": "res", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 2, + "name": "res", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + } + ], + "expected": [ + { + "claim_id": "ccc", + "height": 1, + "name": "res", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "claim_id": "ccc", + "height": 2, + "name": "res", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "claim_id": "ccc", + "height": 3, + "name": "res", + "nout": 0, + "txid": "aecfaewcfa" + } + ] + }, + { + "description": "sort by name", + "results": [ + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res3", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res2", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + } + ], + "expected": [ + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res2", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res3", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + } + ] + }, + { + "description": "sort by outpoint", + "results": [ + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 2, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 1, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 3, + "txid": "aecfaewcfa" + } + ], + "expected": [ + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 1, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 2, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 3, + "txid": "aecfaewcfa" + } + ] } ] } \ No newline at end of file From 65fed3f5aa2295bf330460de36a12b30ad8dc142 Mon Sep 17 00:00:00 2001 From: Sergey Rozhnov Date: Fri, 18 May 2018 12:53:32 +0400 Subject: [PATCH 11/29] refactored sorting of claims and unit tests --- lbrynet/daemon/Daemon.py | 7 +- lbrynet/daemon/claims_comparator.py | 10 - .../unit/daemon/claims_comparator_cases.json | 200 ---------------- .../unit/daemon/test_claims_comparator.py | 217 ++++++++++++++++-- 4 files changed, 208 insertions(+), 226 deletions(-) delete mode 100644 lbrynet/daemon/claims_comparator.py delete mode 100644 lbrynet/tests/unit/daemon/claims_comparator_cases.json diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index 9da91e320..c8b7e376f 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -36,7 +36,6 @@ from lbrynet.daemon.Downloader import GetStream from lbrynet.daemon.Publisher import Publisher from lbrynet.daemon.ExchangeRateManager import ExchangeRateManager from lbrynet.daemon.auth.server import AuthJSONRPCServer -from lbrynet.daemon.claims_comparator import arrange_results from lbrynet.core.PaymentRateManager import OnlyFreePaymentsManager from lbrynet.core import utils, system_info from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob @@ -164,6 +163,12 @@ class AlwaysSend(object): return d +def arrange_results(claims): + for claim in claims: + claim['result'].sort(key=lambda d: (d['height'], d['name'], d['claim_id'], d['txid'], d['nout'])) + return claims + + class Daemon(AuthJSONRPCServer): """ LBRYnet daemon, a jsonrpc interface to lbry functions diff --git a/lbrynet/daemon/claims_comparator.py b/lbrynet/daemon/claims_comparator.py deleted file mode 100644 index 44d27a955..000000000 --- a/lbrynet/daemon/claims_comparator.py +++ /dev/null @@ -1,10 +0,0 @@ -def arrange_results(claims): - for claim in claims: - results = claim['result'] - sorted_results = sorted(results, key=lambda d: (d['height'], d['name'], d['claim_id'], _outpoint(d))) - claim['result'] = sorted_results - return claims - - -def _outpoint(claim): - return '{}:{}'.format(claim['txid'], claim['nout']) diff --git a/lbrynet/tests/unit/daemon/claims_comparator_cases.json b/lbrynet/tests/unit/daemon/claims_comparator_cases.json deleted file mode 100644 index 11592fbf1..000000000 --- a/lbrynet/tests/unit/daemon/claims_comparator_cases.json +++ /dev/null @@ -1,200 +0,0 @@ -{ - "cases": [ - { - "description": "sort by claim_id", - "results": [ - { - "height": 1, - "name": "res", - "claim_id": "ccc", - "nout": 0, - "txid": "fdsafa" - }, - { - "height": 1, - "name": "res", - "claim_id": "aaa", - "nout": 0, - "txid": "w5tv8uorgt" - }, - { - "height": 1, - "name": "res", - "claim_id": "bbb", - "nout": 0, - "txid": "aecfaewcfa" - } - ], - "expected": [ - { - "height": 1, - "name": "res", - "claim_id": "aaa", - "nout": 0, - "txid": "w5tv8uorgt" - }, - { - "height": 1, - "name": "res", - "claim_id": "bbb", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res", - "claim_id": "ccc", - "nout": 0, - "txid": "fdsafa" - } - ] - }, - { - "description": "sort by height", - "results": [ - { - "height": 1, - "name": "res", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 3, - "name": "res", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 2, - "name": "res", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - } - ], - "expected": [ - { - "claim_id": "ccc", - "height": 1, - "name": "res", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "claim_id": "ccc", - "height": 2, - "name": "res", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "claim_id": "ccc", - "height": 3, - "name": "res", - "nout": 0, - "txid": "aecfaewcfa" - } - ] - }, - { - "description": "sort by name", - "results": [ - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res3", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res2", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - } - ], - "expected": [ - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res2", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res3", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - } - ] - }, - { - "description": "sort by outpoint", - "results": [ - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 2, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 1, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 3, - "txid": "aecfaewcfa" - } - ], - "expected": [ - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 1, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 2, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 3, - "txid": "aecfaewcfa" - } - ] - } - ] -} \ No newline at end of file diff --git a/lbrynet/tests/unit/daemon/test_claims_comparator.py b/lbrynet/tests/unit/daemon/test_claims_comparator.py index 07fcbc503..416a394d5 100644 --- a/lbrynet/tests/unit/daemon/test_claims_comparator.py +++ b/lbrynet/tests/unit/daemon/test_claims_comparator.py @@ -1,26 +1,213 @@ -import json import unittest -from lbrynet.daemon.claims_comparator import arrange_results +from lbrynet.daemon.Daemon import arrange_results class ClaimsComparatorTest(unittest.TestCase): - def setUp(self): - with open('claims_comparator_cases.json') as f: - document = json.load(f) - self.cases = document['cases'] + def test_arrange_results_when_sorted_by_claim_id(self): + self.run_test( + [ + { + "height": 1, + "name": "res", + "claim_id": "ccc", + "nout": 0, + "txid": "fdsafa" + }, + { + "height": 1, + "name": "res", + "claim_id": "aaa", + "nout": 0, + "txid": "w5tv8uorgt" + }, + { + "height": 1, + "name": "res", + "claim_id": "bbb", + "nout": 0, + "txid": "aecfaewcfa" + } + ], + [ + { + "height": 1, + "name": "res", + "claim_id": "aaa", + "nout": 0, + "txid": "w5tv8uorgt" + }, + { + "height": 1, + "name": "res", + "claim_id": "bbb", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res", + "claim_id": "ccc", + "nout": 0, + "txid": "fdsafa" + } + ]) - def test_arrange_results(self): - for case in self.cases: - results = case['results'] - data = {'result': results} - expected = case['expected'] + def test_arrange_results_when_sorted_by_height(self): + self.run_test( + [ + { + "height": 1, + "name": "res", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 3, + "name": "res", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 2, + "name": "res", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + } + ], + [ + { + "claim_id": "ccc", + "height": 1, + "name": "res", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "claim_id": "ccc", + "height": 2, + "name": "res", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "claim_id": "ccc", + "height": 3, + "name": "res", + "nout": 0, + "txid": "aecfaewcfa" + } + ]) - claims = arrange_results([data]) - claim = claims[0] - actual = claim['result'] + def test_arrange_results_when_sorted_by_name(self): + self.run_test( + [ + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res3", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res2", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + } + ], + [ + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res2", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res3", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + } + ]) - self.assertEqual(expected, actual, case['description']) + def test_arrange_results_when_sort_by_outpoint(self): + self.run_test( + [ + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 2, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 1, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 3, + "txid": "aecfaewcfa" + } + ], + [ + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 1, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 2, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 3, + "txid": "aecfaewcfa" + } + ]) + + def run_test(self, results, expected): + data = {'result': results} + + claims = arrange_results([data]) + claim = claims[0] + actual = claim['result'] + + self.assertEqual(expected, actual) if __name__ == '__main__': From e220a57f0e5a14de934ff45e723fc061c129066e Mon Sep 17 00:00:00 2001 From: Sergey Rozhnov Date: Fri, 18 May 2018 18:51:28 +0400 Subject: [PATCH 12/29] refactored unit test for sort_claim_results --- lbrynet/daemon/Daemon.py | 6 +- .../unit/daemon/test_claims_comparator.py | 228 +++--------------- 2 files changed, 33 insertions(+), 201 deletions(-) diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index c8b7e376f..ac497c37c 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -163,7 +163,7 @@ class AlwaysSend(object): return d -def arrange_results(claims): +def sort_claim_results(claims): for claim in claims: claim['result'].sort(key=lambda d: (d['height'], d['name'], d['claim_id'], d['txid'], d['nout'])) return claims @@ -2299,7 +2299,7 @@ class Daemon(AuthJSONRPCServer): """ d = self.session.wallet.get_name_claims() - d.addCallback(arrange_results) + d.addCallback(sort_claim_results) d.addCallback(lambda claims: self._render_response(claims)) return d @@ -2338,7 +2338,7 @@ class Daemon(AuthJSONRPCServer): """ claims = yield self.session.wallet.get_claims_for_name(name) - result = arrange_results(claims) + result = sort_claim_results(claims) defer.returnValue(result) @defer.inlineCallbacks diff --git a/lbrynet/tests/unit/daemon/test_claims_comparator.py b/lbrynet/tests/unit/daemon/test_claims_comparator.py index 416a394d5..772ecb5e5 100644 --- a/lbrynet/tests/unit/daemon/test_claims_comparator.py +++ b/lbrynet/tests/unit/daemon/test_claims_comparator.py @@ -1,213 +1,45 @@ import unittest -from lbrynet.daemon.Daemon import arrange_results +from lbrynet.daemon.Daemon import sort_claim_results class ClaimsComparatorTest(unittest.TestCase): - def test_arrange_results_when_sorted_by_claim_id(self): - self.run_test( - [ - { - "height": 1, - "name": "res", - "claim_id": "ccc", - "nout": 0, - "txid": "fdsafa" - }, - { - "height": 1, - "name": "res", - "claim_id": "aaa", - "nout": 0, - "txid": "w5tv8uorgt" - }, - { - "height": 1, - "name": "res", - "claim_id": "bbb", - "nout": 0, - "txid": "aecfaewcfa" - } - ], - [ - { - "height": 1, - "name": "res", - "claim_id": "aaa", - "nout": 0, - "txid": "w5tv8uorgt" - }, - { - "height": 1, - "name": "res", - "claim_id": "bbb", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res", - "claim_id": "ccc", - "nout": 0, - "txid": "fdsafa" - } - ]) + def test_sort_claim_results_when_sorted_by_claim_id(self): + results = [{"height": 1, "name": "res", "claim_id": "ccc", "nout": 0, "txid": "fdsafa"}, + {"height": 1, "name": "res", "claim_id": "aaa", "nout": 0, "txid": "w5tv8uorgt"}, + {"height": 1, "name": "res", "claim_id": "bbb", "nout": 0, "txid": "aecfaewcfa"}] + self.run_test(results, 'claim_id', ['aaa', 'bbb', 'ccc']) - def test_arrange_results_when_sorted_by_height(self): - self.run_test( - [ - { - "height": 1, - "name": "res", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 3, - "name": "res", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 2, - "name": "res", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - } - ], - [ - { - "claim_id": "ccc", - "height": 1, - "name": "res", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "claim_id": "ccc", - "height": 2, - "name": "res", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "claim_id": "ccc", - "height": 3, - "name": "res", - "nout": 0, - "txid": "aecfaewcfa" - } - ]) + def test_sort_claim_results_when_sorted_by_height(self): + results = [{"height": 1, "name": "res", "claim_id": "ccc", "nout": 0, "txid": "aecfaewcfa"}, + {"height": 3, "name": "res", "claim_id": "ccc", "nout": 0, "txid": "aecfaewcfa"}, + {"height": 2, "name": "res", "claim_id": "ccc", "nout": 0, "txid": "aecfaewcfa"}] + self.run_test(results, 'height', [1, 2, 3]) - def test_arrange_results_when_sorted_by_name(self): - self.run_test( - [ - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res3", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res2", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - } - ], - [ - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res2", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res3", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - } - ]) + def test_sort_claim_results_when_sorted_by_name(self): + results = [{"height": 1, "name": "res1", "claim_id": "ccc", "nout": 0, "txid": "aecfaewcfa"}, + {"height": 1, "name": "res3", "claim_id": "ccc", "nout": 0, "txid": "aecfaewcfa"}, + {"height": 1, "name": "res2", "claim_id": "ccc", "nout": 0, "txid": "aecfaewcfa"}] + self.run_test(results, 'name', ['res1', 'res2', 'res3']) - def test_arrange_results_when_sort_by_outpoint(self): - self.run_test( - [ - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 2, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 1, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 3, - "txid": "aecfaewcfa" - } - ], - [ - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 1, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 2, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 3, - "txid": "aecfaewcfa" - } - ]) + def test_sort_claim_results_when_sorted_by_txid(self): + results = [{"height": 1, "name": "res1", "claim_id": "ccc", "nout": 2, "txid": "111"}, + {"height": 1, "name": "res1", "claim_id": "ccc", "nout": 1, "txid": "222"}, + {"height": 1, "name": "res1", "claim_id": "ccc", "nout": 3, "txid": "333"}] + self.run_test(results, 'txid', ['111', '222', '333']) - def run_test(self, results, expected): + def test_sort_claim_results_when_sorted_by_nout(self): + results = [{"height": 1, "name": "res1", "claim_id": "ccc", "nout": 2, "txid": "aecfaewcfa"}, + {"height": 1, "name": "res1", "claim_id": "ccc", "nout": 1, "txid": "aecfaewcfa"}, + {"height": 1, "name": "res1", "claim_id": "ccc", "nout": 3, "txid": "aecfaewcfa"}] + self.run_test(results, 'nout', [1, 2, 3]) + + def run_test(self, results, field, expected): data = {'result': results} - - claims = arrange_results([data]) + claims = sort_claim_results([data]) claim = claims[0] actual = claim['result'] - - self.assertEqual(expected, actual) + self.assertEqual(expected, [r[field] for r in actual]) if __name__ == '__main__': From 25c36346b6fad57f2dc121fa028d2ab108a89954 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 22 May 2018 17:15:34 -0400 Subject: [PATCH 13/29] Revert "WIP: feature/1098/sorted-claim-results" --- CHANGELOG.md | 1 - lbrynet/daemon/Daemon.py | 10 +--- lbrynet/tests/unit/daemon/__init__.py | 0 .../unit/daemon/test_claims_comparator.py | 46 ------------------- 4 files changed, 1 insertion(+), 56 deletions(-) delete mode 100644 lbrynet/tests/unit/daemon/__init__.py delete mode 100644 lbrynet/tests/unit/daemon/test_claims_comparator.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b95a4f1eb..b94d5fd44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,7 +52,6 @@ at anytime. * download blockchain headers from s3 before starting the wallet when the local height is more than `s3_headers_depth` (a config setting) blocks behind * track successful reflector uploads in sqlite to minimize how many streams are attempted by auto re-reflect * increase the default `auto_re_reflect_interval` to a day - * `claim_list` and `claim_list_mine` in Daemon `return` sorted results ### Added * virtual kademlia network and mock udp transport for dht integration tests diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index ac497c37c..43c67e0d7 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -163,12 +163,6 @@ class AlwaysSend(object): return d -def sort_claim_results(claims): - for claim in claims: - claim['result'].sort(key=lambda d: (d['height'], d['name'], d['claim_id'], d['txid'], d['nout'])) - return claims - - class Daemon(AuthJSONRPCServer): """ LBRYnet daemon, a jsonrpc interface to lbry functions @@ -2299,7 +2293,6 @@ class Daemon(AuthJSONRPCServer): """ d = self.session.wallet.get_name_claims() - d.addCallback(sort_claim_results) d.addCallback(lambda claims: self._render_response(claims)) return d @@ -2338,8 +2331,7 @@ class Daemon(AuthJSONRPCServer): """ claims = yield self.session.wallet.get_claims_for_name(name) - result = sort_claim_results(claims) - defer.returnValue(result) + defer.returnValue(claims) @defer.inlineCallbacks def jsonrpc_claim_list_by_channel(self, page=0, page_size=10, uri=None, uris=[]): diff --git a/lbrynet/tests/unit/daemon/__init__.py b/lbrynet/tests/unit/daemon/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/lbrynet/tests/unit/daemon/test_claims_comparator.py b/lbrynet/tests/unit/daemon/test_claims_comparator.py deleted file mode 100644 index 772ecb5e5..000000000 --- a/lbrynet/tests/unit/daemon/test_claims_comparator.py +++ /dev/null @@ -1,46 +0,0 @@ -import unittest - -from lbrynet.daemon.Daemon import sort_claim_results - - -class ClaimsComparatorTest(unittest.TestCase): - def test_sort_claim_results_when_sorted_by_claim_id(self): - results = [{"height": 1, "name": "res", "claim_id": "ccc", "nout": 0, "txid": "fdsafa"}, - {"height": 1, "name": "res", "claim_id": "aaa", "nout": 0, "txid": "w5tv8uorgt"}, - {"height": 1, "name": "res", "claim_id": "bbb", "nout": 0, "txid": "aecfaewcfa"}] - self.run_test(results, 'claim_id', ['aaa', 'bbb', 'ccc']) - - def test_sort_claim_results_when_sorted_by_height(self): - results = [{"height": 1, "name": "res", "claim_id": "ccc", "nout": 0, "txid": "aecfaewcfa"}, - {"height": 3, "name": "res", "claim_id": "ccc", "nout": 0, "txid": "aecfaewcfa"}, - {"height": 2, "name": "res", "claim_id": "ccc", "nout": 0, "txid": "aecfaewcfa"}] - self.run_test(results, 'height', [1, 2, 3]) - - def test_sort_claim_results_when_sorted_by_name(self): - results = [{"height": 1, "name": "res1", "claim_id": "ccc", "nout": 0, "txid": "aecfaewcfa"}, - {"height": 1, "name": "res3", "claim_id": "ccc", "nout": 0, "txid": "aecfaewcfa"}, - {"height": 1, "name": "res2", "claim_id": "ccc", "nout": 0, "txid": "aecfaewcfa"}] - self.run_test(results, 'name', ['res1', 'res2', 'res3']) - - def test_sort_claim_results_when_sorted_by_txid(self): - results = [{"height": 1, "name": "res1", "claim_id": "ccc", "nout": 2, "txid": "111"}, - {"height": 1, "name": "res1", "claim_id": "ccc", "nout": 1, "txid": "222"}, - {"height": 1, "name": "res1", "claim_id": "ccc", "nout": 3, "txid": "333"}] - self.run_test(results, 'txid', ['111', '222', '333']) - - def test_sort_claim_results_when_sorted_by_nout(self): - results = [{"height": 1, "name": "res1", "claim_id": "ccc", "nout": 2, "txid": "aecfaewcfa"}, - {"height": 1, "name": "res1", "claim_id": "ccc", "nout": 1, "txid": "aecfaewcfa"}, - {"height": 1, "name": "res1", "claim_id": "ccc", "nout": 3, "txid": "aecfaewcfa"}] - self.run_test(results, 'nout', [1, 2, 3]) - - def run_test(self, results, field, expected): - data = {'result': results} - claims = sort_claim_results([data]) - claim = claims[0] - actual = claim['result'] - self.assertEqual(expected, [r[field] for r in actual]) - - -if __name__ == '__main__': - unittest.main() From 0936bdbdd3e2d3f3d31c2018fd220971b46b8ed3 Mon Sep 17 00:00:00 2001 From: jleute Date: Sat, 9 Jun 2018 00:47:34 +0200 Subject: [PATCH 14/29] doesn't belong here. --- lbrynet/tests/util.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lbrynet/tests/util.py b/lbrynet/tests/util.py index 25af63697..68b445c8e 100644 --- a/lbrynet/tests/util.py +++ b/lbrynet/tests/util.py @@ -10,11 +10,6 @@ import mock DEFAULT_TIMESTAMP = datetime.datetime(2016, 1, 1) DEFAULT_ISO_TIME = time.mktime(DEFAULT_TIMESTAMP.timetuple()) -<<<<<<< HEAD -log = logging.getLogger("lbrynet.tests.util") -======= ->>>>>>> 2619c396d980944802a3f7e0800e92794cc3de5a - def mk_db_and_blob_dir(): db_dir = tempfile.mkdtemp() From c7ba82dc0704d92988e63923bd91d945dddda7de Mon Sep 17 00:00:00 2001 From: Sergey Rozhnov Date: Wed, 16 May 2018 18:29:44 +0400 Subject: [PATCH 15/29] claim_list and claim_list_mine in Daemon return sorted results --- lbrynet/daemon/Daemon.py | 2 + lbrynet/daemon/claims_comparator.py | 36 +++++++++++++ lbrynet/tests/unit/daemon/__init__.py | 0 .../unit/daemon/test_claims_comparator.py | 52 +++++++++++++++++++ 4 files changed, 90 insertions(+) create mode 100644 lbrynet/daemon/claims_comparator.py create mode 100644 lbrynet/tests/unit/daemon/__init__.py create mode 100644 lbrynet/tests/unit/daemon/test_claims_comparator.py diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index 6fa9fb148..648b8a70b 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -36,6 +36,7 @@ from lbrynet.daemon.Downloader import GetStream from lbrynet.daemon.Publisher import Publisher from lbrynet.daemon.ExchangeRateManager import ExchangeRateManager from lbrynet.daemon.auth.server import AuthJSONRPCServer +from lbrynet.daemon.claims_comparator import arrange_results from lbrynet.core.PaymentRateManager import OnlyFreePaymentsManager from lbrynet.core import utils, system_info from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob @@ -2329,6 +2330,7 @@ class Daemon(AuthJSONRPCServer): """ d = self.session.wallet.get_name_claims() + d.addCallback(arrange_results) d.addCallback(lambda claims: self._render_response(claims)) return d diff --git a/lbrynet/daemon/claims_comparator.py b/lbrynet/daemon/claims_comparator.py new file mode 100644 index 000000000..4a1ea9870 --- /dev/null +++ b/lbrynet/daemon/claims_comparator.py @@ -0,0 +1,36 @@ +_comparison_order = ['height', 'name', 'claim_id'] # TODO outpoint + + +def arrange_results(claims): + for claim in claims: + results = claim['result'] + sorted_results = sorted(results, cmp=_compare_results) + claim['result'] = sorted_results + return claims + + +def _compare_results(left, right): + """ + :type left: dict + :type right: dict + """ + result = 0 + + for attribute in _comparison_order: + left_value = left[attribute] + right_value = right[attribute] + sub_result = _cmp(left_value, right_value) + if sub_result is not 0: + result = sub_result + break + + return result + + +def _cmp(left, right): + if left == right: + return 0 + elif left < right: + return -1 + else: + return 1 diff --git a/lbrynet/tests/unit/daemon/__init__.py b/lbrynet/tests/unit/daemon/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lbrynet/tests/unit/daemon/test_claims_comparator.py b/lbrynet/tests/unit/daemon/test_claims_comparator.py new file mode 100644 index 000000000..9651c2137 --- /dev/null +++ b/lbrynet/tests/unit/daemon/test_claims_comparator.py @@ -0,0 +1,52 @@ +import unittest + +from lbrynet.daemon.claims_comparator import arrange_results + + +class ClaimsComparatorTest(unittest.TestCase): + def test_arrange_results(self): + results = [ + { + 'height': 1, + 'name': 'res', + 'claim_id': 'ccc' + }, + { + 'height': 1, + 'name': 'res', + 'claim_id': 'aaa' + }, + { + 'height': 1, + 'name': 'res', + 'claim_id': 'bbb' + } + ] + data = {'result': results} + + expected = [ + { + 'height': 1, + 'name': 'res', + 'claim_id': 'aaa' + }, + { + 'height': 1, + 'name': 'res', + 'claim_id': 'bbb' + }, + { + 'height': 1, + 'name': 'res', + 'claim_id': 'ccc' + } + ] + claims = arrange_results([data]) + claim = claims[0] + actual = claim['result'] + + self.assertEqual(expected, actual) + + +if __name__ == '__main__': + unittest.main() From b770b2de6a705838e21665b7672239532522ef83 Mon Sep 17 00:00:00 2001 From: Sergey Rozhnov Date: Thu, 17 May 2018 13:36:32 +0400 Subject: [PATCH 16/29] simplified claims comparison logic; refactored unit tests --- lbrynet/daemon/claims_comparator.py | 32 ++--------- .../unit/daemon/claims_comparator_cases.json | 53 +++++++++++++++++++ .../unit/daemon/test_claims_comparator.py | 53 +++++-------------- 3 files changed, 70 insertions(+), 68 deletions(-) create mode 100644 lbrynet/tests/unit/daemon/claims_comparator_cases.json diff --git a/lbrynet/daemon/claims_comparator.py b/lbrynet/daemon/claims_comparator.py index 4a1ea9870..44d27a955 100644 --- a/lbrynet/daemon/claims_comparator.py +++ b/lbrynet/daemon/claims_comparator.py @@ -1,36 +1,10 @@ -_comparison_order = ['height', 'name', 'claim_id'] # TODO outpoint - - def arrange_results(claims): for claim in claims: results = claim['result'] - sorted_results = sorted(results, cmp=_compare_results) + sorted_results = sorted(results, key=lambda d: (d['height'], d['name'], d['claim_id'], _outpoint(d))) claim['result'] = sorted_results return claims -def _compare_results(left, right): - """ - :type left: dict - :type right: dict - """ - result = 0 - - for attribute in _comparison_order: - left_value = left[attribute] - right_value = right[attribute] - sub_result = _cmp(left_value, right_value) - if sub_result is not 0: - result = sub_result - break - - return result - - -def _cmp(left, right): - if left == right: - return 0 - elif left < right: - return -1 - else: - return 1 +def _outpoint(claim): + return '{}:{}'.format(claim['txid'], claim['nout']) diff --git a/lbrynet/tests/unit/daemon/claims_comparator_cases.json b/lbrynet/tests/unit/daemon/claims_comparator_cases.json new file mode 100644 index 000000000..dbcb732b7 --- /dev/null +++ b/lbrynet/tests/unit/daemon/claims_comparator_cases.json @@ -0,0 +1,53 @@ +{ + "cases": [ + { + "description": "sort by claim_id", + "results": [ + { + "height": 1, + "name": "res", + "claim_id": "ccc", + "nout": 0, + "txid": "fdsafa" + }, + { + "height": 1, + "name": "res", + "claim_id": "aaa", + "nout": 0, + "txid": "w5tv8uorgt" + }, + { + "height": 1, + "name": "res", + "claim_id": "bbb", + "nout": 0, + "txid": "aecfaewcfa" + } + ], + "expected": [ + { + "height": 1, + "name": "res", + "claim_id": "aaa", + "nout": 0, + "txid": "w5tv8uorgt" + }, + { + "height": 1, + "name": "res", + "claim_id": "bbb", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res", + "claim_id": "ccc", + "nout": 0, + "txid": "fdsafa" + } + ] + } + ] +} \ No newline at end of file diff --git a/lbrynet/tests/unit/daemon/test_claims_comparator.py b/lbrynet/tests/unit/daemon/test_claims_comparator.py index 9651c2137..07fcbc503 100644 --- a/lbrynet/tests/unit/daemon/test_claims_comparator.py +++ b/lbrynet/tests/unit/daemon/test_claims_comparator.py @@ -1,51 +1,26 @@ +import json import unittest from lbrynet.daemon.claims_comparator import arrange_results class ClaimsComparatorTest(unittest.TestCase): + def setUp(self): + with open('claims_comparator_cases.json') as f: + document = json.load(f) + self.cases = document['cases'] + def test_arrange_results(self): - results = [ - { - 'height': 1, - 'name': 'res', - 'claim_id': 'ccc' - }, - { - 'height': 1, - 'name': 'res', - 'claim_id': 'aaa' - }, - { - 'height': 1, - 'name': 'res', - 'claim_id': 'bbb' - } - ] - data = {'result': results} + for case in self.cases: + results = case['results'] + data = {'result': results} + expected = case['expected'] - expected = [ - { - 'height': 1, - 'name': 'res', - 'claim_id': 'aaa' - }, - { - 'height': 1, - 'name': 'res', - 'claim_id': 'bbb' - }, - { - 'height': 1, - 'name': 'res', - 'claim_id': 'ccc' - } - ] - claims = arrange_results([data]) - claim = claims[0] - actual = claim['result'] + claims = arrange_results([data]) + claim = claims[0] + actual = claim['result'] - self.assertEqual(expected, actual) + self.assertEqual(expected, actual, case['description']) if __name__ == '__main__': From 156b1320707290b57182f246de6327c3ba27a004 Mon Sep 17 00:00:00 2001 From: Sergey Rozhnov Date: Thu, 17 May 2018 13:49:09 +0400 Subject: [PATCH 17/29] implemented additional test cases for claims sorting; updated changelog --- CHANGELOG.md | 1 + .../unit/daemon/claims_comparator_cases.json | 147 ++++++++++++++++++ 2 files changed, 148 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7acba6ca..539595609 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,7 @@ at anytime. * refactored dht iterativeFind * sort dht contacts returned by `findCloseNodes` in the routing table * disabled Cryptonator price feed + * `claim_list` and `claim_list_mine` in Daemon `return` sorted results ### Added * virtual kademlia network and mock udp transport for dht integration tests diff --git a/lbrynet/tests/unit/daemon/claims_comparator_cases.json b/lbrynet/tests/unit/daemon/claims_comparator_cases.json index dbcb732b7..11592fbf1 100644 --- a/lbrynet/tests/unit/daemon/claims_comparator_cases.json +++ b/lbrynet/tests/unit/daemon/claims_comparator_cases.json @@ -48,6 +48,153 @@ "txid": "fdsafa" } ] + }, + { + "description": "sort by height", + "results": [ + { + "height": 1, + "name": "res", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 3, + "name": "res", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 2, + "name": "res", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + } + ], + "expected": [ + { + "claim_id": "ccc", + "height": 1, + "name": "res", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "claim_id": "ccc", + "height": 2, + "name": "res", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "claim_id": "ccc", + "height": 3, + "name": "res", + "nout": 0, + "txid": "aecfaewcfa" + } + ] + }, + { + "description": "sort by name", + "results": [ + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res3", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res2", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + } + ], + "expected": [ + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res2", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res3", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + } + ] + }, + { + "description": "sort by outpoint", + "results": [ + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 2, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 1, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 3, + "txid": "aecfaewcfa" + } + ], + "expected": [ + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 1, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 2, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 3, + "txid": "aecfaewcfa" + } + ] } ] } \ No newline at end of file From c290fb5908bae0cf77472f1dfb2591f3cef2e544 Mon Sep 17 00:00:00 2001 From: Sergey Rozhnov Date: Fri, 18 May 2018 12:53:32 +0400 Subject: [PATCH 18/29] refactored sorting of claims and unit tests --- lbrynet/daemon/Daemon.py | 1 - lbrynet/daemon/claims_comparator.py | 10 - .../unit/daemon/claims_comparator_cases.json | 200 ---------------- .../unit/daemon/test_claims_comparator.py | 217 ++++++++++++++++-- 4 files changed, 202 insertions(+), 226 deletions(-) delete mode 100644 lbrynet/daemon/claims_comparator.py delete mode 100644 lbrynet/tests/unit/daemon/claims_comparator_cases.json diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index 648b8a70b..909c538e3 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -36,7 +36,6 @@ from lbrynet.daemon.Downloader import GetStream from lbrynet.daemon.Publisher import Publisher from lbrynet.daemon.ExchangeRateManager import ExchangeRateManager from lbrynet.daemon.auth.server import AuthJSONRPCServer -from lbrynet.daemon.claims_comparator import arrange_results from lbrynet.core.PaymentRateManager import OnlyFreePaymentsManager from lbrynet.core import utils, system_info from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob diff --git a/lbrynet/daemon/claims_comparator.py b/lbrynet/daemon/claims_comparator.py deleted file mode 100644 index 44d27a955..000000000 --- a/lbrynet/daemon/claims_comparator.py +++ /dev/null @@ -1,10 +0,0 @@ -def arrange_results(claims): - for claim in claims: - results = claim['result'] - sorted_results = sorted(results, key=lambda d: (d['height'], d['name'], d['claim_id'], _outpoint(d))) - claim['result'] = sorted_results - return claims - - -def _outpoint(claim): - return '{}:{}'.format(claim['txid'], claim['nout']) diff --git a/lbrynet/tests/unit/daemon/claims_comparator_cases.json b/lbrynet/tests/unit/daemon/claims_comparator_cases.json deleted file mode 100644 index 11592fbf1..000000000 --- a/lbrynet/tests/unit/daemon/claims_comparator_cases.json +++ /dev/null @@ -1,200 +0,0 @@ -{ - "cases": [ - { - "description": "sort by claim_id", - "results": [ - { - "height": 1, - "name": "res", - "claim_id": "ccc", - "nout": 0, - "txid": "fdsafa" - }, - { - "height": 1, - "name": "res", - "claim_id": "aaa", - "nout": 0, - "txid": "w5tv8uorgt" - }, - { - "height": 1, - "name": "res", - "claim_id": "bbb", - "nout": 0, - "txid": "aecfaewcfa" - } - ], - "expected": [ - { - "height": 1, - "name": "res", - "claim_id": "aaa", - "nout": 0, - "txid": "w5tv8uorgt" - }, - { - "height": 1, - "name": "res", - "claim_id": "bbb", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res", - "claim_id": "ccc", - "nout": 0, - "txid": "fdsafa" - } - ] - }, - { - "description": "sort by height", - "results": [ - { - "height": 1, - "name": "res", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 3, - "name": "res", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 2, - "name": "res", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - } - ], - "expected": [ - { - "claim_id": "ccc", - "height": 1, - "name": "res", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "claim_id": "ccc", - "height": 2, - "name": "res", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "claim_id": "ccc", - "height": 3, - "name": "res", - "nout": 0, - "txid": "aecfaewcfa" - } - ] - }, - { - "description": "sort by name", - "results": [ - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res3", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res2", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - } - ], - "expected": [ - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res2", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res3", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - } - ] - }, - { - "description": "sort by outpoint", - "results": [ - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 2, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 1, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 3, - "txid": "aecfaewcfa" - } - ], - "expected": [ - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 1, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 2, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 3, - "txid": "aecfaewcfa" - } - ] - } - ] -} \ No newline at end of file diff --git a/lbrynet/tests/unit/daemon/test_claims_comparator.py b/lbrynet/tests/unit/daemon/test_claims_comparator.py index 07fcbc503..416a394d5 100644 --- a/lbrynet/tests/unit/daemon/test_claims_comparator.py +++ b/lbrynet/tests/unit/daemon/test_claims_comparator.py @@ -1,26 +1,213 @@ -import json import unittest -from lbrynet.daemon.claims_comparator import arrange_results +from lbrynet.daemon.Daemon import arrange_results class ClaimsComparatorTest(unittest.TestCase): - def setUp(self): - with open('claims_comparator_cases.json') as f: - document = json.load(f) - self.cases = document['cases'] + def test_arrange_results_when_sorted_by_claim_id(self): + self.run_test( + [ + { + "height": 1, + "name": "res", + "claim_id": "ccc", + "nout": 0, + "txid": "fdsafa" + }, + { + "height": 1, + "name": "res", + "claim_id": "aaa", + "nout": 0, + "txid": "w5tv8uorgt" + }, + { + "height": 1, + "name": "res", + "claim_id": "bbb", + "nout": 0, + "txid": "aecfaewcfa" + } + ], + [ + { + "height": 1, + "name": "res", + "claim_id": "aaa", + "nout": 0, + "txid": "w5tv8uorgt" + }, + { + "height": 1, + "name": "res", + "claim_id": "bbb", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res", + "claim_id": "ccc", + "nout": 0, + "txid": "fdsafa" + } + ]) - def test_arrange_results(self): - for case in self.cases: - results = case['results'] - data = {'result': results} - expected = case['expected'] + def test_arrange_results_when_sorted_by_height(self): + self.run_test( + [ + { + "height": 1, + "name": "res", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 3, + "name": "res", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 2, + "name": "res", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + } + ], + [ + { + "claim_id": "ccc", + "height": 1, + "name": "res", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "claim_id": "ccc", + "height": 2, + "name": "res", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "claim_id": "ccc", + "height": 3, + "name": "res", + "nout": 0, + "txid": "aecfaewcfa" + } + ]) - claims = arrange_results([data]) - claim = claims[0] - actual = claim['result'] + def test_arrange_results_when_sorted_by_name(self): + self.run_test( + [ + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res3", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res2", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + } + ], + [ + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res2", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res3", + "claim_id": "ccc", + "nout": 0, + "txid": "aecfaewcfa" + } + ]) - self.assertEqual(expected, actual, case['description']) + def test_arrange_results_when_sort_by_outpoint(self): + self.run_test( + [ + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 2, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 1, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 3, + "txid": "aecfaewcfa" + } + ], + [ + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 1, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 2, + "txid": "aecfaewcfa" + }, + { + "height": 1, + "name": "res1", + "claim_id": "ccc", + "nout": 3, + "txid": "aecfaewcfa" + } + ]) + + def run_test(self, results, expected): + data = {'result': results} + + claims = arrange_results([data]) + claim = claims[0] + actual = claim['result'] + + self.assertEqual(expected, actual) if __name__ == '__main__': From d66bce814c4017f139e2427121de8798dfd660d2 Mon Sep 17 00:00:00 2001 From: Sergey Rozhnov Date: Fri, 18 May 2018 18:51:28 +0400 Subject: [PATCH 19/29] refactored unit test for sort_claim_results --- lbrynet/daemon/Daemon.py | 2 +- .../unit/daemon/test_claims_comparator.py | 228 +++--------------- 2 files changed, 31 insertions(+), 199 deletions(-) diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index 909c538e3..23ecc7d67 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -2329,7 +2329,7 @@ class Daemon(AuthJSONRPCServer): """ d = self.session.wallet.get_name_claims() - d.addCallback(arrange_results) + d.addCallback(sort_claim_results) d.addCallback(lambda claims: self._render_response(claims)) return d diff --git a/lbrynet/tests/unit/daemon/test_claims_comparator.py b/lbrynet/tests/unit/daemon/test_claims_comparator.py index 416a394d5..772ecb5e5 100644 --- a/lbrynet/tests/unit/daemon/test_claims_comparator.py +++ b/lbrynet/tests/unit/daemon/test_claims_comparator.py @@ -1,213 +1,45 @@ import unittest -from lbrynet.daemon.Daemon import arrange_results +from lbrynet.daemon.Daemon import sort_claim_results class ClaimsComparatorTest(unittest.TestCase): - def test_arrange_results_when_sorted_by_claim_id(self): - self.run_test( - [ - { - "height": 1, - "name": "res", - "claim_id": "ccc", - "nout": 0, - "txid": "fdsafa" - }, - { - "height": 1, - "name": "res", - "claim_id": "aaa", - "nout": 0, - "txid": "w5tv8uorgt" - }, - { - "height": 1, - "name": "res", - "claim_id": "bbb", - "nout": 0, - "txid": "aecfaewcfa" - } - ], - [ - { - "height": 1, - "name": "res", - "claim_id": "aaa", - "nout": 0, - "txid": "w5tv8uorgt" - }, - { - "height": 1, - "name": "res", - "claim_id": "bbb", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res", - "claim_id": "ccc", - "nout": 0, - "txid": "fdsafa" - } - ]) + def test_sort_claim_results_when_sorted_by_claim_id(self): + results = [{"height": 1, "name": "res", "claim_id": "ccc", "nout": 0, "txid": "fdsafa"}, + {"height": 1, "name": "res", "claim_id": "aaa", "nout": 0, "txid": "w5tv8uorgt"}, + {"height": 1, "name": "res", "claim_id": "bbb", "nout": 0, "txid": "aecfaewcfa"}] + self.run_test(results, 'claim_id', ['aaa', 'bbb', 'ccc']) - def test_arrange_results_when_sorted_by_height(self): - self.run_test( - [ - { - "height": 1, - "name": "res", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 3, - "name": "res", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 2, - "name": "res", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - } - ], - [ - { - "claim_id": "ccc", - "height": 1, - "name": "res", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "claim_id": "ccc", - "height": 2, - "name": "res", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "claim_id": "ccc", - "height": 3, - "name": "res", - "nout": 0, - "txid": "aecfaewcfa" - } - ]) + def test_sort_claim_results_when_sorted_by_height(self): + results = [{"height": 1, "name": "res", "claim_id": "ccc", "nout": 0, "txid": "aecfaewcfa"}, + {"height": 3, "name": "res", "claim_id": "ccc", "nout": 0, "txid": "aecfaewcfa"}, + {"height": 2, "name": "res", "claim_id": "ccc", "nout": 0, "txid": "aecfaewcfa"}] + self.run_test(results, 'height', [1, 2, 3]) - def test_arrange_results_when_sorted_by_name(self): - self.run_test( - [ - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res3", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res2", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - } - ], - [ - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res2", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res3", - "claim_id": "ccc", - "nout": 0, - "txid": "aecfaewcfa" - } - ]) + def test_sort_claim_results_when_sorted_by_name(self): + results = [{"height": 1, "name": "res1", "claim_id": "ccc", "nout": 0, "txid": "aecfaewcfa"}, + {"height": 1, "name": "res3", "claim_id": "ccc", "nout": 0, "txid": "aecfaewcfa"}, + {"height": 1, "name": "res2", "claim_id": "ccc", "nout": 0, "txid": "aecfaewcfa"}] + self.run_test(results, 'name', ['res1', 'res2', 'res3']) - def test_arrange_results_when_sort_by_outpoint(self): - self.run_test( - [ - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 2, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 1, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 3, - "txid": "aecfaewcfa" - } - ], - [ - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 1, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 2, - "txid": "aecfaewcfa" - }, - { - "height": 1, - "name": "res1", - "claim_id": "ccc", - "nout": 3, - "txid": "aecfaewcfa" - } - ]) + def test_sort_claim_results_when_sorted_by_txid(self): + results = [{"height": 1, "name": "res1", "claim_id": "ccc", "nout": 2, "txid": "111"}, + {"height": 1, "name": "res1", "claim_id": "ccc", "nout": 1, "txid": "222"}, + {"height": 1, "name": "res1", "claim_id": "ccc", "nout": 3, "txid": "333"}] + self.run_test(results, 'txid', ['111', '222', '333']) - def run_test(self, results, expected): + def test_sort_claim_results_when_sorted_by_nout(self): + results = [{"height": 1, "name": "res1", "claim_id": "ccc", "nout": 2, "txid": "aecfaewcfa"}, + {"height": 1, "name": "res1", "claim_id": "ccc", "nout": 1, "txid": "aecfaewcfa"}, + {"height": 1, "name": "res1", "claim_id": "ccc", "nout": 3, "txid": "aecfaewcfa"}] + self.run_test(results, 'nout', [1, 2, 3]) + + def run_test(self, results, field, expected): data = {'result': results} - - claims = arrange_results([data]) + claims = sort_claim_results([data]) claim = claims[0] actual = claim['result'] - - self.assertEqual(expected, actual) + self.assertEqual(expected, [r[field] for r in actual]) if __name__ == '__main__': From 0ae89f01e6cc9c65f02e907cd95f982343827274 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 22 May 2018 17:15:34 -0400 Subject: [PATCH 20/29] Revert "WIP: feature/1098/sorted-claim-results" --- lbrynet/daemon/Daemon.py | 1 - lbrynet/tests/unit/daemon/__init__.py | 0 .../unit/daemon/test_claims_comparator.py | 46 ------------------- 3 files changed, 47 deletions(-) delete mode 100644 lbrynet/tests/unit/daemon/__init__.py delete mode 100644 lbrynet/tests/unit/daemon/test_claims_comparator.py diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index 23ecc7d67..6fa9fb148 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -2329,7 +2329,6 @@ class Daemon(AuthJSONRPCServer): """ d = self.session.wallet.get_name_claims() - d.addCallback(sort_claim_results) d.addCallback(lambda claims: self._render_response(claims)) return d diff --git a/lbrynet/tests/unit/daemon/__init__.py b/lbrynet/tests/unit/daemon/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/lbrynet/tests/unit/daemon/test_claims_comparator.py b/lbrynet/tests/unit/daemon/test_claims_comparator.py deleted file mode 100644 index 772ecb5e5..000000000 --- a/lbrynet/tests/unit/daemon/test_claims_comparator.py +++ /dev/null @@ -1,46 +0,0 @@ -import unittest - -from lbrynet.daemon.Daemon import sort_claim_results - - -class ClaimsComparatorTest(unittest.TestCase): - def test_sort_claim_results_when_sorted_by_claim_id(self): - results = [{"height": 1, "name": "res", "claim_id": "ccc", "nout": 0, "txid": "fdsafa"}, - {"height": 1, "name": "res", "claim_id": "aaa", "nout": 0, "txid": "w5tv8uorgt"}, - {"height": 1, "name": "res", "claim_id": "bbb", "nout": 0, "txid": "aecfaewcfa"}] - self.run_test(results, 'claim_id', ['aaa', 'bbb', 'ccc']) - - def test_sort_claim_results_when_sorted_by_height(self): - results = [{"height": 1, "name": "res", "claim_id": "ccc", "nout": 0, "txid": "aecfaewcfa"}, - {"height": 3, "name": "res", "claim_id": "ccc", "nout": 0, "txid": "aecfaewcfa"}, - {"height": 2, "name": "res", "claim_id": "ccc", "nout": 0, "txid": "aecfaewcfa"}] - self.run_test(results, 'height', [1, 2, 3]) - - def test_sort_claim_results_when_sorted_by_name(self): - results = [{"height": 1, "name": "res1", "claim_id": "ccc", "nout": 0, "txid": "aecfaewcfa"}, - {"height": 1, "name": "res3", "claim_id": "ccc", "nout": 0, "txid": "aecfaewcfa"}, - {"height": 1, "name": "res2", "claim_id": "ccc", "nout": 0, "txid": "aecfaewcfa"}] - self.run_test(results, 'name', ['res1', 'res2', 'res3']) - - def test_sort_claim_results_when_sorted_by_txid(self): - results = [{"height": 1, "name": "res1", "claim_id": "ccc", "nout": 2, "txid": "111"}, - {"height": 1, "name": "res1", "claim_id": "ccc", "nout": 1, "txid": "222"}, - {"height": 1, "name": "res1", "claim_id": "ccc", "nout": 3, "txid": "333"}] - self.run_test(results, 'txid', ['111', '222', '333']) - - def test_sort_claim_results_when_sorted_by_nout(self): - results = [{"height": 1, "name": "res1", "claim_id": "ccc", "nout": 2, "txid": "aecfaewcfa"}, - {"height": 1, "name": "res1", "claim_id": "ccc", "nout": 1, "txid": "aecfaewcfa"}, - {"height": 1, "name": "res1", "claim_id": "ccc", "nout": 3, "txid": "aecfaewcfa"}] - self.run_test(results, 'nout', [1, 2, 3]) - - def run_test(self, results, field, expected): - data = {'result': results} - claims = sort_claim_results([data]) - claim = claims[0] - actual = claim['result'] - self.assertEqual(expected, [r[field] for r in actual]) - - -if __name__ == '__main__': - unittest.main() From c411d8700beeeaddd7174df2f4382be1117904cd Mon Sep 17 00:00:00 2001 From: jleute Date: Thu, 17 May 2018 23:41:23 +0200 Subject: [PATCH 21/29] fixes #1109 --- CHANGELOG.md | 1 + lbrynet/conf.py | 27 +++++++++++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 539595609..82c854868 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ at anytime. * `use_auth_http` in a config file being overridden by the default command line argument to `lbrynet-daemon`, now the command line value will only override the config file value if it is provided * `lbrynet-cli` not automatically switching to the authenticated client if the server is detected to be using authentication. This resulted in `lbrynet-cli` failing to run when `lbrynet-daemon` was run with the `--http-auth` flag * fixed error when using `claim_show` with `txid` and `nout` arguments + * fixed error when saving server list to conf file (issue #1109) ### Deprecated * diff --git a/lbrynet/conf.py b/lbrynet/conf.py index ddd4b4c7d..1ef435ede 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -168,6 +168,8 @@ def server_port(server_and_port): def server_list(servers): return [server_port(server) for server in servers] +def server_list_reverse(servers): + return ["%s:%s" % (server, port) for server, port in servers] class Env(envparse.Env): """An Env parser that automatically namespaces the variables with LBRY""" @@ -267,7 +269,7 @@ ADJUSTABLE_SETTINGS = { 'is_generous_host': (bool, True), 'announce_head_blobs_only': (bool, True), 'concurrent_announcers': (int, DEFAULT_CONCURRENT_ANNOUNCERS), - 'known_dht_nodes': (list, DEFAULT_DHT_NODES, server_list), + 'known_dht_nodes': (list, DEFAULT_DHT_NODES, server_list, server_list_reverse), 'lbryum_wallet_dir': (str, default_lbryum_dir), 'max_connections_per_stream': (int, 5), 'seek_head_blob_first': (bool, True), @@ -285,7 +287,7 @@ ADJUSTABLE_SETTINGS = { # event the initial upload failed or was disconnected part way through, provided the auto_re_reflect_interval > 0) 'reflect_uploads': (bool, True), 'auto_re_reflect_interval': (int, 86400), # set to 0 to disable - 'reflector_servers': (list, [('reflector2.lbry.io', 5566)], server_list), + 'reflector_servers': (list, [('reflector2.lbry.io', 5566)], server_list, server_list_reverse), 'run_reflector_server': (bool, False), 'sd_download_timeout': (int, 3), 'share_usage_data': (bool, True), # whether to share usage stats and diagnostic info with LBRY @@ -295,7 +297,8 @@ ADJUSTABLE_SETTINGS = { 'use_keyring': (bool, False), 'wallet': (str, LBRYUM_WALLET), 'blockchain_name': (str, 'lbrycrd_main'), - 'lbryum_servers': (list, [('lbryum8.lbry.io', 50001), ('lbryum9.lbry.io', 50001)], server_list), + 'lbryum_servers': (list, [('lbryum8.lbry.io', 50001), ('lbryum9.lbry.io', + 50001)], server_list, server_list_reverse), 's3_headers_depth': (int, 96 * 10) # download headers from s3 when the local height is more than 10 chunks behind } @@ -497,18 +500,30 @@ class Config(object): path = conf_file else: path = self.get_conf_filename() - + # reverse the conversions done after loading the settings from the conf + # file + rev = self._convert_conf_file_lists_reverse(self._data[TYPE_PERSISTED]) ext = os.path.splitext(path)[1] encoder = settings_encoders.get(ext, False) assert encoder is not False, 'Unknown settings format %s' % ext with open(path, 'w') as settings_file: - settings_file.write(encoder(self._data[TYPE_PERSISTED])) + settings_file.write(encoder(rev)) + + @staticmethod + def _convert_conf_file_lists_reverse(converted): + rev = {} + for k in converted.iterkeys(): + if k in ADJUSTABLE_SETTINGS and len(ADJUSTABLE_SETTINGS[k]) == 4: + rev[k] = ADJUSTABLE_SETTINGS[k][3](converted[k]) + else: + rev[k] = converted[k] + return rev @staticmethod def _convert_conf_file_lists(decoded): converted = {} for k, v in decoded.iteritems(): - if k in ADJUSTABLE_SETTINGS and len(ADJUSTABLE_SETTINGS[k]) == 3: + if k in ADJUSTABLE_SETTINGS and len(ADJUSTABLE_SETTINGS[k]) >= 3: converted[k] = ADJUSTABLE_SETTINGS[k][2](v) else: converted[k] = v From 2e3ceedc1706a13d6d618f043019e3a247dea26f Mon Sep 17 00:00:00 2001 From: jleute Date: Fri, 18 May 2018 02:54:40 +0200 Subject: [PATCH 22/29] added unit test for reversal of conversion of config file entries --- lbrynet/tests/unit/test_conf.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lbrynet/tests/unit/test_conf.py b/lbrynet/tests/unit/test_conf.py index c18ed1f83..3baa5f7ab 100644 --- a/lbrynet/tests/unit/test_conf.py +++ b/lbrynet/tests/unit/test_conf.py @@ -77,3 +77,18 @@ class SettingsTest(unittest.TestCase): self.assertEqual(str, type(conf.default_download_dir)) self.assertEqual(str, type(conf.default_data_dir)) self.assertEqual(str, type(conf.default_lbryum_dir)) + + def test_conversion_reversal(self): + # simulate decoding, conversion, conversion reversal and encoding of + # server list settings from the config file. + settings = self.get_mock_config_instance() + encoder = conf.settings_encoders['.yml'] + decoder = conf.settings_decoders['.yml'] + conf_file_entry = "lbryum_servers: ['localhost:5001', 'localhost:5002']" + decoded = decoder(conf_file_entry) + converted = settings._convert_conf_file_lists(decoded) + converted_reversed = settings._convert_conf_file_lists_reverse(converted) + encoded = encoder(converted_reversed) + self.assertEqual(conf_file_entry, encoded.strip()) + self.assertEqual(decoded, converted_reversed) + From 5926b17871b3af82fa062d55e164ae8686e98a65 Mon Sep 17 00:00:00 2001 From: jleute Date: Sat, 19 May 2018 01:32:51 +0200 Subject: [PATCH 23/29] Updated unittest for saving of server lists in the conf file --- lbrynet/conf.py | 3 ++- lbrynet/tests/unit/test_conf.py | 29 ++++++++++++++++------------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 1ef435ede..7b068a017 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -538,7 +538,6 @@ class Config(object): path = conf_file else: path = self.get_conf_filename() - ext = os.path.splitext(path)[1] decoder = settings_decoders.get(ext, False) assert decoder is not False, 'Unknown settings format %s' % ext @@ -645,6 +644,8 @@ def get_default_env(): for k, v in ADJUSTABLE_SETTINGS.iteritems(): if len(v) == 3: env_defaults[k] = (v[0], None, v[2]) + elif len(v) == 4: + env_defaults[k] = (v[0], None, v[2], v[3]) else: env_defaults[k] = (v[0], None) return Env(**env_defaults) diff --git a/lbrynet/tests/unit/test_conf.py b/lbrynet/tests/unit/test_conf.py index 3baa5f7ab..87d5c03fc 100644 --- a/lbrynet/tests/unit/test_conf.py +++ b/lbrynet/tests/unit/test_conf.py @@ -4,6 +4,8 @@ import json from twisted.trial import unittest from lbrynet import conf from lbrynet.core.Error import InvalidCurrencyError +from lbrynet.tests import mocks +from lbrynet.tests.util import create_conf_file class SettingsTest(unittest.TestCase): def setUp(self): @@ -78,17 +80,18 @@ class SettingsTest(unittest.TestCase): self.assertEqual(str, type(conf.default_data_dir)) self.assertEqual(str, type(conf.default_lbryum_dir)) - def test_conversion_reversal(self): - # simulate decoding, conversion, conversion reversal and encoding of - # server list settings from the config file. - settings = self.get_mock_config_instance() - encoder = conf.settings_encoders['.yml'] - decoder = conf.settings_decoders['.yml'] - conf_file_entry = "lbryum_servers: ['localhost:5001', 'localhost:5002']" - decoded = decoder(conf_file_entry) - converted = settings._convert_conf_file_lists(decoded) - converted_reversed = settings._convert_conf_file_lists_reverse(converted) - encoded = encoder(converted_reversed) - self.assertEqual(conf_file_entry, encoded.strip()) - self.assertEqual(decoded, converted_reversed) + def test_load_save_load_config_file(self): + #settings = self.get_mock_config_instance() + conf_entry = 'lbryum_servers: ["localhost:50001", "localhost:50002"]\n' + conf_temp = create_conf_file(conf_entry) + conf.conf_file = conf_temp + adjustable_settings={'data_dir': (str, conf.default_data_dir), + 'lbryum_servers': (list, [('localhost', 5001)], + conf.server_list, conf.server_list_reverse)} + env = conf.Env(**adjustable_settings) + settings = conf.Config({}, adjustable_settings, environment=env) + conf.settings = settings + settings.load_conf_file_settings() + settings.save_conf_file_settings() + settings.load_conf_file_settings() From e00238427b5cb3a091f0fd1e47f9bf149fc674bd Mon Sep 17 00:00:00 2001 From: jleute Date: Sat, 19 May 2018 02:15:37 +0200 Subject: [PATCH 24/29] add assert to test_load_save_load_config_file test --- lbrynet/tests/unit/test_conf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lbrynet/tests/unit/test_conf.py b/lbrynet/tests/unit/test_conf.py index 87d5c03fc..9159244ec 100644 --- a/lbrynet/tests/unit/test_conf.py +++ b/lbrynet/tests/unit/test_conf.py @@ -81,7 +81,6 @@ class SettingsTest(unittest.TestCase): self.assertEqual(str, type(conf.default_lbryum_dir)) def test_load_save_load_config_file(self): - #settings = self.get_mock_config_instance() conf_entry = 'lbryum_servers: ["localhost:50001", "localhost:50002"]\n' conf_temp = create_conf_file(conf_entry) conf.conf_file = conf_temp @@ -92,6 +91,8 @@ class SettingsTest(unittest.TestCase): settings = conf.Config({}, adjustable_settings, environment=env) conf.settings = settings settings.load_conf_file_settings() + first = settings.get('lbryum_servers', data_type=conf.TYPE_PERSISTED) settings.save_conf_file_settings() settings.load_conf_file_settings() - + second = settings.get('lbryum_servers', data_type=conf.TYPE_PERSISTED) + self.assertEqual(first, second) From cfe5eb5d56c9a4d15e5829ce078474bc04bab665 Mon Sep 17 00:00:00 2001 From: jleute Date: Sat, 19 May 2018 02:42:45 +0200 Subject: [PATCH 25/29] clean up temporary conf file --- lbrynet/tests/unit/test_conf.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lbrynet/tests/unit/test_conf.py b/lbrynet/tests/unit/test_conf.py index 9159244ec..aedf4f6cc 100644 --- a/lbrynet/tests/unit/test_conf.py +++ b/lbrynet/tests/unit/test_conf.py @@ -5,7 +5,7 @@ from twisted.trial import unittest from lbrynet import conf from lbrynet.core.Error import InvalidCurrencyError from lbrynet.tests import mocks -from lbrynet.tests.util import create_conf_file +from lbrynet.tests.util import create_conf_file, remove_conf_file class SettingsTest(unittest.TestCase): def setUp(self): @@ -84,7 +84,7 @@ class SettingsTest(unittest.TestCase): conf_entry = 'lbryum_servers: ["localhost:50001", "localhost:50002"]\n' conf_temp = create_conf_file(conf_entry) conf.conf_file = conf_temp - adjustable_settings={'data_dir': (str, conf.default_data_dir), + adjustable_settings = {'data_dir': (str, conf.default_data_dir), 'lbryum_servers': (list, [('localhost', 5001)], conf.server_list, conf.server_list_reverse)} env = conf.Env(**adjustable_settings) @@ -93,6 +93,12 @@ class SettingsTest(unittest.TestCase): settings.load_conf_file_settings() first = settings.get('lbryum_servers', data_type=conf.TYPE_PERSISTED) settings.save_conf_file_settings() - settings.load_conf_file_settings() + try: + settings.load_conf_file_settings() + except Exception, e: + remove_conf_file(conf_temp) + raise Exception(e) second = settings.get('lbryum_servers', data_type=conf.TYPE_PERSISTED) + remove_conf_file(conf_temp) self.assertEqual(first, second) + From ea7ef541749dea6c4f94c5fc90ae0ca1eb3fad53 Mon Sep 17 00:00:00 2001 From: jleute Date: Sat, 19 May 2018 09:44:15 +0200 Subject: [PATCH 26/29] remove unused import --- lbrynet/tests/unit/test_conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lbrynet/tests/unit/test_conf.py b/lbrynet/tests/unit/test_conf.py index aedf4f6cc..06be39600 100644 --- a/lbrynet/tests/unit/test_conf.py +++ b/lbrynet/tests/unit/test_conf.py @@ -4,7 +4,6 @@ import json from twisted.trial import unittest from lbrynet import conf from lbrynet.core.Error import InvalidCurrencyError -from lbrynet.tests import mocks from lbrynet.tests.util import create_conf_file, remove_conf_file class SettingsTest(unittest.TestCase): From acbe3b0f6593f2aa55036938ea5da69340609001 Mon Sep 17 00:00:00 2001 From: jleute Date: Mon, 21 May 2018 18:50:25 +0200 Subject: [PATCH 27/29] updated test to compare original and saved config file --- lbrynet/tests/unit/test_conf.py | 34 ++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/lbrynet/tests/unit/test_conf.py b/lbrynet/tests/unit/test_conf.py index 06be39600..d605bf6dc 100644 --- a/lbrynet/tests/unit/test_conf.py +++ b/lbrynet/tests/unit/test_conf.py @@ -1,10 +1,10 @@ import os import json +import tempfile from twisted.trial import unittest from lbrynet import conf from lbrynet.core.Error import InvalidCurrencyError -from lbrynet.tests.util import create_conf_file, remove_conf_file class SettingsTest(unittest.TestCase): def setUp(self): @@ -79,25 +79,29 @@ class SettingsTest(unittest.TestCase): self.assertEqual(str, type(conf.default_data_dir)) self.assertEqual(str, type(conf.default_lbryum_dir)) - def test_load_save_load_config_file(self): - conf_entry = 'lbryum_servers: ["localhost:50001", "localhost:50002"]\n' - conf_temp = create_conf_file(conf_entry) - conf.conf_file = conf_temp + def test_load_save_config_file(self): + # setup settings adjustable_settings = {'data_dir': (str, conf.default_data_dir), 'lbryum_servers': (list, [('localhost', 5001)], conf.server_list, conf.server_list_reverse)} env = conf.Env(**adjustable_settings) settings = conf.Config({}, adjustable_settings, environment=env) conf.settings = settings - settings.load_conf_file_settings() - first = settings.get('lbryum_servers', data_type=conf.TYPE_PERSISTED) - settings.save_conf_file_settings() - try: + # setup tempfile + conf_entry = "lbryum_servers: ['localhost:50001', 'localhost:50002']\n" + with tempfile.NamedTemporaryFile(suffix='.yml') as conf_file: + conf_file.write(conf_entry) + conf_file.seek(0) + conf.conf_file = conf_file.name + # load and save settings from conf file settings.load_conf_file_settings() - except Exception, e: - remove_conf_file(conf_temp) - raise Exception(e) - second = settings.get('lbryum_servers', data_type=conf.TYPE_PERSISTED) - remove_conf_file(conf_temp) - self.assertEqual(first, second) + settings.save_conf_file_settings() + # test if overwritten entry equals original entry + # use decoded versions, because format might change without + # changing the interpretation + decoder = conf.settings_decoders['.yml'] + conf_decoded = decoder(conf_entry) + conf_entry_new = conf_file.read() + conf_decoded_new = decoder(conf_entry_new) + self.assertEqual(conf_decoded, conf_decoded_new) From 4b9079bf25cd4ba47d9c8a049af958a400056713 Mon Sep 17 00:00:00 2001 From: jleute Date: Sat, 9 Jun 2018 00:47:34 +0200 Subject: [PATCH 28/29] doesn't belong here. From 9735373eefca2ac77f8085352206472651ddaf1d Mon Sep 17 00:00:00 2001 From: jleute Date: Tue, 12 Jun 2018 00:10:23 +0200 Subject: [PATCH 29/29] remove dependency on server_list_reverse --- lbrynet/tests/unit/test_conf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lbrynet/tests/unit/test_conf.py b/lbrynet/tests/unit/test_conf.py index d605bf6dc..1fa493351 100644 --- a/lbrynet/tests/unit/test_conf.py +++ b/lbrynet/tests/unit/test_conf.py @@ -82,8 +82,7 @@ class SettingsTest(unittest.TestCase): def test_load_save_config_file(self): # setup settings adjustable_settings = {'data_dir': (str, conf.default_data_dir), - 'lbryum_servers': (list, [('localhost', 5001)], - conf.server_list, conf.server_list_reverse)} + 'lbryum_servers': (list, [])} env = conf.Env(**adjustable_settings) settings = conf.Config({}, adjustable_settings, environment=env) conf.settings = settings