From fb34476cd7695a01b9b5617f70321b2b1f6d6677 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Thu, 18 Oct 2018 15:10:00 -0400 Subject: [PATCH 1/4] add upnp status dictionary to status response -log upnp debugging information as an error --- CHANGELOG.md | 1 + lbrynet/daemon/Components.py | 14 +++++++++++++- lbrynet/daemon/Daemon.py | 6 ++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5220e444f..4dda8bc9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ most commands. * added `address_unused` command to get existing or generate a new unused address. * added pagination support for `address_list`, `channel_list`, `claim_list_mine`, `transaction_list` and `utxo_list`. + * added `upnp` field to `status` response * removed `send_amount_to_address` command previously marked as deprecated * removed `channel_list_mine` command previously marked as deprecated * removed `get_availability` command previously marked as deprecated diff --git a/lbrynet/daemon/Components.py b/lbrynet/daemon/Components.py index f85bcde66..3c2bb95c0 100644 --- a/lbrynet/daemon/Components.py +++ b/lbrynet/daemon/Components.py @@ -718,12 +718,18 @@ class UPnPComponent(Component): else: log.info("got external ip from upnp: %s", self.external_ip) yield self._setup_redirects() - log.info("set up upnp port redirects") except Exception as err: log.warning("error trying to set up upnp: %s", err) self.external_ip = CS.get_external_ip() else: self.external_ip = CS.get_external_ip() + if self.upnp: + if not self.upnp_redirects: + log.error("failed to setup upnp, debugging infomation: %s", self.upnp.zipped_debugging_info) + else: + log.info("set up upnp port redirects for gateway: %s", self.upnp.gateway.manufacturer_string) + else: + log.error("failed to setup upnp") def stop(self): return defer.DeferredList( @@ -731,6 +737,12 @@ class UPnPComponent(Component): for protocol, port in self.upnp_redirects.items()] ) + def get_status(self): + return { + 'redirects': self.upnp_redirects, + 'gateway': '' if not self.upnp else self.upnp.gateway.manufacturer_string, + } + class ExchangeRateManagerComponent(Component): component_name = EXCHANGE_RATE_MANAGER_COMPONENT diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index 3e1c033fd..6738e78c2 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -771,6 +771,12 @@ class Daemon(AuthJSONRPCServer): }, 'file_manager': { 'managed_files': (int) count of files in the file manager, + }, + 'upnp': { + 'redirects': { + : (int) external_port, + }, + 'gateway': (str) manufacturer and model, } } """ From 6d8ab97ca9e633556ba9ba511328e52e3b96cfbe Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Thu, 18 Oct 2018 15:57:15 -0400 Subject: [PATCH 2/4] fix analytics --- lbrynet/core/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/core/utils.py b/lbrynet/core/utils.py index 9b45ea4c6..64f8e7478 100644 --- a/lbrynet/core/utils.py +++ b/lbrynet/core/utils.py @@ -94,7 +94,7 @@ def rot13(some_str): def deobfuscate(obfustacated): - return base64.b64decode(rot13(obfustacated)) + return base64.b64decode(rot13(obfustacated)).decode() def obfuscate(plain): From 213bdac55fbb6dd351c34c994a1d3be17e6a581f Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Thu, 18 Oct 2018 17:41:49 -0400 Subject: [PATCH 3/4] check upnp redirects and external ip in a looping call -add upnp setup success/fail analytics event --- lbrynet/analytics.py | 9 ++++ lbrynet/daemon/Components.py | 102 +++++++++++++++++++++++++++-------- lbrynet/daemon/Daemon.py | 3 ++ 3 files changed, 91 insertions(+), 23 deletions(-) diff --git a/lbrynet/analytics.py b/lbrynet/analytics.py index 4b7645e69..10de2bbaa 100644 --- a/lbrynet/analytics.py +++ b/lbrynet/analytics.py @@ -19,6 +19,7 @@ CLAIM_ACTION = 'Claim Action' # publish/create/update/abandon NEW_CHANNEL = 'New Channel' CREDITS_SENT = 'Credits Sent' NEW_DOWNLOAD_STAT = 'Download' +UPNP_SETUP = "UPnP Setup" BLOB_BYTES_UPLOADED = 'Blob Bytes Uploaded' @@ -69,6 +70,14 @@ class Manager: 'timestamp': utils.isonow(), }) + def send_upnp_setup_success_fail(self, success, status): + self.analytics_api.track( + self._event(UPNP_SETUP, { + 'success': success, + 'status': status, + }) + ) + def send_server_startup(self): self.analytics_api.track(self._event(SERVER_STARTUP)) diff --git a/lbrynet/daemon/Components.py b/lbrynet/daemon/Components.py index 3c2bb95c0..66f2fef0e 100644 --- a/lbrynet/daemon/Components.py +++ b/lbrynet/daemon/Components.py @@ -6,9 +6,10 @@ import math import binascii from hashlib import sha256 from types import SimpleNamespace -from twisted.internet import defer, threads, reactor, error +from twisted.internet import defer, threads, reactor, error, task import lbryschema from aioupnp.upnp import UPnP +from aioupnp.fault import UPnPError from lbrynet import conf from lbrynet.core.utils import DeferredDict from lbrynet.core.PaymentRateManager import OnlyFreePaymentsManager @@ -684,6 +685,8 @@ class UPnPComponent(Component): self.upnp = None self.upnp_redirects = {} self.external_ip = None + self._maintain_redirects_lc = task.LoopingCall(self._maintain_redirects) + self._maintain_redirects_lc.clock = self.component_manager.reactor @property def component(self): @@ -697,41 +700,91 @@ class UPnPComponent(Component): }) self.upnp_redirects.update(upnp_redirects) + @defer.inlineCallbacks + def _maintain_redirects(self): + # setup the gateway if necessary + if not self.upnp: + try: + self.upnp = yield from_future(UPnP.discover()) + log.info("found upnp gateway: %s", self.upnp.gateway.manufacturer_string) + except Exception as err: + log.warning("upnp discovery failed: %s", err) + return + + # update the external ip + try: + external_ip = yield from_future(self.upnp.get_external_ip()) + if external_ip == "0.0.0.0": + log.warning("upnp doesn't know the external ip address (returned 0.0.0.0), using fallback") + external_ip = CS.get_external_ip() + if self.external_ip and self.external_ip != external_ip: + log.info("external ip changed from %s to %s", self.external_ip, external_ip) + elif not self.external_ip: + log.info("got external ip: %s", external_ip) + self.external_ip = external_ip + except (asyncio.TimeoutError, UPnPError): + pass + + if not self.upnp_redirects: # setup missing redirects + try: + upnp_redirects = yield DeferredDict({ + "UDP": from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")), + "TCP": from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")) + }) + self.upnp_redirects.update(upnp_redirects) + except (asyncio.TimeoutError, UPnPError): + self.upnp = None + return self._maintain_redirects() + else: # check existing redirects are still active + found = set() + mappings = yield from_future(self.upnp.get_redirects()) + for mapping in mappings: + proto = mapping['NewProtocol'] + if proto in self.upnp_redirects and mapping['NewExternalPort'] == self.upnp_redirects[proto]: + if mapping['NewInternalClient'] == self.upnp.lan_address: + found.add(proto) + if 'UDP' not in found: + try: + udp_port = yield from_future( + self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port") + ) + self.upnp_redirects['UDP'] = udp_port + log.info("refreshed upnp redirect for dht port: %i", udp_port) + except (asyncio.TimeoutError, UPnPError): + del self.upnp_redirects['UDP'] + if 'TCP' not in found: + try: + tcp_port = yield from_future( + self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port") + ) + self.upnp_redirects['TCP'] = tcp_port + log.info("refreshed upnp redirect for peer port: %i", tcp_port) + except (asyncio.TimeoutError, UPnPError): + del self.upnp_redirects['TCP'] + if 'TCP' in self.upnp_redirects and 'UDP' in self.upnp_redirects: + log.debug("upnp redirects are still active") + @defer.inlineCallbacks def start(self): if not self.use_upnp: self.external_ip = CS.get_external_ip() return - try: - self.upnp = yield from_future(UPnP.discover()) - log.info("found upnp gateway") - found = True - except Exception as err: - log.warning("upnp discovery failed: %s", err) - found = False - if found: - try: - self.external_ip = yield from_future(self.upnp.get_external_ip()) - if self.external_ip == "0.0.0.0": - log.warning("upnp doesn't know the external ip address (returned 0.0.0.0), using fallback") - self.external_ip = CS.get_external_ip() - else: - log.info("got external ip from upnp: %s", self.external_ip) - yield self._setup_redirects() - except Exception as err: - log.warning("error trying to set up upnp: %s", err) - self.external_ip = CS.get_external_ip() - else: - self.external_ip = CS.get_external_ip() + success = False + yield self._maintain_redirects() if self.upnp: if not self.upnp_redirects: log.error("failed to setup upnp, debugging infomation: %s", self.upnp.zipped_debugging_info) else: - log.info("set up upnp port redirects for gateway: %s", self.upnp.gateway.manufacturer_string) + success = True + log.debug("set up upnp port redirects for gateway: %s", self.upnp.gateway.manufacturer_string) else: log.error("failed to setup upnp") + self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, self.get_status()) + self._maintain_redirects_lc.start(360, now=False) def stop(self): + if self._maintain_redirects_lc.running: + self._maintain_redirects_lc.stop() return defer.DeferredList( [from_future(self.upnp.delete_port_mapping(port, protocol)) for protocol, port in self.upnp_redirects.items()] @@ -741,6 +794,9 @@ class UPnPComponent(Component): return { 'redirects': self.upnp_redirects, 'gateway': '' if not self.upnp else self.upnp.gateway.manufacturer_string, + 'dht_redirect_set': 'UDP' in self.upnp_redirects, + 'peer_redirect_set': 'TCP' in self.upnp_redirects, + 'external_ip': self.external_ip } diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index 6738e78c2..e040ed912 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -777,6 +777,9 @@ class Daemon(AuthJSONRPCServer): : (int) external_port, }, 'gateway': (str) manufacturer and model, + 'dht_redirect_set': (bool), + 'peer_redirect_set': (bool), + 'external_ip': (str) external ip address, } } """ From 0ed56bbe1ad21e8d9d88a430ffa8c95ec856fe08 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Thu, 18 Oct 2018 17:53:16 -0400 Subject: [PATCH 4/4] tests --- tests/unit/core/test_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/core/test_utils.py b/tests/unit/core/test_utils.py index 3ca5817b2..208b45ad3 100644 --- a/tests/unit/core/test_utils.py +++ b/tests/unit/core/test_utils.py @@ -23,13 +23,13 @@ class CompareVersionTest(unittest.TestCase): class ObfuscationTest(unittest.TestCase): def test_deobfuscation_reverses_obfuscation(self): - plain = "my_test_string".encode() - obf = utils.obfuscate(plain) + plain = "my_test_string" + obf = utils.obfuscate(plain.encode()) self.assertEqual(plain, utils.deobfuscate(obf)) def test_can_use_unicode(self): - plain = '☃'.encode() - obf = utils.obfuscate(plain) + plain = '☃' + obf = utils.obfuscate(plain.encode()) self.assertEqual(plain, utils.deobfuscate(obf))