fallback to getting external ip from spv servers instead of internal apis

lbry/dht/peer.py

@@ -1,14 +1,14 @@
 import typing
 import asyncio
 import logging
-import ipaddress
 from binascii import hexlify
 from dataclasses import dataclass, field
 from functools import lru_cache
-
+from lbry.utils import is_valid_public_ipv4 as _is_valid_public_ipv4
 from lbry.dht import constants
 from lbry.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
 
+ALLOW_LOCALHOST = False
 log = logging.getLogger(__name__)
 
 
@@ -20,28 +20,9 @@ def make_kademlia_peer(node_id: typing.Optional[bytes], address: typing.Optional
     return KademliaPeer(address, node_id, udp_port, tcp_port=tcp_port, allow_localhost=allow_localhost)
 
 
-# the ipaddress module does not show these subnets as reserved
-CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
-IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
-
-ALLOW_LOCALHOST = False
-
-
 def is_valid_public_ipv4(address, allow_localhost: bool = False):
     allow_localhost = bool(allow_localhost or ALLOW_LOCALHOST)
-    try:
+    return _is_valid_public_ipv4(address, allow_localhost)
-        parsed_ip = ipaddress.ip_address(address)
-        if parsed_ip.is_loopback and allow_localhost:
-            return True
-
-        if any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local, parsed_ip.is_loopback,
-                parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private, parsed_ip.is_reserved)):
-            return False
-        else:
-            return not any((CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
-                            IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
-    except (ipaddress.AddressValueError, ValueError):
-        return False
 
 
 class PeerManager:

lbry/extras/daemon/analytics.py

@@ -132,7 +132,7 @@ class AnalyticsManager:
     async def run(self):
         while True:
             if self.enabled:
-                self.external_ip = await utils.get_external_ip()
+                self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
                 await self._send_heartbeat()
             await asyncio.sleep(1800)
 

lbry/extras/daemon/components.py

@@ -275,7 +275,7 @@ class DHTComponent(Component):
         external_ip = upnp_component.external_ip
         storage = self.component_manager.get_component(DATABASE_COMPONENT)
         if not external_ip:
-            external_ip = await utils.get_external_ip()
+            external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
             if not external_ip:
                 log.warning("failed to get external ip")
 
@@ -476,7 +476,7 @@ class UPnPComponent(Component):
                 pass
         if external_ip and not is_valid_public_ipv4(external_ip):
             log.warning("UPnP returned a private/reserved ip - %s, checking lbry.com fallback", external_ip)
-            external_ip = await utils.get_external_ip()
+            external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
         if self.external_ip and self.external_ip != external_ip:
             log.info("external ip changed from %s to %s", self.external_ip, external_ip)
         if external_ip:
@@ -534,7 +534,7 @@ class UPnPComponent(Component):
     async def start(self):
         log.info("detecting external ip")
         if not self.use_upnp:
-            self.external_ip = await utils.get_external_ip()
+            self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
             return
         success = False
         await self._maintain_redirects()
@@ -549,9 +549,9 @@ class UPnPComponent(Component):
         else:
             log.error("failed to setup upnp")
         if not self.external_ip:
-            self.external_ip = await utils.get_external_ip()
+            self.external_ip, probed_url = await utils.get_external_ip(self.conf.lbryum_servers)
             if self.external_ip:
-                log.info("detected external ip using lbry.com fallback")
+                log.info("detected external ip using %s fallback", probed_url)
         if self.component_manager.analytics_manager:
             self.component_manager.loop.create_task(
                 self.component_manager.analytics_manager.send_upnp_setup_success_fail(

lbry/utils.py

@@ -379,14 +379,80 @@ async def aiohttp_request(method, url, **kwargs) -> typing.AsyncContextManager[a
             yield response
 
 
-async def get_external_ip() -> typing.Optional[str]:  # used if upnp is disabled or non-functioning
+# the ipaddress module does not show these subnets as reserved
+CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
+IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
+
+
+def is_valid_public_ipv4(address, allow_localhost: bool = False):
+    try:
+        parsed_ip = ipaddress.ip_address(address)
+        if parsed_ip.is_loopback and allow_localhost:
+            return True
+        if any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local, parsed_ip.is_loopback,
+                parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private, parsed_ip.is_reserved)):
+            return False
+        else:
+            return not any((CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
+                            IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
+    except (ipaddress.AddressValueError, ValueError):
+        return False
+
+
+async def fallback_get_external_ip():  # used if spv servers can't be used for ip detection
     try:
         async with aiohttp_request("get", "https://api.lbry.com/ip") as resp:
             response = await resp.json()
             if response['success']:
-                return response['data']['ip']
+                return response['data']['ip'], None
     except Exception:
-        return
+        return None, None
+
+
+async def _get_external_ip(default_servers) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]:
+    # used if upnp is disabled or non-functioning
+    from lbry.wallet.server.udp import SPVStatusClientProtocol  # pylint: disable=C0415
+
+    hostname_to_ip = {}
+    ip_to_hostnames = collections.defaultdict(list)
+
+    async def resolve_spv(server, port):
+        try:
+            server_addr = await resolve_host(server, port, 'udp')
+            hostname_to_ip[server] = (server_addr, port)
+            ip_to_hostnames[(server_addr, port)].append(server)
+        except Exception:
+            log.exception("error looking up dns for spv servers")
+
+    # accumulate the dns results
+    await asyncio.gather(*(resolve_spv(server, port) for (server, port) in default_servers))
+
+    loop = asyncio.get_event_loop()
+    pong_responses = asyncio.Queue()
+    connection = SPVStatusClientProtocol(pong_responses)
+    try:
+        await loop.create_datagram_endpoint(lambda: connection, ('0.0.0.0', 0))
+        # could raise OSError if it cant bind
+        randomized_servers = list(ip_to_hostnames.keys())
+        random.shuffle(randomized_servers)
+        for server in randomized_servers:
+            connection.ping(server)
+            try:
+                _, pong = await asyncio.wait_for(pong_responses.get(), 1)
+                if is_valid_public_ipv4(pong.ip_address):
+                    return pong.ip_address, ip_to_hostnames[server][0]
+            except asyncio.TimeoutError:
+                pass
+        return None, None
+    finally:
+        connection.close()
+
+
+async def get_external_ip(default_servers) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]:
+    ip_from_spv_servers = await _get_external_ip(default_servers)
+    if not ip_from_spv_servers[1]:
+        return await fallback_get_external_ip()
+    return ip_from_spv_servers
 
 
 def is_running_from_bundle():