Madiator2011/lbry-sdk
1
0
Fork 0
forked from LBRYCommunity/lbry-sdk
Code Pull requests Activity

Merge pull request #3148 from lbryio/udp-wallet-server-status

Browse source 
UDP based spv server status, improved server selection
This commit is contained in:
Jack Robison 2021-01-21 20:55:16 -05:00 committed by GitHub
commit 50e17eb1ab
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 631 additions and 291 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
lbry/blob/blob_manager.py
View file

@ -2,7 +2,7 @@ import os
import typing
import asyncio
import logging
from lbry.utils import LRUCache
from lbry.utils import LRUCacheWithMetrics
from lbry.blob.blob_file import is_valid_blobhash, BlobFile, BlobBuffer, AbstractBlob
from lbry.stream.descriptor import StreamDescriptor
from lbry.connection_manager import ConnectionManager
@ -32,7 +32,7 @@ class BlobManager:
else self._node_data_store.completed_blobs
self.blobs: typing.Dict[str, AbstractBlob] = {}
self.config = config
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCache(
self.decrypted_blob_lru_cache = None if not self.config.blob_lru_cache_size else LRUCacheWithMetrics(
self.config.blob_lru_cache_size)
self.connection_manager = ConnectionManager(loop)

25
lbry/dht/peer.py
View file

@ -1,14 +1,14 @@
import typing
import asyncio
import logging
import ipaddress
from binascii import hexlify
from dataclasses import dataclass, field
from functools import lru_cache
from lbry.utils import is_valid_public_ipv4 as _is_valid_public_ipv4
from lbry.dht import constants
from lbry.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address
ALLOW_LOCALHOST = False
log = logging.getLogger(__name__)
@ -20,28 +20,9 @@ def make_kademlia_peer(node_id: typing.Optional[bytes], address: typing.Optional
return KademliaPeer(address, node_id, udp_port, tcp_port=tcp_port, allow_localhost=allow_localhost)
# the ipaddress module does not show these subnets as reserved
CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
ALLOW_LOCALHOST = False
def is_valid_public_ipv4(address, allow_localhost: bool = False):
allow_localhost = bool(allow_localhost or ALLOW_LOCALHOST)
try:
parsed_ip = ipaddress.ip_address(address)
if parsed_ip.is_loopback and allow_localhost:
return True
if any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local, parsed_ip.is_loopback,
parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private, parsed_ip.is_reserved)):
return False
else:
return not any((CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
except (ipaddress.AddressValueError, ValueError):
return False
return _is_valid_public_ipv4(address, allow_localhost)
class PeerManager:

2
lbry/extras/daemon/analytics.py
View file

@ -132,7 +132,7 @@ class AnalyticsManager:
async def run(self):
while True:
if self.enabled:
self.external_ip = await utils.get_external_ip()
self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
await self._send_heartbeat()
await asyncio.sleep(1800)

36
lbry/extras/daemon/components.py
View file

@ -119,13 +119,14 @@ class WalletComponent(Component):
async def get_status(self):
if self.wallet_manager is None:
return
session_pool = self.wallet_manager.ledger.network.session_pool
sessions = session_pool.sessions
is_connected = self.wallet_manager.ledger.network.is_connected
sessions = []
connected = None
if self.wallet_manager.ledger.network.client:
addr_and_port = self.wallet_manager.ledger.network.client.server_address_and_port
if addr_and_port:
connected = f"{addr_and_port[0]}:{addr_and_port[1]}"
if is_connected:
addr, port = self.wallet_manager.ledger.network.client.server
connected = f"{addr}:{port}"
sessions.append(self.wallet_manager.ledger.network.client)
result = {
'connected': connected,
'connected_features': self.wallet_manager.ledger.network.server_features,
@ -137,8 +138,8 @@ class WalletComponent(Component):
'availability': session.available,
} for session in sessions
],
'known_servers': len(sessions),
'available_servers': len(list(session_pool.available_sessions))
'known_servers': len(self.wallet_manager.ledger.network.config['default_servers']),
'available_servers': 1 if is_connected else 0
}
if self.wallet_manager.ledger.network.remote_height:
@ -274,7 +275,7 @@ class DHTComponent(Component):
external_ip = upnp_component.external_ip
storage = self.component_manager.get_component(DATABASE_COMPONENT)
if not external_ip:
external_ip = await utils.get_external_ip()
external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
if not external_ip:
log.warning("failed to get external ip")
@ -328,7 +329,7 @@ class HashAnnouncerComponent(Component):
class FileManagerComponent(Component):
component_name = FILE_MANAGER_COMPONENT
depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT, LIBTORRENT_COMPONENT]
depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
@ -351,7 +352,10 @@ class FileManagerComponent(Component):
wallet = self.component_manager.get_component(WALLET_COMPONENT)
node = self.component_manager.get_component(DHT_COMPONENT) \
if self.component_manager.has_component(DHT_COMPONENT) else None
torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT) if TorrentSession else None
try:
torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT) if TorrentSession else None
except NameError:
torrent = None
log.info('Starting the file manager')
loop = asyncio.get_event_loop()
self.file_manager = FileManager(
@ -360,7 +364,7 @@ class FileManagerComponent(Component):
self.file_manager.source_managers['stream'] = StreamManager(
loop, self.conf, blob_manager, wallet, storage, node,
)
if TorrentSession:
if TorrentSession and LIBTORRENT_COMPONENT not in self.conf.components_to_skip:
self.file_manager.source_managers['torrent'] = TorrentManager(
loop, self.conf, torrent, storage, self.component_manager.analytics_manager
)
@ -472,7 +476,7 @@ class UPnPComponent(Component):
pass
if external_ip and not is_valid_public_ipv4(external_ip):
log.warning("UPnP returned a private/reserved ip - %s, checking lbry.com fallback", external_ip)
external_ip = await utils.get_external_ip()
external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
if self.external_ip and self.external_ip != external_ip:
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
if external_ip:
@ -530,7 +534,7 @@ class UPnPComponent(Component):
async def start(self):
log.info("detecting external ip")
if not self.use_upnp:
self.external_ip = await utils.get_external_ip()
self.external_ip, _ = await utils.get_external_ip(self.conf.lbryum_servers)
return
success = False
await self._maintain_redirects()
@ -545,9 +549,9 @@ class UPnPComponent(Component):
else:
log.error("failed to setup upnp")
if not self.external_ip:
self.external_ip = await utils.get_external_ip()
self.external_ip, probed_url = await utils.get_external_ip(self.conf.lbryum_servers)
if self.external_ip:
log.info("detected external ip using lbry.com fallback")
log.info("detected external ip using %s fallback", probed_url)
if self.component_manager.analytics_manager:
self.component_manager.loop.create_task(
self.component_manager.analytics_manager.send_upnp_setup_success_fail(

8
lbry/testcase.py
View file

@ -23,8 +23,9 @@ from lbry.wallet.orchstr8.node import BlockchainNode, WalletNode
from lbry.extras.daemon.daemon import Daemon, jsonrpc_dumps_pretty
from lbry.extras.daemon.components import Component, WalletComponent
from lbry.extras.daemon.components import (
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
DHT_COMPONENT,
HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, LIBTORRENT_COMPONENT
)
from lbry.extras.daemon.componentmanager import ComponentManager
from lbry.extras.daemon.exchange_rate_manager import (
@ -320,6 +321,7 @@ class CommandTestCase(IntegrationTestCase):
self.server_blob_manager = None
self.server = None
self.reflector = None
self.skip_libtorrent = True
async def asyncSetUp(self):
await super().asyncSetUp()
@ -395,6 +397,8 @@ class CommandTestCase(IntegrationTestCase):
DHT_COMPONENT, UPNP_COMPONENT, HASH_ANNOUNCER_COMPONENT,
PEER_PROTOCOL_SERVER_COMPONENT
]
if self.skip_libtorrent:
conf.components_to_skip.append(LIBTORRENT_COMPONENT)
wallet_node.manager.config = conf
def wallet_maker(component_manager):

133
lbry/utils.py
View file

@ -192,6 +192,8 @@ def cache_concurrent(async_fn):
async def resolve_host(url: str, port: int, proto: str) -> str:
if proto not in ['udp', 'tcp']:
raise Exception("invalid protocol")
if url.lower() == 'localhost':
return '127.0.0.1'
try:
if ipaddress.ip_address(url):
return url
@ -206,7 +208,7 @@ async def resolve_host(url: str, port: int, proto: str) -> str:
))[0][4][0]
class LRUCache:
class LRUCacheWithMetrics:
__slots__ = [
'capacity',
'cache',
@ -231,7 +233,7 @@ class LRUCache:
f"{metric_name}_cache_miss_count", "Number of cache misses", namespace=namespace
)
except ValueError as err:
log.warning("failed to set up prometheus %s_cache_miss_count metric: %s", metric_name, err)
log.debug("failed to set up prometheus %s_cache_miss_count metric: %s", metric_name, err)
self._track_metrics = False
self.hits = self.misses = None
@ -286,12 +288,63 @@ class LRUCache:
pass
class LRUCache:
__slots__ = [
'capacity',
'cache'
]
def __init__(self, capacity: int):
self.capacity = capacity
self.cache = collections.OrderedDict()
def get(self, key, default=None):
try:
value = self.cache.pop(key)
except KeyError:
return default
self.cache[key] = value
return value
def set(self, key, value):
try:
self.cache.pop(key)
except KeyError:
if len(self.cache) >= self.capacity:
self.cache.popitem(last=False)
self.cache[key] = value
def clear(self):
self.cache.clear()
def pop(self, key):
return self.cache.pop(key)
def __setitem__(self, key, value):
return self.set(key, value)
def __getitem__(self, item):
return self.get(item)
def __contains__(self, item) -> bool:
return item in self.cache
def __len__(self):
return len(self.cache)
def __delitem__(self, key):
self.cache.pop(key)
def __del__(self):
self.clear()
def lru_cache_concurrent(cache_size: typing.Optional[int] = None,
override_lru_cache: typing.Optional[LRUCache] = None):
override_lru_cache: typing.Optional[LRUCacheWithMetrics] = None):
if not cache_size and override_lru_cache is None:
raise ValueError("invalid cache size")
concurrent_cache = {}
lru_cache = override_lru_cache if override_lru_cache is not None else LRUCache(cache_size)
lru_cache = override_lru_cache if override_lru_cache is not None else LRUCacheWithMetrics(cache_size)
def wrapper(async_fn):
@ -326,14 +379,80 @@ async def aiohttp_request(method, url, **kwargs) -> typing.AsyncContextManager[a
yield response
async def get_external_ip() -> typing.Optional[str]: # used if upnp is disabled or non-functioning
# the ipaddress module does not show these subnets as reserved
CARRIER_GRADE_NAT_SUBNET = ipaddress.ip_network('100.64.0.0/10')
IPV4_TO_6_RELAY_SUBNET = ipaddress.ip_network('192.88.99.0/24')
def is_valid_public_ipv4(address, allow_localhost: bool = False):
try:
parsed_ip = ipaddress.ip_address(address)
if parsed_ip.is_loopback and allow_localhost:
return True
if any((parsed_ip.version != 4, parsed_ip.is_unspecified, parsed_ip.is_link_local, parsed_ip.is_loopback,
parsed_ip.is_multicast, parsed_ip.is_reserved, parsed_ip.is_private, parsed_ip.is_reserved)):
return False
else:
return not any((CARRIER_GRADE_NAT_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32")),
IPV4_TO_6_RELAY_SUBNET.supernet_of(ipaddress.ip_network(f"{address}/32"))))
except (ipaddress.AddressValueError, ValueError):
return False
async def fallback_get_external_ip(): # used if spv servers can't be used for ip detection
try:
async with aiohttp_request("get", "https://api.lbry.com/ip") as resp:
response = await resp.json()
if response['success']:
return response['data']['ip']
return response['data']['ip'], None
except Exception:
return
return None, None
async def _get_external_ip(default_servers) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]:
# used if upnp is disabled or non-functioning
from lbry.wallet.server.udp import SPVStatusClientProtocol # pylint: disable=C0415
hostname_to_ip = {}
ip_to_hostnames = collections.defaultdict(list)
async def resolve_spv(server, port):
try:
server_addr = await resolve_host(server, port, 'udp')
hostname_to_ip[server] = (server_addr, port)
ip_to_hostnames[(server_addr, port)].append(server)
except Exception:
log.exception("error looking up dns for spv servers")
# accumulate the dns results
await asyncio.gather(*(resolve_spv(server, port) for (server, port) in default_servers))
loop = asyncio.get_event_loop()
pong_responses = asyncio.Queue()
connection = SPVStatusClientProtocol(pong_responses)
try:
await loop.create_datagram_endpoint(lambda: connection, ('0.0.0.0', 0))
# could raise OSError if it cant bind
randomized_servers = list(ip_to_hostnames.keys())
random.shuffle(randomized_servers)
for server in randomized_servers:
connection.ping(server)
try:
_, pong = await asyncio.wait_for(pong_responses.get(), 1)
if is_valid_public_ipv4(pong.ip_address):
return pong.ip_address, ip_to_hostnames[server][0]
except asyncio.TimeoutError:
pass
return None, None
finally:
connection.close()
async def get_external_ip(default_servers) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]:
ip_from_spv_servers = await _get_external_ip(default_servers)
if not ip_from_spv_servers[1]:
return await fallback_get_external_ip()
return ip_from_spv_servers
def is_running_from_bundle():

6
lbry/wallet/ledger.py
View file

@ -14,7 +14,7 @@ from lbry.schema.result import Outputs, INVALID, NOT_FOUND
from lbry.schema.url import URL
from lbry.crypto.hash import hash160, double_sha256, sha256
from lbry.crypto.base58 import Base58
from lbry.utils import LRUCache
from lbry.utils import LRUCacheWithMetrics
from .tasks import TaskGroup
from .database import Database
@ -155,7 +155,7 @@ class Ledger(metaclass=LedgerRegistry):
self._on_ready_controller = StreamController()
self.on_ready = self._on_ready_controller.stream
self._tx_cache = LRUCache(self.config.get("tx_cache_size", 1024), metric_name='tx')
self._tx_cache = LRUCacheWithMetrics(self.config.get("tx_cache_size", 1024), metric_name='tx')
self._update_tasks = TaskGroup()
self._other_tasks = TaskGroup() # that we dont need to start
self._utxo_reservation_lock = asyncio.Lock()
@ -167,7 +167,7 @@ class Ledger(metaclass=LedgerRegistry):
self._known_addresses_out_of_sync = set()
self.fee_per_name_char = self.config.get('fee_per_name_char', self.default_fee_per_name_char)
self._balance_cache = LRUCache(2 ** 15)
self._balance_cache = LRUCacheWithMetrics(2 ** 15)
@classmethod
def get_id(cls):

356
lbry/wallet/network.py
View file

@ -1,26 +1,27 @@
import logging
import asyncio
import json
import socket
from time import perf_counter
from operator import itemgetter
from collections import defaultdict
from typing import Dict, Optional, Tuple
import aiohttp
from lbry import __version__
from lbry.utils import resolve_host
from lbry.error import IncompatibleWalletServerError
from lbry.wallet.rpc import RPCSession as BaseClientSession, Connector, RPCError, ProtocolError
from lbry.wallet.stream import StreamController
from lbry.wallet.server.udp import SPVStatusClientProtocol, SPVPong
log = logging.getLogger(__name__)
class ClientSession(BaseClientSession):
def __init__(self, *args, network, server, timeout=30, on_connect_callback=None, **kwargs):
def __init__(self, *args, network: 'Network', server, timeout=30, **kwargs):
self.network = network
self.server = server
super().__init__(*args, **kwargs)
self._on_disconnect_controller = StreamController()
self.on_disconnected = self._on_disconnect_controller.stream
self.framer.max_size = self.max_errors = 1 << 32
self.timeout = timeout
self.max_seconds_idle = timeout * 2
@ -28,8 +29,6 @@ class ClientSession(BaseClientSession):
self.connection_latency: Optional[float] = None
self._response_samples = 0
self.pending_amount = 0
self._on_connect_cb = on_connect_callback or (lambda: None)
self.trigger_urgent_reconnect = asyncio.Event()
@property
def available(self):
@ -56,7 +55,7 @@ class ClientSession(BaseClientSession):
async def send_request(self, method, args=()):
self.pending_amount += 1
log.debug("send %s%s to %s:%i", method, tuple(args), *self.server)
log.debug("send %s%s to %s:%i (%i timeout)", method, tuple(args), self.server[0], self.server[1], self.timeout)
try:
if method == 'server.version':
return await self.send_timed_server_version_request(args, self.timeout)
@ -67,7 +66,7 @@ class ClientSession(BaseClientSession):
log.debug("Time since last packet: %s", perf_counter() - self.last_packet_received)
if (perf_counter() - self.last_packet_received) < self.timeout:
continue
log.info("timeout sending %s to %s:%i", method, *self.server)
log.warning("timeout sending %s to %s:%i", method, *self.server)
raise asyncio.TimeoutError
if done:
try:
@ -87,44 +86,12 @@ class ClientSession(BaseClientSession):
self.synchronous_close()
raise
except asyncio.CancelledError:
log.info("cancelled sending %s to %s:%i", method, *self.server)
log.warning("cancelled sending %s to %s:%i", method, *self.server)
# self.synchronous_close()
raise
finally:
self.pending_amount -= 1
async def ensure_session(self):
# Handles reconnecting and maintaining a session alive
# TODO: change to 'ping' on newer protocol (above 1.2)
retry_delay = default_delay = 1.0
while True:
try:
if self.is_closing():
await self.create_connection(self.timeout)
await self.ensure_server_version()
self._on_connect_cb()
if (perf_counter() - self.last_send) > self.max_seconds_idle or self.response_time is None:
await self.ensure_server_version()
retry_delay = default_delay
except RPCError as e:
await self.close()
log.debug("Server error, ignoring for 1h: %s:%d -- %s", *self.server, e.message)
retry_delay = 60 * 60
except IncompatibleWalletServerError:
await self.close()
retry_delay = 60 * 60
log.debug("Wallet server has an incompatible version, retrying in 1h: %s:%d", *self.server)
except (asyncio.TimeoutError, OSError):
await self.close()
retry_delay = min(60, retry_delay * 2)
log.debug("Wallet server timeout (retry in %s seconds): %s:%d", retry_delay, *self.server)
try:
await asyncio.wait_for(self.trigger_urgent_reconnect.wait(), timeout=retry_delay)
except asyncio.TimeoutError:
pass
finally:
self.trigger_urgent_reconnect.clear()
async def ensure_server_version(self, required=None, timeout=3):
required = required or self.network.PROTOCOL_VERSION
response = await asyncio.wait_for(
@ -134,6 +101,25 @@ class ClientSession(BaseClientSession):
raise IncompatibleWalletServerError(*self.server)
return response
async def keepalive_loop(self, timeout=3, max_idle=60):
try:
while True:
now = perf_counter()
if min(self.last_send, self.last_packet_received) + max_idle < now:
await asyncio.wait_for(
self.send_request('server.ping', []), timeout=timeout
)
else:
await asyncio.sleep(max(0, max_idle - (now - self.last_send)))
except Exception as err:
if isinstance(err, asyncio.CancelledError):
log.warning("closing connection to %s:%i", *self.server)
else:
log.exception("lost connection to spv")
finally:
if not self.is_closing():
self._close()
async def create_connection(self, timeout=6):
connector = Connector(lambda: self, *self.server)
start = perf_counter()
@ -145,12 +131,14 @@ class ClientSession(BaseClientSession):
controller.add(request.args)
def connection_lost(self, exc):
log.debug("Connection lost: %s:%d", *self.server)
log.warning("Connection lost: %s:%d", *self.server)
super().connection_lost(exc)
self.response_time = None
self.connection_latency = None
self._response_samples = 0
self._on_disconnect_controller.add(True)
# self._on_disconnect_controller.add(True)
if self.network:
self.network.disconnect()
class Network:
@ -160,10 +148,9 @@ class Network:
def __init__(self, ledger):
self.ledger = ledger
self.session_pool = SessionPool(network=self, timeout=self.config.get('connect_timeout', 6))
self.client: Optional[ClientSession] = None
self.server_features = None
self._switch_task: Optional[asyncio.Task] = None
# self._switch_task: Optional[asyncio.Task] = None
self.running = False
self.remote_height: int = 0
self._concurrency = asyncio.Semaphore(16)
@ -183,58 +170,170 @@ class Network:
}
self.aiohttp_session: Optional[aiohttp.ClientSession] = None
self._urgent_need_reconnect = asyncio.Event()
self._loop_task: Optional[asyncio.Task] = None
self._keepalive_task: Optional[asyncio.Task] = None
@property
def config(self):
return self.ledger.config
async def switch_forever(self):
while self.running:
if self.is_connected:
await self.client.on_disconnected.first
self.server_features = None
self.client = None
continue
self.client = await self.session_pool.wait_for_fastest_session()
log.info("Switching to SPV wallet server: %s:%d", *self.client.server)
try:
self.server_features = await self.get_server_features()
self._update_remote_height((await self.subscribe_headers(),))
self._on_connected_controller.add(True)
log.info("Subscribed to headers: %s:%d", *self.client.server)
except (asyncio.TimeoutError, ConnectionError):
log.info("Switching to %s:%d timed out, closing and retrying.", *self.client.server)
self.client.synchronous_close()
self.server_features = None
self.client = None
def disconnect(self):
if self._keepalive_task and not self._keepalive_task.done():
self._keepalive_task.cancel()
self._keepalive_task = None
async def start(self):
self.running = True
self.aiohttp_session = aiohttp.ClientSession()
self._switch_task = asyncio.ensure_future(self.switch_forever())
# this may become unnecessary when there are no more bugs found,
# but for now it helps understanding log reports
self._switch_task.add_done_callback(lambda _: log.info("Wallet client switching task stopped."))
self.session_pool.start(self.config['default_servers'])
self.on_header.listen(self._update_remote_height)
if not self.running:
self.running = True
self.aiohttp_session = aiohttp.ClientSession()
self.on_header.listen(self._update_remote_height)
self._loop_task = asyncio.create_task(self.network_loop())
self._urgent_need_reconnect.set()
def loop_task_done_callback(f):
try:
f.result()
except Exception:
if self.running:
log.exception("wallet server connection loop crashed")
self._loop_task.add_done_callback(loop_task_done_callback)
async def resolve_spv_dns(self):
hostname_to_ip = {}
ip_to_hostnames = defaultdict(list)
async def resolve_spv(server, port):
try:
server_addr = await resolve_host(server, port, 'udp')
hostname_to_ip[server] = (server_addr, port)
ip_to_hostnames[(server_addr, port)].append(server)
except socket.error:
log.warning("error looking up dns for spv server %s:%i", server, port)
except Exception:
log.exception("error looking up dns for spv server %s:%i", server, port)
# accumulate the dns results
await asyncio.gather(*(resolve_spv(server, port) for (server, port) in self.config['default_servers']))
return hostname_to_ip, ip_to_hostnames
async def get_n_fastest_spvs(self, n=5, timeout=3.0) -> Dict[Tuple[str, int], SPVPong]:
loop = asyncio.get_event_loop()
pong_responses = asyncio.Queue()
connection = SPVStatusClientProtocol(pong_responses)
sent_ping_timestamps = {}
_, ip_to_hostnames = await self.resolve_spv_dns()
log.info("%i possible spv servers to try (%i urls in config)", len(ip_to_hostnames),
len(self.config['default_servers']))
pongs = {}
try:
await loop.create_datagram_endpoint(lambda: connection, ('0.0.0.0', 0))
# could raise OSError if it cant bind
start = perf_counter()
for server in ip_to_hostnames:
connection.ping(server)
sent_ping_timestamps[server] = perf_counter()
while len(pongs) < n:
(remote, ts), pong = await asyncio.wait_for(pong_responses.get(), timeout - (perf_counter() - start))
latency = ts - start
log.info("%s:%i has latency of %sms (available: %s, height: %i)",
'/'.join(ip_to_hostnames[remote]), remote[1], round(latency * 1000, 2),
pong.available, pong.height)
if pong.available:
pongs[remote] = pong
return pongs
except asyncio.TimeoutError:
if pongs:
log.info("%i/%i probed spv servers are accepting connections", len(pongs), len(ip_to_hostnames))
else:
log.warning("%i spv status probes failed, retrying later. servers tried: %s",
len(sent_ping_timestamps),
', '.join('/'.join(hosts) + f' ({ip})' for ip, hosts in ip_to_hostnames.items()))
return pongs
finally:
connection.close()
async def connect_to_fastest(self) -> Optional[ClientSession]:
fastest_spvs = await self.get_n_fastest_spvs()
for (host, port) in fastest_spvs:
client = ClientSession(network=self, server=(host, port))
try:
await client.create_connection()
log.warning("Connected to spv server %s:%i", host, port)
await client.ensure_server_version()
return client
except (asyncio.TimeoutError, ConnectionError, OSError, IncompatibleWalletServerError, RPCError):
log.warning("Connecting to %s:%d failed", host, port)
client._close()
return
async def network_loop(self):
sleep_delay = 30
while self.running:
await asyncio.wait(
[asyncio.sleep(30), self._urgent_need_reconnect.wait()], return_when=asyncio.FIRST_COMPLETED
)
if self._urgent_need_reconnect.is_set():
sleep_delay = 30
self._urgent_need_reconnect.clear()
if not self.is_connected:
client = await self.connect_to_fastest()
if not client:
log.warning("failed to connect to any spv servers, retrying later")
sleep_delay *= 2
sleep_delay = min(sleep_delay, 300)
continue
log.debug("get spv server features %s:%i", *client.server)
features = await client.send_request('server.features', [])
self.client, self.server_features = client, features
log.info("subscribe to headers %s:%i", *client.server)
self._update_remote_height((await self.subscribe_headers(),))
self._on_connected_controller.add(True)
server_str = "%s:%i" % client.server
log.info("maintaining connection to spv server %s", server_str)
self._keepalive_task = asyncio.create_task(self.client.keepalive_loop())
try:
await asyncio.wait(
[self._keepalive_task, self._urgent_need_reconnect.wait()],
return_when=asyncio.FIRST_COMPLETED
)
if self._urgent_need_reconnect.is_set():
log.warning("urgent reconnect needed")
self._urgent_need_reconnect.clear()
if self._keepalive_task and not self._keepalive_task.done():
self._keepalive_task.cancel()
except asyncio.CancelledError:
pass
finally:
self._keepalive_task = None
self.client = None
self.server_features = None
log.warning("connection lost to %s", server_str)
log.info("network loop finished")
async def stop(self):
if self.running:
self.running = False
self.running = False
self.disconnect()
if self._loop_task and not self._loop_task.done():
self._loop_task.cancel()
self._loop_task = None
if self.aiohttp_session:
await self.aiohttp_session.close()
self._switch_task.cancel()
self.session_pool.stop()
self.aiohttp_session = None
@property
def is_connected(self):
return self.client and not self.client.is_closing()
def rpc(self, list_or_method, args, restricted=True, session=None):
session = session or (self.client if restricted else self.session_pool.fastest_session)
if session and not session.is_closing():
def rpc(self, list_or_method, args, restricted=True, session: Optional[ClientSession] = None):
if session or self.is_connected:
session = session or self.client
return session.send_request(list_or_method, args)
else:
self.session_pool.trigger_nodelay_connect()
self._urgent_need_reconnect.set()
raise ConnectionError("Attempting to send rpc request when connection is not available.")
async def retriable_call(self, function, *args, **kwargs):
@ -242,14 +341,15 @@ class Network:
while self.running:
if not self.is_connected:
log.warning("Wallet server unavailable, waiting for it to come back and retry.")
self._urgent_need_reconnect.set()
await self.on_connected.first
await self.session_pool.wait_for_fastest_session()
try:
return await function(*args, **kwargs)
except asyncio.TimeoutError:
log.warning("Wallet server call timed out, retrying.")
except ConnectionError:
pass
log.warning("connection error")
raise asyncio.CancelledError() # if we got here, we are shutting down
def _update_remote_height(self, header_args):
@ -339,95 +439,3 @@ class Network:
async with self.aiohttp_session.post(server, json=message) as r:
result = await r.json()
return result['result']
class SessionPool:
def __init__(self, network: Network, timeout: float):
self.network = network
self.sessions: Dict[ClientSession, Optional[asyncio.Task]] = dict()
self.timeout = timeout
self.new_connection_event = asyncio.Event()
@property
def online(self):
return any(not session.is_closing() for session in self.sessions)
@property
def available_sessions(self):
return (session for session in self.sessions if session.available)
@property
def fastest_session(self):
if not self.online:
return None
return min(
[((session.response_time + session.connection_latency) * (session.pending_amount + 1), session)
for session in self.available_sessions] or [(0, None)],
key=itemgetter(0)
)[1]
def _get_session_connect_callback(self, session: ClientSession):
loop = asyncio.get_event_loop()
def callback():
duplicate_connections = [
s for s in self.sessions
if s is not session and s.server_address_and_port == session.server_address_and_port
]
already_connected = None if not duplicate_connections else duplicate_connections[0]
if already_connected:
self.sessions.pop(session).cancel()
session.synchronous_close()
log.debug("wallet server %s resolves to the same server as %s, rechecking in an hour",
session.server[0], already_connected.server[0])
loop.call_later(3600, self._connect_session, session.server)
return
self.new_connection_event.set()
log.info("connected to %s:%i", *session.server)
return callback
def _connect_session(self, server: Tuple[str, int]):
session = None
for s in self.sessions:
if s.server == server:
session = s
break
if not session:
session = ClientSession(
network=self.network, server=server
)
session._on_connect_cb = self._get_session_connect_callback(session)
task = self.sessions.get(session, None)
if not task or task.done():
task = asyncio.create_task(session.ensure_session())
task.add_done_callback(lambda _: self.ensure_connections())
self.sessions[session] = task
def start(self, default_servers):
for server in default_servers:
self._connect_session(server)
def stop(self):
for session, task in self.sessions.items():
task.cancel()
session.synchronous_close()
self.sessions.clear()
def ensure_connections(self):
for session in self.sessions:
self._connect_session(session.server)
def trigger_nodelay_connect(self):
# used when other parts of the system sees we might have internet back
# bypasses the retry interval
for session in self.sessions:
session.trigger_urgent_reconnect.set()
async def wait_for_fastest_session(self):
while not self.fastest_session:
self.trigger_nodelay_connect()
self.new_connection_event.clear()
await self.new_connection_event.wait()
return self.fastest_session

7
lbry/wallet/server/block_processor.py
View file

@ -11,6 +11,7 @@ from lbry.wallet.server.daemon import DaemonError
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
from lbry.wallet.server.util import chunks, class_logger
from lbry.wallet.server.leveldb import FlushData
from lbry.wallet.server.udp import StatusServer
class Prefetcher:
@ -185,6 +186,7 @@ class BlockProcessor:
self.search_cache = {}
self.history_cache = {}
self.status_server = StatusServer()
async def run_in_thread_with_lock(self, func, *args):
# Run in a thread to prevent blocking. Shielded so that
@ -221,6 +223,7 @@ class BlockProcessor:
processed_time = time.perf_counter() - start
self.block_count_metric.set(self.height)
self.block_update_time_metric.observe(processed_time)
self.status_server.set_height(self.db.fs_height, self.db.db_tip)
if not self.db.first_sync:
s = '' if len(blocks) == 1 else 's'
self.logger.info('processed {:,d} block{} in {:.1f}s'.format(len(blocks), s, processed_time))
@ -682,9 +685,11 @@ class BlockProcessor:
disk before exiting, as otherwise a significant amount of work
could be lost.
"""
self._caught_up_event = caught_up_event
try:
await self._first_open_dbs()
self.status_server.set_height(self.db.fs_height, self.db.db_tip)
await asyncio.wait([
self.prefetcher.main_loop(self.height),
self._process_prefetched_blocks()
@ -695,6 +700,7 @@ class BlockProcessor:
self.logger.exception("Block processing failed!")
raise
finally:
self.status_server.stop()
# Shut down block processing
self.logger.info('flushing to DB for a clean shutdown...')
await self.flush(True)
@ -714,7 +720,6 @@ class BlockProcessor:
class Timer:
def __init__(self, name):
self.name = name
self.total = 0

6
lbry/wallet/server/daemon.py
View file

@ -6,7 +6,7 @@ from functools import wraps
import aiohttp
from prometheus_client import Gauge, Histogram
from lbry.utils import LRUCache
from lbry.utils import LRUCacheWithMetrics
from lbry.wallet.rpc.jsonrpc import RPCError
from lbry.wallet.server.util import hex_to_bytes, class_logger
from lbry.wallet.rpc import JSONRPC
@ -54,8 +54,8 @@ class Daemon:
self._height = None
self.available_rpcs = {}
self.connector = aiohttp.TCPConnector()
self._block_hash_cache = LRUCache(100000)
self._block_cache = LRUCache(2**16, metric_name='block', namespace=NAMESPACE)
self._block_hash_cache = LRUCacheWithMetrics(100000)
self._block_cache = LRUCacheWithMetrics(2 ** 16, metric_name='block', namespace=NAMESPACE)
async def close(self):
if self.connector:

4
lbry/wallet/server/leveldb.py
View file

@ -24,7 +24,7 @@ from glob import glob
from struct import pack, unpack
from concurrent.futures.thread import ThreadPoolExecutor
import attr
from lbry.utils import LRUCache
from lbry.utils import LRUCacheWithMetrics
from lbry.wallet.server import util
from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN
from lbry.wallet.server.merkle import Merkle, MerkleCache
@ -93,7 +93,7 @@ class LevelDB:
self.headers_db = None
self.tx_db = None
self._tx_and_merkle_cache = LRUCache(2**17, metric_name='tx_and_merkle', namespace="wallet_server")
self._tx_and_merkle_cache = LRUCacheWithMetrics(2 ** 17, metric_name='tx_and_merkle', namespace="wallet_server")
self.total_transactions = None
async def _read_tx_counts(self):

5
lbry/wallet/server/server.py
View file

@ -110,11 +110,14 @@ class Server:
self.cancellable_tasks.append(asyncio.ensure_future(run(*args, _flag)))
return _flag.wait()
await self.start_prometheus()
await self.bp.status_server.start(0, bytes.fromhex(self.bp.coin.GENESIS_HASH)[::-1]
, self.env.host, self.env.tcp_port)
await _start_cancellable(self.bp.fetch_and_process_blocks)
await self.db.populate_header_merkle_cache()
await _start_cancellable(self.mempool.keep_synchronized)
await _start_cancellable(self.session_mgr.serve, self.notifications)
await self.start_prometheus()
async def stop(self):
for task in reversed(self.cancellable_tasks):

25
lbry/wallet/server/session.py
View file

@ -21,7 +21,7 @@ from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from prometheus_client import Counter, Info, Histogram, Gauge
import lbry
from lbry.utils import LRUCache
from lbry.utils import LRUCacheWithMetrics
from lbry.build_info import BUILD, COMMIT_HASH, DOCKER_TAG
from lbry.wallet.server.block_processor import LBRYBlockProcessor
from lbry.wallet.server.db.writer import LBRYLevelDB
@ -247,11 +247,12 @@ class SessionManager:
async def _manage_servers(self):
paused = False
max_sessions = self.env.max_sessions
low_watermark = max_sessions * 19 // 20
low_watermark = int(max_sessions * 0.95)
while True:
await self.session_event.wait()
self.session_event.clear()
if not paused and len(self.sessions) >= max_sessions:
self.bp.status_server.set_unavailable()
self.logger.info(f'maximum sessions {max_sessions:,d} '
f'reached, stopping new connections until '
f'count drops to {low_watermark:,d}')
@ -260,6 +261,7 @@ class SessionManager:
# Start listening for incoming connections if paused and
# session count has fallen
if paused and len(self.sessions) <= low_watermark:
self.bp.status_server.set_available()
self.logger.info('resuming listening for incoming connections')
await self._start_external_servers()
paused = False
@ -572,6 +574,7 @@ class SessionManager:
await self.start_other()
await self._start_external_servers()
server_listening_event.set()
self.bp.status_server.set_available()
# Peer discovery should start after the external servers
# because we connect to ourself
await asyncio.wait([
@ -582,6 +585,7 @@ class SessionManager:
])
finally:
await self._close_servers(list(self.servers.keys()))
log.warning("disconnect %i sessions", len(self.sessions))
if self.sessions:
await asyncio.wait([
session.close(force_after=1) for session in self.sessions.values()
@ -810,8 +814,8 @@ class LBRYSessionManager(SessionManager):
if self.env.websocket_host is not None and self.env.websocket_port is not None:
self.websocket = AdminWebSocket(self)
self.search_cache = self.bp.search_cache
self.search_cache['search'] = LRUCache(2**14, metric_name='search', namespace=NAMESPACE)
self.search_cache['resolve'] = LRUCache(2**16, metric_name='resolve', namespace=NAMESPACE)
self.search_cache['search'] = LRUCacheWithMetrics(2 ** 14, metric_name='search', namespace=NAMESPACE)
self.search_cache['resolve'] = LRUCacheWithMetrics(2 ** 16, metric_name='resolve', namespace=NAMESPACE)
async def process_metrics(self):
while self.running:
@ -864,6 +868,7 @@ class LBRYElectrumX(SessionBase):
max_errors = math.inf # don't disconnect people for errors! let them happen...
session_mgr: LBRYSessionManager
version = lbry.__version__
cached_server_features = {}
@classmethod
def initialize_request_handlers(cls):
@ -910,6 +915,8 @@ class LBRYElectrumX(SessionBase):
super().__init__(*args, **kwargs)
if not LBRYElectrumX.request_handlers:
LBRYElectrumX.initialize_request_handlers()
if not LBRYElectrumX.cached_server_features:
LBRYElectrumX.set_server_features(self.env)
self.subscribe_headers = False
self.subscribe_headers_raw = False
self.connection.max_response_size = self.env.max_send
@ -927,10 +934,10 @@ class LBRYElectrumX(SessionBase):
for ver in (cls.PROTOCOL_MIN, cls.PROTOCOL_MAX)]
@classmethod
def server_features(cls, env):
def set_server_features(cls, env):
"""Return the server features dictionary."""
min_str, max_str = cls.protocol_min_max_strings()
return {
cls.cached_server_features.update({
'hosts': env.hosts_dict(),
'pruning': None,
'server_version': cls.version,
@ -943,10 +950,10 @@ class LBRYElectrumX(SessionBase):
'daily_fee': env.daily_fee,
'hash_function': 'sha256',
'trending_algorithm': env.trending_algorithms[0]
}
})
async def server_features_async(self):
return self.server_features(self.env)
return self.cached_server_features
@classmethod
def server_version_args(cls):
@ -1271,7 +1278,7 @@ class LBRYElectrumX(SessionBase):
hashXes = [
(self.address_to_hashX(address), address) for address in addresses
]
return await asyncio.gather(*(self.hashX_subscribe(*args) for args in hashXes))
return [await self.hashX_subscribe(*args) for args in hashXes]
async def address_unsubscribe(self, address):
"""Unsubscribe an address.

192
lbry/wallet/server/udp.py Normal file
View file

@ -0,0 +1,192 @@
import asyncio
import struct
from time import perf_counter
import logging
from typing import Optional, Tuple, NamedTuple
from lbry.utils import LRUCache
# from prometheus_client import Counter
log = logging.getLogger(__name__)
_MAGIC = 1446058291 # genesis blocktime (which is actually wrong)
# ping_count_metric = Counter("ping_count", "Number of pings received", namespace='wallet_server_status')
_PAD_BYTES = b'\x00' * 64
class SPVPing(NamedTuple):
magic: int
protocol_version: int
pad_bytes: bytes
def encode(self):
return struct.pack(b'!lB64s', *self)
@staticmethod
def make(protocol_version=1) -> bytes:
return SPVPing(_MAGIC, protocol_version, _PAD_BYTES).encode()
@classmethod
def decode(cls, packet: bytes):
decoded = cls(*struct.unpack(b'!lB64s', packet[:69]))
if decoded.magic != _MAGIC:
raise ValueError("invalid magic bytes")
return decoded
class SPVPong(NamedTuple):
protocol_version: int
flags: int
height: int
tip: bytes
source_address_raw: bytes
def encode(self):
return struct.pack(b'!BBl32s4s', *self)
@staticmethod
def make(height: int, tip: bytes, flags: int, protocol_version: int = 1) -> bytes:
# note: drops the last 4 bytes so the result can be cached and have addresses added to it as needed
return SPVPong(protocol_version, flags, height, tip, b'\x00\x00\x00\x00').encode()[:38]
@classmethod
def decode(cls, packet: bytes):
return cls(*struct.unpack(b'!BBl32s4s', packet[:42]))
@property
def available(self) -> bool:
return (self.flags & 0b00000001) > 0
@property
def ip_address(self) -> str:
return ".".join(map(str, self.source_address_raw))
def __repr__(self) -> str:
return f"SPVPong(external_ip={self.ip_address}, version={self.protocol_version}, " \
f"available={'True' if self.flags & 1 > 0 else 'False'}," \
f" height={self.height}, tip={self.tip[::-1].hex()})"
class SPVServerStatusProtocol(asyncio.DatagramProtocol):
PROTOCOL_VERSION = 1
def __init__(self, height: int, tip: bytes, throttle_cache_size: int = 1024, throttle_reqs_per_sec: int = 10):
super().__init__()
self.transport: Optional[asyncio.transports.DatagramTransport] = None
self._height = height
self._tip = tip
self._flags = 0
self._cached_response = None
self.update_cached_response()
self._throttle = LRUCache(throttle_cache_size)
self._should_log = LRUCache(throttle_cache_size)
self._min_delay = 1 / throttle_reqs_per_sec
def update_cached_response(self):
self._cached_response = SPVPong.make(self._height, self._tip, self._flags, self.PROTOCOL_VERSION)
def set_unavailable(self):
self._flags &= 0b11111110
self.update_cached_response()
def set_available(self):
self._flags |= 0b00000001
self.update_cached_response()
def set_height(self, height: int, tip: bytes):
self._height, self._tip = height, tip
self.update_cached_response()
def should_throttle(self, host: str):
now = perf_counter()
last_requested = self._throttle.get(host, default=0)
self._throttle[host] = now
if now - last_requested < self._min_delay:
log_cnt = self._should_log.get(host, default=0) + 1
if log_cnt % 100 == 0:
log.warning("throttle spv status to %s", host)
self._should_log[host] = log_cnt
return True
return False
def make_pong(self, host):
return self._cached_response + bytes(int(b) for b in host.split("."))
def datagram_received(self, data: bytes, addr: Tuple[str, int]):
if self.should_throttle(addr[0]):
return
try:
SPVPing.decode(data)
except (ValueError, struct.error, AttributeError, TypeError):
# log.exception("derp")
return
self.transport.sendto(self.make_pong(addr[0]), addr)
# ping_count_metric.inc()
def connection_made(self, transport) -> None:
self.transport = transport
def connection_lost(self, exc: Optional[Exception]) -> None:
self.transport = None
def close(self):
if self.transport:
self.transport.close()
class StatusServer:
def __init__(self):
self._protocol: Optional[SPVServerStatusProtocol] = None
async def start(self, height: int, tip: bytes, interface: str, port: int):
if self._protocol:
return
loop = asyncio.get_event_loop()
self._protocol = SPVServerStatusProtocol(height, tip)
interface = interface if interface.lower() != 'localhost' else '127.0.0.1'
await loop.create_datagram_endpoint(lambda: self._protocol, (interface, port))
log.info("started udp status server on %s:%i", interface, port)
def stop(self):
if self._protocol:
self._protocol.close()
self._protocol = None
def set_unavailable(self):
self._protocol.set_unavailable()
def set_available(self):
self._protocol.set_available()
def set_height(self, height: int, tip: bytes):
self._protocol.set_height(height, tip)
class SPVStatusClientProtocol(asyncio.DatagramProtocol):
PROTOCOL_VERSION = 1
def __init__(self, responses: asyncio.Queue):
super().__init__()
self.transport: Optional[asyncio.transports.DatagramTransport] = None
self.responses = responses
self._ping_packet = SPVPing.make(self.PROTOCOL_VERSION)
def datagram_received(self, data: bytes, addr: Tuple[str, int]):
try:
self.responses.put_nowait(((addr, perf_counter()), SPVPong.decode(data)))
except (ValueError, struct.error, AttributeError, TypeError, RuntimeError):
return
def connection_made(self, transport) -> None:
self.transport = transport
def connection_lost(self, exc: Optional[Exception]) -> None:
self.transport = None
log.info("closed udp spv server selection client")
def ping(self, server: Tuple[str, int]):
self.transport.sendto(self._ping_packet, server)
def close(self):
# log.info("close udp client")
if self.transport:
self.transport.close()

80
tests/integration/blockchain/test_network.py
View file

@ -6,6 +6,7 @@ from unittest.mock import Mock
from lbry.wallet.network import Network
from lbry.wallet.orchstr8.node import SPVNode
from lbry.wallet.rpc import RPCSession
from lbry.wallet.server.udp import StatusServer
from lbry.testcase import IntegrationTestCase, AsyncioTestCase
@ -32,18 +33,17 @@ class NetworkTests(IntegrationTestCase):
'server_version': lbry.__version__,
'trending_algorithm': 'zscore',
}, await self.ledger.network.get_server_features())
await self.conductor.spv_node.stop()
# await self.conductor.spv_node.stop()
payment_address, donation_address = await self.account.get_addresses(limit=2)
await self.conductor.spv_node.start(
self.conductor.blockchain_node,
extraconf={
'DESCRIPTION': 'Fastest server in the west.',
'PAYMENT_ADDRESS': payment_address,
'DONATION_ADDRESS': donation_address,
'DAILY_FEE': '42'
}
)
await self.ledger.network.on_connected.first
self.conductor.spv_node.server.env.payment_address = payment_address
self.conductor.spv_node.server.env.donation_address = donation_address
self.conductor.spv_node.server.env.description = 'Fastest server in the west.'
self.conductor.spv_node.server.env.daily_fee = '42'
from lbry.wallet.server.session import LBRYElectrumX
LBRYElectrumX.set_server_features(self.conductor.spv_node.server.env)
# await self.ledger.network.on_connected.first
self.assertDictEqual({
'genesis_hash': self.conductor.spv_node.coin_class.GENESIS_HASH,
'hash_function': 'sha256',
@ -65,22 +65,21 @@ class ReconnectTests(IntegrationTestCase):
async def test_multiple_servers(self):
# we have a secondary node that connects later, so
node2 = SPVNode(self.conductor.spv_module, node_number=2)
self.ledger.network.config['default_servers'].append((node2.hostname, node2.port))
await asyncio.wait_for(self.ledger.stop(), timeout=1)
await asyncio.wait_for(self.ledger.start(), timeout=1)
self.ledger.network.session_pool.new_connection_event.clear()
await node2.start(self.blockchain)
# this is only to speed up the test as retrying would take 4+ seconds
for session in self.ledger.network.session_pool.sessions:
session.trigger_urgent_reconnect.set()
await asyncio.wait_for(self.ledger.network.session_pool.new_connection_event.wait(), timeout=1)
self.assertEqual(2, len(list(self.ledger.network.session_pool.available_sessions)))
self.ledger.network.config['default_servers'].append((node2.hostname, node2.port))
self.ledger.network.config['default_servers'].reverse()
self.assertEqual(50002, self.ledger.network.client.server[1])
await self.ledger.stop()
await self.ledger.start()
self.assertTrue(self.ledger.network.is_connected)
switch_event = self.ledger.network.on_connected.first
self.assertEqual(50003, self.ledger.network.client.server[1])
await node2.stop(True)
self.assertFalse(self.ledger.network.is_connected)
await self.ledger.resolve([], ['derp'])
self.assertEqual(50002, self.ledger.network.client.server[1])
await node2.stop(True)
# secondary down, but primary is ok, do not switch! (switches trigger new on_connected events)
with self.assertRaises(asyncio.TimeoutError):
await asyncio.wait_for(switch_event, timeout=1)
async def test_direct_sync(self):
await self.ledger.stop()
@ -98,10 +97,13 @@ class ReconnectTests(IntegrationTestCase):
async def test_connection_drop_still_receives_events_after_reconnected(self):
address1 = await self.account.receiving.get_or_create_usable_address()
# disconnect and send a new tx, should reconnect and get it
self.ledger.network.client.connection_lost(Exception())
self.ledger.network.client.transport.close()
self.assertFalse(self.ledger.network.is_connected)
await self.ledger.resolve([], 'derp')
sendtxid = await self.blockchain.send_to_address(address1, 1.1337)
await asyncio.wait_for(self.on_transaction_id(sendtxid), 1.0) # mempool
# await self.ledger.resolve([], 'derp')
# self.assertTrue(self.ledger.network.is_connected)
await asyncio.wait_for(self.on_transaction_id(sendtxid), 10.0) # mempool
await self.blockchain.generate(1)
await self.on_transaction_id(sendtxid) # confirmed
self.assertLess(self.ledger.network.client.response_time, 1) # response time properly set lower, we are fine
@ -123,7 +125,7 @@ class ReconnectTests(IntegrationTestCase):
await self.blockchain.generate(1)
# (this is just so the test doesn't hang forever if it doesn't reconnect)
if not self.ledger.network.is_connected:
await asyncio.wait_for(self.ledger.network.on_connected.first, timeout=1.0)
await asyncio.wait_for(self.ledger.network.on_connected.first, timeout=10.0)
# omg, the burned cable still works! torba is fire proof!
await self.ledger.network.get_transaction(sendtxid)
@ -136,15 +138,19 @@ class ReconnectTests(IntegrationTestCase):
await self.ledger.network.on_connected.first
self.assertTrue(self.ledger.network.is_connected)
async def test_online_but_still_unavailable(self):
# Edge case. See issue #2445 for context
self.assertIsNotNone(self.ledger.network.session_pool.fastest_session)
for session in self.ledger.network.session_pool.sessions:
session.response_time = None
self.assertIsNone(self.ledger.network.session_pool.fastest_session)
# async def test_online_but_still_unavailable(self):
# # Edge case. See issue #2445 for context
# self.assertIsNotNone(self.ledger.network.session_pool.fastest_session)
# for session in self.ledger.network.session_pool.sessions:
# session.response_time = None
# self.assertIsNone(self.ledger.network.session_pool.fastest_session)
class ServerPickingTestCase(AsyncioTestCase):
async def _make_udp_server(self, port):
s = StatusServer()
await s.start(0, b'\x00' * 32, '127.0.0.1', port)
self.addCleanup(s.stop)
async def _make_fake_server(self, latency=1.0, port=1):
# local fake server with artificial latency
@ -156,6 +162,7 @@ class ServerPickingTestCase(AsyncioTestCase):
return {'height': 1}
server = await self.loop.create_server(lambda: FakeSession(), host='127.0.0.1', port=port)
self.addCleanup(server.close)
await self._make_udp_server(port)
return '127.0.0.1', port
async def _make_bad_server(self, port=42420):
@ -164,9 +171,10 @@ class ServerPickingTestCase(AsyncioTestCase):
writer.write(await reader.read())
server = await asyncio.start_server(echo, host='127.0.0.1', port=port)
self.addCleanup(server.close)
await self._make_udp_server(port)
return '127.0.0.1', port
async def test_pick_fastest(self):
async def _test_pick_fastest(self):
ledger = Mock(config={
'default_servers': [
# fast but unhealthy, should be discarded
@ -182,8 +190,8 @@ class ServerPickingTestCase(AsyncioTestCase):
network = Network(ledger)
self.addCleanup(network.stop)
asyncio.ensure_future(network.start())
await asyncio.wait_for(network.on_connected.first, timeout=1)
await network.start()
await asyncio.wait_for(network.on_connected.first, timeout=10)
self.assertTrue(network.is_connected)
self.assertTupleEqual(network.client.server, ('127.0.0.1', 1337))
self.assertTrue(all([not session.is_closing() for session in network.session_pool.available_sessions]))

6
tests/integration/blockchain/test_wallet_commands.py
View file

@ -43,12 +43,14 @@ class WalletCommands(CommandTestCase):
)
async def test_wallet_reconnect(self):
status = await self.daemon.jsonrpc_status()
self.assertEqual(len(status['wallet']['servers']), 1)
self.assertEqual(status['wallet']['servers'][0]['port'], 50002)
await self.conductor.spv_node.stop(True)
self.conductor.spv_node.port = 54320
await self.conductor.spv_node.start(self.conductor.blockchain_node)
status = await self.daemon.jsonrpc_status()
self.assertEqual(len(status['wallet']['servers']), 1)
self.assertEqual(status['wallet']['servers'][0]['port'], 50002)
self.assertEqual(len(status['wallet']['servers']), 0)
self.daemon.jsonrpc_settings_set('lbryum_servers', ['localhost:54320'])
await self.daemon.jsonrpc_wallet_reconnect()
status = await self.daemon.jsonrpc_status()

17
tests/integration/blockchain/test_wallet_server_sessions.py
View file

@ -4,6 +4,7 @@ import lbry
import lbry.wallet
from lbry.error import ServerPaymentFeeAboveMaxAllowedError
from lbry.wallet.network import ClientSession
from lbry.wallet.server.session import LBRYElectrumX
from lbry.testcase import IntegrationTestCase, CommandTestCase
from lbry.wallet.orchstr8.node import SPVNode
@ -46,7 +47,7 @@ class TestSessions(IntegrationTestCase):
class TestUsagePayment(CommandTestCase):
async def test_single_server_payment(self):
async def _test_single_server_payment(self):
wallet_pay_service = self.daemon.component_manager.get_component('wallet_server_payments')
wallet_pay_service.payment_period = 1
# only starts with a positive max key fee
@ -68,21 +69,21 @@ class TestUsagePayment(CommandTestCase):
self.addCleanup(node.stop)
self.daemon.jsonrpc_settings_set('lbryum_servers', [f"{node.hostname}:{node.port}"])
await self.daemon.jsonrpc_wallet_reconnect()
LBRYElectrumX.set_server_features(node.server.env)
features = await self.ledger.network.get_server_features()
self.assertEqual(features["payment_address"], address)
self.assertEqual(features["daily_fee"], "1.1")
with self.assertRaises(ServerPaymentFeeAboveMaxAllowedError):
await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=8)
await node.stop(False)
await node.start(self.blockchain, extraconf={"PAYMENT_ADDRESS": address, "DAILY_FEE": "1.0"})
self.daemon.jsonrpc_settings_set('lbryum_servers', [f"{node.hostname}:{node.port}"])
await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=30)
node.server.env.daily_fee = "1.0"
node.server.env.payment_address = address
LBRYElectrumX.set_server_features(node.server.env)
# self.daemon.jsonrpc_settings_set('lbryum_servers', [f"{node.hostname}:{node.port}"])
await self.daemon.jsonrpc_wallet_reconnect()
features = await self.ledger.network.get_server_features()
self.assertEqual(features["payment_address"], address)
self.assertEqual(features["daily_fee"], "1.0")
tx = await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=8)
tx = await asyncio.wait_for(wallet_pay_service.on_payment.first, timeout=30)
self.assertIsNotNone(await self.blockchain.get_raw_transaction(tx.id)) # verify its broadcasted
self.assertEqual(tx.outputs[0].amount, 100000000)
self.assertEqual(tx.outputs[0].get_address(self.ledger), address)

4
tests/integration/datanetwork/test_file_commands.py
View file

@ -9,6 +9,10 @@ from lbry.wallet import Transaction
class FileCommands(CommandTestCase):
def __init__(self, *a, **kw):
super().__init__(*a, **kw)
self.skip_libtorrent = False
async def initialize_torrent(self, tx_to_update=None):
if not hasattr(self, 'seeder_session'):
self.seeder_session = TorrentSession(self.loop, None)

6
tests/integration/other/test_cli.py
View file

@ -7,7 +7,8 @@ from lbry.extras import cli
from lbry.extras.daemon.components import (
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT,
LIBTORRENT_COMPONENT
)
from lbry.extras.daemon.daemon import Daemon
@ -22,7 +23,8 @@ class CLIIntegrationTest(AsyncioTestCase):
conf.components_to_skip = (
DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT,
HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT,
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT
UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT,
LIBTORRENT_COMPONENT
)
Daemon.component_attributes = {}
self.daemon = Daemon(conf)