From 6254f53716e29a27536ae157ab661d7fca731d16 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 8 Jul 2021 03:41:51 -0300 Subject: [PATCH 1/8] propagate external ip changes from upnp component to dht node protocol --- lbry/extras/daemon/components.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lbry/extras/daemon/components.py b/lbry/extras/daemon/components.py index e55ec342d..822022f32 100644 --- a/lbry/extras/daemon/components.py +++ b/lbry/extras/daemon/components.py @@ -481,6 +481,10 @@ class UPnPComponent(Component): log.info("external ip changed from %s to %s", self.external_ip, external_ip) if external_ip: self.external_ip = external_ip + dht_component = self.component_manager.get_component(DHT_COMPONENT) + if dht_component: + dht_node = dht_component.component + dht_node.protocol.external_ip = external_ip # assert self.external_ip is not None # TODO: handle going/starting offline if not self.upnp_redirects and self.upnp: # setup missing redirects From 9b3b609e40fa39b9fb5a9e1403e3adafea9607ff Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 8 Jul 2021 03:46:48 -0300 Subject: [PATCH 2/8] re-enable test_losing_connection --- tests/unit/dht/test_node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/dht/test_node.py b/tests/unit/dht/test_node.py index a5e599f9b..5c502f9a7 100644 --- a/tests/unit/dht/test_node.py +++ b/tests/unit/dht/test_node.py @@ -92,7 +92,7 @@ class TestNodePingQueueDiscover(AsyncioTestCase): class TestTemporarilyLosingConnection(AsyncioTestCase): - @unittest.SkipTest + TIMEOUT = None # not supported as it advances time async def test_losing_connection(self): async def wait_for(check_ok, insist, timeout=20): start = loop.time() From c519d4651bf44df18561345615446455e9b2d3fd Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 8 Jul 2021 03:55:21 -0300 Subject: [PATCH 3/8] loop.time is not usable on advance time, use wall time --- tests/unit/dht/test_node.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/unit/dht/test_node.py b/tests/unit/dht/test_node.py index 5c502f9a7..75f999a75 100644 --- a/tests/unit/dht/test_node.py +++ b/tests/unit/dht/test_node.py @@ -1,4 +1,5 @@ import asyncio +import time import unittest import typing from lbry.testcase import AsyncioTestCase @@ -95,8 +96,8 @@ class TestTemporarilyLosingConnection(AsyncioTestCase): TIMEOUT = None # not supported as it advances time async def test_losing_connection(self): async def wait_for(check_ok, insist, timeout=20): - start = loop.time() - while loop.time() - start < timeout: + start = time.time() + while time.time() - start < timeout: if check_ok(): break await asyncio.sleep(0) From 4ebe4ce1b79d9fb844c3bedf40b07676bd55b4d3 Mon Sep 17 00:00:00 2001 From: belikor Date: Tue, 6 Jul 2021 00:54:42 -0500 Subject: [PATCH 4/8] scripts: note to further investigate in `download_blob_from_peer` Currently `lbrynet blob get ` does not work to download single blobs which are not already present in the system. The function locks up and never returns. It only works for blobs that are in the `blobfiles` directory already. This bug is reported in lbryio/lbry-sdk, issue #2070. Maybe this script can be investigated, and certain parts can be added to `lbry.extras.daemon.daemon.jsonrpc_blob_get` in order to solve the previous issue, and finally download single blobs from the network (peers or reflector servers). --- scripts/download_blob_from_peer.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/scripts/download_blob_from_peer.py b/scripts/download_blob_from_peer.py index 3418ab1f8..ae6721dbf 100644 --- a/scripts/download_blob_from_peer.py +++ b/scripts/download_blob_from_peer.py @@ -1,3 +1,19 @@ +"""A simple script that attempts to directly download a single blob. + +To Do: +------ +Currently `lbrynet blob get ` does not work to download single blobs +which are not already present in the system. The function locks up and +never returns. +It only works for blobs that are in the `blobfiles` directory already. + +This bug is reported in lbryio/lbry-sdk, issue #2070. + +Maybe this script can be investigated, and certain parts can be added to +`lbry.extras.daemon.daemon.jsonrpc_blob_get` +in order to solve the previous issue, and finally download single blobs +from the network (peers or reflector servers). +""" import sys import os import asyncio @@ -47,7 +63,11 @@ async def main(blob_hash: str, url: str): print(f"deleted {blob_hash}") -if __name__ == "__main__": # usage: python download_blob_from_peer.py [host url:port] +if __name__ == "__main__": + if len(sys.argv) < 2: + print("usage: download_blob_from_peer.py [host_url:port]") + sys.exit(1) + url = 'reflector.lbry.com:5567' if len(sys.argv) > 2: url = sys.argv[2] From d151a82d78ee01b5efd271bfb7af61eea2dc91ee Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 15 Jul 2021 17:07:47 -0300 Subject: [PATCH 5/8] add libtool and automake to the dockerfiles so they can build coincurve --- docker/Dockerfile.wallet_server | 1 + docker/Dockerfile.web | 1 + 2 files changed, 2 insertions(+) diff --git a/docker/Dockerfile.wallet_server b/docker/Dockerfile.wallet_server index 1d913793b..a3ed8b60e 100644 --- a/docker/Dockerfile.wallet_server +++ b/docker/Dockerfile.wallet_server @@ -13,6 +13,7 @@ RUN apt-get update && \ wget \ tar unzip \ build-essential \ + automake libtool \ pkg-config \ libleveldb-dev \ python3.7 \ diff --git a/docker/Dockerfile.web b/docker/Dockerfile.web index 3e81e235f..683a7ac82 100644 --- a/docker/Dockerfile.web +++ b/docker/Dockerfile.web @@ -11,6 +11,7 @@ ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT RUN apt-get update && \ apt-get -y --no-install-recommends install \ wget \ + automake libtool \ tar unzip \ build-essential \ pkg-config \ From b7791d284569daf8cdf1be8e4aa6632720758d08 Mon Sep 17 00:00:00 2001 From: belikor Date: Tue, 13 Jul 2021 11:54:14 -0500 Subject: [PATCH 6/8] exchange_rate_manager: raise exception if `'error'` is in `json_response` If the error is not handled, the running daemon will continuously print the following error message: ``` Traceback (most recent call last): File "lbry/extras/daemon/exchange_rate_manager.py", line 77, in get_rate File "lbry/extras/daemon/exchange_rate_manager.py", line 189, in get_rate_from_response KeyError: 0 ``` This started happening when the UPBit exchange decided to delist the LBC coin. Normally `json_response` should be a dictionary, not a list, so `json_response[0]` causes an error. By checking for the `'error'` key, we can raise the proper exception. Once this is done, the message will be a warning, not a traceback. ``` WARNING lbry.extras.daemon.exchange_rate_manager:92: Failed to get exchange rate from UPbit: result not found ``` --- lbry/extras/daemon/exchange_rate_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbry/extras/daemon/exchange_rate_manager.py b/lbry/extras/daemon/exchange_rate_manager.py index a80f50575..500c6f24e 100644 --- a/lbry/extras/daemon/exchange_rate_manager.py +++ b/lbry/extras/daemon/exchange_rate_manager.py @@ -186,7 +186,7 @@ class UPbitBTCFeed(MarketFeed): params = {"markets": "BTC-LBC"} def get_rate_from_response(self, json_response): - if len(json_response) != 1 or 'trade_price' not in json_response[0]: + if "error" in json_response or len(json_response) != 1 or 'trade_price' not in json_response[0]: raise InvalidExchangeRateResponseError(self.name, 'result not found') return 1.0 / float(json_response[0]['trade_price']) From a74685d66d42f20827f62d788e7aa741336065f2 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 14 Jul 2021 16:40:26 -0300 Subject: [PATCH 7/8] add script to troubleshoot p2p/dht --- .../troubleshoot_p2p_and_dht_webservice.py | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 scripts/troubleshoot_p2p_and_dht_webservice.py diff --git a/scripts/troubleshoot_p2p_and_dht_webservice.py b/scripts/troubleshoot_p2p_and_dht_webservice.py new file mode 100644 index 000000000..dc004ff7f --- /dev/null +++ b/scripts/troubleshoot_p2p_and_dht_webservice.py @@ -0,0 +1,71 @@ +import asyncio +from aiohttp import web + +from lbry.blob_exchange.serialization import BlobRequest, BlobResponse +from lbry.dht.constants import generate_id +from lbry.dht.node import Node +from lbry.dht.peer import make_kademlia_peer, PeerManager +from lbry.extras.daemon.storage import SQLiteStorage + +loop = asyncio.get_event_loop() +NODE = Node( + loop, PeerManager(loop), generate_id(), 60600, 60600, 3333, None, + storage=SQLiteStorage(None, ":memory:", loop, loop.time) +) + + +async def check_p2p(ip, port): + writer = None + try: + reader, writer = await asyncio.open_connection(ip, port) + writer.write(BlobRequest.make_request_for_blob_hash('0'*96).serialize()) + return BlobResponse.deserialize(await reader.readuntil(b'}')).get_address_response().lbrycrd_address + except OSError: + return None + finally: + if writer: + writer.close() + await writer.wait_closed() + + +async def check_dht(ip, port): + peer = make_kademlia_peer(None, ip, udp_port=int(port)) + return await NODE.protocol.get_rpc_peer(peer).ping() + + +async def endpoint_p2p(request): + p2p_port = request.match_info.get('p2p_port', "3333") + try: + address = await asyncio.wait_for(check_p2p(request.remote, p2p_port), 3) + except asyncio.TimeoutError: + address = None + return {"status": address is not None, "port": p2p_port, "payment_address": address} + + +async def endpoint_dht(request): + dht_port = request.match_info.get('dht_port', "3333") + try: + response = await check_dht(request.remote, dht_port) + except asyncio.TimeoutError: + response = None + return {"status": response == b'pong', "port": dht_port} + + +async def endpoint_default(request): + return {"dht_status": await endpoint_dht(request), "p2p_status": await endpoint_p2p(request)} + + +def as_json_response_wrapper(endpoint): + async def json_endpoint(*args, **kwargs): + return web.json_response(await endpoint(*args, **kwargs)) + return json_endpoint + + +app = web.Application() +app.add_routes([web.get('/', as_json_response_wrapper(endpoint_default)), + web.get('/dht/{dht_port}', as_json_response_wrapper(endpoint_dht)), + web.get('/p2p/{p2p_port}', as_json_response_wrapper(endpoint_p2p))]) + +if __name__ == '__main__': + loop.create_task(NODE.start_listening("0.0.0.0")) + web.run_app(app, port=60666) \ No newline at end of file From 0ccafd5b534c234596d2e18183cb52246070f3ce Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 8 Jul 2021 03:31:20 -0300 Subject: [PATCH 8/8] make get_or_create_usable_address respect the generator lock --- lbry/wallet/account.py | 3 ++- tests/unit/wallet/test_account.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lbry/wallet/account.py b/lbry/wallet/account.py index b673a5cb5..a61878403 100644 --- a/lbry/wallet/account.py +++ b/lbry/wallet/account.py @@ -96,7 +96,8 @@ class AddressManager: return [r['address'] for r in records] async def get_or_create_usable_address(self) -> str: - addresses = await self.get_addresses(only_usable=True, limit=10) + async with self.address_generator_lock: + addresses = await self.get_addresses(only_usable=True, limit=10) if addresses: return random.choice(addresses) addresses = await self.ensure_address_gap() diff --git a/tests/unit/wallet/test_account.py b/tests/unit/wallet/test_account.py index 5ce68f430..894762c68 100644 --- a/tests/unit/wallet/test_account.py +++ b/tests/unit/wallet/test_account.py @@ -1,3 +1,4 @@ +import asyncio from binascii import hexlify from lbry.testcase import AsyncioTestCase from lbry.wallet import Wallet, Ledger, Database, Headers, Account, SingleKey, HierarchicalDeterministic @@ -37,6 +38,18 @@ class TestAccount(AsyncioTestCase): addresses = await account.change.get_addresses() self.assertEqual(len(addresses), 6) + async def test_unused_address_on_account_creation_does_not_cause_a_race(self): + account = Account.generate(self.ledger, Wallet(), 'lbryum') + await account.ledger.db.db.executescript("update pubkey_address set used_times=10") + await account.receiving.address_generator_lock.acquire() + delayed1 = asyncio.ensure_future(account.receiving.ensure_address_gap()) + delayed = asyncio.ensure_future(account.receiving.get_or_create_usable_address()) + await asyncio.sleep(0) + # wallet being created and queried at the same time + account.receiving.address_generator_lock.release() + await delayed1 + await delayed + async def test_generate_keys_over_batch_threshold_saves_it_properly(self): account = Account.generate(self.ledger, Wallet(), 'lbryum') async with account.receiving.address_generator_lock: