From 0961cad716f75614766158a4281771ece7a58640 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 29 Jul 2022 21:55:35 -0300 Subject: [PATCH 1/7] remove dead code --- lbry/torrent/session.py | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/lbry/torrent/session.py b/lbry/torrent/session.py index feff53f75..857540d4c 100644 --- a/lbry/torrent/session.py +++ b/lbry/torrent/session.py @@ -10,47 +10,13 @@ from typing import Optional import libtorrent -NOTIFICATION_MASKS = [ - "error", - "peer", - "port_mapping", - "storage", - "tracker", - "debug", - "status", - "progress", - "ip_block", - "dht", - "stats", - "session_log", - "torrent_log", - "peer_log", - "incoming_request", - "dht_log", - "dht_operation", - "port_mapping_log", - "picker_log", - "file_progress", - "piece_progress", - "upload", - "block_progress" -] log = logging.getLogger(__name__) - - DEFAULT_FLAGS = ( # fixme: somehow the logic here is inverted? libtorrent.add_torrent_params_flags_t.flag_auto_managed | libtorrent.add_torrent_params_flags_t.flag_update_subscribe ) -def get_notification_type(notification) -> str: - for i, notification_type in enumerate(NOTIFICATION_MASKS): - if (1 << i) & notification: - return notification_type - raise ValueError("unrecognized notification type") - - class TorrentHandle: def __init__(self, loop, executor, handle): self._loop = loop From 84d89ce5afc68dbb65acea0703f397528b2d3ee2 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 29 Jul 2022 21:56:09 -0300 Subject: [PATCH 2/7] torrent: disable upnp, natpmp. --- lbry/torrent/session.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lbry/torrent/session.py b/lbry/torrent/session.py index 857540d4c..887dc70da 100644 --- a/lbry/torrent/session.py +++ b/lbry/torrent/session.py @@ -122,10 +122,8 @@ class TorrentSession: async def bind(self, interface: str = '0.0.0.0', port: int = 10889): settings = { 'listen_interfaces': f"{interface}:{port}", - 'enable_outgoing_utp': True, - 'enable_incoming_utp': True, - 'enable_outgoing_tcp': False, - 'enable_incoming_tcp': False + 'enable_natpmp': False, + 'enable_upnp': False } self._session = await self._loop.run_in_executor( self._executor, libtorrent.session, settings # pylint: disable=c-extension-no-member From 3021962e3d4eb185fd023465b74f3e154c1b17b4 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 29 Jul 2022 21:56:45 -0300 Subject: [PATCH 3/7] tests: add peer directly instead of relying on torrent dht --- tests/integration/datanetwork/test_file_commands.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/integration/datanetwork/test_file_commands.py b/tests/integration/datanetwork/test_file_commands.py index 0e1ac141e..f0185d2b8 100644 --- a/tests/integration/datanetwork/test_file_commands.py +++ b/tests/integration/datanetwork/test_file_commands.py @@ -18,11 +18,17 @@ class FileCommands(CommandTestCase): super().__init__(*a, **kw) self.skip_libtorrent = False + async def add_forever(self): + while True: + for handle in self.client_session._handles.values(): + handle._handle.connect_peer(('127.0.0.1', 4040)) + await asyncio.sleep(.1) + async def initialize_torrent(self, tx_to_update=None): if not hasattr(self, 'seeder_session'): self.seeder_session = TorrentSession(self.loop, None) self.addCleanup(self.seeder_session.stop) - await self.seeder_session.bind(port=4040) + await self.seeder_session.bind('127.0.0.1', port=4040) btih = await self.seeder_session.add_fake_torrent() address = await self.account.receiving.get_or_create_usable_address() if not tx_to_update: @@ -40,8 +46,9 @@ class FileCommands(CommandTestCase): await tx.sign([self.account]) await self.broadcast_and_confirm(tx) self.client_session = self.daemon.file_manager.source_managers['torrent'].torrent_session - self.client_session._session.add_dht_node(('localhost', 4040)) self.client_session.wait_start = False # fixme: this is super slow on tests + task = asyncio.create_task(self.add_forever()) + self.addCleanup(task.cancel) return tx, btih @skipIf(TorrentSession is None, "libtorrent not installed") From 8f26010c047192beb9d8c3da60932dfef9bcb497 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 29 Jul 2022 22:06:31 -0300 Subject: [PATCH 4/7] make libtorrent a normal dependency --- .github/workflows/main.yml | 4 ++-- setup.py | 2 +- tox.ini | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index df2988216..39991c1c7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,7 +18,7 @@ jobs: key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} restore-keys: ${{ runner.os }}-pip- - run: pip install --user --upgrade pip wheel - - run: pip install -e .[torrent,lint] + - run: pip install -e .[lint] - run: make lint tests-unit: @@ -50,7 +50,7 @@ jobs: string: ${{ runner.os }} - run: python -m pip install --user --upgrade pip wheel - if: startsWith(runner.os, 'linux') - run: pip install -e .[torrent,test] + run: pip install -e .[test] - if: startsWith(runner.os, 'linux') env: HOME: /tmp diff --git a/setup.py b/setup.py index c8d3f6018..bc2ca6013 100644 --- a/setup.py +++ b/setup.py @@ -46,9 +46,9 @@ setup( 'coincurve==15.0.0', 'pbkdf2==1.3', 'filetype==1.0.9', + 'libtorrent==2.0.6', ], extras_require={ - 'torrent': ['lbry-libtorrent'], 'lint': [ 'pylint==2.10.0' ], diff --git a/tox.ini b/tox.ini index 6795a8a1a..2e562f4ea 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,6 @@ deps = extras = test hub - torrent changedir = {toxinidir}/tests setenv = HOME=/tmp From 5be990fc55fd717f1bb29b5ae54dfcb6f0fe48e3 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 15 Aug 2022 23:22:21 -0300 Subject: [PATCH 5/7] do not ignore libtorrent import error --- lbry/extras/daemon/components.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lbry/extras/daemon/components.py b/lbry/extras/daemon/components.py index 637ede42e..116fb3607 100644 --- a/lbry/extras/daemon/components.py +++ b/lbry/extras/daemon/components.py @@ -28,11 +28,7 @@ from lbry.torrent.torrent_manager import TorrentManager from lbry.wallet import WalletManager from lbry.wallet.usage_payment import WalletServerPayer from lbry.torrent.tracker import TrackerClient - -try: - from lbry.torrent.session import TorrentSession -except ImportError: - TorrentSession = None +from lbry.torrent.session import TorrentSession log = logging.getLogger(__name__) From c07c369a28c463796ef5e4e95c6a948980f5fe38 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 16 Aug 2022 17:26:57 -0300 Subject: [PATCH 6/7] add libtorrent pyinstaller hook --- scripts/hook-libtorrent.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 scripts/hook-libtorrent.py diff --git a/scripts/hook-libtorrent.py b/scripts/hook-libtorrent.py new file mode 100644 index 000000000..bd8b7da16 --- /dev/null +++ b/scripts/hook-libtorrent.py @@ -0,0 +1,24 @@ +""" +Hook for libtorrent. +""" + +import os +import glob +import os.path +from PyInstaller.utils.hooks import get_module_file_attribute +from PyInstaller import compat + + +def get_binaries(): + if compat.is_win: + files = ('c:/Windows/System32/libssl-1_1-x64.dll', 'c:/Windows/System32/libcrypto-1_1-x64.dll') + for file in files: + if not os.path.isfile(file): + print(f"MISSING {file}") + return [(file, '.') for file in files] + return [] + + +binaries = get_binaries() +for file in glob.glob(os.path.join(get_module_file_attribute('libtorrent'), 'libtorrent*pyd*')): + binaries.append((file, 'libtorrent')) From 8a033d58df516883f250b81b9a94b867152f7e49 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 18 Aug 2022 19:29:51 -0300 Subject: [PATCH 7/7] fix torrent component --- lbry/extras/daemon/components.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/lbry/extras/daemon/components.py b/lbry/extras/daemon/components.py index 116fb3607..33deccee9 100644 --- a/lbry/extras/daemon/components.py +++ b/lbry/extras/daemon/components.py @@ -357,10 +357,6 @@ class FileManagerComponent(Component): wallet = self.component_manager.get_component(WALLET_COMPONENT) node = self.component_manager.get_component(DHT_COMPONENT) \ if self.component_manager.has_component(DHT_COMPONENT) else None - try: - torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT) if TorrentSession else None - except NameError: - torrent = None log.info('Starting the file manager') loop = asyncio.get_event_loop() self.file_manager = FileManager( @@ -369,7 +365,8 @@ class FileManagerComponent(Component): self.file_manager.source_managers['stream'] = StreamManager( loop, self.conf, blob_manager, wallet, storage, node, ) - if TorrentSession and LIBTORRENT_COMPONENT not in self.conf.components_to_skip: + if self.component_manager.has_component(LIBTORRENT_COMPONENT): + torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT) self.file_manager.source_managers['torrent'] = TorrentManager( loop, self.conf, torrent, storage, self.component_manager.analytics_manager ) @@ -498,9 +495,8 @@ class TorrentComponent(Component): } async def start(self): - if TorrentSession: - self.torrent_session = TorrentSession(asyncio.get_event_loop(), None) - await self.torrent_session.bind() # TODO: specify host/port + self.torrent_session = TorrentSession(asyncio.get_event_loop(), None) + await self.torrent_session.bind() # TODO: specify host/port async def stop(self): if self.torrent_session: