From 9432e1b5b23f723ee3b0cf6eb9576e8d80f56385 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Mon, 20 Apr 2020 11:57:09 -0400 Subject: [PATCH 01/86] add `uploading_to_reflector` to `file_list` results --- lbry/extras/daemon/json_response_encoder.py | 6 ++++-- lbry/stream/managed_stream.py | 6 +++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/lbry/extras/daemon/json_response_encoder.py b/lbry/extras/daemon/json_response_encoder.py index a4246bfab..b7702f541 100644 --- a/lbry/extras/daemon/json_response_encoder.py +++ b/lbry/extras/daemon/json_response_encoder.py @@ -109,7 +109,8 @@ def encode_file_doc(): 'channel_claim_id': '(str) None if claim is not found or not signed', 'channel_name': '(str) None if claim is not found or not signed', 'claim_name': '(str) None if claim is not found else the claim name', - 'reflector_progress': '(int) reflector upload progress, 0 to 100' + 'reflector_progress': '(int) reflector upload progress, 0 to 100', + 'uploading_to_reflector': '(bool) set to True when currently uploading to reflector' } @@ -309,7 +310,8 @@ class JSONResponseEncoder(JSONEncoder): 'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height, 'timestamp': self.ledger.headers.estimated_timestamp(tx_height), 'is_fully_reflected': managed_stream.is_fully_reflected, - 'reflector_progress': managed_stream.reflector_progress + 'reflector_progress': managed_stream.reflector_progress, + 'uploading_to_reflector': managed_stream.uploading_to_reflector } def encode_claim(self, claim): diff --git a/lbry/stream/managed_stream.py b/lbry/stream/managed_stream.py index 5071d879e..8339f6a67 100644 --- a/lbry/stream/managed_stream.py +++ b/lbry/stream/managed_stream.py @@ -74,7 +74,8 @@ class ManagedStream: 'saving', 'finished_writing', 'started_writing', - 'finished_write_attempt' + 'finished_write_attempt', + 'uploading_to_reflector' ] def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager', @@ -103,6 +104,7 @@ class ManagedStream: self.fully_reflected = asyncio.Event(loop=self.loop) self.reflector_progress = 0 + self.uploading_to_reflector = False self.file_output_task: typing.Optional[asyncio.Task] = None self.delayed_stop_task: typing.Optional[asyncio.Task] = None self.streaming_responses: typing.List[typing.Tuple[Request, StreamResponse]] = [] @@ -432,6 +434,7 @@ class ManagedStream: sent = [] protocol = StreamReflectorClient(self.blob_manager, self.descriptor) try: + self.uploading_to_reflector = True await self.loop.create_connection(lambda: protocol, host, port) await protocol.send_handshake() sent_sd, needed = await protocol.send_descriptor() @@ -458,6 +461,7 @@ class ManagedStream: finally: if protocol.transport: protocol.transport.close() + self.uploading_to_reflector = False if not self.fully_reflected.is_set(): self.fully_reflected.set() await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}") From 21c112d0599348375bb0d423759b3a920188f4ac Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Mon, 20 Apr 2020 12:16:31 -0400 Subject: [PATCH 02/86] lbrycrd url --- lbry/wallet/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbry/wallet/__init__.py b/lbry/wallet/__init__.py index 7ed88527b..98cfc8bb6 100644 --- a/lbry/wallet/__init__.py +++ b/lbry/wallet/__init__.py @@ -2,7 +2,7 @@ __node_daemon__ = 'lbrycrdd' __node_cli__ = 'lbrycrd-cli' __node_bin__ = '' __node_url__ = ( - 'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.4/lbrycrd-linux-1744.zip' + 'https://github.com/lbryio/lbrycrd/releases/download/v0.17.4.5/lbrycrd-linux-1745.zip' ) __spvserver__ = 'lbry.wallet.server.coin.LBCRegTest' From 51f573f1eaf8030d9df0cbe9f02c7a3ac2a38cbd Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Mon, 20 Apr 2020 13:37:40 -0400 Subject: [PATCH 03/86] v0.70.0 --- lbry/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbry/__init__.py b/lbry/__init__.py index 1591ec860..e29ac73ee 100644 --- a/lbry/__init__.py +++ b/lbry/__init__.py @@ -1,2 +1,2 @@ -__version__ = "0.69.1" +__version__ = "0.70.0" version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name From 9a6326b027060cae8cfe5b5ab029bb5b5a4ffb40 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Wed, 22 Apr 2020 10:36:09 -0400 Subject: [PATCH 04/86] fix for claim_list incorrectly handling --is_spent flag --- lbry/extras/daemon/daemon.py | 2 +- tests/integration/blockchain/test_claim_commands.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index 9789bfa14..6e2ef7aa9 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -2237,7 +2237,7 @@ class Daemon(metaclass=JSONRPCServerType): Returns: {Paginated[Output]} """ kwargs['type'] = claim_type or CLAIM_TYPE_NAMES - if 'is_spent' not in kwargs: + if not kwargs.get('is_spent', False): kwargs['is_not_spent'] = True return self.jsonrpc_txo_list(**kwargs) diff --git a/tests/integration/blockchain/test_claim_commands.py b/tests/integration/blockchain/test_claim_commands.py index eda369674..4873e0cde 100644 --- a/tests/integration/blockchain/test_claim_commands.py +++ b/tests/integration/blockchain/test_claim_commands.py @@ -661,12 +661,15 @@ class ClaimCommands(ClaimTestCase): channel_id = self.get_claim_id(await self.channel_create()) stream_id = self.get_claim_id(await self.stream_create()) + await self.stream_update(stream_id, title='foo') + # type filtering r = await self.claim_list(claim_type='channel') self.assertEqual(1, len(r)) self.assertEqual('channel', r[0]['value_type']) - r = await self.claim_list(claim_type='stream') + # catch a bug where cli sends is_spent=False by default + r = await self.claim_list(claim_type='stream', is_spent=False) self.assertEqual(1, len(r)) self.assertEqual('stream', r[0]['value_type']) From 49458d1085afbbee2a1ea8fbaf762d9210636aca Mon Sep 17 00:00:00 2001 From: Tom Date: Wed, 22 Apr 2020 23:16:30 -0400 Subject: [PATCH 05/86] fix: reposts being returned for single tags --- lbry/wallet/server/db/reader.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lbry/wallet/server/db/reader.py b/lbry/wallet/server/db/reader.py index a98330efb..ae252355a 100644 --- a/lbry/wallet/server/db/reader.py +++ b/lbry/wallet/server/db/reader.py @@ -547,10 +547,10 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun if for_count or attr == 'tag': if attr == 'tag': any_queries[f'#_any_{attr}'] = f""" - (claim.claim_type != {CLAIM_TYPES['repost']} + ((claim.claim_type != {CLAIM_TYPES['repost']} AND claim.claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) OR (claim.claim_type == {CLAIM_TYPES['repost']} AND - claim.reposted_claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) + claim.reposted_claim_hash IN (SELECT claim_hash FROM tag WHERE tag IN ({values})))) """ else: any_queries[f'#_any_{attr}'] = f""" @@ -606,10 +606,10 @@ def _apply_constraints_for_array_attributes(constraints, attr, cleaner, for_coun if for_count: if attr == 'tag': constraints[f'#_not_{attr}'] = f""" - (claim.claim_type != {CLAIM_TYPES['repost']} - AND claim.claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) AND + ((claim.claim_type != {CLAIM_TYPES['repost']} + AND claim.claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) OR (claim.claim_type == {CLAIM_TYPES['repost']} AND - claim.reposted_claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values}))) + claim.reposted_claim_hash NOT IN (SELECT claim_hash FROM tag WHERE tag IN ({values})))) """ else: constraints[f'#_not_{attr}'] = f""" From fbe0f886b621b6396efd42f99438c6cdb1c2fb67 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 22 Apr 2020 16:10:23 -0400 Subject: [PATCH 06/86] non blocking blob creation --- lbry/blob/blob_file.py | 32 +++++++++++++------ lbry/blob_exchange/client.py | 2 ++ lbry/stream/managed_stream.py | 19 +++++------ tests/unit/blob/test_blob_manager.py | 3 +- .../unit/blob_exchange/test_transfer_blob.py | 9 ++++-- 5 files changed, 44 insertions(+), 21 deletions(-) diff --git a/lbry/blob/blob_file.py b/lbry/blob/blob_file.py index 65e0d4a43..b8c7c461c 100644 --- a/lbry/blob/blob_file.py +++ b/lbry/blob/blob_file.py @@ -110,7 +110,7 @@ class AbstractBlob: if reader in self.readers: self.readers.remove(reader) - def _write_blob(self, blob_bytes: bytes): + def _write_blob(self, blob_bytes: bytes) -> asyncio.Task: raise NotImplementedError() def set_length(self, length) -> None: @@ -198,11 +198,17 @@ class AbstractBlob: def save_verified_blob(self, verified_bytes: bytes): if self.verified.is_set(): return - if self.is_writeable(): - self._write_blob(verified_bytes) + + def update_events(_): self.verified.set() + self.writing.clear() + + if self.is_writeable(): + self.writing.set() + task = self._write_blob(verified_bytes) + task.add_done_callback(update_events) if self.blob_completed_callback: - self.blob_completed_callback(self) + task.add_done_callback(lambda _: self.blob_completed_callback(self)) def get_blob_writer(self, peer_address: typing.Optional[str] = None, peer_port: typing.Optional[int] = None) -> HashBlobWriter: @@ -261,9 +267,11 @@ class BlobBuffer(AbstractBlob): self.verified.clear() def _write_blob(self, blob_bytes: bytes): - if self._verified_bytes: - raise OSError("already have bytes for blob") - self._verified_bytes = BytesIO(blob_bytes) + async def write(): + if self._verified_bytes: + raise OSError("already have bytes for blob") + self._verified_bytes = BytesIO(blob_bytes) + return self.loop.create_task(write()) def delete(self): if self._verified_bytes: @@ -319,8 +327,14 @@ class BlobFile(AbstractBlob): handle.close() def _write_blob(self, blob_bytes: bytes): - with open(self.file_path, 'wb') as f: - f.write(blob_bytes) + def _write_blob(): + with open(self.file_path, 'wb') as f: + f.write(blob_bytes) + + async def write_blob(): + await self.loop.run_in_executor(None, _write_blob) + + return self.loop.create_task(write_blob()) def delete(self): if os.path.isfile(self.file_path): diff --git a/lbry/blob_exchange/client.py b/lbry/blob_exchange/client.py index 408c0d323..61920c5b7 100644 --- a/lbry/blob_exchange/client.py +++ b/lbry/blob_exchange/client.py @@ -152,6 +152,8 @@ class BlobExchangeClientProtocol(asyncio.Protocol): log.debug(msg) msg = f"downloaded {self.blob.blob_hash[:8]} from {self.peer_address}:{self.peer_port}" await asyncio.wait_for(self.writer.finished, self.peer_timeout, loop=self.loop) + # wait for the io to finish + await self.blob.verified.wait() log.info("%s at %fMB/s", msg, round((float(self._blob_bytes_received) / float(time.perf_counter() - start_time)) / 1000000.0, 2)) diff --git a/lbry/stream/managed_stream.py b/lbry/stream/managed_stream.py index 8339f6a67..c449fe232 100644 --- a/lbry/stream/managed_stream.py +++ b/lbry/stream/managed_stream.py @@ -343,9 +343,10 @@ class ManagedStream: self.streaming.clear() @staticmethod - def _write_decrypted_blob(handle: typing.IO, data: bytes): - handle.write(data) - handle.flush() + def _write_decrypted_blob(output_path: str, data: bytes): + with open(output_path, 'ab') as handle: + handle.write(data) + handle.flush() async def _save_file(self, output_path: str): log.info("save file for lbry://%s#%s (sd hash %s...) -> %s", self.claim_name, self.claim_id, self.sd_hash[:6], @@ -355,12 +356,12 @@ class ManagedStream: self.finished_writing.clear() self.started_writing.clear() try: - with open(output_path, 'wb') as file_write_handle: - async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID): - log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1) - await self.loop.run_in_executor(None, self._write_decrypted_blob, file_write_handle, decrypted) - if not self.started_writing.is_set(): - self.started_writing.set() + open(output_path, 'wb').close() + async for blob_info, decrypted in self._aiter_read_stream(connection_id=self.SAVING_ID): + log.info("write blob %i/%i", blob_info.blob_num + 1, len(self.descriptor.blobs) - 1) + await self.loop.run_in_executor(None, self._write_decrypted_blob, output_path, decrypted) + if not self.started_writing.is_set(): + self.started_writing.set() await self.update_status(ManagedStream.STATUS_FINISHED) if self.analytics_manager: self.loop.create_task(self.analytics_manager.send_download_finished( diff --git a/tests/unit/blob/test_blob_manager.py b/tests/unit/blob/test_blob_manager.py index c868890f1..788ab0953 100644 --- a/tests/unit/blob/test_blob_manager.py +++ b/tests/unit/blob/test_blob_manager.py @@ -16,7 +16,7 @@ class TestBlobManager(AsyncioTestCase): self.blob_manager = BlobManager(self.loop, tmp_dir, self.storage, self.config) await self.storage.open() - async def test_memory_blobs_arent_verifie_but_real_ones_are(self): + async def test_memory_blobs_arent_verified_but_real_ones_are(self): for save_blobs in (False, True): await self.setup_blob_manager(save_blobs=save_blobs) # add a blob file @@ -24,6 +24,7 @@ class TestBlobManager(AsyncioTestCase): blob_bytes = b'1' * ((2 * 2 ** 20) - 1) blob = self.blob_manager.get_blob(blob_hash, len(blob_bytes)) blob.save_verified_blob(blob_bytes) + await blob.verified.wait() self.assertTrue(blob.get_is_verified()) self.blob_manager.blob_completed(blob) self.assertEqual(self.blob_manager.is_blob_verified(blob_hash), save_blobs) diff --git a/tests/unit/blob_exchange/test_transfer_blob.py b/tests/unit/blob_exchange/test_transfer_blob.py index f7c011e3b..fab7a4db0 100644 --- a/tests/unit/blob_exchange/test_transfer_blob.py +++ b/tests/unit/blob_exchange/test_transfer_blob.py @@ -130,10 +130,14 @@ class TestBlobExchange(BlobExchangeTestBase): write_blob = blob._write_blob write_called_count = 0 - def wrap_write_blob(blob_bytes): + async def _wrap_write_blob(blob_bytes): nonlocal write_called_count write_called_count += 1 - write_blob(blob_bytes) + await write_blob(blob_bytes) + + def wrap_write_blob(blob_bytes): + return asyncio.create_task(_wrap_write_blob(blob_bytes)) + blob._write_blob = wrap_write_blob writer1 = blob.get_blob_writer(peer_port=1) @@ -166,6 +170,7 @@ class TestBlobExchange(BlobExchangeTestBase): self.assertDictEqual({1: mock_blob_bytes, 2: mock_blob_bytes}, results) self.assertEqual(1, write_called_count) + await blob.verified.wait() self.assertTrue(blob.get_is_verified()) self.assertDictEqual({}, blob.writers) From decc5c74ef02e3e54790a08a3b9c377e70fe9db2 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 22 Apr 2020 18:17:01 -0400 Subject: [PATCH 07/86] don't block when reading a file when creating a stream --- lbry/stream/descriptor.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/lbry/stream/descriptor.py b/lbry/stream/descriptor.py index 1c0305ddc..2e2626b9b 100644 --- a/lbry/stream/descriptor.py +++ b/lbry/stream/descriptor.py @@ -44,18 +44,25 @@ def random_iv_generator() -> typing.Generator[bytes, None, None]: yield os.urandom(AES.block_size // 8) -def file_reader(file_path: str): +def read_bytes(file_path: str, offset: int, to_read: int): + with open(file_path, 'rb') as f: + f.seek(offset) + return f.read(to_read) + + +async def file_reader(file_path: str): length = int(os.stat(file_path).st_size) offset = 0 - with open(file_path, 'rb') as stream_file: - while offset < length: - bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1) - if not bytes_to_read: - break - blob_bytes = stream_file.read(bytes_to_read) - yield blob_bytes - offset += bytes_to_read + while offset < length: + bytes_to_read = min((length - offset), MAX_BLOB_SIZE - 1) + if not bytes_to_read: + break + blob_bytes = await asyncio.get_event_loop().run_in_executor( + None, read_bytes, file_path, offset, bytes_to_read + ) + yield blob_bytes + offset += bytes_to_read def sanitize_file_name(dirty_name: str, default_file_name: str = 'lbry_download'): @@ -245,7 +252,7 @@ class StreamDescriptor: iv_generator = iv_generator or random_iv_generator() key = key or os.urandom(AES.block_size // 8) blob_num = -1 - for blob_bytes in file_reader(file_path): + async for blob_bytes in file_reader(file_path): blob_num += 1 blob_info = await BlobFile.create_from_unencrypted( loop, blob_dir, key, next(iv_generator), blob_bytes, blob_num, blob_completed_callback From 08197a327e6a0af802cdbec6fe8881de0560d6ee Mon Sep 17 00:00:00 2001 From: Noah Date: Mon, 13 Apr 2020 07:13:19 -0400 Subject: [PATCH 08/86] fix Missing xdg download location Fixes an error in detection xdg config locations when XDG_DOWNLOAD_DIR is not present in `user-dirs.dirs` --- lbry/conf.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lbry/conf.py b/lbry/conf.py index fa395d8a9..6e1081a91 100644 --- a/lbry/conf.py +++ b/lbry/conf.py @@ -691,9 +691,10 @@ def get_darwin_directories() -> typing.Tuple[str, str, str]: def get_linux_directories() -> typing.Tuple[str, str, str]: try: with open(os.path.join(user_config_dir(), 'user-dirs.dirs'), 'r') as xdg: - down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()).group(1) - down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir) - download_dir = re.sub('\"', '', down_dir) + down_dir = re.search(r'XDG_DOWNLOAD_DIR=(.+)', xdg.read()) + if down_dir: + down_dir = re.sub(r'\$HOME', os.getenv('HOME') or os.path.expanduser("~/"), down_dir.group(1)) + download_dir = re.sub('\"', '', down_dir) except OSError: download_dir = os.getenv('XDG_DOWNLOAD_DIR') if not download_dir: From 523d22262b890da41eb695f411ed9ec70aac52cf Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Mon, 27 Apr 2020 10:18:39 -0400 Subject: [PATCH 09/86] pin pylint to 2.4.4 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d189bb6fa..23b3f84e8 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ install: pip install -e . tools: - pip install mypy==0.701 + pip install mypy==0.701 pylint==2.4.4 pip install coverage astroid pylint lint: From ced368db311d356f09d647de5ba07eb5700921b9 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 21 Apr 2020 19:21:15 -0300 Subject: [PATCH 10/86] hold headers file in memory during runtime --- lbry/wallet/header.py | 74 +++++++++++++------------------ tests/unit/wallet/test_headers.py | 1 + 2 files changed, 33 insertions(+), 42 deletions(-) diff --git a/lbry/wallet/header.py b/lbry/wallet/header.py index fd5a85c61..de366d802 100644 --- a/lbry/wallet/header.py +++ b/lbry/wallet/header.py @@ -42,23 +42,19 @@ class Headers: validate_difficulty: bool = True def __init__(self, path) -> None: - if path == ':memory:': - self.io = BytesIO() + self.io = BytesIO() self.path = path self._size: Optional[int] = None self.chunk_getter: Optional[Callable] = None - self.executor = ThreadPoolExecutor(1) self.known_missing_checkpointed_chunks = set() self.check_chunk_lock = asyncio.Lock() async def open(self): - if not self.executor: - self.executor = ThreadPoolExecutor(1) if self.path != ':memory:': - if not os.path.exists(self.path): - self.io = open(self.path, 'w+b') - else: - self.io = open(self.path, 'r+b') + if os.path.exists(self.path): + with open(self.path, 'r+b') as header_file: + self.io.seek(0) + self.io.write(header_file.read()) bytes_size = self.io.seek(0, os.SEEK_END) self._size = bytes_size // self.header_size max_checkpointed_height = max(self.checkpoints.keys() or [-1]) + 1000 @@ -72,10 +68,12 @@ class Headers: await self.get_all_missing_headers() async def close(self): - if self.executor: - self.executor.shutdown() - self.executor = None - self.io.close() + if self.io is not None: + flags = 'r+b' if os.path.exists(self.path) else 'w+b' + with open(self.path, flags) as header_file: + header_file.write(self.io.getbuffer()) + self.io.close() + self.io = None @staticmethod def serialize(header): @@ -148,15 +146,14 @@ class Headers: await self.ensure_chunk_at(height) if not 0 <= height <= self.height: raise IndexError(f"{height} is out of bounds, current height: {self.height}") - return await asyncio.get_running_loop().run_in_executor(self.executor, self._read, height) + return self._read(height) def _read(self, height, count=1): - self.io.seek(height * self.header_size, os.SEEK_SET) - return self.io.read(self.header_size * count) + offset = height * self.header_size + return bytes(self.io.getbuffer()[offset: offset + self.header_size * count]) def chunk_hash(self, start, count): - self.io.seek(start * self.header_size, os.SEEK_SET) - return self.hash_header(self.io.read(count * self.header_size)).decode() + return self.hash_header(self._read(start, count)).decode() async def ensure_checkpointed_size(self): max_checkpointed_height = max(self.checkpoints.keys() or [-1]) @@ -179,7 +176,7 @@ class Headers: ) chunk_hash = self.hash_header(chunk).decode() if self.checkpoints.get(start) == chunk_hash: - await asyncio.get_running_loop().run_in_executor(self.executor, self._write, start, chunk) + self._write(start, chunk) if start in self.known_missing_checkpointed_chunks: self.known_missing_checkpointed_chunks.remove(start) return @@ -194,22 +191,18 @@ class Headers: if normalized_height in self.checkpoints: return normalized_height not in self.known_missing_checkpointed_chunks - def _has_header(height): - empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d' - all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b' - return self.chunk_hash(height, 1) not in (empty, all_zeroes) - return await asyncio.get_running_loop().run_in_executor(self.executor, _has_header, height) + empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d' + all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b' + return self.chunk_hash(height, 1) not in (empty, all_zeroes) async def get_all_missing_headers(self): # Heavy operation done in one optimized shot - def _io_checkall(): - for chunk_height, expected_hash in reversed(list(self.checkpoints.items())): - if chunk_height in self.known_missing_checkpointed_chunks: - continue - if self.chunk_hash(chunk_height, 1000) != expected_hash: - self.known_missing_checkpointed_chunks.add(chunk_height) - return self.known_missing_checkpointed_chunks - return await asyncio.get_running_loop().run_in_executor(self.executor, _io_checkall) + for chunk_height, expected_hash in reversed(list(self.checkpoints.items())): + if chunk_height in self.known_missing_checkpointed_chunks: + continue + if self.chunk_hash(chunk_height, 1000) != expected_hash: + self.known_missing_checkpointed_chunks.add(chunk_height) + return self.known_missing_checkpointed_chunks @property def height(self) -> int: @@ -241,7 +234,7 @@ class Headers: bail = True chunk = chunk[:(height-e.height)*self.header_size] if chunk: - added += await asyncio.get_running_loop().run_in_executor(self.executor, self._write, height, chunk) + added += self._write(height, chunk) if bail: break return added @@ -306,9 +299,7 @@ class Headers: previous_header_hash = fail = None batch_size = 36 for height in range(start_height, self.height, batch_size): - headers = await asyncio.get_running_loop().run_in_executor( - self.executor, self._read, height, batch_size - ) + headers = self._read(height, batch_size) if len(headers) % self.header_size != 0: headers = headers[:(len(headers) // self.header_size) * self.header_size] for header_hash, header in self._iterate_headers(height, headers): @@ -324,12 +315,11 @@ class Headers: assert start_height > 0 and height == start_height if fail: log.warning("Header file corrupted at height %s, truncating it.", height - 1) - def __truncate(at_height): - self.io.seek(max(0, (at_height - 1)) * self.header_size, os.SEEK_SET) - self.io.truncate() - self.io.flush() - self._size = self.io.seek(0, os.SEEK_END) // self.header_size - return await asyncio.get_running_loop().run_in_executor(self.executor, __truncate, height) + self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET) + self.io.truncate() + self.io.flush() + self._size = self.io.seek(0, os.SEEK_END) // self.header_size + return previous_header_hash = header_hash @classmethod diff --git a/tests/unit/wallet/test_headers.py b/tests/unit/wallet/test_headers.py index e014f6e46..1b0e7628b 100644 --- a/tests/unit/wallet/test_headers.py +++ b/tests/unit/wallet/test_headers.py @@ -192,6 +192,7 @@ class TestHeaders(AsyncioTestCase): reader_task = asyncio.create_task(reader()) await writer() await reader_task + await headers.close() HEADERS = unhexlify( From 239ee2437c78ce48fbd957b11ba068b8cfb8a60f Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 22 Apr 2020 03:27:10 -0300 Subject: [PATCH 11/86] estimate only whats not downloaded --- lbry/wallet/header.py | 7 +++++-- tests/unit/wallet/test_headers.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/lbry/wallet/header.py b/lbry/wallet/header.py index de366d802..fc9018562 100644 --- a/lbry/wallet/header.py +++ b/lbry/wallet/header.py @@ -136,6 +136,9 @@ class Headers: def estimated_timestamp(self, height): if height <= 0: return + if self.has_header(height): + offset = height * self.header_size + return struct.unpack(' Date: Wed, 22 Apr 2020 16:57:37 -0300 Subject: [PATCH 12/86] test fixes + leave tx plot always on estimations --- lbry/wallet/header.py | 10 ++++----- tests/unit/wallet/test_database.py | 1 + tests/unit/wallet/test_headers.py | 34 +++++++++++++++++++----------- tests/unit/wallet/test_ledger.py | 2 ++ 4 files changed, 30 insertions(+), 17 deletions(-) diff --git a/lbry/wallet/header.py b/lbry/wallet/header.py index fc9018562..6094aa017 100644 --- a/lbry/wallet/header.py +++ b/lbry/wallet/header.py @@ -5,7 +5,6 @@ import asyncio import logging import zlib from datetime import date -from concurrent.futures.thread import ThreadPoolExecutor from io import BytesIO from typing import Optional, Iterator, Tuple, Callable @@ -42,7 +41,7 @@ class Headers: validate_difficulty: bool = True def __init__(self, path) -> None: - self.io = BytesIO() + self.io = None self.path = path self._size: Optional[int] = None self.chunk_getter: Optional[Callable] = None @@ -50,6 +49,7 @@ class Headers: self.check_chunk_lock = asyncio.Lock() async def open(self): + self.io = BytesIO() if self.path != ':memory:': if os.path.exists(self.path): with open(self.path, 'r+b') as header_file: @@ -133,16 +133,16 @@ class Headers: except struct.error: raise IndexError(f"failed to get {height}, at {len(self)}") - def estimated_timestamp(self, height): + def estimated_timestamp(self, height, try_real_headers=True): if height <= 0: return - if self.has_header(height): + if try_real_headers and self.has_header(height): offset = height * self.header_size return struct.unpack(' bytes: if self.chunk_getter: diff --git a/tests/unit/wallet/test_database.py b/tests/unit/wallet/test_database.py index 7ceed23d7..b796a9c1a 100644 --- a/tests/unit/wallet/test_database.py +++ b/tests/unit/wallet/test_database.py @@ -211,6 +211,7 @@ class TestQueries(AsyncioTestCase): 'db': Database(':memory:'), 'headers': Headers(':memory:') }) + await self.ledger.headers.open() self.wallet = Wallet() await self.ledger.db.open() diff --git a/tests/unit/wallet/test_headers.py b/tests/unit/wallet/test_headers.py index 578b85512..da433724d 100644 --- a/tests/unit/wallet/test_headers.py +++ b/tests/unit/wallet/test_headers.py @@ -21,8 +21,8 @@ class TestHeaders(AsyncioTestCase): async def test_deserialize(self): self.maxDiff = None h = Headers(':memory:') - h.io.write(HEADERS) await h.open() + await h.connect(0, HEADERS) self.assertEqual(await h.get(0), { 'bits': 520159231, 'block_height': 0, @@ -52,8 +52,11 @@ class TestHeaders(AsyncioTestCase): self.assertEqual(headers.height, 19) async def test_connect_from_middle(self): - h = Headers(':memory:') - h.io.write(HEADERS[:block_bytes(10)]) + headers_temporary_file = tempfile.mktemp() + self.addCleanup(os.remove, headers_temporary_file) + with open(headers_temporary_file, 'w+b') as headers_file: + headers_file.write(HEADERS[:block_bytes(10)]) + h = Headers(headers_temporary_file) await h.open() self.assertEqual(h.height, 9) await h.connect(len(h), HEADERS[block_bytes(10):block_bytes(20)]) @@ -115,6 +118,7 @@ class TestHeaders(AsyncioTestCase): async def test_bounds(self): headers = Headers(':memory:') + await headers.open() await headers.connect(0, HEADERS) self.assertEqual(19, headers.height) with self.assertRaises(IndexError): @@ -126,6 +130,7 @@ class TestHeaders(AsyncioTestCase): async def test_repair(self): headers = Headers(':memory:') + await headers.open() await headers.connect(0, HEADERS[:block_bytes(11)]) self.assertEqual(10, headers.height) await headers.repair() @@ -147,8 +152,9 @@ class TestHeaders(AsyncioTestCase): await headers.repair(start_height=10) self.assertEqual(19, headers.height) - def test_do_not_estimate_unconfirmed(self): + async def test_do_not_estimate_unconfirmed(self): headers = Headers(':memory:') + await headers.open() self.assertIsNone(headers.estimated_timestamp(-1)) self.assertIsNone(headers.estimated_timestamp(0)) self.assertIsNotNone(headers.estimated_timestamp(1)) @@ -164,17 +170,21 @@ class TestHeaders(AsyncioTestCase): self.assertEqual(after_downloading_header_estimated, real_time) async def test_misalignment_triggers_repair_on_open(self): - headers = Headers(':memory:') - headers.io.seek(0) - headers.io.write(HEADERS) + headers_temporary_file = tempfile.mktemp() + self.addCleanup(os.remove, headers_temporary_file) + with open(headers_temporary_file, 'w+b') as headers_file: + headers_file.write(HEADERS) + headers = Headers(headers_temporary_file) with self.assertLogs(level='WARN') as cm: await headers.open() + await headers.close() self.assertEqual(cm.output, []) - headers.io.seek(0) - headers.io.truncate() - headers.io.write(HEADERS[:block_bytes(10)]) - headers.io.write(b'ops') - headers.io.write(HEADERS[block_bytes(10):]) + with open(headers_temporary_file, 'w+b') as headers_file: + headers_file.seek(0) + headers_file.truncate() + headers_file.write(HEADERS[:block_bytes(10)]) + headers_file.write(b'ops') + headers_file.write(HEADERS[block_bytes(10):]) await headers.open() self.assertEqual( cm.output, [ diff --git a/tests/unit/wallet/test_ledger.py b/tests/unit/wallet/test_ledger.py index 0244de987..bfe5cc71b 100644 --- a/tests/unit/wallet/test_ledger.py +++ b/tests/unit/wallet/test_ledger.py @@ -48,6 +48,8 @@ class LedgerTestCase(AsyncioTestCase): 'db': Database(':memory:'), 'headers': Headers(':memory:') }) + self.ledger.headers.checkpoints = {} + await self.ledger.headers.open() self.account = Account.generate(self.ledger, Wallet(), "lbryum") await self.ledger.db.open() From 58f77b2a1c132d321f5ea410b336cdcc665d59db Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sat, 25 Apr 2020 04:24:19 -0300 Subject: [PATCH 13/86] load/dump header file using executor --- lbry/wallet/header.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/lbry/wallet/header.py b/lbry/wallet/header.py index 6094aa017..c67b0dd1d 100644 --- a/lbry/wallet/header.py +++ b/lbry/wallet/header.py @@ -51,10 +51,12 @@ class Headers: async def open(self): self.io = BytesIO() if self.path != ':memory:': - if os.path.exists(self.path): - with open(self.path, 'r+b') as header_file: - self.io.seek(0) - self.io.write(header_file.read()) + def _readit(): + if os.path.exists(self.path): + with open(self.path, 'r+b') as header_file: + self.io.seek(0) + self.io.write(header_file.read()) + await asyncio.get_event_loop().run_in_executor(None, _readit) bytes_size = self.io.seek(0, os.SEEK_END) self._size = bytes_size // self.header_size max_checkpointed_height = max(self.checkpoints.keys() or [-1]) + 1000 @@ -69,9 +71,11 @@ class Headers: async def close(self): if self.io is not None: - flags = 'r+b' if os.path.exists(self.path) else 'w+b' - with open(self.path, flags) as header_file: - header_file.write(self.io.getbuffer()) + def _close(): + flags = 'r+b' if os.path.exists(self.path) else 'w+b' + with open(self.path, flags) as header_file: + header_file.write(self.io.getbuffer()) + await asyncio.get_event_loop().run_in_executor(None, _close) self.io.close() self.io = None From 00c0f48b029d6c03a128b61f572e88875cdd7b00 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Mon, 27 Apr 2020 12:21:04 -0400 Subject: [PATCH 14/86] v0.71.0 --- lbry/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbry/__init__.py b/lbry/__init__.py index e29ac73ee..75bafa71b 100644 --- a/lbry/__init__.py +++ b/lbry/__init__.py @@ -1,2 +1,2 @@ -__version__ = "0.70.0" +__version__ = "0.71.0" version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name From ff7bed720aee2335cd8510becb05d47beec3780e Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 29 Apr 2020 12:31:38 -0400 Subject: [PATCH 15/86] don't close the connection upon a cancelled request --- lbry/wallet/network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbry/wallet/network.py b/lbry/wallet/network.py index b117a0164..94ffd5e4f 100644 --- a/lbry/wallet/network.py +++ b/lbry/wallet/network.py @@ -87,7 +87,7 @@ class ClientSession(BaseClientSession): raise except asyncio.CancelledError: log.info("cancelled sending %s to %s:%i", method, *self.server) - self.synchronous_close() + # self.synchronous_close() raise finally: self.pending_amount -= 1 From 0a9d4de1265df4f96a9c6a58c2881891ee8d6209 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Fri, 1 May 2020 11:40:52 -0400 Subject: [PATCH 16/86] include write lock in try/finally --- lbry/wallet/database.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lbry/wallet/database.py b/lbry/wallet/database.py index 5aec29649..ec92d6522 100644 --- a/lbry/wallet/database.py +++ b/lbry/wallet/database.py @@ -137,15 +137,15 @@ class AIOSQLite: async def run(self, fun, *args, **kwargs): self.writers += 1 self.read_ready.clear() - async with self.write_lock: - try: + try: + async with self.write_lock: return await asyncio.get_event_loop().run_in_executor( self.writer_executor, lambda: self.__run_transaction(fun, *args, **kwargs) ) - finally: - self.writers -= 1 - if not self.writers: - self.read_ready.set() + finally: + self.writers -= 1 + if not self.writers: + self.read_ready.set() def __run_transaction(self, fun: Callable[[sqlite3.Connection, Any, Any], Any], *args, **kwargs): self.writer_connection.execute('begin') From 79624febc09bd2dcd93453a83b74f98458b9889c Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Fri, 1 May 2020 12:48:41 -0400 Subject: [PATCH 17/86] prevent pileup of writes blocking reads --- lbry/wallet/database.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/lbry/wallet/database.py b/lbry/wallet/database.py index ec92d6522..446f53ee5 100644 --- a/lbry/wallet/database.py +++ b/lbry/wallet/database.py @@ -73,6 +73,7 @@ class AIOSQLite: self.write_lock = asyncio.Lock() self.writers = 0 self.read_ready = asyncio.Event() + self.urgent_read_done = asyncio.Event() @classmethod async def connect(cls, path: Union[bytes, str], *args, **kwargs): @@ -88,6 +89,7 @@ class AIOSQLite: ) await asyncio.get_event_loop().run_in_executor(db.writer_executor, _connect_writer) db.read_ready.set() + db.urgent_read_done.set() return db async def close(self): @@ -112,12 +114,25 @@ class AIOSQLite: read_only=False, fetch_all: bool = False) -> List[dict]: read_only_fn = run_read_only_fetchall if fetch_all else run_read_only_fetchone parameters = parameters if parameters is not None else [] + still_waiting = False + urgent_read = False if read_only: - while self.writers: - await self.read_ready.wait() - return await asyncio.get_event_loop().run_in_executor( - self.reader_executor, read_only_fn, sql, parameters - ) + try: + while self.writers: # more writes can come in while we are waiting for the first + if not urgent_read and still_waiting and self.urgent_read_done.is_set(): + # throttle the writes if they pile up + self.urgent_read_done.clear() + urgent_read = True + # wait until the running writes have finished + await self.read_ready.wait() + still_waiting = True + return await asyncio.get_event_loop().run_in_executor( + self.reader_executor, read_only_fn, sql, parameters + ) + finally: + if urgent_read: + # unthrottle the writers if they had to be throttled + self.urgent_read_done.set() if fetch_all: return await self.run(lambda conn: conn.execute(sql, parameters).fetchall()) return await self.run(lambda conn: conn.execute(sql, parameters).fetchone()) @@ -135,6 +150,7 @@ class AIOSQLite: return self.run(lambda conn: conn.execute(sql, parameters)) async def run(self, fun, *args, **kwargs): + await self.urgent_read_done.wait() self.writers += 1 self.read_ready.clear() try: From 36c05fc4b97e7092b185fd48c56334f20b12cc32 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Thu, 23 Apr 2020 20:21:27 -0400 Subject: [PATCH 18/86] move wallet server prometheus -only run wallet server metrics for the wallet server --- lbry/prometheus.py | 32 +++ lbry/wallet/rpc/session.py | 10 +- lbry/wallet/server/block_processor.py | 8 +- lbry/wallet/server/daemon.py | 11 +- lbry/wallet/server/prometheus.py | 188 +++++++++++------- lbry/wallet/server/server.py | 7 +- lbry/wallet/server/session.py | 28 ++- .../test_blockchain_reorganization.py | 8 +- 8 files changed, 181 insertions(+), 111 deletions(-) create mode 100644 lbry/prometheus.py diff --git a/lbry/prometheus.py b/lbry/prometheus.py new file mode 100644 index 000000000..220ee97bd --- /dev/null +++ b/lbry/prometheus.py @@ -0,0 +1,32 @@ +import logging +from aiohttp import web +from prometheus_client import generate_latest as prom_generate_latest + + +class PrometheusServer: + def __init__(self, logger=None): + self.runner = None + self.logger = logger or logging.getLogger(__name__) + + async def start(self, interface: str, port: int): + prom_app = web.Application() + prom_app.router.add_get('/metrics', self.handle_metrics_get_request) + self.runner = web.AppRunner(prom_app) + await self.runner.setup() + + metrics_site = web.TCPSite(self.runner, interface, port, shutdown_timeout=.5) + await metrics_site.start() + self.logger.info('metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2]) + + async def handle_metrics_get_request(self, request: web.Request): + try: + return web.Response( + text=prom_generate_latest().decode(), + content_type='text/plain; version=0.0.4' + ) + except Exception: + self.logger.exception('could not generate prometheus data') + raise + + async def stop(self): + await self.runner.cleanup() diff --git a/lbry/wallet/rpc/session.py b/lbry/wallet/rpc/session.py index 53c164f4f..a454e3a30 100644 --- a/lbry/wallet/rpc/session.py +++ b/lbry/wallet/rpc/session.py @@ -39,7 +39,7 @@ from lbry.wallet.tasks import TaskGroup from .jsonrpc import Request, JSONRPCConnection, JSONRPCv2, JSONRPC, Batch, Notification from .jsonrpc import RPCError, ProtocolError from .framing import BadMagicError, BadChecksumError, OversizedPayloadError, BitcoinFramer, NewlineFramer -from lbry.wallet.server.prometheus import NOTIFICATION_COUNT, RESPONSE_TIMES, REQUEST_ERRORS_COUNT, RESET_CONNECTIONS +from lbry.wallet.server import prometheus class Connector: @@ -388,7 +388,7 @@ class RPCSession(SessionBase): except MemoryError: self.logger.warning('received oversized message from %s:%s, dropping connection', self._address[0], self._address[1]) - RESET_CONNECTIONS.labels(version=self.client_version).inc() + prometheus.METRICS.RESET_CONNECTIONS.labels(version=self.client_version).inc() self._close() return @@ -422,7 +422,7 @@ class RPCSession(SessionBase): 'internal server error') if isinstance(request, Request): message = request.send_result(result) - RESPONSE_TIMES.labels( + prometheus.METRICS.RESPONSE_TIMES.labels( method=request.method, version=self.client_version ).observe(time.perf_counter() - start) @@ -430,7 +430,7 @@ class RPCSession(SessionBase): await self._send_message(message) if isinstance(result, Exception): self._bump_errors() - REQUEST_ERRORS_COUNT.labels( + prometheus.METRICS.REQUEST_ERRORS_COUNT.labels( method=request.method, version=self.client_version ).inc() @@ -467,7 +467,7 @@ class RPCSession(SessionBase): async def send_notification(self, method, args=()): """Send an RPC notification over the network.""" message = self.connection.send_notification(Notification(method, args)) - NOTIFICATION_COUNT.labels(method=method, version=self.client_version).inc() + prometheus.METRICS.NOTIFICATION_COUNT.labels(method=method, version=self.client_version).inc() await self._send_message(message) def send_batch(self, raise_errors=False): diff --git a/lbry/wallet/server/block_processor.py b/lbry/wallet/server/block_processor.py index 44eba7d1a..48a4519d0 100644 --- a/lbry/wallet/server/block_processor.py +++ b/lbry/wallet/server/block_processor.py @@ -10,7 +10,7 @@ from lbry.wallet.server.daemon import DaemonError from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN from lbry.wallet.server.util import chunks, class_logger from lbry.wallet.server.leveldb import FlushData -from lbry.wallet.server.prometheus import BLOCK_COUNT, BLOCK_UPDATE_TIMES, REORG_COUNT +from lbry.wallet.server import prometheus class Prefetcher: @@ -199,8 +199,8 @@ class BlockProcessor: cache.clear() await self._maybe_flush() processed_time = time.perf_counter() - start - BLOCK_COUNT.set(self.height) - BLOCK_UPDATE_TIMES.observe(processed_time) + prometheus.METRICS.BLOCK_COUNT.set(self.height) + prometheus.METRICS.BLOCK_UPDATE_TIMES.observe(processed_time) if not self.db.first_sync: s = '' if len(blocks) == 1 else 's' self.logger.info('processed {:,d} block{} in {:.1f}s'.format(len(blocks), s, processed_time)) @@ -255,7 +255,7 @@ class BlockProcessor: last -= len(raw_blocks) await self.run_in_thread_with_lock(self.db.sql.delete_claims_above_height, self.height) await self.prefetcher.reset_height(self.height) - REORG_COUNT.inc() + prometheus.METRICS.REORG_COUNT.inc() async def reorg_hashes(self, count): """Return a pair (start, last, hashes) of blocks to back up during a diff --git a/lbry/wallet/server/daemon.py b/lbry/wallet/server/daemon.py index 960c47024..39487f367 100644 --- a/lbry/wallet/server/daemon.py +++ b/lbry/wallet/server/daemon.py @@ -10,7 +10,8 @@ import aiohttp from lbry.wallet.rpc.jsonrpc import RPCError from lbry.wallet.server.util import hex_to_bytes, class_logger from lbry.wallet.rpc import JSONRPC -from lbry.wallet.server.prometheus import LBRYCRD_REQUEST_TIMES, LBRYCRD_PENDING_COUNT +from lbry.wallet.server import prometheus + class DaemonError(Exception): """Raised when the daemon returns an error in its results.""" @@ -129,7 +130,7 @@ class Daemon: while True: try: for method in methods: - LBRYCRD_PENDING_COUNT.labels(method=method).inc() + prometheus.METRICS.LBRYCRD_PENDING_COUNT.labels(method=method).inc() result = await self._send_data(data) result = processor(result) if on_good_message: @@ -154,7 +155,7 @@ class Daemon: on_good_message = 'running normally' finally: for method in methods: - LBRYCRD_PENDING_COUNT.labels(method=method).dec() + prometheus.METRICS.LBRYCRD_PENDING_COUNT.labels(method=method).dec() await asyncio.sleep(retry) retry = max(min(self.max_retry, retry * 2), self.init_retry) @@ -175,7 +176,7 @@ class Daemon: if params: payload['params'] = params result = await self._send(payload, processor) - LBRYCRD_REQUEST_TIMES.labels(method=method).observe(time.perf_counter() - start) + prometheus.METRICS.LBRYCRD_REQUEST_TIMES.labels(method=method).observe(time.perf_counter() - start) return result async def _send_vector(self, method, params_iterable, replace_errs=False): @@ -200,7 +201,7 @@ class Daemon: result = [] if payload: result = await self._send(payload, processor) - LBRYCRD_REQUEST_TIMES.labels(method=method).observe(time.perf_counter()-start) + prometheus.METRICS.LBRYCRD_REQUEST_TIMES.labels(method=method).observe(time.perf_counter()-start) return result async def _is_rpc_available(self, method): diff --git a/lbry/wallet/server/prometheus.py b/lbry/wallet/server/prometheus.py index e28976bf9..ee3e268cf 100644 --- a/lbry/wallet/server/prometheus.py +++ b/lbry/wallet/server/prometheus.py @@ -1,89 +1,125 @@ import os -from aiohttp import web -from prometheus_client import Counter, Info, generate_latest as prom_generate_latest, Histogram, Gauge +from prometheus_client import Counter, Info, Histogram, Gauge from lbry import __version__ as version from lbry.build_info import BUILD, COMMIT_HASH, DOCKER_TAG from lbry.wallet.server import util import lbry.wallet.server.version as wallet_server_version -NAMESPACE = "wallet_server" -CPU_COUNT = f"{os.cpu_count()}" -VERSION_INFO = Info('build', 'Wallet server build info (e.g. version, commit hash)', namespace=NAMESPACE) -VERSION_INFO.info({ - 'build': BUILD, - "commit": COMMIT_HASH, - "docker_tag": DOCKER_TAG, - 'version': version, - "min_version": util.version_string(wallet_server_version.PROTOCOL_MIN), - "cpu_count": CPU_COUNT -}) -SESSIONS_COUNT = Gauge("session_count", "Number of connected client sessions", namespace=NAMESPACE, - labelnames=("version", )) -REQUESTS_COUNT = Counter("requests_count", "Number of requests received", namespace=NAMESPACE, - labelnames=("method", "version")) -RESPONSE_TIMES = Histogram("response_time", "Response times", namespace=NAMESPACE, labelnames=("method", "version")) -NOTIFICATION_COUNT = Counter("notification", "Number of notifications sent (for subscriptions)", - namespace=NAMESPACE, labelnames=("method", "version")) -REQUEST_ERRORS_COUNT = Counter("request_error", "Number of requests that returned errors", namespace=NAMESPACE, - labelnames=("method", "version")) -SQLITE_INTERRUPT_COUNT = Counter("interrupt", "Number of interrupted queries", namespace=NAMESPACE) -SQLITE_OPERATIONAL_ERROR_COUNT = Counter( - "operational_error", "Number of queries that raised operational errors", namespace=NAMESPACE -) -SQLITE_INTERNAL_ERROR_COUNT = Counter( - "internal_error", "Number of queries raising unexpected errors", namespace=NAMESPACE -) -SQLITE_EXECUTOR_TIMES = Histogram("executor_time", "SQLite executor times", namespace=NAMESPACE) -SQLITE_PENDING_COUNT = Gauge( - "pending_queries_count", "Number of pending and running sqlite queries", namespace=NAMESPACE -) -LBRYCRD_REQUEST_TIMES = Histogram( - "lbrycrd_request", "lbrycrd requests count", namespace=NAMESPACE, labelnames=("method",) -) -LBRYCRD_PENDING_COUNT = Gauge( - "lbrycrd_pending_count", "Number of lbrycrd rpcs that are in flight", namespace=NAMESPACE, labelnames=("method",) -) -CLIENT_VERSIONS = Counter( - "clients", "Number of connections received per client version", - namespace=NAMESPACE, labelnames=("version",) -) -BLOCK_COUNT = Gauge( - "block_count", "Number of processed blocks", namespace=NAMESPACE -) -BLOCK_UPDATE_TIMES = Histogram("block_time", "Block update times", namespace=NAMESPACE) -REORG_COUNT = Gauge( - "reorg_count", "Number of reorgs", namespace=NAMESPACE -) -RESET_CONNECTIONS = Counter( - "reset_clients", "Number of reset connections by client version", - namespace=NAMESPACE, labelnames=("version",) -) +class PrometheusMetrics: + VERSION_INFO: Info + SESSIONS_COUNT: Gauge + REQUESTS_COUNT: Counter + RESPONSE_TIMES: Histogram + NOTIFICATION_COUNT: Counter + REQUEST_ERRORS_COUNT: Counter + SQLITE_INTERRUPT_COUNT: Counter + SQLITE_OPERATIONAL_ERROR_COUNT: Counter + SQLITE_INTERNAL_ERROR_COUNT: Counter + SQLITE_EXECUTOR_TIMES: Histogram + SQLITE_PENDING_COUNT: Gauge + LBRYCRD_REQUEST_TIMES: Histogram + LBRYCRD_PENDING_COUNT: Gauge + CLIENT_VERSIONS: Counter + BLOCK_COUNT: Gauge + BLOCK_UPDATE_TIMES: Histogram + REORG_COUNT: Gauge + RESET_CONNECTIONS: Counter + + __slots__ = [ + 'VERSION_INFO', + 'SESSIONS_COUNT', + 'REQUESTS_COUNT', + 'RESPONSE_TIMES', + 'NOTIFICATION_COUNT', + 'REQUEST_ERRORS_COUNT', + 'SQLITE_INTERRUPT_COUNT', + 'SQLITE_OPERATIONAL_ERROR_COUNT', + 'SQLITE_INTERNAL_ERROR_COUNT', + 'SQLITE_EXECUTOR_TIMES', + 'SQLITE_PENDING_COUNT', + 'LBRYCRD_REQUEST_TIMES', + 'LBRYCRD_PENDING_COUNT', + 'CLIENT_VERSIONS', + 'BLOCK_COUNT', + 'BLOCK_UPDATE_TIMES', + 'REORG_COUNT', + 'RESET_CONNECTIONS', + '_installed', + 'namespace', + 'cpu_count' + ] -class PrometheusServer: def __init__(self): - self.logger = util.class_logger(__name__, self.__class__.__name__) - self.runner = None + self._installed = False + self.namespace = "wallet_server" + self.cpu_count = f"{os.cpu_count()}" - async def start(self, port: int): - prom_app = web.Application() - prom_app.router.add_get('/metrics', self.handle_metrics_get_request) - self.runner = web.AppRunner(prom_app) - await self.runner.setup() + def uninstall(self): + self._installed = False + for item in self.__slots__: + if not item.startswith('_') and item not in ('namespace', 'cpu_count'): + current = getattr(self, item, None) + if current: + setattr(self, item, None) + del current - metrics_site = web.TCPSite(self.runner, "0.0.0.0", port, shutdown_timeout=.5) - await metrics_site.start() - self.logger.info('metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2]) + def install(self): + if self._installed: + return + self._installed = True + self.VERSION_INFO = Info('build', 'Wallet server build info (e.g. version, commit hash)', namespace=self.namespace) + self.VERSION_INFO.info({ + 'build': BUILD, + "commit": COMMIT_HASH, + "docker_tag": DOCKER_TAG, + 'version': version, + "min_version": util.version_string(wallet_server_version.PROTOCOL_MIN), + "cpu_count": self.cpu_count + }) + self.SESSIONS_COUNT = Gauge("session_count", "Number of connected client sessions", namespace=self.namespace, + labelnames=("version",)) + self.REQUESTS_COUNT = Counter("requests_count", "Number of requests received", namespace=self.namespace, + labelnames=("method", "version")) + self.RESPONSE_TIMES = Histogram("response_time", "Response times", namespace=self.namespace, + labelnames=("method", "version")) + self.NOTIFICATION_COUNT = Counter("notification", "Number of notifications sent (for subscriptions)", + namespace=self.namespace, labelnames=("method", "version")) + self.REQUEST_ERRORS_COUNT = Counter("request_error", "Number of requests that returned errors", namespace=self.namespace, + labelnames=("method", "version")) + self.SQLITE_INTERRUPT_COUNT = Counter("interrupt", "Number of interrupted queries", namespace=self.namespace) + self.SQLITE_OPERATIONAL_ERROR_COUNT = Counter( + "operational_error", "Number of queries that raised operational errors", namespace=self.namespace + ) + self.SQLITE_INTERNAL_ERROR_COUNT = Counter( + "internal_error", "Number of queries raising unexpected errors", namespace=self.namespace + ) + self.SQLITE_EXECUTOR_TIMES = Histogram("executor_time", "SQLite executor times", namespace=self.namespace) + self.SQLITE_PENDING_COUNT = Gauge( + "pending_queries_count", "Number of pending and running sqlite queries", namespace=self.namespace + ) + self.LBRYCRD_REQUEST_TIMES = Histogram( + "lbrycrd_request", "lbrycrd requests count", namespace=self.namespace, labelnames=("method",) + ) + self.LBRYCRD_PENDING_COUNT = Gauge( + "lbrycrd_pending_count", "Number of lbrycrd rpcs that are in flight", namespace=self.namespace, + labelnames=("method",) + ) + self.CLIENT_VERSIONS = Counter( + "clients", "Number of connections received per client version", + namespace=self.namespace, labelnames=("version",) + ) + self.BLOCK_COUNT = Gauge( + "block_count", "Number of processed blocks", namespace=self.namespace + ) + self.BLOCK_UPDATE_TIMES = Histogram("block_time", "Block update times", namespace=self.namespace) + self.REORG_COUNT = Gauge( + "reorg_count", "Number of reorgs", namespace=self.namespace + ) + self.RESET_CONNECTIONS = Counter( + "reset_clients", "Number of reset connections by client version", + namespace=self.namespace, labelnames=("version",) + ) - async def handle_metrics_get_request(self, request: web.Request): - try: - return web.Response( - text=prom_generate_latest().decode(), - content_type='text/plain; version=0.0.4' - ) - except Exception: - self.logger.exception('could not generate prometheus data') - raise - async def stop(self): - await self.runner.cleanup() +METRICS = PrometheusMetrics() diff --git a/lbry/wallet/server/server.py b/lbry/wallet/server/server.py index 4d0374ba4..98000ace9 100644 --- a/lbry/wallet/server/server.py +++ b/lbry/wallet/server/server.py @@ -6,7 +6,8 @@ import typing import lbry from lbry.wallet.server.mempool import MemPool, MemPoolAPI -from lbry.wallet.server.prometheus import PrometheusServer +from lbry.prometheus import PrometheusServer +from lbry.wallet.server.prometheus import METRICS class Notifications: @@ -92,6 +93,7 @@ class Server: ) async def start(self): + METRICS.install() env = self.env min_str, max_str = env.coin.SESSIONCLS.protocol_min_max_strings() self.log.info(f'software version: {lbry.__version__}') @@ -121,6 +123,7 @@ class Server: self.prometheus_server = None self.shutdown_event.set() await self.daemon.close() + METRICS.uninstall() def run(self): loop = asyncio.get_event_loop() @@ -143,4 +146,4 @@ class Server: async def start_prometheus(self): if not self.prometheus_server and self.env.prometheus_port: self.prometheus_server = PrometheusServer() - await self.prometheus_server.start(self.env.prometheus_port) + await self.prometheus_server.start("0.0.0.0", self.env.prometheus_port) diff --git a/lbry/wallet/server/session.py b/lbry/wallet/server/session.py index 9a9e23558..0dccce853 100644 --- a/lbry/wallet/server/session.py +++ b/lbry/wallet/server/session.py @@ -27,9 +27,7 @@ from lbry.wallet.server.db.writer import LBRYLevelDB from lbry.wallet.server.db import reader from lbry.wallet.server.websocket import AdminWebSocket from lbry.wallet.server.metrics import ServerLoadData, APICallMetrics -from lbry.wallet.server.prometheus import REQUESTS_COUNT, SQLITE_INTERRUPT_COUNT, SQLITE_INTERNAL_ERROR_COUNT -from lbry.wallet.server.prometheus import SQLITE_OPERATIONAL_ERROR_COUNT, SQLITE_EXECUTOR_TIMES, SESSIONS_COUNT -from lbry.wallet.server.prometheus import SQLITE_PENDING_COUNT, CLIENT_VERSIONS +from lbry.wallet.server import prometheus from lbry.wallet.rpc.framing import NewlineFramer import lbry.wallet.server.version as VERSION @@ -677,7 +675,7 @@ class SessionBase(RPCSession): context = {'conn_id': f'{self.session_id}'} self.logger = util.ConnectionLogger(self.logger, context) self.group = self.session_mgr.add_session(self) - SESSIONS_COUNT.labels(version=self.client_version).inc() + prometheus.METRICS.SESSIONS_COUNT.labels(version=self.client_version).inc() peer_addr_str = self.peer_address_str() self.logger.info(f'{self.kind} {peer_addr_str}, ' f'{self.session_mgr.session_count():,d} total') @@ -686,7 +684,7 @@ class SessionBase(RPCSession): """Handle client disconnection.""" super().connection_lost(exc) self.session_mgr.remove_session(self) - SESSIONS_COUNT.labels(version=self.client_version).dec() + prometheus.METRICS.SESSIONS_COUNT.labels(version=self.client_version).dec() msg = '' if not self._can_send.is_set(): msg += ' whilst paused' @@ -710,7 +708,7 @@ class SessionBase(RPCSession): """Handle an incoming request. ElectrumX doesn't receive notifications from client sessions. """ - REQUESTS_COUNT.labels(method=request.method, version=self.client_version).inc() + prometheus.METRICS.REQUESTS_COUNT.labels(method=request.method, version=self.client_version).inc() if isinstance(request, Request): handler = self.request_handlers.get(request.method) handler = partial(handler, self) @@ -946,7 +944,7 @@ class LBRYElectrumX(SessionBase): async def run_in_executor(self, query_name, func, kwargs): start = time.perf_counter() try: - SQLITE_PENDING_COUNT.inc() + prometheus.METRICS.SQLITE_PENDING_COUNT.inc() result = await asyncio.get_running_loop().run_in_executor( self.session_mgr.query_executor, func, kwargs ) @@ -955,18 +953,18 @@ class LBRYElectrumX(SessionBase): except reader.SQLiteInterruptedError as error: metrics = self.get_metrics_or_placeholder_for_api(query_name) metrics.query_interrupt(start, error.metrics) - SQLITE_INTERRUPT_COUNT.inc() + prometheus.METRICS.prometheus.METRICS.SQLITE_INTERRUPT_COUNT.inc() raise RPCError(JSONRPC.QUERY_TIMEOUT, 'sqlite query timed out') except reader.SQLiteOperationalError as error: metrics = self.get_metrics_or_placeholder_for_api(query_name) metrics.query_error(start, error.metrics) - SQLITE_OPERATIONAL_ERROR_COUNT.inc() + prometheus.METRICS.SQLITE_OPERATIONAL_ERROR_COUNT.inc() raise RPCError(JSONRPC.INTERNAL_ERROR, 'query failed to execute') except Exception: log.exception("dear devs, please handle this exception better") metrics = self.get_metrics_or_placeholder_for_api(query_name) metrics.query_error(start, {}) - SQLITE_INTERNAL_ERROR_COUNT.inc() + prometheus.METRICS.SQLITE_INTERNAL_ERROR_COUNT.inc() raise RPCError(JSONRPC.INTERNAL_ERROR, 'unknown server error') else: if self.env.track_metrics: @@ -975,8 +973,8 @@ class LBRYElectrumX(SessionBase): metrics.query_response(start, metrics_data) return base64.b64encode(result).decode() finally: - SQLITE_PENDING_COUNT.dec() - SQLITE_EXECUTOR_TIMES.observe(time.perf_counter() - start) + prometheus.METRICS.SQLITE_PENDING_COUNT.dec() + prometheus.METRICS.SQLITE_EXECUTOR_TIMES.observe(time.perf_counter() - start) async def run_and_cache_query(self, query_name, function, kwargs): metrics = self.get_metrics_or_placeholder_for_api(query_name) @@ -1443,10 +1441,10 @@ class LBRYElectrumX(SessionBase): raise RPCError(BAD_REQUEST, f'unsupported client: {client_name}') if self.client_version != client_name[:17]: - SESSIONS_COUNT.labels(version=self.client_version).dec() + prometheus.METRICS.SESSIONS_COUNT.labels(version=self.client_version).dec() self.client_version = client_name[:17] - SESSIONS_COUNT.labels(version=self.client_version).inc() - CLIENT_VERSIONS.labels(version=self.client_version).inc() + prometheus.METRICS.SESSIONS_COUNT.labels(version=self.client_version).inc() + prometheus.METRICS.CLIENT_VERSIONS.labels(version=self.client_version).inc() # Find the highest common protocol version. Disconnect if # that protocol version in unsupported. diff --git a/tests/integration/blockchain/test_blockchain_reorganization.py b/tests/integration/blockchain/test_blockchain_reorganization.py index 216030839..5764c49c0 100644 --- a/tests/integration/blockchain/test_blockchain_reorganization.py +++ b/tests/integration/blockchain/test_blockchain_reorganization.py @@ -2,7 +2,7 @@ import logging import asyncio from binascii import hexlify from lbry.testcase import CommandTestCase -from lbry.wallet.server.prometheus import REORG_COUNT +from lbry.wallet.server import prometheus class BlockchainReorganizationTests(CommandTestCase): @@ -16,7 +16,7 @@ class BlockchainReorganizationTests(CommandTestCase): ) async def test_reorg(self): - REORG_COUNT.set(0) + prometheus.METRICS.REORG_COUNT.set(0) # invalidate current block, move forward 2 self.assertEqual(self.ledger.headers.height, 206) await self.assertBlockHash(206) @@ -26,7 +26,7 @@ class BlockchainReorganizationTests(CommandTestCase): self.assertEqual(self.ledger.headers.height, 207) await self.assertBlockHash(206) await self.assertBlockHash(207) - self.assertEqual(1, REORG_COUNT._samples()[0][2]) + self.assertEqual(1, prometheus.METRICS.REORG_COUNT._samples()[0][2]) # invalidate current block, move forward 3 await self.blockchain.invalidate_block((await self.ledger.headers.hash(206)).decode()) @@ -36,7 +36,7 @@ class BlockchainReorganizationTests(CommandTestCase): await self.assertBlockHash(206) await self.assertBlockHash(207) await self.assertBlockHash(208) - self.assertEqual(2, REORG_COUNT._samples()[0][2]) + self.assertEqual(2, prometheus.METRICS.REORG_COUNT._samples()[0][2]) async def test_reorg_change_claim_height(self): # sanity check From 797364ee5cebd1a8e1ef54a178bcc7c6837c4726 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Thu, 23 Apr 2020 21:17:44 -0400 Subject: [PATCH 19/86] refactor prometheus metrics --- lbry/extras/daemon/daemon.py | 24 +++- lbry/wallet/database.py | 19 ++- lbry/wallet/rpc/session.py | 27 +++- lbry/wallet/server/block_processor.py | 19 ++- lbry/wallet/server/daemon.py | 21 ++- lbry/wallet/server/prometheus.py | 125 ------------------ lbry/wallet/server/server.py | 3 - lbry/wallet/server/session.py | 63 +++++++-- .../test_blockchain_reorganization.py | 8 +- 9 files changed, 145 insertions(+), 164 deletions(-) delete mode 100644 lbry/wallet/server/prometheus.py diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index 6e2ef7aa9..94186abc0 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -19,7 +19,7 @@ from functools import wraps, partial import ecdsa import base58 from aiohttp import web -from prometheus_client import generate_latest as prom_generate_latest +from prometheus_client import generate_latest as prom_generate_latest, Gauge, Histogram, Counter from google.protobuf.message import DecodeError from lbry.wallet import ( Wallet, ENCRYPT_ON_DISK, SingleKey, HierarchicalDeterministic, @@ -297,6 +297,20 @@ class Daemon(metaclass=JSONRPCServerType): callable_methods: dict deprecated_methods: dict + pending_requests_metric = Gauge( + "pending_requests", "Number of running api requests", namespace="daemon_api", + labelnames=("method",) + ) + + requests_count_metric = Counter( + "requests_count", "Number of requests received", namespace="daemon_api", + labelnames=("method",) + ) + response_time_metric = Histogram( + "response_time", "Response times", namespace="daemon_api", + labelnames=("method",) + ) + def __init__(self, conf: Config, component_manager: typing.Optional[ComponentManager] = None): self.conf = conf self.platform_info = system_info.get_platform() @@ -457,7 +471,6 @@ class Daemon(metaclass=JSONRPCServerType): log.info("Starting LBRYNet Daemon") log.debug("Settings: %s", json.dumps(self.conf.settings_dict, indent=2)) log.info("Platform: %s", json.dumps(self.platform_info, indent=2)) - self.need_connection_status_refresh.set() self._connection_status_task = self.component_manager.loop.create_task( self.keep_connection_status_up_to_date() @@ -663,7 +676,9 @@ class Daemon(metaclass=JSONRPCServerType): JSONRPCError.CODE_INVALID_PARAMS, params_error_message, ) - + self.pending_requests_metric.labels(method=function_name).inc() + self.requests_count_metric.labels(method=function_name).inc() + start = time.perf_counter() try: result = method(self, *_args, **_kwargs) if asyncio.iscoroutine(result): @@ -677,6 +692,9 @@ class Daemon(metaclass=JSONRPCServerType): return JSONRPCError.create_command_exception( command=function_name, args=_args, kwargs=_kwargs, exception=e, traceback=format_exc() ) + finally: + self.pending_requests_metric.labels(method=function_name).dec() + self.response_time_metric.labels(method=function_name).observe(time.perf_counter() - start) def _verify_method_is_callable(self, function_path): if function_path not in self.callable_methods: diff --git a/lbry/wallet/database.py b/lbry/wallet/database.py index 446f53ee5..55f45ff63 100644 --- a/lbry/wallet/database.py +++ b/lbry/wallet/database.py @@ -3,6 +3,7 @@ import logging import asyncio import sqlite3 import platform +import time from binascii import hexlify from dataclasses import dataclass from contextvars import ContextVar @@ -10,6 +11,7 @@ from concurrent.futures.thread import ThreadPoolExecutor from concurrent.futures.process import ProcessPoolExecutor from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable, Dict, Optional from datetime import date +from prometheus_client import Gauge from .bip32 import PubKey from .transaction import Transaction, Output, OutputScript, TXRefImmutable @@ -64,6 +66,13 @@ else: class AIOSQLite: reader_executor: ReaderExecutorClass + waiting_writes_metric = Gauge( + "waiting_writes_count", "Number of waiting db writes", namespace="daemon_database" + ) + waiting_reads_metric = Gauge( + "waiting_reads_count", "Number of waiting db writes", namespace="daemon_database" + ) + def __init__(self): # has to be single threaded as there is no mapping of thread:connection self.writer_executor = ThreadPoolExecutor(max_workers=1) @@ -117,6 +126,7 @@ class AIOSQLite: still_waiting = False urgent_read = False if read_only: + self.waiting_reads_metric.inc() try: while self.writers: # more writes can come in while we are waiting for the first if not urgent_read and still_waiting and self.urgent_read_done.is_set(): @@ -133,6 +143,7 @@ class AIOSQLite: if urgent_read: # unthrottle the writers if they had to be throttled self.urgent_read_done.set() + self.waiting_reads_metric.dec() if fetch_all: return await self.run(lambda conn: conn.execute(sql, parameters).fetchall()) return await self.run(lambda conn: conn.execute(sql, parameters).fetchone()) @@ -150,7 +161,12 @@ class AIOSQLite: return self.run(lambda conn: conn.execute(sql, parameters)) async def run(self, fun, *args, **kwargs): - await self.urgent_read_done.wait() + self.waiting_writes_metric.inc() + try: + await self.urgent_read_done.wait() + except Exception as e: + self.waiting_writes_metric.dec() + raise e self.writers += 1 self.read_ready.clear() try: @@ -160,6 +176,7 @@ class AIOSQLite: ) finally: self.writers -= 1 + self.waiting_writes_metric.dec() if not self.writers: self.read_ready.set() diff --git a/lbry/wallet/rpc/session.py b/lbry/wallet/rpc/session.py index a454e3a30..dc353b50e 100644 --- a/lbry/wallet/rpc/session.py +++ b/lbry/wallet/rpc/session.py @@ -33,13 +33,12 @@ from asyncio import Event, CancelledError import logging import time from contextlib import suppress - +from prometheus_client import Counter, Histogram from lbry.wallet.tasks import TaskGroup from .jsonrpc import Request, JSONRPCConnection, JSONRPCv2, JSONRPC, Batch, Notification from .jsonrpc import RPCError, ProtocolError from .framing import BadMagicError, BadChecksumError, OversizedPayloadError, BitcoinFramer, NewlineFramer -from lbry.wallet.server import prometheus class Connector: @@ -372,10 +371,26 @@ class BatchRequest: raise BatchError(self) +NAMESPACE = "wallet_server" + + class RPCSession(SessionBase): """Base class for protocols where a message can lead to a response, for example JSON RPC.""" + RESPONSE_TIMES = Histogram("response_time", "Response times", namespace=NAMESPACE, + labelnames=("method", "version")) + NOTIFICATION_COUNT = Counter("notification", "Number of notifications sent (for subscriptions)", + namespace=NAMESPACE, labelnames=("method", "version")) + REQUEST_ERRORS_COUNT = Counter( + "request_error", "Number of requests that returned errors", namespace=NAMESPACE, + labelnames=("method", "version") + ) + RESET_CONNECTIONS = Counter( + "reset_clients", "Number of reset connections by client version", + namespace=NAMESPACE, labelnames=("version",) + ) + def __init__(self, *, framer=None, loop=None, connection=None): super().__init__(framer=framer, loop=loop) self.connection = connection or self.default_connection() @@ -388,7 +403,7 @@ class RPCSession(SessionBase): except MemoryError: self.logger.warning('received oversized message from %s:%s, dropping connection', self._address[0], self._address[1]) - prometheus.METRICS.RESET_CONNECTIONS.labels(version=self.client_version).inc() + self.RESET_CONNECTIONS.labels(version=self.client_version).inc() self._close() return @@ -422,7 +437,7 @@ class RPCSession(SessionBase): 'internal server error') if isinstance(request, Request): message = request.send_result(result) - prometheus.METRICS.RESPONSE_TIMES.labels( + self.RESPONSE_TIMES.labels( method=request.method, version=self.client_version ).observe(time.perf_counter() - start) @@ -430,7 +445,7 @@ class RPCSession(SessionBase): await self._send_message(message) if isinstance(result, Exception): self._bump_errors() - prometheus.METRICS.REQUEST_ERRORS_COUNT.labels( + self.REQUEST_ERRORS_COUNT.labels( method=request.method, version=self.client_version ).inc() @@ -467,7 +482,7 @@ class RPCSession(SessionBase): async def send_notification(self, method, args=()): """Send an RPC notification over the network.""" message = self.connection.send_notification(Notification(method, args)) - prometheus.METRICS.NOTIFICATION_COUNT.labels(method=method, version=self.client_version).inc() + self.NOTIFICATION_COUNT.labels(method=method, version=self.client_version).inc() await self._send_message(message) def send_batch(self, raise_errors=False): diff --git a/lbry/wallet/server/block_processor.py b/lbry/wallet/server/block_processor.py index 48a4519d0..cb6a32f55 100644 --- a/lbry/wallet/server/block_processor.py +++ b/lbry/wallet/server/block_processor.py @@ -3,6 +3,7 @@ import asyncio from struct import pack, unpack from concurrent.futures.thread import ThreadPoolExecutor from typing import Optional +from prometheus_client import Gauge, Histogram import lbry from lbry.schema.claim import Claim from lbry.wallet.server.db.writer import SQLDB @@ -10,7 +11,6 @@ from lbry.wallet.server.daemon import DaemonError from lbry.wallet.server.hash import hash_to_hex_str, HASHX_LEN from lbry.wallet.server.util import chunks, class_logger from lbry.wallet.server.leveldb import FlushData -from lbry.wallet.server import prometheus class Prefetcher: @@ -129,6 +129,9 @@ class ChainError(Exception): """Raised on error processing blocks.""" +NAMESPACE = "wallet_server" + + class BlockProcessor: """Process blocks and update the DB state to match. @@ -136,6 +139,14 @@ class BlockProcessor: Coordinate backing up in case of chain reorganisations. """ + block_count_metric = Gauge( + "block_count", "Number of processed blocks", namespace=NAMESPACE + ) + block_update_time_metric = Histogram("block_time", "Block update times", namespace=NAMESPACE) + reorg_count_metric = Gauge( + "reorg_count", "Number of reorgs", namespace=NAMESPACE + ) + def __init__(self, env, db, daemon, notifications): self.env = env self.db = db @@ -199,8 +210,8 @@ class BlockProcessor: cache.clear() await self._maybe_flush() processed_time = time.perf_counter() - start - prometheus.METRICS.BLOCK_COUNT.set(self.height) - prometheus.METRICS.BLOCK_UPDATE_TIMES.observe(processed_time) + self.block_count_metric.set(self.height) + self.block_update_time_metric.observe(processed_time) if not self.db.first_sync: s = '' if len(blocks) == 1 else 's' self.logger.info('processed {:,d} block{} in {:.1f}s'.format(len(blocks), s, processed_time)) @@ -255,7 +266,7 @@ class BlockProcessor: last -= len(raw_blocks) await self.run_in_thread_with_lock(self.db.sql.delete_claims_above_height, self.height) await self.prefetcher.reset_height(self.height) - prometheus.METRICS.REORG_COUNT.inc() + self.reorg_count_metric.inc() async def reorg_hashes(self, count): """Return a pair (start, last, hashes) of blocks to back up during a diff --git a/lbry/wallet/server/daemon.py b/lbry/wallet/server/daemon.py index 39487f367..44a366b6a 100644 --- a/lbry/wallet/server/daemon.py +++ b/lbry/wallet/server/daemon.py @@ -6,11 +6,11 @@ from functools import wraps from pylru import lrucache import aiohttp +from prometheus_client import Gauge, Histogram from lbry.wallet.rpc.jsonrpc import RPCError from lbry.wallet.server.util import hex_to_bytes, class_logger from lbry.wallet.rpc import JSONRPC -from lbry.wallet.server import prometheus class DaemonError(Exception): @@ -25,12 +25,23 @@ class WorkQueueFullError(Exception): """Internal - when the daemon's work queue is full.""" +NAMESPACE = "wallet_server" + + class Daemon: """Handles connections to a daemon at the given URL.""" WARMING_UP = -28 id_counter = itertools.count() + lbrycrd_request_time_metric = Histogram( + "lbrycrd_request", "lbrycrd requests count", namespace=NAMESPACE, labelnames=("method",) + ) + lbrycrd_pending_count_metric = Gauge( + "lbrycrd_pending_count", "Number of lbrycrd rpcs that are in flight", namespace=NAMESPACE, + labelnames=("method",) + ) + def __init__(self, coin, url, max_workqueue=10, init_retry=0.25, max_retry=4.0): self.coin = coin @@ -130,7 +141,7 @@ class Daemon: while True: try: for method in methods: - prometheus.METRICS.LBRYCRD_PENDING_COUNT.labels(method=method).inc() + self.lbrycrd_pending_count_metric.labels(method=method).inc() result = await self._send_data(data) result = processor(result) if on_good_message: @@ -155,7 +166,7 @@ class Daemon: on_good_message = 'running normally' finally: for method in methods: - prometheus.METRICS.LBRYCRD_PENDING_COUNT.labels(method=method).dec() + self.lbrycrd_pending_count_metric.labels(method=method).dec() await asyncio.sleep(retry) retry = max(min(self.max_retry, retry * 2), self.init_retry) @@ -176,7 +187,7 @@ class Daemon: if params: payload['params'] = params result = await self._send(payload, processor) - prometheus.METRICS.LBRYCRD_REQUEST_TIMES.labels(method=method).observe(time.perf_counter() - start) + self.lbrycrd_request_time_metric.labels(method=method).observe(time.perf_counter() - start) return result async def _send_vector(self, method, params_iterable, replace_errs=False): @@ -201,7 +212,7 @@ class Daemon: result = [] if payload: result = await self._send(payload, processor) - prometheus.METRICS.LBRYCRD_REQUEST_TIMES.labels(method=method).observe(time.perf_counter()-start) + self.lbrycrd_request_time_metric.labels(method=method).observe(time.perf_counter() - start) return result async def _is_rpc_available(self, method): diff --git a/lbry/wallet/server/prometheus.py b/lbry/wallet/server/prometheus.py deleted file mode 100644 index ee3e268cf..000000000 --- a/lbry/wallet/server/prometheus.py +++ /dev/null @@ -1,125 +0,0 @@ -import os -from prometheus_client import Counter, Info, Histogram, Gauge -from lbry import __version__ as version -from lbry.build_info import BUILD, COMMIT_HASH, DOCKER_TAG -from lbry.wallet.server import util -import lbry.wallet.server.version as wallet_server_version - - -class PrometheusMetrics: - VERSION_INFO: Info - SESSIONS_COUNT: Gauge - REQUESTS_COUNT: Counter - RESPONSE_TIMES: Histogram - NOTIFICATION_COUNT: Counter - REQUEST_ERRORS_COUNT: Counter - SQLITE_INTERRUPT_COUNT: Counter - SQLITE_OPERATIONAL_ERROR_COUNT: Counter - SQLITE_INTERNAL_ERROR_COUNT: Counter - SQLITE_EXECUTOR_TIMES: Histogram - SQLITE_PENDING_COUNT: Gauge - LBRYCRD_REQUEST_TIMES: Histogram - LBRYCRD_PENDING_COUNT: Gauge - CLIENT_VERSIONS: Counter - BLOCK_COUNT: Gauge - BLOCK_UPDATE_TIMES: Histogram - REORG_COUNT: Gauge - RESET_CONNECTIONS: Counter - - __slots__ = [ - 'VERSION_INFO', - 'SESSIONS_COUNT', - 'REQUESTS_COUNT', - 'RESPONSE_TIMES', - 'NOTIFICATION_COUNT', - 'REQUEST_ERRORS_COUNT', - 'SQLITE_INTERRUPT_COUNT', - 'SQLITE_OPERATIONAL_ERROR_COUNT', - 'SQLITE_INTERNAL_ERROR_COUNT', - 'SQLITE_EXECUTOR_TIMES', - 'SQLITE_PENDING_COUNT', - 'LBRYCRD_REQUEST_TIMES', - 'LBRYCRD_PENDING_COUNT', - 'CLIENT_VERSIONS', - 'BLOCK_COUNT', - 'BLOCK_UPDATE_TIMES', - 'REORG_COUNT', - 'RESET_CONNECTIONS', - '_installed', - 'namespace', - 'cpu_count' - ] - - def __init__(self): - self._installed = False - self.namespace = "wallet_server" - self.cpu_count = f"{os.cpu_count()}" - - def uninstall(self): - self._installed = False - for item in self.__slots__: - if not item.startswith('_') and item not in ('namespace', 'cpu_count'): - current = getattr(self, item, None) - if current: - setattr(self, item, None) - del current - - def install(self): - if self._installed: - return - self._installed = True - self.VERSION_INFO = Info('build', 'Wallet server build info (e.g. version, commit hash)', namespace=self.namespace) - self.VERSION_INFO.info({ - 'build': BUILD, - "commit": COMMIT_HASH, - "docker_tag": DOCKER_TAG, - 'version': version, - "min_version": util.version_string(wallet_server_version.PROTOCOL_MIN), - "cpu_count": self.cpu_count - }) - self.SESSIONS_COUNT = Gauge("session_count", "Number of connected client sessions", namespace=self.namespace, - labelnames=("version",)) - self.REQUESTS_COUNT = Counter("requests_count", "Number of requests received", namespace=self.namespace, - labelnames=("method", "version")) - self.RESPONSE_TIMES = Histogram("response_time", "Response times", namespace=self.namespace, - labelnames=("method", "version")) - self.NOTIFICATION_COUNT = Counter("notification", "Number of notifications sent (for subscriptions)", - namespace=self.namespace, labelnames=("method", "version")) - self.REQUEST_ERRORS_COUNT = Counter("request_error", "Number of requests that returned errors", namespace=self.namespace, - labelnames=("method", "version")) - self.SQLITE_INTERRUPT_COUNT = Counter("interrupt", "Number of interrupted queries", namespace=self.namespace) - self.SQLITE_OPERATIONAL_ERROR_COUNT = Counter( - "operational_error", "Number of queries that raised operational errors", namespace=self.namespace - ) - self.SQLITE_INTERNAL_ERROR_COUNT = Counter( - "internal_error", "Number of queries raising unexpected errors", namespace=self.namespace - ) - self.SQLITE_EXECUTOR_TIMES = Histogram("executor_time", "SQLite executor times", namespace=self.namespace) - self.SQLITE_PENDING_COUNT = Gauge( - "pending_queries_count", "Number of pending and running sqlite queries", namespace=self.namespace - ) - self.LBRYCRD_REQUEST_TIMES = Histogram( - "lbrycrd_request", "lbrycrd requests count", namespace=self.namespace, labelnames=("method",) - ) - self.LBRYCRD_PENDING_COUNT = Gauge( - "lbrycrd_pending_count", "Number of lbrycrd rpcs that are in flight", namespace=self.namespace, - labelnames=("method",) - ) - self.CLIENT_VERSIONS = Counter( - "clients", "Number of connections received per client version", - namespace=self.namespace, labelnames=("version",) - ) - self.BLOCK_COUNT = Gauge( - "block_count", "Number of processed blocks", namespace=self.namespace - ) - self.BLOCK_UPDATE_TIMES = Histogram("block_time", "Block update times", namespace=self.namespace) - self.REORG_COUNT = Gauge( - "reorg_count", "Number of reorgs", namespace=self.namespace - ) - self.RESET_CONNECTIONS = Counter( - "reset_clients", "Number of reset connections by client version", - namespace=self.namespace, labelnames=("version",) - ) - - -METRICS = PrometheusMetrics() diff --git a/lbry/wallet/server/server.py b/lbry/wallet/server/server.py index 98000ace9..cca84c852 100644 --- a/lbry/wallet/server/server.py +++ b/lbry/wallet/server/server.py @@ -7,7 +7,6 @@ import typing import lbry from lbry.wallet.server.mempool import MemPool, MemPoolAPI from lbry.prometheus import PrometheusServer -from lbry.wallet.server.prometheus import METRICS class Notifications: @@ -93,7 +92,6 @@ class Server: ) async def start(self): - METRICS.install() env = self.env min_str, max_str = env.coin.SESSIONCLS.protocol_min_max_strings() self.log.info(f'software version: {lbry.__version__}') @@ -123,7 +121,6 @@ class Server: self.prometheus_server = None self.shutdown_event.set() await self.daemon.close() - METRICS.uninstall() def run(self): loop = asyncio.get_event_loop() diff --git a/lbry/wallet/server/session.py b/lbry/wallet/server/session.py index 0dccce853..1cce96ff9 100644 --- a/lbry/wallet/server/session.py +++ b/lbry/wallet/server/session.py @@ -20,14 +20,15 @@ from functools import partial from binascii import hexlify from pylru import lrucache from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor +from prometheus_client import Counter, Info, Histogram, Gauge import lbry +from lbry.build_info import BUILD, COMMIT_HASH, DOCKER_TAG from lbry.wallet.server.block_processor import LBRYBlockProcessor from lbry.wallet.server.db.writer import LBRYLevelDB from lbry.wallet.server.db import reader from lbry.wallet.server.websocket import AdminWebSocket from lbry.wallet.server.metrics import ServerLoadData, APICallMetrics -from lbry.wallet.server import prometheus from lbry.wallet.rpc.framing import NewlineFramer import lbry.wallet.server.version as VERSION @@ -117,9 +118,45 @@ class SessionGroup: self.semaphore = asyncio.Semaphore(20) +NAMESPACE = "wallet_server" + + class SessionManager: """Holds global state about all sessions.""" + version_info_metric = Info( + 'build', 'Wallet server build info (e.g. version, commit hash)', namespace=NAMESPACE + ) + version_info_metric.info({ + 'build': BUILD, + "commit": COMMIT_HASH, + "docker_tag": DOCKER_TAG, + 'version': lbry.__version__, + "min_version": util.version_string(VERSION.PROTOCOL_MIN), + "cpu_count": os.cpu_count() + }) + session_count_metric = Gauge("session_count", "Number of connected client sessions", namespace=NAMESPACE, + labelnames=("version",)) + request_count_metric = Counter("requests_count", "Number of requests received", namespace=NAMESPACE, + labelnames=("method", "version")) + + interrupt_count_metric = Counter("interrupt", "Number of interrupted queries", namespace=NAMESPACE) + db_operational_error_metric = Counter( + "operational_error", "Number of queries that raised operational errors", namespace=NAMESPACE + ) + db_error_metric = Counter( + "internal_error", "Number of queries raising unexpected errors", namespace=NAMESPACE + ) + executor_time_metric = Histogram("executor_time", "SQLite executor times", namespace=NAMESPACE) + pending_query_metric = Gauge( + "pending_queries_count", "Number of pending and running sqlite queries", namespace=NAMESPACE + ) + + client_version_metric = Counter( + "clients", "Number of connections received per client version", + namespace=NAMESPACE, labelnames=("version",) + ) + def __init__(self, env: 'Env', db: LBRYLevelDB, bp: LBRYBlockProcessor, daemon: 'Daemon', mempool: 'MemPool', shutdown_event: asyncio.Event): env.max_send = max(350000, env.max_send) @@ -675,7 +712,7 @@ class SessionBase(RPCSession): context = {'conn_id': f'{self.session_id}'} self.logger = util.ConnectionLogger(self.logger, context) self.group = self.session_mgr.add_session(self) - prometheus.METRICS.SESSIONS_COUNT.labels(version=self.client_version).inc() + self.session_mgr.session_count_metric.labels(version=self.client_version).inc() peer_addr_str = self.peer_address_str() self.logger.info(f'{self.kind} {peer_addr_str}, ' f'{self.session_mgr.session_count():,d} total') @@ -684,7 +721,7 @@ class SessionBase(RPCSession): """Handle client disconnection.""" super().connection_lost(exc) self.session_mgr.remove_session(self) - prometheus.METRICS.SESSIONS_COUNT.labels(version=self.client_version).dec() + self.session_mgr.session_count_metric.labels(version=self.client_version).dec() msg = '' if not self._can_send.is_set(): msg += ' whilst paused' @@ -708,7 +745,7 @@ class SessionBase(RPCSession): """Handle an incoming request. ElectrumX doesn't receive notifications from client sessions. """ - prometheus.METRICS.REQUESTS_COUNT.labels(method=request.method, version=self.client_version).inc() + self.session_mgr.request_count_metric.labels(method=request.method, version=self.client_version).inc() if isinstance(request, Request): handler = self.request_handlers.get(request.method) handler = partial(handler, self) @@ -944,7 +981,7 @@ class LBRYElectrumX(SessionBase): async def run_in_executor(self, query_name, func, kwargs): start = time.perf_counter() try: - prometheus.METRICS.SQLITE_PENDING_COUNT.inc() + self.session_mgr.pending_query_metric.inc() result = await asyncio.get_running_loop().run_in_executor( self.session_mgr.query_executor, func, kwargs ) @@ -953,18 +990,18 @@ class LBRYElectrumX(SessionBase): except reader.SQLiteInterruptedError as error: metrics = self.get_metrics_or_placeholder_for_api(query_name) metrics.query_interrupt(start, error.metrics) - prometheus.METRICS.prometheus.METRICS.SQLITE_INTERRUPT_COUNT.inc() + self.session_mgr.self.session_mgr.SQLITE_INTERRUPT_COUNT.inc() raise RPCError(JSONRPC.QUERY_TIMEOUT, 'sqlite query timed out') except reader.SQLiteOperationalError as error: metrics = self.get_metrics_or_placeholder_for_api(query_name) metrics.query_error(start, error.metrics) - prometheus.METRICS.SQLITE_OPERATIONAL_ERROR_COUNT.inc() + self.session_mgr.db_operational_error_metric.inc() raise RPCError(JSONRPC.INTERNAL_ERROR, 'query failed to execute') except Exception: log.exception("dear devs, please handle this exception better") metrics = self.get_metrics_or_placeholder_for_api(query_name) metrics.query_error(start, {}) - prometheus.METRICS.SQLITE_INTERNAL_ERROR_COUNT.inc() + self.session_mgr.db_error_metric.inc() raise RPCError(JSONRPC.INTERNAL_ERROR, 'unknown server error') else: if self.env.track_metrics: @@ -973,8 +1010,8 @@ class LBRYElectrumX(SessionBase): metrics.query_response(start, metrics_data) return base64.b64encode(result).decode() finally: - prometheus.METRICS.SQLITE_PENDING_COUNT.dec() - prometheus.METRICS.SQLITE_EXECUTOR_TIMES.observe(time.perf_counter() - start) + self.session_mgr.pending_query_metric.dec() + self.session_mgr.executor_time_metric.observe(time.perf_counter() - start) async def run_and_cache_query(self, query_name, function, kwargs): metrics = self.get_metrics_or_placeholder_for_api(query_name) @@ -1441,10 +1478,10 @@ class LBRYElectrumX(SessionBase): raise RPCError(BAD_REQUEST, f'unsupported client: {client_name}') if self.client_version != client_name[:17]: - prometheus.METRICS.SESSIONS_COUNT.labels(version=self.client_version).dec() + self.session_mgr.session_count_metric.labels(version=self.client_version).dec() self.client_version = client_name[:17] - prometheus.METRICS.SESSIONS_COUNT.labels(version=self.client_version).inc() - prometheus.METRICS.CLIENT_VERSIONS.labels(version=self.client_version).inc() + self.session_mgr.session_count_metric.labels(version=self.client_version).inc() + self.session_mgr.client_version_metric.labels(version=self.client_version).inc() # Find the highest common protocol version. Disconnect if # that protocol version in unsupported. diff --git a/tests/integration/blockchain/test_blockchain_reorganization.py b/tests/integration/blockchain/test_blockchain_reorganization.py index 5764c49c0..af9349e67 100644 --- a/tests/integration/blockchain/test_blockchain_reorganization.py +++ b/tests/integration/blockchain/test_blockchain_reorganization.py @@ -2,7 +2,6 @@ import logging import asyncio from binascii import hexlify from lbry.testcase import CommandTestCase -from lbry.wallet.server import prometheus class BlockchainReorganizationTests(CommandTestCase): @@ -16,7 +15,8 @@ class BlockchainReorganizationTests(CommandTestCase): ) async def test_reorg(self): - prometheus.METRICS.REORG_COUNT.set(0) + bp = self.conductor.spv_node.server.bp + bp.reorg_count_metric.set(0) # invalidate current block, move forward 2 self.assertEqual(self.ledger.headers.height, 206) await self.assertBlockHash(206) @@ -26,7 +26,7 @@ class BlockchainReorganizationTests(CommandTestCase): self.assertEqual(self.ledger.headers.height, 207) await self.assertBlockHash(206) await self.assertBlockHash(207) - self.assertEqual(1, prometheus.METRICS.REORG_COUNT._samples()[0][2]) + self.assertEqual(1, bp.reorg_count_metric._samples()[0][2]) # invalidate current block, move forward 3 await self.blockchain.invalidate_block((await self.ledger.headers.hash(206)).decode()) @@ -36,7 +36,7 @@ class BlockchainReorganizationTests(CommandTestCase): await self.assertBlockHash(206) await self.assertBlockHash(207) await self.assertBlockHash(208) - self.assertEqual(2, prometheus.METRICS.REORG_COUNT._samples()[0][2]) + self.assertEqual(2, bp.reorg_count_metric._samples()[0][2]) async def test_reorg_change_claim_height(self): # sanity check From 3469abaefdcc1c7f9cd03b6a34f05727121e36e6 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Sat, 2 May 2020 21:23:17 -0400 Subject: [PATCH 20/86] write lock metrics --- lbry/utils.py | 24 ++++++++++++++++++++++ lbry/wallet/database.py | 44 ++++++++++++++++++++++++++++++++++------- 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/lbry/utils.py b/lbry/utils.py index c24d8a971..4aa1215e3 100644 --- a/lbry/utils.py +++ b/lbry/utils.py @@ -3,6 +3,7 @@ import codecs import datetime import random import socket +import time import string import sys import json @@ -19,6 +20,7 @@ import pkg_resources import certifi import aiohttp +from prometheus_client import Histogram from lbry.schema.claim import Claim log = logging.getLogger(__name__) @@ -282,3 +284,25 @@ async def get_external_ip() -> typing.Optional[str]: # used if upnp is disabled def is_running_from_bundle(): # see https://pyinstaller.readthedocs.io/en/stable/runtime-information.html return getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS') + + +class LockWithMetrics(asyncio.Lock): + def __init__(self, acquire_metric, held_time_metric, loop=None): + super().__init__(loop=loop) + self._acquire_metric = acquire_metric + self._lock_held_time_metric = held_time_metric + self._lock_acquired_time = None + + async def acquire(self): + start = time.perf_counter() + try: + return await super().acquire() + finally: + self._lock_acquired_time = time.perf_counter() + self._acquire_metric.observe(self._lock_acquired_time - start) + + def release(self): + try: + return super().release() + finally: + self._lock_held_time_metric.observe(time.perf_counter() - self._lock_acquired_time) diff --git a/lbry/wallet/database.py b/lbry/wallet/database.py index 55f45ff63..8a39ee22a 100644 --- a/lbry/wallet/database.py +++ b/lbry/wallet/database.py @@ -3,7 +3,6 @@ import logging import asyncio import sqlite3 import platform -import time from binascii import hexlify from dataclasses import dataclass from contextvars import ContextVar @@ -11,7 +10,8 @@ from concurrent.futures.thread import ThreadPoolExecutor from concurrent.futures.process import ProcessPoolExecutor from typing import Tuple, List, Union, Callable, Any, Awaitable, Iterable, Dict, Optional from datetime import date -from prometheus_client import Gauge +from prometheus_client import Gauge, Counter, Histogram +from lbry.utils import LockWithMetrics from .bip32 import PubKey from .transaction import Transaction, Output, OutputScript, TXRefImmutable @@ -72,6 +72,18 @@ class AIOSQLite: waiting_reads_metric = Gauge( "waiting_reads_count", "Number of waiting db writes", namespace="daemon_database" ) + write_count_metric = Counter( + "write_count", "Number of database writes", namespace="daemon_database" + ) + read_count_metric = Counter( + "read_count", "Number of database reads", namespace="daemon_database" + ) + acquire_write_lock_metric = Histogram( + f'write_lock_acquired', 'Time to acquire the write lock', namespace="daemon_database" + ) + held_write_lock_metric = Histogram( + f'write_lock_held', 'Length of time the write lock is held for', namespace="daemon_database" + ) def __init__(self): # has to be single threaded as there is no mapping of thread:connection @@ -79,7 +91,7 @@ class AIOSQLite: self.writer_connection: Optional[sqlite3.Connection] = None self._closing = False self.query_count = 0 - self.write_lock = asyncio.Lock() + self.write_lock = LockWithMetrics(self.acquire_write_lock_metric, self.held_write_lock_metric) self.writers = 0 self.read_ready = asyncio.Event() self.urgent_read_done = asyncio.Event() @@ -127,6 +139,7 @@ class AIOSQLite: urgent_read = False if read_only: self.waiting_reads_metric.inc() + self.read_count_metric.inc() try: while self.writers: # more writes can come in while we are waiting for the first if not urgent_read and still_waiting and self.urgent_read_done.is_set(): @@ -161,6 +174,7 @@ class AIOSQLite: return self.run(lambda conn: conn.execute(sql, parameters)) async def run(self, fun, *args, **kwargs): + self.write_count_metric.inc() self.waiting_writes_metric.inc() try: await self.urgent_read_done.wait() @@ -193,10 +207,26 @@ class AIOSQLite: log.warning("rolled back") raise - def run_with_foreign_keys_disabled(self, fun, *args, **kwargs) -> Awaitable: - return asyncio.get_event_loop().run_in_executor( - self.writer_executor, self.__run_transaction_with_foreign_keys_disabled, fun, args, kwargs - ) + async def run_with_foreign_keys_disabled(self, fun, *args, **kwargs): + self.write_count_metric.inc() + self.waiting_writes_metric.inc() + try: + await self.urgent_read_done.wait() + except Exception as e: + self.waiting_writes_metric.dec() + raise e + self.writers += 1 + self.read_ready.clear() + try: + async with self.write_lock: + return await asyncio.get_event_loop().run_in_executor( + self.writer_executor, self.__run_transaction_with_foreign_keys_disabled, fun, args, kwargs + ) + finally: + self.writers -= 1 + self.waiting_writes_metric.dec() + if not self.writers: + self.read_ready.set() def __run_transaction_with_foreign_keys_disabled(self, fun: Callable[[sqlite3.Connection, Any, Any], Any], From 87f751188e12413e0d7c34ec29008df475cd2bff Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Sat, 2 May 2020 21:58:41 -0400 Subject: [PATCH 21/86] cancelled and failed api request metrics --- lbry/extras/daemon/daemon.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index 94186abc0..e27cd32cd 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -306,6 +306,14 @@ class Daemon(metaclass=JSONRPCServerType): "requests_count", "Number of requests received", namespace="daemon_api", labelnames=("method",) ) + failed_request_metric = Counter( + "failed_request_count", "Number of failed requests", namespace="daemon_api", + labelnames=("method",) + ) + cancelled_request_metric = Counter( + "cancelled_request_count", "Number of cancelled requests", namespace="daemon_api", + labelnames=("method",) + ) response_time_metric = Histogram( "response_time", "Response times", namespace="daemon_api", labelnames=("method",) @@ -685,9 +693,11 @@ class Daemon(metaclass=JSONRPCServerType): result = await result return result except asyncio.CancelledError: + self.cancelled_request_metric.labels(method=function_name).inc() log.info("cancelled API call for: %s", function_name) raise except Exception as e: # pylint: disable=broad-except + self.failed_request_metric.labels(method=function_name).inc() log.exception("error handling api request") return JSONRPCError.create_command_exception( command=function_name, args=_args, kwargs=_kwargs, exception=e, traceback=format_exc() From d3ffae72fb529cc7d581797024ecf773d7f79fa5 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Sat, 2 May 2020 22:30:25 -0400 Subject: [PATCH 22/86] buckets --- lbry/extras/daemon/daemon.py | 7 ++++++- lbry/wallet/database.py | 10 +++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index e27cd32cd..5c84f8227 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -290,6 +290,11 @@ class JSONRPCServerType(type): return klass +HISTOGRAM_BUCKETS = ( + .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf') +) + + class Daemon(metaclass=JSONRPCServerType): """ LBRYnet daemon, a jsonrpc interface to lbry functions @@ -315,7 +320,7 @@ class Daemon(metaclass=JSONRPCServerType): labelnames=("method",) ) response_time_metric = Histogram( - "response_time", "Response times", namespace="daemon_api", + "response_time", "Response times", namespace="daemon_api", buckets=HISTOGRAM_BUCKETS, labelnames=("method",) ) diff --git a/lbry/wallet/database.py b/lbry/wallet/database.py index 8a39ee22a..4f92ad445 100644 --- a/lbry/wallet/database.py +++ b/lbry/wallet/database.py @@ -22,6 +22,10 @@ from .util import date_to_julian_day log = logging.getLogger(__name__) sqlite3.enable_callback_tracebacks(True) +HISTOGRAM_BUCKETS = ( + .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf') +) + @dataclass class ReaderProcessState: @@ -79,10 +83,10 @@ class AIOSQLite: "read_count", "Number of database reads", namespace="daemon_database" ) acquire_write_lock_metric = Histogram( - f'write_lock_acquired', 'Time to acquire the write lock', namespace="daemon_database" + f'write_lock_acquired', 'Time to acquire the write lock', namespace="daemon_database", buckets=HISTOGRAM_BUCKETS ) held_write_lock_metric = Histogram( - f'write_lock_held', 'Length of time the write lock is held for', namespace="daemon_database" + f'write_lock_held', 'Length of time the write lock is held for', namespace="daemon_database", buckets=HISTOGRAM_BUCKETS ) def __init__(self): @@ -642,7 +646,7 @@ class Database(SQLiteMixin): return self.db.run(__many) async def reserve_outputs(self, txos, is_reserved=True): - txoids = ((is_reserved, txo.id) for txo in txos) + txoids = [(is_reserved, txo.id) for txo in txos] await self.db.executemany("UPDATE txo SET is_reserved = ? WHERE txoid = ?", txoids) async def release_outputs(self, txos): From e3abab6d4d28e924dbdddeb03c1bdc643391062a Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Mon, 4 May 2020 12:09:09 -0400 Subject: [PATCH 23/86] pylint --- lbry/utils.py | 1 - lbry/wallet/database.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lbry/utils.py b/lbry/utils.py index 4aa1215e3..0db443cd9 100644 --- a/lbry/utils.py +++ b/lbry/utils.py @@ -20,7 +20,6 @@ import pkg_resources import certifi import aiohttp -from prometheus_client import Histogram from lbry.schema.claim import Claim log = logging.getLogger(__name__) diff --git a/lbry/wallet/database.py b/lbry/wallet/database.py index 4f92ad445..31d078cec 100644 --- a/lbry/wallet/database.py +++ b/lbry/wallet/database.py @@ -86,7 +86,8 @@ class AIOSQLite: f'write_lock_acquired', 'Time to acquire the write lock', namespace="daemon_database", buckets=HISTOGRAM_BUCKETS ) held_write_lock_metric = Histogram( - f'write_lock_held', 'Length of time the write lock is held for', namespace="daemon_database", buckets=HISTOGRAM_BUCKETS + f'write_lock_held', 'Length of time the write lock is held for', namespace="daemon_database", + buckets=HISTOGRAM_BUCKETS ) def __init__(self): From 750ff448ad3d2b7b8528d8d8a3a3d343e95a0d96 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Mon, 4 May 2020 13:47:37 -0400 Subject: [PATCH 24/86] comments --- lbry/wallet/database.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lbry/wallet/database.py b/lbry/wallet/database.py index 31d078cec..9c8ad7695 100644 --- a/lbry/wallet/database.py +++ b/lbry/wallet/database.py @@ -181,12 +181,18 @@ class AIOSQLite: async def run(self, fun, *args, **kwargs): self.write_count_metric.inc() self.waiting_writes_metric.inc() + # it's possible many writes are coming in one after the other, these can + # block reader calls for a long time + # if the reader waits for the writers to finish and then has to wait for + # yet more, it will clear the urgent_read_done event to block more writers + # piling on try: await self.urgent_read_done.wait() except Exception as e: self.waiting_writes_metric.dec() raise e self.writers += 1 + # block readers self.read_ready.clear() try: async with self.write_lock: @@ -197,6 +203,7 @@ class AIOSQLite: self.writers -= 1 self.waiting_writes_metric.dec() if not self.writers: + # unblock the readers once the last enqueued writer finishes self.read_ready.set() def __run_transaction(self, fun: Callable[[sqlite3.Connection, Any, Any], Any], *args, **kwargs): From 8be1c8310d5ed2069ed4f97620e9297fd59eb449 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Mon, 4 May 2020 13:53:13 -0400 Subject: [PATCH 25/86] v0.72.0 --- lbry/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbry/__init__.py b/lbry/__init__.py index 75bafa71b..bc64b8354 100644 --- a/lbry/__init__.py +++ b/lbry/__init__.py @@ -1,2 +1,2 @@ -__version__ = "0.71.0" +__version__ = "0.72.0" version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name From 179383540f73617587271e47c4c30b1681b0c10c Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 15 Jan 2020 10:18:38 -0500 Subject: [PATCH 26/86] ManagedDownloadSource and SourceManager refactor --- lbry/file/__init__.py | 0 lbry/file/source.py | 175 +++++++++++++++++++ lbry/file/source_manager.py | 125 ++++++++++++++ lbry/stream/managed_stream.py | 195 +++++----------------- lbry/stream/stream_manager.py | 304 +++++----------------------------- 5 files changed, 387 insertions(+), 412 deletions(-) create mode 100644 lbry/file/__init__.py create mode 100644 lbry/file/source.py create mode 100644 lbry/file/source_manager.py diff --git a/lbry/file/__init__.py b/lbry/file/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lbry/file/source.py b/lbry/file/source.py new file mode 100644 index 000000000..05d8fb35d --- /dev/null +++ b/lbry/file/source.py @@ -0,0 +1,175 @@ +import os +import asyncio +import typing +import logging +import binascii +from typing import Optional +from lbry.utils import generate_id +from lbry.extras.daemon.storage import StoredContentClaim + +if typing.TYPE_CHECKING: + from lbry.conf import Config + from lbry.extras.daemon.analytics import AnalyticsManager + from lbry.wallet.transaction import Transaction + from lbry.extras.daemon.storage import SQLiteStorage + +log = logging.getLogger(__name__) + + +# def _get_next_available_file_name(download_directory: str, file_name: str) -> str: +# base_name, ext = os.path.splitext(os.path.basename(file_name)) +# i = 0 +# while os.path.isfile(os.path.join(download_directory, file_name)): +# i += 1 +# file_name = "%s_%i%s" % (base_name, i, ext) +# +# return file_name +# +# +# async def get_next_available_file_name(loop: asyncio.AbstractEventLoop, download_directory: str, file_name: str) -> str: +# return await loop.run_in_executor(None, _get_next_available_file_name, download_directory, file_name) + + +class ManagedDownloadSource: + STATUS_RUNNING = "running" + STATUS_STOPPED = "stopped" + STATUS_FINISHED = "finished" + + SAVING_ID = 1 + STREAMING_ID = 2 + + def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage', identifier: str, + file_name: Optional[str] = None, download_directory: Optional[str] = None, + status: Optional[str] = STATUS_STOPPED, claim: Optional[StoredContentClaim] = None, + download_id: Optional[str] = None, rowid: Optional[int] = None, + content_fee: Optional['Transaction'] = None, + analytics_manager: Optional['AnalyticsManager'] = None, + added_on: Optional[int] = None): + self.loop = loop + self.storage = storage + self.config = config + self.identifier = identifier + self.download_directory = download_directory + self._file_name = file_name + self._status = status + self.stream_claim_info = claim + self.download_id = download_id or binascii.hexlify(generate_id()).decode() + self.rowid = rowid + self.content_fee = content_fee + self.purchase_receipt = None + self._added_on = added_on + self.analytics_manager = analytics_manager + + self.saving = asyncio.Event(loop=self.loop) + self.finished_writing = asyncio.Event(loop=self.loop) + self.started_writing = asyncio.Event(loop=self.loop) + self.finished_write_attempt = asyncio.Event(loop=self.loop) + + # @classmethod + # async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', file_path: str, + # key: Optional[bytes] = None, + # iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedDownloadSource': + # raise NotImplementedError() + + async def start(self, timeout: Optional[float] = None): + raise NotImplementedError() + + async def stop(self, finished: bool = False): + raise NotImplementedError() + + async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None): + raise NotImplementedError() + + def stop_tasks(self): + raise NotImplementedError() + + # def set_claim(self, claim_info: typing.Dict, claim: 'Claim'): + # self.stream_claim_info = StoredContentClaim( + # f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'], + # claim_info['name'], claim_info['amount'], claim_info['height'], + # binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'], + # claim_info['claim_sequence'], claim_info.get('channel_name') + # ) + # + # async def update_content_claim(self, claim_info: Optional[typing.Dict] = None): + # if not claim_info: + # claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash) + # self.set_claim(claim_info, claim_info['value']) + + @property + def file_name(self) -> Optional[str]: + return self._file_name + + @property + def added_on(self) -> Optional[int]: + return self._added_on + + @property + def status(self) -> str: + return self._status + + @property + def completed(self): + raise NotImplementedError() + + # @property + # def stream_url(self): + # return f"http://{self.config.streaming_host}:{self.config.streaming_port}/stream/{self.sd_hash} + + @property + def finished(self) -> bool: + return self.status == self.STATUS_FINISHED + + @property + def running(self) -> bool: + return self.status == self.STATUS_RUNNING + + @property + def claim_id(self) -> Optional[str]: + return None if not self.stream_claim_info else self.stream_claim_info.claim_id + + @property + def txid(self) -> Optional[str]: + return None if not self.stream_claim_info else self.stream_claim_info.txid + + @property + def nout(self) -> Optional[int]: + return None if not self.stream_claim_info else self.stream_claim_info.nout + + @property + def outpoint(self) -> Optional[str]: + return None if not self.stream_claim_info else self.stream_claim_info.outpoint + + @property + def claim_height(self) -> Optional[int]: + return None if not self.stream_claim_info else self.stream_claim_info.height + + @property + def channel_claim_id(self) -> Optional[str]: + return None if not self.stream_claim_info else self.stream_claim_info.channel_claim_id + + @property + def channel_name(self) -> Optional[str]: + return None if not self.stream_claim_info else self.stream_claim_info.channel_name + + @property + def claim_name(self) -> Optional[str]: + return None if not self.stream_claim_info else self.stream_claim_info.claim_name + + @property + def metadata(self) -> Optional[typing.Dict]: + return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict() + + @property + def metadata_protobuf(self) -> bytes: + if self.stream_claim_info: + return binascii.hexlify(self.stream_claim_info.claim.to_bytes()) + + @property + def full_path(self) -> Optional[str]: + return os.path.join(self.download_directory, os.path.basename(self.file_name)) \ + if self.file_name and self.download_directory else None + + @property + def output_file_exists(self): + return os.path.isfile(self.full_path) if self.full_path else False diff --git a/lbry/file/source_manager.py b/lbry/file/source_manager.py new file mode 100644 index 000000000..56ba5fd5f --- /dev/null +++ b/lbry/file/source_manager.py @@ -0,0 +1,125 @@ +import os +import asyncio +import binascii +import logging +import typing +from typing import Optional +from lbry.file.source import ManagedDownloadSource +if typing.TYPE_CHECKING: + from lbry.conf import Config + from lbry.extras.daemon.analytics import AnalyticsManager + from lbry.extras.daemon.storage import SQLiteStorage + +log = logging.getLogger(__name__) + +comparison_operators = { + 'eq': lambda a, b: a == b, + 'ne': lambda a, b: a != b, + 'g': lambda a, b: a > b, + 'l': lambda a, b: a < b, + 'ge': lambda a, b: a >= b, + 'le': lambda a, b: a <= b, +} + + +def path_or_none(p) -> Optional[str]: + if not p: + return + return binascii.unhexlify(p).decode() + + +class SourceManager: + filter_fields = { + 'rowid', + 'status', + 'file_name', + 'added_on', + 'claim_name', + 'claim_height', + 'claim_id', + 'outpoint', + 'txid', + 'nout', + 'channel_claim_id', + 'channel_name' + } + + source_class = ManagedDownloadSource + + def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage', + analytics_manager: Optional['AnalyticsManager'] = None): + self.loop = loop + self.config = config + self.storage = storage + self.analytics_manager = analytics_manager + self._sources: typing.Dict[str, ManagedDownloadSource] = {} + self.started = asyncio.Event(loop=self.loop) + + def add(self, source: ManagedDownloadSource): + self._sources[source.identifier] = source + + def remove(self, source: ManagedDownloadSource): + if source.identifier not in self._sources: + return + self._sources.pop(source.identifier) + source.stop_tasks() + + async def initialize_from_database(self): + raise NotImplementedError() + + async def start(self): + await self.initialize_from_database() + self.started.set() + + def stop(self): + while self._sources: + _, source = self._sources.popitem() + source.stop_tasks() + self.started.clear() + + async def create(self, file_path: str, key: Optional[bytes] = None, **kw) -> ManagedDownloadSource: + raise NotImplementedError() + + async def _delete(self, source: ManagedDownloadSource): + raise NotImplementedError() + + async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False): + await self._delete(source) + self.remove(source) + if delete_file and source.output_file_exists: + os.remove(source.full_path) + + def get_filtered(self, sort_by: Optional[str] = None, reverse: Optional[bool] = False, + comparison: Optional[str] = None, **search_by) -> typing.List[ManagedDownloadSource]: + """ + Get a list of filtered and sorted ManagedStream objects + + :param sort_by: field to sort by + :param reverse: reverse sorting + :param comparison: comparison operator used for filtering + :param search_by: fields and values to filter by + """ + if sort_by and sort_by not in self.filter_fields: + raise ValueError(f"'{sort_by}' is not a valid field to sort by") + if comparison and comparison not in comparison_operators: + raise ValueError(f"'{comparison}' is not a valid comparison") + if 'full_status' in search_by: + del search_by['full_status'] + for search in search_by.keys(): + if search not in self.filter_fields: + raise ValueError(f"'{search}' is not a valid search operation") + if search_by: + comparison = comparison or 'eq' + sources = [] + for stream in self._sources.values(): + for search, val in search_by.items(): + if comparison_operators[comparison](getattr(stream, search), val): + sources.append(stream) + break + else: + sources = list(self._sources.values()) + if sort_by: + sources.sort(key=lambda s: getattr(s, sort_by)) + if reverse: + sources.reverse() + return sources diff --git a/lbry/stream/managed_stream.py b/lbry/stream/managed_stream.py index c449fe232..c530550c6 100644 --- a/lbry/stream/managed_stream.py +++ b/lbry/stream/managed_stream.py @@ -4,6 +4,7 @@ import time import typing import logging import binascii +from typing import Optional from aiohttp.web import Request, StreamResponse, HTTPRequestRangeNotSatisfiable from lbry.utils import generate_id from lbry.error import DownloadSDTimeoutError @@ -13,12 +14,14 @@ from lbry.stream.descriptor import StreamDescriptor, sanitize_file_name from lbry.stream.reflector.client import StreamReflectorClient from lbry.extras.daemon.storage import StoredContentClaim from lbry.blob import MAX_BLOB_SIZE +from lbry.file.source import ManagedDownloadSource if typing.TYPE_CHECKING: from lbry.conf import Config from lbry.schema.claim import Claim from lbry.blob.blob_manager import BlobManager from lbry.blob.blob_info import BlobInfo + from lbry.extras.daemon.storage import SQLiteStorage from lbry.dht.node import Node from lbry.extras.daemon.analytics import AnalyticsManager from lbry.wallet.transaction import Transaction @@ -40,65 +43,20 @@ async def get_next_available_file_name(loop: asyncio.AbstractEventLoop, download return await loop.run_in_executor(None, _get_next_available_file_name, download_directory, file_name) -class ManagedStream: - STATUS_RUNNING = "running" - STATUS_STOPPED = "stopped" - STATUS_FINISHED = "finished" - - SAVING_ID = 1 - STREAMING_ID = 2 - - __slots__ = [ - 'loop', - 'config', - 'blob_manager', - 'sd_hash', - 'download_directory', - '_file_name', - '_added_on', - '_status', - 'stream_claim_info', - 'download_id', - 'rowid', - 'content_fee', - 'purchase_receipt', - 'downloader', - 'analytics_manager', - 'fully_reflected', - 'reflector_progress', - 'file_output_task', - 'delayed_stop_task', - 'streaming_responses', - 'streaming', - '_running', - 'saving', - 'finished_writing', - 'started_writing', - 'finished_write_attempt', - 'uploading_to_reflector' - ] - +class ManagedStream(ManagedDownloadSource): def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager', - sd_hash: str, download_directory: typing.Optional[str] = None, file_name: typing.Optional[str] = None, - status: typing.Optional[str] = STATUS_STOPPED, claim: typing.Optional[StoredContentClaim] = None, - download_id: typing.Optional[str] = None, rowid: typing.Optional[int] = None, - descriptor: typing.Optional[StreamDescriptor] = None, - content_fee: typing.Optional['Transaction'] = None, - analytics_manager: typing.Optional['AnalyticsManager'] = None, - added_on: typing.Optional[int] = None): - self.loop = loop - self.config = config + sd_hash: str, download_directory: Optional[str] = None, file_name: Optional[str] = None, + status: Optional[str] = ManagedDownloadSource.STATUS_STOPPED, + claim: Optional[StoredContentClaim] = None, + download_id: Optional[str] = None, rowid: Optional[int] = None, + descriptor: Optional[StreamDescriptor] = None, + content_fee: Optional['Transaction'] = None, + analytics_manager: Optional['AnalyticsManager'] = None, + added_on: Optional[int] = None): + super().__init__(loop, config, blob_manager.storage, sd_hash, file_name, download_directory, status, claim, + download_id, rowid, content_fee, analytics_manager, added_on) self.blob_manager = blob_manager - self.sd_hash = sd_hash - self.download_directory = download_directory - self._file_name = file_name - self._status = status - self.stream_claim_info = claim - self.download_id = download_id or binascii.hexlify(generate_id()).decode() - self.rowid = rowid - self.content_fee = content_fee self.purchase_receipt = None - self._added_on = added_on self.downloader = StreamDownloader(self.loop, self.config, self.blob_manager, sd_hash, descriptor) self.analytics_manager = analytics_manager @@ -108,12 +66,13 @@ class ManagedStream: self.file_output_task: typing.Optional[asyncio.Task] = None self.delayed_stop_task: typing.Optional[asyncio.Task] = None self.streaming_responses: typing.List[typing.Tuple[Request, StreamResponse]] = [] + self.fully_reflected = asyncio.Event(loop=self.loop) self.streaming = asyncio.Event(loop=self.loop) self._running = asyncio.Event(loop=self.loop) - self.saving = asyncio.Event(loop=self.loop) - self.finished_writing = asyncio.Event(loop=self.loop) - self.started_writing = asyncio.Event(loop=self.loop) - self.finished_write_attempt = asyncio.Event(loop=self.loop) + + @property + def sd_hash(self) -> str: + return self.identifier @property def is_fully_reflected(self) -> bool: @@ -128,17 +87,9 @@ class ManagedStream: return self.descriptor.stream_hash @property - def file_name(self) -> typing.Optional[str]: + def file_name(self) -> Optional[str]: return self._file_name or (self.descriptor.suggested_file_name if self.descriptor else None) - @property - def added_on(self) -> typing.Optional[int]: - return self._added_on - - @property - def status(self) -> str: - return self._status - @property def written_bytes(self) -> int: return 0 if not self.output_file_exists else os.stat(self.full_path).st_size @@ -156,55 +107,6 @@ class ManagedStream: self._status = status await self.blob_manager.storage.change_file_status(self.stream_hash, status) - @property - def finished(self) -> bool: - return self.status == self.STATUS_FINISHED - - @property - def running(self) -> bool: - return self.status == self.STATUS_RUNNING - - @property - def claim_id(self) -> typing.Optional[str]: - return None if not self.stream_claim_info else self.stream_claim_info.claim_id - - @property - def txid(self) -> typing.Optional[str]: - return None if not self.stream_claim_info else self.stream_claim_info.txid - - @property - def nout(self) -> typing.Optional[int]: - return None if not self.stream_claim_info else self.stream_claim_info.nout - - @property - def outpoint(self) -> typing.Optional[str]: - return None if not self.stream_claim_info else self.stream_claim_info.outpoint - - @property - def claim_height(self) -> typing.Optional[int]: - return None if not self.stream_claim_info else self.stream_claim_info.height - - @property - def channel_claim_id(self) -> typing.Optional[str]: - return None if not self.stream_claim_info else self.stream_claim_info.channel_claim_id - - @property - def channel_name(self) -> typing.Optional[str]: - return None if not self.stream_claim_info else self.stream_claim_info.channel_name - - @property - def claim_name(self) -> typing.Optional[str]: - return None if not self.stream_claim_info else self.stream_claim_info.claim_name - - @property - def metadata(self) -> typing.Optional[typing.Dict]: - return None if not self.stream_claim_info else self.stream_claim_info.claim.stream.to_dict() - - @property - def metadata_protobuf(self) -> bytes: - if self.stream_claim_info: - return binascii.hexlify(self.stream_claim_info.claim.to_bytes()) - @property def blobs_completed(self) -> int: return sum([1 if b.blob_hash in self.blob_manager.completed_blob_hashes else 0 @@ -218,39 +120,30 @@ class ManagedStream: def blobs_remaining(self) -> int: return self.blobs_in_stream - self.blobs_completed - @property - def full_path(self) -> typing.Optional[str]: - return os.path.join(self.download_directory, os.path.basename(self.file_name)) \ - if self.file_name and self.download_directory else None - - @property - def output_file_exists(self): - return os.path.isfile(self.full_path) if self.full_path else False - @property def mime_type(self): return guess_media_type(os.path.basename(self.descriptor.suggested_file_name))[0] - @classmethod - async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager', - file_path: str, key: typing.Optional[bytes] = None, - iv_generator: typing.Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedStream': - """ - Generate a stream from a file and save it to the db - """ - descriptor = await StreamDescriptor.create_stream( - loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator, - blob_completed_callback=blob_manager.blob_completed - ) - await blob_manager.storage.store_stream( - blob_manager.get_blob(descriptor.sd_hash), descriptor - ) - row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path), - os.path.dirname(file_path), 0) - return cls(loop, config, blob_manager, descriptor.sd_hash, os.path.dirname(file_path), - os.path.basename(file_path), status=cls.STATUS_FINISHED, rowid=row_id, descriptor=descriptor) + # @classmethod + # async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', + # file_path: str, key: Optional[bytes] = None, + # iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedDownloadSource': + # """ + # Generate a stream from a file and save it to the db + # """ + # descriptor = await StreamDescriptor.create_stream( + # loop, blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator, + # blob_completed_callback=blob_manager.blob_completed + # ) + # await blob_manager.storage.store_stream( + # blob_manager.get_blob(descriptor.sd_hash), descriptor + # ) + # row_id = await blob_manager.storage.save_published_file(descriptor.stream_hash, os.path.basename(file_path), + # os.path.dirname(file_path), 0) + # return cls(loop, config, blob_manager, descriptor.sd_hash, os.path.dirname(file_path), + # os.path.basename(file_path), status=cls.STATUS_FINISHED, rowid=row_id, descriptor=descriptor) - async def start(self, node: typing.Optional['Node'] = None, timeout: typing.Optional[float] = None, + async def start(self, node: Optional['Node'] = None, timeout: Optional[float] = None, save_now: bool = False): timeout = timeout or self.config.download_timeout if self._running.is_set(): @@ -287,7 +180,7 @@ class ManagedStream: if (finished and self.status != self.STATUS_FINISHED) or self.status == self.STATUS_RUNNING: await self.update_status(self.STATUS_FINISHED if finished else self.STATUS_STOPPED) - async def _aiter_read_stream(self, start_blob_num: typing.Optional[int] = 0, connection_id: int = 0)\ + async def _aiter_read_stream(self, start_blob_num: Optional[int] = 0, connection_id: int = 0)\ -> typing.AsyncIterator[typing.Tuple['BlobInfo', bytes]]: if start_blob_num >= len(self.descriptor.blobs[:-1]): raise IndexError(start_blob_num) @@ -299,7 +192,7 @@ class ManagedStream: decrypted = await self.downloader.read_blob(blob_info, connection_id) yield (blob_info, decrypted) - async def stream_file(self, request: Request, node: typing.Optional['Node'] = None) -> StreamResponse: + async def stream_file(self, request: Request, node: Optional['Node'] = None) -> StreamResponse: log.info("stream file to browser for lbry://%s#%s (sd hash %s...)", self.claim_name, self.claim_id, self.sd_hash[:6]) headers, size, skip_blobs, first_blob_start_offset = self._prepare_range_response_headers( @@ -391,8 +284,8 @@ class ManagedStream: self.saving.clear() self.finished_write_attempt.set() - async def save_file(self, file_name: typing.Optional[str] = None, download_directory: typing.Optional[str] = None, - node: typing.Optional['Node'] = None): + async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None, + node: Optional['Node'] = None): await self.start(node) if self.file_output_task and not self.file_output_task.done(): # cancel an already running save task self.file_output_task.cancel() @@ -476,7 +369,7 @@ class ManagedStream: claim_info['claim_sequence'], claim_info.get('channel_name') ) - async def update_content_claim(self, claim_info: typing.Optional[typing.Dict] = None): + async def update_content_claim(self, claim_info: Optional[typing.Dict] = None): if not claim_info: claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash) self.set_claim(claim_info, claim_info['value']) diff --git a/lbry/stream/stream_manager.py b/lbry/stream/stream_manager.py index 4fb37e99a..58035e174 100644 --- a/lbry/stream/stream_manager.py +++ b/lbry/stream/stream_manager.py @@ -15,11 +15,14 @@ from lbry.schema.claim import Claim from lbry.schema.url import URL from lbry.wallet.dewies import dewies_to_lbc from lbry.wallet import Output - +from lbry.source_manager import SourceManager +from lbry.file.source import ManagedDownloadSource if typing.TYPE_CHECKING: from lbry.conf import Config from lbry.blob.blob_manager import BlobManager from lbry.dht.node import Node + from lbry.wallet.wallet import WalletManager + from lbry.wallet.transaction import Transaction from lbry.extras.daemon.analytics import AnalyticsManager from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager @@ -29,32 +32,12 @@ if typing.TYPE_CHECKING: log = logging.getLogger(__name__) -FILTER_FIELDS = [ - 'rowid', - 'status', - 'file_name', - 'added_on', - 'sd_hash', - 'stream_hash', - 'claim_name', - 'claim_height', - 'claim_id', - 'outpoint', - 'txid', - 'nout', - 'channel_claim_id', - 'channel_name', - 'full_status', # TODO: remove - 'blobs_remaining', - 'blobs_in_stream' -] SET_FILTER_FIELDS = { "claim_ids": "claim_id", "channel_claim_ids": "channel_claim_id", "outpoints": "outpoint" } - COMPARISON_OPERATORS = { 'eq': lambda a, b: a == b, 'ne': lambda a, b: a != b, @@ -64,35 +47,44 @@ COMPARISON_OPERATORS = { 'le': lambda a, b: a <= b, 'in': lambda a, b: a in b } - - -def path_or_none(path) -> Optional[str]: - if not path: +def path_or_none(p) -> Optional[str]: + if not p: return - return binascii.unhexlify(path).decode() + return binascii.unhexlify(p).decode() -class StreamManager: +class StreamManager(SourceManager): + _sources: typing.Dict[str, ManagedStream] + + filter_fields = set(SourceManager.filter_fields) + filter_fields.update({ + 'sd_hash', + 'stream_hash', + 'full_status', # TODO: remove + 'blobs_remaining', + 'blobs_in_stream' + }) + def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager', wallet_manager: 'WalletManager', storage: 'SQLiteStorage', node: Optional['Node'], analytics_manager: Optional['AnalyticsManager'] = None): - self.loop = loop - self.config = config + super().__init__(loop, config, storage, analytics_manager) self.blob_manager = blob_manager self.wallet_manager = wallet_manager - self.storage = storage self.node = node - self.analytics_manager = analytics_manager - self.streams: typing.Dict[str, ManagedStream] = {} self.resume_saving_task: Optional[asyncio.Task] = None self.re_reflect_task: Optional[asyncio.Task] = None self.update_stream_finished_futs: typing.List[asyncio.Future] = [] self.running_reflector_uploads: typing.Dict[str, asyncio.Task] = {} self.started = asyncio.Event(loop=self.loop) + def add(self, source: ManagedStream): + super().add(source) + self.storage.content_claim_callbacks[source.stream_hash] = lambda: self._update_content_claim(source) + async def _update_content_claim(self, stream: ManagedStream): claim_info = await self.storage.get_content_claim(stream.stream_hash) - self.streams.setdefault(stream.sd_hash, stream).set_claim(claim_info, claim_info['value']) + self._sources.setdefault(stream.sd_hash, stream).set_claim(claim_info, claim_info['value']) async def recover_streams(self, file_infos: typing.List[typing.Dict]): to_restore = [] @@ -123,10 +115,10 @@ class StreamManager: # if self.blob_manager._save_blobs: # log.info("Recovered %i/%i attempted streams", len(to_restore), len(file_infos)) - async def add_stream(self, rowid: int, sd_hash: str, file_name: Optional[str], - download_directory: Optional[str], status: str, - claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'], - added_on: Optional[int], fully_reflected: bool): + async def _load_stream(self, rowid: int, sd_hash: str, file_name: Optional[str], + download_directory: Optional[str], status: str, + claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'], + added_on: Optional[int]): try: descriptor = await self.blob_manager.get_stream_descriptor(sd_hash) except InvalidStreamDescriptorError as err: @@ -139,10 +131,9 @@ class StreamManager: ) if fully_reflected: stream.fully_reflected.set() - self.streams[sd_hash] = stream - self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream) + self.add(stream) - async def load_and_resume_streams_from_database(self): + async def initialize_from_database(self): to_recover = [] to_start = [] @@ -156,7 +147,6 @@ class StreamManager: to_recover.append(file_info) to_start.append(file_info) if to_recover: - log.info("Recover %i files", len(to_recover)) await self.recover_streams(to_recover) log.info("Initializing %i files", len(to_start)) @@ -167,7 +157,7 @@ class StreamManager: download_directory = path_or_none(file_info['download_directory']) if file_name and download_directory and not file_info['saved_file'] and file_info['status'] == 'running': to_resume_saving.append((file_name, download_directory, file_info['sd_hash'])) - add_stream_tasks.append(self.loop.create_task(self.add_stream( + add_stream_tasks.append(self.loop.create_task(self._load_stream( file_info['rowid'], file_info['sd_hash'], file_name, download_directory, file_info['status'], file_info['claim'], file_info['content_fee'], @@ -175,25 +165,22 @@ class StreamManager: ))) if add_stream_tasks: await asyncio.gather(*add_stream_tasks, loop=self.loop) - log.info("Started stream manager with %i files", len(self.streams)) + log.info("Started stream manager with %i files", len(self._sources)) if not self.node: log.info("no DHT node given, resuming downloads trusting that we can contact reflector") if to_resume_saving: - self.resume_saving_task = self.loop.create_task(self.resume(to_resume_saving)) - - async def resume(self, to_resume_saving): - log.info("Resuming saving %i files", len(to_resume_saving)) - await asyncio.gather( - *(self.streams[sd_hash].save_file(file_name, download_directory, node=self.node) - for (file_name, download_directory, sd_hash) in to_resume_saving), - loop=self.loop - ) + log.info("Resuming saving %i files", len(to_resume_saving)) + self.resume_saving_task = self.loop.create_task(asyncio.gather( + *(self._sources[sd_hash].save_file(file_name, download_directory, node=self.node) + for (file_name, download_directory, sd_hash) in to_resume_saving), + loop=self.loop + )) async def reflect_streams(self): while True: if self.config.reflect_streams and self.config.reflector_servers: sd_hashes = await self.storage.get_streams_to_re_reflect() - sd_hashes = [sd for sd in sd_hashes if sd in self.streams] + sd_hashes = [sd for sd in sd_hashes if sd in self._sources] batch = [] while sd_hashes: stream = self.streams[sd_hashes.pop()] @@ -209,18 +196,14 @@ class StreamManager: await asyncio.sleep(300, loop=self.loop) async def start(self): - await self.load_and_resume_streams_from_database() + await super().start() self.re_reflect_task = self.loop.create_task(self.reflect_streams()) - self.started.set() def stop(self): if self.resume_saving_task and not self.resume_saving_task.done(): self.resume_saving_task.cancel() if self.re_reflect_task and not self.re_reflect_task.done(): self.re_reflect_task.cancel() - while self.streams: - _, stream = self.streams.popitem() - stream.stop_tasks() while self.update_stream_finished_futs: self.update_stream_finished_futs.pop().cancel() while self.running_reflector_uploads: @@ -260,14 +243,7 @@ class StreamManager: del self.streams[stream.sd_hash] blob_hashes = [stream.sd_hash] + [b.blob_hash for b in stream.descriptor.blobs[:-1]] await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False) - await self.storage.delete_stream(stream.descriptor) - if delete_file and stream.output_file_exists: - os.remove(stream.full_path) - - def get_stream_by_stream_hash(self, stream_hash: str) -> Optional[ManagedStream]: - streams = tuple(filter(lambda stream: stream.stream_hash == stream_hash, self.streams.values())) - if streams: - return streams[0] + await self.storage.delete(stream.descriptor) def get_filtered_streams(self, sort_by: Optional[str] = None, reverse: Optional[bool] = False, comparison: Optional[str] = None, @@ -324,199 +300,5 @@ class StreamManager: streams.reverse() return streams - async def _check_update_or_replace(self, outpoint: str, claim_id: str, claim: Claim - ) -> typing.Tuple[Optional[ManagedStream], Optional[ManagedStream]]: - existing = self.get_filtered_streams(outpoint=outpoint) - if existing: - return existing[0], None - existing = self.get_filtered_streams(sd_hash=claim.stream.source.sd_hash) - if existing and existing[0].claim_id != claim_id: - raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {claim_id}") - if existing: - log.info("claim contains a metadata only update to a stream we have") - await self.storage.save_content_claim( - existing[0].stream_hash, outpoint - ) - await self._update_content_claim(existing[0]) - return existing[0], None - else: - existing_for_claim_id = self.get_filtered_streams(claim_id=claim_id) - if existing_for_claim_id: - log.info("claim contains an update to a stream we have, downloading it") - return None, existing_for_claim_id[0] - return None, None - - @staticmethod - def _convert_to_old_resolve_output(wallet_manager, resolves): - result = {} - for url, txo in resolves.items(): - if isinstance(txo, Output): - tx_height = txo.tx_ref.height - best_height = wallet_manager.ledger.headers.height - result[url] = { - 'name': txo.claim_name, - 'value': txo.claim, - 'protobuf': binascii.hexlify(txo.claim.to_bytes()), - 'claim_id': txo.claim_id, - 'txid': txo.tx_ref.id, - 'nout': txo.position, - 'amount': dewies_to_lbc(txo.amount), - 'effective_amount': txo.meta.get('effective_amount', 0), - 'height': tx_height, - 'confirmations': (best_height+1) - tx_height if tx_height > 0 else tx_height, - 'claim_sequence': -1, - 'address': txo.get_address(wallet_manager.ledger), - 'valid_at_height': txo.meta.get('activation_height', None), - 'timestamp': wallet_manager.ledger.headers.estimated_timestamp(tx_height), - 'supports': [] - } - else: - result[url] = txo - return result - - @cache_concurrent - async def download_stream_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager', - timeout: Optional[float] = None, - file_name: Optional[str] = None, - download_directory: Optional[str] = None, - save_file: Optional[bool] = None, - resolve_timeout: float = 3.0, - wallet: Optional['Wallet'] = None) -> ManagedStream: - manager = self.wallet_manager - wallet = wallet or manager.default_wallet - timeout = timeout or self.config.download_timeout - start_time = self.loop.time() - resolved_time = None - stream = None - txo: Optional[Output] = None - error = None - outpoint = None - if save_file is None: - save_file = self.config.save_files - if file_name and not save_file: - save_file = True - if save_file: - download_directory = download_directory or self.config.download_dir - else: - download_directory = None - - payment = None - try: - # resolve the claim - if not URL.parse(uri).has_stream: - raise ResolveError("cannot download a channel claim, specify a /path") - try: - response = await asyncio.wait_for( - manager.ledger.resolve(wallet.accounts, [uri], include_purchase_receipt=True), - resolve_timeout - ) - resolved_result = self._convert_to_old_resolve_output(manager, response) - except asyncio.TimeoutError: - raise ResolveTimeoutError(uri) - except Exception as err: - if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8 - raise - log.exception("Unexpected error resolving stream:") - raise ResolveError(f"Unexpected error resolving stream: {str(err)}") - await self.storage.save_claims_for_resolve([ - value for value in resolved_result.values() if 'error' not in value - ]) - resolved = resolved_result.get(uri, {}) - resolved = resolved if 'value' in resolved else resolved.get('claim') - if not resolved: - raise ResolveError(f"Failed to resolve stream at '{uri}'") - if 'error' in resolved: - raise ResolveError(f"error resolving stream: {resolved['error']}") - txo = response[uri] - - claim = Claim.from_bytes(binascii.unhexlify(resolved['protobuf'])) - outpoint = f"{resolved['txid']}:{resolved['nout']}" - resolved_time = self.loop.time() - start_time - - # resume or update an existing stream, if the stream changed: download it and delete the old one after - updated_stream, to_replace = await self._check_update_or_replace(outpoint, resolved['claim_id'], claim) - if updated_stream: - log.info("already have stream for %s", uri) - if save_file and updated_stream.output_file_exists: - save_file = False - await updated_stream.start(node=self.node, timeout=timeout, save_now=save_file) - if not updated_stream.output_file_exists and (save_file or file_name or download_directory): - await updated_stream.save_file( - file_name=file_name, download_directory=download_directory, node=self.node - ) - return updated_stream - - if not to_replace and txo.has_price and not txo.purchase_receipt: - payment = await manager.create_purchase_transaction( - wallet.accounts, txo, exchange_rate_manager - ) - - stream = ManagedStream( - self.loop, self.config, self.blob_manager, claim.stream.source.sd_hash, download_directory, - file_name, ManagedStream.STATUS_RUNNING, content_fee=payment, - analytics_manager=self.analytics_manager - ) - log.info("starting download for %s", uri) - - before_download = self.loop.time() - await stream.start(self.node, timeout) - stream.set_claim(resolved, claim) - if to_replace: # delete old stream now that the replacement has started downloading - await self.delete_stream(to_replace) - - if payment is not None: - await manager.broadcast_or_release(payment) - payment = None # to avoid releasing in `finally` later - log.info("paid fee of %s for %s", dewies_to_lbc(stream.content_fee.outputs[0].amount), uri) - await self.storage.save_content_fee(stream.stream_hash, stream.content_fee) - - self.streams[stream.sd_hash] = stream - self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream) - await self.storage.save_content_claim(stream.stream_hash, outpoint) - if save_file: - await asyncio.wait_for(stream.save_file(node=self.node), timeout - (self.loop.time() - before_download), - loop=self.loop) - return stream - except asyncio.TimeoutError: - error = DownloadDataTimeoutError(stream.sd_hash) - raise error - except Exception as err: # forgive data timeout, don't delete stream - expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError, - KeyFeeAboveMaxAllowedError) - if isinstance(err, expected): - log.warning("Failed to download %s: %s", uri, str(err)) - elif isinstance(err, asyncio.CancelledError): - pass - else: - log.exception("Unexpected error downloading stream:") - error = err - raise - finally: - if payment is not None: - # payment is set to None after broadcasting, if we're here an exception probably happened - await manager.ledger.release_tx(payment) - if self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or - stream.downloader.time_to_first_bytes))): - server = self.wallet_manager.ledger.network.client.server - self.loop.create_task( - self.analytics_manager.send_time_to_first_bytes( - resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id, - uri, outpoint, - None if not stream else len(stream.downloader.blob_downloader.active_connections), - None if not stream else len(stream.downloader.blob_downloader.scores), - None if not stream else len(stream.downloader.blob_downloader.connection_failures), - False if not stream else stream.downloader.added_fixed_peers, - self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay, - None if not stream else stream.sd_hash, - None if not stream else stream.downloader.time_to_descriptor, - None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash, - None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length, - None if not stream else stream.downloader.time_to_first_bytes, - None if not error else error.__class__.__name__, - None if not error else str(error), - None if not server else f"{server[0]}:{server[1]}" - ) - ) - async def stream_partial_content(self, request: Request, sd_hash: str): - return await self.streams[sd_hash].stream_file(request, self.node) + return await self._sources[sd_hash].stream_file(request, self.node) From 814a0a123fb6f27bdbaa399b48a075a4990761ec Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 15 Jan 2020 10:19:29 -0500 Subject: [PATCH 27/86] file manager refactor --- lbry/file/file_manager.py | 435 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 435 insertions(+) create mode 100644 lbry/file/file_manager.py diff --git a/lbry/file/file_manager.py b/lbry/file/file_manager.py new file mode 100644 index 000000000..dc95829d2 --- /dev/null +++ b/lbry/file/file_manager.py @@ -0,0 +1,435 @@ +import time +import asyncio +import binascii +import logging +import typing +from typing import Optional +from aiohttp.web import Request +from lbry.error import ResolveError, InvalidStreamDescriptorError, DownloadSDTimeoutError, InsufficientFundsError +from lbry.error import ResolveTimeoutError, DownloadDataTimeoutError, KeyFeeAboveMaxAllowedError +from lbry.utils import cache_concurrent +from lbry.schema.claim import Claim +from lbry.schema.url import URL +from lbry.wallet.dewies import dewies_to_lbc +from lbry.wallet.transaction import Output +from lbry.file.source_manager import SourceManager +from lbry.file.source import ManagedDownloadSource +if typing.TYPE_CHECKING: + from lbry.conf import Config + from lbry.extras.daemon.analytics import AnalyticsManager + from lbry.extras.daemon.storage import SQLiteStorage + from lbry.wallet import LbryWalletManager + from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager + +log = logging.getLogger(__name__) + + +def path_or_none(p) -> Optional[str]: + if not p: + return + return binascii.unhexlify(p).decode() + + +class FileManager: + def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', wallet_manager: 'LbryWalletManager', + storage: 'SQLiteStorage', analytics_manager: Optional['AnalyticsManager'] = None): + self.loop = loop + self.config = config + self.wallet_manager = wallet_manager + self.storage = storage + self.analytics_manager = analytics_manager + self.source_managers: typing.Dict[str, SourceManager] = {} + + async def start(self): + await asyncio.gather(*(source_manager.start() for source_manager in self.source_managers.values())) + + def stop(self): + while self.source_managers: + _, source_manager = self.source_managers.popitem() + source_manager.stop() + + @cache_concurrent + async def download_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager', + timeout: Optional[float] = None, file_name: Optional[str] = None, + download_directory: Optional[str] = None, + save_file: Optional[bool] = None, resolve_timeout: float = 3.0, + wallet: Optional['Wallet'] = None) -> ManagedDownloadSource: + + wallet = wallet or self.wallet_manager.default_wallet + timeout = timeout or self.config.download_timeout + start_time = self.loop.time() + resolved_time = None + stream = None + error = None + outpoint = None + if save_file is None: + save_file = self.config.save_files + if file_name and not save_file: + save_file = True + if save_file: + download_directory = download_directory or self.config.download_dir + else: + download_directory = None + + payment = None + try: + # resolve the claim + if not URL.parse(uri).has_stream: + raise ResolveError("cannot download a channel claim, specify a /path") + try: + resolved_result = await asyncio.wait_for( + self.wallet_manager.ledger.resolve(wallet.accounts, [uri]), + resolve_timeout + ) + except asyncio.TimeoutError: + raise ResolveTimeoutError(uri) + except Exception as err: + if isinstance(err, asyncio.CancelledError): + raise + log.exception("Unexpected error resolving stream:") + raise ResolveError(f"Unexpected error resolving stream: {str(err)}") + if not resolved_result: + raise ResolveError(f"Failed to resolve stream at '{uri}'") + if 'error' in resolved_result: + raise ResolveError(f"Unexpected error resolving uri for download: {resolved_result['error']}") + + await self.storage.save_claims( + resolved_result, self.wallet_manager.ledger + ) + + txo = resolved_result[uri] + claim = txo.claim + outpoint = f"{txo.tx_ref.id}:{txo.position}" + resolved_time = self.loop.time() - start_time + + #################### + # update or replace + #################### + + if claim.stream.source.bt_infohash: + source_manager = self.source_managers['torrent'] + else: + source_manager = self.source_managers['stream'] + + # resume or update an existing stream, if the stream changed: download it and delete the old one after + existing = self.get_filtered(sd_hash=claim.stream.source.sd_hash) + if existing and existing[0].claim_id != txo.claim_id: + raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {txo.claim_id}") + if existing: + log.info("claim contains a metadata only update to a stream we have") + await self.storage.save_content_claim( + existing[0].stream_hash, outpoint + ) + await source_manager._update_content_claim(existing[0]) + return existing[0] + else: + existing_for_claim_id = self.get_filtered(claim_id=txo.claim_id) + if existing_for_claim_id: + log.info("claim contains an update to a stream we have, downloading it") + if save_file and existing_for_claim_id[0].output_file_exists: + save_file = False + await existing_for_claim_id[0].start(node=self.node, timeout=timeout, save_now=save_file) + if not existing_for_claim_id[0].output_file_exists and (save_file or file_name or download_directory): + await existing_for_claim_id[0].save_file( + file_name=file_name, download_directory=download_directory, node=self.node + ) + return existing_for_claim_id[0] + + + + + + + if updated_stream: + + + #################### + # pay fee + #################### + + if not to_replace and txo.has_price and not txo.purchase_receipt: + payment = await manager.create_purchase_transaction( + wallet.accounts, txo, exchange_rate_manager + ) + + #################### + # make downloader and wait for start + #################### + + stream = ManagedStream( + self.loop, self.config, self.blob_manager, claim.stream.source.sd_hash, download_directory, + file_name, ManagedStream.STATUS_RUNNING, content_fee=payment, + analytics_manager=self.analytics_manager + ) + log.info("starting download for %s", uri) + + before_download = self.loop.time() + await stream.start(self.node, timeout) + stream.set_claim(resolved, claim) + + #################### + # success case: delete to_replace if applicable, broadcast fee payment + #################### + + if to_replace: # delete old stream now that the replacement has started downloading + await self.delete(to_replace) + + if payment is not None: + await self.wallet_manager.broadcast_or_release(payment) + payment = None # to avoid releasing in `finally` later + log.info("paid fee of %s for %s", dewies_to_lbc(stream.content_fee.outputs[0].amount), uri) + await self.storage.save_content_fee(stream.stream_hash, stream.content_fee) + + self._sources[stream.sd_hash] = stream + self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream) + + await self.storage.save_content_claim(stream.stream_hash, outpoint) + if save_file: + await asyncio.wait_for(stream.save_file(node=self.node), timeout - (self.loop.time() - before_download), + loop=self.loop) + return stream + except asyncio.TimeoutError: + error = DownloadDataTimeoutError(stream.sd_hash) + raise error + except Exception as err: # forgive data timeout, don't delete stream + expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError, + KeyFeeAboveMaxAllowedError) + if isinstance(err, expected): + log.warning("Failed to download %s: %s", uri, str(err)) + elif isinstance(err, asyncio.CancelledError): + pass + else: + log.exception("Unexpected error downloading stream:") + error = err + raise + finally: + if payment is not None: + # payment is set to None after broadcasting, if we're here an exception probably happened + await self.wallet_manager.ledger.release_tx(payment) + if self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or + stream.downloader.time_to_first_bytes))): + server = self.wallet_manager.ledger.network.client.server + self.loop.create_task( + self.analytics_manager.send_time_to_first_bytes( + resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id, + uri, outpoint, + None if not stream else len(stream.downloader.blob_downloader.active_connections), + None if not stream else len(stream.downloader.blob_downloader.scores), + None if not stream else len(stream.downloader.blob_downloader.connection_failures), + False if not stream else stream.downloader.added_fixed_peers, + self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay, + None if not stream else stream.sd_hash, + None if not stream else stream.downloader.time_to_descriptor, + None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash, + None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length, + None if not stream else stream.downloader.time_to_first_bytes, + None if not error else error.__class__.__name__, + None if not error else str(error), + None if not server else f"{server[0]}:{server[1]}" + ) + ) + + async def stream_partial_content(self, request: Request, sd_hash: str): + return await self._sources[sd_hash].stream_file(request, self.node) + + def get_filtered(self, sort_by: Optional[str] = None, reverse: Optional[bool] = False, + comparison: Optional[str] = None, **search_by) -> typing.List[ManagedDownloadSource]: + """ + Get a list of filtered and sorted ManagedStream objects + + :param sort_by: field to sort by + :param reverse: reverse sorting + :param comparison: comparison operator used for filtering + :param search_by: fields and values to filter by + """ + if sort_by and sort_by not in self.filter_fields: + raise ValueError(f"'{sort_by}' is not a valid field to sort by") + if comparison and comparison not in comparison_operators: + raise ValueError(f"'{comparison}' is not a valid comparison") + if 'full_status' in search_by: + del search_by['full_status'] + for search in search_by.keys(): + if search not in self.filter_fields: + raise ValueError(f"'{search}' is not a valid search operation") + if search_by: + comparison = comparison or 'eq' + sources = [] + for stream in self._sources.values(): + for search, val in search_by.items(): + if comparison_operators[comparison](getattr(stream, search), val): + sources.append(stream) + break + else: + sources = list(self._sources.values()) + if sort_by: + sources.sort(key=lambda s: getattr(s, sort_by)) + if reverse: + sources.reverse() + return sources + + + + # @cache_concurrent + # async def download_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager', + # timeout: Optional[float] = None, file_name: Optional[str] = None, + # download_directory: Optional[str] = None, + # save_file: Optional[bool] = None, resolve_timeout: float = 3.0, + # wallet: Optional['Wallet'] = None) -> ManagedDownloadSource: + # wallet = wallet or self.wallet_manager.default_wallet + # timeout = timeout or self.config.download_timeout + # start_time = self.loop.time() + # resolved_time = None + # stream = None + # txo: Optional[Output] = None + # error = None + # outpoint = None + # if save_file is None: + # save_file = self.config.save_files + # if file_name and not save_file: + # save_file = True + # if save_file: + # download_directory = download_directory or self.config.download_dir + # else: + # download_directory = None + # + # payment = None + # try: + # # resolve the claim + # if not URL.parse(uri).has_stream: + # raise ResolveError("cannot download a channel claim, specify a /path") + # try: + # response = await asyncio.wait_for( + # self.wallet_manager.ledger.resolve(wallet.accounts, [uri]), + # resolve_timeout + # ) + # resolved_result = {} + # for url, txo in response.items(): + # if isinstance(txo, Output): + # tx_height = txo.tx_ref.height + # best_height = self.wallet_manager.ledger.headers.height + # resolved_result[url] = { + # 'name': txo.claim_name, + # 'value': txo.claim, + # 'protobuf': binascii.hexlify(txo.claim.to_bytes()), + # 'claim_id': txo.claim_id, + # 'txid': txo.tx_ref.id, + # 'nout': txo.position, + # 'amount': dewies_to_lbc(txo.amount), + # 'effective_amount': txo.meta.get('effective_amount', 0), + # 'height': tx_height, + # 'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height, + # 'claim_sequence': -1, + # 'address': txo.get_address(self.wallet_manager.ledger), + # 'valid_at_height': txo.meta.get('activation_height', None), + # 'timestamp': self.wallet_manager.ledger.headers[tx_height]['timestamp'], + # 'supports': [] + # } + # else: + # resolved_result[url] = txo + # except asyncio.TimeoutError: + # raise ResolveTimeoutError(uri) + # except Exception as err: + # if isinstance(err, asyncio.CancelledError): + # raise + # log.exception("Unexpected error resolving stream:") + # raise ResolveError(f"Unexpected error resolving stream: {str(err)}") + # await self.storage.save_claims_for_resolve([ + # value for value in resolved_result.values() if 'error' not in value + # ]) + # + # resolved = resolved_result.get(uri, {}) + # resolved = resolved if 'value' in resolved else resolved.get('claim') + # if not resolved: + # raise ResolveError(f"Failed to resolve stream at '{uri}'") + # if 'error' in resolved: + # raise ResolveError(f"error resolving stream: {resolved['error']}") + # txo = response[uri] + # + # claim = Claim.from_bytes(binascii.unhexlify(resolved['protobuf'])) + # outpoint = f"{resolved['txid']}:{resolved['nout']}" + # resolved_time = self.loop.time() - start_time + # + # # resume or update an existing stream, if the stream changed: download it and delete the old one after + # updated_stream, to_replace = await self._check_update_or_replace(outpoint, resolved['claim_id'], claim) + # if updated_stream: + # log.info("already have stream for %s", uri) + # if save_file and updated_stream.output_file_exists: + # save_file = False + # await updated_stream.start(node=self.node, timeout=timeout, save_now=save_file) + # if not updated_stream.output_file_exists and (save_file or file_name or download_directory): + # await updated_stream.save_file( + # file_name=file_name, download_directory=download_directory, node=self.node + # ) + # return updated_stream + # + # if not to_replace and txo.has_price and not txo.purchase_receipt: + # payment = await manager.create_purchase_transaction( + # wallet.accounts, txo, exchange_rate_manager + # ) + # + # stream = ManagedStream( + # self.loop, self.config, self.blob_manager, claim.stream.source.sd_hash, download_directory, + # file_name, ManagedStream.STATUS_RUNNING, content_fee=payment, + # analytics_manager=self.analytics_manager + # ) + # log.info("starting download for %s", uri) + # + # before_download = self.loop.time() + # await stream.start(self.node, timeout) + # stream.set_claim(resolved, claim) + # if to_replace: # delete old stream now that the replacement has started downloading + # await self.delete(to_replace) + # + # if payment is not None: + # await manager.broadcast_or_release(payment) + # payment = None # to avoid releasing in `finally` later + # log.info("paid fee of %s for %s", dewies_to_lbc(stream.content_fee.outputs[0].amount), uri) + # await self.storage.save_content_fee(stream.stream_hash, stream.content_fee) + # + # self._sources[stream.sd_hash] = stream + # self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream) + # await self.storage.save_content_claim(stream.stream_hash, outpoint) + # if save_file: + # await asyncio.wait_for(stream.save_file(node=self.node), timeout - (self.loop.time() - before_download), + # loop=self.loop) + # return stream + # except asyncio.TimeoutError: + # error = DownloadDataTimeoutError(stream.sd_hash) + # raise error + # except Exception as err: # forgive data timeout, don't delete stream + # expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError, + # KeyFeeAboveMaxAllowedError) + # if isinstance(err, expected): + # log.warning("Failed to download %s: %s", uri, str(err)) + # elif isinstance(err, asyncio.CancelledError): + # pass + # else: + # log.exception("Unexpected error downloading stream:") + # error = err + # raise + # finally: + # if payment is not None: + # # payment is set to None after broadcasting, if we're here an exception probably happened + # await manager.ledger.release_tx(payment) + # if self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or + # stream.downloader.time_to_first_bytes))): + # server = self.wallet_manager.ledger.network.client.server + # self.loop.create_task( + # self.analytics_manager.send_time_to_first_bytes( + # resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id, + # uri, outpoint, + # None if not stream else len(stream.downloader.blob_downloader.active_connections), + # None if not stream else len(stream.downloader.blob_downloader.scores), + # None if not stream else len(stream.downloader.blob_downloader.connection_failures), + # False if not stream else stream.downloader.added_fixed_peers, + # self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay, + # None if not stream else stream.sd_hash, + # None if not stream else stream.downloader.time_to_descriptor, + # None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash, + # None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length, + # None if not stream else stream.downloader.time_to_first_bytes, + # None if not error else error.__class__.__name__, + # None if not error else str(error), + # None if not server else f"{server[0]}:{server[1]}" + # ) + # ) From f2cc19e6aa551b524f7d538074bfeb3ad2d2ad31 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 15 Jan 2020 10:20:36 -0500 Subject: [PATCH 28/86] add lbry.torrent --- lbry/torrent/__init__.py | 0 lbry/torrent/torrent.py | 70 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 lbry/torrent/__init__.py create mode 100644 lbry/torrent/torrent.py diff --git a/lbry/torrent/__init__.py b/lbry/torrent/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lbry/torrent/torrent.py b/lbry/torrent/torrent.py new file mode 100644 index 000000000..bbbc487bf --- /dev/null +++ b/lbry/torrent/torrent.py @@ -0,0 +1,70 @@ +import asyncio +import logging +import typing + + +log = logging.getLogger(__name__) + + +class TorrentInfo: + __slots__ = ('dht_seeds', 'http_seeds', 'trackers', 'total_size') + + def __init__(self, dht_seeds: typing.Tuple[typing.Tuple[str, int]], + http_seeds: typing.Tuple[typing.Dict[str, typing.Any]], + trackers: typing.Tuple[typing.Tuple[str, int]], total_size: int): + self.dht_seeds = dht_seeds + self.http_seeds = http_seeds + self.trackers = trackers + self.total_size = total_size + + @classmethod + def from_libtorrent_info(cls, ti): + return cls( + ti.nodes(), tuple( + { + 'url': web_seed['url'], + 'type': web_seed['type'], + 'auth': web_seed['auth'] + } for web_seed in ti.web_seeds() + ), tuple( + (tracker.url, tracker.tier) for tracker in ti.trackers() + ), ti.total_size() + ) + + +class Torrent: + def __init__(self, loop, handle): + self._loop = loop + self._handle = handle + self.finished = asyncio.Event(loop=loop) + + def _threaded_update_status(self): + status = self._handle.status() + if not status.is_seeding: + log.info('%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d) %s' % ( + status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000, + status.num_peers, status.state)) + elif not self.finished.is_set(): + self.finished.set() + + async def wait_for_finished(self): + while True: + await self._loop.run_in_executor( + None, self._threaded_update_status + ) + if self.finished.is_set(): + log.info("finished downloading torrent!") + await self.pause() + break + await asyncio.sleep(1, loop=self._loop) + + async def pause(self): + log.info("pause torrent") + await self._loop.run_in_executor( + None, self._handle.pause + ) + + async def resume(self): + await self._loop.run_in_executor( + None, self._handle.resume + ) From b09c46f6f76917386bf4e68d5a404286366638ac Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 24 Jan 2020 10:58:01 -0300 Subject: [PATCH 29/86] add torrent component --- lbry/extras/daemon/components.py | 36 +++++++- lbry/torrent/session.py | 139 +++++++++++++++++++++++++++++++ 2 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 lbry/torrent/session.py diff --git a/lbry/extras/daemon/components.py b/lbry/extras/daemon/components.py index 5271c1558..1ac385c60 100644 --- a/lbry/extras/daemon/components.py +++ b/lbry/extras/daemon/components.py @@ -22,6 +22,11 @@ from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager from lbry.extras.daemon.storage import SQLiteStorage from lbry.wallet import WalletManager from lbry.wallet.usage_payment import WalletServerPayer +try: + import libtorrent + from lbry.torrent.session import TorrentSession +except ImportError: + libtorrent = None log = logging.getLogger(__name__) @@ -37,6 +42,7 @@ STREAM_MANAGER_COMPONENT = "stream_manager" PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server" UPNP_COMPONENT = "upnp" EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager" +LIBTORRENT_COMPONENT = "libtorrent_component" class DatabaseComponent(Component): @@ -335,7 +341,7 @@ class StreamManagerComponent(Component): if not self.stream_manager: return return { - 'managed_files': len(self.stream_manager.streams), + 'managed_files': len(self.stream_manager._sources), } async def start(self): @@ -356,6 +362,34 @@ class StreamManagerComponent(Component): self.stream_manager.stop() +class TorrentComponent(Component): + component_name = LIBTORRENT_COMPONENT + + def __init__(self, component_manager): + super().__init__(component_manager) + self.torrent_session = None + + @property + def component(self) -> typing.Optional[StreamManager]: + return self.torrent_session + + async def get_status(self): + if not self.torrent_session: + return + return { + 'running': True, # TODO: what to return here? + } + + async def start(self): + if libtorrent: + self.torrent_session = TorrentSession(asyncio.get_event_loop(), None) + await self.torrent_session.bind() # TODO: specify host/port + + async def stop(self): + if self.torrent_session: + await self.torrent_session.pause() + + class PeerProtocolServerComponent(Component): component_name = PEER_PROTOCOL_SERVER_COMPONENT depends_on = [UPNP_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT] diff --git a/lbry/torrent/session.py b/lbry/torrent/session.py new file mode 100644 index 000000000..01e4cafa3 --- /dev/null +++ b/lbry/torrent/session.py @@ -0,0 +1,139 @@ +import asyncio +import binascii +import libtorrent + + +NOTIFICATION_MASKS = [ + "error", + "peer", + "port_mapping", + "storage", + "tracker", + "debug", + "status", + "progress", + "ip_block", + "dht", + "stats", + "session_log", + "torrent_log", + "peer_log", + "incoming_request", + "dht_log", + "dht_operation", + "port_mapping_log", + "picker_log", + "file_progress", + "piece_progress", + "upload", + "block_progress" +] + + +def get_notification_type(notification) -> str: + for i, notification_type in enumerate(NOTIFICATION_MASKS): + if (1 << i) & notification: + return notification_type + raise ValueError("unrecognized notification type") + + +class TorrentHandle: + def __init__(self, loop, executor, handle): + self._loop = loop + self._executor = executor + self._handle = handle + self.finished = asyncio.Event(loop=loop) + + def _show_status(self): + status = self._handle.status() + if not status.is_seeding: + print('%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d) %s' % ( + status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000, + status.num_peers, status.state)) + elif not self.finished.is_set(): + self.finished.set() + print("finished!") + + async def status_loop(self): + while True: + await self._loop.run_in_executor( + self._executor, self._show_status + ) + if self.finished.is_set(): + break + await asyncio.sleep(1, loop=self._loop) + + async def pause(self): + await self._loop.run_in_executor( + self._executor, self._handle.pause + ) + + async def resume(self): + await self._loop.run_in_executor( + self._executor, self._handle.resume + ) + + +class TorrentSession: + def __init__(self, loop, executor): + self._loop = loop + self._executor = executor + self._session = None + self._handles = {} + + async def bind(self, interface: str = '0.0.0.0', port: int = 6881): + settings = { + 'listen_interfaces': f"{interface}:{port}", + 'enable_outgoing_utp': True, + 'enable_incoming_utp': True, + 'enable_outgoing_tcp': True, + 'enable_incoming_tcp': True + } + self._session = await self._loop.run_in_executor( + self._executor, libtorrent.session, settings + ) + await self._loop.run_in_executor( + self._executor, self._session.add_dht_router, "router.utorrent.com", 6881 + ) + self._loop.create_task(self.process_alerts()) + + def _pop_alerts(self): + for alert in self._session.pop_alerts(): + print("alert: ", alert) + + async def process_alerts(self): + while True: + await self._loop.run_in_executor( + self._executor, self._pop_alerts + ) + await asyncio.sleep(1, loop=self._loop) + + async def pause(self): + state = await self._loop.run_in_executor( + self._executor, self._session.save_state + ) + # print(f"state:\n{state}") + await self._loop.run_in_executor( + self._executor, self._session.pause + ) + + async def resume(self): + await self._loop.run_in_executor( + self._executor, self._session.resume + ) + + def _add_torrent(self, btih: str, download_directory: str): + self._handles[btih] = TorrentHandle(self._loop, self._executor, self._session.add_torrent( + {'info_hash': binascii.unhexlify(btih.encode()), 'save_path': download_directory} + )) + + async def add_torrent(self, btih, download_path): + await self._loop.run_in_executor( + self._executor, self._add_torrent, btih, download_path + ) + self._loop.create_task(self._handles[btih].status_loop()) + await self._handles[btih].finished.wait() + + +def get_magnet_uri(btih): + return f"magnet:?xt=urn:btih:{btih}" From 543c75b293a2838f44fa31749dc1a62e29c5ad5c Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 27 Jan 2020 02:10:55 -0300 Subject: [PATCH 30/86] wip --- lbry/extras/daemon/components.py | 22 ++++--- lbry/extras/daemon/daemon.py | 2 +- lbry/extras/daemon/storage.py | 15 ++++- lbry/file/file_manager.py | 98 +++++++++++++++++--------------- lbry/stream/stream_manager.py | 14 +---- 5 files changed, 81 insertions(+), 70 deletions(-) diff --git a/lbry/extras/daemon/components.py b/lbry/extras/daemon/components.py index 1ac385c60..de43eb926 100644 --- a/lbry/extras/daemon/components.py +++ b/lbry/extras/daemon/components.py @@ -17,6 +17,7 @@ from lbry.dht.blob_announcer import BlobAnnouncer from lbry.blob.blob_manager import BlobManager from lbry.blob_exchange.server import BlobServer from lbry.stream.stream_manager import StreamManager +from lbry.file.file_manager import FileManager from lbry.extras.daemon.component import Component from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager from lbry.extras.daemon.storage import SQLiteStorage @@ -331,17 +332,17 @@ class StreamManagerComponent(Component): def __init__(self, component_manager): super().__init__(component_manager) - self.stream_manager: typing.Optional[StreamManager] = None + self.file_manager: typing.Optional[FileManager] = None @property - def component(self) -> typing.Optional[StreamManager]: - return self.stream_manager + def component(self) -> typing.Optional[FileManager]: + return self.file_manager async def get_status(self): - if not self.stream_manager: + if not self.file_manager: return return { - 'managed_files': len(self.stream_manager._sources), + 'managed_files': len(self.file_manager._sources), } async def start(self): @@ -352,14 +353,17 @@ class StreamManagerComponent(Component): if self.component_manager.has_component(DHT_COMPONENT) else None log.info('Starting the file manager') loop = asyncio.get_event_loop() - self.stream_manager = StreamManager( + self.file_manager = FileManager( + loop, self.conf, wallet, storage, self.component_manager.analytics_manager + ) + self.file_manager.source_managers['stream'] = StreamManager( loop, self.conf, blob_manager, wallet, storage, node, self.component_manager.analytics_manager ) - await self.stream_manager.start() + await self.file_manager.start() log.info('Done setting up file manager') async def stop(self): - self.stream_manager.stop() + self.file_manager.stop() class TorrentComponent(Component): @@ -370,7 +374,7 @@ class TorrentComponent(Component): self.torrent_session = None @property - def component(self) -> typing.Optional[StreamManager]: + def component(self) -> typing.Optional[TorrentSession]: return self.torrent_session async def get_status(self): diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index 5c84f8227..276e1693e 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -1103,7 +1103,7 @@ class Daemon(metaclass=JSONRPCServerType): if download_directory and not os.path.isdir(download_directory): return {"error": f"specified download directory \"{download_directory}\" does not exist"} try: - stream = await self.stream_manager.download_stream_from_uri( + stream = await self.stream_manager.download_from_uri( uri, self.exchange_rate_manager, timeout, file_name, download_directory, save_file=save_file, wallet=wallet ) diff --git a/lbry/extras/daemon/storage.py b/lbry/extras/daemon/storage.py index 11a61e45e..426985a48 100644 --- a/lbry/extras/daemon/storage.py +++ b/lbry/extras/daemon/storage.py @@ -9,7 +9,7 @@ from typing import Optional from lbry.wallet import SQLiteMixin from lbry.conf import Config from lbry.wallet.dewies import dewies_to_lbc, lbc_to_dewies -from lbry.wallet.transaction import Transaction +from lbry.wallet.transaction import Transaction, Output from lbry.schema.claim import Claim from lbry.dht.constants import DATA_EXPIRATION from lbry.blob.blob_info import BlobInfo @@ -727,6 +727,19 @@ class SQLiteStorage(SQLiteMixin): if claim_id_to_supports: await self.save_supports(claim_id_to_supports) + def save_claim_from_output(self, ledger, output: Output): + return self.save_claims([{ + "claim_id": output.claim_id, + "name": output.claim_name, + "amount": dewies_to_lbc(output.amount), + "address": output.get_address(ledger), + "txid": output.tx_ref.id, + "nout": output.position, + "value": output.claim, + "height": -1, + "claim_sequence": -1, + }]) + def save_claims_for_resolve(self, claim_infos): to_save = {} for info in claim_infos: diff --git a/lbry/file/file_manager.py b/lbry/file/file_manager.py index dc95829d2..443c94a69 100644 --- a/lbry/file/file_manager.py +++ b/lbry/file/file_manager.py @@ -7,6 +7,7 @@ from typing import Optional from aiohttp.web import Request from lbry.error import ResolveError, InvalidStreamDescriptorError, DownloadSDTimeoutError, InsufficientFundsError from lbry.error import ResolveTimeoutError, DownloadDataTimeoutError, KeyFeeAboveMaxAllowedError +from lbry.stream.managed_stream import ManagedStream from lbry.utils import cache_concurrent from lbry.schema.claim import Claim from lbry.schema.url import URL @@ -93,14 +94,11 @@ class FileManager: if 'error' in resolved_result: raise ResolveError(f"Unexpected error resolving uri for download: {resolved_result['error']}") - await self.storage.save_claims( - resolved_result, self.wallet_manager.ledger - ) - txo = resolved_result[uri] claim = txo.claim outpoint = f"{txo.tx_ref.id}:{txo.position}" resolved_time = self.loop.time() - start_time + await self.storage.save_claim_from_output(self.wallet_manager.ledger, txo) #################### # update or replace @@ -113,6 +111,7 @@ class FileManager: # resume or update an existing stream, if the stream changed: download it and delete the old one after existing = self.get_filtered(sd_hash=claim.stream.source.sd_hash) + to_replace, updated_stream = None, None if existing and existing[0].claim_id != txo.claim_id: raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {txo.claim_id}") if existing: @@ -121,7 +120,7 @@ class FileManager: existing[0].stream_hash, outpoint ) await source_manager._update_content_claim(existing[0]) - return existing[0] + updated_stream = existing[0] else: existing_for_claim_id = self.get_filtered(claim_id=txo.claim_id) if existing_for_claim_id: @@ -133,14 +132,19 @@ class FileManager: await existing_for_claim_id[0].save_file( file_name=file_name, download_directory=download_directory, node=self.node ) - return existing_for_claim_id[0] - - - - - + to_replace = existing_for_claim_id[0] + # resume or update an existing stream, if the stream changed: download it and delete the old one after if updated_stream: + log.info("already have stream for %s", uri) + if save_file and updated_stream.output_file_exists: + save_file = False + await updated_stream.start(node=self.node, timeout=timeout, save_now=save_file) + if not updated_stream.output_file_exists and (save_file or file_name or download_directory): + await updated_stream.save_file( + file_name=file_name, download_directory=download_directory, node=self.node + ) + return updated_stream #################### @@ -156,23 +160,25 @@ class FileManager: # make downloader and wait for start #################### - stream = ManagedStream( - self.loop, self.config, self.blob_manager, claim.stream.source.sd_hash, download_directory, - file_name, ManagedStream.STATUS_RUNNING, content_fee=payment, - analytics_manager=self.analytics_manager - ) + if not claim.stream.source.bt_infohash: + stream = ManagedStream( + self.loop, self.config, source_manager.blob_manager, claim.stream.source.sd_hash, download_directory, + file_name, ManagedStream.STATUS_RUNNING, content_fee=payment, + analytics_manager=self.analytics_manager + ) + else: + stream = None log.info("starting download for %s", uri) before_download = self.loop.time() - await stream.start(self.node, timeout) - stream.set_claim(resolved, claim) + await stream.start(source_manager.node, timeout) #################### # success case: delete to_replace if applicable, broadcast fee payment #################### if to_replace: # delete old stream now that the replacement has started downloading - await self.delete(to_replace) + await source_manager.delete(to_replace) if payment is not None: await self.wallet_manager.broadcast_or_release(payment) @@ -180,12 +186,11 @@ class FileManager: log.info("paid fee of %s for %s", dewies_to_lbc(stream.content_fee.outputs[0].amount), uri) await self.storage.save_content_fee(stream.stream_hash, stream.content_fee) - self._sources[stream.sd_hash] = stream - self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream) + source_manager.add(stream) await self.storage.save_content_claim(stream.stream_hash, outpoint) if save_file: - await asyncio.wait_for(stream.save_file(node=self.node), timeout - (self.loop.time() - before_download), + await asyncio.wait_for(stream.save_file(node=source_manager.node), timeout - (self.loop.time() - before_download), loop=self.loop) return stream except asyncio.TimeoutError: @@ -232,8 +237,7 @@ class FileManager: async def stream_partial_content(self, request: Request, sd_hash: str): return await self._sources[sd_hash].stream_file(request, self.node) - def get_filtered(self, sort_by: Optional[str] = None, reverse: Optional[bool] = False, - comparison: Optional[str] = None, **search_by) -> typing.List[ManagedDownloadSource]: + def get_filtered(self, *args, **kwargs) -> typing.List[ManagedDownloadSource]: """ Get a list of filtered and sorted ManagedStream objects @@ -242,30 +246,30 @@ class FileManager: :param comparison: comparison operator used for filtering :param search_by: fields and values to filter by """ - if sort_by and sort_by not in self.filter_fields: - raise ValueError(f"'{sort_by}' is not a valid field to sort by") - if comparison and comparison not in comparison_operators: - raise ValueError(f"'{comparison}' is not a valid comparison") - if 'full_status' in search_by: - del search_by['full_status'] - for search in search_by.keys(): - if search not in self.filter_fields: - raise ValueError(f"'{search}' is not a valid search operation") - if search_by: - comparison = comparison or 'eq' - sources = [] - for stream in self._sources.values(): - for search, val in search_by.items(): - if comparison_operators[comparison](getattr(stream, search), val): - sources.append(stream) - break + return sum(*(manager.get_filtered(*args, **kwargs) for manager in self.source_managers.values()), []) + + async def _check_update_or_replace( + self, outpoint: str, claim_id: str, claim: Claim + ) -> typing.Tuple[Optional[ManagedDownloadSource], Optional[ManagedDownloadSource]]: + existing = self.get_filtered(outpoint=outpoint) + if existing: + return existing[0], None + existing = self.get_filtered(sd_hash=claim.stream.source.sd_hash) + if existing and existing[0].claim_id != claim_id: + raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {claim_id}") + if existing: + log.info("claim contains a metadata only update to a stream we have") + await self.storage.save_content_claim( + existing[0].stream_hash, outpoint + ) + await self._update_content_claim(existing[0]) + return existing[0], None else: - sources = list(self._sources.values()) - if sort_by: - sources.sort(key=lambda s: getattr(s, sort_by)) - if reverse: - sources.reverse() - return sources + existing_for_claim_id = self.get_filtered(claim_id=claim_id) + if existing_for_claim_id: + log.info("claim contains an update to a stream we have, downloading it") + return None, existing_for_claim_id[0] + return None, None diff --git a/lbry/stream/stream_manager.py b/lbry/stream/stream_manager.py index 58035e174..d5fc9202b 100644 --- a/lbry/stream/stream_manager.py +++ b/lbry/stream/stream_manager.py @@ -6,16 +6,10 @@ import random import typing from typing import Optional from aiohttp.web import Request -from lbry.error import ResolveError, InvalidStreamDescriptorError, DownloadSDTimeoutError, InsufficientFundsError -from lbry.error import ResolveTimeoutError, DownloadDataTimeoutError, KeyFeeAboveMaxAllowedError -from lbry.utils import cache_concurrent +from lbry.error import InvalidStreamDescriptorError +from lbry.file.source_manager import SourceManager from lbry.stream.descriptor import StreamDescriptor from lbry.stream.managed_stream import ManagedStream -from lbry.schema.claim import Claim -from lbry.schema.url import URL -from lbry.wallet.dewies import dewies_to_lbc -from lbry.wallet import Output -from lbry.source_manager import SourceManager from lbry.file.source import ManagedDownloadSource if typing.TYPE_CHECKING: from lbry.conf import Config @@ -25,10 +19,6 @@ if typing.TYPE_CHECKING: from lbry.wallet.transaction import Transaction from lbry.extras.daemon.analytics import AnalyticsManager from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim - from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager - from lbry.wallet.transaction import Transaction - from lbry.wallet.manager import WalletManager - from lbry.wallet.wallet import Wallet log = logging.getLogger(__name__) From 698ee271d6673e8d53289bde5eceecf3e2b184d6 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 27 Jan 2020 19:02:31 -0300 Subject: [PATCH 31/86] stream manager component becomes file manager component --- lbry/extras/daemon/components.py | 7 ++- lbry/extras/daemon/daemon.py | 60 +++++++++---------- .../datanetwork/test_file_commands.py | 12 ++-- .../integration/datanetwork/test_streaming.py | 4 +- tests/integration/other/test_cli.py | 6 +- .../unit/components/test_component_manager.py | 2 +- 6 files changed, 46 insertions(+), 45 deletions(-) diff --git a/lbry/extras/daemon/components.py b/lbry/extras/daemon/components.py index de43eb926..ff6de3c61 100644 --- a/lbry/extras/daemon/components.py +++ b/lbry/extras/daemon/components.py @@ -28,6 +28,7 @@ try: from lbry.torrent.session import TorrentSession except ImportError: libtorrent = None + TorrentSession = None log = logging.getLogger(__name__) @@ -39,7 +40,7 @@ WALLET_COMPONENT = "wallet" WALLET_SERVER_PAYMENTS_COMPONENT = "wallet_server_payments" DHT_COMPONENT = "dht" HASH_ANNOUNCER_COMPONENT = "hash_announcer" -STREAM_MANAGER_COMPONENT = "stream_manager" +FILE_MANAGER_COMPONENT = "file_manager" PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server" UPNP_COMPONENT = "upnp" EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager" @@ -326,8 +327,8 @@ class HashAnnouncerComponent(Component): } -class StreamManagerComponent(Component): - component_name = STREAM_MANAGER_COMPONENT +class FileManagerComponent(Component): + component_name = FILE_MANAGER_COMPONENT depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT] def __init__(self, component_manager): diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index 276e1693e..5432cd79f 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -40,7 +40,7 @@ from lbry.error import ( from lbry.extras import system_info from lbry.extras.daemon import analytics from lbry.extras.daemon.components import WALLET_COMPONENT, DATABASE_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT -from lbry.extras.daemon.components import STREAM_MANAGER_COMPONENT +from lbry.extras.daemon.components import FILE_MANAGER_COMPONENT from lbry.extras.daemon.components import EXCHANGE_RATE_MANAGER_COMPONENT, UPNP_COMPONENT from lbry.extras.daemon.componentmanager import RequiredCondition from lbry.extras.daemon.componentmanager import ComponentManager @@ -372,8 +372,8 @@ class Daemon(metaclass=JSONRPCServerType): return self.component_manager.get_component(DATABASE_COMPONENT) @property - def stream_manager(self) -> typing.Optional['StreamManager']: - return self.component_manager.get_component(STREAM_MANAGER_COMPONENT) + def file_manager(self) -> typing.Optional['StreamManager']: + return self.component_manager.get_component(FILE_MANAGER_COMPONENT) @property def exchange_rate_manager(self) -> typing.Optional['ExchangeRateManager']: @@ -609,8 +609,8 @@ class Daemon(metaclass=JSONRPCServerType): else: name, claim_id = name_and_claim_id.split("/") uri = f"lbry://{name}#{claim_id}" - if not self.stream_manager.started.is_set(): - await self.stream_manager.started.wait() + if not self.file_manager.started.is_set(): + await self.file_manager.started.wait() stream = await self.jsonrpc_get(uri) if isinstance(stream, dict): raise web.HTTPServerError(text=stream['error']) @@ -634,11 +634,11 @@ class Daemon(metaclass=JSONRPCServerType): async def _handle_stream_range_request(self, request: web.Request): sd_hash = request.path.split("/stream/")[1] - if not self.stream_manager.started.is_set(): - await self.stream_manager.started.wait() - if sd_hash not in self.stream_manager.streams: + if not self.file_manager.started.is_set(): + await self.file_manager.started.wait() + if sd_hash not in self.file_manager.streams: return web.HTTPNotFound() - return await self.stream_manager.stream_partial_content(request, sd_hash) + return await self.file_manager.stream_partial_content(request, sd_hash) async def _process_rpc_call(self, data): args = data.get('params', {}) @@ -1077,7 +1077,7 @@ class Daemon(metaclass=JSONRPCServerType): return results @requires(WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, - STREAM_MANAGER_COMPONENT) + FILE_MANAGER_COMPONENT) async def jsonrpc_get( self, uri, file_name=None, download_directory=None, timeout=None, save_file=None, wallet_id=None): """ @@ -1103,7 +1103,7 @@ class Daemon(metaclass=JSONRPCServerType): if download_directory and not os.path.isdir(download_directory): return {"error": f"specified download directory \"{download_directory}\" does not exist"} try: - stream = await self.stream_manager.download_from_uri( + stream = await self.file_manager.download_from_uri( uri, self.exchange_rate_manager, timeout, file_name, download_directory, save_file=save_file, wallet=wallet ) @@ -1949,7 +1949,7 @@ class Daemon(metaclass=JSONRPCServerType): File management. """ - @requires(STREAM_MANAGER_COMPONENT) + @requires(FILE_MANAGER_COMPONENT) async def jsonrpc_file_list(self, sort=None, reverse=False, comparison=None, wallet_id=None, page=None, page_size=None, **kwargs): """ @@ -1994,7 +1994,7 @@ class Daemon(metaclass=JSONRPCServerType): comparison = comparison or 'eq' paginated = paginate_list( - self.stream_manager.get_filtered_streams(sort, reverse, comparison, **kwargs), page, page_size + self.file_manager.get_filtered_streams(sort, reverse, comparison, **kwargs), page, page_size ) if paginated['items']: receipts = { @@ -2008,7 +2008,7 @@ class Daemon(metaclass=JSONRPCServerType): stream.purchase_receipt = receipts.get(stream.claim_id) return paginated - @requires(STREAM_MANAGER_COMPONENT) + @requires(FILE_MANAGER_COMPONENT) async def jsonrpc_file_set_status(self, status, **kwargs): """ Start or stop downloading a file @@ -2032,12 +2032,12 @@ class Daemon(metaclass=JSONRPCServerType): if status not in ['start', 'stop']: raise Exception('Status must be "start" or "stop".') - streams = self.stream_manager.get_filtered_streams(**kwargs) + streams = self.file_manager.get_filtered_streams(**kwargs) if not streams: raise Exception(f'Unable to find a file for {kwargs}') stream = streams[0] if status == 'start' and not stream.running: - await stream.save_file(node=self.stream_manager.node) + await stream.save_file(node=self.file_manager.node) msg = "Resumed download" elif status == 'stop' and stream.running: await stream.stop() @@ -2049,7 +2049,7 @@ class Daemon(metaclass=JSONRPCServerType): ) return msg - @requires(STREAM_MANAGER_COMPONENT) + @requires(FILE_MANAGER_COMPONENT) async def jsonrpc_file_delete(self, delete_from_download_dir=False, delete_all=False, **kwargs): """ Delete a LBRY file @@ -2081,7 +2081,7 @@ class Daemon(metaclass=JSONRPCServerType): (bool) true if deletion was successful """ - streams = self.stream_manager.get_filtered_streams(**kwargs) + streams = self.file_manager.get_filtered_streams(**kwargs) if len(streams) > 1: if not delete_all: @@ -2098,12 +2098,12 @@ class Daemon(metaclass=JSONRPCServerType): else: for stream in streams: message = f"Deleted file {stream.file_name}" - await self.stream_manager.delete_stream(stream, delete_file=delete_from_download_dir) + await self.file_manager.delete_stream(stream, delete_file=delete_from_download_dir) log.info(message) result = True return result - @requires(STREAM_MANAGER_COMPONENT) + @requires(FILE_MANAGER_COMPONENT) async def jsonrpc_file_save(self, file_name=None, download_directory=None, **kwargs): """ Start saving a file to disk. @@ -2130,7 +2130,7 @@ class Daemon(metaclass=JSONRPCServerType): Returns: {File} """ - streams = self.stream_manager.get_filtered_streams(**kwargs) + streams = self.file_manager.get_filtered_streams(**kwargs) if len(streams) > 1: log.warning("There are %i matching files, use narrower filters to select one", len(streams)) @@ -2905,7 +2905,7 @@ class Daemon(metaclass=JSONRPCServerType): Create, update, abandon, list and inspect your stream claims. """ - @requires(WALLET_COMPONENT, STREAM_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT) + @requires(WALLET_COMPONENT, FILE_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT) async def jsonrpc_publish(self, name, **kwargs): """ Create or replace a stream claim at a given name (use 'stream create/update' for more control). @@ -3027,7 +3027,7 @@ class Daemon(metaclass=JSONRPCServerType): f"to update a specific stream claim." ) - @requires(WALLET_COMPONENT, STREAM_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT) + @requires(WALLET_COMPONENT, FILE_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT) async def jsonrpc_stream_repost(self, name, bid, claim_id, allow_duplicate_name=False, channel_id=None, channel_name=None, channel_account_id=None, account_id=None, wallet_id=None, claim_address=None, funding_account_ids=None, preview=False, blocking=False): @@ -3099,7 +3099,7 @@ class Daemon(metaclass=JSONRPCServerType): return tx - @requires(WALLET_COMPONENT, STREAM_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT) + @requires(WALLET_COMPONENT, FILE_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT) async def jsonrpc_stream_create( self, name, bid, file_path, allow_duplicate_name=False, channel_id=None, channel_name=None, channel_account_id=None, @@ -3237,7 +3237,7 @@ class Daemon(metaclass=JSONRPCServerType): file_stream = None if not preview: - file_stream = await self.stream_manager.create_stream(file_path) + file_stream = await self.file_manager.create_stream(file_path) claim.stream.source.sd_hash = file_stream.sd_hash new_txo.script.generate() @@ -3257,7 +3257,7 @@ class Daemon(metaclass=JSONRPCServerType): return tx - @requires(WALLET_COMPONENT, STREAM_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT) + @requires(WALLET_COMPONENT, FILE_MANAGER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT) async def jsonrpc_stream_update( self, claim_id, bid=None, file_path=None, channel_id=None, channel_name=None, channel_account_id=None, clear_channel=False, @@ -4583,9 +4583,9 @@ class Daemon(metaclass=JSONRPCServerType): """ if not blob_hash or not is_valid_blobhash(blob_hash): return f"Invalid blob hash to delete '{blob_hash}'" - streams = self.stream_manager.get_filtered_streams(sd_hash=blob_hash) + streams = self.file_manager.get_filtered_streams(sd_hash=blob_hash) if streams: - await self.stream_manager.delete_stream(streams[0]) + await self.file_manager.delete_stream(streams[0]) else: await self.blob_manager.delete_blobs([blob_hash]) return "Deleted %s" % blob_hash @@ -4758,7 +4758,7 @@ class Daemon(metaclass=JSONRPCServerType): raise NotImplementedError() - @requires(STREAM_MANAGER_COMPONENT) + @requires(FILE_MANAGER_COMPONENT) async def jsonrpc_file_reflect(self, **kwargs): """ Reflect all the blobs in a file matching the filter criteria @@ -5334,7 +5334,7 @@ class Daemon(metaclass=JSONRPCServerType): results = await self.ledger.resolve(accounts, urls, **kwargs) if self.conf.save_resolved_claims and results: try: - claims = self.stream_manager._convert_to_old_resolve_output(self.wallet_manager, results) + claims = self.file_manager._convert_to_old_resolve_output(self.wallet_manager, results) await self.storage.save_claims_for_resolve([ value for value in claims.values() if 'error' not in value ]) diff --git a/tests/integration/datanetwork/test_file_commands.py b/tests/integration/datanetwork/test_file_commands.py index 1ab06d088..861040f91 100644 --- a/tests/integration/datanetwork/test_file_commands.py +++ b/tests/integration/datanetwork/test_file_commands.py @@ -228,11 +228,11 @@ class FileCommands(CommandTestCase): await self.daemon.jsonrpc_get('lbry://foo') with open(original_path, 'wb') as handle: handle.write(b'some other stuff was there instead') - self.daemon.stream_manager.stop() - await self.daemon.stream_manager.start() + self.daemon.file_manager.stop() + await self.daemon.file_manager.start() await asyncio.wait_for(self.wait_files_to_complete(), timeout=5) # if this hangs, file didn't get set completed # check that internal state got through up to the file list API - stream = self.daemon.stream_manager.get_stream_by_stream_hash(file_info['stream_hash']) + stream = self.daemon.file_manager.get_stream_by_stream_hash(file_info['stream_hash']) file_info = (await self.file_list())[0] self.assertEqual(stream.file_name, file_info['file_name']) # checks if what the API shows is what he have at the very internal level. @@ -255,7 +255,7 @@ class FileCommands(CommandTestCase): resp = await self.out(self.daemon.jsonrpc_get('lbry://foo', timeout=2)) self.assertNotIn('error', resp) self.assertTrue(os.path.isfile(path)) - self.daemon.stream_manager.stop() + self.daemon.file_manager.stop() await asyncio.sleep(0.01, loop=self.loop) # FIXME: this sleep should not be needed self.assertFalse(os.path.isfile(path)) @@ -348,8 +348,8 @@ class FileCommands(CommandTestCase): # restart the daemon and make sure the fee is still there - self.daemon.stream_manager.stop() - await self.daemon.stream_manager.start() + self.daemon.file_manager.stop() + await self.daemon.file_manager.start() self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1) self.assertEqual((await self.daemon.jsonrpc_file_list())['items'][0].content_fee.raw, raw_content_fee) await self.daemon.jsonrpc_file_delete(claim_name='icanpay') diff --git a/tests/integration/datanetwork/test_streaming.py b/tests/integration/datanetwork/test_streaming.py index e6d572e94..856a3c090 100644 --- a/tests/integration/datanetwork/test_streaming.py +++ b/tests/integration/datanetwork/test_streaming.py @@ -21,8 +21,8 @@ def get_random_bytes(n: int) -> bytes: class RangeRequests(CommandTestCase): async def _restart_stream_manager(self): - self.daemon.stream_manager.stop() - await self.daemon.stream_manager.start() + self.daemon.file_manager.stop() + await self.daemon.file_manager.start() return async def _setup_stream(self, data: bytes, save_blobs: bool = True, save_files: bool = False, file_size=0): diff --git a/tests/integration/other/test_cli.py b/tests/integration/other/test_cli.py index 59b629747..459d2171a 100644 --- a/tests/integration/other/test_cli.py +++ b/tests/integration/other/test_cli.py @@ -6,7 +6,7 @@ from lbry.conf import Config from lbry.extras import cli from lbry.extras.daemon.components import ( DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT, - HASH_ANNOUNCER_COMPONENT, STREAM_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT, + HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT, UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT ) from lbry.extras.daemon.daemon import Daemon @@ -21,7 +21,7 @@ class CLIIntegrationTest(AsyncioTestCase): conf.api = 'localhost:5299' conf.components_to_skip = ( DATABASE_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT, - HASH_ANNOUNCER_COMPONENT, STREAM_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT, + HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT, UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, WALLET_SERVER_PAYMENTS_COMPONENT ) Daemon.component_attributes = {} @@ -34,4 +34,4 @@ class CLIIntegrationTest(AsyncioTestCase): with contextlib.redirect_stdout(actual_output): cli.main(["--api", "localhost:5299", "status"]) actual_output = actual_output.getvalue() - self.assertIn("connection_status", actual_output) \ No newline at end of file + self.assertIn("connection_status", actual_output) diff --git a/tests/unit/components/test_component_manager.py b/tests/unit/components/test_component_manager.py index d8d2ed5a9..6738c14e4 100644 --- a/tests/unit/components/test_component_manager.py +++ b/tests/unit/components/test_component_manager.py @@ -26,7 +26,7 @@ class TestComponentManager(AsyncioTestCase): [ components.HashAnnouncerComponent, components.PeerProtocolServerComponent, - components.StreamManagerComponent, + components.FileManagerComponent, components.WalletServerPaymentsComponent ] ] From 27739e036419ead9b3f1b1ee2a4b716bc4693de8 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 28 Jan 2020 21:24:05 -0300 Subject: [PATCH 32/86] fix save from resolve --- lbry/extras/daemon/daemon.py | 24 +- lbry/extras/daemon/storage.py | 6 +- lbry/file/file_manager.py | 205 +++--------------- lbry/stream/managed_stream.py | 11 +- lbry/stream/stream_manager.py | 4 +- lbry/torrent/session.py | 7 +- .../datanetwork/test_file_commands.py | 2 +- tests/unit/stream/test_managed_stream.py | 6 +- 8 files changed, 62 insertions(+), 203 deletions(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index 5432cd79f..fd3153f9e 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -1994,7 +1994,7 @@ class Daemon(metaclass=JSONRPCServerType): comparison = comparison or 'eq' paginated = paginate_list( - self.file_manager.get_filtered_streams(sort, reverse, comparison, **kwargs), page, page_size + self.file_manager.get_filtered(sort, reverse, comparison, **kwargs), page, page_size ) if paginated['items']: receipts = { @@ -2032,12 +2032,12 @@ class Daemon(metaclass=JSONRPCServerType): if status not in ['start', 'stop']: raise Exception('Status must be "start" or "stop".') - streams = self.file_manager.get_filtered_streams(**kwargs) + streams = self.file_manager.get_filtered(**kwargs) if not streams: raise Exception(f'Unable to find a file for {kwargs}') stream = streams[0] if status == 'start' and not stream.running: - await stream.save_file(node=self.file_manager.node) + await stream.save_file() msg = "Resumed download" elif status == 'stop' and stream.running: await stream.stop() @@ -2081,7 +2081,7 @@ class Daemon(metaclass=JSONRPCServerType): (bool) true if deletion was successful """ - streams = self.file_manager.get_filtered_streams(**kwargs) + streams = self.file_manager.get_filtered(**kwargs) if len(streams) > 1: if not delete_all: @@ -2098,7 +2098,7 @@ class Daemon(metaclass=JSONRPCServerType): else: for stream in streams: message = f"Deleted file {stream.file_name}" - await self.file_manager.delete_stream(stream, delete_file=delete_from_download_dir) + await self.file_manager.delete(stream, delete_file=delete_from_download_dir) log.info(message) result = True return result @@ -2130,7 +2130,7 @@ class Daemon(metaclass=JSONRPCServerType): Returns: {File} """ - streams = self.file_manager.get_filtered_streams(**kwargs) + streams = self.file_manager.get_filtered(**kwargs) if len(streams) > 1: log.warning("There are %i matching files, use narrower filters to select one", len(streams)) @@ -4583,9 +4583,9 @@ class Daemon(metaclass=JSONRPCServerType): """ if not blob_hash or not is_valid_blobhash(blob_hash): return f"Invalid blob hash to delete '{blob_hash}'" - streams = self.file_manager.get_filtered_streams(sd_hash=blob_hash) + streams = self.file_manager.get_filtered(sd_hash=blob_hash) if streams: - await self.file_manager.delete_stream(streams[0]) + await self.file_manager.delete(streams[0]) else: await self.blob_manager.delete_blobs([blob_hash]) return "Deleted %s" % blob_hash @@ -5334,10 +5334,10 @@ class Daemon(metaclass=JSONRPCServerType): results = await self.ledger.resolve(accounts, urls, **kwargs) if self.conf.save_resolved_claims and results: try: - claims = self.file_manager._convert_to_old_resolve_output(self.wallet_manager, results) - await self.storage.save_claims_for_resolve([ - value for value in claims.values() if 'error' not in value - ]) + await self.storage.save_claim_from_output( + self.ledger, + *(result for result in results.values() if isinstance(result, Output)) + ) except DecodeError: pass return results diff --git a/lbry/extras/daemon/storage.py b/lbry/extras/daemon/storage.py index 426985a48..8a03a456c 100644 --- a/lbry/extras/daemon/storage.py +++ b/lbry/extras/daemon/storage.py @@ -727,7 +727,7 @@ class SQLiteStorage(SQLiteMixin): if claim_id_to_supports: await self.save_supports(claim_id_to_supports) - def save_claim_from_output(self, ledger, output: Output): + def save_claim_from_output(self, ledger, *outputs: Output): return self.save_claims([{ "claim_id": output.claim_id, "name": output.claim_name, @@ -736,9 +736,9 @@ class SQLiteStorage(SQLiteMixin): "txid": output.tx_ref.id, "nout": output.position, "value": output.claim, - "height": -1, + "height": output.tx_ref.height, "claim_sequence": -1, - }]) + } for output in outputs]) def save_claims_for_resolve(self, claim_infos): to_save = {} diff --git a/lbry/file/file_manager.py b/lbry/file/file_manager.py index 443c94a69..871b94d47 100644 --- a/lbry/file/file_manager.py +++ b/lbry/file/file_manager.py @@ -40,14 +40,28 @@ class FileManager: self.storage = storage self.analytics_manager = analytics_manager self.source_managers: typing.Dict[str, SourceManager] = {} + self.started = asyncio.Event() + + @property + def streams(self): + return self.source_managers['stream']._sources + + async def create_stream(self, file_path: str, key: Optional[bytes] = None, **kwargs) -> ManagedDownloadSource: + if 'stream' in self.source_managers: + return await self.source_managers['stream'].create(file_path, key, **kwargs) + raise NotImplementedError async def start(self): await asyncio.gather(*(source_manager.start() for source_manager in self.source_managers.values())) + for manager in self.source_managers.values(): + await manager.started.wait() + self.started.set() def stop(self): - while self.source_managers: - _, source_manager = self.source_managers.popitem() - source_manager.stop() + for manager in self.source_managers.values(): + # fixme: pop or not? + manager.stop() + self.started.clear() @cache_concurrent async def download_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager', @@ -130,7 +144,7 @@ class FileManager: await existing_for_claim_id[0].start(node=self.node, timeout=timeout, save_now=save_file) if not existing_for_claim_id[0].output_file_exists and (save_file or file_name or download_directory): await existing_for_claim_id[0].save_file( - file_name=file_name, download_directory=download_directory, node=self.node + file_name=file_name, download_directory=download_directory ) to_replace = existing_for_claim_id[0] @@ -139,10 +153,10 @@ class FileManager: log.info("already have stream for %s", uri) if save_file and updated_stream.output_file_exists: save_file = False - await updated_stream.start(node=self.node, timeout=timeout, save_now=save_file) + await updated_stream.start(timeout=timeout, save_now=save_file) if not updated_stream.output_file_exists and (save_file or file_name or download_directory): await updated_stream.save_file( - file_name=file_name, download_directory=download_directory, node=self.node + file_name=file_name, download_directory=download_directory ) return updated_stream @@ -152,7 +166,7 @@ class FileManager: #################### if not to_replace and txo.has_price and not txo.purchase_receipt: - payment = await manager.create_purchase_transaction( + payment = await self.wallet_manager.create_purchase_transaction( wallet.accounts, txo, exchange_rate_manager ) @@ -171,7 +185,7 @@ class FileManager: log.info("starting download for %s", uri) before_download = self.loop.time() - await stream.start(source_manager.node, timeout) + await stream.start(timeout, save_file) #################### # success case: delete to_replace if applicable, broadcast fee payment @@ -190,7 +204,7 @@ class FileManager: await self.storage.save_content_claim(stream.stream_hash, outpoint) if save_file: - await asyncio.wait_for(stream.save_file(node=source_manager.node), timeout - (self.loop.time() - before_download), + await asyncio.wait_for(stream.save_file(), timeout - (self.loop.time() - before_download), loop=self.loop) return stream except asyncio.TimeoutError: @@ -235,7 +249,7 @@ class FileManager: ) async def stream_partial_content(self, request: Request, sd_hash: str): - return await self._sources[sd_hash].stream_file(request, self.node) + return await self.source_managers['stream'].stream_partial_content(request, sd_hash) def get_filtered(self, *args, **kwargs) -> typing.List[ManagedDownloadSource]: """ @@ -246,7 +260,7 @@ class FileManager: :param comparison: comparison operator used for filtering :param search_by: fields and values to filter by """ - return sum(*(manager.get_filtered(*args, **kwargs) for manager in self.source_managers.values()), []) + return sum((manager.get_filtered(*args, **kwargs) for manager in self.source_managers.values()), []) async def _check_update_or_replace( self, outpoint: str, claim_id: str, claim: Claim @@ -271,169 +285,6 @@ class FileManager: return None, existing_for_claim_id[0] return None, None - - - # @cache_concurrent - # async def download_from_uri(self, uri, exchange_rate_manager: 'ExchangeRateManager', - # timeout: Optional[float] = None, file_name: Optional[str] = None, - # download_directory: Optional[str] = None, - # save_file: Optional[bool] = None, resolve_timeout: float = 3.0, - # wallet: Optional['Wallet'] = None) -> ManagedDownloadSource: - # wallet = wallet or self.wallet_manager.default_wallet - # timeout = timeout or self.config.download_timeout - # start_time = self.loop.time() - # resolved_time = None - # stream = None - # txo: Optional[Output] = None - # error = None - # outpoint = None - # if save_file is None: - # save_file = self.config.save_files - # if file_name and not save_file: - # save_file = True - # if save_file: - # download_directory = download_directory or self.config.download_dir - # else: - # download_directory = None - # - # payment = None - # try: - # # resolve the claim - # if not URL.parse(uri).has_stream: - # raise ResolveError("cannot download a channel claim, specify a /path") - # try: - # response = await asyncio.wait_for( - # self.wallet_manager.ledger.resolve(wallet.accounts, [uri]), - # resolve_timeout - # ) - # resolved_result = {} - # for url, txo in response.items(): - # if isinstance(txo, Output): - # tx_height = txo.tx_ref.height - # best_height = self.wallet_manager.ledger.headers.height - # resolved_result[url] = { - # 'name': txo.claim_name, - # 'value': txo.claim, - # 'protobuf': binascii.hexlify(txo.claim.to_bytes()), - # 'claim_id': txo.claim_id, - # 'txid': txo.tx_ref.id, - # 'nout': txo.position, - # 'amount': dewies_to_lbc(txo.amount), - # 'effective_amount': txo.meta.get('effective_amount', 0), - # 'height': tx_height, - # 'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height, - # 'claim_sequence': -1, - # 'address': txo.get_address(self.wallet_manager.ledger), - # 'valid_at_height': txo.meta.get('activation_height', None), - # 'timestamp': self.wallet_manager.ledger.headers[tx_height]['timestamp'], - # 'supports': [] - # } - # else: - # resolved_result[url] = txo - # except asyncio.TimeoutError: - # raise ResolveTimeoutError(uri) - # except Exception as err: - # if isinstance(err, asyncio.CancelledError): - # raise - # log.exception("Unexpected error resolving stream:") - # raise ResolveError(f"Unexpected error resolving stream: {str(err)}") - # await self.storage.save_claims_for_resolve([ - # value for value in resolved_result.values() if 'error' not in value - # ]) - # - # resolved = resolved_result.get(uri, {}) - # resolved = resolved if 'value' in resolved else resolved.get('claim') - # if not resolved: - # raise ResolveError(f"Failed to resolve stream at '{uri}'") - # if 'error' in resolved: - # raise ResolveError(f"error resolving stream: {resolved['error']}") - # txo = response[uri] - # - # claim = Claim.from_bytes(binascii.unhexlify(resolved['protobuf'])) - # outpoint = f"{resolved['txid']}:{resolved['nout']}" - # resolved_time = self.loop.time() - start_time - # - # # resume or update an existing stream, if the stream changed: download it and delete the old one after - # updated_stream, to_replace = await self._check_update_or_replace(outpoint, resolved['claim_id'], claim) - # if updated_stream: - # log.info("already have stream for %s", uri) - # if save_file and updated_stream.output_file_exists: - # save_file = False - # await updated_stream.start(node=self.node, timeout=timeout, save_now=save_file) - # if not updated_stream.output_file_exists and (save_file or file_name or download_directory): - # await updated_stream.save_file( - # file_name=file_name, download_directory=download_directory, node=self.node - # ) - # return updated_stream - # - # if not to_replace and txo.has_price and not txo.purchase_receipt: - # payment = await manager.create_purchase_transaction( - # wallet.accounts, txo, exchange_rate_manager - # ) - # - # stream = ManagedStream( - # self.loop, self.config, self.blob_manager, claim.stream.source.sd_hash, download_directory, - # file_name, ManagedStream.STATUS_RUNNING, content_fee=payment, - # analytics_manager=self.analytics_manager - # ) - # log.info("starting download for %s", uri) - # - # before_download = self.loop.time() - # await stream.start(self.node, timeout) - # stream.set_claim(resolved, claim) - # if to_replace: # delete old stream now that the replacement has started downloading - # await self.delete(to_replace) - # - # if payment is not None: - # await manager.broadcast_or_release(payment) - # payment = None # to avoid releasing in `finally` later - # log.info("paid fee of %s for %s", dewies_to_lbc(stream.content_fee.outputs[0].amount), uri) - # await self.storage.save_content_fee(stream.stream_hash, stream.content_fee) - # - # self._sources[stream.sd_hash] = stream - # self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream) - # await self.storage.save_content_claim(stream.stream_hash, outpoint) - # if save_file: - # await asyncio.wait_for(stream.save_file(node=self.node), timeout - (self.loop.time() - before_download), - # loop=self.loop) - # return stream - # except asyncio.TimeoutError: - # error = DownloadDataTimeoutError(stream.sd_hash) - # raise error - # except Exception as err: # forgive data timeout, don't delete stream - # expected = (DownloadSDTimeoutError, DownloadDataTimeoutError, InsufficientFundsError, - # KeyFeeAboveMaxAllowedError) - # if isinstance(err, expected): - # log.warning("Failed to download %s: %s", uri, str(err)) - # elif isinstance(err, asyncio.CancelledError): - # pass - # else: - # log.exception("Unexpected error downloading stream:") - # error = err - # raise - # finally: - # if payment is not None: - # # payment is set to None after broadcasting, if we're here an exception probably happened - # await manager.ledger.release_tx(payment) - # if self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or - # stream.downloader.time_to_first_bytes))): - # server = self.wallet_manager.ledger.network.client.server - # self.loop.create_task( - # self.analytics_manager.send_time_to_first_bytes( - # resolved_time, self.loop.time() - start_time, None if not stream else stream.download_id, - # uri, outpoint, - # None if not stream else len(stream.downloader.blob_downloader.active_connections), - # None if not stream else len(stream.downloader.blob_downloader.scores), - # None if not stream else len(stream.downloader.blob_downloader.connection_failures), - # False if not stream else stream.downloader.added_fixed_peers, - # self.config.fixed_peer_delay if not stream else stream.downloader.fixed_peers_delay, - # None if not stream else stream.sd_hash, - # None if not stream else stream.downloader.time_to_descriptor, - # None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].blob_hash, - # None if not (stream and stream.descriptor) else stream.descriptor.blobs[0].length, - # None if not stream else stream.downloader.time_to_first_bytes, - # None if not error else error.__class__.__name__, - # None if not error else str(error), - # None if not server else f"{server[0]}:{server[1]}" - # ) - # ) + async def delete(self, source: ManagedDownloadSource, delete_file=False): + for manager in self.source_managers.values(): + return await manager.delete(source, delete_file) diff --git a/lbry/stream/managed_stream.py b/lbry/stream/managed_stream.py index c530550c6..34696c442 100644 --- a/lbry/stream/managed_stream.py +++ b/lbry/stream/managed_stream.py @@ -143,7 +143,7 @@ class ManagedStream(ManagedDownloadSource): # return cls(loop, config, blob_manager, descriptor.sd_hash, os.path.dirname(file_path), # os.path.basename(file_path), status=cls.STATUS_FINISHED, rowid=row_id, descriptor=descriptor) - async def start(self, node: Optional['Node'] = None, timeout: Optional[float] = None, + async def start(self, timeout: Optional[float] = None, save_now: bool = False): timeout = timeout or self.config.download_timeout if self._running.is_set(): @@ -151,7 +151,7 @@ class ManagedStream(ManagedDownloadSource): log.info("start downloader for stream (sd hash: %s)", self.sd_hash) self._running.set() try: - await asyncio.wait_for(self.downloader.start(node), timeout, loop=self.loop) + await asyncio.wait_for(self.downloader.start(), timeout, loop=self.loop) except asyncio.TimeoutError: self._running.clear() raise DownloadSDTimeoutError(self.sd_hash) @@ -161,6 +161,11 @@ class ManagedStream(ManagedDownloadSource): self.delayed_stop_task = self.loop.create_task(self._delayed_stop()) if not await self.blob_manager.storage.file_exists(self.sd_hash): if save_now: + if not self._file_name: + self._file_name = await get_next_available_file_name( + self.loop, self.download_directory, + self._file_name or sanitize_file_name(self.descriptor.suggested_file_name) + ) file_name, download_dir = self._file_name, self.download_directory else: file_name, download_dir = None, None @@ -286,7 +291,7 @@ class ManagedStream(ManagedDownloadSource): async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None, node: Optional['Node'] = None): - await self.start(node) + await self.start() if self.file_output_task and not self.file_output_task.done(): # cancel an already running save task self.file_output_task.cancel() self.download_directory = download_directory or self.download_directory or self.config.download_dir diff --git a/lbry/stream/stream_manager.py b/lbry/stream/stream_manager.py index d5fc9202b..caee74b73 100644 --- a/lbry/stream/stream_manager.py +++ b/lbry/stream/stream_manager.py @@ -160,8 +160,8 @@ class StreamManager(SourceManager): log.info("no DHT node given, resuming downloads trusting that we can contact reflector") if to_resume_saving: log.info("Resuming saving %i files", len(to_resume_saving)) - self.resume_saving_task = self.loop.create_task(asyncio.gather( - *(self._sources[sd_hash].save_file(file_name, download_directory, node=self.node) + self.resume_saving_task = asyncio.ensure_future(asyncio.gather( + *(self._sources[sd_hash].save_file(file_name, download_directory) for (file_name, download_directory, sd_hash) in to_resume_saving), loop=self.loop )) diff --git a/lbry/torrent/session.py b/lbry/torrent/session.py index 01e4cafa3..5214042e7 100644 --- a/lbry/torrent/session.py +++ b/lbry/torrent/session.py @@ -93,7 +93,8 @@ class TorrentSession: self._executor, libtorrent.session, settings ) await self._loop.run_in_executor( - self._executor, self._session.add_dht_router, "router.utorrent.com", 6881 + self._executor, + lambda: self._session.add_dht_router("router.utorrent.com", 6881) ) self._loop.create_task(self.process_alerts()) @@ -110,11 +111,11 @@ class TorrentSession: async def pause(self): state = await self._loop.run_in_executor( - self._executor, self._session.save_state + self._executor, lambda: self._session.save_state() ) # print(f"state:\n{state}") await self._loop.run_in_executor( - self._executor, self._session.pause + self._executor, lambda: self._session.pause() ) async def resume(self): diff --git a/tests/integration/datanetwork/test_file_commands.py b/tests/integration/datanetwork/test_file_commands.py index 861040f91..542be228c 100644 --- a/tests/integration/datanetwork/test_file_commands.py +++ b/tests/integration/datanetwork/test_file_commands.py @@ -232,7 +232,7 @@ class FileCommands(CommandTestCase): await self.daemon.file_manager.start() await asyncio.wait_for(self.wait_files_to_complete(), timeout=5) # if this hangs, file didn't get set completed # check that internal state got through up to the file list API - stream = self.daemon.file_manager.get_stream_by_stream_hash(file_info['stream_hash']) + stream = self.daemon.file_manager.get_filtered(stream_hash=file_info['stream_hash'])[0] file_info = (await self.file_list())[0] self.assertEqual(stream.file_name, file_info['file_name']) # checks if what the API shows is what he have at the very internal level. diff --git a/tests/unit/stream/test_managed_stream.py b/tests/unit/stream/test_managed_stream.py index dbdfa5157..64e3e3ea2 100644 --- a/tests/unit/stream/test_managed_stream.py +++ b/tests/unit/stream/test_managed_stream.py @@ -76,7 +76,8 @@ class TestManagedStream(BlobExchangeTestBase): return q2, self.loop.create_task(_task()) mock_node.accumulate_peers = mock_accumulate_peers or _mock_accumulate_peers - await self.stream.save_file(node=mock_node) + self.stream.node = mock_node + await self.stream.save_file() await self.stream.finished_write_attempt.wait() self.assertTrue(os.path.isfile(self.stream.full_path)) if stop_when_done: @@ -123,7 +124,8 @@ class TestManagedStream(BlobExchangeTestBase): mock_node.accumulate_peers = _mock_accumulate_peers - await self.stream.save_file(node=mock_node) + self.stream.node = mock_node + await self.stream.save_file() await self.stream.finished_writing.wait() self.assertTrue(os.path.isfile(self.stream.full_path)) with open(self.stream.full_path, 'rb') as f: From 2089059792149b9be0a45fae9e432fa832ec7275 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 28 Jan 2020 22:37:52 -0300 Subject: [PATCH 33/86] pylint --- lbry/extras/daemon/components.py | 6 +- lbry/file/file_manager.py | 48 +++------------ lbry/file/source.py | 16 +---- lbry/file/source_manager.py | 20 +++--- lbry/stream/managed_stream.py | 5 +- lbry/stream/stream_manager.py | 78 +++--------------------- lbry/torrent/session.py | 60 +++++++++--------- lbry/torrent/torrent.py | 16 ++--- tests/unit/stream/test_stream_manager.py | 3 +- 9 files changed, 67 insertions(+), 185 deletions(-) diff --git a/lbry/extras/daemon/components.py b/lbry/extras/daemon/components.py index ff6de3c61..046faca2a 100644 --- a/lbry/extras/daemon/components.py +++ b/lbry/extras/daemon/components.py @@ -24,10 +24,8 @@ from lbry.extras.daemon.storage import SQLiteStorage from lbry.wallet import WalletManager from lbry.wallet.usage_payment import WalletServerPayer try: - import libtorrent from lbry.torrent.session import TorrentSession except ImportError: - libtorrent = None TorrentSession = None log = logging.getLogger(__name__) @@ -343,7 +341,7 @@ class FileManagerComponent(Component): if not self.file_manager: return return { - 'managed_files': len(self.file_manager._sources), + 'managed_files': len(self.file_manager.get_filtered()), } async def start(self): @@ -386,7 +384,7 @@ class TorrentComponent(Component): } async def start(self): - if libtorrent: + if TorrentSession: self.torrent_session = TorrentSession(asyncio.get_event_loop(), None) await self.torrent_session.bind() # TODO: specify host/port diff --git a/lbry/file/file_manager.py b/lbry/file/file_manager.py index 871b94d47..8da9a5619 100644 --- a/lbry/file/file_manager.py +++ b/lbry/file/file_manager.py @@ -1,38 +1,28 @@ -import time import asyncio -import binascii import logging import typing from typing import Optional from aiohttp.web import Request -from lbry.error import ResolveError, InvalidStreamDescriptorError, DownloadSDTimeoutError, InsufficientFundsError +from lbry.error import ResolveError, DownloadSDTimeoutError, InsufficientFundsError from lbry.error import ResolveTimeoutError, DownloadDataTimeoutError, KeyFeeAboveMaxAllowedError from lbry.stream.managed_stream import ManagedStream from lbry.utils import cache_concurrent -from lbry.schema.claim import Claim from lbry.schema.url import URL from lbry.wallet.dewies import dewies_to_lbc -from lbry.wallet.transaction import Output from lbry.file.source_manager import SourceManager from lbry.file.source import ManagedDownloadSource if typing.TYPE_CHECKING: from lbry.conf import Config from lbry.extras.daemon.analytics import AnalyticsManager from lbry.extras.daemon.storage import SQLiteStorage - from lbry.wallet import LbryWalletManager + from lbry.wallet import WalletManager from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager log = logging.getLogger(__name__) -def path_or_none(p) -> Optional[str]: - if not p: - return - return binascii.unhexlify(p).decode() - - class FileManager: - def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', wallet_manager: 'LbryWalletManager', + def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', wallet_manager: 'WalletManager', storage: 'SQLiteStorage', analytics_manager: Optional['AnalyticsManager'] = None): self.loop = loop self.config = config @@ -141,8 +131,9 @@ class FileManager: log.info("claim contains an update to a stream we have, downloading it") if save_file and existing_for_claim_id[0].output_file_exists: save_file = False - await existing_for_claim_id[0].start(node=self.node, timeout=timeout, save_now=save_file) - if not existing_for_claim_id[0].output_file_exists and (save_file or file_name or download_directory): + await existing_for_claim_id[0].start(timeout=timeout, save_now=save_file) + if not existing_for_claim_id[0].output_file_exists and ( + save_file or file_name or download_directory): await existing_for_claim_id[0].save_file( file_name=file_name, download_directory=download_directory ) @@ -176,8 +167,8 @@ class FileManager: if not claim.stream.source.bt_infohash: stream = ManagedStream( - self.loop, self.config, source_manager.blob_manager, claim.stream.source.sd_hash, download_directory, - file_name, ManagedStream.STATUS_RUNNING, content_fee=payment, + self.loop, self.config, source_manager.blob_manager, claim.stream.source.sd_hash, + download_directory, file_name, ManagedStream.STATUS_RUNNING, content_fee=payment, analytics_manager=self.analytics_manager ) else: @@ -262,29 +253,6 @@ class FileManager: """ return sum((manager.get_filtered(*args, **kwargs) for manager in self.source_managers.values()), []) - async def _check_update_or_replace( - self, outpoint: str, claim_id: str, claim: Claim - ) -> typing.Tuple[Optional[ManagedDownloadSource], Optional[ManagedDownloadSource]]: - existing = self.get_filtered(outpoint=outpoint) - if existing: - return existing[0], None - existing = self.get_filtered(sd_hash=claim.stream.source.sd_hash) - if existing and existing[0].claim_id != claim_id: - raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {claim_id}") - if existing: - log.info("claim contains a metadata only update to a stream we have") - await self.storage.save_content_claim( - existing[0].stream_hash, outpoint - ) - await self._update_content_claim(existing[0]) - return existing[0], None - else: - existing_for_claim_id = self.get_filtered(claim_id=claim_id) - if existing_for_claim_id: - log.info("claim contains an update to a stream we have, downloading it") - return None, existing_for_claim_id[0] - return None, None - async def delete(self, source: ManagedDownloadSource, delete_file=False): for manager in self.source_managers.values(): return await manager.delete(source, delete_file) diff --git a/lbry/file/source.py b/lbry/file/source.py index 05d8fb35d..81ab0ee82 100644 --- a/lbry/file/source.py +++ b/lbry/file/source.py @@ -16,20 +16,6 @@ if typing.TYPE_CHECKING: log = logging.getLogger(__name__) -# def _get_next_available_file_name(download_directory: str, file_name: str) -> str: -# base_name, ext = os.path.splitext(os.path.basename(file_name)) -# i = 0 -# while os.path.isfile(os.path.join(download_directory, file_name)): -# i += 1 -# file_name = "%s_%i%s" % (base_name, i, ext) -# -# return file_name -# -# -# async def get_next_available_file_name(loop: asyncio.AbstractEventLoop, download_directory: str, file_name: str) -> str: -# return await loop.run_in_executor(None, _get_next_available_file_name, download_directory, file_name) - - class ManagedDownloadSource: STATUS_RUNNING = "running" STATUS_STOPPED = "stopped" @@ -71,7 +57,7 @@ class ManagedDownloadSource: # iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> 'ManagedDownloadSource': # raise NotImplementedError() - async def start(self, timeout: Optional[float] = None): + async def start(self, timeout: Optional[float] = None, save_now: Optional[bool] = False): raise NotImplementedError() async def stop(self, finished: bool = False): diff --git a/lbry/file/source_manager.py b/lbry/file/source_manager.py index 56ba5fd5f..9eada3cca 100644 --- a/lbry/file/source_manager.py +++ b/lbry/file/source_manager.py @@ -1,6 +1,5 @@ import os import asyncio -import binascii import logging import typing from typing import Optional @@ -12,7 +11,7 @@ if typing.TYPE_CHECKING: log = logging.getLogger(__name__) -comparison_operators = { +COMPARISON_OPERATORS = { 'eq': lambda a, b: a == b, 'ne': lambda a, b: a != b, 'g': lambda a, b: a > b, @@ -22,12 +21,6 @@ comparison_operators = { } -def path_or_none(p) -> Optional[str]: - if not p: - return - return binascii.unhexlify(p).decode() - - class SourceManager: filter_fields = { 'rowid', @@ -77,10 +70,11 @@ class SourceManager: source.stop_tasks() self.started.clear() - async def create(self, file_path: str, key: Optional[bytes] = None, **kw) -> ManagedDownloadSource: + async def create(self, file_path: str, key: Optional[bytes] = None, + iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedDownloadSource: raise NotImplementedError() - async def _delete(self, source: ManagedDownloadSource): + async def _delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False): raise NotImplementedError() async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False): @@ -101,11 +95,11 @@ class SourceManager: """ if sort_by and sort_by not in self.filter_fields: raise ValueError(f"'{sort_by}' is not a valid field to sort by") - if comparison and comparison not in comparison_operators: + if comparison and comparison not in COMPARISON_OPERATORS: raise ValueError(f"'{comparison}' is not a valid comparison") if 'full_status' in search_by: del search_by['full_status'] - for search in search_by.keys(): + for search in search_by: if search not in self.filter_fields: raise ValueError(f"'{search}' is not a valid search operation") if search_by: @@ -113,7 +107,7 @@ class SourceManager: sources = [] for stream in self._sources.values(): for search, val in search_by.items(): - if comparison_operators[comparison](getattr(stream, search), val): + if COMPARISON_OPERATORS[comparison](getattr(stream, search), val): sources.append(stream) break else: diff --git a/lbry/stream/managed_stream.py b/lbry/stream/managed_stream.py index 34696c442..2e00a8d65 100644 --- a/lbry/stream/managed_stream.py +++ b/lbry/stream/managed_stream.py @@ -6,7 +6,6 @@ import logging import binascii from typing import Optional from aiohttp.web import Request, StreamResponse, HTTPRequestRangeNotSatisfiable -from lbry.utils import generate_id from lbry.error import DownloadSDTimeoutError from lbry.schema.mime_types import guess_media_type from lbry.stream.downloader import StreamDownloader @@ -21,7 +20,6 @@ if typing.TYPE_CHECKING: from lbry.schema.claim import Claim from lbry.blob.blob_manager import BlobManager from lbry.blob.blob_info import BlobInfo - from lbry.extras.daemon.storage import SQLiteStorage from lbry.dht.node import Node from lbry.extras.daemon.analytics import AnalyticsManager from lbry.wallet.transaction import Transaction @@ -289,8 +287,7 @@ class ManagedStream(ManagedDownloadSource): self.saving.clear() self.finished_write_attempt.set() - async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None, - node: Optional['Node'] = None): + async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None): await self.start() if self.file_output_task and not self.file_output_task.done(): # cancel an already running save task self.file_output_task.cancel() diff --git a/lbry/stream/stream_manager.py b/lbry/stream/stream_manager.py index caee74b73..ad232b28b 100644 --- a/lbry/stream/stream_manager.py +++ b/lbry/stream/stream_manager.py @@ -23,24 +23,10 @@ if typing.TYPE_CHECKING: log = logging.getLogger(__name__) -SET_FILTER_FIELDS = { - "claim_ids": "claim_id", - "channel_claim_ids": "channel_claim_id", - "outpoints": "outpoint" -} -COMPARISON_OPERATORS = { - 'eq': lambda a, b: a == b, - 'ne': lambda a, b: a != b, - 'g': lambda a, b: a > b, - 'l': lambda a, b: a < b, - 'ge': lambda a, b: a >= b, - 'le': lambda a, b: a <= b, - 'in': lambda a, b: a in b -} -def path_or_none(p) -> Optional[str]: - if not p: +def path_or_none(encoded_path) -> Optional[str]: + if not encoded_path: return - return binascii.unhexlify(p).decode() + return binascii.unhexlify(encoded_path).decode() class StreamManager(SourceManager): @@ -235,60 +221,10 @@ class StreamManager(SourceManager): await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False) await self.storage.delete(stream.descriptor) - def get_filtered_streams(self, sort_by: Optional[str] = None, reverse: Optional[bool] = False, - comparison: Optional[str] = None, - **search_by) -> typing.List[ManagedStream]: - """ - Get a list of filtered and sorted ManagedStream objects - - :param sort_by: field to sort by - :param reverse: reverse sorting - :param comparison: comparison operator used for filtering - :param search_by: fields and values to filter by - """ - if sort_by and sort_by not in FILTER_FIELDS: - raise ValueError(f"'{sort_by}' is not a valid field to sort by") - if comparison and comparison not in COMPARISON_OPERATORS: - raise ValueError(f"'{comparison}' is not a valid comparison") - if 'full_status' in search_by: - del search_by['full_status'] - - for search in search_by: - if search not in FILTER_FIELDS: - raise ValueError(f"'{search}' is not a valid search operation") - - compare_sets = {} - if isinstance(search_by.get('claim_id'), list): - compare_sets['claim_ids'] = search_by.pop('claim_id') - if isinstance(search_by.get('outpoint'), list): - compare_sets['outpoints'] = search_by.pop('outpoint') - if isinstance(search_by.get('channel_claim_id'), list): - compare_sets['channel_claim_ids'] = search_by.pop('channel_claim_id') - - if search_by: - comparison = comparison or 'eq' - streams = [] - for stream in self.streams.values(): - matched = False - for set_search, val in compare_sets.items(): - if COMPARISON_OPERATORS[comparison](getattr(stream, SET_FILTER_FIELDS[set_search]), val): - streams.append(stream) - matched = True - break - if matched: - continue - for search, val in search_by.items(): - this_stream = getattr(stream, search) - if COMPARISON_OPERATORS[comparison](this_stream, val): - streams.append(stream) - break - else: - streams = list(self.streams.values()) - if sort_by: - streams.sort(key=lambda s: getattr(s, sort_by)) - if reverse: - streams.reverse() - return streams + async def _delete(self, source: ManagedStream, delete_file: Optional[bool] = False): + blob_hashes = [source.sd_hash] + [b.blob_hash for b in source.descriptor.blobs[:-1]] + await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False) + await self.storage.delete_stream(source.descriptor) async def stream_partial_content(self, request: Request, sd_hash: str): return await self._sources[sd_hash].stream_file(request, self.node) diff --git a/lbry/torrent/session.py b/lbry/torrent/session.py index 5214042e7..294a2cba5 100644 --- a/lbry/torrent/session.py +++ b/lbry/torrent/session.py @@ -4,29 +4,29 @@ import libtorrent NOTIFICATION_MASKS = [ - "error", - "peer", - "port_mapping", - "storage", - "tracker", - "debug", - "status", - "progress", - "ip_block", - "dht", - "stats", - "session_log", - "torrent_log", - "peer_log", - "incoming_request", - "dht_log", - "dht_operation", - "port_mapping_log", - "picker_log", - "file_progress", - "piece_progress", - "upload", - "block_progress" + "error", + "peer", + "port_mapping", + "storage", + "tracker", + "debug", + "status", + "progress", + "ip_block", + "dht", + "stats", + "session_log", + "torrent_log", + "peer_log", + "incoming_request", + "dht_log", + "dht_operation", + "port_mapping_log", + "picker_log", + "file_progress", + "piece_progress", + "upload", + "block_progress" ] @@ -90,11 +90,12 @@ class TorrentSession: 'enable_incoming_tcp': True } self._session = await self._loop.run_in_executor( - self._executor, libtorrent.session, settings + self._executor, libtorrent.session, settings # pylint: disable=c-extension-no-member ) await self._loop.run_in_executor( self._executor, - lambda: self._session.add_dht_router("router.utorrent.com", 6881) + # lambda necessary due boost functions raising errors when asyncio inspects them. try removing later + lambda: self._session.add_dht_router("router.utorrent.com", 6881) # pylint: disable=unnecessary-lambda ) self._loop.create_task(self.process_alerts()) @@ -110,12 +111,11 @@ class TorrentSession: await asyncio.sleep(1, loop=self._loop) async def pause(self): - state = await self._loop.run_in_executor( - self._executor, lambda: self._session.save_state() - ) - # print(f"state:\n{state}") await self._loop.run_in_executor( - self._executor, lambda: self._session.pause() + self._executor, lambda: self._session.save_state() # pylint: disable=unnecessary-lambda + ) + await self._loop.run_in_executor( + self._executor, lambda: self._session.pause() # pylint: disable=unnecessary-lambda ) async def resume(self): diff --git a/lbry/torrent/torrent.py b/lbry/torrent/torrent.py index bbbc487bf..04a8544c7 100644 --- a/lbry/torrent/torrent.py +++ b/lbry/torrent/torrent.py @@ -18,17 +18,17 @@ class TorrentInfo: self.total_size = total_size @classmethod - def from_libtorrent_info(cls, ti): + def from_libtorrent_info(cls, torrent_info): return cls( - ti.nodes(), tuple( + torrent_info.nodes(), tuple( { 'url': web_seed['url'], 'type': web_seed['type'], 'auth': web_seed['auth'] - } for web_seed in ti.web_seeds() + } for web_seed in torrent_info.web_seeds() ), tuple( - (tracker.url, tracker.tier) for tracker in ti.trackers() - ), ti.total_size() + (tracker.url, tracker.tier) for tracker in torrent_info.trackers() + ), torrent_info.total_size() ) @@ -41,9 +41,11 @@ class Torrent: def _threaded_update_status(self): status = self._handle.status() if not status.is_seeding: - log.info('%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d) %s' % ( + log.info( + '%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d) %s', status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000, - status.num_peers, status.state)) + status.num_peers, status.state + ) elif not self.finished.is_set(): self.finished.set() diff --git a/tests/unit/stream/test_stream_manager.py b/tests/unit/stream/test_stream_manager.py index e33064503..b14f63de6 100644 --- a/tests/unit/stream/test_stream_manager.py +++ b/tests/unit/stream/test_stream_manager.py @@ -302,7 +302,8 @@ class TestStreamManager(BlobExchangeTestBase): ) self.assertEqual(stored_status, "stopped") - await stream.save_file(node=self.stream_manager.node) + stream.node = self.stream_manager.node + await stream.save_file() await stream.finished_writing.wait() await asyncio.sleep(0, loop=self.loop) self.assertTrue(stream.finished) From e888e69d4dc3cba0b16fdde953e74a494f96e976 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 29 Jan 2020 13:49:14 -0300 Subject: [PATCH 34/86] fix unit tests --- lbry/file/file_manager.py | 6 ++- lbry/stream/downloader.py | 4 +- lbry/stream/stream_manager.py | 4 ++ .../unit/components/test_component_manager.py | 23 +++++----- tests/unit/stream/test_managed_stream.py | 5 +-- tests/unit/stream/test_reflector.py | 2 +- tests/unit/stream/test_stream_manager.py | 42 +++++++++++-------- 7 files changed, 50 insertions(+), 36 deletions(-) diff --git a/lbry/file/file_manager.py b/lbry/file/file_manager.py index 8da9a5619..765cd1b53 100644 --- a/lbry/file/file_manager.py +++ b/lbry/file/file_manager.py @@ -93,10 +93,10 @@ class FileManager: raise log.exception("Unexpected error resolving stream:") raise ResolveError(f"Unexpected error resolving stream: {str(err)}") - if not resolved_result: - raise ResolveError(f"Failed to resolve stream at '{uri}'") if 'error' in resolved_result: raise ResolveError(f"Unexpected error resolving uri for download: {resolved_result['error']}") + if not resolved_result or uri not in resolved_result: + raise ResolveError(f"Failed to resolve stream at '{uri}'") txo = resolved_result[uri] claim = txo.claim @@ -166,11 +166,13 @@ class FileManager: #################### if not claim.stream.source.bt_infohash: + # fixme: this shouldnt be here stream = ManagedStream( self.loop, self.config, source_manager.blob_manager, claim.stream.source.sd_hash, download_directory, file_name, ManagedStream.STATUS_RUNNING, content_fee=payment, analytics_manager=self.analytics_manager ) + stream.downloader.node = source_manager.node else: stream = None log.info("starting download for %s", uri) diff --git a/lbry/stream/downloader.py b/lbry/stream/downloader.py index 9fe98ac54..588263b0e 100644 --- a/lbry/stream/downloader.py +++ b/lbry/stream/downloader.py @@ -92,8 +92,8 @@ class StreamDownloader: async def start(self, node: typing.Optional['Node'] = None, connection_id: int = 0): # set up peer accumulation - if node: - self.node = node + self.node = node or self.node # fixme: this shouldnt be set here! + if self.node: if self.accumulate_task and not self.accumulate_task.done(): self.accumulate_task.cancel() _, self.accumulate_task = self.node.accumulate_peers(self.search_queue, self.peer_queue) diff --git a/lbry/stream/stream_manager.py b/lbry/stream/stream_manager.py index ad232b28b..c7a3c51e0 100644 --- a/lbry/stream/stream_manager.py +++ b/lbry/stream/stream_manager.py @@ -54,6 +54,10 @@ class StreamManager(SourceManager): self.running_reflector_uploads: typing.Dict[str, asyncio.Task] = {} self.started = asyncio.Event(loop=self.loop) + @property + def streams(self): + return self._sources + def add(self, source: ManagedStream): super().add(source) self.storage.content_claim_callbacks[source.stream_hash] = lambda: self._update_content_claim(source) diff --git a/tests/unit/components/test_component_manager.py b/tests/unit/components/test_component_manager.py index 6738c14e4..b4e81fed7 100644 --- a/tests/unit/components/test_component_manager.py +++ b/tests/unit/components/test_component_manager.py @@ -16,6 +16,7 @@ class TestComponentManager(AsyncioTestCase): [ components.DatabaseComponent, components.ExchangeRateManagerComponent, + components.TorrentComponent, components.UPnPComponent ], [ @@ -24,9 +25,9 @@ class TestComponentManager(AsyncioTestCase): components.WalletComponent ], [ + components.FileManagerComponent, components.HashAnnouncerComponent, components.PeerProtocolServerComponent, - components.FileManagerComponent, components.WalletServerPaymentsComponent ] ] @@ -135,8 +136,8 @@ class FakeDelayedBlobManager(FakeComponent): await asyncio.sleep(1) -class FakeDelayedStreamManager(FakeComponent): - component_name = "stream_manager" +class FakeDelayedFileManager(FakeComponent): + component_name = "file_manager" depends_on = [FakeDelayedBlobManager.component_name] async def start(self): @@ -153,7 +154,7 @@ class TestComponentManagerProperStart(AdvanceTimeTestCase): PEER_PROTOCOL_SERVER_COMPONENT, UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT], wallet=FakeDelayedWallet, - stream_manager=FakeDelayedStreamManager, + file_manager=FakeDelayedFileManager, blob_manager=FakeDelayedBlobManager ) @@ -163,17 +164,17 @@ class TestComponentManagerProperStart(AdvanceTimeTestCase): await self.advance(0) self.assertTrue(self.component_manager.get_component('wallet').running) self.assertFalse(self.component_manager.get_component('blob_manager').running) - self.assertFalse(self.component_manager.get_component('stream_manager').running) + self.assertFalse(self.component_manager.get_component('file_manager').running) await self.advance(1) self.assertTrue(self.component_manager.get_component('wallet').running) self.assertTrue(self.component_manager.get_component('blob_manager').running) - self.assertFalse(self.component_manager.get_component('stream_manager').running) + self.assertFalse(self.component_manager.get_component('file_manager').running) await self.advance(1) self.assertTrue(self.component_manager.get_component('wallet').running) self.assertTrue(self.component_manager.get_component('blob_manager').running) - self.assertTrue(self.component_manager.get_component('stream_manager').running) + self.assertTrue(self.component_manager.get_component('file_manager').running) async def test_proper_stopping_of_components(self): asyncio.create_task(self.component_manager.start()) @@ -182,18 +183,18 @@ class TestComponentManagerProperStart(AdvanceTimeTestCase): await self.advance(1) self.assertTrue(self.component_manager.get_component('wallet').running) self.assertTrue(self.component_manager.get_component('blob_manager').running) - self.assertTrue(self.component_manager.get_component('stream_manager').running) + self.assertTrue(self.component_manager.get_component('file_manager').running) asyncio.create_task(self.component_manager.stop()) await self.advance(0) - self.assertFalse(self.component_manager.get_component('stream_manager').running) + self.assertFalse(self.component_manager.get_component('file_manager').running) self.assertTrue(self.component_manager.get_component('blob_manager').running) self.assertTrue(self.component_manager.get_component('wallet').running) await self.advance(1) - self.assertFalse(self.component_manager.get_component('stream_manager').running) + self.assertFalse(self.component_manager.get_component('file_manager').running) self.assertFalse(self.component_manager.get_component('blob_manager').running) self.assertTrue(self.component_manager.get_component('wallet').running) await self.advance(1) - self.assertFalse(self.component_manager.get_component('stream_manager').running) + self.assertFalse(self.component_manager.get_component('file_manager').running) self.assertFalse(self.component_manager.get_component('blob_manager').running) self.assertFalse(self.component_manager.get_component('wallet').running) diff --git a/tests/unit/stream/test_managed_stream.py b/tests/unit/stream/test_managed_stream.py index 64e3e3ea2..3542c60e4 100644 --- a/tests/unit/stream/test_managed_stream.py +++ b/tests/unit/stream/test_managed_stream.py @@ -76,7 +76,7 @@ class TestManagedStream(BlobExchangeTestBase): return q2, self.loop.create_task(_task()) mock_node.accumulate_peers = mock_accumulate_peers or _mock_accumulate_peers - self.stream.node = mock_node + self.stream.downloader.node = mock_node await self.stream.save_file() await self.stream.finished_write_attempt.wait() self.assertTrue(os.path.isfile(self.stream.full_path)) @@ -110,7 +110,6 @@ class TestManagedStream(BlobExchangeTestBase): await self.setup_stream(2) mock_node = mock.Mock(spec=Node) - q = asyncio.Queue() bad_peer = make_kademlia_peer(b'2' * 48, "127.0.0.1", tcp_port=3334, allow_localhost=True) @@ -124,7 +123,7 @@ class TestManagedStream(BlobExchangeTestBase): mock_node.accumulate_peers = _mock_accumulate_peers - self.stream.node = mock_node + self.stream.downloader.node = mock_node await self.stream.save_file() await self.stream.finished_writing.wait() self.assertTrue(os.path.isfile(self.stream.full_path)) diff --git a/tests/unit/stream/test_reflector.py b/tests/unit/stream/test_reflector.py index 4845948d1..b47cf31d0 100644 --- a/tests/unit/stream/test_reflector.py +++ b/tests/unit/stream/test_reflector.py @@ -39,7 +39,7 @@ class TestStreamAssembler(AsyncioTestCase): with open(file_path, 'wb') as f: f.write(self.cleartext) - self.stream = await self.stream_manager.create_stream(file_path) + self.stream = await self.stream_manager.create(file_path) async def _test_reflect_stream(self, response_chunk_size): reflector = ReflectorServer(self.server_blob_manager, response_chunk_size=response_chunk_size) diff --git a/tests/unit/stream/test_stream_manager.py b/tests/unit/stream/test_stream_manager.py index b14f63de6..3299bcb4d 100644 --- a/tests/unit/stream/test_stream_manager.py +++ b/tests/unit/stream/test_stream_manager.py @@ -5,6 +5,8 @@ from unittest import mock import asyncio import json from decimal import Decimal + +from lbry.file.file_manager import FileManager from tests.unit.blob_exchange.test_transfer_blob import BlobExchangeTestBase from lbry.testcase import get_fake_exchange_rate_manager from lbry.utils import generate_id @@ -110,10 +112,7 @@ async def get_mock_wallet(sd_hash, storage, balance=10.0, fee=None): async def mock_resolve(*args, **kwargs): result = {txo.meta['permanent_url']: txo} - claims = [ - StreamManager._convert_to_old_resolve_output(manager, result)[txo.meta['permanent_url']] - ] - await storage.save_claims(claims) + await storage.save_claim_from_output(ledger, txo) return result manager.ledger.resolve = mock_resolve @@ -138,11 +137,20 @@ class TestStreamManager(BlobExchangeTestBase): ) self.sd_hash = descriptor.sd_hash self.mock_wallet, self.uri = await get_mock_wallet(self.sd_hash, self.client_storage, balance, fee) - self.stream_manager = StreamManager(self.loop, self.client_config, self.client_blob_manager, self.mock_wallet, - self.client_storage, get_mock_node(self.server_from_client), - AnalyticsManager(self.client_config, - binascii.hexlify(generate_id()).decode(), - binascii.hexlify(generate_id()).decode())) + analytics_manager = AnalyticsManager( + self.client_config, + binascii.hexlify(generate_id()).decode(), + binascii.hexlify(generate_id()).decode() + ) + self.stream_manager = StreamManager( + self.loop, self.client_config, self.client_blob_manager, self.mock_wallet, + self.client_storage, get_mock_node(self.server_from_client), + analytics_manager + ) + self.file_manager = FileManager( + self.loop, self.client_config, self.mock_wallet, self.client_storage, analytics_manager + ) + self.file_manager.source_managers['stream'] = self.stream_manager self.exchange_rate_manager = get_fake_exchange_rate_manager() async def _test_time_to_first_bytes(self, check_post, error=None, after_setup=None): @@ -159,9 +167,9 @@ class TestStreamManager(BlobExchangeTestBase): self.stream_manager.analytics_manager._post = _check_post if error: with self.assertRaises(error): - await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager) + await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager) else: - await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager) + await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager) await asyncio.sleep(0, loop=self.loop) self.assertTrue(checked_analytics_event) @@ -281,7 +289,7 @@ class TestStreamManager(BlobExchangeTestBase): self.stream_manager.analytics_manager._post = check_post self.assertDictEqual(self.stream_manager.streams, {}) - stream = await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager) + stream = await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager) stream_hash = stream.stream_hash self.assertDictEqual(self.stream_manager.streams, {stream.sd_hash: stream}) self.assertTrue(stream.running) @@ -302,7 +310,7 @@ class TestStreamManager(BlobExchangeTestBase): ) self.assertEqual(stored_status, "stopped") - stream.node = self.stream_manager.node + stream.downloader.node = self.stream_manager.node await stream.save_file() await stream.finished_writing.wait() await asyncio.sleep(0, loop=self.loop) @@ -314,7 +322,7 @@ class TestStreamManager(BlobExchangeTestBase): ) self.assertEqual(stored_status, "finished") - await self.stream_manager.delete_stream(stream, True) + await self.stream_manager.delete(stream, True) self.assertDictEqual(self.stream_manager.streams, {}) self.assertFalse(os.path.isfile(os.path.join(self.client_dir, "test_file"))) stored_status = await self.client_storage.run_and_return_one_or_none( @@ -326,7 +334,7 @@ class TestStreamManager(BlobExchangeTestBase): async def _test_download_error_on_start(self, expected_error, timeout=None): error = None try: - await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager, timeout) + await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager, timeout) except Exception as err: if isinstance(err, asyncio.CancelledError): # TODO: remove when updated to 3.8 raise @@ -402,7 +410,7 @@ class TestStreamManager(BlobExchangeTestBase): last_blob_hash = json.loads(sdf.read())['blobs'][-2]['blob_hash'] self.server_blob_manager.delete_blob(last_blob_hash) self.client_config.blob_download_timeout = 0.1 - stream = await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager) + stream = await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager) await stream.started_writing.wait() self.assertEqual('running', stream.status) self.assertIsNotNone(stream.full_path) @@ -434,7 +442,7 @@ class TestStreamManager(BlobExchangeTestBase): self.stream_manager.analytics_manager._post = check_post self.assertDictEqual(self.stream_manager.streams, {}) - stream = await self.stream_manager.download_stream_from_uri(self.uri, self.exchange_rate_manager) + stream = await self.file_manager.download_from_uri(self.uri, self.exchange_rate_manager) await stream.finished_writing.wait() await asyncio.sleep(0, loop=self.loop) self.stream_manager.stop() From 6865ddfc12cd36f2513df416faa8425cbde93a63 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 5 Feb 2020 12:29:26 -0300 Subject: [PATCH 35/86] torrent manager and torrent source --- lbry/extras/daemon/components.py | 10 ++- lbry/file/file_manager.py | 13 +++- lbry/torrent/session.py | 26 ++++++-- lbry/torrent/torrent_manager.py | 111 +++++++++++++++++++++++++++++++ 4 files changed, 151 insertions(+), 9 deletions(-) create mode 100644 lbry/torrent/torrent_manager.py diff --git a/lbry/extras/daemon/components.py b/lbry/extras/daemon/components.py index 046faca2a..38c4d4650 100644 --- a/lbry/extras/daemon/components.py +++ b/lbry/extras/daemon/components.py @@ -21,6 +21,7 @@ from lbry.file.file_manager import FileManager from lbry.extras.daemon.component import Component from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager from lbry.extras.daemon.storage import SQLiteStorage +from lbry.torrent.torrent_manager import TorrentManager from lbry.wallet import WalletManager from lbry.wallet.usage_payment import WalletServerPayer try: @@ -327,7 +328,7 @@ class HashAnnouncerComponent(Component): class FileManagerComponent(Component): component_name = FILE_MANAGER_COMPONENT - depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT] + depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT, LIBTORRENT_COMPONENT] def __init__(self, component_manager): super().__init__(component_manager) @@ -350,14 +351,19 @@ class FileManagerComponent(Component): wallet = self.component_manager.get_component(WALLET_COMPONENT) node = self.component_manager.get_component(DHT_COMPONENT) \ if self.component_manager.has_component(DHT_COMPONENT) else None + torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT) if TorrentSession else None log.info('Starting the file manager') loop = asyncio.get_event_loop() self.file_manager = FileManager( loop, self.conf, wallet, storage, self.component_manager.analytics_manager ) self.file_manager.source_managers['stream'] = StreamManager( - loop, self.conf, blob_manager, wallet, storage, node, self.component_manager.analytics_manager + loop, self.conf, blob_manager, wallet, storage, node, ) + if TorrentSession: + self.file_manager.source_managers['torrent'] = TorrentManager( + loop, self.conf, torrent, storage, self.component_manager.analytics_manager + ) await self.file_manager.start() log.info('Done setting up file manager') diff --git a/lbry/file/file_manager.py b/lbry/file/file_manager.py index 765cd1b53..50b21e9b2 100644 --- a/lbry/file/file_manager.py +++ b/lbry/file/file_manager.py @@ -6,6 +6,7 @@ from aiohttp.web import Request from lbry.error import ResolveError, DownloadSDTimeoutError, InsufficientFundsError from lbry.error import ResolveTimeoutError, DownloadDataTimeoutError, KeyFeeAboveMaxAllowedError from lbry.stream.managed_stream import ManagedStream +from lbry.torrent.torrent_manager import TorrentSource from lbry.utils import cache_concurrent from lbry.schema.url import URL from lbry.wallet.dewies import dewies_to_lbc @@ -110,11 +111,12 @@ class FileManager: if claim.stream.source.bt_infohash: source_manager = self.source_managers['torrent'] + existing = source_manager.get_filtered(bt_infohash=claim.stream.source.bt_infohash) else: source_manager = self.source_managers['stream'] + existing = source_manager.get_filtered(sd_hash=claim.stream.source.sd_hash) # resume or update an existing stream, if the stream changed: download it and delete the old one after - existing = self.get_filtered(sd_hash=claim.stream.source.sd_hash) to_replace, updated_stream = None, None if existing and existing[0].claim_id != txo.claim_id: raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {txo.claim_id}") @@ -151,7 +153,6 @@ class FileManager: ) return updated_stream - #################### # pay fee #################### @@ -174,7 +175,13 @@ class FileManager: ) stream.downloader.node = source_manager.node else: - stream = None + stream = TorrentSource( + self.loop, self.config, self.storage, identifier=claim.stream.source.bt_infohash, + file_name=file_name, download_directory=download_directory or self.config.download_dir, + status=ManagedStream.STATUS_RUNNING, + claim=claim, analytics_manager=self.analytics_manager, + torrent_session=source_manager.torrent_session + ) log.info("starting download for %s", uri) before_download = self.loop.time() diff --git a/lbry/torrent/session.py b/lbry/torrent/session.py index 294a2cba5..0a33c0bf6 100644 --- a/lbry/torrent/session.py +++ b/lbry/torrent/session.py @@ -1,5 +1,7 @@ import asyncio import binascii +from typing import Optional + import libtorrent @@ -30,6 +32,15 @@ NOTIFICATION_MASKS = [ ] +DEFAULT_FLAGS = ( # fixme: somehow the logic here is inverted? + libtorrent.add_torrent_params_flags_t.flag_paused + | libtorrent.add_torrent_params_flags_t.flag_auto_managed + | libtorrent.add_torrent_params_flags_t.flag_duplicate_is_error + | libtorrent.add_torrent_params_flags_t.flag_upload_mode + | libtorrent.add_torrent_params_flags_t.flag_update_subscribe +) + + def get_notification_type(notification) -> str: for i, notification_type in enumerate(NOTIFICATION_MASKS): if (1 << i) & notification: @@ -123,10 +134,11 @@ class TorrentSession: self._executor, self._session.resume ) - def _add_torrent(self, btih: str, download_directory: str): - self._handles[btih] = TorrentHandle(self._loop, self._executor, self._session.add_torrent( - {'info_hash': binascii.unhexlify(btih.encode()), 'save_path': download_directory} - )) + def _add_torrent(self, btih: str, download_directory: Optional[str]): + params = {'info_hash': binascii.unhexlify(btih.encode()), 'flags': DEFAULT_FLAGS} + if download_directory: + params['save_path'] = download_directory + self._handles[btih] = TorrentHandle(self._loop, self._executor, self._session.add_torrent(params)) async def add_torrent(self, btih, download_path): await self._loop.run_in_executor( @@ -135,6 +147,12 @@ class TorrentSession: self._loop.create_task(self._handles[btih].status_loop()) await self._handles[btih].finished.wait() + async def remove_torrent(self, btih, remove_files=False): + if btih in self._handles: + handle = self._handles[btih] + self._session.remove_torrent(handle, 1 if remove_files else 0) + self._handles.pop(btih) + def get_magnet_uri(btih): return f"magnet:?xt=urn:btih:{btih}" diff --git a/lbry/torrent/torrent_manager.py b/lbry/torrent/torrent_manager.py new file mode 100644 index 000000000..02f6b7cf9 --- /dev/null +++ b/lbry/torrent/torrent_manager.py @@ -0,0 +1,111 @@ +import asyncio +import binascii +import logging +import typing +from typing import Optional +from aiohttp.web import Request +from lbry.file.source_manager import SourceManager +from lbry.file.source import ManagedDownloadSource + +if typing.TYPE_CHECKING: + from lbry.torrent.session import TorrentSession + from lbry.conf import Config + from lbry.wallet.transaction import Transaction + from lbry.extras.daemon.analytics import AnalyticsManager + from lbry.extras.daemon.storage import SQLiteStorage, StoredContentClaim + from lbry.extras.daemon.storage import StoredContentClaim + +log = logging.getLogger(__name__) + + +def path_or_none(encoded_path) -> Optional[str]: + if not encoded_path: + return + return binascii.unhexlify(encoded_path).decode() + + +class TorrentSource(ManagedDownloadSource): + STATUS_STOPPED = "stopped" + + def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage', identifier: str, + file_name: Optional[str] = None, download_directory: Optional[str] = None, + status: Optional[str] = STATUS_STOPPED, claim: Optional['StoredContentClaim'] = None, + download_id: Optional[str] = None, rowid: Optional[int] = None, + content_fee: Optional['Transaction'] = None, + analytics_manager: Optional['AnalyticsManager'] = None, + added_on: Optional[int] = None, torrent_session: Optional['TorrentSession'] = None): + super().__init__(loop, config, storage, identifier, file_name, download_directory, status, claim, download_id, + rowid, content_fee, analytics_manager, added_on) + self.torrent_session = torrent_session + + async def start(self, timeout: Optional[float] = None, save_now: Optional[bool] = False): + await self.torrent_session.add_torrent(self.identifier, self.download_directory) + + async def stop(self, finished: bool = False): + await self.torrent_session.remove_torrent(self.identifier) + + async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None): + raise NotImplementedError() + + def stop_tasks(self): + raise NotImplementedError() + + @property + def completed(self): + raise NotImplementedError() + +class TorrentManager(SourceManager): + _sources: typing.Dict[str, ManagedDownloadSource] + + filter_fields = set(SourceManager.filter_fields) + filter_fields.update({ + 'bt_infohash', + 'blobs_remaining', # TODO: here they call them "parts", but its pretty much the same concept + 'blobs_in_stream' + }) + + def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', torrent_session: 'TorrentSession', + storage: 'SQLiteStorage', analytics_manager: Optional['AnalyticsManager'] = None): + super().__init__(loop, config, storage, analytics_manager) + self.torrent_session: 'TorrentSession' = torrent_session + + def add(self, source: ManagedDownloadSource): + super().add(source) + + async def recover_streams(self, file_infos: typing.List[typing.Dict]): + raise NotImplementedError + + async def _load_stream(self, rowid: int, bt_infohash: str, file_name: Optional[str], + download_directory: Optional[str], status: str, + claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'], + added_on: Optional[int]): + stream = TorrentSource( + self.loop, self.config, self.storage, identifier=bt_infohash, file_name=file_name, + download_directory=download_directory, status=status, claim=claim, rowid=rowid, + content_fee=content_fee, analytics_manager=self.analytics_manager, added_on=added_on, + torrent_session=self.torrent_session + ) + self.add(stream) + + async def initialize_from_database(self): + pass + + async def start(self): + await super().start() + + def stop(self): + super().stop() + log.info("finished stopping the torrent manager") + + async def create(self, file_path: str, key: Optional[bytes] = None, + iv_generator: Optional[typing.Generator[bytes, None, None]] = None): + raise NotImplementedError + + async def _delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False): + raise NotImplementedError + # blob_hashes = [source.sd_hash] + [b.blob_hash for b in source.descriptor.blobs[:-1]] + # await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False) + # await self.storage.delete_stream(source.descriptor) + + async def stream_partial_content(self, request: Request, sd_hash: str): + raise NotImplementedError From dd26a968285b56e62e938cb4fa8c4732598d9882 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 7 Feb 2020 12:32:39 -0300 Subject: [PATCH 36/86] adds more torrent parts --- lbry/extras/daemon/daemon.py | 10 ++-- lbry/file/file_manager.py | 5 +- lbry/file/source_manager.py | 4 -- lbry/stream/stream_manager.py | 34 +++++++---- lbry/torrent/session.py | 102 ++++++++++++++++++++++++++------ lbry/torrent/torrent_manager.py | 4 ++ 6 files changed, 118 insertions(+), 41 deletions(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index fd3153f9e..cb88142ab 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -57,8 +57,8 @@ if typing.TYPE_CHECKING: from lbry.extras.daemon.components import UPnPComponent from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager from lbry.extras.daemon.storage import SQLiteStorage - from lbry.stream.stream_manager import StreamManager from lbry.wallet import WalletManager, Ledger + from lbry.file.file_manager import FileManager log = logging.getLogger(__name__) @@ -372,7 +372,7 @@ class Daemon(metaclass=JSONRPCServerType): return self.component_manager.get_component(DATABASE_COMPONENT) @property - def file_manager(self) -> typing.Optional['StreamManager']: + def file_manager(self) -> typing.Optional['FileManager']: return self.component_manager.get_component(FILE_MANAGER_COMPONENT) @property @@ -3447,11 +3447,11 @@ class Daemon(metaclass=JSONRPCServerType): stream_hash = None if not preview: - old_stream = self.stream_manager.streams.get(old_txo.claim.stream.source.sd_hash, None) + old_stream = self.file_manager.get_filtered(sd_hash=old_txo.claim.stream.source.sd_hash)[0] if file_path is not None: if old_stream: - await self.stream_manager.delete_stream(old_stream, delete_file=False) - file_stream = await self.stream_manager.create_stream(file_path) + await self.file_manager.delete(old_stream, delete_file=False) + file_stream = await self.file_manager.create_stream(file_path) new_txo.claim.stream.source.sd_hash = file_stream.sd_hash new_txo.script.generate() stream_hash = file_stream.stream_hash diff --git a/lbry/file/file_manager.py b/lbry/file/file_manager.py index 50b21e9b2..1ab72f7c1 100644 --- a/lbry/file/file_manager.py +++ b/lbry/file/file_manager.py @@ -98,8 +98,9 @@ class FileManager: raise ResolveError(f"Unexpected error resolving uri for download: {resolved_result['error']}") if not resolved_result or uri not in resolved_result: raise ResolveError(f"Failed to resolve stream at '{uri}'") - txo = resolved_result[uri] + if isinstance(txo, dict): + raise ResolveError(f"Failed to resolve stream at '{uri}': {txo}") claim = txo.claim outpoint = f"{txo.tx_ref.id}:{txo.position}" resolved_time = self.loop.time() - start_time @@ -179,7 +180,7 @@ class FileManager: self.loop, self.config, self.storage, identifier=claim.stream.source.bt_infohash, file_name=file_name, download_directory=download_directory or self.config.download_dir, status=ManagedStream.STATUS_RUNNING, - claim=claim, analytics_manager=self.analytics_manager, + analytics_manager=self.analytics_manager, torrent_session=source_manager.torrent_session ) log.info("starting download for %s", uri) diff --git a/lbry/file/source_manager.py b/lbry/file/source_manager.py index 9eada3cca..e3f7d4ad3 100644 --- a/lbry/file/source_manager.py +++ b/lbry/file/source_manager.py @@ -74,11 +74,7 @@ class SourceManager: iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedDownloadSource: raise NotImplementedError() - async def _delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False): - raise NotImplementedError() - async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False): - await self._delete(source) self.remove(source) if delete_file and source.output_file_exists: os.remove(source.full_path) diff --git a/lbry/stream/stream_manager.py b/lbry/stream/stream_manager.py index c7a3c51e0..84ef5b2a1 100644 --- a/lbry/stream/stream_manager.py +++ b/lbry/stream/stream_manager.py @@ -32,7 +32,7 @@ def path_or_none(encoded_path) -> Optional[str]: class StreamManager(SourceManager): _sources: typing.Dict[str, ManagedStream] - filter_fields = set(SourceManager.filter_fields) + filter_fields = SourceManager.filter_fields filter_fields.update({ 'sd_hash', 'stream_hash', @@ -180,6 +180,7 @@ class StreamManager(SourceManager): self.re_reflect_task = self.loop.create_task(self.reflect_streams()) def stop(self): + super().stop() if self.resume_saving_task and not self.resume_saving_task.done(): self.resume_saving_task.cancel() if self.re_reflect_task and not self.re_reflect_task.done(): @@ -206,16 +207,30 @@ class StreamManager(SourceManager): ) return task - async def create_stream(self, file_path: str, key: Optional[bytes] = None, - iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream: - stream = await ManagedStream.create(self.loop, self.config, self.blob_manager, file_path, key, iv_generator) + async def create(self, file_path: str, key: Optional[bytes] = None, + iv_generator: Optional[typing.Generator[bytes, None, None]] = None) -> ManagedStream: + descriptor = await StreamDescriptor.create_stream( + self.loop, self.blob_manager.blob_dir, file_path, key=key, iv_generator=iv_generator, + blob_completed_callback=self.blob_manager.blob_completed + ) + await self.storage.store_stream( + self.blob_manager.get_blob(descriptor.sd_hash), descriptor + ) + row_id = await self.storage.save_published_file( + descriptor.stream_hash, os.path.basename(file_path), os.path.dirname(file_path), 0 + ) + stream = ManagedStream( + self.loop, self.config, self.blob_manager, descriptor.sd_hash, os.path.dirname(file_path), + os.path.basename(file_path), status=ManagedDownloadSource.STATUS_FINISHED, + rowid=row_id, descriptor=descriptor + ) self.streams[stream.sd_hash] = stream self.storage.content_claim_callbacks[stream.stream_hash] = lambda: self._update_content_claim(stream) if self.config.reflect_streams and self.config.reflector_servers: self.reflect_stream(stream) return stream - async def delete_stream(self, stream: ManagedStream, delete_file: Optional[bool] = False): + async def delete(self, stream: ManagedStream, delete_file: Optional[bool] = False): if stream.sd_hash in self.running_reflector_uploads: self.running_reflector_uploads[stream.sd_hash].cancel() stream.stop_tasks() @@ -223,12 +238,9 @@ class StreamManager(SourceManager): del self.streams[stream.sd_hash] blob_hashes = [stream.sd_hash] + [b.blob_hash for b in stream.descriptor.blobs[:-1]] await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False) - await self.storage.delete(stream.descriptor) - - async def _delete(self, source: ManagedStream, delete_file: Optional[bool] = False): - blob_hashes = [source.sd_hash] + [b.blob_hash for b in source.descriptor.blobs[:-1]] - await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False) - await self.storage.delete_stream(source.descriptor) + await self.storage.delete_stream(stream.descriptor) + if delete_file and stream.output_file_exists: + os.remove(stream.full_path) async def stream_partial_content(self, request: Request, sd_hash: str): return await self._sources[sd_hash].stream_file(request, self.node) diff --git a/lbry/torrent/session.py b/lbry/torrent/session.py index 0a33c0bf6..af2663e64 100644 --- a/lbry/torrent/session.py +++ b/lbry/torrent/session.py @@ -1,5 +1,8 @@ import asyncio import binascii +import os +from hashlib import sha1 +from tempfile import mkdtemp from typing import Optional import libtorrent @@ -33,10 +36,8 @@ NOTIFICATION_MASKS = [ DEFAULT_FLAGS = ( # fixme: somehow the logic here is inverted? - libtorrent.add_torrent_params_flags_t.flag_paused - | libtorrent.add_torrent_params_flags_t.flag_auto_managed + libtorrent.add_torrent_params_flags_t.flag_auto_managed | libtorrent.add_torrent_params_flags_t.flag_duplicate_is_error - | libtorrent.add_torrent_params_flags_t.flag_upload_mode | libtorrent.add_torrent_params_flags_t.flag_update_subscribe ) @@ -52,15 +53,23 @@ class TorrentHandle: def __init__(self, loop, executor, handle): self._loop = loop self._executor = executor - self._handle = handle + self._handle: libtorrent.torrent_handle = handle self.finished = asyncio.Event(loop=loop) + self.metadata_completed = asyncio.Event(loop=loop) def _show_status(self): + # fixme: cleanup status = self._handle.status() + if status.has_metadata: + self.metadata_completed.set() + # metadata: libtorrent.torrent_info = self._handle.get_torrent_info() + # print(metadata) + # print(metadata.files()) + # print(type(self._handle)) if not status.is_seeding: - print('%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d) %s' % ( + print('%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d seeds: %d) %s - %s' % ( status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000, - status.num_peers, status.state)) + status.num_peers, status.num_seeds, status.state, status.save_path)) elif not self.finished.is_set(): self.finished.set() print("finished!") @@ -72,7 +81,7 @@ class TorrentHandle: ) if self.finished.is_set(): break - await asyncio.sleep(1, loop=self._loop) + await asyncio.sleep(0.1, loop=self._loop) async def pause(self): await self._loop.run_in_executor( @@ -89,25 +98,44 @@ class TorrentSession: def __init__(self, loop, executor): self._loop = loop self._executor = executor - self._session = None + self._session: Optional[libtorrent.session] = None self._handles = {} - async def bind(self, interface: str = '0.0.0.0', port: int = 6881): + async def add_fake_torrent(self): + dir = mkdtemp() + info, btih = self._create_fake(dir) + flags = libtorrent.add_torrent_params_flags_t.flag_seed_mode + handle = self._session.add_torrent({ + 'ti': info, 'save_path': dir, 'flags': flags + }) + self._handles[btih] = TorrentHandle(self._loop, self._executor, handle) + return btih + + def _create_fake(self, dir): + # beware, that's just for testing + path = os.path.join(dir, 'tmp') + with open(path, 'wb') as myfile: + size = myfile.write(b'0' * 40 * 1024 * 1024) + fs = libtorrent.file_storage() + fs.add_file('tmp', size) + print(fs.file_path(0)) + t = libtorrent.create_torrent(fs, 0, 4 * 1024 * 1024) + libtorrent.set_piece_hashes(t, dir) + info = libtorrent.torrent_info(t.generate()) + btih = sha1(info.metadata()).hexdigest() + return info, btih + + async def bind(self, interface: str = '0.0.0.0', port: int = 10889): settings = { 'listen_interfaces': f"{interface}:{port}", 'enable_outgoing_utp': True, 'enable_incoming_utp': True, - 'enable_outgoing_tcp': True, - 'enable_incoming_tcp': True + 'enable_outgoing_tcp': False, + 'enable_incoming_tcp': False } self._session = await self._loop.run_in_executor( self._executor, libtorrent.session, settings # pylint: disable=c-extension-no-member ) - await self._loop.run_in_executor( - self._executor, - # lambda necessary due boost functions raising errors when asyncio inspects them. try removing later - lambda: self._session.add_dht_router("router.utorrent.com", 6881) # pylint: disable=unnecessary-lambda - ) self._loop.create_task(self.process_alerts()) def _pop_alerts(self): @@ -135,17 +163,25 @@ class TorrentSession: ) def _add_torrent(self, btih: str, download_directory: Optional[str]): - params = {'info_hash': binascii.unhexlify(btih.encode()), 'flags': DEFAULT_FLAGS} + params = {'info_hash': binascii.unhexlify(btih.encode())} + flags = DEFAULT_FLAGS + print(bin(flags)) + flags ^= libtorrent.add_torrent_params_flags_t.flag_paused + # flags ^= libtorrent.add_torrent_params_flags_t.flag_auto_managed + # flags ^= libtorrent.add_torrent_params_flags_t.flag_stop_when_ready + print(bin(flags)) + # params['flags'] = flags if download_directory: params['save_path'] = download_directory - self._handles[btih] = TorrentHandle(self._loop, self._executor, self._session.add_torrent(params)) + handle = self._handles[btih] = TorrentHandle(self._loop, self._executor, self._session.add_torrent(params)) + handle._handle.force_dht_announce() async def add_torrent(self, btih, download_path): await self._loop.run_in_executor( self._executor, self._add_torrent, btih, download_path ) self._loop.create_task(self._handles[btih].status_loop()) - await self._handles[btih].finished.wait() + await self._handles[btih].metadata_completed.wait() async def remove_torrent(self, btih, remove_files=False): if btih in self._handles: @@ -156,3 +192,31 @@ class TorrentSession: def get_magnet_uri(btih): return f"magnet:?xt=urn:btih:{btih}" + + +async def main(): + if os.path.exists("~/Downloads/ubuntu-18.04.3-live-server-amd64.torrent"): + os.remove("~/Downloads/ubuntu-18.04.3-live-server-amd64.torrent") + if os.path.exists("~/Downloads/ubuntu-18.04.3-live-server-amd64.iso"): + os.remove("~/Downloads/ubuntu-18.04.3-live-server-amd64.iso") + + btih = "dd8255ecdc7ca55fb0bbf81323d87062db1f6d1c" + + executor = None + session = TorrentSession(asyncio.get_event_loop(), executor) + session2 = TorrentSession(asyncio.get_event_loop(), executor) + await session.bind('localhost', port=4040) + await session2.bind('localhost', port=4041) + btih = await session.add_fake_torrent() + session2._session.add_dht_node(('localhost', 4040)) + await session2.add_torrent(btih, "/tmp/down") + print('added') + while True: + print("idling") + await asyncio.sleep(100) + await session.pause() + executor.shutdown() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/lbry/torrent/torrent_manager.py b/lbry/torrent/torrent_manager.py index 02f6b7cf9..24ed651be 100644 --- a/lbry/torrent/torrent_manager.py +++ b/lbry/torrent/torrent_manager.py @@ -26,6 +26,10 @@ def path_or_none(encoded_path) -> Optional[str]: class TorrentSource(ManagedDownloadSource): STATUS_STOPPED = "stopped" + filter_fields = SourceManager.filter_fields + filter_fields.update({ + 'bt_infohash' + }) def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage', identifier: str, file_name: Optional[str] = None, download_directory: Optional[str] = None, From b930c3fc937c7b0d2a9f71d67211891118b4a8b3 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 7 Feb 2020 14:42:25 -0300 Subject: [PATCH 37/86] fix torrent and stream manager reference leftovers --- lbry/extras/daemon/daemon.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index cb88142ab..4e81b93e4 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -858,7 +858,8 @@ class Daemon(metaclass=JSONRPCServerType): 'exchange_rate_manager': (bool), 'hash_announcer': (bool), 'peer_protocol_server': (bool), - 'stream_manager': (bool), + 'file_manager': (bool), + 'libtorrent_component': (bool), 'upnp': (bool), 'wallet': (bool), }, @@ -885,6 +886,9 @@ class Daemon(metaclass=JSONRPCServerType): } ], }, + 'libtorrent_component': { + 'running': (bool) libtorrent was detected and started successfully, + }, 'dht': { 'node_id': (str) lbry dht node id - hex encoded, 'peers_in_routing_table': (int) the number of peers in the routing table, @@ -906,7 +910,7 @@ class Daemon(metaclass=JSONRPCServerType): 'hash_announcer': { 'announce_queue_size': (int) number of blobs currently queued to be announced }, - 'stream_manager': { + 'file_manager': { 'managed_files': (int) count of files in the stream manager, }, 'upnp': { @@ -4787,8 +4791,8 @@ class Daemon(metaclass=JSONRPCServerType): else: server, port = random.choice(self.conf.reflector_servers) reflected = await asyncio.gather(*[ - self.stream_manager.reflect_stream(stream, server, port) - for stream in self.stream_manager.get_filtered_streams(**kwargs) + self.file_manager['stream'].reflect_stream(stream, server, port) + for stream in self.file_manager.get_filtered_streams(**kwargs) ]) total = [] for reflected_for_stream in reflected: From cf985486e52ffde83ebeeaf04c0f13414bca5ec5 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 10 Feb 2020 21:50:16 -0300 Subject: [PATCH 38/86] torrent test and misc fixes --- Makefile | 1 + lbry/file/file_manager.py | 9 ++++-- lbry/torrent/session.py | 26 +++++++++++++---- lbry/torrent/torrent_manager.py | 5 ++-- .../datanetwork/test_file_commands.py | 28 +++++++++++++++++++ tox.ini | 2 ++ 6 files changed, 62 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 23b3f84e8..ab1f20ac4 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ .PHONY: install tools lint test idea install: + pip install -e git+https://github.com/shyba/libtorrent.git#egg=python-libtorrent CFLAGS="-DSQLITE_MAX_VARIABLE_NUMBER=2500000" pip install -U https://github.com/rogerbinns/apsw/releases/download/3.30.1-r1/apsw-3.30.1-r1.zip \ --global-option=fetch \ --global-option=--version --global-option=3.30.1 --global-option=--all \ diff --git a/lbry/file/file_manager.py b/lbry/file/file_manager.py index 1ab72f7c1..926296f6e 100644 --- a/lbry/file/file_manager.py +++ b/lbry/file/file_manager.py @@ -66,6 +66,7 @@ class FileManager: start_time = self.loop.time() resolved_time = None stream = None + claim = None error = None outpoint = None if save_file is None: @@ -203,7 +204,8 @@ class FileManager: source_manager.add(stream) - await self.storage.save_content_claim(stream.stream_hash, outpoint) + if not claim.stream.source.bt_infohash: + await self.storage.save_content_claim(stream.stream_hash, outpoint) if save_file: await asyncio.wait_for(stream.save_file(), timeout - (self.loop.time() - before_download), loop=self.loop) @@ -226,7 +228,10 @@ class FileManager: if payment is not None: # payment is set to None after broadcasting, if we're here an exception probably happened await self.wallet_manager.ledger.release_tx(payment) - if self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or + if self.analytics_manager and claim and claim.stream.source.bt_infohash: + # TODO: analytics for torrents + pass + elif self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or stream.downloader.time_to_first_bytes))): server = self.wallet_manager.ledger.network.client.server self.loop.create_task( diff --git a/lbry/torrent/session.py b/lbry/torrent/session.py index af2663e64..2e4c6a72e 100644 --- a/lbry/torrent/session.py +++ b/lbry/torrent/session.py @@ -37,6 +37,7 @@ NOTIFICATION_MASKS = [ DEFAULT_FLAGS = ( # fixme: somehow the logic here is inverted? libtorrent.add_torrent_params_flags_t.flag_auto_managed + | libtorrent.add_torrent_params_flags_t.flag_paused | libtorrent.add_torrent_params_flags_t.flag_duplicate_is_error | libtorrent.add_torrent_params_flags_t.flag_update_subscribe ) @@ -76,9 +77,7 @@ class TorrentHandle: async def status_loop(self): while True: - await self._loop.run_in_executor( - self._executor, self._show_status - ) + self._show_status() if self.finished.is_set(): break await asyncio.sleep(0.1, loop=self._loop) @@ -100,6 +99,7 @@ class TorrentSession: self._executor = executor self._session: Optional[libtorrent.session] = None self._handles = {} + self.tasks = [] async def add_fake_torrent(self): dir = mkdtemp() @@ -136,7 +136,18 @@ class TorrentSession: self._session = await self._loop.run_in_executor( self._executor, libtorrent.session, settings # pylint: disable=c-extension-no-member ) - self._loop.create_task(self.process_alerts()) + self.tasks.append(self._loop.create_task(self.process_alerts())) + + def stop(self): + while self.tasks: + self.tasks.pop().cancel() + self._session.save_state() + self._session.pause() + self._session.stop_dht() + self._session.stop_lsd() + self._session.stop_natpmp() + self._session.stop_upnp() + self._session = None def _pop_alerts(self): for alert in self._session.pop_alerts(): @@ -167,7 +178,7 @@ class TorrentSession: flags = DEFAULT_FLAGS print(bin(flags)) flags ^= libtorrent.add_torrent_params_flags_t.flag_paused - # flags ^= libtorrent.add_torrent_params_flags_t.flag_auto_managed + flags ^= libtorrent.add_torrent_params_flags_t.flag_auto_managed # flags ^= libtorrent.add_torrent_params_flags_t.flag_stop_when_ready print(bin(flags)) # params['flags'] = flags @@ -189,6 +200,11 @@ class TorrentSession: self._session.remove_torrent(handle, 1 if remove_files else 0) self._handles.pop(btih) + async def save_file(self, btih, download_directory): + return + handle = self._handles[btih] + handle._handle.move_storage(download_directory) + def get_magnet_uri(btih): return f"magnet:?xt=urn:btih:{btih}" diff --git a/lbry/torrent/torrent_manager.py b/lbry/torrent/torrent_manager.py index 24ed651be..118c5d897 100644 --- a/lbry/torrent/torrent_manager.py +++ b/lbry/torrent/torrent_manager.py @@ -49,15 +49,16 @@ class TorrentSource(ManagedDownloadSource): await self.torrent_session.remove_torrent(self.identifier) async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None): - raise NotImplementedError() + await self.torrent_session.save_file(self.identifier, download_directory) def stop_tasks(self): - raise NotImplementedError() + pass @property def completed(self): raise NotImplementedError() + class TorrentManager(SourceManager): _sources: typing.Dict[str, ManagedDownloadSource] diff --git a/tests/integration/datanetwork/test_file_commands.py b/tests/integration/datanetwork/test_file_commands.py index 542be228c..0d8ac3a41 100644 --- a/tests/integration/datanetwork/test_file_commands.py +++ b/tests/integration/datanetwork/test_file_commands.py @@ -2,10 +2,38 @@ import asyncio import os from binascii import hexlify +from lbry.schema import Claim from lbry.testcase import CommandTestCase +from lbry.torrent.session import TorrentSession +from lbry.torrent.torrent_manager import TorrentManager +from lbry.wallet import Transaction class FileCommands(CommandTestCase): + async def initialize_torrent(self): + self.seeder_session = TorrentSession(self.loop, None) + self.addCleanup(self.seeder_session.stop) + await self.seeder_session.bind('localhost', 4040) + self.btih = await self.seeder_session.add_fake_torrent() + address = await self.account.receiving.get_or_create_usable_address() + claim = Claim() + claim.stream.update(bt_infohash=self.btih) + tx = await Transaction.claim_create( + 'torrent', claim, 1, address, [self.account], self.account) + await tx.sign([self.account]) + await self.broadcast(tx) + await self.confirm_tx(tx.id) + client_session = TorrentSession(self.loop, None) + self.daemon.file_manager.source_managers['torrent'] = TorrentManager( + self.loop, self.daemon.conf, client_session, self.daemon.storage, self.daemon.analytics_manager + ) + await self.daemon.file_manager.source_managers['torrent'].start() + await client_session.bind('localhost', 4041) + client_session._session.add_dht_node(('localhost', 4040)) + + async def test_download_torrent(self): + await self.initialize_torrent() + await self.out(self.daemon.jsonrpc_get('torrent')) async def create_streams_in_range(self, *args, **kwargs): self.stream_claim_ids = [] diff --git a/tox.ini b/tox.ini index 3b446a241..697c76ed2 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,8 @@ changedir = {toxinidir}/tests setenv = HOME=/tmp commands = + pip install -U pip + pip install -e 'git+https://github.com/shyba/libtorrent.git#egg=python-libtorrent' pip install https://github.com/rogerbinns/apsw/releases/download/3.30.1-r1/apsw-3.30.1-r1.zip \ --global-option=fetch \ --global-option=--version --global-option=3.30.1 --global-option=--all \ From 4d47873219a96f2aef32a3af1216ab249702e7e8 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 10 Feb 2020 23:15:18 -0300 Subject: [PATCH 39/86] working file list after torrent get --- lbry/extras/daemon/json_response_encoder.py | 30 +++++++++++-------- lbry/extras/daemon/storage.py | 20 ++++++++++++- lbry/file/file_manager.py | 8 ++++- lbry/file/source.py | 16 +++++----- lbry/stream/managed_stream.py | 8 ----- lbry/torrent/session.py | 16 ++++++++++ lbry/torrent/torrent_manager.py | 10 ++++++- .../datanetwork/test_file_commands.py | 1 + 8 files changed, 77 insertions(+), 32 deletions(-) diff --git a/lbry/extras/daemon/json_response_encoder.py b/lbry/extras/daemon/json_response_encoder.py index b7702f541..641a7148f 100644 --- a/lbry/extras/daemon/json_response_encoder.py +++ b/lbry/extras/daemon/json_response_encoder.py @@ -7,6 +7,7 @@ from json import JSONEncoder from google.protobuf.message import DecodeError from lbry.schema.claim import Claim +from lbry.torrent.torrent_manager import TorrentSource from lbry.wallet import Wallet, Ledger, Account, Transaction, Output from lbry.wallet.bip32 import PubKey from lbry.wallet.dewies import dewies_to_lbc @@ -128,6 +129,8 @@ class JSONResponseEncoder(JSONEncoder): return self.encode_wallet(obj) if isinstance(obj, ManagedStream): return self.encode_file(obj) + if isinstance(obj, TorrentSource): + return self.encode_file(obj) if isinstance(obj, Transaction): return self.encode_transaction(obj) if isinstance(obj, Output): @@ -273,26 +276,27 @@ class JSONResponseEncoder(JSONEncoder): output_exists = managed_stream.output_file_exists tx_height = managed_stream.stream_claim_info.height best_height = self.ledger.headers.height + is_stream = hasattr(managed_stream, 'stream_hash') return { - 'streaming_url': managed_stream.stream_url, + 'streaming_url': managed_stream.stream_url if is_stream else None, 'completed': managed_stream.completed, 'file_name': managed_stream.file_name if output_exists else None, 'download_directory': managed_stream.download_directory if output_exists else None, 'download_path': managed_stream.full_path if output_exists else None, 'points_paid': 0.0, 'stopped': not managed_stream.running, - 'stream_hash': managed_stream.stream_hash, - 'stream_name': managed_stream.descriptor.stream_name, - 'suggested_file_name': managed_stream.descriptor.suggested_file_name, - 'sd_hash': managed_stream.descriptor.sd_hash, - 'mime_type': managed_stream.mime_type, - 'key': managed_stream.descriptor.key, - 'total_bytes_lower_bound': managed_stream.descriptor.lower_bound_decrypted_length(), - 'total_bytes': managed_stream.descriptor.upper_bound_decrypted_length(), - 'written_bytes': managed_stream.written_bytes, - 'blobs_completed': managed_stream.blobs_completed, - 'blobs_in_stream': managed_stream.blobs_in_stream, - 'blobs_remaining': managed_stream.blobs_remaining, + 'stream_hash': managed_stream.stream_hash if is_stream else None, + 'stream_name': managed_stream.descriptor.stream_name if is_stream else None, + 'suggested_file_name': managed_stream.descriptor.suggested_file_name if is_stream else None, + 'sd_hash': managed_stream.descriptor.sd_hash if is_stream else None, + 'mime_type': managed_stream.mime_type if is_stream else None, + 'key': managed_stream.descriptor.key if is_stream else None, + 'total_bytes_lower_bound': managed_stream.descriptor.lower_bound_decrypted_length() if is_stream else None, + 'total_bytes': managed_stream.descriptor.upper_bound_decrypted_length() if is_stream else None, + 'written_bytes': managed_stream.written_bytes if is_stream else None, + 'blobs_completed': managed_stream.blobs_completed if is_stream else None, + 'blobs_in_stream': managed_stream.blobs_in_stream if is_stream else None, + 'blobs_remaining': managed_stream.blobs_remaining if is_stream else None, 'status': managed_stream.status, 'claim_id': managed_stream.claim_id, 'txid': managed_stream.txid, diff --git a/lbry/extras/daemon/storage.py b/lbry/extras/daemon/storage.py index 8a03a456c..ce242bfe0 100644 --- a/lbry/extras/daemon/storage.py +++ b/lbry/extras/daemon/storage.py @@ -753,7 +753,8 @@ class SQLiteStorage(SQLiteMixin): return self.save_claims(to_save.values()) @staticmethod - def _save_content_claim(transaction, claim_outpoint, stream_hash): + def _save_content_claim(transaction, claim_outpoint, stream_hash=None, bt_infohash=None): + assert stream_hash or bt_infohash # get the claim id and serialized metadata claim_info = transaction.execute( "select claim_id, serialized_metadata from claim where claim_outpoint=?", (claim_outpoint,) @@ -801,6 +802,19 @@ class SQLiteStorage(SQLiteMixin): if stream_hash in self.content_claim_callbacks: await self.content_claim_callbacks[stream_hash]() + async def save_torrent_content_claim(self, bt_infohash, claim_outpoint, length, name): + def _save_torrent(transaction): + transaction.execute( + "insert into torrent values (?, NULL, ?, ?)", (bt_infohash, length, name) + ).fetchall() + transaction.execute( + "insert into content_claim values (NULL, ?, ?)", (bt_infohash, claim_outpoint) + ).fetchall() + await self.db.run(_save_torrent) + # update corresponding ManagedEncryptedFileDownloader object + if bt_infohash in self.content_claim_callbacks: + await self.content_claim_callbacks[bt_infohash]() + async def get_content_claim(self, stream_hash: str, include_supports: typing.Optional[bool] = True) -> typing.Dict: claims = await self.db.run(get_claims_from_stream_hashes, [stream_hash]) claim = None @@ -812,6 +826,10 @@ class SQLiteStorage(SQLiteMixin): claim['effective_amount'] = calculate_effective_amount(claim['amount'], supports) return claim + async def get_content_claim_for_torrent(self, bt_infohash): + claims = await self.db.run(get_claims_from_torrent_info_hashes, [bt_infohash]) + return claims[bt_infohash].as_dict() if claims else None + # # # # # # # # # reflector functions # # # # # # # # # def update_reflected_stream(self, sd_hash, reflector_address, success=True): diff --git a/lbry/file/file_manager.py b/lbry/file/file_manager.py index 926296f6e..c8ef53be7 100644 --- a/lbry/file/file_manager.py +++ b/lbry/file/file_manager.py @@ -16,7 +16,7 @@ if typing.TYPE_CHECKING: from lbry.conf import Config from lbry.extras.daemon.analytics import AnalyticsManager from lbry.extras.daemon.storage import SQLiteStorage - from lbry.wallet import WalletManager + from lbry.wallet import WalletManager, Output from lbry.extras.daemon.exchange_rate_manager import ExchangeRateManager log = logging.getLogger(__name__) @@ -206,6 +206,12 @@ class FileManager: if not claim.stream.source.bt_infohash: await self.storage.save_content_claim(stream.stream_hash, outpoint) + else: + await self.storage.save_torrent_content_claim( + stream.identifier, outpoint, stream.torrent_length, stream.torrent_name + ) + claim_info = await self.storage.get_content_claim_for_torrent(stream.identifier) + stream.set_claim(claim_info, claim) if save_file: await asyncio.wait_for(stream.save_file(), timeout - (self.loop.time() - before_download), loop=self.loop) diff --git a/lbry/file/source.py b/lbry/file/source.py index 81ab0ee82..b661eb594 100644 --- a/lbry/file/source.py +++ b/lbry/file/source.py @@ -69,14 +69,14 @@ class ManagedDownloadSource: def stop_tasks(self): raise NotImplementedError() - # def set_claim(self, claim_info: typing.Dict, claim: 'Claim'): - # self.stream_claim_info = StoredContentClaim( - # f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'], - # claim_info['name'], claim_info['amount'], claim_info['height'], - # binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'], - # claim_info['claim_sequence'], claim_info.get('channel_name') - # ) - # + def set_claim(self, claim_info: typing.Dict, claim: 'Claim'): + self.stream_claim_info = StoredContentClaim( + f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'], + claim_info['name'], claim_info['amount'], claim_info['height'], + binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'], + claim_info['claim_sequence'], claim_info.get('channel_name') + ) + # async def update_content_claim(self, claim_info: Optional[typing.Dict] = None): # if not claim_info: # claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash) diff --git a/lbry/stream/managed_stream.py b/lbry/stream/managed_stream.py index 2e00a8d65..8147b5a7a 100644 --- a/lbry/stream/managed_stream.py +++ b/lbry/stream/managed_stream.py @@ -363,14 +363,6 @@ class ManagedStream(ManagedDownloadSource): await self.blob_manager.storage.update_reflected_stream(self.sd_hash, f"{host}:{port}") return sent - def set_claim(self, claim_info: typing.Dict, claim: 'Claim'): - self.stream_claim_info = StoredContentClaim( - f"{claim_info['txid']}:{claim_info['nout']}", claim_info['claim_id'], - claim_info['name'], claim_info['amount'], claim_info['height'], - binascii.hexlify(claim.to_bytes()).decode(), claim.signing_channel_id, claim_info['address'], - claim_info['claim_sequence'], claim_info.get('channel_name') - ) - async def update_content_claim(self, claim_info: Optional[typing.Dict] = None): if not claim_info: claim_info = await self.blob_manager.storage.get_content_claim(self.stream_hash) diff --git a/lbry/torrent/session.py b/lbry/torrent/session.py index 2e4c6a72e..e37b2e68f 100644 --- a/lbry/torrent/session.py +++ b/lbry/torrent/session.py @@ -57,12 +57,19 @@ class TorrentHandle: self._handle: libtorrent.torrent_handle = handle self.finished = asyncio.Event(loop=loop) self.metadata_completed = asyncio.Event(loop=loop) + self.size = 0 + self.total_wanted_done = 0 + self.name = '' def _show_status(self): # fixme: cleanup status = self._handle.status() if status.has_metadata: self.metadata_completed.set() + self._handle.pause() + self.size = status.total_wanted + self.total_wanted_done = status.total_wanted_done + self.name = status.name # metadata: libtorrent.torrent_info = self._handle.get_torrent_info() # print(metadata) # print(metadata.files()) @@ -205,6 +212,15 @@ class TorrentSession: handle = self._handles[btih] handle._handle.move_storage(download_directory) + def get_size(self, btih): + return self._handles[btih].size + + def get_name(self, btih): + return self._handles[btih].name + + def get_downloaded(self, btih): + return self._handles[btih].total_wanted_done + def get_magnet_uri(btih): return f"magnet:?xt=urn:btih:{btih}" diff --git a/lbry/torrent/torrent_manager.py b/lbry/torrent/torrent_manager.py index 118c5d897..c8f0f0b36 100644 --- a/lbry/torrent/torrent_manager.py +++ b/lbry/torrent/torrent_manager.py @@ -51,12 +51,20 @@ class TorrentSource(ManagedDownloadSource): async def save_file(self, file_name: Optional[str] = None, download_directory: Optional[str] = None): await self.torrent_session.save_file(self.identifier, download_directory) + @property + def torrent_length(self): + return self.torrent_session.get_size(self.identifier) + + @property + def torrent_name(self): + return self.torrent_session.get_name(self.identifier) + def stop_tasks(self): pass @property def completed(self): - raise NotImplementedError() + return self.torrent_session.get_downloaded(self.identifier) == self.torrent_length class TorrentManager(SourceManager): diff --git a/tests/integration/datanetwork/test_file_commands.py b/tests/integration/datanetwork/test_file_commands.py index 0d8ac3a41..b5a7ba405 100644 --- a/tests/integration/datanetwork/test_file_commands.py +++ b/tests/integration/datanetwork/test_file_commands.py @@ -34,6 +34,7 @@ class FileCommands(CommandTestCase): async def test_download_torrent(self): await self.initialize_torrent() await self.out(self.daemon.jsonrpc_get('torrent')) + self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1) async def create_streams_in_range(self, *args, **kwargs): self.stream_claim_ids = [] From a2f8e7068e152eebc3f599a29b5cb7f93fe66522 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 14 Feb 2020 13:23:33 -0300 Subject: [PATCH 40/86] pylint --- lbry/file/file_manager.py | 2 +- lbry/stream/managed_stream.py | 1 - lbry/stream/stream_manager.py | 20 ++++++++++---------- lbry/torrent/torrent_manager.py | 3 --- setup.cfg | 1 + 5 files changed, 12 insertions(+), 15 deletions(-) diff --git a/lbry/file/file_manager.py b/lbry/file/file_manager.py index c8ef53be7..fb841f401 100644 --- a/lbry/file/file_manager.py +++ b/lbry/file/file_manager.py @@ -238,7 +238,7 @@ class FileManager: # TODO: analytics for torrents pass elif self.analytics_manager and (error or (stream and (stream.downloader.time_to_descriptor or - stream.downloader.time_to_first_bytes))): + stream.downloader.time_to_first_bytes))): server = self.wallet_manager.ledger.network.client.server self.loop.create_task( self.analytics_manager.send_time_to_first_bytes( diff --git a/lbry/stream/managed_stream.py b/lbry/stream/managed_stream.py index 8147b5a7a..7d87577a8 100644 --- a/lbry/stream/managed_stream.py +++ b/lbry/stream/managed_stream.py @@ -3,7 +3,6 @@ import asyncio import time import typing import logging -import binascii from typing import Optional from aiohttp.web import Request, StreamResponse, HTTPRequestRangeNotSatisfiable from lbry.error import DownloadSDTimeoutError diff --git a/lbry/stream/stream_manager.py b/lbry/stream/stream_manager.py index 84ef5b2a1..a120316cc 100644 --- a/lbry/stream/stream_manager.py +++ b/lbry/stream/stream_manager.py @@ -230,17 +230,17 @@ class StreamManager(SourceManager): self.reflect_stream(stream) return stream - async def delete(self, stream: ManagedStream, delete_file: Optional[bool] = False): - if stream.sd_hash in self.running_reflector_uploads: - self.running_reflector_uploads[stream.sd_hash].cancel() - stream.stop_tasks() - if stream.sd_hash in self.streams: - del self.streams[stream.sd_hash] - blob_hashes = [stream.sd_hash] + [b.blob_hash for b in stream.descriptor.blobs[:-1]] + async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False): + if source.sd_hash in self.running_reflector_uploads: + self.running_reflector_uploads[source.sd_hash].cancel() + source.stop_tasks() + if source.sd_hash in self.streams: + del self.streams[source.sd_hash] + blob_hashes = [source.sd_hash] + [b.blob_hash for b in source.descriptor.blobs[:-1]] await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False) - await self.storage.delete_stream(stream.descriptor) - if delete_file and stream.output_file_exists: - os.remove(stream.full_path) + await self.storage.delete_stream(source.descriptor) + if delete_file and source.output_file_exists: + os.remove(source.full_path) async def stream_partial_content(self, request: Request, sd_hash: str): return await self._sources[sd_hash].stream_file(request, self.node) diff --git a/lbry/torrent/torrent_manager.py b/lbry/torrent/torrent_manager.py index c8f0f0b36..4868c2060 100644 --- a/lbry/torrent/torrent_manager.py +++ b/lbry/torrent/torrent_manager.py @@ -82,9 +82,6 @@ class TorrentManager(SourceManager): super().__init__(loop, config, storage, analytics_manager) self.torrent_session: 'TorrentSession' = torrent_session - def add(self, source: ManagedDownloadSource): - super().add(source) - async def recover_streams(self, file_infos: typing.List[typing.Dict]): raise NotImplementedError diff --git a/setup.cfg b/setup.cfg index c5d268dbb..8bad7f04e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,6 +12,7 @@ ignore_missing_imports = True [pylint] jobs=8 ignore=words,server,rpc,schema,winpaths.py,migrator,undecorated.py +extension-pkg-whitelist=libtorrent max-parents=10 max-args=10 max-line-length=120 From abaac8ef480d913f47de5954716957ed84acf6d6 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 24 Feb 2020 13:03:42 -0300 Subject: [PATCH 41/86] fixes from rebase, install libtorrent from s3 --- lbry/extras/daemon/json_response_encoder.py | 6 ++-- lbry/stream/stream_manager.py | 2 +- lbry/torrent/session.py | 30 ++++++------------- lbry/torrent/torrent_manager.py | 4 +++ .../datanetwork/test_file_commands.py | 4 +-- tox.ini | 1 + 6 files changed, 20 insertions(+), 27 deletions(-) diff --git a/lbry/extras/daemon/json_response_encoder.py b/lbry/extras/daemon/json_response_encoder.py index 641a7148f..6545a34d5 100644 --- a/lbry/extras/daemon/json_response_encoder.py +++ b/lbry/extras/daemon/json_response_encoder.py @@ -313,9 +313,9 @@ class JSONResponseEncoder(JSONEncoder): 'height': tx_height, 'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height, 'timestamp': self.ledger.headers.estimated_timestamp(tx_height), - 'is_fully_reflected': managed_stream.is_fully_reflected, - 'reflector_progress': managed_stream.reflector_progress, - 'uploading_to_reflector': managed_stream.uploading_to_reflector + 'is_fully_reflected': managed_stream.is_fully_reflected if is_stream else False, + 'reflector_progress': managed_stream.reflector_progress if is_stream else False, + 'uploading_to_reflector': managed_stream.uploading_to_reflector if is_stream else False } def encode_claim(self, claim): diff --git a/lbry/stream/stream_manager.py b/lbry/stream/stream_manager.py index a120316cc..3c8fcf57b 100644 --- a/lbry/stream/stream_manager.py +++ b/lbry/stream/stream_manager.py @@ -98,7 +98,7 @@ class StreamManager(SourceManager): async def _load_stream(self, rowid: int, sd_hash: str, file_name: Optional[str], download_directory: Optional[str], status: str, claim: Optional['StoredContentClaim'], content_fee: Optional['Transaction'], - added_on: Optional[int]): + added_on: Optional[int], fully_reflected: Optional[bool]): try: descriptor = await self.blob_manager.get_stream_descriptor(sd_hash) except InvalidStreamDescriptorError as err: diff --git a/lbry/torrent/session.py b/lbry/torrent/session.py index e37b2e68f..89cda059a 100644 --- a/lbry/torrent/session.py +++ b/lbry/torrent/session.py @@ -1,6 +1,7 @@ import asyncio import binascii import os +import logging from hashlib import sha1 from tempfile import mkdtemp from typing import Optional @@ -33,6 +34,7 @@ NOTIFICATION_MASKS = [ "upload", "block_progress" ] +log = logging.getLogger(__name__) DEFAULT_FLAGS = ( # fixme: somehow the logic here is inverted? @@ -65,22 +67,19 @@ class TorrentHandle: # fixme: cleanup status = self._handle.status() if status.has_metadata: - self.metadata_completed.set() - self._handle.pause() self.size = status.total_wanted self.total_wanted_done = status.total_wanted_done self.name = status.name - # metadata: libtorrent.torrent_info = self._handle.get_torrent_info() - # print(metadata) - # print(metadata.files()) - # print(type(self._handle)) + if not self.metadata_completed.is_set(): + self.metadata_completed.set() + log.info("Metadata completed for btih:%s - %s", status.info_hash, self.name) if not status.is_seeding: - print('%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d seeds: %d) %s - %s' % ( + log.debug('%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d seeds: %d) %s - %s' % ( status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000, status.num_peers, status.num_seeds, status.state, status.save_path)) elif not self.finished.is_set(): self.finished.set() - print("finished!") + log.info("Torrent finished: %s", self.name) async def status_loop(self): while True: @@ -125,7 +124,6 @@ class TorrentSession: size = myfile.write(b'0' * 40 * 1024 * 1024) fs = libtorrent.file_storage() fs.add_file('tmp', size) - print(fs.file_path(0)) t = libtorrent.create_torrent(fs, 0, 4 * 1024 * 1024) libtorrent.set_piece_hashes(t, dir) info = libtorrent.torrent_info(t.generate()) @@ -158,7 +156,7 @@ class TorrentSession: def _pop_alerts(self): for alert in self._session.pop_alerts(): - print("alert: ", alert) + log.info("torrent alert: %s", alert) async def process_alerts(self): while True: @@ -182,13 +180,6 @@ class TorrentSession: def _add_torrent(self, btih: str, download_directory: Optional[str]): params = {'info_hash': binascii.unhexlify(btih.encode())} - flags = DEFAULT_FLAGS - print(bin(flags)) - flags ^= libtorrent.add_torrent_params_flags_t.flag_paused - flags ^= libtorrent.add_torrent_params_flags_t.flag_auto_managed - # flags ^= libtorrent.add_torrent_params_flags_t.flag_stop_when_ready - print(bin(flags)) - # params['flags'] = flags if download_directory: params['save_path'] = download_directory handle = self._handles[btih] = TorrentHandle(self._loop, self._executor, self._session.add_torrent(params)) @@ -208,9 +199,8 @@ class TorrentSession: self._handles.pop(btih) async def save_file(self, btih, download_directory): - return handle = self._handles[btih] - handle._handle.move_storage(download_directory) + await handle.resume() def get_size(self, btih): return self._handles[btih].size @@ -242,9 +232,7 @@ async def main(): btih = await session.add_fake_torrent() session2._session.add_dht_node(('localhost', 4040)) await session2.add_torrent(btih, "/tmp/down") - print('added') while True: - print("idling") await asyncio.sleep(100) await session.pause() executor.shutdown() diff --git a/lbry/torrent/torrent_manager.py b/lbry/torrent/torrent_manager.py index 4868c2060..9d47a55e9 100644 --- a/lbry/torrent/torrent_manager.py +++ b/lbry/torrent/torrent_manager.py @@ -59,6 +59,10 @@ class TorrentSource(ManagedDownloadSource): def torrent_name(self): return self.torrent_session.get_name(self.identifier) + @property + def bt_infohash(self): + return self.identifier + def stop_tasks(self): pass diff --git a/tests/integration/datanetwork/test_file_commands.py b/tests/integration/datanetwork/test_file_commands.py index b5a7ba405..8aafb60f0 100644 --- a/tests/integration/datanetwork/test_file_commands.py +++ b/tests/integration/datanetwork/test_file_commands.py @@ -13,7 +13,7 @@ class FileCommands(CommandTestCase): async def initialize_torrent(self): self.seeder_session = TorrentSession(self.loop, None) self.addCleanup(self.seeder_session.stop) - await self.seeder_session.bind('localhost', 4040) + await self.seeder_session.bind(port=4040) self.btih = await self.seeder_session.add_fake_torrent() address = await self.account.receiving.get_or_create_usable_address() claim = Claim() @@ -28,7 +28,7 @@ class FileCommands(CommandTestCase): self.loop, self.daemon.conf, client_session, self.daemon.storage, self.daemon.analytics_manager ) await self.daemon.file_manager.source_managers['torrent'].start() - await client_session.bind('localhost', 4041) + await client_session.bind(port=4041) client_session._session.add_dht_node(('localhost', 4040)) async def test_download_torrent(self): diff --git a/tox.ini b/tox.ini index 697c76ed2..b055e7178 100644 --- a/tox.ini +++ b/tox.ini @@ -13,6 +13,7 @@ commands = --global-option=fetch \ --global-option=--version --global-option=3.30.1 --global-option=--all \ --global-option=build --global-option=--enable --global-option=fts5 + pip install https://s3.amazonaws.com/files.lbry.io/python_libtorrent-1.2.4-py2.py3-none-any.whl orchstr8 download blockchain: coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration.blockchain {posargs} datanetwork: coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration.datanetwork {posargs} From b73c00943c199e3090f1ada6a877f66d495ac53e Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 24 Feb 2020 13:34:13 -0300 Subject: [PATCH 42/86] linting and minor refactor --- .github/workflows/main.yml | 2 ++ Makefile | 2 +- lbry/torrent/session.py | 41 +++++++++++++++++++------------------- setup.cfg | 4 ++-- tox.ini | 3 +-- 5 files changed, 27 insertions(+), 25 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bea9fa1be..24e18f0dd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -45,6 +45,8 @@ jobs: run: | sudo apt-get update sudo apt-get install -y --no-install-recommends ffmpeg + - if: matrix.test == 'datanetwork' + run: sudo apt install -y --no-install-recommends libboost1.67-all-dev - run: pip install tox-travis - run: tox -e ${{ matrix.test }} diff --git a/Makefile b/Makefile index ab1f20ac4..65615b8b6 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: install tools lint test idea install: - pip install -e git+https://github.com/shyba/libtorrent.git#egg=python-libtorrent + pip install https://s3.amazonaws.com/files.lbry.io/python_libtorrent-1.2.4-py2.py3-none-any.whl CFLAGS="-DSQLITE_MAX_VARIABLE_NUMBER=2500000" pip install -U https://github.com/rogerbinns/apsw/releases/download/3.30.1-r1/apsw-3.30.1-r1.zip \ --global-option=fetch \ --global-option=--version --global-option=3.30.1 --global-option=--all \ diff --git a/lbry/torrent/session.py b/lbry/torrent/session.py index 89cda059a..f3bbd2c53 100644 --- a/lbry/torrent/session.py +++ b/lbry/torrent/session.py @@ -74,9 +74,9 @@ class TorrentHandle: self.metadata_completed.set() log.info("Metadata completed for btih:%s - %s", status.info_hash, self.name) if not status.is_seeding: - log.debug('%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d seeds: %d) %s - %s' % ( - status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000, - status.num_peers, status.num_seeds, status.state, status.save_path)) + log.debug('%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d seeds: %d) %s - %s', + status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000, + status.num_peers, status.num_seeds, status.state, status.save_path) elif not self.finished.is_set(): self.finished.set() log.info("Torrent finished: %s", self.name) @@ -95,7 +95,7 @@ class TorrentHandle: async def resume(self): await self._loop.run_in_executor( - self._executor, self._handle.resume + self._executor, lambda: self._handle.resume() # pylint: disable=unnecessary-lambda ) @@ -108,28 +108,15 @@ class TorrentSession: self.tasks = [] async def add_fake_torrent(self): - dir = mkdtemp() - info, btih = self._create_fake(dir) + tmpdir = mkdtemp() + info, btih = _create_fake_torrent(tmpdir) flags = libtorrent.add_torrent_params_flags_t.flag_seed_mode handle = self._session.add_torrent({ - 'ti': info, 'save_path': dir, 'flags': flags + 'ti': info, 'save_path': tmpdir, 'flags': flags }) self._handles[btih] = TorrentHandle(self._loop, self._executor, handle) return btih - def _create_fake(self, dir): - # beware, that's just for testing - path = os.path.join(dir, 'tmp') - with open(path, 'wb') as myfile: - size = myfile.write(b'0' * 40 * 1024 * 1024) - fs = libtorrent.file_storage() - fs.add_file('tmp', size) - t = libtorrent.create_torrent(fs, 0, 4 * 1024 * 1024) - libtorrent.set_piece_hashes(t, dir) - info = libtorrent.torrent_info(t.generate()) - btih = sha1(info.metadata()).hexdigest() - return info, btih - async def bind(self, interface: str = '0.0.0.0', port: int = 10889): settings = { 'listen_interfaces': f"{interface}:{port}", @@ -216,6 +203,20 @@ def get_magnet_uri(btih): return f"magnet:?xt=urn:btih:{btih}" +def _create_fake_torrent(tmpdir): + # beware, that's just for testing + path = os.path.join(tmpdir, 'tmp') + with open(path, 'wb') as myfile: + size = myfile.write(b'0' * 40 * 1024 * 1024) + file_storage = libtorrent.file_storage() + file_storage.add_file('tmp', size) + t = libtorrent.create_torrent(file_storage, 0, 4 * 1024 * 1024) + libtorrent.set_piece_hashes(t, tmpdir) + info = libtorrent.torrent_info(t.generate()) + btih = sha1(info.metadata()).hexdigest() + return info, btih + + async def main(): if os.path.exists("~/Downloads/ubuntu-18.04.3-live-server-amd64.torrent"): os.remove("~/Downloads/ubuntu-18.04.3-live-server-amd64.torrent") diff --git a/setup.cfg b/setup.cfg index 8bad7f04e..e8bc1920b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,19 +6,19 @@ source = lbry .tox/*/lib/python*/site-packages/lbry -[cryptography.*,coincurve.*,pbkdf2] +[cryptography.*,coincurve.*,pbkdf2, libtorrent] ignore_missing_imports = True [pylint] jobs=8 ignore=words,server,rpc,schema,winpaths.py,migrator,undecorated.py -extension-pkg-whitelist=libtorrent max-parents=10 max-args=10 max-line-length=120 good-names=T,t,n,i,j,k,x,y,s,f,d,h,c,e,op,db,tx,io,cachedproperty,log,id,r,iv,ts,l valid-metaclass-classmethod-first-arg=mcs disable= + c-extension-no-member, fixme, broad-except, no-else-return, diff --git a/tox.ini b/tox.ini index b055e7178..b2485aecb 100644 --- a/tox.ini +++ b/tox.ini @@ -7,8 +7,7 @@ changedir = {toxinidir}/tests setenv = HOME=/tmp commands = - pip install -U pip - pip install -e 'git+https://github.com/shyba/libtorrent.git#egg=python-libtorrent' + pip install https://s3.amazonaws.com/files.lbry.io/python_libtorrent-1.2.4-py2.py3-none-any.whl pip install https://github.com/rogerbinns/apsw/releases/download/3.30.1-r1/apsw-3.30.1-r1.zip \ --global-option=fetch \ --global-option=--version --global-option=3.30.1 --global-option=--all \ From 6d83f7e7bdb61d52dcefabb2c7c76e45b618baa2 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 24 Feb 2020 17:46:52 -0300 Subject: [PATCH 43/86] correct wheel with boost 1.65 --- .github/workflows/main.yml | 2 +- Makefile | 2 +- tox.ini | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 24e18f0dd..f50692e5c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -46,7 +46,7 @@ jobs: sudo apt-get update sudo apt-get install -y --no-install-recommends ffmpeg - if: matrix.test == 'datanetwork' - run: sudo apt install -y --no-install-recommends libboost1.67-all-dev + run: sudo apt install -y --no-install-recommends libboost1.65-all-dev - run: pip install tox-travis - run: tox -e ${{ matrix.test }} diff --git a/Makefile b/Makefile index 65615b8b6..a6221fa03 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: install tools lint test idea install: - pip install https://s3.amazonaws.com/files.lbry.io/python_libtorrent-1.2.4-py2.py3-none-any.whl + pip install https://s3.amazonaws.com/files.lbry.io/python_libtorrent-1.2.4-py3-none-any.whl CFLAGS="-DSQLITE_MAX_VARIABLE_NUMBER=2500000" pip install -U https://github.com/rogerbinns/apsw/releases/download/3.30.1-r1/apsw-3.30.1-r1.zip \ --global-option=fetch \ --global-option=--version --global-option=3.30.1 --global-option=--all \ diff --git a/tox.ini b/tox.ini index b2485aecb..55a4f4d79 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ changedir = {toxinidir}/tests setenv = HOME=/tmp commands = - pip install https://s3.amazonaws.com/files.lbry.io/python_libtorrent-1.2.4-py2.py3-none-any.whl + pip install https://s3.amazonaws.com/files.lbry.io/python_libtorrent-1.2.4-py3-none-any.whl pip install https://github.com/rogerbinns/apsw/releases/download/3.30.1-r1/apsw-3.30.1-r1.zip \ --global-option=fetch \ --global-option=--version --global-option=3.30.1 --global-option=--all \ From ce7a985df6d3eb4ca12f6c26b193e1267c3aea32 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 24 Feb 2020 18:08:41 -0300 Subject: [PATCH 44/86] add boost on gitlab, fix failing test, add libtorrent to linux build --- .gitlab-ci.yml | 5 ++++- lbry/extras/daemon/daemon.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ea24541b9..822301f64 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,6 +40,8 @@ test:unit: test:datanetwork-integration: stage: test script: + - apt-get update + - apt-get install -y libboost1.65-all-dev - pip install tox-travis - tox -e datanetwork @@ -92,8 +94,9 @@ build:linux: - apt-get install -y --no-install-recommends software-properties-common zip curl build-essential - add-apt-repository -y ppa:deadsnakes/ppa - apt-get update - - apt-get install -y --no-install-recommends python3.7-dev + - apt-get install -y --no-install-recommends python3.7-dev libboost1.65-all-dev - python3.7 <(curl -q https://bootstrap.pypa.io/get-pip.py) # make sure we get pip with python3.7 + - pip install https://s3.amazonaws.com/files.lbry.io/python_libtorrent-1.2.4-py3-none-any.whl # temporarly only on linux build:mac: extends: .build diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index 4e81b93e4..9a8c5c92c 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -3451,7 +3451,8 @@ class Daemon(metaclass=JSONRPCServerType): stream_hash = None if not preview: - old_stream = self.file_manager.get_filtered(sd_hash=old_txo.claim.stream.source.sd_hash)[0] + old_stream = self.file_manager.get_filtered(sd_hash=old_txo.claim.stream.source.sd_hash) + old_stream = old_stream[0] if old_stream else None if file_path is not None: if old_stream: await self.file_manager.delete(old_stream, delete_file=False) From 3f718e6efcc9b77be80adcdcd05a6fdf35a14bd0 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 25 Feb 2020 12:39:28 -0300 Subject: [PATCH 45/86] gitlab: use ubuntu on datanetwork tests --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 822301f64..9b6e28d56 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -39,9 +39,10 @@ test:unit: test:datanetwork-integration: stage: test + image: ubuntu:18.04 script: - apt-get update - - apt-get install -y libboost1.65-all-dev + - apt-get install -y libboost1.65-all-dev python3.7-dev - pip install tox-travis - tox -e datanetwork From f602541edee0bf28f1adb35b49173c9b21c013d8 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 25 Feb 2020 18:18:48 -0300 Subject: [PATCH 46/86] fix not knowing a torrent exists --- lbry/extras/daemon/storage.py | 4 ++-- lbry/file/file_manager.py | 15 +++++++++++---- .../integration/datanetwork/test_file_commands.py | 4 +++- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/lbry/extras/daemon/storage.py b/lbry/extras/daemon/storage.py index ce242bfe0..1387f94a7 100644 --- a/lbry/extras/daemon/storage.py +++ b/lbry/extras/daemon/storage.py @@ -805,10 +805,10 @@ class SQLiteStorage(SQLiteMixin): async def save_torrent_content_claim(self, bt_infohash, claim_outpoint, length, name): def _save_torrent(transaction): transaction.execute( - "insert into torrent values (?, NULL, ?, ?)", (bt_infohash, length, name) + "insert or replace into torrent values (?, NULL, ?, ?)", (bt_infohash, length, name) ).fetchall() transaction.execute( - "insert into content_claim values (NULL, ?, ?)", (bt_infohash, claim_outpoint) + "insert or replace into content_claim values (NULL, ?, ?)", (bt_infohash, claim_outpoint) ).fetchall() await self.db.run(_save_torrent) # update corresponding ManagedEncryptedFileDownloader object diff --git a/lbry/file/file_manager.py b/lbry/file/file_manager.py index fb841f401..ec18d273d 100644 --- a/lbry/file/file_manager.py +++ b/lbry/file/file_manager.py @@ -124,10 +124,17 @@ class FileManager: raise ResolveError(f"stream for {existing[0].claim_id} collides with existing download {txo.claim_id}") if existing: log.info("claim contains a metadata only update to a stream we have") - await self.storage.save_content_claim( - existing[0].stream_hash, outpoint - ) - await source_manager._update_content_claim(existing[0]) + if claim.stream.source.bt_infohash: + await self.storage.save_torrent_content_claim( + existing[0].identifier, outpoint, existing[0].torrent_length, existing[0].torrent_name + ) + claim_info = await self.storage.get_content_claim_for_torrent(existing[0].identifier) + existing[0].set_claim(claim_info, claim) + else: + await self.storage.save_content_claim( + existing[0].stream_hash, outpoint + ) + await source_manager._update_content_claim(existing[0]) updated_stream = existing[0] else: existing_for_claim_id = self.get_filtered(claim_id=txo.claim_id) diff --git a/tests/integration/datanetwork/test_file_commands.py b/tests/integration/datanetwork/test_file_commands.py index 8aafb60f0..15783f5f6 100644 --- a/tests/integration/datanetwork/test_file_commands.py +++ b/tests/integration/datanetwork/test_file_commands.py @@ -33,7 +33,9 @@ class FileCommands(CommandTestCase): async def test_download_torrent(self): await self.initialize_torrent() - await self.out(self.daemon.jsonrpc_get('torrent')) + self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent'))) + self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1) + self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent'))) self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1) async def create_streams_in_range(self, *args, **kwargs): From ce1eabaed6cc3aabf35bed6af6a3df9f7a060273 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 25 Feb 2020 21:18:01 -0300 Subject: [PATCH 47/86] fix moving to a new btih --- lbry/torrent/session.py | 7 +-- lbry/torrent/torrent_manager.py | 4 ++ .../datanetwork/test_file_commands.py | 50 ++++++++++++------- 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/lbry/torrent/session.py b/lbry/torrent/session.py index f3bbd2c53..d714bdbfb 100644 --- a/lbry/torrent/session.py +++ b/lbry/torrent/session.py @@ -2,6 +2,7 @@ import asyncio import binascii import os import logging +import random from hashlib import sha1 from tempfile import mkdtemp from typing import Optional @@ -179,10 +180,10 @@ class TorrentSession: self._loop.create_task(self._handles[btih].status_loop()) await self._handles[btih].metadata_completed.wait() - async def remove_torrent(self, btih, remove_files=False): + def remove_torrent(self, btih, remove_files=False): if btih in self._handles: handle = self._handles[btih] - self._session.remove_torrent(handle, 1 if remove_files else 0) + self._session.remove_torrent(handle._handle, 1 if remove_files else 0) self._handles.pop(btih) async def save_file(self, btih, download_directory): @@ -207,7 +208,7 @@ def _create_fake_torrent(tmpdir): # beware, that's just for testing path = os.path.join(tmpdir, 'tmp') with open(path, 'wb') as myfile: - size = myfile.write(b'0' * 40 * 1024 * 1024) + size = myfile.write(bytes([random.randint(0, 255) for _ in range(40)]) * 1024) file_storage = libtorrent.file_storage() file_storage.add_file('tmp', size) t = libtorrent.create_torrent(file_storage, 0, 4 * 1024 * 1024) diff --git a/lbry/torrent/torrent_manager.py b/lbry/torrent/torrent_manager.py index 9d47a55e9..c524599c7 100644 --- a/lbry/torrent/torrent_manager.py +++ b/lbry/torrent/torrent_manager.py @@ -111,6 +111,10 @@ class TorrentManager(SourceManager): super().stop() log.info("finished stopping the torrent manager") + async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False): + await super().delete(source, delete_file) + self.torrent_session.remove_torrent(source.identifier, delete_file) + async def create(self, file_path: str, key: Optional[bytes] = None, iv_generator: Optional[typing.Generator[bytes, None, None]] = None): raise NotImplementedError diff --git a/tests/integration/datanetwork/test_file_commands.py b/tests/integration/datanetwork/test_file_commands.py index 15783f5f6..095d70765 100644 --- a/tests/integration/datanetwork/test_file_commands.py +++ b/tests/integration/datanetwork/test_file_commands.py @@ -10,33 +10,49 @@ from lbry.wallet import Transaction class FileCommands(CommandTestCase): - async def initialize_torrent(self): - self.seeder_session = TorrentSession(self.loop, None) - self.addCleanup(self.seeder_session.stop) - await self.seeder_session.bind(port=4040) - self.btih = await self.seeder_session.add_fake_torrent() + async def initialize_torrent(self, tx_to_update=None): + if not hasattr(self, 'seeder_session'): + self.seeder_session = TorrentSession(self.loop, None) + self.addCleanup(self.seeder_session.stop) + await self.seeder_session.bind(port=4040) + btih = await self.seeder_session.add_fake_torrent() address = await self.account.receiving.get_or_create_usable_address() - claim = Claim() - claim.stream.update(bt_infohash=self.btih) - tx = await Transaction.claim_create( - 'torrent', claim, 1, address, [self.account], self.account) + if not tx_to_update: + claim = Claim() + claim.stream.update(bt_infohash=btih) + tx = await Transaction.claim_create( + 'torrent', claim, 1, address, [self.account], self.account + ) + else: + claim = tx_to_update.outputs[0].claim + claim.stream.update(bt_infohash=btih) + tx = await Transaction.claim_update( + tx_to_update.outputs[0], claim, 1, address, [self.account], self.account + ) await tx.sign([self.account]) await self.broadcast(tx) await self.confirm_tx(tx.id) - client_session = TorrentSession(self.loop, None) - self.daemon.file_manager.source_managers['torrent'] = TorrentManager( - self.loop, self.daemon.conf, client_session, self.daemon.storage, self.daemon.analytics_manager - ) - await self.daemon.file_manager.source_managers['torrent'].start() - await client_session.bind(port=4041) - client_session._session.add_dht_node(('localhost', 4040)) + self.client_session = self.daemon.file_manager.source_managers['torrent'].torrent_session + self.client_session._session.add_dht_node(('localhost', 4040)) + return tx, btih async def test_download_torrent(self): - await self.initialize_torrent() + tx, btih = await self.initialize_torrent() self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent'))) self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1) + # second call, see its there and move on self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent'))) self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1) + self.assertEqual((await self.daemon.jsonrpc_file_list())['items'][0].identifier, btih) + self.assertIn(btih, self.client_session._handles) + tx, new_btih = await self.initialize_torrent(tx) + self.assertNotEqual(btih, new_btih) + # claim now points to another torrent, update to it + self.assertNotIn('error', await self.out(self.daemon.jsonrpc_get('torrent'))) + self.assertEqual((await self.daemon.jsonrpc_file_list())['items'][0].identifier, new_btih) + self.assertIn(new_btih, self.client_session._handles) + self.assertNotIn(btih, self.client_session._handles) + self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1) async def create_streams_in_range(self, *args, **kwargs): self.stream_claim_ids = [] From a7c2408c0a592de1aa5c2f35b1deb7c575acdb0e Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 26 Feb 2020 03:20:26 -0300 Subject: [PATCH 48/86] fix and test delete with torrents --- lbry/file/file_manager.py | 2 +- lbry/stream/stream_manager.py | 12 +++++++----- lbry/torrent/session.py | 8 +++++++- tests/integration/datanetwork/test_file_commands.py | 3 +++ 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/lbry/file/file_manager.py b/lbry/file/file_manager.py index ec18d273d..cf4dfe5ec 100644 --- a/lbry/file/file_manager.py +++ b/lbry/file/file_manager.py @@ -283,4 +283,4 @@ class FileManager: async def delete(self, source: ManagedDownloadSource, delete_file=False): for manager in self.source_managers.values(): - return await manager.delete(source, delete_file) + await manager.delete(source, delete_file) diff --git a/lbry/stream/stream_manager.py b/lbry/stream/stream_manager.py index 3c8fcf57b..8df388452 100644 --- a/lbry/stream/stream_manager.py +++ b/lbry/stream/stream_manager.py @@ -231,12 +231,14 @@ class StreamManager(SourceManager): return stream async def delete(self, source: ManagedDownloadSource, delete_file: Optional[bool] = False): - if source.sd_hash in self.running_reflector_uploads: - self.running_reflector_uploads[source.sd_hash].cancel() + if not isinstance(source, ManagedStream): + return + if source.identifier in self.running_reflector_uploads: + self.running_reflector_uploads[source.identifier].cancel() source.stop_tasks() - if source.sd_hash in self.streams: - del self.streams[source.sd_hash] - blob_hashes = [source.sd_hash] + [b.blob_hash for b in source.descriptor.blobs[:-1]] + if source.identifier in self.streams: + del self.streams[source.identifier] + blob_hashes = [source.identifier] + [b.blob_hash for b in source.descriptor.blobs[:-1]] await self.blob_manager.delete_blobs(blob_hashes, delete_from_db=False) await self.storage.delete_stream(source.descriptor) if delete_file and source.output_file_exists: diff --git a/lbry/torrent/session.py b/lbry/torrent/session.py index d714bdbfb..e119583ed 100644 --- a/lbry/torrent/session.py +++ b/lbry/torrent/session.py @@ -63,6 +63,11 @@ class TorrentHandle: self.size = 0 self.total_wanted_done = 0 self.name = '' + self.tasks = [] + + def stop_tasks(self): + while self.tasks: + self.tasks.pop().cancel() def _show_status(self): # fixme: cleanup @@ -177,12 +182,13 @@ class TorrentSession: await self._loop.run_in_executor( self._executor, self._add_torrent, btih, download_path ) - self._loop.create_task(self._handles[btih].status_loop()) + self._handles[btih].tasks.append(self._loop.create_task(self._handles[btih].status_loop())) await self._handles[btih].metadata_completed.wait() def remove_torrent(self, btih, remove_files=False): if btih in self._handles: handle = self._handles[btih] + handle.stop_tasks() self._session.remove_torrent(handle._handle, 1 if remove_files else 0) self._handles.pop(btih) diff --git a/tests/integration/datanetwork/test_file_commands.py b/tests/integration/datanetwork/test_file_commands.py index 095d70765..bbc0dca55 100644 --- a/tests/integration/datanetwork/test_file_commands.py +++ b/tests/integration/datanetwork/test_file_commands.py @@ -53,6 +53,9 @@ class FileCommands(CommandTestCase): self.assertIn(new_btih, self.client_session._handles) self.assertNotIn(btih, self.client_session._handles) self.assertItemCount(await self.daemon.jsonrpc_file_list(), 1) + await self.daemon.jsonrpc_file_delete(delete_all=True) + self.assertItemCount(await self.daemon.jsonrpc_file_list(), 0) + self.assertNotIn(new_btih, self.client_session._handles) async def create_streams_in_range(self, *args, **kwargs): self.stream_claim_ids = [] From 6ad0242617bd7e4f93844a19ff01483a4d682357 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 26 Feb 2020 12:40:11 -0300 Subject: [PATCH 49/86] find and show largest file --- lbry/torrent/session.py | 18 ++++++++++++++++++ lbry/torrent/torrent_manager.py | 4 ++++ 2 files changed, 22 insertions(+) diff --git a/lbry/torrent/session.py b/lbry/torrent/session.py index e119583ed..6f559386c 100644 --- a/lbry/torrent/session.py +++ b/lbry/torrent/session.py @@ -64,6 +64,19 @@ class TorrentHandle: self.total_wanted_done = 0 self.name = '' self.tasks = [] + self.torrent_file: Optional[libtorrent.torrent_info] = None + self._base_path = None + + @property + def largest_file(self) -> Optional[str]: + if not self.torrent_file: + return None + largest_size, path = 0, None + for file_num in range(self.torrent_file.num_files()): + if self.torrent_file.file_size(file_num) > largest_size: + largest_size = self.torrent_file.file_size(file_num) + path = self.torrent_file.at(file_num).path + return os.path.join(self._base_path, path) def stop_tasks(self): while self.tasks: @@ -79,6 +92,8 @@ class TorrentHandle: if not self.metadata_completed.is_set(): self.metadata_completed.set() log.info("Metadata completed for btih:%s - %s", status.info_hash, self.name) + self.torrent_file = self._handle.get_torrent_info().files() + self._base_path = status.save_path if not status.is_seeding: log.debug('%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d seeds: %d) %s - %s', status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000, @@ -178,6 +193,9 @@ class TorrentSession: handle = self._handles[btih] = TorrentHandle(self._loop, self._executor, self._session.add_torrent(params)) handle._handle.force_dht_announce() + def full_path(self, btih): + return self._handles[btih].largest_file + async def add_torrent(self, btih, download_path): await self._loop.run_in_executor( self._executor, self._add_torrent, btih, download_path diff --git a/lbry/torrent/torrent_manager.py b/lbry/torrent/torrent_manager.py index c524599c7..ff38e51c6 100644 --- a/lbry/torrent/torrent_manager.py +++ b/lbry/torrent/torrent_manager.py @@ -42,6 +42,10 @@ class TorrentSource(ManagedDownloadSource): rowid, content_fee, analytics_manager, added_on) self.torrent_session = torrent_session + @property + def full_path(self) -> Optional[str]: + return self.torrent_session.full_path(self.identifier) + async def start(self, timeout: Optional[float] = None, save_now: Optional[bool] = False): await self.torrent_session.add_torrent(self.identifier, self.download_directory) From 53382b7e15c2f058108467a84bb06b904a3136cf Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 28 Feb 2020 14:58:59 -0300 Subject: [PATCH 50/86] wait started event --- lbry/extras/daemon/json_response_encoder.py | 2 +- lbry/torrent/session.py | 37 ++++++++++++++++--- lbry/torrent/torrent_manager.py | 7 +++- .../datanetwork/test_file_commands.py | 2 +- 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/lbry/extras/daemon/json_response_encoder.py b/lbry/extras/daemon/json_response_encoder.py index 6545a34d5..6bb1ed0b2 100644 --- a/lbry/extras/daemon/json_response_encoder.py +++ b/lbry/extras/daemon/json_response_encoder.py @@ -278,7 +278,7 @@ class JSONResponseEncoder(JSONEncoder): best_height = self.ledger.headers.height is_stream = hasattr(managed_stream, 'stream_hash') return { - 'streaming_url': managed_stream.stream_url if is_stream else None, + 'streaming_url': managed_stream.stream_url if is_stream else f'file://{managed_stream.full_path}', 'completed': managed_stream.completed, 'file_name': managed_stream.file_name if output_exists else None, 'download_directory': managed_stream.download_directory if output_exists else None, diff --git a/lbry/torrent/session.py b/lbry/torrent/session.py index 6f559386c..b4b47bcdb 100644 --- a/lbry/torrent/session.py +++ b/lbry/torrent/session.py @@ -58,25 +58,32 @@ class TorrentHandle: self._loop = loop self._executor = executor self._handle: libtorrent.torrent_handle = handle + self.started = asyncio.Event(loop=loop) self.finished = asyncio.Event(loop=loop) self.metadata_completed = asyncio.Event(loop=loop) self.size = 0 self.total_wanted_done = 0 self.name = '' self.tasks = [] - self.torrent_file: Optional[libtorrent.torrent_info] = None + self.torrent_file: Optional[libtorrent.file_storage] = None self._base_path = None + self._handle.set_sequential_download(1) @property def largest_file(self) -> Optional[str]: if not self.torrent_file: return None - largest_size, path = 0, None + index = self.largest_file_index + return os.path.join(self._base_path, self.torrent_file.at(index).path) + + @property + def largest_file_index(self): + largest_size, index = 0, 0 for file_num in range(self.torrent_file.num_files()): if self.torrent_file.file_size(file_num) > largest_size: largest_size = self.torrent_file.file_size(file_num) - path = self.torrent_file.at(file_num).path - return os.path.join(self._base_path, path) + index = file_num + return index def stop_tasks(self): while self.tasks: @@ -84,6 +91,8 @@ class TorrentHandle: def _show_status(self): # fixme: cleanup + if not self._handle.is_valid(): + return status = self._handle.status() if status.has_metadata: self.size = status.total_wanted @@ -94,6 +103,14 @@ class TorrentHandle: log.info("Metadata completed for btih:%s - %s", status.info_hash, self.name) self.torrent_file = self._handle.get_torrent_info().files() self._base_path = status.save_path + first_piece = self.torrent_file.at(self.largest_file_index).offset + self._handle.read_piece(first_piece) + if not self.started.is_set(): + if self._handle.have_piece(first_piece): + self.started.set() + else: + # prioritize it + self._handle.set_piece_deadline(first_piece, 100) if not status.is_seeding: log.debug('%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d seeds: %d) %s - %s', status.progress * 100, status.download_rate / 1000, status.upload_rate / 1000, @@ -127,6 +144,7 @@ class TorrentSession: self._session: Optional[libtorrent.session] = None self._handles = {} self.tasks = [] + self.wait_start = True async def add_fake_torrent(self): tmpdir = mkdtemp() @@ -190,8 +208,9 @@ class TorrentSession: params = {'info_hash': binascii.unhexlify(btih.encode())} if download_directory: params['save_path'] = download_directory - handle = self._handles[btih] = TorrentHandle(self._loop, self._executor, self._session.add_torrent(params)) - handle._handle.force_dht_announce() + handle = self._session.add_torrent(params) + handle.force_dht_announce() + self._handles[btih] = TorrentHandle(self._loop, self._executor, handle) def full_path(self, btih): return self._handles[btih].largest_file @@ -202,6 +221,9 @@ class TorrentSession: ) self._handles[btih].tasks.append(self._loop.create_task(self._handles[btih].status_loop())) await self._handles[btih].metadata_completed.wait() + if self.wait_start: + # fixme: temporary until we add streaming support, otherwise playback fails! + await self._handles[btih].started.wait() def remove_torrent(self, btih, remove_files=False): if btih in self._handles: @@ -223,6 +245,9 @@ class TorrentSession: def get_downloaded(self, btih): return self._handles[btih].total_wanted_done + def is_completed(self, btih): + return self._handles[btih].finished.is_set() + def get_magnet_uri(btih): return f"magnet:?xt=urn:btih:{btih}" diff --git a/lbry/torrent/torrent_manager.py b/lbry/torrent/torrent_manager.py index ff38e51c6..a2e1edbe4 100644 --- a/lbry/torrent/torrent_manager.py +++ b/lbry/torrent/torrent_manager.py @@ -1,6 +1,7 @@ import asyncio import binascii import logging +import os import typing from typing import Optional from aiohttp.web import Request @@ -44,7 +45,9 @@ class TorrentSource(ManagedDownloadSource): @property def full_path(self) -> Optional[str]: - return self.torrent_session.full_path(self.identifier) + full_path = self.torrent_session.full_path(self.identifier) + self.download_directory = os.path.dirname(full_path) + return full_path async def start(self, timeout: Optional[float] = None, save_now: Optional[bool] = False): await self.torrent_session.add_torrent(self.identifier, self.download_directory) @@ -72,7 +75,7 @@ class TorrentSource(ManagedDownloadSource): @property def completed(self): - return self.torrent_session.get_downloaded(self.identifier) == self.torrent_length + return self.torrent_session.is_completed(self.identifier) class TorrentManager(SourceManager): diff --git a/tests/integration/datanetwork/test_file_commands.py b/tests/integration/datanetwork/test_file_commands.py index bbc0dca55..df46c6fab 100644 --- a/tests/integration/datanetwork/test_file_commands.py +++ b/tests/integration/datanetwork/test_file_commands.py @@ -5,7 +5,6 @@ from binascii import hexlify from lbry.schema import Claim from lbry.testcase import CommandTestCase from lbry.torrent.session import TorrentSession -from lbry.torrent.torrent_manager import TorrentManager from lbry.wallet import Transaction @@ -34,6 +33,7 @@ class FileCommands(CommandTestCase): await self.confirm_tx(tx.id) self.client_session = self.daemon.file_manager.source_managers['torrent'].torrent_session self.client_session._session.add_dht_node(('localhost', 4040)) + self.client_session.wait_start = False # fixme: this is super slow on tests return tx, btih async def test_download_torrent(self): From f145d08c10d7c29f682edb78f9af6b4895629819 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 28 Feb 2020 15:22:57 -0300 Subject: [PATCH 51/86] tell progress, stop trying to read first piece --- lbry/extras/daemon/json_response_encoder.py | 6 +++--- lbry/torrent/session.py | 5 +---- lbry/torrent/torrent_manager.py | 4 ++++ 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/lbry/extras/daemon/json_response_encoder.py b/lbry/extras/daemon/json_response_encoder.py index 6bb1ed0b2..fcffb0d29 100644 --- a/lbry/extras/daemon/json_response_encoder.py +++ b/lbry/extras/daemon/json_response_encoder.py @@ -291,9 +291,9 @@ class JSONResponseEncoder(JSONEncoder): 'sd_hash': managed_stream.descriptor.sd_hash if is_stream else None, 'mime_type': managed_stream.mime_type if is_stream else None, 'key': managed_stream.descriptor.key if is_stream else None, - 'total_bytes_lower_bound': managed_stream.descriptor.lower_bound_decrypted_length() if is_stream else None, - 'total_bytes': managed_stream.descriptor.upper_bound_decrypted_length() if is_stream else None, - 'written_bytes': managed_stream.written_bytes if is_stream else None, + 'total_bytes_lower_bound': managed_stream.descriptor.lower_bound_decrypted_length() if is_stream else managed_stream.torrent_length, + 'total_bytes': managed_stream.descriptor.upper_bound_decrypted_length() if is_stream else managed_stream.torrent_length, + 'written_bytes': managed_stream.written_bytes if is_stream else managed_stream.written_bytes, 'blobs_completed': managed_stream.blobs_completed if is_stream else None, 'blobs_in_stream': managed_stream.blobs_in_stream if is_stream else None, 'blobs_remaining': managed_stream.blobs_remaining if is_stream else None, diff --git a/lbry/torrent/session.py b/lbry/torrent/session.py index b4b47bcdb..feff53f75 100644 --- a/lbry/torrent/session.py +++ b/lbry/torrent/session.py @@ -40,8 +40,6 @@ log = logging.getLogger(__name__) DEFAULT_FLAGS = ( # fixme: somehow the logic here is inverted? libtorrent.add_torrent_params_flags_t.flag_auto_managed - | libtorrent.add_torrent_params_flags_t.flag_paused - | libtorrent.add_torrent_params_flags_t.flag_duplicate_is_error | libtorrent.add_torrent_params_flags_t.flag_update_subscribe ) @@ -104,7 +102,6 @@ class TorrentHandle: self.torrent_file = self._handle.get_torrent_info().files() self._base_path = status.save_path first_piece = self.torrent_file.at(self.largest_file_index).offset - self._handle.read_piece(first_piece) if not self.started.is_set(): if self._handle.have_piece(first_piece): self.started.set() @@ -205,7 +202,7 @@ class TorrentSession: ) def _add_torrent(self, btih: str, download_directory: Optional[str]): - params = {'info_hash': binascii.unhexlify(btih.encode())} + params = {'info_hash': binascii.unhexlify(btih.encode()), 'flags': DEFAULT_FLAGS} if download_directory: params['save_path'] = download_directory handle = self._session.add_torrent(params) diff --git a/lbry/torrent/torrent_manager.py b/lbry/torrent/torrent_manager.py index a2e1edbe4..cf9106731 100644 --- a/lbry/torrent/torrent_manager.py +++ b/lbry/torrent/torrent_manager.py @@ -62,6 +62,10 @@ class TorrentSource(ManagedDownloadSource): def torrent_length(self): return self.torrent_session.get_size(self.identifier) + @property + def written_bytes(self): + return self.torrent_session.get_downloaded(self.identifier) + @property def torrent_name(self): return self.torrent_session.get_name(self.identifier) From 190b01fdf958e173af0268e8d0785a3257c0c3a9 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 28 Feb 2020 15:49:58 -0300 Subject: [PATCH 52/86] calculate total bytes outside of dict --- lbry/extras/daemon/json_response_encoder.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lbry/extras/daemon/json_response_encoder.py b/lbry/extras/daemon/json_response_encoder.py index fcffb0d29..7997eba0f 100644 --- a/lbry/extras/daemon/json_response_encoder.py +++ b/lbry/extras/daemon/json_response_encoder.py @@ -277,6 +277,11 @@ class JSONResponseEncoder(JSONEncoder): tx_height = managed_stream.stream_claim_info.height best_height = self.ledger.headers.height is_stream = hasattr(managed_stream, 'stream_hash') + if is_stream: + total_bytes_lower_bound = managed_stream.descriptor.lower_bound_decrypted_length() + total_bytes = managed_stream.descriptor.upper_bound_decrypted_length() + else: + total_bytes_lower_bound = total_bytes = managed_stream.torrent_length return { 'streaming_url': managed_stream.stream_url if is_stream else f'file://{managed_stream.full_path}', 'completed': managed_stream.completed, @@ -291,8 +296,8 @@ class JSONResponseEncoder(JSONEncoder): 'sd_hash': managed_stream.descriptor.sd_hash if is_stream else None, 'mime_type': managed_stream.mime_type if is_stream else None, 'key': managed_stream.descriptor.key if is_stream else None, - 'total_bytes_lower_bound': managed_stream.descriptor.lower_bound_decrypted_length() if is_stream else managed_stream.torrent_length, - 'total_bytes': managed_stream.descriptor.upper_bound_decrypted_length() if is_stream else managed_stream.torrent_length, + 'total_bytes_lower_bound': total_bytes_lower_bound, + 'total_bytes': total_bytes, 'written_bytes': managed_stream.written_bytes if is_stream else managed_stream.written_bytes, 'blobs_completed': managed_stream.blobs_completed if is_stream else None, 'blobs_in_stream': managed_stream.blobs_in_stream if is_stream else None, From 8811b8c1fdd3a38cd7a7ea8c4d83d919278d8e6f Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 28 Feb 2020 15:23:52 -0300 Subject: [PATCH 53/86] use ubuntu 18 on gitlab, temporarly --- .gitlab-ci.yml | 9 ++++----- tox.ini | 3 ++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9b6e28d56..ad0698a7b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -39,12 +39,11 @@ test:unit: test:datanetwork-integration: stage: test - image: ubuntu:18.04 script: - apt-get update - - apt-get install -y libboost1.65-all-dev python3.7-dev - - pip install tox-travis - - tox -e datanetwork + - apt-get install -y libboost-all-dev + - python3.7 -m pip install tox-travis + - LIBTORRENT_URL=https://s3.amazonaws.com/files.lbry.io/python_libtorrent-1.2.4-py2.py3-none-any.whl tox -e datanetwork --recreate test:blockchain-integration: stage: test @@ -87,7 +86,7 @@ test:json-api: build:linux: extends: .build - image: ubuntu:16.04 + image: ubuntu:18.04 variables: OS: linux before_script: diff --git a/tox.ini b/tox.ini index 55a4f4d79..c9d0dae71 100644 --- a/tox.ini +++ b/tox.ini @@ -6,8 +6,9 @@ extras = test changedir = {toxinidir}/tests setenv = HOME=/tmp + LIBTORRENT_URL={env:LIBTORRENT_URL:https://s3.amazonaws.com/files.lbry.io/python_libtorrent-1.2.4-py3-none-any.whl} commands = - pip install https://s3.amazonaws.com/files.lbry.io/python_libtorrent-1.2.4-py3-none-any.whl + pip install {env:LIBTORRENT_URL} pip install https://github.com/rogerbinns/apsw/releases/download/3.30.1-r1/apsw-3.30.1-r1.zip \ --global-option=fetch \ --global-option=--version --global-option=3.30.1 --global-option=--all \ From 8a4fe4f3add9b338cbf1dc4f244fafc50302d176 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 10 Mar 2020 20:24:57 -0300 Subject: [PATCH 54/86] lbry-libtorrent is now on pypi --- .github/workflows/main.yml | 2 -- .gitlab-ci.yml | 12 +++++------- tox.ini | 4 +--- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f50692e5c..bea9fa1be 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -45,8 +45,6 @@ jobs: run: | sudo apt-get update sudo apt-get install -y --no-install-recommends ffmpeg - - if: matrix.test == 'datanetwork' - run: sudo apt install -y --no-install-recommends libboost1.65-all-dev - run: pip install tox-travis - run: tox -e ${{ matrix.test }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ad0698a7b..46aa3a845 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,10 +40,8 @@ test:unit: test:datanetwork-integration: stage: test script: - - apt-get update - - apt-get install -y libboost-all-dev - - python3.7 -m pip install tox-travis - - LIBTORRENT_URL=https://s3.amazonaws.com/files.lbry.io/python_libtorrent-1.2.4-py2.py3-none-any.whl tox -e datanetwork --recreate + - pip install tox-travis + - tox -e datanetwork --recreate test:blockchain-integration: stage: test @@ -86,7 +84,7 @@ test:json-api: build:linux: extends: .build - image: ubuntu:18.04 + image: ubuntu:16.04 variables: OS: linux before_script: @@ -94,9 +92,9 @@ build:linux: - apt-get install -y --no-install-recommends software-properties-common zip curl build-essential - add-apt-repository -y ppa:deadsnakes/ppa - apt-get update - - apt-get install -y --no-install-recommends python3.7-dev libboost1.65-all-dev + - apt-get install -y --no-install-recommends python3.7-dev - python3.7 <(curl -q https://bootstrap.pypa.io/get-pip.py) # make sure we get pip with python3.7 - - pip install https://s3.amazonaws.com/files.lbry.io/python_libtorrent-1.2.4-py3-none-any.whl # temporarly only on linux + - pip install lbry-libtorrent build:mac: extends: .build diff --git a/tox.ini b/tox.ini index c9d0dae71..73d57410b 100644 --- a/tox.ini +++ b/tox.ini @@ -6,14 +6,12 @@ extras = test changedir = {toxinidir}/tests setenv = HOME=/tmp - LIBTORRENT_URL={env:LIBTORRENT_URL:https://s3.amazonaws.com/files.lbry.io/python_libtorrent-1.2.4-py3-none-any.whl} commands = - pip install {env:LIBTORRENT_URL} pip install https://github.com/rogerbinns/apsw/releases/download/3.30.1-r1/apsw-3.30.1-r1.zip \ --global-option=fetch \ --global-option=--version --global-option=3.30.1 --global-option=--all \ --global-option=build --global-option=--enable --global-option=fts5 - pip install https://s3.amazonaws.com/files.lbry.io/python_libtorrent-1.2.4-py2.py3-none-any.whl + pip install lbry-libtorrent orchstr8 download blockchain: coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration.blockchain {posargs} datanetwork: coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration.datanetwork {posargs} From 64c25b049c61b114c696ee3d0bde5b4bab2c1399 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sun, 26 Apr 2020 05:45:18 -0300 Subject: [PATCH 55/86] fixup get_filtered from rebase --- lbry/file/source_manager.py | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/lbry/file/source_manager.py b/lbry/file/source_manager.py index e3f7d4ad3..87f0a17f1 100644 --- a/lbry/file/source_manager.py +++ b/lbry/file/source_manager.py @@ -95,21 +95,40 @@ class SourceManager: raise ValueError(f"'{comparison}' is not a valid comparison") if 'full_status' in search_by: del search_by['full_status'] + for search in search_by: if search not in self.filter_fields: raise ValueError(f"'{search}' is not a valid search operation") + + compare_sets = {} + if isinstance(search_by.get('claim_id'), list): + compare_sets['claim_ids'] = search_by.pop('claim_id') + if isinstance(search_by.get('outpoint'), list): + compare_sets['outpoints'] = search_by.pop('outpoint') + if isinstance(search_by.get('channel_claim_id'), list): + compare_sets['channel_claim_ids'] = search_by.pop('channel_claim_id') + if search_by: comparison = comparison or 'eq' - sources = [] + streams = [] for stream in self._sources.values(): + matched = False + for set_search, val in compare_sets.items(): + if COMPARISON_OPERATORS[comparison](getattr(stream, self.filter_fields[set_search]), val): + streams.append(stream) + matched = True + break + if matched: + continue for search, val in search_by.items(): - if COMPARISON_OPERATORS[comparison](getattr(stream, search), val): - sources.append(stream) + this_stream = getattr(stream, search) + if COMPARISON_OPERATORS[comparison](this_stream, val): + streams.append(stream) break else: - sources = list(self._sources.values()) + streams = list(self._sources.values()) if sort_by: - sources.sort(key=lambda s: getattr(s, sort_by)) + streams.sort(key=lambda s: getattr(s, sort_by)) if reverse: - sources.reverse() - return sources + streams.reverse() + return streams From de78876b1ab7a7adff75a3edae155ae1256c76d2 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sun, 26 Apr 2020 06:15:36 -0300 Subject: [PATCH 56/86] fix test purchase --- lbry/file/file_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbry/file/file_manager.py b/lbry/file/file_manager.py index cf4dfe5ec..0c1eb7069 100644 --- a/lbry/file/file_manager.py +++ b/lbry/file/file_manager.py @@ -85,7 +85,7 @@ class FileManager: raise ResolveError("cannot download a channel claim, specify a /path") try: resolved_result = await asyncio.wait_for( - self.wallet_manager.ledger.resolve(wallet.accounts, [uri]), + self.wallet_manager.ledger.resolve(wallet.accounts, [uri], include_purchase_receipt=True), resolve_timeout ) except asyncio.TimeoutError: From c3b8f366edba1d35b49a65404193435b691fc0b9 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Sun, 26 Apr 2020 06:26:08 -0300 Subject: [PATCH 57/86] fixes from review --- lbry/extras/daemon/json_response_encoder.py | 67 ++++++++++++++------- 1 file changed, 46 insertions(+), 21 deletions(-) diff --git a/lbry/extras/daemon/json_response_encoder.py b/lbry/extras/daemon/json_response_encoder.py index 7997eba0f..99d487cd2 100644 --- a/lbry/extras/daemon/json_response_encoder.py +++ b/lbry/extras/daemon/json_response_encoder.py @@ -127,9 +127,7 @@ class JSONResponseEncoder(JSONEncoder): return self.encode_account(obj) if isinstance(obj, Wallet): return self.encode_wallet(obj) - if isinstance(obj, ManagedStream): - return self.encode_file(obj) - if isinstance(obj, TorrentSource): + if isinstance(obj, (ManagedStream, TorrentSource)): return self.encode_file(obj) if isinstance(obj, Transaction): return self.encode_transaction(obj) @@ -282,26 +280,26 @@ class JSONResponseEncoder(JSONEncoder): total_bytes = managed_stream.descriptor.upper_bound_decrypted_length() else: total_bytes_lower_bound = total_bytes = managed_stream.torrent_length - return { - 'streaming_url': managed_stream.stream_url if is_stream else f'file://{managed_stream.full_path}', + result = { + 'streaming_url': None, 'completed': managed_stream.completed, - 'file_name': managed_stream.file_name if output_exists else None, - 'download_directory': managed_stream.download_directory if output_exists else None, - 'download_path': managed_stream.full_path if output_exists else None, + 'file_name': None, + 'download_directory': None, + 'download_path': None, 'points_paid': 0.0, 'stopped': not managed_stream.running, - 'stream_hash': managed_stream.stream_hash if is_stream else None, - 'stream_name': managed_stream.descriptor.stream_name if is_stream else None, - 'suggested_file_name': managed_stream.descriptor.suggested_file_name if is_stream else None, - 'sd_hash': managed_stream.descriptor.sd_hash if is_stream else None, - 'mime_type': managed_stream.mime_type if is_stream else None, - 'key': managed_stream.descriptor.key if is_stream else None, + 'stream_hash': None, + 'stream_name': None, + 'suggested_file_name': None, + 'sd_hash': None, + 'mime_type': None, + 'key': None, 'total_bytes_lower_bound': total_bytes_lower_bound, 'total_bytes': total_bytes, - 'written_bytes': managed_stream.written_bytes if is_stream else managed_stream.written_bytes, - 'blobs_completed': managed_stream.blobs_completed if is_stream else None, - 'blobs_in_stream': managed_stream.blobs_in_stream if is_stream else None, - 'blobs_remaining': managed_stream.blobs_remaining if is_stream else None, + 'written_bytes': managed_stream.written_bytes, + 'blobs_completed': None, + 'blobs_in_stream': None, + 'blobs_remaining': None, 'status': managed_stream.status, 'claim_id': managed_stream.claim_id, 'txid': managed_stream.txid, @@ -318,10 +316,37 @@ class JSONResponseEncoder(JSONEncoder): 'height': tx_height, 'confirmations': (best_height + 1) - tx_height if tx_height > 0 else tx_height, 'timestamp': self.ledger.headers.estimated_timestamp(tx_height), - 'is_fully_reflected': managed_stream.is_fully_reflected if is_stream else False, - 'reflector_progress': managed_stream.reflector_progress if is_stream else False, - 'uploading_to_reflector': managed_stream.uploading_to_reflector if is_stream else False + 'is_fully_reflected': False, + 'reflector_progress': False, + 'uploading_to_reflector': False } + if is_stream: + result.update({ + 'streaming_url': managed_stream.stream_url, + 'stream_hash': managed_stream.stream_hash, + 'stream_name': managed_stream.descriptor.stream_name, + 'suggested_file_name': managed_stream.descriptor.suggested_file_name, + 'sd_hash': managed_stream.descriptor.sd_hash, + 'mime_type': managed_stream.mime_type, + 'key': managed_stream.descriptor.key, + 'blobs_completed': managed_stream.blobs_completed, + 'blobs_in_stream': managed_stream.blobs_in_stream, + 'blobs_remaining': managed_stream.blobs_remaining, + 'is_fully_reflected': managed_stream.is_fully_reflected, + 'reflector_progress': managed_stream.reflector_progress, + 'uploading_to_reflector': managed_stream.uploading_to_reflector + }) + else: + result.update({ + 'streaming_url': f'file://{managed_stream.full_path}', + }) + if output_exists: + result.update({ + 'file_name': managed_stream.file_name, + 'download_directory': managed_stream.download_directory, + 'download_path': managed_stream.full_path, + }) + return result def encode_claim(self, claim): encoded = getattr(claim, claim.claim_type).to_dict() From ced3c7efe43753651b57c26590097590791f27eb Mon Sep 17 00:00:00 2001 From: Luiz <34900176+lpessin@users.noreply.github.com> Date: Thu, 7 May 2020 13:37:46 -0300 Subject: [PATCH 58/86] fix duplicate line on api doc delete line 3616 (duplicate line 3618) --- lbry/extras/daemon/daemon.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index 5c84f8227..594142391 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -3613,7 +3613,6 @@ class Daemon(metaclass=JSONRPCServerType): given name. default: false. --title= : (str) title of the collection --description=<description> : (str) description of the collection - --clear_languages : (bool) clear existing languages (prior to adding new ones) --tags=<tags> : (list) content tags --clear_languages : (bool) clear existing languages (prior to adding new ones) --languages=<languages> : (list) languages used by the collection, From 1cd5377b45ddd1f03d6423f8294879b2b9368930 Mon Sep 17 00:00:00 2001 From: Jack Robison <jackrobison@lbry.io> Date: Fri, 8 May 2020 10:58:29 -0400 Subject: [PATCH 59/86] split fixed peer setting out from reflector_servers --- lbry/conf.py | 7 ++++++- lbry/extras/daemon/analytics.py | 2 +- lbry/stream/downloader.py | 8 ++++---- lbry/testcase.py | 1 + tests/unit/blob_exchange/test_transfer_blob.py | 4 ++-- tests/unit/stream/test_stream_manager.py | 4 ++-- 6 files changed, 16 insertions(+), 10 deletions(-) diff --git a/lbry/conf.py b/lbry/conf.py index 6e1081a91..2b0e7003f 100644 --- a/lbry/conf.py +++ b/lbry/conf.py @@ -577,9 +577,14 @@ class Config(CLIConfig): ) # servers - reflector_servers = Servers("Reflector re-hosting servers", [ + reflector_servers = Servers("Reflector re-hosting servers for mirroring publishes", [ ('reflector.lbry.com', 5566) ]) + + fixed_peers = Servers("Fixed peers to fall back to if none are found on P2P for a blob", [ + ('reflector.lbry.com', 5567) + ]) + lbryum_servers = Servers("SPV wallet servers", [ ('spv11.lbry.com', 50001), ('spv12.lbry.com', 50001), diff --git a/lbry/extras/daemon/analytics.py b/lbry/extras/daemon/analytics.py index 828112396..f6983016c 100644 --- a/lbry/extras/daemon/analytics.py +++ b/lbry/extras/daemon/analytics.py @@ -66,7 +66,7 @@ def _download_properties(conf: Config, external_ip: str, resolve_duration: float "node_rpc_timeout": conf.node_rpc_timeout, "peer_connect_timeout": conf.peer_connect_timeout, "blob_download_timeout": conf.blob_download_timeout, - "use_fixed_peers": len(conf.reflector_servers) > 0, + "use_fixed_peers": len(conf.fixed_peers) > 0, "fixed_peer_delay": fixed_peer_delay, "added_fixed_peers": added_fixed_peers, "active_peer_count": active_peer_count, diff --git a/lbry/stream/downloader.py b/lbry/stream/downloader.py index 588263b0e..94537e034 100644 --- a/lbry/stream/downloader.py +++ b/lbry/stream/downloader.py @@ -51,15 +51,15 @@ class StreamDownloader: def _delayed_add_fixed_peers(): self.added_fixed_peers = True self.peer_queue.put_nowait([ - make_kademlia_peer(None, address, None, tcp_port=port + 1, allow_localhost=True) + make_kademlia_peer(None, address, None, tcp_port=port, allow_localhost=True) for address, port in addresses ]) - if not self.config.reflector_servers: + if not self.config.fixed_peers: return addresses = [ - (await resolve_host(url, port + 1, proto='tcp'), port) - for url, port in self.config.reflector_servers + (await resolve_host(url, port, proto='tcp'), port) + for url, port in self.config.fixed_peers ] if 'dht' in self.config.components_to_skip or not self.node or not \ len(self.node.protocol.routing_table.get_peers()) > 0: diff --git a/lbry/testcase.py b/lbry/testcase.py index dcdaa83e5..6dc1e2eb9 100644 --- a/lbry/testcase.py +++ b/lbry/testcase.py @@ -386,6 +386,7 @@ class CommandTestCase(IntegrationTestCase): conf.blockchain_name = 'lbrycrd_regtest' conf.lbryum_servers = [('127.0.0.1', 50001)] conf.reflector_servers = [('127.0.0.1', 5566)] + conf.fixed_peers = [('127.0.0.1', 5567)] conf.known_dht_nodes = [] conf.blob_lru_cache_size = self.blob_lru_cache_size conf.components_to_skip = [ diff --git a/tests/unit/blob_exchange/test_transfer_blob.py b/tests/unit/blob_exchange/test_transfer_blob.py index fab7a4db0..b6339f375 100644 --- a/tests/unit/blob_exchange/test_transfer_blob.py +++ b/tests/unit/blob_exchange/test_transfer_blob.py @@ -34,13 +34,13 @@ class BlobExchangeTestBase(AsyncioTestCase): self.addCleanup(shutil.rmtree, self.client_dir) self.addCleanup(shutil.rmtree, self.server_dir) self.server_config = Config(data_dir=self.server_dir, download_dir=self.server_dir, wallet=self.server_dir, - reflector_servers=[]) + fixed_peers=[]) self.server_storage = SQLiteStorage(self.server_config, os.path.join(self.server_dir, "lbrynet.sqlite")) self.server_blob_manager = BlobManager(self.loop, self.server_dir, self.server_storage, self.server_config) self.server = BlobServer(self.loop, self.server_blob_manager, 'bQEaw42GXsgCAGio1nxFncJSyRmnztSCjP') self.client_config = Config(data_dir=self.client_dir, download_dir=self.client_dir, wallet=self.client_dir, - reflector_servers=[]) + fixed_peers=[]) self.client_storage = SQLiteStorage(self.client_config, os.path.join(self.client_dir, "lbrynet.sqlite")) self.client_blob_manager = BlobManager(self.loop, self.client_dir, self.client_storage, self.client_config) self.client_peer_manager = PeerManager(self.loop) diff --git a/tests/unit/stream/test_stream_manager.py b/tests/unit/stream/test_stream_manager.py index 3299bcb4d..91bece2eb 100644 --- a/tests/unit/stream/test_stream_manager.py +++ b/tests/unit/stream/test_stream_manager.py @@ -187,7 +187,7 @@ class TestStreamManager(BlobExchangeTestBase): await self._test_time_to_first_bytes(check_post) async def test_fixed_peer_delay_dht_peers_found(self): - self.client_config.reflector_servers = [(self.server_from_client.address, self.server_from_client.tcp_port - 1)] + self.client_config.fixed_peers = [(self.server_from_client.address, self.server_from_client.tcp_port - 1)] server_from_client = None self.server_from_client, server_from_client = server_from_client, self.server_from_client @@ -231,7 +231,7 @@ class TestStreamManager(BlobExchangeTestBase): await self._test_time_to_first_bytes(check_post, DownloadSDTimeoutError, after_setup=after_setup) async def test_override_fixed_peer_delay_dht_disabled(self): - self.client_config.reflector_servers = [(self.server_from_client.address, self.server_from_client.tcp_port - 1)] + self.client_config.fixed_peers = [(self.server_from_client.address, self.server_from_client.tcp_port - 1)] self.client_config.components_to_skip = ['dht', 'hash_announcer'] self.client_config.fixed_peer_delay = 9001.0 self.server_from_client = None From ebbb182537cd33e222fe3b8d7e13eb0e1e5b674b Mon Sep 17 00:00:00 2001 From: Jack Robison <jackrobison@lbry.io> Date: Mon, 11 May 2020 14:06:23 -0400 Subject: [PATCH 60/86] fix test --- tests/unit/stream/test_stream_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/stream/test_stream_manager.py b/tests/unit/stream/test_stream_manager.py index 91bece2eb..8c17f61e4 100644 --- a/tests/unit/stream/test_stream_manager.py +++ b/tests/unit/stream/test_stream_manager.py @@ -187,7 +187,7 @@ class TestStreamManager(BlobExchangeTestBase): await self._test_time_to_first_bytes(check_post) async def test_fixed_peer_delay_dht_peers_found(self): - self.client_config.fixed_peers = [(self.server_from_client.address, self.server_from_client.tcp_port - 1)] + self.client_config.fixed_peers = [(self.server_from_client.address, self.server_from_client.tcp_port)] server_from_client = None self.server_from_client, server_from_client = server_from_client, self.server_from_client @@ -231,7 +231,7 @@ class TestStreamManager(BlobExchangeTestBase): await self._test_time_to_first_bytes(check_post, DownloadSDTimeoutError, after_setup=after_setup) async def test_override_fixed_peer_delay_dht_disabled(self): - self.client_config.fixed_peers = [(self.server_from_client.address, self.server_from_client.tcp_port - 1)] + self.client_config.fixed_peers = [(self.server_from_client.address, self.server_from_client.tcp_port)] self.client_config.components_to_skip = ['dht', 'hash_announcer'] self.client_config.fixed_peer_delay = 9001.0 self.server_from_client = None From 4d58648c024d16d0e02622d80d1d4622d0072e78 Mon Sep 17 00:00:00 2001 From: Jack Robison <jackrobison@lbry.io> Date: Mon, 11 May 2020 14:52:31 -0400 Subject: [PATCH 61/86] update default fixed peer to cdn.reflector.lbry.com --- lbry/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbry/conf.py b/lbry/conf.py index 2b0e7003f..afd648193 100644 --- a/lbry/conf.py +++ b/lbry/conf.py @@ -582,7 +582,7 @@ class Config(CLIConfig): ]) fixed_peers = Servers("Fixed peers to fall back to if none are found on P2P for a blob", [ - ('reflector.lbry.com', 5567) + ('cdn.reflector.lbry.com', 5567) ]) lbryum_servers = Servers("SPV wallet servers", [ From c22482f90781aa03fb55ad555d5a6bbae0834cb0 Mon Sep 17 00:00:00 2001 From: Jack Robison <jackrobison@lbry.io> Date: Mon, 11 May 2020 14:10:34 -0400 Subject: [PATCH 62/86] channel private key generation in a thread pool --- lbry/extras/daemon/daemon.py | 4 +-- lbry/wallet/account.py | 6 ++-- lbry/wallet/database.py | 2 +- lbry/wallet/transaction.py | 7 +++-- .../test_internal_transaction_api.py | 2 +- tests/unit/comments/test_comment_signing.py | 20 ++++++------- tests/unit/wallet/test_schema_signing.py | 28 +++++++++---------- 7 files changed, 37 insertions(+), 32 deletions(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index d83f73abb..67df0ae8d 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -2550,7 +2550,7 @@ class Daemon(metaclass=JSONRPCServerType): name, claim, amount, claim_address, funding_accounts, funding_accounts[0] ) txo = tx.outputs[0] - txo.generate_channel_private_key() + await txo.generate_channel_private_key() await tx.sign(funding_accounts) @@ -2702,7 +2702,7 @@ class Daemon(metaclass=JSONRPCServerType): new_txo = tx.outputs[0] if new_signing_key: - new_txo.generate_channel_private_key() + await new_txo.generate_channel_private_key() else: new_txo.private_key = old_txo.private_key diff --git a/lbry/wallet/account.py b/lbry/wallet/account.py index 3a3d4c3f3..fdefde985 100644 --- a/lbry/wallet/account.py +++ b/lbry/wallet/account.py @@ -525,11 +525,13 @@ class Account: channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes) self.channel_keys[channel_pubkey_hash] = private_key.to_pem().decode() - def get_channel_private_key(self, public_key_bytes): + async def get_channel_private_key(self, public_key_bytes): channel_pubkey_hash = self.ledger.public_key_to_address(public_key_bytes) private_key_pem = self.channel_keys.get(channel_pubkey_hash) if private_key_pem: - return ecdsa.SigningKey.from_pem(private_key_pem, hashfunc=sha256) + return await asyncio.get_event_loop().run_in_executor( + None, ecdsa.SigningKey.from_pem, private_key_pem, sha256 + ) async def maybe_migrate_certificates(self): def to_der(private_key_pem): diff --git a/lbry/wallet/database.py b/lbry/wallet/database.py index 9c8ad7695..15c866017 100644 --- a/lbry/wallet/database.py +++ b/lbry/wallet/database.py @@ -918,7 +918,7 @@ class Database(SQLiteMixin): channel_ids.add(txo.claim.signing_channel_id) if txo.claim.is_channel and wallet: for account in wallet.accounts: - private_key = account.get_channel_private_key( + private_key = await account.get_channel_private_key( txo.claim.channel.public_key_bytes ) if private_key: diff --git a/lbry/wallet/transaction.py b/lbry/wallet/transaction.py index df9beddcf..0f32ffdd3 100644 --- a/lbry/wallet/transaction.py +++ b/lbry/wallet/transaction.py @@ -2,6 +2,7 @@ import struct import hashlib import logging import typing +import asyncio from binascii import hexlify, unhexlify from typing import List, Iterable, Optional, Tuple @@ -412,8 +413,10 @@ class Output(InputOutput): self.channel = None self.claim.clear_signature() - def generate_channel_private_key(self): - self.private_key = ecdsa.SigningKey.generate(curve=ecdsa.SECP256k1, hashfunc=hashlib.sha256) + async def generate_channel_private_key(self): + self.private_key = await asyncio.get_event_loop().run_in_executor( + None, ecdsa.SigningKey.generate, ecdsa.SECP256k1, None, hashlib.sha256 + ) self.claim.channel.public_key_bytes = self.private_key.get_verifying_key().to_der() self.script.generate() return self.private_key diff --git a/tests/integration/blockchain/test_internal_transaction_api.py b/tests/integration/blockchain/test_internal_transaction_api.py index 2bb4ac944..6eba5e229 100644 --- a/tests/integration/blockchain/test_internal_transaction_api.py +++ b/tests/integration/blockchain/test_internal_transaction_api.py @@ -31,7 +31,7 @@ class BasicTransactionTest(IntegrationTestCase): channel_txo = Output.pay_claim_name_pubkey_hash( l2d('1.0'), '@bar', channel, self.account.ledger.address_to_hash160(address1) ) - channel_txo.generate_channel_private_key() + await channel_txo.generate_channel_private_key() channel_txo.script.generate() channel_tx = await Transaction.create([], [channel_txo], [self.account], self.account) diff --git a/tests/unit/comments/test_comment_signing.py b/tests/unit/comments/test_comment_signing.py index 9cdfd3d69..ceee8a8a2 100644 --- a/tests/unit/comments/test_comment_signing.py +++ b/tests/unit/comments/test_comment_signing.py @@ -18,31 +18,31 @@ class TestSigningComments(AsyncioTestCase): 'comment_id': hashlib.sha256(comment.encode()).hexdigest() } - def test01_successful_create_sign_and_validate_comment(self): - channel = get_channel('@BusterBluth') + async def test01_successful_create_sign_and_validate_comment(self): + channel = await get_channel('@BusterBluth') stream = get_stream('pop secret') comment = self.create_claim_comment_body('Cool stream', stream, channel) sign_comment(comment, channel) self.assertTrue(is_comment_signed_by_channel(comment, channel)) - def test02_fail_to_validate_spoofed_channel(self): - pdiddy = get_channel('@PDitty') - channel2 = get_channel('@TomHaverford') + async def test02_fail_to_validate_spoofed_channel(self): + pdiddy = await get_channel('@PDitty') + channel2 = await get_channel('@TomHaverford') stream = get_stream() comment = self.create_claim_comment_body('Woahh This is Sick!! Shout out 2 my boy Tommy H', stream, pdiddy) sign_comment(comment, channel2) self.assertFalse(is_comment_signed_by_channel(comment, pdiddy)) - def test03_successful_sign_abandon_comment(self): - rswanson = get_channel('@RonSwanson') + async def test03_successful_sign_abandon_comment(self): + rswanson = await get_channel('@RonSwanson') dsilver = get_stream('Welcome to the Pawnee, and give a big round for Ron Swanson, AKA Duke Silver') comment_body = self.create_claim_comment_body('COMPUTER, DELETE ALL VIDEOS OF RON.', dsilver, rswanson) sign_comment(comment_body, rswanson, abandon=True) self.assertTrue(is_comment_signed_by_channel(comment_body, rswanson, abandon=True)) - def test04_invalid_signature(self): - rswanson = get_channel('@RonSwanson') - jeanralphio = get_channel('@JeanRalphio') + async def test04_invalid_signature(self): + rswanson = await get_channel('@RonSwanson') + jeanralphio = await get_channel('@JeanRalphio') chair = get_stream('This is a nice chair. I made it with Mahogany wood and this electric saw') chair_comment = self.create_claim_comment_body( 'Hah. You use an electric saw? Us swansons have been making chairs with handsaws just three after birth.', diff --git a/tests/unit/wallet/test_schema_signing.py b/tests/unit/wallet/test_schema_signing.py index dbe31943e..08b61ce9d 100644 --- a/tests/unit/wallet/test_schema_signing.py +++ b/tests/unit/wallet/test_schema_signing.py @@ -21,9 +21,9 @@ def get_tx(): return Transaction().add_inputs([get_input()]) -def get_channel(claim_name='@foo'): +async def get_channel(claim_name='@foo'): channel_txo = Output.pay_claim_name_pubkey_hash(CENT, claim_name, Claim(), b'abc') - channel_txo.generate_channel_private_key() + await channel_txo.generate_channel_private_key() get_tx().add_outputs([channel_txo]) return channel_txo @@ -36,32 +36,32 @@ def get_stream(claim_name='foo'): class TestSigningAndValidatingClaim(AsyncioTestCase): - def test_successful_create_sign_and_validate(self): - channel = get_channel() + async def test_successful_create_sign_and_validate(self): + channel = await get_channel() stream = get_stream() stream.sign(channel) self.assertTrue(stream.is_signed_by(channel)) - def test_fail_to_validate_on_wrong_channel(self): + async def test_fail_to_validate_on_wrong_channel(self): stream = get_stream() - stream.sign(get_channel()) - self.assertFalse(stream.is_signed_by(get_channel())) + stream.sign(await get_channel()) + self.assertFalse(stream.is_signed_by(await get_channel())) - def test_fail_to_validate_altered_claim(self): - channel = get_channel() + async def test_fail_to_validate_altered_claim(self): + channel = await get_channel() stream = get_stream() stream.sign(channel) self.assertTrue(stream.is_signed_by(channel)) stream.claim.stream.title = 'hello' self.assertFalse(stream.is_signed_by(channel)) - def test_valid_private_key_for_cert(self): - channel = get_channel() + async def test_valid_private_key_for_cert(self): + channel = await get_channel() self.assertTrue(channel.is_channel_private_key(channel.private_key)) - def test_fail_to_load_wrong_private_key_for_cert(self): - channel = get_channel() - self.assertFalse(channel.is_channel_private_key(get_channel().private_key)) + async def test_fail_to_load_wrong_private_key_for_cert(self): + channel = await get_channel() + self.assertFalse(channel.is_channel_private_key((await get_channel()).private_key)) class TestValidatingOldSignatures(AsyncioTestCase): From f20ca70c013534266c9cbf6bcae4ce4d7b625fa0 Mon Sep 17 00:00:00 2001 From: Jack Robison <jackrobison@lbry.io> Date: Mon, 11 May 2020 15:48:34 -0400 Subject: [PATCH 63/86] add `uploading_to_reflector` and `is_fully_reflected` filter arguments to `file_list` --- lbry/extras/daemon/daemon.py | 5 ++++- lbry/stream/managed_stream.py | 1 - lbry/stream/stream_manager.py | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index 67df0ae8d..c0b333fcd 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -1965,7 +1965,8 @@ class Daemon(metaclass=JSONRPCServerType): [--outpoint=<outpoint>] [--txid=<txid>] [--nout=<nout>] [--channel_claim_id=<channel_claim_id>] [--channel_name=<channel_name>] [--claim_name=<claim_name>] [--blobs_in_stream=<blobs_in_stream>] - [--blobs_remaining=<blobs_remaining>] [--sort=<sort_by>] + [--blobs_remaining=<blobs_remaining>] [--uploading_to_reflector=<uploading_to_reflector>] + [--is_fully_reflected=<is_fully_reflected>] [--sort=<sort_by>] [--comparison=<comparison>] [--full_status=<full_status>] [--reverse] [--page=<page>] [--page_size=<page_size>] [--wallet_id=<wallet_id>] @@ -1984,6 +1985,8 @@ class Daemon(metaclass=JSONRPCServerType): --channel_name=<channel_name> : (str) get file with matching channel name --claim_name=<claim_name> : (str) get file with matching claim name --blobs_in_stream<blobs_in_stream> : (int) get file with matching blobs in stream + --uploading_to_reflector=<uploading_to_reflector> : (bool) get files currently uploading to reflector + --is_fully_reflected=<is_fully_reflected> : (bool) get files that have been uploaded to reflector --blobs_remaining=<blobs_remaining> : (int) amount of remaining blobs to download --sort=<sort_by> : (str) field to sort by (one of the above filter fields) --comparison=<comparison> : (str) logical comparison, (eq | ne | g | ge | l | le | in) diff --git a/lbry/stream/managed_stream.py b/lbry/stream/managed_stream.py index 7d87577a8..debd987a4 100644 --- a/lbry/stream/managed_stream.py +++ b/lbry/stream/managed_stream.py @@ -57,7 +57,6 @@ class ManagedStream(ManagedDownloadSource): self.downloader = StreamDownloader(self.loop, self.config, self.blob_manager, sd_hash, descriptor) self.analytics_manager = analytics_manager - self.fully_reflected = asyncio.Event(loop=self.loop) self.reflector_progress = 0 self.uploading_to_reflector = False self.file_output_task: typing.Optional[asyncio.Task] = None diff --git a/lbry/stream/stream_manager.py b/lbry/stream/stream_manager.py index 8df388452..4d0d1093b 100644 --- a/lbry/stream/stream_manager.py +++ b/lbry/stream/stream_manager.py @@ -38,7 +38,9 @@ class StreamManager(SourceManager): 'stream_hash', 'full_status', # TODO: remove 'blobs_remaining', - 'blobs_in_stream' + 'blobs_in_stream', + 'uploading_to_reflector', + 'is_fully_reflected' }) def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', blob_manager: 'BlobManager', From 78b8261a3a1216cccf775950cf92a354acd7fb71 Mon Sep 17 00:00:00 2001 From: Jack Robison <jackrobison@lbry.io> Date: Mon, 11 May 2020 16:05:13 -0400 Subject: [PATCH 64/86] cancel pending reflector request when connection is lost -add 180s timeout --- lbry/stream/reflector/client.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lbry/stream/reflector/client.py b/lbry/stream/reflector/client.py index 7a8032c99..fbc084b9b 100644 --- a/lbry/stream/reflector/client.py +++ b/lbry/stream/reflector/client.py @@ -35,6 +35,8 @@ class StreamReflectorClient(asyncio.Protocol): def connection_lost(self, exc: typing.Optional[Exception]): self.transport = None self.connected.clear() + if self.pending_request: + self.pending_request.cancel() if self.reflected_blobs: log.info("Finished sending reflector %i blobs", len(self.reflected_blobs)) @@ -56,11 +58,11 @@ class StreamReflectorClient(asyncio.Protocol): self.response_buff = b'' return - async def send_request(self, request_dict: typing.Dict): + async def send_request(self, request_dict: typing.Dict, timeout: int = 180): msg = json.dumps(request_dict) self.transport.write(msg.encode()) try: - self.pending_request = self.loop.create_task(self.response_queue.get()) + self.pending_request = self.loop.create_task(asyncio.wait_for(self.response_queue.get(), timeout)) return await self.pending_request finally: self.pending_request = None @@ -87,7 +89,7 @@ class StreamReflectorClient(asyncio.Protocol): sent_sd = False if response['send_sd_blob']: await sd_blob.sendfile(self) - received = await self.response_queue.get() + received = await asyncio.wait_for(self.response_queue.get(), 30) if received.get('received_sd_blob'): sent_sd = True if not needed: @@ -111,7 +113,7 @@ class StreamReflectorClient(asyncio.Protocol): raise ValueError("I don't know whether to send the blob or not!") if response['send_blob']: await blob.sendfile(self) - received = await self.response_queue.get() + received = await asyncio.wait_for(self.response_queue.get(), 30) if received.get('received_blob'): self.reflected_blobs.append(blob.blob_hash) log.info("Sent reflector blob %s", blob.blob_hash[:8]) From a469b8bc042f7d01a8bd80050049d671acf9e470 Mon Sep 17 00:00:00 2001 From: Jack Robison <jackrobison@lbry.io> Date: Mon, 11 May 2020 18:43:47 -0400 Subject: [PATCH 65/86] return streams matching all file_list filters rather than those matching any -fix filter fields when using sets --- lbry/file/source_manager.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/lbry/file/source_manager.py b/lbry/file/source_manager.py index 87f0a17f1..0fadeb346 100644 --- a/lbry/file/source_manager.py +++ b/lbry/file/source_manager.py @@ -37,6 +37,12 @@ class SourceManager: 'channel_name' } + set_filter_fields = { + "claim_ids": "claim_id", + "channel_claim_ids": "channel_claim_id", + "outpoints": "outpoint" + } + source_class = ManagedDownloadSource def __init__(self, loop: asyncio.AbstractEventLoop, config: 'Config', storage: 'SQLiteStorage', @@ -108,23 +114,19 @@ class SourceManager: if isinstance(search_by.get('channel_claim_id'), list): compare_sets['channel_claim_ids'] = search_by.pop('channel_claim_id') - if search_by: + if search_by or compare_sets: comparison = comparison or 'eq' streams = [] for stream in self._sources.values(): - matched = False - for set_search, val in compare_sets.items(): - if COMPARISON_OPERATORS[comparison](getattr(stream, self.filter_fields[set_search]), val): - streams.append(stream) - matched = True - break - if matched: + if compare_sets and not all( + getattr(stream, self.set_filter_fields[set_search]) in val + for set_search, val in compare_sets.items()): continue - for search, val in search_by.items(): - this_stream = getattr(stream, search) - if COMPARISON_OPERATORS[comparison](this_stream, val): - streams.append(stream) - break + if search_by and not all( + COMPARISON_OPERATORS[comparison](getattr(stream, search), val) + for search, val in search_by.items()): + continue + streams.append(stream) else: streams = list(self._sources.values()) if sort_by: From 3c85322523a2e4665b37c026c6af5e0aadda6edc Mon Sep 17 00:00:00 2001 From: Jack Robison <jackrobison@lbry.io> Date: Mon, 11 May 2020 19:16:08 -0400 Subject: [PATCH 66/86] add `status` arg to `file_list` cli --- lbry/extras/daemon/daemon.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index c0b333fcd..8352bf047 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -1966,7 +1966,7 @@ class Daemon(metaclass=JSONRPCServerType): [--channel_claim_id=<channel_claim_id>] [--channel_name=<channel_name>] [--claim_name=<claim_name>] [--blobs_in_stream=<blobs_in_stream>] [--blobs_remaining=<blobs_remaining>] [--uploading_to_reflector=<uploading_to_reflector>] - [--is_fully_reflected=<is_fully_reflected>] [--sort=<sort_by>] + [--is_fully_reflected=<is_fully_reflected>] [--status=<status>] [--sort=<sort_by>] [--comparison=<comparison>] [--full_status=<full_status>] [--reverse] [--page=<page>] [--page_size=<page_size>] [--wallet_id=<wallet_id>] @@ -1987,6 +1987,7 @@ class Daemon(metaclass=JSONRPCServerType): --blobs_in_stream<blobs_in_stream> : (int) get file with matching blobs in stream --uploading_to_reflector=<uploading_to_reflector> : (bool) get files currently uploading to reflector --is_fully_reflected=<is_fully_reflected> : (bool) get files that have been uploaded to reflector + --status=<status> : (str) match by status, ( running | finished | stopped ) --blobs_remaining=<blobs_remaining> : (int) amount of remaining blobs to download --sort=<sort_by> : (str) field to sort by (one of the above filter fields) --comparison=<comparison> : (str) logical comparison, (eq | ne | g | ge | l | le | in) From b000a40f28e2101c16f1e3ff5ff745f669b005a5 Mon Sep 17 00:00:00 2001 From: Jack Robison <jackrobison@lbry.io> Date: Mon, 11 May 2020 19:22:53 -0400 Subject: [PATCH 67/86] add `completed` filter arg to `file_list` --- lbry/extras/daemon/daemon.py | 5 +++-- lbry/file/source_manager.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index 8352bf047..fd4627987 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -1966,8 +1966,8 @@ class Daemon(metaclass=JSONRPCServerType): [--channel_claim_id=<channel_claim_id>] [--channel_name=<channel_name>] [--claim_name=<claim_name>] [--blobs_in_stream=<blobs_in_stream>] [--blobs_remaining=<blobs_remaining>] [--uploading_to_reflector=<uploading_to_reflector>] - [--is_fully_reflected=<is_fully_reflected>] [--status=<status>] [--sort=<sort_by>] - [--comparison=<comparison>] [--full_status=<full_status>] [--reverse] + [--is_fully_reflected=<is_fully_reflected>] [--status=<status>] [--completed=<completed>] + [--sort=<sort_by>] [--comparison=<comparison>] [--full_status=<full_status>] [--reverse] [--page=<page>] [--page_size=<page_size>] [--wallet_id=<wallet_id>] Options: @@ -1988,6 +1988,7 @@ class Daemon(metaclass=JSONRPCServerType): --uploading_to_reflector=<uploading_to_reflector> : (bool) get files currently uploading to reflector --is_fully_reflected=<is_fully_reflected> : (bool) get files that have been uploaded to reflector --status=<status> : (str) match by status, ( running | finished | stopped ) + --completed=<completed> : (bool) match only completed --blobs_remaining=<blobs_remaining> : (int) amount of remaining blobs to download --sort=<sort_by> : (str) field to sort by (one of the above filter fields) --comparison=<comparison> : (str) logical comparison, (eq | ne | g | ge | l | le | in) diff --git a/lbry/file/source_manager.py b/lbry/file/source_manager.py index 0fadeb346..bf2846a00 100644 --- a/lbry/file/source_manager.py +++ b/lbry/file/source_manager.py @@ -34,7 +34,8 @@ class SourceManager: 'txid', 'nout', 'channel_claim_id', - 'channel_name' + 'channel_name', + 'completed' } set_filter_fields = { From e8ba5d7606de735020854ce7777ffb70cb562247 Mon Sep 17 00:00:00 2001 From: Lex Berezhny <lex@damoti.com> Date: Mon, 11 May 2020 19:47:08 -0400 Subject: [PATCH 68/86] v0.73.0 --- lbry/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbry/__init__.py b/lbry/__init__.py index bc64b8354..ec898acc5 100644 --- a/lbry/__init__.py +++ b/lbry/__init__.py @@ -1,2 +1,2 @@ -__version__ = "0.72.0" +__version__ = "0.73.0" version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name From bbded12923ebfe3d99c1481694f426b50b92e35c Mon Sep 17 00:00:00 2001 From: Jack Robison <jackrobison@lbry.io> Date: Tue, 12 May 2020 00:32:36 -0400 Subject: [PATCH 69/86] fix node not being set on the downloader in some cases --- lbry/extras/daemon/daemon.py | 4 ++++ lbry/file/file_manager.py | 4 ++++ lbry/stream/managed_stream.py | 4 ++-- lbry/stream/stream_manager.py | 5 ++++- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index fd4627987..441d4b93a 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -2046,6 +2046,8 @@ class Daemon(metaclass=JSONRPCServerType): raise Exception(f'Unable to find a file for {kwargs}') stream = streams[0] if status == 'start' and not stream.running: + if not hasattr(stream, 'bt_infohash') and 'dht' not in self.conf.components_to_skip: + stream.downloader.node = self.dht_node await stream.save_file() msg = "Resumed download" elif status == 'stop' and stream.running: @@ -2148,6 +2150,8 @@ class Daemon(metaclass=JSONRPCServerType): log.warning("There is no file to save") return False stream = streams[0] + if not hasattr(stream, 'bt_infohash') and 'dht' not in self.conf.components_to_skip: + stream.downloader.node = self.dht_node await stream.save_file(file_name, download_directory) return stream diff --git a/lbry/file/file_manager.py b/lbry/file/file_manager.py index 0c1eb7069..906362858 100644 --- a/lbry/file/file_manager.py +++ b/lbry/file/file_manager.py @@ -142,6 +142,8 @@ class FileManager: log.info("claim contains an update to a stream we have, downloading it") if save_file and existing_for_claim_id[0].output_file_exists: save_file = False + if not claim.stream.source.bt_infohash: + existing_for_claim_id[0].downloader.node = source_manager.node await existing_for_claim_id[0].start(timeout=timeout, save_now=save_file) if not existing_for_claim_id[0].output_file_exists and ( save_file or file_name or download_directory): @@ -155,6 +157,8 @@ class FileManager: log.info("already have stream for %s", uri) if save_file and updated_stream.output_file_exists: save_file = False + if not claim.stream.source.bt_infohash: + updated_stream.downloader.node = source_manager.node await updated_stream.start(timeout=timeout, save_now=save_file) if not updated_stream.output_file_exists and (save_file or file_name or download_directory): await updated_stream.save_file( diff --git a/lbry/stream/managed_stream.py b/lbry/stream/managed_stream.py index debd987a4..73976a8a3 100644 --- a/lbry/stream/managed_stream.py +++ b/lbry/stream/managed_stream.py @@ -193,13 +193,13 @@ class ManagedStream(ManagedDownloadSource): decrypted = await self.downloader.read_blob(blob_info, connection_id) yield (blob_info, decrypted) - async def stream_file(self, request: Request, node: Optional['Node'] = None) -> StreamResponse: + async def stream_file(self, request: Request) -> StreamResponse: log.info("stream file to browser for lbry://%s#%s (sd hash %s...)", self.claim_name, self.claim_id, self.sd_hash[:6]) headers, size, skip_blobs, first_blob_start_offset = self._prepare_range_response_headers( request.headers.get('range', 'bytes=0-') ) - await self.start(node) + await self.start() response = StreamResponse( status=206, headers=headers diff --git a/lbry/stream/stream_manager.py b/lbry/stream/stream_manager.py index 4d0d1093b..8e66aff96 100644 --- a/lbry/stream/stream_manager.py +++ b/lbry/stream/stream_manager.py @@ -247,4 +247,7 @@ class StreamManager(SourceManager): os.remove(source.full_path) async def stream_partial_content(self, request: Request, sd_hash: str): - return await self._sources[sd_hash].stream_file(request, self.node) + stream = self._sources[sd_hash] + if not stream.downloader.node: + stream.downloader.node = self.node + return await stream.stream_file(request) From a0fea30a11c3b750908a1fce38565ddeeaef41f9 Mon Sep 17 00:00:00 2001 From: Victor Shyba <victor1984@riseup.net> Date: Sat, 9 May 2020 20:45:25 -0300 Subject: [PATCH 70/86] make wait check every second instead of once --- lbry/wallet/ledger.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/lbry/wallet/ledger.py b/lbry/wallet/ledger.py index 79e7ef6b2..92cbf93a5 100644 --- a/lbry/wallet/ledger.py +++ b/lbry/wallet/ledger.py @@ -1,5 +1,6 @@ import os import copy +import time import asyncio import logging from io import StringIO @@ -639,6 +640,7 @@ class Ledger(metaclass=LedgerRegistry): return self.network.broadcast(hexlify(tx.raw).decode()) async def wait(self, tx: Transaction, height=-1, timeout=1): + timeout = timeout or 600 # after 10 minutes there is almost 0 hope addresses = set() for txi in tx.inputs: if txi.txo_ref.txo is not None: @@ -648,13 +650,20 @@ class Ledger(metaclass=LedgerRegistry): for txo in tx.outputs: if txo.has_address: addresses.add(self.hash160_to_address(txo.pubkey_hash)) + start = int(time.perf_counter()) + while timeout and (int(time.perf_counter()) - start) <= timeout: + if await self._wait_round(tx, height, addresses): + return + raise asyncio.TimeoutError('Timed out waiting for transaction.') + + async def _wait_round(self, tx: Transaction, height: int, addresses: Iterable[str]): records = await self.db.get_addresses(address__in=addresses) _, pending = await asyncio.wait([ self.on_transaction.where(partial( lambda a, e: a == e.address and e.tx.height >= height and e.tx.id == tx.id, address_record['address'] )) for address_record in records - ], timeout=timeout) + ], timeout=1) if pending: records = await self.db.get_addresses(address__in=addresses) for record in records: @@ -666,8 +675,9 @@ class Ledger(metaclass=LedgerRegistry): if txid == tx.id and local_height >= height: found = True if not found: - print(record['history'], addresses, tx.id) - raise asyncio.TimeoutError('Timed out waiting for transaction.') + log.debug("timeout: %s, %s, %s", record['history'], addresses, tx.id) + return False + return True async def _inflate_outputs( self, query, accounts, From 77d19af3595792968ed58893635a2dc6dd2b9eb4 Mon Sep 17 00:00:00 2001 From: Jack Robison <jackrobison@lbry.io> Date: Wed, 13 May 2020 09:21:48 -0400 Subject: [PATCH 71/86] v0.73.1 --- lbry/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbry/__init__.py b/lbry/__init__.py index ec898acc5..4fddf1e5c 100644 --- a/lbry/__init__.py +++ b/lbry/__init__.py @@ -1,2 +1,2 @@ -__version__ = "0.73.0" +__version__ = "0.73.1" version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name From 26964ecf0f5cd01df894c59dd455286f2d721809 Mon Sep 17 00:00:00 2001 From: Jack Robison <jackrobison@lbry.io> Date: Wed, 13 May 2020 09:24:35 -0400 Subject: [PATCH 72/86] fix download_blob_from_peer.py --- scripts/download_blob_from_peer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/download_blob_from_peer.py b/scripts/download_blob_from_peer.py index 08f046abc..3418ab1f8 100644 --- a/scripts/download_blob_from_peer.py +++ b/scripts/download_blob_from_peer.py @@ -3,6 +3,7 @@ import os import asyncio import socket import ipaddress +import lbry.wallet from lbry.conf import Config from lbry.extras.daemon.storage import SQLiteStorage from lbry.blob.blob_manager import BlobManager From af94687d453fdaac31a47ba9cdd0080c9b1a2bc5 Mon Sep 17 00:00:00 2001 From: Akinwale Ariwodola <akinwale@gmail.com> Date: Sun, 17 May 2020 10:32:26 +0100 Subject: [PATCH 73/86] add download_path as a filter field for file_list --- lbry/file/source_manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lbry/file/source_manager.py b/lbry/file/source_manager.py index bf2846a00..b4babc7a9 100644 --- a/lbry/file/source_manager.py +++ b/lbry/file/source_manager.py @@ -27,6 +27,7 @@ class SourceManager: 'status', 'file_name', 'added_on', + 'download_path', 'claim_name', 'claim_height', 'claim_id', From 68ed9f4ffc10334082473c230e4081a302e93c90 Mon Sep 17 00:00:00 2001 From: Akinwale Ariwodola <akinwale@gmail.com> Date: Sun, 17 May 2020 12:12:31 +0100 Subject: [PATCH 74/86] add download_path property to managed_stream --- lbry/stream/managed_stream.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lbry/stream/managed_stream.py b/lbry/stream/managed_stream.py index 73976a8a3..da625c381 100644 --- a/lbry/stream/managed_stream.py +++ b/lbry/stream/managed_stream.py @@ -120,6 +120,10 @@ class ManagedStream(ManagedDownloadSource): def mime_type(self): return guess_media_type(os.path.basename(self.descriptor.suggested_file_name))[0] + @property + def download_path(self): + return f"{self.download_directory}/{self._file_name}" if self.download_directory and self._file_name else None + # @classmethod # async def create(cls, loop: asyncio.AbstractEventLoop, config: 'Config', # file_path: str, key: Optional[bytes] = None, From e49fcea6e3fac49d37e03d7fba2bfcef2d07c199 Mon Sep 17 00:00:00 2001 From: thebubbleindex <21062721+thebubbleindex@users.noreply.github.com> Date: Thu, 7 May 2020 08:59:03 -0700 Subject: [PATCH 75/86] fix issue with specifying ports via env vars make sure tcp and udp port for dht are int type --- lbry/conf.py | 15 ++++++++++++--- tests/unit/test_conf.py | 5 +++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/lbry/conf.py b/lbry/conf.py index afd648193..b6c11af4d 100644 --- a/lbry/conf.py +++ b/lbry/conf.py @@ -277,8 +277,17 @@ class Strings(ListSetting): class EnvironmentAccess: PREFIX = 'LBRY_' - def __init__(self, environ: dict): - self.environ = environ + def __init__(self, config: 'BaseConfig', environ: dict): + self.configuration = config + self.environ = {} + if environ: + self.load(environ) + + def load(self, environ): + for setting in self.configuration.get_settings(): + value = environ.get(f'{self.PREFIX}{setting.name.upper()}', NOT_SET) + if value != NOT_SET and not (isinstance(setting, ListSetting) and value is None): + self.environ[f'{self.PREFIX}{setting.name.upper()}'] = setting.deserialize(value) def __contains__(self, item: str): return f'{self.PREFIX}{item.upper()}' in self.environ @@ -443,7 +452,7 @@ class BaseConfig: self.arguments = ArgumentAccess(self, args) def set_environment(self, environ=None): - self.environment = EnvironmentAccess(environ or os.environ) + self.environment = EnvironmentAccess(self, dict(environ or os.environ)) def set_persisted(self, config_file_path=None): if config_file_path is None: diff --git a/tests/unit/test_conf.py b/tests/unit/test_conf.py index cae6e6a90..138354604 100644 --- a/tests/unit/test_conf.py +++ b/tests/unit/test_conf.py @@ -93,6 +93,11 @@ class ConfigurationTests(unittest.TestCase): self.assertEqual(c.test_str, 'the default') c.set_environment({'LBRY_TEST_STR': 'from environ'}) self.assertEqual(c.test_str, 'from environ') + + c = TestConfig() + self.assertEqual(c.test_int, 9) + c.set_environment({'LBRY_TEST_INT': '1'}) + self.assertEqual(c.test_int, 1) def test_persisted(self): with tempfile.TemporaryDirectory() as temp_dir: From b09eabc478b3f626be7e419e6e3b41ef4a7d3a70 Mon Sep 17 00:00:00 2001 From: Lex Berezhny <lex@damoti.com> Date: Mon, 18 May 2020 08:51:06 -0400 Subject: [PATCH 76/86] minor simplifcation --- lbry/conf.py | 10 +++++----- tests/unit/test_conf.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lbry/conf.py b/lbry/conf.py index b6c11af4d..008742157 100644 --- a/lbry/conf.py +++ b/lbry/conf.py @@ -279,7 +279,7 @@ class EnvironmentAccess: def __init__(self, config: 'BaseConfig', environ: dict): self.configuration = config - self.environ = {} + self.data = {} if environ: self.load(environ) @@ -287,13 +287,13 @@ class EnvironmentAccess: for setting in self.configuration.get_settings(): value = environ.get(f'{self.PREFIX}{setting.name.upper()}', NOT_SET) if value != NOT_SET and not (isinstance(setting, ListSetting) and value is None): - self.environ[f'{self.PREFIX}{setting.name.upper()}'] = setting.deserialize(value) + self.data[setting.name] = setting.deserialize(value) def __contains__(self, item: str): - return f'{self.PREFIX}{item.upper()}' in self.environ + return item in self.data def __getitem__(self, item: str): - return self.environ[f'{self.PREFIX}{item.upper()}'] + return self.data[item] class ArgumentAccess: @@ -452,7 +452,7 @@ class BaseConfig: self.arguments = ArgumentAccess(self, args) def set_environment(self, environ=None): - self.environment = EnvironmentAccess(self, dict(environ or os.environ)) + self.environment = EnvironmentAccess(self, environ or os.environ) def set_persisted(self, config_file_path=None): if config_file_path is None: diff --git a/tests/unit/test_conf.py b/tests/unit/test_conf.py index 138354604..6abeef642 100644 --- a/tests/unit/test_conf.py +++ b/tests/unit/test_conf.py @@ -90,11 +90,11 @@ class ConfigurationTests(unittest.TestCase): def test_environment(self): c = TestConfig() + self.assertEqual(c.test_str, 'the default') c.set_environment({'LBRY_TEST_STR': 'from environ'}) self.assertEqual(c.test_str, 'from environ') - - c = TestConfig() + self.assertEqual(c.test_int, 9) c.set_environment({'LBRY_TEST_INT': '1'}) self.assertEqual(c.test_int, 1) From ff8a50c366f09ef4a75703187f89597a5237c447 Mon Sep 17 00:00:00 2001 From: Lex Berezhny <lex@damoti.com> Date: Mon, 18 May 2020 11:15:08 -0400 Subject: [PATCH 77/86] fixed bug with leaky information between outputs --- lbry/wallet/ledger.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/lbry/wallet/ledger.py b/lbry/wallet/ledger.py index 92cbf93a5..ffbc7148b 100644 --- a/lbry/wallet/ledger.py +++ b/lbry/wallet/ledger.py @@ -694,14 +694,24 @@ class Ledger(metaclass=LedgerRegistry): self.cache_transaction(*tx) for tx in outputs.txs )) - txos, blocked = outputs.inflate(txs) + _txos, blocked = outputs.inflate(txs) + + txos = [] + for txo in _txos: + if isinstance(txo, Output): + # transactions and outputs are cached and shared between wallets + # we don't want to leak informaion between wallet so we add the + # wallet specific metadata on throw away copies of the txos + txo = copy.copy(txo) + txo.purchase_receipt = None + txo.update_annotations(None) + txos.append(txo) includes = ( include_purchase_receipt, include_is_my_output, include_sent_supports, include_sent_tips ) if accounts and any(includes): - copies = [] receipts = {} if include_purchase_receipt: priced_claims = [] @@ -718,46 +728,38 @@ class Ledger(metaclass=LedgerRegistry): } for txo in txos: if isinstance(txo, Output) and txo.can_decode_claim: - # transactions and outputs are cached and shared between wallets - # we don't want to leak informaion between wallet so we add the - # wallet specific metadata on throw away copies of the txos - txo_copy = copy.copy(txo) - copies.append(txo_copy) if include_purchase_receipt: - txo_copy.purchase_receipt = receipts.get(txo.claim_id) + txo.purchase_receipt = receipts.get(txo.claim_id) if include_is_my_output: mine = await self.db.get_txo_count( claim_id=txo.claim_id, txo_type__in=CLAIM_TYPES, is_my_output=True, is_spent=False, accounts=accounts ) if mine: - txo_copy.is_my_output = True + txo.is_my_output = True else: - txo_copy.is_my_output = False + txo.is_my_output = False if include_sent_supports: supports = await self.db.get_txo_sum( claim_id=txo.claim_id, txo_type=TXO_TYPES['support'], is_my_input=True, is_my_output=True, is_spent=False, accounts=accounts ) - txo_copy.sent_supports = supports + txo.sent_supports = supports if include_sent_tips: tips = await self.db.get_txo_sum( claim_id=txo.claim_id, txo_type=TXO_TYPES['support'], is_my_input=True, is_my_output=False, accounts=accounts ) - txo_copy.sent_tips = tips + txo.sent_tips = tips if include_received_tips: tips = await self.db.get_txo_sum( claim_id=txo.claim_id, txo_type=TXO_TYPES['support'], is_my_input=False, is_my_output=True, accounts=accounts ) - txo_copy.received_tips = tips - else: - copies.append(txo) - txos = copies + txo.received_tips = tips return txos, blocked, outputs.offset, outputs.total async def resolve(self, accounts, urls, **kwargs): From 590c892a6a39b45d619e33616dd09e3d16d8ea19 Mon Sep 17 00:00:00 2001 From: Lex Berezhny <lex@damoti.com> Date: Mon, 18 May 2020 12:27:22 -0400 Subject: [PATCH 78/86] re-set channel on txo --- lbry/wallet/ledger.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lbry/wallet/ledger.py b/lbry/wallet/ledger.py index ffbc7148b..56656ae4b 100644 --- a/lbry/wallet/ledger.py +++ b/lbry/wallet/ledger.py @@ -703,8 +703,10 @@ class Ledger(metaclass=LedgerRegistry): # we don't want to leak informaion between wallet so we add the # wallet specific metadata on throw away copies of the txos txo = copy.copy(txo) + channel = txo.channel txo.purchase_receipt = None txo.update_annotations(None) + txo.channel = channel txos.append(txo) includes = ( From 01280c8d0473da30d1e979c5ae099464710b9a6d Mon Sep 17 00:00:00 2001 From: Akinwale Ariwodola <akinwale@gmail.com> Date: Mon, 18 May 2020 18:52:13 +0100 Subject: [PATCH 79/86] update docstring for download_path --- lbry/extras/daemon/daemon.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index 441d4b93a..756ff6bc7 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -1965,10 +1965,11 @@ class Daemon(metaclass=JSONRPCServerType): [--outpoint=<outpoint>] [--txid=<txid>] [--nout=<nout>] [--channel_claim_id=<channel_claim_id>] [--channel_name=<channel_name>] [--claim_name=<claim_name>] [--blobs_in_stream=<blobs_in_stream>] - [--blobs_remaining=<blobs_remaining>] [--uploading_to_reflector=<uploading_to_reflector>] - [--is_fully_reflected=<is_fully_reflected>] [--status=<status>] [--completed=<completed>] - [--sort=<sort_by>] [--comparison=<comparison>] [--full_status=<full_status>] [--reverse] - [--page=<page>] [--page_size=<page_size>] [--wallet_id=<wallet_id>] + [--download_path=<download_path>] [--blobs_remaining=<blobs_remaining>] + [--uploading_to_reflector=<uploading_to_reflector>] [--is_fully_reflected=<is_fully_reflected>] + [--status=<status>] [--completed=<completed>] [--sort=<sort_by>] [--comparison=<comparison>] + [--full_status=<full_status>] [--reverse] [--page=<page>] [--page_size=<page_size>] + [--wallet_id=<wallet_id>] Options: --sd_hash=<sd_hash> : (str) get file with matching sd hash @@ -1985,6 +1986,7 @@ class Daemon(metaclass=JSONRPCServerType): --channel_name=<channel_name> : (str) get file with matching channel name --claim_name=<claim_name> : (str) get file with matching claim name --blobs_in_stream<blobs_in_stream> : (int) get file with matching blobs in stream + --download_path=<download_path> : (str) get file with matching download path --uploading_to_reflector=<uploading_to_reflector> : (bool) get files currently uploading to reflector --is_fully_reflected=<is_fully_reflected> : (bool) get files that have been uploaded to reflector --status=<status> : (str) match by status, ( running | finished | stopped ) From 7f6b2fe4f1048fa4ea431223339304f34bddfc98 Mon Sep 17 00:00:00 2001 From: Lex Berezhny <lex@damoti.com> Date: Mon, 18 May 2020 16:56:58 -0400 Subject: [PATCH 80/86] v0.74.0 --- lbry/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbry/__init__.py b/lbry/__init__.py index 4fddf1e5c..c7e039eed 100644 --- a/lbry/__init__.py +++ b/lbry/__init__.py @@ -1,2 +1,2 @@ -__version__ = "0.73.1" +__version__ = "0.74.0" version = tuple(map(int, __version__.split('.'))) # pylint: disable=invalid-name From cae7792a1e837701daa9db7fc241779ea49c0a4d Mon Sep 17 00:00:00 2001 From: Jack Robison <jackrobison@lbry.io> Date: Mon, 25 May 2020 10:16:18 -0400 Subject: [PATCH 81/86] add `transaction_cache_size` to config --- lbry/conf.py | 1 + lbry/wallet/ledger.py | 2 +- lbry/wallet/manager.py | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lbry/conf.py b/lbry/conf.py index 008742157..6945e537a 100644 --- a/lbry/conf.py +++ b/lbry/conf.py @@ -636,6 +636,7 @@ class Config(CLIConfig): "Strategy to use when selecting UTXOs for a transaction", STRATEGIES, "standard") + transaction_cache_size = Integer("Transaction cache size", 100_000) save_resolved_claims = Toggle( "Save content claims to the database when they are resolved to keep file_list up to date, " "only disable this if file_x commands are not needed", True diff --git a/lbry/wallet/ledger.py b/lbry/wallet/ledger.py index 56656ae4b..50adf7467 100644 --- a/lbry/wallet/ledger.py +++ b/lbry/wallet/ledger.py @@ -158,7 +158,7 @@ class Ledger(metaclass=LedgerRegistry): self._on_ready_controller = StreamController() self.on_ready = self._on_ready_controller.stream - self._tx_cache = pylru.lrucache(100000) + self._tx_cache = pylru.lrucache(self.config.get("tx_cache_size", 100_000)) self._update_tasks = TaskGroup() self._other_tasks = TaskGroup() # that we dont need to start self._utxo_reservation_lock = asyncio.Lock() diff --git a/lbry/wallet/manager.py b/lbry/wallet/manager.py index 20658a2e5..22f0e5d21 100644 --- a/lbry/wallet/manager.py +++ b/lbry/wallet/manager.py @@ -184,6 +184,7 @@ class WalletManager: 'auto_connect': True, 'default_servers': config.lbryum_servers, 'data_path': config.wallet_dir, + 'tx_cache_size': config.transaction_cache_size } wallets_directory = os.path.join(config.wallet_dir, 'wallets') From c94cc293c28fc6959394ecc5aaf14e714bbf02ea Mon Sep 17 00:00:00 2001 From: Jack Robison <jackrobison@lbry.io> Date: Mon, 25 May 2020 10:21:36 -0400 Subject: [PATCH 82/86] fix uncaught errors in test_component_manager --- tests/unit/components/test_component_manager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/components/test_component_manager.py b/tests/unit/components/test_component_manager.py index b4e81fed7..48d844168 100644 --- a/tests/unit/components/test_component_manager.py +++ b/tests/unit/components/test_component_manager.py @@ -120,6 +120,8 @@ class FakeComponent: class FakeDelayedWallet(FakeComponent): component_name = "wallet" depends_on = [] + ledger = None + default_wallet = None async def stop(self): await asyncio.sleep(1) From 6a0302fec64bba1e95bf9107142e4fbd5168bb00 Mon Sep 17 00:00:00 2001 From: Jack Robison <jackrobison@lbry.io> Date: Mon, 25 May 2020 10:23:11 -0400 Subject: [PATCH 83/86] fix uncaught dht DecodeError --- lbry/dht/protocol/protocol.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lbry/dht/protocol/protocol.py b/lbry/dht/protocol/protocol.py index 016ad50bf..7b90b5644 100644 --- a/lbry/dht/protocol/protocol.py +++ b/lbry/dht/protocol/protocol.py @@ -10,6 +10,7 @@ from asyncio.protocols import DatagramProtocol from asyncio.transports import DatagramTransport from lbry.dht import constants +from lbry.dht.serialization.bencoding import DecodeError from lbry.dht.serialization.datagram import decode_datagram, ErrorDatagram, ResponseDatagram, RequestDatagram from lbry.dht.serialization.datagram import RESPONSE_TYPE, ERROR_TYPE, PAGE_KEY from lbry.dht.error import RemoteException, TransportNotConnected @@ -554,7 +555,7 @@ class KademliaProtocol(DatagramProtocol): def datagram_received(self, datagram: bytes, address: typing.Tuple[str, int]) -> None: # pylint: disable=arguments-differ try: message = decode_datagram(datagram) - except (ValueError, TypeError): + except (ValueError, TypeError, DecodeError): self.peer_manager.report_failure(address[0], address[1]) log.warning("Couldn't decode dht datagram from %s: %s", address, binascii.hexlify(datagram).decode()) return From 34eae6e6086a1fb413a1b372fb9ed1715f0f3385 Mon Sep 17 00:00:00 2001 From: Jack Robison <jackrobison@lbry.io> Date: Mon, 25 May 2020 10:24:31 -0400 Subject: [PATCH 84/86] fix wallet server prometheus bucket sizes --- lbry/wallet/rpc/session.py | 6 +++++- lbry/wallet/server/block_processor.py | 7 ++++++- lbry/wallet/server/session.py | 10 +++++++--- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/lbry/wallet/rpc/session.py b/lbry/wallet/rpc/session.py index dc353b50e..58d8a10fc 100644 --- a/lbry/wallet/rpc/session.py +++ b/lbry/wallet/rpc/session.py @@ -40,6 +40,10 @@ from .jsonrpc import Request, JSONRPCConnection, JSONRPCv2, JSONRPC, Batch, Noti from .jsonrpc import RPCError, ProtocolError from .framing import BadMagicError, BadChecksumError, OversizedPayloadError, BitcoinFramer, NewlineFramer +HISTOGRAM_BUCKETS = ( + .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf') +) + class Connector: @@ -379,7 +383,7 @@ class RPCSession(SessionBase): for example JSON RPC.""" RESPONSE_TIMES = Histogram("response_time", "Response times", namespace=NAMESPACE, - labelnames=("method", "version")) + labelnames=("method", "version"), buckets=HISTOGRAM_BUCKETS) NOTIFICATION_COUNT = Counter("notification", "Number of notifications sent (for subscriptions)", namespace=NAMESPACE, labelnames=("method", "version")) REQUEST_ERRORS_COUNT = Counter( diff --git a/lbry/wallet/server/block_processor.py b/lbry/wallet/server/block_processor.py index cb6a32f55..69a57a2eb 100644 --- a/lbry/wallet/server/block_processor.py +++ b/lbry/wallet/server/block_processor.py @@ -130,6 +130,9 @@ class ChainError(Exception): NAMESPACE = "wallet_server" +HISTOGRAM_BUCKETS = ( + .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf') +) class BlockProcessor: @@ -142,7 +145,9 @@ class BlockProcessor: block_count_metric = Gauge( "block_count", "Number of processed blocks", namespace=NAMESPACE ) - block_update_time_metric = Histogram("block_time", "Block update times", namespace=NAMESPACE) + block_update_time_metric = Histogram( + "block_time", "Block update times", namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS + ) reorg_count_metric = Gauge( "reorg_count", "Number of reorgs", namespace=NAMESPACE ) diff --git a/lbry/wallet/server/session.py b/lbry/wallet/server/session.py index 1cce96ff9..c72278992 100644 --- a/lbry/wallet/server/session.py +++ b/lbry/wallet/server/session.py @@ -119,7 +119,9 @@ class SessionGroup: NAMESPACE = "wallet_server" - +HISTOGRAM_BUCKETS = ( + .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, 15.0, 20.0, 30.0, 60.0, float('inf') +) class SessionManager: """Holds global state about all sessions.""" @@ -147,7 +149,9 @@ class SessionManager: db_error_metric = Counter( "internal_error", "Number of queries raising unexpected errors", namespace=NAMESPACE ) - executor_time_metric = Histogram("executor_time", "SQLite executor times", namespace=NAMESPACE) + executor_time_metric = Histogram( + "executor_time", "SQLite executor times", namespace=NAMESPACE, buckets=HISTOGRAM_BUCKETS + ) pending_query_metric = Gauge( "pending_queries_count", "Number of pending and running sqlite queries", namespace=NAMESPACE ) @@ -990,7 +994,7 @@ class LBRYElectrumX(SessionBase): except reader.SQLiteInterruptedError as error: metrics = self.get_metrics_or_placeholder_for_api(query_name) metrics.query_interrupt(start, error.metrics) - self.session_mgr.self.session_mgr.SQLITE_INTERRUPT_COUNT.inc() + self.session_mgr.interrupt_count_metric.inc() raise RPCError(JSONRPC.QUERY_TIMEOUT, 'sqlite query timed out') except reader.SQLiteOperationalError as error: metrics = self.get_metrics_or_placeholder_for_api(query_name) From 4bbd850898e42319db751feead7f71f8c1a05ea8 Mon Sep 17 00:00:00 2001 From: Jack Robison <jackrobison@lbry.io> Date: Mon, 25 May 2020 10:25:04 -0400 Subject: [PATCH 85/86] fix uncaught ValueError in hashX_unsubscribe --- lbry/wallet/server/session.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lbry/wallet/server/session.py b/lbry/wallet/server/session.py index c72278992..186755b73 100644 --- a/lbry/wallet/server/session.py +++ b/lbry/wallet/server/session.py @@ -1220,7 +1220,10 @@ class LBRYElectrumX(SessionBase): return await self.address_status(hashX) async def hashX_unsubscribe(self, hashX, alias): - del self.hashX_subs[hashX] + try: + del self.hashX_subs[hashX] + except ValueError: + pass def address_to_hashX(self, address): try: From ce7816a968a4b572e8aaf6c88b65712479d331b4 Mon Sep 17 00:00:00 2001 From: Thomas Zarebczan <tzarebczan@users.noreply.github.com> Date: Mon, 25 May 2020 21:45:59 -0400 Subject: [PATCH 86/86] more aggressive video transdoing Have noticed the defaults aren't aggressive enough to stream smoothly (yet). Downgrade max rate to 5500K, higher crf = smaller file size for now. --- lbry/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lbry/conf.py b/lbry/conf.py index 6945e537a..e9b293933 100644 --- a/lbry/conf.py +++ b/lbry/conf.py @@ -478,12 +478,12 @@ class TranscodeConfig(BaseConfig): '', previous_names=['ffmpeg_folder']) video_encoder = String('FFmpeg codec and parameters for the video encoding. ' 'Example: libaom-av1 -crf 25 -b:v 0 -strict experimental', - 'libx264 -crf 21 -preset faster -pix_fmt yuv420p') + 'libx264 -crf 24 -preset faster -pix_fmt yuv420p') video_bitrate_maximum = Integer('Maximum bits per second allowed for video streams (0 to disable).', 8400000) video_scaler = String('FFmpeg scaling parameters for reducing bitrate. ' 'Example: -vf "scale=-2:720,fps=24" -maxrate 5M -bufsize 3M', r'-vf "scale=if(gte(iw\,ih)\,min(1920\,iw)\,-2):if(lt(iw\,ih)\,min(1920\,ih)\,-2)" ' - r'-maxrate 8400K -bufsize 5000K') + r'-maxrate 5500K -bufsize 5000K') audio_encoder = String('FFmpeg codec and parameters for the audio encoding. ' 'Example: libopus -b:a 128k', 'aac -b:a 160k')