Madiator2011/lbry-sdk
1
0
Fork 0
forked from LBRYCommunity/lbry-sdk
Code Pull requests Activity

logging, cleanup

Browse source
This commit is contained in:
Jack Robison 2019-01-28 17:29:08 -05:00
parent 7d33b4f1f3
commit 64dffa306f
No known key found for this signature in database
GPG key ID: DF25C68FE0239BB2
3 changed files with 27 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
lbrynet/blob_exchange/client.py
View file

@ -19,15 +19,14 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
self.writer: 'HashBlobWriter' = None self.writer: 'HashBlobWriter' = None
self.blob: 'BlobFile' = None self.blob: 'BlobFile' = None
self.download_running = asyncio.Event(loop=self.loop)
self._blob_bytes_received = 0 self._blob_bytes_received = 0
self._response_fut: asyncio.Future = None self._response_fut: asyncio.Future = None
self._request_lock = asyncio.Lock(loop=self.loop) self._request_lock = asyncio.Lock(loop=self.loop)
def handle_data_received(self, data: bytes): def data_received(self, data: bytes):
if self.transport.is_closing(): if self.transport.is_closing(): # TODO: is this needed?
if self._response_fut and not (self._response_fut.done() or self._response_fut.cancelled()): if self._response_fut and not self._response_fut.done():
self._response_fut.cancel() self._response_fut.cancel()
return return
@ -36,27 +35,36 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
if response.responses and self.blob: if response.responses and self.blob:
blob_response = response.get_blob_response() blob_response = response.get_blob_response()
if blob_response and not blob_response.error and blob_response.blob_hash == self.blob.blob_hash: if blob_response and not blob_response.error and blob_response.blob_hash == self.blob.blob_hash:
# set the expected length for the incoming blob if we didn't know it
self.blob.set_length(blob_response.length) self.blob.set_length(blob_response.length)
elif blob_response and not blob_response.error and self.blob.blob_hash != blob_response.blob_hash: elif blob_response and not blob_response.error and self.blob.blob_hash != blob_response.blob_hash:
# the server started sending a blob we didn't request
log.warning("mismatch with self.blob %s", self.blob.blob_hash) log.warning("mismatch with self.blob %s", self.blob.blob_hash)
return return
if response.responses: if response.responses:
log.debug("got response from %s:%i <- %s", self.peer_address, self.peer_port, response.to_dict())
# fire the Future with the response to our request
self._response_fut.set_result(response) self._response_fut.set_result(response)
if response.blob_data and self.writer and not self.writer.closed(): if response.blob_data and self.writer and not self.writer.closed():
log.debug("got %i blob bytes from %s:%i", len(response.blob_data), self.peer_address, self.peer_port)
# write blob bytes if we're writing a blob and have blob bytes to write
self._blob_bytes_received += len(response.blob_data) self._blob_bytes_received += len(response.blob_data)
try: try:
self.writer.write(response.blob_data) self.writer.write(response.blob_data)
return
except IOError as err: except IOError as err:
log.error("error downloading blob: %s", err) log.error("error downloading blob from %s:%i: %s", self.peer_address, self.peer_port, err)
if self._response_fut and not self._response_fut.done():
def data_received(self, data): self._response_fut.set_exception(err)
try: except (asyncio.CancelledError, asyncio.TimeoutError) as err: # TODO: is this needed?
return self.handle_data_received(data) log.error("%s downloading blob from %s:%i", str(err), self.peer_address, self.peer_port)
except (asyncio.CancelledError, asyncio.TimeoutError) as err:
if self._response_fut and not self._response_fut.done(): if self._response_fut and not self._response_fut.done():
self._response_fut.set_exception(err) self._response_fut.set_exception(err)
async def _download_blob(self) -> typing.Tuple[bool, bool]: async def _download_blob(self) -> typing.Tuple[bool, bool]:
"""
:return: download success (bool), keep connection (bool)
"""
request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash) request = BlobRequest.make_request_for_blob_hash(self.blob.blob_hash)
try: try:
msg = request.serialize() msg = request.serialize()
@ -109,7 +117,6 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
self.writer.close_handle() self.writer.close_handle()
if self.blob: if self.blob:
await self.blob.close() await self.blob.close()
self.download_running.clear()
self._response_fut = None self._response_fut = None
self.writer = None self.writer = None
self.blob = None self.blob = None
@ -122,11 +129,7 @@ class BlobExchangeClientProtocol(asyncio.Protocol):
return False, True return False, True
async with self._request_lock: async with self._request_lock:
try: try:
if self.download_running.is_set():
log.info("wait for download already running")
await self.download_running.wait()
self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0 self.blob, self.writer, self._blob_bytes_received = blob, blob.open_for_writing(), 0
self.download_running.set()
self._response_fut = asyncio.Future(loop=self.loop) self._response_fut = asyncio.Future(loop=self.loop)
return await self._download_blob() return await self._download_blob()
except OSError: except OSError:
@ -161,9 +164,6 @@ async def request_blob(loop: asyncio.BaseEventLoop, blob: 'BlobFile', protocol:
""" """
if blob.get_is_verified(): if blob.get_is_verified():
return False, True return False, True
if blob.get_is_verified():
log.info("already verified")
return False, True
try: try:
await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port), await asyncio.wait_for(loop.create_connection(lambda: protocol, address, tcp_port),
peer_connect_timeout, loop=loop) peer_connect_timeout, loop=loop)

12
lbrynet/stream/downloader.py
View file

@ -51,7 +51,7 @@ class StreamDownloader(StreamAssembler): # TODO: reduce duplication, refactor t
async def _request_blob(self, peer: 'KademliaPeer'): async def _request_blob(self, peer: 'KademliaPeer'):
if self.current_blob.get_is_verified(): if self.current_blob.get_is_verified():
log.info("already verified") log.debug("already verified")
return return
if peer not in self.active_connections: if peer not in self.active_connections:
log.warning("not active, adding: %s", str(peer)) log.warning("not active, adding: %s", str(peer))
@ -61,12 +61,12 @@ class StreamDownloader(StreamAssembler): # TODO: reduce duplication, refactor t
peer.address, peer.tcp_port, self.peer_connect_timeout) peer.address, peer.tcp_port, self.peer_connect_timeout)
await protocol.close() await protocol.close()
if not keep_connection: if not keep_connection:
log.info("drop peer %s:%i", peer.address, peer.tcp_port) log.debug("drop peer %s:%i", peer.address, peer.tcp_port)
if peer in self.active_connections: if peer in self.active_connections:
async with self._lock: async with self._lock:
del self.active_connections[peer] del self.active_connections[peer]
return return
log.info("keep peer %s:%i", peer.address, peer.tcp_port) log.debug("keep peer %s:%i", peer.address, peer.tcp_port)
def _update_requests(self): def _update_requests(self):
self.new_peer_event.clear() self.new_peer_event.clear()
@ -77,7 +77,7 @@ class StreamDownloader(StreamAssembler): # TODO: reduce duplication, refactor t
if peer not in self.requested_from[self.current_blob.blob_hash] and peer not in to_add: if peer not in self.requested_from[self.current_blob.blob_hash] and peer not in to_add:
to_add.append(peer) to_add.append(peer)
if to_add or self.running_download_requests: if to_add or self.running_download_requests:
log.info("adding download probes for %i peers to %i already active", log.debug("adding download probes for %i peers to %i already active",
min(len(to_add), 8 - len(self.running_download_requests)), min(len(to_add), 8 - len(self.running_download_requests)),
len(self.running_download_requests)) len(self.running_download_requests))
else: else:
@ -176,9 +176,7 @@ class StreamDownloader(StreamAssembler): # TODO: reduce duplication, refactor t
try: try:
async with node.stream_peer_search_junction(blob_queue) as search_junction: async with node.stream_peer_search_junction(blob_queue) as search_junction:
async for peers in search_junction: async for peers in search_junction:
if not isinstance(peers, list): # TODO: what's up with this? if peers:
log.error("not a list: %s %s", peers, str(type(peers)))
else:
self._add_peer_protocols(peers) self._add_peer_protocols(peers)
if not added_peers.is_set(): if not added_peers.is_set():
added_peers.set() added_peers.set()

1
lbrynet/stream/managed_stream.py
View file

@ -30,7 +30,6 @@ class ManagedStream:
self.stream_hash = descriptor.stream_hash self.stream_hash = descriptor.stream_hash
self.stream_claim_info = claim self.stream_claim_info = claim
self._status = status self._status = status
self._store_after_finished: asyncio.Task = None
self.fully_reflected = asyncio.Event(loop=self.loop) self.fully_reflected = asyncio.Event(loop=self.loop)
@property @property