From 191f661b357f4993115149cbe51e9da8232036a4 Mon Sep 17 00:00:00 2001 From: Jimmy Kiselak Date: Wed, 3 Aug 2016 23:03:14 -0400 Subject: [PATCH 01/44] skeleton for reflector protocol --- lbrynet/reflector/__init__.py | 0 lbrynet/reflector/client/__init__.py | 0 lbrynet/reflector/client/client.py | 140 +++++++++++++++++++++++++++ lbrynet/reflector/server/__init__.py | 0 lbrynet/reflector/server/server.py | 86 ++++++++++++++++ 5 files changed, 226 insertions(+) create mode 100644 lbrynet/reflector/__init__.py create mode 100644 lbrynet/reflector/client/__init__.py create mode 100644 lbrynet/reflector/client/client.py create mode 100644 lbrynet/reflector/server/__init__.py create mode 100644 lbrynet/reflector/server/server.py diff --git a/lbrynet/reflector/__init__.py b/lbrynet/reflector/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lbrynet/reflector/client/__init__.py b/lbrynet/reflector/client/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lbrynet/reflector/client/client.py b/lbrynet/reflector/client/client.py new file mode 100644 index 000000000..940dc7a27 --- /dev/null +++ b/lbrynet/reflector/client/client.py @@ -0,0 +1,140 @@ +""" +The reflector protocol (all dicts encoded in json): + +Client Handshake (sent once per connection, at the start of the connection): + +{ + 'version': 0, +} + + +Server Handshake (sent once per connection, after receiving the client handshake): + +{ + 'version': 0, +} + + +Client Info Request: + +{ + 'blob_hash': "", + 'blob_size': +} + + +Server Info Response (sent in response to Client Info Request): + +{ + 'response': ['YES', 'NO'] +} + +If response is 'YES', client may send a Client Blob Request or a Client Info Request. +If response is 'NO', client may only send a Client Info Request + + +Client Blob Request: + +{} # Yes, this is an empty dictionary, in case something needs to go here in the future + # this blob data must match the info sent in the most recent Client Info Request + + +Server Blob Response (sent in response to Client Blob Request): +{ + 'received': True +} + +Client may now send another Client Info Request + +""" +import json +import logging +from twisted.internet.protocol import Protocol, ClientFactory + + +log = logging.getLogger(__name__) + + +class IncompleteResponseError(Exception): + pass + + +class LBRYFileReflectorClient(Protocol): + def connectionMade(self): + self.peer = self.factory.peer + self.response_buff = '' + self.outgoing_buff = '' + self.blob_hashes_to_send = [] + self.next_blob_to_send = None + self.received_handshake_response = False + d = self.get_blobs_to_send(self.factory.stream_info_manager, self.factory.stream_hash) + d.addCallback(lambda _: self.send_handshake()) + + def dataReceived(self, data): + self.response_buff += data + try: + msg = self.parse_response(self.response_buff) + except IncompleteResponseError: + pass + else: + self.response_buff = '' + d = self.handle_response(msg) + d.addCallbacks(lambda _: self.send_next_request(), self.response_failure_handler) + + def connectionLost(self, reason): + pass + + def get_blobs_to_send(self, stream_info_manager, stream_hash): + d = stream_info_manager.get_blobs_for_stream(stream_hash) + + def set_blobs(blob_hashes): + for blob_hash, position, iv, length in blob_hashes: + self.blob_hashes_to_send.append(blob_hash) + + d.addCallback(set_blobs) + return d + + def parse_response(self, buff): + try: + return json.loads(buff) + except ValueError: + raise IncompleteResponseError() + + def handle_response(self, response_dict): + if self.received_handshake_response is False: + self.handle_handshake_response(response_dict) + else: + self.handle_normal_response(response_dict) + + def handle_handshake_response(self, response_dict): + pass + + def handle_normal_response(self, response_dict): + pass + + def send_next_request(self): + if self.next_blob_to_send is not None: + # send the blob + pass + elif self.blobs_to_send: + # send the server the next blob hash + length + pass + else: + # close connection + pass + + +class LBRYFileReflectorClientFactory(ClientFactory): + protocol = LBRYFileReflectorClient + + def __init__(self, stream_info_manager, peer, stream_hash): + self.peer = peer + self.stream_info_manager = stream_info_manager + self.stream_hash = stream_hash + self.p = None + + def buildProtocol(self, addr): + p = self.protocol() + p.factory = self + self.p = p + return p \ No newline at end of file diff --git a/lbrynet/reflector/server/__init__.py b/lbrynet/reflector/server/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lbrynet/reflector/server/server.py b/lbrynet/reflector/server/server.py new file mode 100644 index 000000000..ca43e224b --- /dev/null +++ b/lbrynet/reflector/server/server.py @@ -0,0 +1,86 @@ +import logging +from twisted.python import failure +from twisted.internet import error +from twisted.internet.protocol import Protocol, ServerFactory +import json + + +log = logging.getLogger(__name__) + + +class IncompleteMessageError(Exception): + pass + + +class ReflectorServer(Protocol): + """ + """ + + def connectionMade(self): + peer_info = self.transport.getPeer() + self.peer = self.factory.peer_manager.get_peer(peer_info.host, peer_info.port) + self.received_handshake = False + self.peer_version = None + self.receiving_blob = False + self.blob_write = None + self.blob_finished_d = None + self.request_buff = "" + + def connectionLost(self, reason=failure.Failure(error.ConnectionDone())): + pass + + def dataReceived(self, data): + if self.receiving_blob is False: + self.request_buff += data + try: + msg = self.parse_request(self.request_buff) + except IncompleteMessageError: + pass + else: + self.request_buff = '' + d = self.handle_request(msg) + d.addCallbacks(self.send_response, self.handle_error) + else: + self.blob_write(data) + + def parse_request(self, buff): + try: + return json.loads(buff) + except ValueError: + raise IncompleteMessageError() + + def handle_request(self, request_dict): + if self.received_handshake is False: + return self.handle_handshake(request_dict) + else: + return self.handle_normal_request(request_dict) + + def handle_handshake(self, request_dict): + pass + + def handle_normal_request(self, request_dict): + if self.blob_write is None: + # we haven't opened a blob yet, meaning we must be waiting for the + # next message containing a blob hash and a length. this message + # should be it. if it's one we want, open the blob for writing, and + # return a nice response dict (in a Deferred) saying go ahead + pass + else: + # we have a blob open already, so this message should have nothing + # important in it. to the deferred that fires when the blob is done, + # add a callback which returns a nice response dict saying to keep + # sending, and then return that deferred + pass + + def send_response(self, response_dict): + pass + + def handle_error(self, err): + pass + + +class ReflectorServerFactory(ServerFactory): + protocol = ReflectorServer + + def __init__(self, peer_manager): + self.peer_manager = peer_manager \ No newline at end of file From 1dea20a358864dc392268d70554a6f1711fbf852 Mon Sep 17 00:00:00 2001 From: Jimmy Kiselak Date: Wed, 3 Aug 2016 23:11:29 -0400 Subject: [PATCH 02/44] include sd blob hashes in list of blob hashes to send via reflector --- lbrynet/reflector/client/client.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lbrynet/reflector/client/client.py b/lbrynet/reflector/client/client.py index 940dc7a27..379a86a96 100644 --- a/lbrynet/reflector/client/client.py +++ b/lbrynet/reflector/client/client.py @@ -92,6 +92,13 @@ class LBRYFileReflectorClient(Protocol): self.blob_hashes_to_send.append(blob_hash) d.addCallback(set_blobs) + + d.addCallback(lambda _: stream_info_manager.get_sd_blob_hashes_for_stream) + + def set_sd_blobs(sd_blob_hashes): + for sd_blob_hash, in sd_blob_hashes: + self.blob_hashes_to_send.append(sd_blob_hash) + d.addCallback(set_sd_blobs) return d def parse_response(self, buff): From b7e2e87ac148c0807e47e3f5ce107b91e07ca3ec Mon Sep 17 00:00:00 2001 From: Jimmy Kiselak Date: Sun, 7 Aug 2016 22:33:40 -0400 Subject: [PATCH 03/44] fill in much of the skeleton for the reflector protocol --- lbrynet/reflector/client/client.py | 99 +++++++++++++++++++++++++----- lbrynet/reflector/server/server.py | 89 ++++++++++++++++++++------- 2 files changed, 152 insertions(+), 36 deletions(-) diff --git a/lbrynet/reflector/client/client.py b/lbrynet/reflector/client/client.py index 379a86a96..2bccb4fe3 100644 --- a/lbrynet/reflector/client/client.py +++ b/lbrynet/reflector/client/client.py @@ -26,7 +26,7 @@ Client Info Request: Server Info Response (sent in response to Client Info Request): { - 'response': ['YES', 'NO'] + 'send_blob': True|False } If response is 'YES', client may send a Client Blob Request or a Client Info Request. @@ -41,7 +41,7 @@ Client Blob Request: Server Blob Response (sent in response to Client Blob Request): { - 'received': True + 'received_blob': True } Client may now send another Client Info Request @@ -49,6 +49,7 @@ Client may now send another Client Info Request """ import json import logging +from twisted.protocols.basic import FileSender from twisted.internet.protocol import Protocol, ClientFactory @@ -60,13 +61,19 @@ class IncompleteResponseError(Exception): class LBRYFileReflectorClient(Protocol): + + # Protocol stuff + def connectionMade(self): - self.peer = self.factory.peer + self.blob_manager = self.factory.blob_manager self.response_buff = '' self.outgoing_buff = '' self.blob_hashes_to_send = [] self.next_blob_to_send = None + self.blob_read_handle = None self.received_handshake_response = False + self.protocol_version = None + self.file_sender = None d = self.get_blobs_to_send(self.factory.stream_info_manager, self.factory.stream_hash) d.addCallback(lambda _: self.send_handshake()) @@ -84,6 +91,17 @@ class LBRYFileReflectorClient(Protocol): def connectionLost(self, reason): pass + # IConsumer stuff + + def registerProducer(self, producer, streaming): + assert streaming is True + + def unregisterProducer(self): + self.transport.loseConnection() + + def write(self, data): + self.transport.write(data) + def get_blobs_to_send(self, stream_info_manager, stream_hash): d = stream_info_manager.get_blobs_for_stream(stream_hash) @@ -109,33 +127,86 @@ class LBRYFileReflectorClient(Protocol): def handle_response(self, response_dict): if self.received_handshake_response is False: - self.handle_handshake_response(response_dict) + return self.handle_handshake_response(response_dict) else: - self.handle_normal_response(response_dict) + return self.handle_normal_response(response_dict) + + def set_not_uploading(self): + if self.currently_uploading is not None: + self.currently_uploading.close_read_handle(self.read_handle) + self.read_handle = None + self.currently_uploading = None + self.file_sender = None + + def start_transfer(self): + self.write(json.dumps("{}")) + log.info("Starting the file upload") + assert self.read_handle is not None, "self.read_handle was None when trying to start the transfer" + d = self.file_sender.beginFileTransfer(self.read_handle, self) + return d def handle_handshake_response(self, response_dict): - pass + if 'version' not in response_dict: + raise ValueError("Need protocol version number!") + self.protocol_version = int(response_dict['version']) + if self.protocol_version != 0: + raise ValueError("I can't handle protocol version {}!".format(self.protocol_version)) + self.received_handshake_response = True def handle_normal_response(self, response_dict): - pass + if self.next_blob_to_send is not None: # Expecting Server Info Response + if 'send_blob' not in response_dict: + raise ValueError("I don't know whether to send the blob or not!") + if response_dict['send_blob'] is True: + self.file_sender = FileSender() + return True + else: + return self.set_not_uploading() + else: # Expecting Server Blob Response + if 'received_blob' not in response_dict: + raise ValueError("I don't know if the blob made it to the intended destination!") + else: + return self.set_not_uploading() + + def open_blob_for_reading(self, blob): + if blob.is_validated(): + read_handle = blob.open_for_reading() + if read_handle is not None: + self.next_blob_to_send = blob + self.read_handle = read_handle + return None + raise ValueError("Couldn't open that blob for some reason") + + def send_blob_info(self): + assert self.next_blob_to_send is not None, "need to have a next blob to send at this point" + self.write(json.dumps({ + 'blob_hash': self.next_blob_to_send.blob_hash, + 'blob_length': self.next_blob_to_send.length + })) def send_next_request(self): - if self.next_blob_to_send is not None: + if self.file_sender is not None: # send the blob - pass - elif self.blobs_to_send: + return self.start_transfer() + elif self.blob_hashes_to_send: + # open the next blob to send + blob_hash = self.blob_hashes_to_send[0] + self.blob_hashes_to_send = self.blob_hashes_to_send[1:] + d = self.blob_manager.get_blob(blob_hash, True) + d.addCallback(self.open_blob_for_reading) # send the server the next blob hash + length - pass + d.addCallback(lambda _: self.send_blob_info()) + return d else: # close connection - pass + self.transport.loseConnection() class LBRYFileReflectorClientFactory(ClientFactory): protocol = LBRYFileReflectorClient - def __init__(self, stream_info_manager, peer, stream_hash): - self.peer = peer + def __init__(self, blob_manager, stream_info_manager, stream_hash): + self.blob_manager = blob_manager self.stream_info_manager = stream_info_manager self.stream_hash = stream_hash self.p = None diff --git a/lbrynet/reflector/server/server.py b/lbrynet/reflector/server/server.py index ca43e224b..f0ee24f87 100644 --- a/lbrynet/reflector/server/server.py +++ b/lbrynet/reflector/server/server.py @@ -4,26 +4,25 @@ from twisted.internet import error from twisted.internet.protocol import Protocol, ServerFactory import json +from lbrynet.core.utils import is_valid_blobhash + log = logging.getLogger(__name__) -class IncompleteMessageError(Exception): - pass - - class ReflectorServer(Protocol): - """ - """ def connectionMade(self): peer_info = self.transport.getPeer() self.peer = self.factory.peer_manager.get_peer(peer_info.host, peer_info.port) + self.blob_manager = self.factory.blob_manager self.received_handshake = False self.peer_version = None self.receiving_blob = False + self.incoming_blob = None self.blob_write = None self.blob_finished_d = None + self.cancel_write = None self.request_buff = "" def connectionLost(self, reason=failure.Failure(error.ConnectionDone())): @@ -32,22 +31,34 @@ class ReflectorServer(Protocol): def dataReceived(self, data): if self.receiving_blob is False: self.request_buff += data - try: - msg = self.parse_request(self.request_buff) - except IncompleteMessageError: - pass - else: + msg, extra_data = self._get_valid_response(self.request_buff) + if msg is not None: self.request_buff = '' d = self.handle_request(msg) d.addCallbacks(self.send_response, self.handle_error) + if self.receiving_blob is True and len(extra_data) != 0: + self.blob_write(extra_data) else: self.blob_write(data) - def parse_request(self, buff): - try: - return json.loads(buff) - except ValueError: - raise IncompleteMessageError() + def _get_valid_response(self, response_msg): + extra_data = None + response = None + curr_pos = 0 + while 1: + next_close_paren = response_msg.find('}', curr_pos) + if next_close_paren != -1: + curr_pos = next_close_paren + 1 + try: + response = json.loads(response_msg[:curr_pos]) + except ValueError: + pass + else: + extra_data = response_msg[curr_pos:] + break + else: + break + return response, extra_data def handle_request(self, request_dict): if self.received_handshake is False: @@ -56,7 +67,26 @@ class ReflectorServer(Protocol): return self.handle_normal_request(request_dict) def handle_handshake(self, request_dict): - pass + if 'version' not in request_dict: + raise ValueError("Client should send version") + self.peer_version = int(request_dict['version']) + if self.peer_version != 0: + raise ValueError("I don't know that version!") + return {'version': 0} + + def determine_blob_needed(self, blob): + if blob.is_validated(): + return {'send_blob': False} + else: + self.incoming_blob = blob + self.blob_finished_d, self.blob_write, self.cancel_write = blob.open_for_writing(self.peer) + return {'send_blob': True} + + def close_blob(self): + self.blob_finished_d = None + self.blob_write = None + self.cancel_write = None + self.incoming_blob = None def handle_normal_request(self, request_dict): if self.blob_write is None: @@ -64,16 +94,30 @@ class ReflectorServer(Protocol): # next message containing a blob hash and a length. this message # should be it. if it's one we want, open the blob for writing, and # return a nice response dict (in a Deferred) saying go ahead - pass + if not 'blob_hash' in request_dict or not 'blob_size' in request_dict: + raise ValueError("Expected a blob hash and a blob size") + if not is_valid_blobhash(request_dict['blob_hash']): + raise ValueError("Got a bad blob hash: {}".format(request_dict['blob_hash'])) + d = self.blob_manager.get_blob( + request_dict['blob_hash'], + True, + int(request_dict['blob_size']) + ) + d.addCallback(self.determine_blob_needed) else: # we have a blob open already, so this message should have nothing # important in it. to the deferred that fires when the blob is done, # add a callback which returns a nice response dict saying to keep # sending, and then return that deferred - pass + self.receiving_blob = True + d = self.blob_finished_d + d.addCallback(lambda _: self.close_blob()) + d.addCallback(lambda _: {'received_blob': True}) + d.addCallback(self.send_response) + return d def send_response(self, response_dict): - pass + self.write(json.dumps(response_dict)) def handle_error(self, err): pass @@ -82,5 +126,6 @@ class ReflectorServer(Protocol): class ReflectorServerFactory(ServerFactory): protocol = ReflectorServer - def __init__(self, peer_manager): - self.peer_manager = peer_manager \ No newline at end of file + def __init__(self, peer_manager, blob_manager): + self.peer_manager = peer_manager + self.blob_manager = blob_manager \ No newline at end of file From 7e2ad58eddb65ebaeef325ca30df4052e5341cbc Mon Sep 17 00:00:00 2001 From: Jimmy Kiselak Date: Tue, 9 Aug 2016 00:59:50 -0400 Subject: [PATCH 04/44] get reflector client and server working; add func test to prove it --- lbrynet/reflector/client/client.py | 57 +++++++---- lbrynet/reflector/server/server.py | 9 +- tests/functional_tests.py | 149 ++++++++++++++++++++++++++++- 3 files changed, 194 insertions(+), 21 deletions(-) diff --git a/lbrynet/reflector/client/client.py b/lbrynet/reflector/client/client.py index 2bccb4fe3..ce2102f95 100644 --- a/lbrynet/reflector/client/client.py +++ b/lbrynet/reflector/client/client.py @@ -51,6 +51,7 @@ import json import logging from twisted.protocols.basic import FileSender from twisted.internet.protocol import Protocol, ClientFactory +from twisted.internet import defer, error log = logging.getLogger(__name__) @@ -74,8 +75,11 @@ class LBRYFileReflectorClient(Protocol): self.received_handshake_response = False self.protocol_version = None self.file_sender = None + self.producer = None + self.streaming = False d = self.get_blobs_to_send(self.factory.stream_info_manager, self.factory.stream_hash) d.addCallback(lambda _: self.send_handshake()) + d.addErrback(lambda err: log.warning("An error occurred immediately: %s", err.getTraceback())) def dataReceived(self, data): self.response_buff += data @@ -86,45 +90,64 @@ class LBRYFileReflectorClient(Protocol): else: self.response_buff = '' d = self.handle_response(msg) - d.addCallbacks(lambda _: self.send_next_request(), self.response_failure_handler) + d.addCallback(lambda _: self.send_next_request()) + d.addErrback(self.response_failure_handler) def connectionLost(self, reason): - pass + if reason.check(error.ConnectionDone): + self.factory.finished_deferred.callback(True) + else: + self.factory.finished_deferred.callback(reason) # IConsumer stuff def registerProducer(self, producer, streaming): - assert streaming is True + self.producer = producer + self.streaming = streaming + if self.streaming is False: + from twisted.internet import reactor + reactor.callLater(0, self.producer.resumeProducing) def unregisterProducer(self): - self.transport.loseConnection() + self.producer = None def write(self, data): self.transport.write(data) + if self.producer is not None and self.streaming is False: + from twisted.internet import reactor + reactor.callLater(0, self.producer.resumeProducing) def get_blobs_to_send(self, stream_info_manager, stream_hash): d = stream_info_manager.get_blobs_for_stream(stream_hash) def set_blobs(blob_hashes): for blob_hash, position, iv, length in blob_hashes: - self.blob_hashes_to_send.append(blob_hash) + if blob_hash is not None: + self.blob_hashes_to_send.append(blob_hash) d.addCallback(set_blobs) - d.addCallback(lambda _: stream_info_manager.get_sd_blob_hashes_for_stream) + d.addCallback(lambda _: stream_info_manager.get_sd_blob_hashes_for_stream(stream_hash)) def set_sd_blobs(sd_blob_hashes): - for sd_blob_hash, in sd_blob_hashes: + for sd_blob_hash in sd_blob_hashes: self.blob_hashes_to_send.append(sd_blob_hash) + d.addCallback(set_sd_blobs) return d + def send_handshake(self): + self.write(json.dumps({'version': 0})) + def parse_response(self, buff): try: return json.loads(buff) except ValueError: raise IncompleteResponseError() + def response_failure_handler(self, err): + log.warning("An error occurred handling the response: %s", err.getTraceback()) + def handle_response(self, response_dict): if self.received_handshake_response is False: return self.handle_handshake_response(response_dict) @@ -132,15 +155,15 @@ class LBRYFileReflectorClient(Protocol): return self.handle_normal_response(response_dict) def set_not_uploading(self): - if self.currently_uploading is not None: - self.currently_uploading.close_read_handle(self.read_handle) + if self.next_blob_to_send is not None: + self.next_blob_to_send.close_read_handle(self.read_handle) self.read_handle = None - self.currently_uploading = None + self.next_blob_to_send = None self.file_sender = None + return defer.succeed(None) def start_transfer(self): - self.write(json.dumps("{}")) - log.info("Starting the file upload") + self.write(json.dumps({})) assert self.read_handle is not None, "self.read_handle was None when trying to start the transfer" d = self.file_sender.beginFileTransfer(self.read_handle, self) return d @@ -152,14 +175,15 @@ class LBRYFileReflectorClient(Protocol): if self.protocol_version != 0: raise ValueError("I can't handle protocol version {}!".format(self.protocol_version)) self.received_handshake_response = True + return defer.succeed(True) def handle_normal_response(self, response_dict): - if self.next_blob_to_send is not None: # Expecting Server Info Response + if self.file_sender is None: # Expecting Server Info Response if 'send_blob' not in response_dict: raise ValueError("I don't know whether to send the blob or not!") if response_dict['send_blob'] is True: self.file_sender = FileSender() - return True + return defer.succeed(True) else: return self.set_not_uploading() else: # Expecting Server Blob Response @@ -175,13 +199,13 @@ class LBRYFileReflectorClient(Protocol): self.next_blob_to_send = blob self.read_handle = read_handle return None - raise ValueError("Couldn't open that blob for some reason") + raise ValueError("Couldn't open that blob for some reason. blob_hash: {}".format(blob.blob_hash)) def send_blob_info(self): assert self.next_blob_to_send is not None, "need to have a next blob to send at this point" self.write(json.dumps({ 'blob_hash': self.next_blob_to_send.blob_hash, - 'blob_length': self.next_blob_to_send.length + 'blob_size': self.next_blob_to_send.length })) def send_next_request(self): @@ -210,6 +234,7 @@ class LBRYFileReflectorClientFactory(ClientFactory): self.stream_info_manager = stream_info_manager self.stream_hash = stream_hash self.p = None + self.finished_deferred = defer.Deferred() def buildProtocol(self, addr): p = self.protocol() diff --git a/lbrynet/reflector/server/server.py b/lbrynet/reflector/server/server.py index f0ee24f87..a8f36ae22 100644 --- a/lbrynet/reflector/server/server.py +++ b/lbrynet/reflector/server/server.py @@ -1,6 +1,6 @@ import logging from twisted.python import failure -from twisted.internet import error +from twisted.internet import error, defer from twisted.internet.protocol import Protocol, ServerFactory import json @@ -72,7 +72,8 @@ class ReflectorServer(Protocol): self.peer_version = int(request_dict['version']) if self.peer_version != 0: raise ValueError("I don't know that version!") - return {'version': 0} + self.received_handshake = True + return defer.succeed({'version': 0}) def determine_blob_needed(self, blob): if blob.is_validated(): @@ -87,6 +88,7 @@ class ReflectorServer(Protocol): self.blob_write = None self.cancel_write = None self.incoming_blob = None + self.receiving_blob = False def handle_normal_request(self, request_dict): if self.blob_write is None: @@ -113,11 +115,10 @@ class ReflectorServer(Protocol): d = self.blob_finished_d d.addCallback(lambda _: self.close_blob()) d.addCallback(lambda _: {'received_blob': True}) - d.addCallback(self.send_response) return d def send_response(self, response_dict): - self.write(json.dumps(response_dict)) + self.transport.write(json.dumps(response_dict)) def handle_error(self, err): pass diff --git a/tests/functional_tests.py b/tests/functional_tests.py index 76af2b5ed..c9e062d1c 100644 --- a/tests/functional_tests.py +++ b/tests/functional_tests.py @@ -26,7 +26,7 @@ from lbrynet.core.StreamDescriptor import download_sd_blob from lbrynet.lbryfilemanager.LBRYFileCreator import create_lbry_file from lbrynet.lbryfile.client.LBRYFileOptions import add_lbry_file_to_sd_identifier from lbrynet.lbryfile.StreamDescriptor import get_sd_info -from twisted.internet import defer, threads, task +from twisted.internet import defer, threads, task, error from twisted.trial.unittest import TestCase from twisted.python.failure import Failure import os @@ -38,6 +38,10 @@ from lbrynet.core.server.ServerProtocol import ServerProtocolFactory from lbrynet.lbrylive.server.LiveBlobInfoQueryHandler import CryptBlobInfoQueryHandlerFactory from lbrynet.lbrylive.client.LiveStreamOptions import add_live_stream_to_sd_identifier from lbrynet.lbrylive.client.LiveStreamDownloader import add_full_live_stream_downloader_to_sd_identifier +from lbrynet.core.BlobManager import TempBlobManager +from lbrynet.reflector.client.client import LBRYFileReflectorClientFactory +from lbrynet.reflector.server.server import ReflectorServerFactory +from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob log_format = "%(funcName)s(): %(message)s" @@ -1369,4 +1373,147 @@ class TestStreamify(TestCase): d.addCallback(lambda _: self.lbry_file_manager.setup()) d.addCallback(lambda _: create_stream()) d.addCallback(combine_stream) + return d + + +class TestReflector(TestCase): + + def setUp(self): + self.session = None + self.stream_info_manager = None + self.lbry_file_manager = None + self.server_blob_manager = None + self.reflector_port = None + self.addCleanup(self.take_down_env) + + def take_down_env(self): + + d = defer.succeed(True) + if self.lbry_file_manager is not None: + d.addCallback(lambda _: self.lbry_file_manager.stop()) + if self.session is not None: + d.addCallback(lambda _: self.session.shut_down()) + if self.stream_info_manager is not None: + d.addCallback(lambda _: self.stream_info_manager.stop()) + if self.server_blob_manager is not None: + d.addCallback(lambda _: self.server_blob_manager.stop()) + if self.reflector_port is not None: + d.addCallback(lambda _: self.reflector_port.stopListening()) + + def delete_test_env(): + shutil.rmtree('client') + + d.addCallback(lambda _: threads.deferToThread(delete_test_env)) + return d + + def test_reflector(self): + + wallet = FakeWallet() + peer_manager = PeerManager() + peer_finder = FakePeerFinder(5553, peer_manager, 2) + hash_announcer = FakeAnnouncer() + rate_limiter = DummyRateLimiter() + sd_identifier = StreamDescriptorIdentifier() + + self.expected_blobs = [ + ('dc4708f76a5e7af0f1cae0ee96b824e2ed9250c9346c093b' + '441f0a20d3607c17948b6fcfb4bc62020fe5286693d08586', 2097152), + ('f4067522c1b49432a2a679512e3917144317caa1abba0c04' + '1e0cd2cf9f635d4cf127ce1824fa04189b63916174951f70', 2097152), + ('305486c434260484fcb2968ce0e963b72f81ba56c11b08b1' + 'af0789b55b44d78422600f9a38e3cf4f2e9569897e5646a9', 1015056), + ] + + db_dir = "client" + os.mkdir(db_dir) + + self.session = LBRYSession(MIN_BLOB_DATA_PAYMENT_RATE, db_dir=db_dir, lbryid="abcd", + peer_finder=peer_finder, hash_announcer=hash_announcer, + blob_dir=None, peer_port=5553, + use_upnp=False, rate_limiter=rate_limiter, wallet=wallet) + + self.stream_info_manager = TempLBRYFileMetadataManager() + + self.lbry_file_manager = LBRYFileManager(self.session, self.stream_info_manager, sd_identifier) + + self.server_blob_manager = TempBlobManager(hash_announcer) + + d = self.session.setup() + d.addCallback(lambda _: self.stream_info_manager.setup()) + d.addCallback(lambda _: add_lbry_file_to_sd_identifier(sd_identifier)) + d.addCallback(lambda _: self.lbry_file_manager.setup()) + d.addCallback(lambda _: self.server_blob_manager.setup()) + + def verify_equal(sd_info): + self.assertEqual(sd_info, test_create_stream_sd_file) + + def save_sd_blob_hash(sd_hash): + self.expected_blobs.append((sd_hash, 923)) + + def verify_stream_descriptor_file(stream_hash): + d = get_sd_info(self.lbry_file_manager.stream_info_manager, stream_hash, True) + d.addCallback(verify_equal) + d.addCallback(lambda _: publish_sd_blob(self.lbry_file_manager.stream_info_manager, self.session.blob_manager, stream_hash)) + d.addCallback(save_sd_blob_hash) + d.addCallback(lambda _: stream_hash) + return d + + def iv_generator(): + iv = 0 + while 1: + iv += 1 + yield "%016d" % iv + + def create_stream(): + test_file = GenFile(5209343, b''.join([chr(i + 3) for i in xrange(0, 64, 6)])) + d = create_lbry_file(self.session, self.lbry_file_manager, "test_file", test_file, + key="0123456701234567", iv_generator=iv_generator()) + return d + + def start_server(): + server_factory = ReflectorServerFactory(peer_manager, self.server_blob_manager) + from twisted.internet import reactor + port = 8943 + while self.reflector_port is None: + try: + self.reflector_port = reactor.listenTCP(port, server_factory) + except error.CannotListenError: + port += 1 + return defer.succeed(port) + + def send_to_server(port, stream_hash): + factory = LBRYFileReflectorClientFactory( + self.session.blob_manager, + self.stream_info_manager, + stream_hash + ) + + from twisted.internet import reactor + reactor.connectTCP('localhost', port, factory) + return factory.finished_deferred + + def verify_blob_completed(blob, blob_size): + self.assertTrue(blob.is_validated()) + self.assertEqual(blob_size, blob.length) + + def verify_have_blob(blob_hash, blob_size): + d = self.server_blob_manager.get_blob(blob_hash, True) + d.addCallback(lambda blob: verify_blob_completed(blob, blob_size)) + return d + + def verify_data_on_reflector(): + check_blob_ds = [] + for blob_hash, blob_size in self.expected_blobs: + check_blob_ds.append(verify_have_blob(blob_hash, blob_size)) + return defer.DeferredList(check_blob_ds) + + def upload_to_reflector(stream_hash): + d = start_server() + d.addCallback(lambda port: send_to_server(port, stream_hash)) + d.addCallback(lambda _: verify_data_on_reflector()) + return d + + d.addCallback(lambda _: create_stream()) + d.addCallback(verify_stream_descriptor_file) + d.addCallback(upload_to_reflector) return d \ No newline at end of file From 14a0252819d74ef55e958c2e6bbc41222b0eada8 Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 9 Aug 2016 12:07:26 -0400 Subject: [PATCH 05/44] hooking up reflector plumbing --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 30 +++++++++++++++++++++++-- lbrynet/lbrynet_daemon/LBRYPublisher.py | 8 +++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 40f75156f..a1671505e 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -56,6 +56,7 @@ from lbrynet.core.PTCWallet import PTCWallet from lbrynet.core.LBRYWallet import LBRYcrdWallet, LBRYumWallet from lbrynet.lbryfilemanager.LBRYFileManager import LBRYFileManager from lbrynet.lbryfile.LBRYFileMetadataManager import DBLBRYFileMetadataManager, TempLBRYFileMetadataManager +from lbrynet.reflector.server import ReflectorServerFactory # from lbryum import LOG_PATH as lbryum_log @@ -203,6 +204,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): 'delete_blobs_on_remove': True, 'peer_port': 3333, 'dht_node_port': 4444, + 'reflector_port': 5566, 'use_upnp': True, 'start_lbrycrdd': True, 'requested_first_run_credits': False, @@ -291,6 +293,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): #### self.delete_blobs_on_remove = self.session_settings['delete_blobs_on_remove'] self.peer_port = self.session_settings['peer_port'] + self.reflector_port = self.session_settings['reflector_port'] self.dht_node_port = self.session_settings['dht_node_port'] self.use_upnp = self.session_settings['use_upnp'] self.start_lbrycrdd = self.session_settings['start_lbrycrdd'] @@ -670,10 +673,10 @@ class LBRYDaemon(jsonrpc.JSONRPC): def _start_server(self): if self.peer_port is not None: - server_factory = ServerProtocolFactory(self.session.rate_limiter, self.query_handlers, self.session.peer_manager) + try: self.lbry_server_port = reactor.listenTCP(self.peer_port, server_factory) except error.CannotListenError as e: @@ -682,6 +685,27 @@ class LBRYDaemon(jsonrpc.JSONRPC): raise ValueError("%s lbrynet may already be running on your computer.", str(e)) return defer.succeed(True) + def _start_reflector(self): + if self.reflector_port is not None: + reflector_factory = ReflectorServerFactory(self.session.peer_manager, self.session.blob_manager) + try: + self.reflector_server_port = reactor.listenTCP(self.reflector_port, reflector_factory) + except error.CannotListenError as e: + import traceback + log.error("Couldn't bind reflector to port %d. %s", self.reflector, traceback.format_exc()) + raise ValueError("%s lbrynet may already be running on your computer.", str(e)) + return defer.succeed(True) + + def _stop_reflector(self): + try: + if self.reflector_server_port is not None: + self.reflector_server_port, p = None, self.reflector_server_port + return defer.maybeDeferred(p.stopListening) + else: + return defer.succeed(True) + except AttributeError: + return defer.succeed(True) + def _stop_server(self): try: if self.lbry_server_port is not None: @@ -695,7 +719,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): def _setup_server(self): def restore_running_status(running): if running is True: - return self._start_server() + d = self._start_server() + d.addCallback(lambda _: self._start_reflector()) return defer.succeed(True) self.startup_status = STARTUP_STAGES[4] @@ -793,6 +818,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d = self._upload_log(log_type="close", exclude_previous=False if self.first_run else True) d.addCallback(lambda _: self._stop_server()) + d.addCallback(lambda _: self._stop_reflector()) d.addErrback(lambda err: True) d.addCallback(lambda _: self.lbry_file_manager.stop()) d.addErrback(lambda err: True) diff --git a/lbrynet/lbrynet_daemon/LBRYPublisher.py b/lbrynet/lbrynet_daemon/LBRYPublisher.py index 3e1b78325..7584d91fe 100644 --- a/lbrynet/lbrynet_daemon/LBRYPublisher.py +++ b/lbrynet/lbrynet_daemon/LBRYPublisher.py @@ -12,6 +12,7 @@ from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob from lbrynet.core.PaymentRateManager import PaymentRateManager from lbrynet.core.LBRYMetadata import Metadata, CURRENT_METADATA_VERSION from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader +from lbrynet.reflector.client import LBRYFileReflectorClientFactory from lbrynet.conf import LOG_FILE_NAME from twisted.internet import threads, defer @@ -41,6 +42,7 @@ class Publisher(object): self.lbry_file = None self.txid = None self.stream_hash = None + self.reflector_client = None self.metadata = {} def start(self, name, file_path, bid, metadata, old_txid): @@ -62,10 +64,16 @@ class Publisher(object): d.addCallback(lambda _: self._create_sd_blob()) d.addCallback(lambda _: self._claim_name()) d.addCallback(lambda _: self.set_status()) + d.addCallback(lambda _: self.start_reflector()) d.addCallbacks(lambda _: _show_result(), self._show_publish_error) return d + def start_reflector(self): + self.reflector_client = LBRYFileReflectorClientFactory(self.session.blob_manager, + self.lbry_file_manager.stream_info_manager, + self.stream_hash) + def _check_file_path(self, file_path): def check_file_threaded(): f = open(file_path) From 82a4fea81a7e0ea24c688d15d0eddf899a73a462 Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 9 Aug 2016 12:12:40 -0400 Subject: [PATCH 06/44] fix log statement --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index a1671505e..43cedfbbf 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -692,7 +692,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.reflector_server_port = reactor.listenTCP(self.reflector_port, reflector_factory) except error.CannotListenError as e: import traceback - log.error("Couldn't bind reflector to port %d. %s", self.reflector, traceback.format_exc()) + log.error("Couldn't bind reflector to port %d. %s", self.reflector_port, traceback.format_exc()) raise ValueError("%s lbrynet may already be running on your computer.", str(e)) return defer.succeed(True) From ef6fe3d1d9b9be3092e72653642562f58de6c06f Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 9 Aug 2016 12:18:46 -0400 Subject: [PATCH 07/44] add variable to enable running reflector server --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 34 ++++++++++++++++------------ 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 43cedfbbf..bbaaa98c0 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -170,6 +170,9 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.first_run_after_update = False self.uploaded_temp_files = [] + # change this to enable reflector server + self.run_reflector_server = False + if os.name == "nt": from lbrynet.winhelpers.knownpaths import get_path, FOLDERID, UserHandle default_download_directory = get_path(FOLDERID.Downloads, UserHandle.current) @@ -686,25 +689,26 @@ class LBRYDaemon(jsonrpc.JSONRPC): return defer.succeed(True) def _start_reflector(self): - if self.reflector_port is not None: - reflector_factory = ReflectorServerFactory(self.session.peer_manager, self.session.blob_manager) - try: - self.reflector_server_port = reactor.listenTCP(self.reflector_port, reflector_factory) - except error.CannotListenError as e: - import traceback - log.error("Couldn't bind reflector to port %d. %s", self.reflector_port, traceback.format_exc()) - raise ValueError("%s lbrynet may already be running on your computer.", str(e)) + if self.run_reflector_server: + if self.reflector_port is not None: + reflector_factory = ReflectorServerFactory(self.session.peer_manager, self.session.blob_manager) + try: + self.reflector_server_port = reactor.listenTCP(self.reflector_port, reflector_factory) + except error.CannotListenError as e: + import traceback + log.error("Couldn't bind reflector to port %d. %s", self.reflector_port, traceback.format_exc()) + raise ValueError("%s lbrynet may already be running on your computer.", str(e)) return defer.succeed(True) def _stop_reflector(self): - try: - if self.reflector_server_port is not None: - self.reflector_server_port, p = None, self.reflector_server_port - return defer.maybeDeferred(p.stopListening) - else: + if self.run_reflector_server: + try: + if self.reflector_server_port is not None: + self.reflector_server_port, p = None, self.reflector_server_port + return defer.maybeDeferred(p.stopListening) + except AttributeError: return defer.succeed(True) - except AttributeError: - return defer.succeed(True) + return defer.succeed(True) def _stop_server(self): try: From 0733d885fb1461b6d9925c98ab04d317dc5378cf Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 9 Aug 2016 12:26:34 -0400 Subject: [PATCH 08/44] start reflector upload in publish --- lbrynet/conf.py | 2 ++ lbrynet/lbrynet_daemon/LBRYPublisher.py | 13 ++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 466c477ea..fe37dd499 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -28,6 +28,8 @@ SEARCH_SERVERS = ["http://lighthouse1.lbry.io:50005", "http://lighthouse2.lbry.io:50005", "http://lighthouse3.lbry.io:50005"] +REFLECTOR_SERVERS = [("http://reflector.lbry.io", 5566)] + LOG_FILE_NAME = "lbrynet.log" LOG_POST_URL = "https://lbry.io/log-upload" diff --git a/lbrynet/lbrynet_daemon/LBRYPublisher.py b/lbrynet/lbrynet_daemon/LBRYPublisher.py index 7584d91fe..1fe6da991 100644 --- a/lbrynet/lbrynet_daemon/LBRYPublisher.py +++ b/lbrynet/lbrynet_daemon/LBRYPublisher.py @@ -2,9 +2,9 @@ import logging import mimetypes import os import sys +import random from appdirs import user_data_dir -from datetime import datetime from lbrynet.core.Error import InsufficientFundsError from lbrynet.lbryfilemanager.LBRYFileCreator import create_lbry_file @@ -13,8 +13,8 @@ from lbrynet.core.PaymentRateManager import PaymentRateManager from lbrynet.core.LBRYMetadata import Metadata, CURRENT_METADATA_VERSION from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader from lbrynet.reflector.client import LBRYFileReflectorClientFactory -from lbrynet.conf import LOG_FILE_NAME -from twisted.internet import threads, defer +from lbrynet.conf import LOG_FILE_NAME, REFLECTOR_SERVERS +from twisted.internet import threads, defer, reactor if sys.platform != "darwin": log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet") @@ -42,7 +42,8 @@ class Publisher(object): self.lbry_file = None self.txid = None self.stream_hash = None - self.reflector_client = None + reflector_server = random.choice(REFLECTOR_SERVERS) + self.reflector_server, self.reflector_port = reflector_server[0], reflector_server[1] self.metadata = {} def start(self, name, file_path, bid, metadata, old_txid): @@ -70,9 +71,11 @@ class Publisher(object): return d def start_reflector(self): - self.reflector_client = LBRYFileReflectorClientFactory(self.session.blob_manager, + factory = LBRYFileReflectorClientFactory(self.session.blob_manager, self.lbry_file_manager.stream_info_manager, self.stream_hash) + reactor.connectTCP(self.reflector_server, self.reflector_port, factory) + return factory.finished_deferred def _check_file_path(self, file_path): def check_file_threaded(): From 0cd92a96c003a8dfe3fde49fa05db05d30765f63 Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 9 Aug 2016 17:46:25 -0400 Subject: [PATCH 09/44] add reflector files to tests directory --- tests/lbrynet/lbrynet/reflector/__init__.py | 0 .../lbrynet/reflector/client/__init__.py | 0 .../lbrynet/reflector/client/client.py | 243 ++++++++++++++++++ .../lbrynet/reflector/server/__init__.py | 0 .../lbrynet/reflector/server/server.py | 132 ++++++++++ 5 files changed, 375 insertions(+) create mode 100644 tests/lbrynet/lbrynet/reflector/__init__.py create mode 100644 tests/lbrynet/lbrynet/reflector/client/__init__.py create mode 100644 tests/lbrynet/lbrynet/reflector/client/client.py create mode 100644 tests/lbrynet/lbrynet/reflector/server/__init__.py create mode 100644 tests/lbrynet/lbrynet/reflector/server/server.py diff --git a/tests/lbrynet/lbrynet/reflector/__init__.py b/tests/lbrynet/lbrynet/reflector/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lbrynet/lbrynet/reflector/client/__init__.py b/tests/lbrynet/lbrynet/reflector/client/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lbrynet/lbrynet/reflector/client/client.py b/tests/lbrynet/lbrynet/reflector/client/client.py new file mode 100644 index 000000000..ce2102f95 --- /dev/null +++ b/tests/lbrynet/lbrynet/reflector/client/client.py @@ -0,0 +1,243 @@ +""" +The reflector protocol (all dicts encoded in json): + +Client Handshake (sent once per connection, at the start of the connection): + +{ + 'version': 0, +} + + +Server Handshake (sent once per connection, after receiving the client handshake): + +{ + 'version': 0, +} + + +Client Info Request: + +{ + 'blob_hash': "", + 'blob_size': +} + + +Server Info Response (sent in response to Client Info Request): + +{ + 'send_blob': True|False +} + +If response is 'YES', client may send a Client Blob Request or a Client Info Request. +If response is 'NO', client may only send a Client Info Request + + +Client Blob Request: + +{} # Yes, this is an empty dictionary, in case something needs to go here in the future + # this blob data must match the info sent in the most recent Client Info Request + + +Server Blob Response (sent in response to Client Blob Request): +{ + 'received_blob': True +} + +Client may now send another Client Info Request + +""" +import json +import logging +from twisted.protocols.basic import FileSender +from twisted.internet.protocol import Protocol, ClientFactory +from twisted.internet import defer, error + + +log = logging.getLogger(__name__) + + +class IncompleteResponseError(Exception): + pass + + +class LBRYFileReflectorClient(Protocol): + + # Protocol stuff + + def connectionMade(self): + self.blob_manager = self.factory.blob_manager + self.response_buff = '' + self.outgoing_buff = '' + self.blob_hashes_to_send = [] + self.next_blob_to_send = None + self.blob_read_handle = None + self.received_handshake_response = False + self.protocol_version = None + self.file_sender = None + self.producer = None + self.streaming = False + d = self.get_blobs_to_send(self.factory.stream_info_manager, self.factory.stream_hash) + d.addCallback(lambda _: self.send_handshake()) + d.addErrback(lambda err: log.warning("An error occurred immediately: %s", err.getTraceback())) + + def dataReceived(self, data): + self.response_buff += data + try: + msg = self.parse_response(self.response_buff) + except IncompleteResponseError: + pass + else: + self.response_buff = '' + d = self.handle_response(msg) + d.addCallback(lambda _: self.send_next_request()) + d.addErrback(self.response_failure_handler) + + def connectionLost(self, reason): + if reason.check(error.ConnectionDone): + self.factory.finished_deferred.callback(True) + else: + self.factory.finished_deferred.callback(reason) + + # IConsumer stuff + + def registerProducer(self, producer, streaming): + self.producer = producer + self.streaming = streaming + if self.streaming is False: + from twisted.internet import reactor + reactor.callLater(0, self.producer.resumeProducing) + + def unregisterProducer(self): + self.producer = None + + def write(self, data): + self.transport.write(data) + if self.producer is not None and self.streaming is False: + from twisted.internet import reactor + reactor.callLater(0, self.producer.resumeProducing) + + def get_blobs_to_send(self, stream_info_manager, stream_hash): + d = stream_info_manager.get_blobs_for_stream(stream_hash) + + def set_blobs(blob_hashes): + for blob_hash, position, iv, length in blob_hashes: + if blob_hash is not None: + self.blob_hashes_to_send.append(blob_hash) + + d.addCallback(set_blobs) + + d.addCallback(lambda _: stream_info_manager.get_sd_blob_hashes_for_stream(stream_hash)) + + def set_sd_blobs(sd_blob_hashes): + for sd_blob_hash in sd_blob_hashes: + self.blob_hashes_to_send.append(sd_blob_hash) + + d.addCallback(set_sd_blobs) + return d + + def send_handshake(self): + self.write(json.dumps({'version': 0})) + + def parse_response(self, buff): + try: + return json.loads(buff) + except ValueError: + raise IncompleteResponseError() + + def response_failure_handler(self, err): + log.warning("An error occurred handling the response: %s", err.getTraceback()) + + def handle_response(self, response_dict): + if self.received_handshake_response is False: + return self.handle_handshake_response(response_dict) + else: + return self.handle_normal_response(response_dict) + + def set_not_uploading(self): + if self.next_blob_to_send is not None: + self.next_blob_to_send.close_read_handle(self.read_handle) + self.read_handle = None + self.next_blob_to_send = None + self.file_sender = None + return defer.succeed(None) + + def start_transfer(self): + self.write(json.dumps({})) + assert self.read_handle is not None, "self.read_handle was None when trying to start the transfer" + d = self.file_sender.beginFileTransfer(self.read_handle, self) + return d + + def handle_handshake_response(self, response_dict): + if 'version' not in response_dict: + raise ValueError("Need protocol version number!") + self.protocol_version = int(response_dict['version']) + if self.protocol_version != 0: + raise ValueError("I can't handle protocol version {}!".format(self.protocol_version)) + self.received_handshake_response = True + return defer.succeed(True) + + def handle_normal_response(self, response_dict): + if self.file_sender is None: # Expecting Server Info Response + if 'send_blob' not in response_dict: + raise ValueError("I don't know whether to send the blob or not!") + if response_dict['send_blob'] is True: + self.file_sender = FileSender() + return defer.succeed(True) + else: + return self.set_not_uploading() + else: # Expecting Server Blob Response + if 'received_blob' not in response_dict: + raise ValueError("I don't know if the blob made it to the intended destination!") + else: + return self.set_not_uploading() + + def open_blob_for_reading(self, blob): + if blob.is_validated(): + read_handle = blob.open_for_reading() + if read_handle is not None: + self.next_blob_to_send = blob + self.read_handle = read_handle + return None + raise ValueError("Couldn't open that blob for some reason. blob_hash: {}".format(blob.blob_hash)) + + def send_blob_info(self): + assert self.next_blob_to_send is not None, "need to have a next blob to send at this point" + self.write(json.dumps({ + 'blob_hash': self.next_blob_to_send.blob_hash, + 'blob_size': self.next_blob_to_send.length + })) + + def send_next_request(self): + if self.file_sender is not None: + # send the blob + return self.start_transfer() + elif self.blob_hashes_to_send: + # open the next blob to send + blob_hash = self.blob_hashes_to_send[0] + self.blob_hashes_to_send = self.blob_hashes_to_send[1:] + d = self.blob_manager.get_blob(blob_hash, True) + d.addCallback(self.open_blob_for_reading) + # send the server the next blob hash + length + d.addCallback(lambda _: self.send_blob_info()) + return d + else: + # close connection + self.transport.loseConnection() + + +class LBRYFileReflectorClientFactory(ClientFactory): + protocol = LBRYFileReflectorClient + + def __init__(self, blob_manager, stream_info_manager, stream_hash): + self.blob_manager = blob_manager + self.stream_info_manager = stream_info_manager + self.stream_hash = stream_hash + self.p = None + self.finished_deferred = defer.Deferred() + + def buildProtocol(self, addr): + p = self.protocol() + p.factory = self + self.p = p + return p \ No newline at end of file diff --git a/tests/lbrynet/lbrynet/reflector/server/__init__.py b/tests/lbrynet/lbrynet/reflector/server/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lbrynet/lbrynet/reflector/server/server.py b/tests/lbrynet/lbrynet/reflector/server/server.py new file mode 100644 index 000000000..a8f36ae22 --- /dev/null +++ b/tests/lbrynet/lbrynet/reflector/server/server.py @@ -0,0 +1,132 @@ +import logging +from twisted.python import failure +from twisted.internet import error, defer +from twisted.internet.protocol import Protocol, ServerFactory +import json + +from lbrynet.core.utils import is_valid_blobhash + + +log = logging.getLogger(__name__) + + +class ReflectorServer(Protocol): + + def connectionMade(self): + peer_info = self.transport.getPeer() + self.peer = self.factory.peer_manager.get_peer(peer_info.host, peer_info.port) + self.blob_manager = self.factory.blob_manager + self.received_handshake = False + self.peer_version = None + self.receiving_blob = False + self.incoming_blob = None + self.blob_write = None + self.blob_finished_d = None + self.cancel_write = None + self.request_buff = "" + + def connectionLost(self, reason=failure.Failure(error.ConnectionDone())): + pass + + def dataReceived(self, data): + if self.receiving_blob is False: + self.request_buff += data + msg, extra_data = self._get_valid_response(self.request_buff) + if msg is not None: + self.request_buff = '' + d = self.handle_request(msg) + d.addCallbacks(self.send_response, self.handle_error) + if self.receiving_blob is True and len(extra_data) != 0: + self.blob_write(extra_data) + else: + self.blob_write(data) + + def _get_valid_response(self, response_msg): + extra_data = None + response = None + curr_pos = 0 + while 1: + next_close_paren = response_msg.find('}', curr_pos) + if next_close_paren != -1: + curr_pos = next_close_paren + 1 + try: + response = json.loads(response_msg[:curr_pos]) + except ValueError: + pass + else: + extra_data = response_msg[curr_pos:] + break + else: + break + return response, extra_data + + def handle_request(self, request_dict): + if self.received_handshake is False: + return self.handle_handshake(request_dict) + else: + return self.handle_normal_request(request_dict) + + def handle_handshake(self, request_dict): + if 'version' not in request_dict: + raise ValueError("Client should send version") + self.peer_version = int(request_dict['version']) + if self.peer_version != 0: + raise ValueError("I don't know that version!") + self.received_handshake = True + return defer.succeed({'version': 0}) + + def determine_blob_needed(self, blob): + if blob.is_validated(): + return {'send_blob': False} + else: + self.incoming_blob = blob + self.blob_finished_d, self.blob_write, self.cancel_write = blob.open_for_writing(self.peer) + return {'send_blob': True} + + def close_blob(self): + self.blob_finished_d = None + self.blob_write = None + self.cancel_write = None + self.incoming_blob = None + self.receiving_blob = False + + def handle_normal_request(self, request_dict): + if self.blob_write is None: + # we haven't opened a blob yet, meaning we must be waiting for the + # next message containing a blob hash and a length. this message + # should be it. if it's one we want, open the blob for writing, and + # return a nice response dict (in a Deferred) saying go ahead + if not 'blob_hash' in request_dict or not 'blob_size' in request_dict: + raise ValueError("Expected a blob hash and a blob size") + if not is_valid_blobhash(request_dict['blob_hash']): + raise ValueError("Got a bad blob hash: {}".format(request_dict['blob_hash'])) + d = self.blob_manager.get_blob( + request_dict['blob_hash'], + True, + int(request_dict['blob_size']) + ) + d.addCallback(self.determine_blob_needed) + else: + # we have a blob open already, so this message should have nothing + # important in it. to the deferred that fires when the blob is done, + # add a callback which returns a nice response dict saying to keep + # sending, and then return that deferred + self.receiving_blob = True + d = self.blob_finished_d + d.addCallback(lambda _: self.close_blob()) + d.addCallback(lambda _: {'received_blob': True}) + return d + + def send_response(self, response_dict): + self.transport.write(json.dumps(response_dict)) + + def handle_error(self, err): + pass + + +class ReflectorServerFactory(ServerFactory): + protocol = ReflectorServer + + def __init__(self, peer_manager, blob_manager): + self.peer_manager = peer_manager + self.blob_manager = blob_manager \ No newline at end of file From 871e6e6f64d0e95a69637a4a0252a96680b51ada Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 9 Aug 2016 17:53:34 -0400 Subject: [PATCH 10/44] remove files that didn't fix test --- tests/lbrynet/lbrynet/reflector/__init__.py | 0 .../lbrynet/reflector/client/__init__.py | 0 .../lbrynet/reflector/client/client.py | 243 ------------------ .../lbrynet/reflector/server/__init__.py | 0 .../lbrynet/reflector/server/server.py | 132 ---------- 5 files changed, 375 deletions(-) delete mode 100644 tests/lbrynet/lbrynet/reflector/__init__.py delete mode 100644 tests/lbrynet/lbrynet/reflector/client/__init__.py delete mode 100644 tests/lbrynet/lbrynet/reflector/client/client.py delete mode 100644 tests/lbrynet/lbrynet/reflector/server/__init__.py delete mode 100644 tests/lbrynet/lbrynet/reflector/server/server.py diff --git a/tests/lbrynet/lbrynet/reflector/__init__.py b/tests/lbrynet/lbrynet/reflector/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/lbrynet/lbrynet/reflector/client/__init__.py b/tests/lbrynet/lbrynet/reflector/client/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/lbrynet/lbrynet/reflector/client/client.py b/tests/lbrynet/lbrynet/reflector/client/client.py deleted file mode 100644 index ce2102f95..000000000 --- a/tests/lbrynet/lbrynet/reflector/client/client.py +++ /dev/null @@ -1,243 +0,0 @@ -""" -The reflector protocol (all dicts encoded in json): - -Client Handshake (sent once per connection, at the start of the connection): - -{ - 'version': 0, -} - - -Server Handshake (sent once per connection, after receiving the client handshake): - -{ - 'version': 0, -} - - -Client Info Request: - -{ - 'blob_hash': "", - 'blob_size': -} - - -Server Info Response (sent in response to Client Info Request): - -{ - 'send_blob': True|False -} - -If response is 'YES', client may send a Client Blob Request or a Client Info Request. -If response is 'NO', client may only send a Client Info Request - - -Client Blob Request: - -{} # Yes, this is an empty dictionary, in case something needs to go here in the future - # this blob data must match the info sent in the most recent Client Info Request - - -Server Blob Response (sent in response to Client Blob Request): -{ - 'received_blob': True -} - -Client may now send another Client Info Request - -""" -import json -import logging -from twisted.protocols.basic import FileSender -from twisted.internet.protocol import Protocol, ClientFactory -from twisted.internet import defer, error - - -log = logging.getLogger(__name__) - - -class IncompleteResponseError(Exception): - pass - - -class LBRYFileReflectorClient(Protocol): - - # Protocol stuff - - def connectionMade(self): - self.blob_manager = self.factory.blob_manager - self.response_buff = '' - self.outgoing_buff = '' - self.blob_hashes_to_send = [] - self.next_blob_to_send = None - self.blob_read_handle = None - self.received_handshake_response = False - self.protocol_version = None - self.file_sender = None - self.producer = None - self.streaming = False - d = self.get_blobs_to_send(self.factory.stream_info_manager, self.factory.stream_hash) - d.addCallback(lambda _: self.send_handshake()) - d.addErrback(lambda err: log.warning("An error occurred immediately: %s", err.getTraceback())) - - def dataReceived(self, data): - self.response_buff += data - try: - msg = self.parse_response(self.response_buff) - except IncompleteResponseError: - pass - else: - self.response_buff = '' - d = self.handle_response(msg) - d.addCallback(lambda _: self.send_next_request()) - d.addErrback(self.response_failure_handler) - - def connectionLost(self, reason): - if reason.check(error.ConnectionDone): - self.factory.finished_deferred.callback(True) - else: - self.factory.finished_deferred.callback(reason) - - # IConsumer stuff - - def registerProducer(self, producer, streaming): - self.producer = producer - self.streaming = streaming - if self.streaming is False: - from twisted.internet import reactor - reactor.callLater(0, self.producer.resumeProducing) - - def unregisterProducer(self): - self.producer = None - - def write(self, data): - self.transport.write(data) - if self.producer is not None and self.streaming is False: - from twisted.internet import reactor - reactor.callLater(0, self.producer.resumeProducing) - - def get_blobs_to_send(self, stream_info_manager, stream_hash): - d = stream_info_manager.get_blobs_for_stream(stream_hash) - - def set_blobs(blob_hashes): - for blob_hash, position, iv, length in blob_hashes: - if blob_hash is not None: - self.blob_hashes_to_send.append(blob_hash) - - d.addCallback(set_blobs) - - d.addCallback(lambda _: stream_info_manager.get_sd_blob_hashes_for_stream(stream_hash)) - - def set_sd_blobs(sd_blob_hashes): - for sd_blob_hash in sd_blob_hashes: - self.blob_hashes_to_send.append(sd_blob_hash) - - d.addCallback(set_sd_blobs) - return d - - def send_handshake(self): - self.write(json.dumps({'version': 0})) - - def parse_response(self, buff): - try: - return json.loads(buff) - except ValueError: - raise IncompleteResponseError() - - def response_failure_handler(self, err): - log.warning("An error occurred handling the response: %s", err.getTraceback()) - - def handle_response(self, response_dict): - if self.received_handshake_response is False: - return self.handle_handshake_response(response_dict) - else: - return self.handle_normal_response(response_dict) - - def set_not_uploading(self): - if self.next_blob_to_send is not None: - self.next_blob_to_send.close_read_handle(self.read_handle) - self.read_handle = None - self.next_blob_to_send = None - self.file_sender = None - return defer.succeed(None) - - def start_transfer(self): - self.write(json.dumps({})) - assert self.read_handle is not None, "self.read_handle was None when trying to start the transfer" - d = self.file_sender.beginFileTransfer(self.read_handle, self) - return d - - def handle_handshake_response(self, response_dict): - if 'version' not in response_dict: - raise ValueError("Need protocol version number!") - self.protocol_version = int(response_dict['version']) - if self.protocol_version != 0: - raise ValueError("I can't handle protocol version {}!".format(self.protocol_version)) - self.received_handshake_response = True - return defer.succeed(True) - - def handle_normal_response(self, response_dict): - if self.file_sender is None: # Expecting Server Info Response - if 'send_blob' not in response_dict: - raise ValueError("I don't know whether to send the blob or not!") - if response_dict['send_blob'] is True: - self.file_sender = FileSender() - return defer.succeed(True) - else: - return self.set_not_uploading() - else: # Expecting Server Blob Response - if 'received_blob' not in response_dict: - raise ValueError("I don't know if the blob made it to the intended destination!") - else: - return self.set_not_uploading() - - def open_blob_for_reading(self, blob): - if blob.is_validated(): - read_handle = blob.open_for_reading() - if read_handle is not None: - self.next_blob_to_send = blob - self.read_handle = read_handle - return None - raise ValueError("Couldn't open that blob for some reason. blob_hash: {}".format(blob.blob_hash)) - - def send_blob_info(self): - assert self.next_blob_to_send is not None, "need to have a next blob to send at this point" - self.write(json.dumps({ - 'blob_hash': self.next_blob_to_send.blob_hash, - 'blob_size': self.next_blob_to_send.length - })) - - def send_next_request(self): - if self.file_sender is not None: - # send the blob - return self.start_transfer() - elif self.blob_hashes_to_send: - # open the next blob to send - blob_hash = self.blob_hashes_to_send[0] - self.blob_hashes_to_send = self.blob_hashes_to_send[1:] - d = self.blob_manager.get_blob(blob_hash, True) - d.addCallback(self.open_blob_for_reading) - # send the server the next blob hash + length - d.addCallback(lambda _: self.send_blob_info()) - return d - else: - # close connection - self.transport.loseConnection() - - -class LBRYFileReflectorClientFactory(ClientFactory): - protocol = LBRYFileReflectorClient - - def __init__(self, blob_manager, stream_info_manager, stream_hash): - self.blob_manager = blob_manager - self.stream_info_manager = stream_info_manager - self.stream_hash = stream_hash - self.p = None - self.finished_deferred = defer.Deferred() - - def buildProtocol(self, addr): - p = self.protocol() - p.factory = self - self.p = p - return p \ No newline at end of file diff --git a/tests/lbrynet/lbrynet/reflector/server/__init__.py b/tests/lbrynet/lbrynet/reflector/server/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/lbrynet/lbrynet/reflector/server/server.py b/tests/lbrynet/lbrynet/reflector/server/server.py deleted file mode 100644 index a8f36ae22..000000000 --- a/tests/lbrynet/lbrynet/reflector/server/server.py +++ /dev/null @@ -1,132 +0,0 @@ -import logging -from twisted.python import failure -from twisted.internet import error, defer -from twisted.internet.protocol import Protocol, ServerFactory -import json - -from lbrynet.core.utils import is_valid_blobhash - - -log = logging.getLogger(__name__) - - -class ReflectorServer(Protocol): - - def connectionMade(self): - peer_info = self.transport.getPeer() - self.peer = self.factory.peer_manager.get_peer(peer_info.host, peer_info.port) - self.blob_manager = self.factory.blob_manager - self.received_handshake = False - self.peer_version = None - self.receiving_blob = False - self.incoming_blob = None - self.blob_write = None - self.blob_finished_d = None - self.cancel_write = None - self.request_buff = "" - - def connectionLost(self, reason=failure.Failure(error.ConnectionDone())): - pass - - def dataReceived(self, data): - if self.receiving_blob is False: - self.request_buff += data - msg, extra_data = self._get_valid_response(self.request_buff) - if msg is not None: - self.request_buff = '' - d = self.handle_request(msg) - d.addCallbacks(self.send_response, self.handle_error) - if self.receiving_blob is True and len(extra_data) != 0: - self.blob_write(extra_data) - else: - self.blob_write(data) - - def _get_valid_response(self, response_msg): - extra_data = None - response = None - curr_pos = 0 - while 1: - next_close_paren = response_msg.find('}', curr_pos) - if next_close_paren != -1: - curr_pos = next_close_paren + 1 - try: - response = json.loads(response_msg[:curr_pos]) - except ValueError: - pass - else: - extra_data = response_msg[curr_pos:] - break - else: - break - return response, extra_data - - def handle_request(self, request_dict): - if self.received_handshake is False: - return self.handle_handshake(request_dict) - else: - return self.handle_normal_request(request_dict) - - def handle_handshake(self, request_dict): - if 'version' not in request_dict: - raise ValueError("Client should send version") - self.peer_version = int(request_dict['version']) - if self.peer_version != 0: - raise ValueError("I don't know that version!") - self.received_handshake = True - return defer.succeed({'version': 0}) - - def determine_blob_needed(self, blob): - if blob.is_validated(): - return {'send_blob': False} - else: - self.incoming_blob = blob - self.blob_finished_d, self.blob_write, self.cancel_write = blob.open_for_writing(self.peer) - return {'send_blob': True} - - def close_blob(self): - self.blob_finished_d = None - self.blob_write = None - self.cancel_write = None - self.incoming_blob = None - self.receiving_blob = False - - def handle_normal_request(self, request_dict): - if self.blob_write is None: - # we haven't opened a blob yet, meaning we must be waiting for the - # next message containing a blob hash and a length. this message - # should be it. if it's one we want, open the blob for writing, and - # return a nice response dict (in a Deferred) saying go ahead - if not 'blob_hash' in request_dict or not 'blob_size' in request_dict: - raise ValueError("Expected a blob hash and a blob size") - if not is_valid_blobhash(request_dict['blob_hash']): - raise ValueError("Got a bad blob hash: {}".format(request_dict['blob_hash'])) - d = self.blob_manager.get_blob( - request_dict['blob_hash'], - True, - int(request_dict['blob_size']) - ) - d.addCallback(self.determine_blob_needed) - else: - # we have a blob open already, so this message should have nothing - # important in it. to the deferred that fires when the blob is done, - # add a callback which returns a nice response dict saying to keep - # sending, and then return that deferred - self.receiving_blob = True - d = self.blob_finished_d - d.addCallback(lambda _: self.close_blob()) - d.addCallback(lambda _: {'received_blob': True}) - return d - - def send_response(self, response_dict): - self.transport.write(json.dumps(response_dict)) - - def handle_error(self, err): - pass - - -class ReflectorServerFactory(ServerFactory): - protocol = ReflectorServer - - def __init__(self, peer_manager, blob_manager): - self.peer_manager = peer_manager - self.blob_manager = blob_manager \ No newline at end of file From b65570fb1605b1fb4303824693816ebde541dd32 Mon Sep 17 00:00:00 2001 From: Jack Date: Tue, 9 Aug 2016 19:53:13 -0400 Subject: [PATCH 11/44] drop connection on response longer than 100 bytes that can't' be decoded --- lbrynet/reflector/server/server.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lbrynet/reflector/server/server.py b/lbrynet/reflector/server/server.py index a8f36ae22..c1098dc40 100644 --- a/lbrynet/reflector/server/server.py +++ b/lbrynet/reflector/server/server.py @@ -45,19 +45,23 @@ class ReflectorServer(Protocol): extra_data = None response = None curr_pos = 0 + size_of_message = len(response_msg) while 1: next_close_paren = response_msg.find('}', curr_pos) if next_close_paren != -1: curr_pos = next_close_paren + 1 try: response = json.loads(response_msg[:curr_pos]) + failed_to_decode = False except ValueError: - pass + failed_to_decode = True else: extra_data = response_msg[curr_pos:] break else: break + if size_of_message > 100 and failed_to_decode: + raise Exception("error decoding response") return response, extra_data def handle_request(self, request_dict): @@ -121,7 +125,8 @@ class ReflectorServer(Protocol): self.transport.write(json.dumps(response_dict)) def handle_error(self, err): - pass + log.error(err.getTraceback()) + self.transport.loseConnection() class ReflectorServerFactory(ServerFactory): From 3fb48318049ef945cd725ebd4f4827bd5a9ec350 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 10 Aug 2016 02:30:41 -0400 Subject: [PATCH 12/44] raise exception when curr_pos > 100 and json fails to decode --- lbrynet/reflector/server/server.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lbrynet/reflector/server/server.py b/lbrynet/reflector/server/server.py index c1098dc40..d2569b6aa 100644 --- a/lbrynet/reflector/server/server.py +++ b/lbrynet/reflector/server/server.py @@ -45,23 +45,22 @@ class ReflectorServer(Protocol): extra_data = None response = None curr_pos = 0 - size_of_message = len(response_msg) while 1: next_close_paren = response_msg.find('}', curr_pos) if next_close_paren != -1: curr_pos = next_close_paren + 1 try: response = json.loads(response_msg[:curr_pos]) - failed_to_decode = False except ValueError: - failed_to_decode = True + if curr_pos > 100: + raise Exception("error decoding response") + else: + pass else: extra_data = response_msg[curr_pos:] break else: break - if size_of_message > 100 and failed_to_decode: - raise Exception("error decoding response") return response, extra_data def handle_request(self, request_dict): From 67909724e7fba4a3bd5f81d891972fc1f39b0e9a Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Wed, 10 Aug 2016 07:44:41 -0500 Subject: [PATCH 13/44] fix up import paths --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 13 +++++++------ lbrynet/lbrynet_daemon/LBRYPublisher.py | 10 ++++++---- lbrynet/reflector/__init__.py | 2 ++ 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 6f0a78a90..d4e52b91e 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -57,8 +57,7 @@ from lbrynet.core.PTCWallet import PTCWallet from lbrynet.core.LBRYWallet import LBRYcrdWallet, LBRYumWallet from lbrynet.lbryfilemanager.LBRYFileManager import LBRYFileManager from lbrynet.lbryfile.LBRYFileMetadataManager import DBLBRYFileMetadataManager, TempLBRYFileMetadataManager -from lbrynet.reflector.server import ReflectorServerFactory -# from lbryum import LOG_PATH as lbryum_log +from lbrynet import reflector # TODO: this code snippet is everywhere. Make it go away @@ -686,13 +685,15 @@ class LBRYDaemon(jsonrpc.JSONRPC): def _start_reflector(self): if self.run_reflector_server: if self.reflector_port is not None: - reflector_factory = ReflectorServerFactory(self.session.peer_manager, self.session.blob_manager) + reflector_factory = reflector.ServerFactory( + self.session.peer_manager, + self.session.blob_manager + ) try: self.reflector_server_port = reactor.listenTCP(self.reflector_port, reflector_factory) except error.CannotListenError as e: - import traceback - log.error("Couldn't bind reflector to port %d. %s", self.reflector_port, traceback.format_exc()) - raise ValueError("%s lbrynet may already be running on your computer.", str(e)) + log.exception("Couldn't bind reflector to port %d", self.reflector_port) + raise ValueError("{} lbrynet may already be running on your computer.".format(e)) return defer.succeed(True) def _stop_reflector(self): diff --git a/lbrynet/lbrynet_daemon/LBRYPublisher.py b/lbrynet/lbrynet_daemon/LBRYPublisher.py index 970db91f6..2caf1d12b 100644 --- a/lbrynet/lbrynet_daemon/LBRYPublisher.py +++ b/lbrynet/lbrynet_daemon/LBRYPublisher.py @@ -12,7 +12,7 @@ from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob from lbrynet.core.PaymentRateManager import PaymentRateManager from lbrynet.core.LBRYMetadata import Metadata, CURRENT_METADATA_VERSION from lbrynet.lbryfilemanager.LBRYFileDownloader import ManagedLBRYFileDownloader -from lbrynet.reflector.client import LBRYFileReflectorClientFactory +from lbrynet import reflector from lbrynet.conf import LOG_FILE_NAME, REFLECTOR_SERVERS from twisted.internet import threads, defer, reactor @@ -70,9 +70,11 @@ class Publisher(object): return d def start_reflector(self): - factory = LBRYFileReflectorClientFactory(self.session.blob_manager, - self.lbry_file_manager.stream_info_manager, - self.stream_hash) + factory = reflector.ClientFactory( + self.session.blob_manager, + self.lbry_file_manager.stream_info_manager, + self.stream_hash + ) reactor.connectTCP(self.reflector_server, self.reflector_port, factory) return factory.finished_deferred diff --git a/lbrynet/reflector/__init__.py b/lbrynet/reflector/__init__.py index e69de29bb..10e8292e7 100644 --- a/lbrynet/reflector/__init__.py +++ b/lbrynet/reflector/__init__.py @@ -0,0 +1,2 @@ +from lbrynet.reflector.server.server import ReflectorServerFactory as ServerFactory +from lbrynet.reflector.client.client import LBRYFileReflectorClientFactory as ClientFactory From d172d43ddf241348fc0417036ae17a331e1acee7 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Wed, 10 Aug 2016 08:29:44 -0500 Subject: [PATCH 14/44] add reflector functional test to travis --- lbrynet/lbryfile/__init__.py | 2 + tests/{lbrynet => functional}/__init__.py | 0 tests/functional/test_reflector.py | 189 ++++++++++++++++++ tests/mocks.py | 164 +++++++++++++++ tests/{lbrynet/core => unit}/__init__.py | 0 .../core/server => unit/core}/__init__.py | 0 .../core/server}/__init__.py | 0 .../core/server/test_BlobRequestHandler.py | 0 .../core/test_LBRYExchangeRateManager.py | 0 .../core/test_LBRYMetadata.py | 0 tests/{lbrynet => unit}/core/test_utils.py | 0 tests/unit/lbrynet_daemon/__init__.py | 0 .../lbrynet_daemon/test_LBRYDaemon.py | 0 13 files changed, 355 insertions(+) rename tests/{lbrynet => functional}/__init__.py (100%) create mode 100644 tests/functional/test_reflector.py create mode 100644 tests/mocks.py rename tests/{lbrynet/core => unit}/__init__.py (100%) rename tests/{lbrynet/core/server => unit/core}/__init__.py (100%) rename tests/{lbrynet/lbrynet_daemon => unit/core/server}/__init__.py (100%) rename tests/{lbrynet => unit}/core/server/test_BlobRequestHandler.py (100%) rename tests/{lbrynet => unit}/core/test_LBRYExchangeRateManager.py (100%) rename tests/{lbrynet => unit}/core/test_LBRYMetadata.py (100%) rename tests/{lbrynet => unit}/core/test_utils.py (100%) create mode 100644 tests/unit/lbrynet_daemon/__init__.py rename tests/{lbrynet => unit}/lbrynet_daemon/test_LBRYDaemon.py (100%) diff --git a/lbrynet/lbryfile/__init__.py b/lbrynet/lbryfile/__init__.py index e69de29bb..8cd10066a 100644 --- a/lbrynet/lbryfile/__init__.py +++ b/lbrynet/lbryfile/__init__.py @@ -0,0 +1,2 @@ +from lbrynet.lbryfile.StreamDescriptor import get_sd_info +from lbrynet.lbryfile.StreamDescriptor import publish_sd_blob diff --git a/tests/lbrynet/__init__.py b/tests/functional/__init__.py similarity index 100% rename from tests/lbrynet/__init__.py rename to tests/functional/__init__.py diff --git a/tests/functional/test_reflector.py b/tests/functional/test_reflector.py new file mode 100644 index 000000000..d77557108 --- /dev/null +++ b/tests/functional/test_reflector.py @@ -0,0 +1,189 @@ +import os +import shutil + +from twisted.internet import defer, threads, error +from twisted.trial import unittest + +from lbrynet import reflector +from lbrynet import conf +from lbrynet import lbryfile +from lbrynet.core import BlobManager +from lbrynet.core import PeerManager +from lbrynet.core import RateLimiter +from lbrynet.core import Session +from lbrynet.core import StreamDescriptor +from lbrynet.lbryfile import LBRYFileMetadataManager +from lbrynet.lbryfile.client import LBRYFileOptions +from lbrynet.lbryfilemanager import LBRYFileCreator +from lbrynet.lbryfilemanager import LBRYFileManager + +from tests import mocks + + +class TestReflector(unittest.TestCase): + def setUp(self): + self.session = None + self.stream_info_manager = None + self.lbry_file_manager = None + self.server_blob_manager = None + self.reflector_port = None + self.addCleanup(self.take_down_env) + + def take_down_env(self): + d = defer.succeed(True) + if self.lbry_file_manager is not None: + d.addCallback(lambda _: self.lbry_file_manager.stop()) + if self.session is not None: + d.addCallback(lambda _: self.session.shut_down()) + if self.stream_info_manager is not None: + d.addCallback(lambda _: self.stream_info_manager.stop()) + if self.server_blob_manager is not None: + d.addCallback(lambda _: self.server_blob_manager.stop()) + if self.reflector_port is not None: + d.addCallback(lambda _: self.reflector_port.stopListening()) + + def delete_test_env(): + shutil.rmtree('client') + + d.addCallback(lambda _: threads.deferToThread(delete_test_env)) + return d + + def test_reflector(self): + wallet = mocks.Wallet() + peer_manager = PeerManager.PeerManager() + peer_finder = mocks.PeerFinder(5553, peer_manager, 2) + hash_announcer = mocks.Announcer() + rate_limiter = RateLimiter.DummyRateLimiter() + sd_identifier = StreamDescriptor.StreamDescriptorIdentifier() + + self.expected_blobs = [ + ( + 'dc4708f76a5e7af0f1cae0ee96b824e2ed9250c9346c093b' + '441f0a20d3607c17948b6fcfb4bc62020fe5286693d08586', + 2097152 + ), + ( + 'f4067522c1b49432a2a679512e3917144317caa1abba0c04' + '1e0cd2cf9f635d4cf127ce1824fa04189b63916174951f70', + 2097152 + ), + ( + '305486c434260484fcb2968ce0e963b72f81ba56c11b08b1' + 'af0789b55b44d78422600f9a38e3cf4f2e9569897e5646a9', + 1015056 + ), + ] + + db_dir = "client" + os.mkdir(db_dir) + + self.session = Session.LBRYSession( + conf.MIN_BLOB_DATA_PAYMENT_RATE, + db_dir=db_dir, + lbryid="abcd", + peer_finder=peer_finder, + hash_announcer=hash_announcer, + blob_dir=None, + peer_port=5553, + use_upnp=False, + rate_limiter=rate_limiter, + wallet=wallet + ) + + self.stream_info_manager = LBRYFileMetadataManager.TempLBRYFileMetadataManager() + + self.lbry_file_manager = LBRYFileManager.LBRYFileManager( + self.session, self.stream_info_manager, sd_identifier) + + self.server_blob_manager = BlobManager.TempBlobManager(hash_announcer) + + d = self.session.setup() + d.addCallback(lambda _: self.stream_info_manager.setup()) + d.addCallback(lambda _: LBRYFileOptions.add_lbry_file_to_sd_identifier(sd_identifier)) + d.addCallback(lambda _: self.lbry_file_manager.setup()) + d.addCallback(lambda _: self.server_blob_manager.setup()) + + def verify_equal(sd_info): + self.assertEqual(sd_info, mocks.create_stream_sd_file) + + def save_sd_blob_hash(sd_hash): + self.expected_blobs.append((sd_hash, 923)) + + def verify_stream_descriptor_file(stream_hash): + d = lbryfile.get_sd_info(self.lbry_file_manager.stream_info_manager, stream_hash, True) + d.addCallback(verify_equal) + d.addCallback( + lambda _: lbryfile.publish_sd_blob( + self.lbry_file_manager.stream_info_manager, + self.session.blob_manager, stream_hash + ) + ) + d.addCallback(save_sd_blob_hash) + d.addCallback(lambda _: stream_hash) + return d + + def iv_generator(): + iv = 0 + while 1: + iv += 1 + yield "%016d" % iv + + def create_stream(): + test_file = mocks.GenFile(5209343, b''.join([chr(i + 3) for i in xrange(0, 64, 6)])) + d = LBRYFileCreator.create_lbry_file( + self.session, + self.lbry_file_manager, + "test_file", + test_file, + key="0123456701234567", + iv_generator=iv_generator() + ) + return d + + def start_server(): + server_factory = reflector.ServerFactory(peer_manager, self.server_blob_manager) + from twisted.internet import reactor + port = 8943 + while self.reflector_port is None: + try: + self.reflector_port = reactor.listenTCP(port, server_factory) + except error.CannotListenError: + port += 1 + return defer.succeed(port) + + def send_to_server(port, stream_hash): + factory = reflector.ClientFactory( + self.session.blob_manager, + self.stream_info_manager, + stream_hash + ) + + from twisted.internet import reactor + reactor.connectTCP('localhost', port, factory) + return factory.finished_deferred + + def verify_blob_completed(blob, blob_size): + self.assertTrue(blob.is_validated()) + self.assertEqual(blob_size, blob.length) + + def verify_have_blob(blob_hash, blob_size): + d = self.server_blob_manager.get_blob(blob_hash, True) + d.addCallback(lambda blob: verify_blob_completed(blob, blob_size)) + return d + + def verify_data_on_reflector(): + check_blob_ds = [] + for blob_hash, blob_size in self.expected_blobs: + check_blob_ds.append(verify_have_blob(blob_hash, blob_size)) + return defer.DeferredList(check_blob_ds) + + def upload_to_reflector(stream_hash): + d = start_server() + d.addCallback(lambda port: send_to_server(port, stream_hash)) + d.addCallback(lambda _: verify_data_on_reflector()) + return d + + d.addCallback(lambda _: create_stream()) + d.addCallback(verify_stream_descriptor_file) + d.addCallback(upload_to_reflector) + return d diff --git a/tests/mocks.py b/tests/mocks.py new file mode 100644 index 000000000..6ea183195 --- /dev/null +++ b/tests/mocks.py @@ -0,0 +1,164 @@ +import io + +from Crypto.PublicKey import RSA +from twisted.internet import defer, threads, task, error + +from lbrynet.core import PTCWallet + + +class Node(object): + def __init__(self, *args, **kwargs): + pass + + def joinNetwork(self, *args): + pass + + def stop(self): + pass + + +class Wallet(object): + def __init__(self): + self.private_key = RSA.generate(1024) + self.encoded_public_key = self.private_key.publickey().exportKey() + + def start(self): + return defer.succeed(True) + + def stop(self): + return defer.succeed(True) + + def get_info_exchanger(self): + return PTCWallet.PointTraderKeyExchanger(self) + + def get_wallet_info_query_handler_factory(self): + return PTCWallet.PointTraderKeyQueryHandlerFactory(self) + + def reserve_points(self, *args): + return True + + def cancel_point_reservation(self, *args): + pass + + def send_points(self, *args): + return defer.succeed(True) + + def add_expected_payment(self, *args): + pass + + def get_balance(self): + return defer.succeed(1000) + + def set_public_key_for_peer(self, peer, public_key): + pass + + +class PeerFinder(object): + def __init__(self, start_port, peer_manager, num_peers): + self.start_port = start_port + self.peer_manager = peer_manager + self.num_peers = num_peers + self.count = 0 + + def find_peers_for_blob(self, *args): + peer_port = self.start_port + self.count + self.count += 1 + if self.count >= self.num_peers: + self.count = 0 + return defer.succeed([self.peer_manager.get_peer("127.0.0.1", peer_port)]) + + def run_manage_loop(self): + pass + + def stop(self): + pass + + +class Announcer(object): + def __init__(self, *args): + pass + + def add_supplier(self, supplier): + pass + + def immediate_announce(self, *args): + pass + + def run_manage_loop(self): + pass + + def stop(self): + pass + + +class GenFile(io.RawIOBase): + def __init__(self, size, pattern): + io.RawIOBase.__init__(self) + self.size = size + self.pattern = pattern + self.read_so_far = 0 + self.buff = b'' + self.last_offset = 0 + + def readable(self): + return True + + def writable(self): + return False + + def read(self, n=-1): + if n > -1: + bytes_to_read = min(n, self.size - self.read_so_far) + else: + bytes_to_read = self.size - self.read_so_far + output, self.buff = self.buff[:bytes_to_read], self.buff[bytes_to_read:] + bytes_to_read -= len(output) + while bytes_to_read > 0: + self.buff = self._generate_chunk() + new_output, self.buff = self.buff[:bytes_to_read], self.buff[bytes_to_read:] + bytes_to_read -= len(new_output) + output += new_output + self.read_so_far += len(output) + return output + + def readall(self): + return self.read() + + def _generate_chunk(self, n=2**10): + output = self.pattern[self.last_offset:self.last_offset + n] + n_left = n - len(output) + whole_patterns = n_left / len(self.pattern) + output += self.pattern * whole_patterns + self.last_offset = n - len(output) + output += self.pattern[:self.last_offset] + return output + + +create_stream_sd_file = { + 'stream_name': '746573745f66696c65', + 'blobs': [ + { + 'length': 2097152, + 'blob_num': 0, + 'blob_hash': 'dc4708f76a5e7af0f1cae0ee96b824e2ed9250c9346c093b441f0a20d3607c17948b6fcfb4bc62020fe5286693d08586', + 'iv': '30303030303030303030303030303031' + }, + { + 'length': 2097152, + 'blob_num': 1, + 'blob_hash': 'f4067522c1b49432a2a679512e3917144317caa1abba0c041e0cd2cf9f635d4cf127ce1824fa04189b63916174951f70', + 'iv': '30303030303030303030303030303032' + }, + { + 'length': 1015056, + 'blob_num': 2, + 'blob_hash': '305486c434260484fcb2968ce0e963b72f81ba56c11b08b1af0789b55b44d78422600f9a38e3cf4f2e9569897e5646a9', + 'iv': '30303030303030303030303030303033' + }, + {'length': 0, 'blob_num': 3, 'iv': '30303030303030303030303030303034'} + ], + 'stream_type': 'lbryfile', + 'key': '30313233343536373031323334353637', + 'suggested_file_name': '746573745f66696c65', + 'stream_hash': '6d27fbe10c86d81aacfb897c7a426d0a2214f5a299455a6d315c0f998c4b3545c2dc60906122d94653c23b1898229e3f' +} diff --git a/tests/lbrynet/core/__init__.py b/tests/unit/__init__.py similarity index 100% rename from tests/lbrynet/core/__init__.py rename to tests/unit/__init__.py diff --git a/tests/lbrynet/core/server/__init__.py b/tests/unit/core/__init__.py similarity index 100% rename from tests/lbrynet/core/server/__init__.py rename to tests/unit/core/__init__.py diff --git a/tests/lbrynet/lbrynet_daemon/__init__.py b/tests/unit/core/server/__init__.py similarity index 100% rename from tests/lbrynet/lbrynet_daemon/__init__.py rename to tests/unit/core/server/__init__.py diff --git a/tests/lbrynet/core/server/test_BlobRequestHandler.py b/tests/unit/core/server/test_BlobRequestHandler.py similarity index 100% rename from tests/lbrynet/core/server/test_BlobRequestHandler.py rename to tests/unit/core/server/test_BlobRequestHandler.py diff --git a/tests/lbrynet/core/test_LBRYExchangeRateManager.py b/tests/unit/core/test_LBRYExchangeRateManager.py similarity index 100% rename from tests/lbrynet/core/test_LBRYExchangeRateManager.py rename to tests/unit/core/test_LBRYExchangeRateManager.py diff --git a/tests/lbrynet/core/test_LBRYMetadata.py b/tests/unit/core/test_LBRYMetadata.py similarity index 100% rename from tests/lbrynet/core/test_LBRYMetadata.py rename to tests/unit/core/test_LBRYMetadata.py diff --git a/tests/lbrynet/core/test_utils.py b/tests/unit/core/test_utils.py similarity index 100% rename from tests/lbrynet/core/test_utils.py rename to tests/unit/core/test_utils.py diff --git a/tests/unit/lbrynet_daemon/__init__.py b/tests/unit/lbrynet_daemon/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lbrynet/lbrynet_daemon/test_LBRYDaemon.py b/tests/unit/lbrynet_daemon/test_LBRYDaemon.py similarity index 100% rename from tests/lbrynet/lbrynet_daemon/test_LBRYDaemon.py rename to tests/unit/lbrynet_daemon/test_LBRYDaemon.py From d6f902653f90fb5405988ebb427a02a9f36a381f Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Wed, 10 Aug 2016 08:35:15 -0500 Subject: [PATCH 15/44] put import in alphabetical order --- tests/functional/test_reflector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/test_reflector.py b/tests/functional/test_reflector.py index d77557108..ee0aeb9e0 100644 --- a/tests/functional/test_reflector.py +++ b/tests/functional/test_reflector.py @@ -4,9 +4,9 @@ import shutil from twisted.internet import defer, threads, error from twisted.trial import unittest -from lbrynet import reflector from lbrynet import conf from lbrynet import lbryfile +from lbrynet import reflector from lbrynet.core import BlobManager from lbrynet.core import PeerManager from lbrynet.core import RateLimiter From 132aa569fe203077f9aec010f6372857ecfed383 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 10 Aug 2016 16:51:46 -0400 Subject: [PATCH 16/44] move run_reflector_server into settings file and add debug lines --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 7 ++++--- lbrynet/lbrynet_daemon/LBRYPublisher.py | 1 + lbrynet/reflector/client/client.py | 3 +++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index d4e52b91e..2afcac7b2 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -175,9 +175,6 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.first_run_after_update = False self.uploaded_temp_files = [] - # change this to enable reflector server - self.run_reflector_server = False - if os.name == "nt": from lbrynet.winhelpers.knownpaths import get_path, FOLDERID, UserHandle default_download_directory = get_path(FOLDERID.Downloads, UserHandle.current) @@ -216,6 +213,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): 'use_upnp': True, 'start_lbrycrdd': True, 'requested_first_run_credits': False, + 'run_reflector_server': False, 'cache_time': DEFAULT_CACHE_TIME, 'startup_scripts': [], 'last_version': {'lbrynet': lbrynet_version, 'lbryum': lbryum_version} @@ -275,6 +273,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.search_timeout = self.session_settings['search_timeout'] self.download_timeout = self.session_settings['download_timeout'] self.max_search_results = self.session_settings['max_search_results'] + self.run_reflector_server = self.session_settings['run_reflector_server'] #### # # Ignore the saved wallet type. Some users will have their wallet type @@ -684,6 +683,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): def _start_reflector(self): if self.run_reflector_server: + log.info("Starting reflector server") if self.reflector_port is not None: reflector_factory = reflector.ServerFactory( self.session.peer_manager, @@ -698,6 +698,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): def _stop_reflector(self): if self.run_reflector_server: + log.info("Stopping reflector server") try: if self.reflector_server_port is not None: self.reflector_server_port, p = None, self.reflector_server_port diff --git a/lbrynet/lbrynet_daemon/LBRYPublisher.py b/lbrynet/lbrynet_daemon/LBRYPublisher.py index 2caf1d12b..de9c558f3 100644 --- a/lbrynet/lbrynet_daemon/LBRYPublisher.py +++ b/lbrynet/lbrynet_daemon/LBRYPublisher.py @@ -70,6 +70,7 @@ class Publisher(object): return d def start_reflector(self): + log.info("Start reflector client") factory = reflector.ClientFactory( self.session.blob_manager, self.lbry_file_manager.stream_info_manager, diff --git a/lbrynet/reflector/client/client.py b/lbrynet/reflector/client/client.py index ce2102f95..5ce017114 100644 --- a/lbrynet/reflector/client/client.py +++ b/lbrynet/reflector/client/client.py @@ -118,10 +118,12 @@ class LBRYFileReflectorClient(Protocol): reactor.callLater(0, self.producer.resumeProducing) def get_blobs_to_send(self, stream_info_manager, stream_hash): + log.info("Get blobs to send to reflector") d = stream_info_manager.get_blobs_for_stream(stream_hash) def set_blobs(blob_hashes): for blob_hash, position, iv, length in blob_hashes: + log.info("Preparing to send %s", blob_hash) if blob_hash is not None: self.blob_hashes_to_send.append(blob_hash) @@ -202,6 +204,7 @@ class LBRYFileReflectorClient(Protocol): raise ValueError("Couldn't open that blob for some reason. blob_hash: {}".format(blob.blob_hash)) def send_blob_info(self): + log.info("Send blob info for %s", self.next_blob_to_send.blob_hash) assert self.next_blob_to_send is not None, "need to have a next blob to send at this point" self.write(json.dumps({ 'blob_hash': self.next_blob_to_send.blob_hash, From 1a42e4357316f4ec80da00dbeda11b1b3aab06e5 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Wed, 10 Aug 2016 19:04:03 -0500 Subject: [PATCH 17/44] log all the things --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 1 + lbrynet/lbrynet_daemon/LBRYDaemonControl.py | 2 +- lbrynet/lbrynet_daemon/LBRYPublisher.py | 25 +++++++++++++---- lbrynet/reflector/client/client.py | 31 +++++++++++++++++++-- tests/functional/test_reflector.py | 15 +++++----- 5 files changed, 57 insertions(+), 17 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 2afcac7b2..026793d43 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -691,6 +691,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): ) try: self.reflector_server_port = reactor.listenTCP(self.reflector_port, reflector_factory) + log.info('Started reflector on port %s', self.reflector_port) except error.CannotListenError as e: log.exception("Couldn't bind reflector to port %d", self.reflector_port) raise ValueError("{} lbrynet may already be running on your computer.".format(e)) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py index 157087f0b..76cbe5152 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py @@ -77,7 +77,7 @@ def start(): log_support.configure_file_handler(lbrynet_log) log_support.configure_loggly_handler() if args.logtoconsole: - log_support.configure_console() + log_support.configure_console(level='DEBUG') try: JSONRPCProxy.from_url(API_CONNECTION_STRING).is_running() diff --git a/lbrynet/lbrynet_daemon/LBRYPublisher.py b/lbrynet/lbrynet_daemon/LBRYPublisher.py index de9c558f3..7eb801cc3 100644 --- a/lbrynet/lbrynet_daemon/LBRYPublisher.py +++ b/lbrynet/lbrynet_daemon/LBRYPublisher.py @@ -47,9 +47,9 @@ class Publisher(object): self.metadata = {} def start(self, name, file_path, bid, metadata): - + log.info('Starting publish for %s', name) def _show_result(): - log.info("Published %s --> lbry://%s txid: %s", self.file_name, self.publish_name, self.txid) + log.info("Success! Published %s --> lbry://%s txid: %s", self.file_name, self.publish_name, self.txid) return defer.succeed(self.txid) self.publish_name = name @@ -99,10 +99,13 @@ class Publisher(object): return d def _create_sd_blob(self): - d = publish_sd_blob(self.lbry_file_manager.stream_info_manager, self.session.blob_manager, + log.debug('Creating stream descriptor blob') + d = publish_sd_blob(self.lbry_file_manager.stream_info_manager, + self.session.blob_manager, self.lbry_file.stream_hash) def set_sd_hash(sd_hash): + log.debug('stream descriptor hash: %s', sd_hash) if 'sources' not in self.metadata: self.metadata['sources'] = {} self.metadata['sources']['lbry_sd_hash'] = sd_hash @@ -111,23 +114,29 @@ class Publisher(object): return d def set_status(self): + log.debug('Setting status') d = self.lbry_file_manager.change_lbry_file_status(self.lbry_file, ManagedLBRYFileDownloader.STATUS_FINISHED) d.addCallback(lambda _: self.lbry_file.restore()) return d def _claim_name(self): - self.metadata['content-type'] = mimetypes.guess_type(os.path.join(self.lbry_file.download_directory, - self.lbry_file.file_name))[0] - self.metadata['ver'] = CURRENT_METADATA_VERSION + log.debug('Claiming name') + self._update_metadata() m = Metadata(self.metadata) def set_tx_hash(txid): + log.debug('Name claimed using txid: %s', txid) self.txid = txid d = self.wallet.claim_name(self.publish_name, self.bid_amount, m) d.addCallback(set_tx_hash) return d + def _update_metadata(self): + filename = os.path.join(self.lbry_file.download_directory, self.lbry_file.file_name) + self.metadata['content-type'] = get_content_type(filename) + self.metadata['ver'] = CURRENT_METADATA_VERSION + def _show_publish_error(self, err): log.info(err.getTraceback()) message = "An error occurred publishing %s to %s. Error: %s." @@ -140,3 +149,7 @@ class Publisher(object): log.error(message, str(self.file_name), str(self.publish_name), err.getTraceback()) return defer.fail(Exception("Publish failed")) + + +def get_content_type(filename): + return mimetypes.guess_type(filename)[0] diff --git a/lbrynet/reflector/client/client.py b/lbrynet/reflector/client/client.py index 5ce017114..d6631b8de 100644 --- a/lbrynet/reflector/client/client.py +++ b/lbrynet/reflector/client/client.py @@ -82,6 +82,7 @@ class LBRYFileReflectorClient(Protocol): d.addErrback(lambda err: log.warning("An error occurred immediately: %s", err.getTraceback())) def dataReceived(self, data): + log.debug('Recieved %s', data) self.response_buff += data try: msg = self.parse_response(self.response_buff) @@ -95,8 +96,10 @@ class LBRYFileReflectorClient(Protocol): def connectionLost(self, reason): if reason.check(error.ConnectionDone): + log.debug('Finished sending data via reflector') self.factory.finished_deferred.callback(True) else: + log.debug('reflector finished: %s', reason) self.factory.finished_deferred.callback(reason) # IConsumer stuff @@ -118,7 +121,7 @@ class LBRYFileReflectorClient(Protocol): reactor.callLater(0, self.producer.resumeProducing) def get_blobs_to_send(self, stream_info_manager, stream_hash): - log.info("Get blobs to send to reflector") + log.debug('Getting blobs from stream hash: %s', stream_hash) d = stream_info_manager.get_blobs_for_stream(stream_hash) def set_blobs(blob_hashes): @@ -139,6 +142,7 @@ class LBRYFileReflectorClient(Protocol): return d def send_handshake(self): + log.debug('Sending handshake') self.write(json.dumps({'version': 0})) def parse_response(self, buff): @@ -198,6 +202,7 @@ class LBRYFileReflectorClient(Protocol): if blob.is_validated(): read_handle = blob.open_for_reading() if read_handle is not None: + log.debug('Getting ready to send %s', blob.blob_hash) self.next_blob_to_send = blob self.read_handle = read_handle return None @@ -206,6 +211,7 @@ class LBRYFileReflectorClient(Protocol): def send_blob_info(self): log.info("Send blob info for %s", self.next_blob_to_send.blob_hash) assert self.next_blob_to_send is not None, "need to have a next blob to send at this point" + log.debug('sending blob info') self.write(json.dumps({ 'blob_hash': self.next_blob_to_send.blob_hash, 'blob_size': self.next_blob_to_send.length @@ -214,10 +220,12 @@ class LBRYFileReflectorClient(Protocol): def send_next_request(self): if self.file_sender is not None: # send the blob + log.debug('Sending the blob') return self.start_transfer() elif self.blob_hashes_to_send: # open the next blob to send blob_hash = self.blob_hashes_to_send[0] + log.debug('No current blob, sending the next one: %s', blob_hash) self.blob_hashes_to_send = self.blob_hashes_to_send[1:] d = self.blob_manager.get_blob(blob_hash, True) d.addCallback(self.open_blob_for_reading) @@ -226,7 +234,8 @@ class LBRYFileReflectorClient(Protocol): return d else: # close connection - self.transport.loseConnection() + log.debug('No more blob hashes, closing connection') + self.transport.closeConnection() class LBRYFileReflectorClientFactory(ClientFactory): @@ -243,4 +252,20 @@ class LBRYFileReflectorClientFactory(ClientFactory): p = self.protocol() p.factory = self self.p = p - return p \ No newline at end of file + return p + + def startFactory(self): + log.debug('Starting reflector factory') + ClientFactory.startFactory(self) + + def startedConnecting(self, connector): + log.debug('Started connecting') + + def clientConnectionLost(self, connector, reason): + """If we get disconnected, reconnect to server.""" + connector.connect() + + def clientConnectionFailed(self, connector, reason): + print("connection failed:", reason) + from twisted.internet import reactor + reactor.stop() diff --git a/tests/functional/test_reflector.py b/tests/functional/test_reflector.py index ee0aeb9e0..691653eae 100644 --- a/tests/functional/test_reflector.py +++ b/tests/functional/test_reflector.py @@ -104,7 +104,7 @@ class TestReflector(unittest.TestCase): d.addCallback(lambda _: self.server_blob_manager.setup()) def verify_equal(sd_info): - self.assertEqual(sd_info, mocks.create_stream_sd_file) + self.assertEqual(mocks.create_stream_sd_file, sd_info) def save_sd_blob_hash(sd_hash): self.expected_blobs.append((sd_hash, 923)) @@ -122,12 +122,6 @@ class TestReflector(unittest.TestCase): d.addCallback(lambda _: stream_hash) return d - def iv_generator(): - iv = 0 - while 1: - iv += 1 - yield "%016d" % iv - def create_stream(): test_file = mocks.GenFile(5209343, b''.join([chr(i + 3) for i in xrange(0, 64, 6)])) d = LBRYFileCreator.create_lbry_file( @@ -187,3 +181,10 @@ class TestReflector(unittest.TestCase): d.addCallback(verify_stream_descriptor_file) d.addCallback(upload_to_reflector) return d + + +def iv_generator(): + iv = 0 + while True: + iv += 1 + yield "%016d" % iv From 0995d864e8fa89fad2c073d9f47e0d6244b8694a Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Wed, 10 Aug 2016 19:09:20 -0500 Subject: [PATCH 18/44] connectTCP wants an ip address, not a url --- lbrynet/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 1b956e092..79d82df03 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -28,7 +28,7 @@ SEARCH_SERVERS = ["http://lighthouse1.lbry.io:50005", "http://lighthouse2.lbry.io:50005", "http://lighthouse3.lbry.io:50005"] -REFLECTOR_SERVERS = [("http://reflector.lbry.io", 5566)] +REFLECTOR_SERVERS = [("reflector.lbry.io", 5566)] LOG_FILE_NAME = "lbrynet.log" LOG_POST_URL = "https://lbry.io/log-upload" From 35481a92f54f5d3bc6378ecf1940601943887e9b Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Wed, 10 Aug 2016 19:36:52 -0500 Subject: [PATCH 19/44] actually cleanup patched time.time --- lbrynet/reflector/client/client.py | 2 +- tests/unit/core/test_LBRYExchangeRateManager.py | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/lbrynet/reflector/client/client.py b/lbrynet/reflector/client/client.py index d6631b8de..72d353085 100644 --- a/lbrynet/reflector/client/client.py +++ b/lbrynet/reflector/client/client.py @@ -235,7 +235,7 @@ class LBRYFileReflectorClient(Protocol): else: # close connection log.debug('No more blob hashes, closing connection') - self.transport.closeConnection() + self.transport.loseConnection() class LBRYFileReflectorClientFactory(ClientFactory): diff --git a/tests/unit/core/test_LBRYExchangeRateManager.py b/tests/unit/core/test_LBRYExchangeRateManager.py index 2a6457536..e07b49194 100644 --- a/tests/unit/core/test_LBRYExchangeRateManager.py +++ b/tests/unit/core/test_LBRYExchangeRateManager.py @@ -19,12 +19,10 @@ class LBRYFeeFormatTest(unittest.TestCase): class LBRYFeeTest(unittest.TestCase): def setUp(self): - self.patcher = mock.patch('time.time') - self.time = self.patcher.start() + patcher = mock.patch('time.time') + self.time = patcher.start() self.time.return_value = 0 - - def tearDown(self): - self.time.stop() + self.addCleanup(patcher.stop) def test_fee_converts_to_lbc(self): fee_dict = { @@ -35,4 +33,4 @@ class LBRYFeeTest(unittest.TestCase): } rates = {'BTCLBC': {'spot': 3.0, 'ts': 2}, 'USDBTC': {'spot': 2.0, 'ts': 3}} manager = LBRYExchangeRateManager.DummyExchangeRateManager(rates) - self.assertEqual(60.0, manager.to_lbc(fee_dict).amount) \ No newline at end of file + self.assertEqual(60.0, manager.to_lbc(fee_dict).amount) From 0a07c8f1312f2f628c78dba289c6afdebfef45d2 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Wed, 10 Aug 2016 19:39:28 -0500 Subject: [PATCH 20/44] log on closed/lost connection --- lbrynet/reflector/client/client.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lbrynet/reflector/client/client.py b/lbrynet/reflector/client/client.py index 72d353085..0fe2c8ce1 100644 --- a/lbrynet/reflector/client/client.py +++ b/lbrynet/reflector/client/client.py @@ -263,9 +263,7 @@ class LBRYFileReflectorClientFactory(ClientFactory): def clientConnectionLost(self, connector, reason): """If we get disconnected, reconnect to server.""" - connector.connect() + log.debug("connection lost: %s", reason) def clientConnectionFailed(self, connector, reason): - print("connection failed:", reason) - from twisted.internet import reactor - reactor.stop() + log.debug("connection failed: %s", reason) From f83daa5a22a92fd0fd46bd347e5b95d7f14e3876 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Thu, 11 Aug 2016 04:44:09 +0000 Subject: [PATCH 21/44] more logging on the server side --- lbrynet/__init__.py | 6 ----- lbrynet/core/log_support.py | 4 +++- lbrynet/lbrynet_daemon/LBRYDaemon.py | 4 ++-- lbrynet/lbrynet_daemon/LBRYDaemonControl.py | 4 ++-- lbrynet/reflector/server/server.py | 25 +++++++++++++++------ 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/lbrynet/__init__.py b/lbrynet/__init__.py index 6b754d3a2..9a7032852 100644 --- a/lbrynet/__init__.py +++ b/lbrynet/__init__.py @@ -1,8 +1,2 @@ -import logging - -log = logging.getLogger(__name__) -logging.getLogger(__name__).addHandler(logging.NullHandler()) -log.setLevel(logging.INFO) - __version__ = "0.3.17" version = tuple(__version__.split('.')) \ No newline at end of file diff --git a/lbrynet/core/log_support.py b/lbrynet/core/log_support.py index 50fb35e83..9644888df 100644 --- a/lbrynet/core/log_support.py +++ b/lbrynet/core/log_support.py @@ -58,8 +58,10 @@ def _log_decorator(fn): handler = fn(*args, **kwargs) if handler.name: remove_handlers(log, handler.name) + handler.setLevel(level) log.addHandler(handler) - log.setLevel(level) + if log.level > level: + log.setLevel(level) return helper diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 026793d43..f9c0a7cbe 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -213,7 +213,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): 'use_upnp': True, 'start_lbrycrdd': True, 'requested_first_run_credits': False, - 'run_reflector_server': False, + 'run_reflector_server': True, 'cache_time': DEFAULT_CACHE_TIME, 'startup_scripts': [], 'last_version': {'lbrynet': lbrynet_version, 'lbryum': lbryum_version} @@ -683,8 +683,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): def _start_reflector(self): if self.run_reflector_server: - log.info("Starting reflector server") if self.reflector_port is not None: + log.info("Starting reflector server listening to %s", self.reflector_port) reflector_factory = reflector.ServerFactory( self.session.peer_manager, self.session.blob_manager diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py index 76cbe5152..4d68424c0 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py @@ -74,8 +74,8 @@ def start(): args = parser.parse_args() log_support.disable_noisy_loggers() - log_support.configure_file_handler(lbrynet_log) - log_support.configure_loggly_handler() + log_support.configure_file_handler(lbrynet_log, level=10) + log_support.configure_loggly_handler(level=10) if args.logtoconsole: log_support.configure_console(level='DEBUG') diff --git a/lbrynet/reflector/server/server.py b/lbrynet/reflector/server/server.py index d2569b6aa..b8a1c6d68 100644 --- a/lbrynet/reflector/server/server.py +++ b/lbrynet/reflector/server/server.py @@ -11,9 +11,9 @@ log = logging.getLogger(__name__) class ReflectorServer(Protocol): - def connectionMade(self): peer_info = self.transport.getPeer() + log.debug('Connection made to %s', peer_info) self.peer = self.factory.peer_manager.get_peer(peer_info.host, peer_info.port) self.blob_manager = self.factory.blob_manager self.received_handshake = False @@ -29,17 +29,21 @@ class ReflectorServer(Protocol): pass def dataReceived(self, data): - if self.receiving_blob is False: + log.debug('Recieved data: %s', data) + if self.receiving_blob: + log.debug('Writing data to blob') + self.blob_write(data) + else: + log.debug('Not yet recieving blob, data needs further processing') self.request_buff += data msg, extra_data = self._get_valid_response(self.request_buff) - if msg is not None: + if msg: self.request_buff = '' d = self.handle_request(msg) d.addCallbacks(self.send_response, self.handle_error) - if self.receiving_blob is True and len(extra_data) != 0: + if self.receiving_blob and extra_data: + log.debug('Writing extra data to blog') self.blob_write(extra_data) - else: - self.blob_write(data) def _get_valid_response(self, response_msg): extra_data = None @@ -70,6 +74,7 @@ class ReflectorServer(Protocol): return self.handle_normal_request(request_dict) def handle_handshake(self, request_dict): + log.debug('Handling handshake') if 'version' not in request_dict: raise ValueError("Client should send version") self.peer_version = int(request_dict['version']) @@ -103,6 +108,7 @@ class ReflectorServer(Protocol): raise ValueError("Expected a blob hash and a blob size") if not is_valid_blobhash(request_dict['blob_hash']): raise ValueError("Got a bad blob hash: {}".format(request_dict['blob_hash'])) + log.debug('Recieved info for blob: %s', request_dict['blob_hash']) d = self.blob_manager.get_blob( request_dict['blob_hash'], True, @@ -114,6 +120,7 @@ class ReflectorServer(Protocol): # important in it. to the deferred that fires when the blob is done, # add a callback which returns a nice response dict saying to keep # sending, and then return that deferred + log.debug('blob is already open') self.receiving_blob = True d = self.blob_finished_d d.addCallback(lambda _: self.close_blob()) @@ -133,4 +140,8 @@ class ReflectorServerFactory(ServerFactory): def __init__(self, peer_manager, blob_manager): self.peer_manager = peer_manager - self.blob_manager = blob_manager \ No newline at end of file + self.blob_manager = blob_manager + + def buildProtocol(self, addr): + log.debug('Creating a protocol for %s', addr) + ServerFactory.buildProtocol(self, addr) From 52859825154384be6df36993bd0728cf9a9be86a Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 11 Aug 2016 01:06:51 -0400 Subject: [PATCH 22/44] resolve reflector ip --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 28 ++++++++++++++++++++++++- lbrynet/lbrynet_daemon/LBRYPublisher.py | 8 +++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index f9c0a7cbe..7a0be1eb1 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -48,7 +48,7 @@ from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, DEFAULT_MAX_SEARCH_RESULTS, \ KNOWN_DHT_NODES, DEFAULT_MAX_KEY_FEE, DEFAULT_WALLET, \ DEFAULT_SEARCH_TIMEOUT, DEFAULT_CACHE_TIME, DEFAULT_UI_BRANCH, \ - LOG_POST_URL, LOG_FILE_NAME + LOG_POST_URL, LOG_FILE_NAME, REFLECTOR_SERVERS from lbrynet.conf import DEFAULT_SD_DOWNLOAD_TIMEOUT from lbrynet.conf import DEFAULT_TIMEOUT from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob, BlobStreamDescriptorReader @@ -1889,6 +1889,27 @@ class LBRYDaemon(jsonrpc.JSONRPC): m['fee'][currency]['address'] = address return m + def _reflect_if_possible(sd_hash, txid): + log.info("Trying to start reflector") + d = self._get_lbry_file('sd_hash', sd_hash, return_json=False) + d.addCallback(lambda r: False if not r else _start_reflector(r.stream_hash)) + d.addCallback(lambda _: txid) + return d + + def _start_reflector(stream_hash): + reflector_server = random.choice(REFLECTOR_SERVERS) + reflector_address, reflector_port = reflector_server[0], reflector_server[1] + log.info("Start reflector client") + factory = reflector.ClientFactory( + self.session.blob_manager, + self.lbry_file_manager.stream_info_manager, + stream_hash + ) + d = reactor.resolve(reflector_address) + d.addCallback(lambda ip: reactor.connectTCP(ip, reflector_port, factory)) + d.addCallback(lambda _: factory.finished_deferred) + return d + name = p['name'] log.info("Publish: ") @@ -1905,8 +1926,10 @@ class LBRYDaemon(jsonrpc.JSONRPC): try: metadata = Metadata(p['metadata']) make_lbry_file = False + sd_hash = metadata['sources']['lbry_sd_hash'] except AssertionError: make_lbry_file = True + sd_hash = None metadata = p['metadata'] file_path = p['file_path'] @@ -1930,6 +1953,9 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda meta: pub.start(name, file_path, bid, meta)) else: d.addCallback(lambda meta: self.session.wallet.claim_name(name, bid, meta)) + if sd_hash: + d.addCallback(lambda txid: _reflect_if_possible(sd_hash, txid)) + d.addCallback(lambda txid: self._add_to_pending_claims(name, txid)) d.addCallback(lambda r: self._render_response(r, OK_CODE)) diff --git a/lbrynet/lbrynet_daemon/LBRYPublisher.py b/lbrynet/lbrynet_daemon/LBRYPublisher.py index 7eb801cc3..5efbe66a9 100644 --- a/lbrynet/lbrynet_daemon/LBRYPublisher.py +++ b/lbrynet/lbrynet_daemon/LBRYPublisher.py @@ -70,14 +70,18 @@ class Publisher(object): return d def start_reflector(self): + reflector_server = random.choice(REFLECTOR_SERVERS) + reflector_address, reflector_port = reflector_server[0], reflector_server[1] log.info("Start reflector client") factory = reflector.ClientFactory( self.session.blob_manager, self.lbry_file_manager.stream_info_manager, self.stream_hash ) - reactor.connectTCP(self.reflector_server, self.reflector_port, factory) - return factory.finished_deferred + d = reactor.resolve(reflector_address) + d.addCallback(lambda ip: reactor.connectTCP(ip, reflector_port, factory)) + d.addCallback(lambda _: factory.finished_deferred) + return d def _check_file_path(self, file_path): def check_file_threaded(): From defbd73b4f24532ccceb98a7082ac05c648e1241 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 11 Aug 2016 01:07:49 -0400 Subject: [PATCH 23/44] squelch very verbose dht error log --- lbrynet/dht/node.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index 8d61d74e2..053f46f27 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -240,9 +240,9 @@ class Node(object): known_nodes = {} def log_error(err, n): - log.error("error storing blob_hash %s at %s", binascii.hexlify(blob_hash), str(n)) - log.error(err.getErrorMessage()) - log.error(err.getTraceback()) + log.debug("error storing blob_hash %s at %s", binascii.hexlify(blob_hash), str(n)) + log.debug(err.getErrorMessage()) + log.debug(err.getTraceback()) def log_success(res): log.debug("Response to store request: %s", str(res)) From 80f27f2b296aee1af4f5f02d97a7aa7f1145f283 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Thu, 11 Aug 2016 05:11:18 +0000 Subject: [PATCH 24/44] Revert "more logging on the server side" This reverts commit f83daa5a22a92fd0fd46bd347e5b95d7f14e3876. --- lbrynet/__init__.py | 6 +++++ lbrynet/core/log_support.py | 4 +--- lbrynet/lbrynet_daemon/LBRYDaemon.py | 4 ++-- lbrynet/lbrynet_daemon/LBRYDaemonControl.py | 4 ++-- lbrynet/reflector/server/server.py | 25 ++++++--------------- 5 files changed, 18 insertions(+), 25 deletions(-) diff --git a/lbrynet/__init__.py b/lbrynet/__init__.py index 9a7032852..6b754d3a2 100644 --- a/lbrynet/__init__.py +++ b/lbrynet/__init__.py @@ -1,2 +1,8 @@ +import logging + +log = logging.getLogger(__name__) +logging.getLogger(__name__).addHandler(logging.NullHandler()) +log.setLevel(logging.INFO) + __version__ = "0.3.17" version = tuple(__version__.split('.')) \ No newline at end of file diff --git a/lbrynet/core/log_support.py b/lbrynet/core/log_support.py index 9644888df..50fb35e83 100644 --- a/lbrynet/core/log_support.py +++ b/lbrynet/core/log_support.py @@ -58,10 +58,8 @@ def _log_decorator(fn): handler = fn(*args, **kwargs) if handler.name: remove_handlers(log, handler.name) - handler.setLevel(level) log.addHandler(handler) - if log.level > level: - log.setLevel(level) + log.setLevel(level) return helper diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 7a0be1eb1..c39e16914 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -213,7 +213,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): 'use_upnp': True, 'start_lbrycrdd': True, 'requested_first_run_credits': False, - 'run_reflector_server': True, + 'run_reflector_server': False, 'cache_time': DEFAULT_CACHE_TIME, 'startup_scripts': [], 'last_version': {'lbrynet': lbrynet_version, 'lbryum': lbryum_version} @@ -683,8 +683,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): def _start_reflector(self): if self.run_reflector_server: + log.info("Starting reflector server") if self.reflector_port is not None: - log.info("Starting reflector server listening to %s", self.reflector_port) reflector_factory = reflector.ServerFactory( self.session.peer_manager, self.session.blob_manager diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py index 4d68424c0..76cbe5152 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py @@ -74,8 +74,8 @@ def start(): args = parser.parse_args() log_support.disable_noisy_loggers() - log_support.configure_file_handler(lbrynet_log, level=10) - log_support.configure_loggly_handler(level=10) + log_support.configure_file_handler(lbrynet_log) + log_support.configure_loggly_handler() if args.logtoconsole: log_support.configure_console(level='DEBUG') diff --git a/lbrynet/reflector/server/server.py b/lbrynet/reflector/server/server.py index b8a1c6d68..d2569b6aa 100644 --- a/lbrynet/reflector/server/server.py +++ b/lbrynet/reflector/server/server.py @@ -11,9 +11,9 @@ log = logging.getLogger(__name__) class ReflectorServer(Protocol): + def connectionMade(self): peer_info = self.transport.getPeer() - log.debug('Connection made to %s', peer_info) self.peer = self.factory.peer_manager.get_peer(peer_info.host, peer_info.port) self.blob_manager = self.factory.blob_manager self.received_handshake = False @@ -29,21 +29,17 @@ class ReflectorServer(Protocol): pass def dataReceived(self, data): - log.debug('Recieved data: %s', data) - if self.receiving_blob: - log.debug('Writing data to blob') - self.blob_write(data) - else: - log.debug('Not yet recieving blob, data needs further processing') + if self.receiving_blob is False: self.request_buff += data msg, extra_data = self._get_valid_response(self.request_buff) - if msg: + if msg is not None: self.request_buff = '' d = self.handle_request(msg) d.addCallbacks(self.send_response, self.handle_error) - if self.receiving_blob and extra_data: - log.debug('Writing extra data to blog') + if self.receiving_blob is True and len(extra_data) != 0: self.blob_write(extra_data) + else: + self.blob_write(data) def _get_valid_response(self, response_msg): extra_data = None @@ -74,7 +70,6 @@ class ReflectorServer(Protocol): return self.handle_normal_request(request_dict) def handle_handshake(self, request_dict): - log.debug('Handling handshake') if 'version' not in request_dict: raise ValueError("Client should send version") self.peer_version = int(request_dict['version']) @@ -108,7 +103,6 @@ class ReflectorServer(Protocol): raise ValueError("Expected a blob hash and a blob size") if not is_valid_blobhash(request_dict['blob_hash']): raise ValueError("Got a bad blob hash: {}".format(request_dict['blob_hash'])) - log.debug('Recieved info for blob: %s', request_dict['blob_hash']) d = self.blob_manager.get_blob( request_dict['blob_hash'], True, @@ -120,7 +114,6 @@ class ReflectorServer(Protocol): # important in it. to the deferred that fires when the blob is done, # add a callback which returns a nice response dict saying to keep # sending, and then return that deferred - log.debug('blob is already open') self.receiving_blob = True d = self.blob_finished_d d.addCallback(lambda _: self.close_blob()) @@ -140,8 +133,4 @@ class ReflectorServerFactory(ServerFactory): def __init__(self, peer_manager, blob_manager): self.peer_manager = peer_manager - self.blob_manager = blob_manager - - def buildProtocol(self, addr): - log.debug('Creating a protocol for %s', addr) - ServerFactory.buildProtocol(self, addr) + self.blob_manager = blob_manager \ No newline at end of file From 623fda3087ad31ea9d8c72f82c1d2c909688d038 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Thu, 11 Aug 2016 05:14:21 +0000 Subject: [PATCH 25/44] improve logging --- lbrynet/core/log_support.py | 10 ++++++++-- lbrynet/lbrynet_daemon/LBRYDaemonControl.py | 3 ++- lbrynet/reflector/server/server.py | 3 ++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/lbrynet/core/log_support.py b/lbrynet/core/log_support.py index 50fb35e83..f393a3210 100644 --- a/lbrynet/core/log_support.py +++ b/lbrynet/core/log_support.py @@ -58,15 +58,21 @@ def _log_decorator(fn): handler = fn(*args, **kwargs) if handler.name: remove_handlers(log, handler.name) + handler.setLevel(level) log.addHandler(handler) - log.setLevel(level) + if log.level > level: + log.setLevel(level) return helper -def disable_noisy_loggers(): +def disable_third_party_loggers(): logging.getLogger('requests').setLevel(logging.WARNING) +def disable_noisy_loggers(): + logging.getLogger('lbrynet.dht').setLevel(logging.INFO) + + @_log_decorator def configure_console(**kwargs): """Convenience function to configure a logger that outputs to stdout""" diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py index 76cbe5152..ce4b0f54a 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py @@ -73,11 +73,12 @@ def start(): parser.set_defaults(branch=False, launchui=True, logtoconsole=False, quiet=False) args = parser.parse_args() - log_support.disable_noisy_loggers() log_support.configure_file_handler(lbrynet_log) log_support.configure_loggly_handler() if args.logtoconsole: log_support.configure_console(level='DEBUG') + log_support.disable_third_party_loggers() + log_support.disable_noisy_loggers() try: JSONRPCProxy.from_url(API_CONNECTION_STRING).is_running() diff --git a/lbrynet/reflector/server/server.py b/lbrynet/reflector/server/server.py index d2569b6aa..fc015cbf6 100644 --- a/lbrynet/reflector/server/server.py +++ b/lbrynet/reflector/server/server.py @@ -14,6 +14,7 @@ class ReflectorServer(Protocol): def connectionMade(self): peer_info = self.transport.getPeer() + log.debug('Connection made to %s', peer_info) self.peer = self.factory.peer_manager.get_peer(peer_info.host, peer_info.port) self.blob_manager = self.factory.blob_manager self.received_handshake = False @@ -45,7 +46,7 @@ class ReflectorServer(Protocol): extra_data = None response = None curr_pos = 0 - while 1: + while True: next_close_paren = response_msg.find('}', curr_pos) if next_close_paren != -1: curr_pos = next_close_paren + 1 From c7430f4ae9de7db916caac35dded71c31be0edec Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Thu, 11 Aug 2016 05:25:45 +0000 Subject: [PATCH 26/44] better more logging on the server side --- lbrynet/reflector/server/server.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/lbrynet/reflector/server/server.py b/lbrynet/reflector/server/server.py index fc015cbf6..c8fb36fa8 100644 --- a/lbrynet/reflector/server/server.py +++ b/lbrynet/reflector/server/server.py @@ -30,17 +30,20 @@ class ReflectorServer(Protocol): pass def dataReceived(self, data): - if self.receiving_blob is False: + if self.receiving_blob: + log.debug('Writing data to blob') + self.blob_write(data) + else: + log.debug('Not yet recieving blob, data needs further processing') self.request_buff += data msg, extra_data = self._get_valid_response(self.request_buff) if msg is not None: self.request_buff = '' d = self.handle_request(msg) d.addCallbacks(self.send_response, self.handle_error) - if self.receiving_blob is True and len(extra_data) != 0: + if self.receiving_blob and extra_data: + log.debug('Writing extra data to blog') self.blob_write(extra_data) - else: - self.blob_write(data) def _get_valid_response(self, response_msg): extra_data = None @@ -71,6 +74,7 @@ class ReflectorServer(Protocol): return self.handle_normal_request(request_dict) def handle_handshake(self, request_dict): + log.debug('Handling handshake') if 'version' not in request_dict: raise ValueError("Client should send version") self.peer_version = int(request_dict['version']) @@ -104,6 +108,7 @@ class ReflectorServer(Protocol): raise ValueError("Expected a blob hash and a blob size") if not is_valid_blobhash(request_dict['blob_hash']): raise ValueError("Got a bad blob hash: {}".format(request_dict['blob_hash'])) + log.debug('Recieved info for blob: %s', request_dict['blob_hash']) d = self.blob_manager.get_blob( request_dict['blob_hash'], True, @@ -115,6 +120,7 @@ class ReflectorServer(Protocol): # important in it. to the deferred that fires when the blob is done, # add a callback which returns a nice response dict saying to keep # sending, and then return that deferred + log.debug('blob is already open') self.receiving_blob = True d = self.blob_finished_d d.addCallback(lambda _: self.close_blob()) @@ -134,4 +140,8 @@ class ReflectorServerFactory(ServerFactory): def __init__(self, peer_manager, blob_manager): self.peer_manager = peer_manager - self.blob_manager = blob_manager \ No newline at end of file + self.blob_manager = blob_manager + + def buildProtocol(self, addr): + log.debug('Creating a protocol for %s', addr) + return ServerFactory.buildProtocol(self, addr) From f4ef92e653b88dcac6b3c6710d9eab4c786104f2 Mon Sep 17 00:00:00 2001 From: Job Evers-Meltzer Date: Thu, 11 Aug 2016 05:37:45 +0000 Subject: [PATCH 27/44] fixup logging some more --- lbrynet/__init__.py | 6 ------ lbrynet/core/log_support.py | 4 ++++ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/lbrynet/__init__.py b/lbrynet/__init__.py index 6b754d3a2..9a7032852 100644 --- a/lbrynet/__init__.py +++ b/lbrynet/__init__.py @@ -1,8 +1,2 @@ -import logging - -log = logging.getLogger(__name__) -logging.getLogger(__name__).addHandler(logging.NullHandler()) -log.setLevel(logging.INFO) - __version__ = "0.3.17" version = tuple(__version__.split('.')) \ No newline at end of file diff --git a/lbrynet/core/log_support.py b/lbrynet/core/log_support.py index f393a3210..e203c9c2a 100644 --- a/lbrynet/core/log_support.py +++ b/lbrynet/core/log_support.py @@ -55,6 +55,10 @@ def _log_decorator(fn): def helper(*args, **kwargs): log = kwargs.pop('log', logging.getLogger()) level = kwargs.pop('level', logging.INFO) + if not isinstance(level, int): + # despite the name, getLevelName returns + # the numeric level when passed a text level + level = logging.getLevelName(level) handler = fn(*args, **kwargs) if handler.name: remove_handlers(log, handler.name) From b8d23d0965a89db47eced6095a37da082004672e Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 11 Aug 2016 12:36:13 -0400 Subject: [PATCH 28/44] add announce_all_blobs_to_dht --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index c39e16914..a3c6046b5 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -2385,6 +2385,20 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda r: self._render_response(r, OK_CODE)) return d + def jsonrpc_announce_all_blobs_to_dht(self): + """ + Announce all blobs to the dht + + Args: + None + Returns: + + """ + + d = self.session.blob_manager.immediate_announce_all_blobs() + d.addCallback(lambda _: self._render_response("Announced", OK_CODE)) + return d + def get_lbrynet_version_from_github(): """Return the latest released version from github.""" From 2d5f8aed24dd3f429b346805fad059df9f84c22b Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 11 Aug 2016 12:38:10 -0400 Subject: [PATCH 29/44] whitespace --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index a3c6046b5..7df728521 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -2398,7 +2398,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): d = self.session.blob_manager.immediate_announce_all_blobs() d.addCallback(lambda _: self._render_response("Announced", OK_CODE)) return d - + def get_lbrynet_version_from_github(): """Return the latest released version from github.""" From ece218785ca8892b7d415db92649ba0006ee67c0 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 17 Aug 2016 20:20:03 -0400 Subject: [PATCH 30/44] reduce logging to sub fire hazard levels --- lbrynet/core/log_support.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lbrynet/core/log_support.py b/lbrynet/core/log_support.py index cc6bba682..54a5a03a1 100644 --- a/lbrynet/core/log_support.py +++ b/lbrynet/core/log_support.py @@ -77,6 +77,9 @@ def disable_third_party_loggers(): def disable_noisy_loggers(): logging.getLogger('lbrynet.dht').setLevel(logging.INFO) + logging.getLogger('lbrynet.core.client.ConnectionManager').setLevel(logging.INFO) + logging.getLogger('lbrynet.core.client.BlobRequester').setLevel(logging.INFO) + logging.getLogger('lbrynet.core.client.ClientProtocol').setLevel(logging.INFO) @_log_decorator From edbda1696e3f97d96cf7feb9a396925f8dabe91e Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 17 Aug 2016 20:24:48 -0400 Subject: [PATCH 31/44] logging fire hazard --- lbrynet/core/log_support.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lbrynet/core/log_support.py b/lbrynet/core/log_support.py index cc6bba682..54a5a03a1 100644 --- a/lbrynet/core/log_support.py +++ b/lbrynet/core/log_support.py @@ -77,6 +77,9 @@ def disable_third_party_loggers(): def disable_noisy_loggers(): logging.getLogger('lbrynet.dht').setLevel(logging.INFO) + logging.getLogger('lbrynet.core.client.ConnectionManager').setLevel(logging.INFO) + logging.getLogger('lbrynet.core.client.BlobRequester').setLevel(logging.INFO) + logging.getLogger('lbrynet.core.client.ClientProtocol').setLevel(logging.INFO) @_log_decorator From a32b6ebf82a9510b84d0dd1353f578e547c6c1f2 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 17 Aug 2016 21:33:41 -0400 Subject: [PATCH 32/44] add reflect jsonrpc command --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 39 ++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index fc4c09c17..225881d50 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -1338,6 +1338,29 @@ class LBRYDaemon(jsonrpc.JSONRPC): d = defer.DeferredList([self._get_lbry_file('sd_hash', l.sd_hash) for l in self.lbry_file_manager.lbry_files]) return d + def _reflect(self, lbry_file): + if not lbry_file: + return defer.fail(Exception("no lbry file given to reflect")) + + sd_hash = lbry_file.sd_hash + stream_hash = lbry_file.stream_hash + + if sd_hash is None or stream_hash is None: + return defer.fail(Exception("unpopulated lbry file fields")) + + reflector_server = random.choice(REFLECTOR_SERVERS) + reflector_address, reflector_port = reflector_server[0], reflector_server[1] + log.info("Start reflector client") + factory = reflector.ClientFactory( + self.session.blob_manager, + self.lbry_file_manager.stream_info_manager, + stream_hash + ) + d = reactor.resolve(reflector_address) + d.addCallback(lambda ip: reactor.connectTCP(ip, reflector_port, factory)) + d.addCallback(lambda _: factory.finished_deferred) + return d + def _log_to_slack(self, msg): URL = "https://hooks.slack.com/services/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" msg = platform.platform() + ": " + base58.b58encode(self.lbryid)[:20] + ", " + msg @@ -2422,6 +2445,22 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallback(lambda _: self._render_response("Announced", OK_CODE)) return d + def jsonrpc_reflect(self, p): + """ + Reflect a stream + + Args: + sd_hash + Returns: + True or traceback + """ + + sd_hash = p['sd_hash'] + d = self._get_lbry_file('sd_hash', sd_hash, return_json=False) + d.addCallback(self._reflect) + d.addCallbacks(lambda _: self._render_response(True, OK_CODE), lambda err: self._render_response(err.getTraceback(), OK_CODE)) + return d + def get_lbrynet_version_from_github(): """Return the latest released version from github.""" From 702698d8aa4eec064a24a55bf86c374ac2d438c4 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 18 Aug 2016 05:25:23 -0400 Subject: [PATCH 33/44] stop heartbeat looping call on shutdown --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 225881d50..80e603d97 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -833,6 +833,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): self.lbry_ui_manager.update_checker.stop() if self.pending_claim_checker.running: self.pending_claim_checker.stop() + if self.send_heartbeat.running: + self.send_heartbeat.stop() self._clean_up_temp_files() From adc2eab6dac939b60938a9dc4e69421e22e80e11 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 18 Aug 2016 05:36:01 -0400 Subject: [PATCH 34/44] log request dict --- lbrynet/core/client/ClientProtocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/core/client/ClientProtocol.py b/lbrynet/core/client/ClientProtocol.py index aa4381e3d..31aa86b74 100644 --- a/lbrynet/core/client/ClientProtocol.py +++ b/lbrynet/core/client/ClientProtocol.py @@ -73,7 +73,7 @@ class ClientProtocol(Protocol): return defer.fail(failure.Failure(ValueError("There is already a request for that response active"))) self._next_request.update(request.request_dict) d = defer.Deferred() - log.debug("Adding a request. Request: %s", str(request)) + log.debug("Adding a request. Request: %s", str(request.request_dict)) self._response_deferreds[request.response_identifier] = d return d From f5508fcdf1b664bb5bdba4e9b2c14e70d5d27613 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 18 Aug 2016 05:36:17 -0400 Subject: [PATCH 35/44] typo --- lbrynet/reflector/server/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/reflector/server/server.py b/lbrynet/reflector/server/server.py index c8fb36fa8..6d309cafa 100644 --- a/lbrynet/reflector/server/server.py +++ b/lbrynet/reflector/server/server.py @@ -42,7 +42,7 @@ class ReflectorServer(Protocol): d = self.handle_request(msg) d.addCallbacks(self.send_response, self.handle_error) if self.receiving_blob and extra_data: - log.debug('Writing extra data to blog') + log.debug('Writing extra data to blob') self.blob_write(extra_data) def _get_valid_response(self, response_msg): From c046cd02d5a29f128946c72b3c3d92707ff16aa7 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 18 Aug 2016 05:58:13 -0400 Subject: [PATCH 36/44] call blob_completed after receiving blob --- lbrynet/core/server/BlobAvailabilityHandler.py | 1 + lbrynet/reflector/server/server.py | 1 + 2 files changed, 2 insertions(+) diff --git a/lbrynet/core/server/BlobAvailabilityHandler.py b/lbrynet/core/server/BlobAvailabilityHandler.py index 5203f72d2..a5d550bdf 100644 --- a/lbrynet/core/server/BlobAvailabilityHandler.py +++ b/lbrynet/core/server/BlobAvailabilityHandler.py @@ -44,6 +44,7 @@ class BlobAvailabilityHandler(object): d = self._get_available_blobs(queries[self.query_identifiers[0]]) def set_field(available_blobs): + log.debug("available blobs: %s", str(available_blobs)) return {'available_blobs': available_blobs} d.addCallback(set_field) diff --git a/lbrynet/reflector/server/server.py b/lbrynet/reflector/server/server.py index 6d309cafa..7ea759f97 100644 --- a/lbrynet/reflector/server/server.py +++ b/lbrynet/reflector/server/server.py @@ -89,6 +89,7 @@ class ReflectorServer(Protocol): else: self.incoming_blob = blob self.blob_finished_d, self.blob_write, self.cancel_write = blob.open_for_writing(self.peer) + self.blob_finished_d.addCallback(lambda _: self.blob_manager.blob_completed(blob)) return {'send_blob': True} def close_blob(self): From 8e7b8af180dc3153df3e5fa4c21e1fb5c1e8dd4b Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 18 Aug 2016 06:23:27 -0400 Subject: [PATCH 37/44] call blob_completed when is_validated is true --- lbrynet/reflector/server/server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lbrynet/reflector/server/server.py b/lbrynet/reflector/server/server.py index 7ea759f97..1728a8ed1 100644 --- a/lbrynet/reflector/server/server.py +++ b/lbrynet/reflector/server/server.py @@ -85,11 +85,12 @@ class ReflectorServer(Protocol): def determine_blob_needed(self, blob): if blob.is_validated(): - return {'send_blob': False} + d = self.blob_manager.blob_completed(blob) + d.addCallback(lambda _: {'send_blob': False}) + return d else: self.incoming_blob = blob self.blob_finished_d, self.blob_write, self.cancel_write = blob.open_for_writing(self.peer) - self.blob_finished_d.addCallback(lambda _: self.blob_manager.blob_completed(blob)) return {'send_blob': True} def close_blob(self): From 929db152143c95c203b5226b70d7c12e173e9db6 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 18 Aug 2016 22:15:49 -0400 Subject: [PATCH 38/44] fix get_history in lbrycrdwallet --- lbrynet/core/LBRYWallet.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/lbrynet/core/LBRYWallet.py b/lbrynet/core/LBRYWallet.py index 0bc584aca..7d275278b 100644 --- a/lbrynet/core/LBRYWallet.py +++ b/lbrynet/core/LBRYWallet.py @@ -489,6 +489,10 @@ class LBRYWallet(object): d.addCallback(self._get_decoded_tx) return d + def get_history(self): + d = self._get_history() + return d + def get_name_and_validity_for_sd_hash(self, sd_hash): d = self._get_claim_metadata_for_sd_hash(sd_hash) d.addCallback(lambda name_txid: self._get_status_of_claim(name_txid[1], name_txid[0], sd_hash) if name_txid is not None else None) @@ -688,6 +692,9 @@ class LBRYWallet(object): def _get_balance_for_address(self, address): return defer.fail(NotImplementedError()) + def _get_history(self): + return defer.fail(NotImplementedError()) + def _start(self): pass @@ -823,6 +830,9 @@ class LBRYcrdWallet(LBRYWallet): def _get_value_for_name(self, name): return threads.deferToThread(self._get_value_for_name_rpc, name) + def _get_history(self): + return threads.deferToThread(self._list_transactions_rpc) + def _get_rpc_conn(self): return AuthServiceProxy(self.rpc_conn_string) @@ -1007,6 +1017,11 @@ class LBRYcrdWallet(LBRYWallet): rpc_conn = self._get_rpc_conn() return rpc_conn.getbestblockhash() + @_catch_connection_error + def _list_transactions_rpc(self): + rpc_conn = self._get_rpc_conn() + return rpc_conn.listtransactions() + @_catch_connection_error def _stop_rpc(self): # check if our lbrycrdd is actually running, or if we connected to one that was already @@ -1294,7 +1309,7 @@ class LBRYumWallet(LBRYWallet): func = getattr(self.cmd_runner, cmd.name) return threads.deferToThread(func) - def get_history(self): + def _get_history(self): cmd = known_commands['history'] func = getattr(self.cmd_runner, cmd.name) return threads.deferToThread(func) From 7a5489401516aa9f0dc1aa7fe46b5818817f91e5 Mon Sep 17 00:00:00 2001 From: Jack Date: Fri, 19 Aug 2016 02:41:23 -0400 Subject: [PATCH 39/44] debugging stuff --- lbrynet/core/client/ConnectionManager.py | 2 +- lbrynet/core/log_support.py | 1 + lbrynet/lbrynet_daemon/LBRYDaemon.py | 9 +++++++++ lbrynet/reflector/server/server.py | 4 +--- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/lbrynet/core/client/ConnectionManager.py b/lbrynet/core/client/ConnectionManager.py index 683935036..f82f19a20 100644 --- a/lbrynet/core/client/ConnectionManager.py +++ b/lbrynet/core/client/ConnectionManager.py @@ -172,7 +172,7 @@ class ConnectionManager(object): def pick_best_peer(peers): # TODO: Eventually rank them based on past performance/reputation. For now # TODO: just pick the first to which we don't have an open connection - log.debug("Got a list of peers to choose from: %s", str(peers)) + log.debug("Got a list of peers to choose from: %s", str(["%s:%i" % (p.host, p.port) for p in peers])) if peers is None: return None for peer in peers: diff --git a/lbrynet/core/log_support.py b/lbrynet/core/log_support.py index 54a5a03a1..c2f652e74 100644 --- a/lbrynet/core/log_support.py +++ b/lbrynet/core/log_support.py @@ -80,6 +80,7 @@ def disable_noisy_loggers(): logging.getLogger('lbrynet.core.client.ConnectionManager').setLevel(logging.INFO) logging.getLogger('lbrynet.core.client.BlobRequester').setLevel(logging.INFO) logging.getLogger('lbrynet.core.client.ClientProtocol').setLevel(logging.INFO) + logging.getLogger('lbrynet.analytics.api').setLevel(logging.INFO) @_log_decorator diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index f3a820399..ac888b757 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -2463,6 +2463,15 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addCallbacks(lambda _: self._render_response(True, OK_CODE), lambda err: self._render_response(err.getTraceback(), OK_CODE)) return d + def jsonrpc_get_blobs(self): + """ + return all blobs + """ + + d = defer.succeed(self.session.blob_manager.blobs) + d.addCallback(lambda r: self._render_response(r, OK_CODE)) + return d + def get_lbrynet_version_from_github(): """Return the latest released version from github.""" diff --git a/lbrynet/reflector/server/server.py b/lbrynet/reflector/server/server.py index 1728a8ed1..6d309cafa 100644 --- a/lbrynet/reflector/server/server.py +++ b/lbrynet/reflector/server/server.py @@ -85,9 +85,7 @@ class ReflectorServer(Protocol): def determine_blob_needed(self, blob): if blob.is_validated(): - d = self.blob_manager.blob_completed(blob) - d.addCallback(lambda _: {'send_blob': False}) - return d + return {'send_blob': False} else: self.incoming_blob = blob self.blob_finished_d, self.blob_write, self.cancel_write = blob.open_for_writing(self.peer) From 31b9d22649f4ed50232eaebe284347034e3ec1a0 Mon Sep 17 00:00:00 2001 From: Sonata Green Date: Fri, 19 Aug 2016 21:12:02 -0500 Subject: [PATCH 40/44] spelling fix --- lbrynet/lbrynet_daemon/LBRYDaemonCLI.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonCLI.py b/lbrynet/lbrynet_daemon/LBRYDaemonCLI.py index 116fd32cb..1d9733b19 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonCLI.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonCLI.py @@ -4,7 +4,7 @@ import json from lbrynet.conf import API_CONNECTION_STRING from jsonrpc.proxy import JSONRPCProxy -help_msg = "Useage: lbrynet-cli method json-args\n" \ +help_msg = "Usage: lbrynet-cli method json-args\n" \ + "Examples: " \ + "lbrynet-cli resolve_name '{\"name\": \"what\"}'\n" \ + "lbrynet-cli get_balance\n" \ From 5022ed2acc28df43699d647472345fa1c3240866 Mon Sep 17 00:00:00 2001 From: Jack Date: Sat, 20 Aug 2016 23:47:12 -0400 Subject: [PATCH 41/44] blob_completed before moving to next one otherwise blob files download to the blobfiles directory, but they are never recorded in blobs.db --- lbrynet/reflector/server/server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lbrynet/reflector/server/server.py b/lbrynet/reflector/server/server.py index 6d309cafa..dea539814 100644 --- a/lbrynet/reflector/server/server.py +++ b/lbrynet/reflector/server/server.py @@ -89,6 +89,7 @@ class ReflectorServer(Protocol): else: self.incoming_blob = blob self.blob_finished_d, self.blob_write, self.cancel_write = blob.open_for_writing(self.peer) + self.blob_finished_d.addCallback(lambda _ :self.blob_manager.blob_completed(blob)) return {'send_blob': True} def close_blob(self): From 3d92413372ab1de6ea751573caf3f87e5324c704 Mon Sep 17 00:00:00 2001 From: Jack Date: Sat, 20 Aug 2016 23:47:41 -0400 Subject: [PATCH 42/44] debug logging --- lbrynet/reflector/server/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lbrynet/reflector/server/server.py b/lbrynet/reflector/server/server.py index dea539814..38127dce2 100644 --- a/lbrynet/reflector/server/server.py +++ b/lbrynet/reflector/server/server.py @@ -27,11 +27,11 @@ class ReflectorServer(Protocol): self.request_buff = "" def connectionLost(self, reason=failure.Failure(error.ConnectionDone())): - pass + log.info("Reflector upload from %s finished" % self.peer.host) def dataReceived(self, data): if self.receiving_blob: - log.debug('Writing data to blob') + # log.debug('Writing data to blob') self.blob_write(data) else: log.debug('Not yet recieving blob, data needs further processing') From 15cfa4564662a7953be6bc5c81c91b0aeaa34859 Mon Sep 17 00:00:00 2001 From: Jack Date: Sun, 21 Aug 2016 00:58:25 -0400 Subject: [PATCH 43/44] remove duplicate function --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 24 +++++------------------- lbrynet/lbrynet_daemon/LBRYPublisher.py | 2 +- 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index ac888b757..8b4cc54fb 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -1344,11 +1344,12 @@ class LBRYDaemon(jsonrpc.JSONRPC): if not lbry_file: return defer.fail(Exception("no lbry file given to reflect")) - sd_hash = lbry_file.sd_hash stream_hash = lbry_file.stream_hash - if sd_hash is None or stream_hash is None: - return defer.fail(Exception("unpopulated lbry file fields")) + if stream_hash is None: + return defer.fail(Exception("no stream hash")) + + log.info("Reflecting stream: %s" % stream_hash) reflector_server = random.choice(REFLECTOR_SERVERS) reflector_address, reflector_port = reflector_server[0], reflector_server[1] @@ -1938,26 +1939,11 @@ class LBRYDaemon(jsonrpc.JSONRPC): return m def _reflect_if_possible(sd_hash, txid): - log.info("Trying to start reflector") d = self._get_lbry_file('sd_hash', sd_hash, return_json=False) - d.addCallback(lambda r: False if not r else _start_reflector(r.stream_hash)) + d.addCallback(self._reflect) d.addCallback(lambda _: txid) return d - def _start_reflector(stream_hash): - reflector_server = random.choice(REFLECTOR_SERVERS) - reflector_address, reflector_port = reflector_server[0], reflector_server[1] - log.info("Start reflector client") - factory = reflector.ClientFactory( - self.session.blob_manager, - self.lbry_file_manager.stream_info_manager, - stream_hash - ) - d = reactor.resolve(reflector_address) - d.addCallback(lambda ip: reactor.connectTCP(ip, reflector_port, factory)) - d.addCallback(lambda _: factory.finished_deferred) - return d - name = p['name'] log.info("Publish: ") diff --git a/lbrynet/lbrynet_daemon/LBRYPublisher.py b/lbrynet/lbrynet_daemon/LBRYPublisher.py index 5efbe66a9..04b9bdaab 100644 --- a/lbrynet/lbrynet_daemon/LBRYPublisher.py +++ b/lbrynet/lbrynet_daemon/LBRYPublisher.py @@ -72,7 +72,7 @@ class Publisher(object): def start_reflector(self): reflector_server = random.choice(REFLECTOR_SERVERS) reflector_address, reflector_port = reflector_server[0], reflector_server[1] - log.info("Start reflector client") + log.info("Reflecting new publication") factory = reflector.ClientFactory( self.session.blob_manager, self.lbry_file_manager.stream_info_manager, From 9c82689f65228a3f6cc0ffc4770f6704dfd60f56 Mon Sep 17 00:00:00 2001 From: Jack Date: Sun, 21 Aug 2016 04:04:11 -0400 Subject: [PATCH 44/44] Bump version: 0.3.19 -> 0.3.20 --- .bumpversion.cfg | 2 +- lbrynet/__init__.py | 2 +- packaging/ubuntu/lbry.desktop | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 78907de6a..686ee8c48 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.19 +current_version = 0.3.20 commit = True tag = True message = Bump version: {current_version} -> {new_version} diff --git a/lbrynet/__init__.py b/lbrynet/__init__.py index d83b7602f..3a92417c3 100644 --- a/lbrynet/__init__.py +++ b/lbrynet/__init__.py @@ -1,2 +1,2 @@ -__version__ = "0.3.19" +__version__ = "0.3.20" version = tuple(__version__.split('.')) \ No newline at end of file diff --git a/packaging/ubuntu/lbry.desktop b/packaging/ubuntu/lbry.desktop index 061299934..1af5a588c 100644 --- a/packaging/ubuntu/lbry.desktop +++ b/packaging/ubuntu/lbry.desktop @@ -1,5 +1,5 @@ [Desktop Entry] -Version=0.3.19 +Version=0.3.20 Name=LBRY Comment=The world's first user-owned content marketplace Icon=lbry