From ecbe4113cea43a3fbc32f529478594ec42f76cf7 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Mon, 23 Oct 2017 13:09:46 -0400 Subject: [PATCH 01/52] move Distance to own file --- lbrynet/dht/distance.py | 22 +++ lbrynet/dht/node.py | 27 +--- lbrynet/tests/dht/test_routing_table.py | 191 ++++++++++++++++++++++++ 3 files changed, 215 insertions(+), 25 deletions(-) create mode 100644 lbrynet/dht/distance.py create mode 100644 lbrynet/tests/dht/test_routing_table.py diff --git a/lbrynet/dht/distance.py b/lbrynet/dht/distance.py new file mode 100644 index 000000000..cda548db2 --- /dev/null +++ b/lbrynet/dht/distance.py @@ -0,0 +1,22 @@ +class Distance(object): + """Calculate the XOR result between two string variables. + + Frequently we re-use one of the points so as an optimization + we pre-calculate the long value of that point. + """ + + def __init__(self, key): + self.key = key + self.val_key_one = long(key.encode('hex'), 16) + + def __call__(self, key_two): + val_key_two = long(key_two.encode('hex'), 16) + return self.val_key_one ^ val_key_two + + def is_closer(self, a, b): + """Returns true is `a` is closer to `key` than `b` is""" + return self(a) < self(b) + + def to_contact(self, contact): + """A convenience function for calculating the distance to a contact""" + return self(contact.id) diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index a13a0227f..cf0cc8e37 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -21,8 +21,9 @@ import protocol from contact import Contact from hashwatcher import HashWatcher -import logging +from distance import Distance +import logging from lbrynet.core.utils import generate_id log = logging.getLogger(__name__) @@ -867,30 +868,6 @@ class _IterativeFindHelper(object): ) -class Distance(object): - """Calculate the XOR result between two string variables. - - Frequently we re-use one of the points so as an optimization - we pre-calculate the long value of that point. - """ - - def __init__(self, key): - self.key = key - self.val_key_one = long(key.encode('hex'), 16) - - def __call__(self, key_two): - val_key_two = long(key_two.encode('hex'), 16) - return self.val_key_one ^ val_key_two - - def is_closer(self, a, b): - """Returns true is `a` is closer to `key` than `b` is""" - return self(a) < self(b) - - def to_contact(self, contact): - """A convenience function for calculating the distance to a contact""" - return self(contact.id) - - class ExpensiveSort(object): """Sort a list in place. diff --git a/lbrynet/tests/dht/test_routing_table.py b/lbrynet/tests/dht/test_routing_table.py new file mode 100644 index 000000000..c93bbc19b --- /dev/null +++ b/lbrynet/tests/dht/test_routing_table.py @@ -0,0 +1,191 @@ + +#!/usr/bin/env python +# +# This library is free software, distributed under the terms of +# the GNU Lesser General Public License Version 3, or any later version. +# See the COPYING file included in this archive + +import hashlib +import unittest + +import lbrynet.dht.constants +import lbrynet.dht.routingtable +import lbrynet.dht.contact +import lbrynet.dht.node +import lbrynet.dht.distance + + +class FakeRPCProtocol(object): + """ Fake RPC protocol; allows lbrynet.dht.contact.Contact objects to "send" RPCs """ + def sendRPC(self, *args, **kwargs): + return FakeDeferred() + + +class FakeDeferred(object): + """ Fake Twisted Deferred object; allows the routing table to add callbacks that do nothing """ + def addCallback(self, *args, **kwargs): + return + + def addErrback(self, *args, **kwargs): + return + + def addCallbacks(self, *args, **kwargs): + return + + +class TreeRoutingTableTest(unittest.TestCase): + """ Test case for the RoutingTable class """ + def setUp(self): + h = hashlib.sha384() + h.update('node1') + self.nodeID = h.digest() + self.protocol = FakeRPCProtocol() + self.routingTable = lbrynet.dht.routingtable.TreeRoutingTable(self.nodeID) + + def testDistance(self): + """ Test to see if distance method returns correct result""" + + # testList holds a couple 3-tuple (variable1, variable2, result) + basicTestList = [('123456789', '123456789', 0L), ('12345', '98765', 34527773184L)] + + for test in basicTestList: + result = lbrynet.dht.distance.Distance(test[0])(test[1]) + self.failIf(result != test[2], 'Result of _distance() should be %s but %s returned' % + (test[2], result)) + + baseIp = '146.64.19.111' + ipTestList = ['146.64.29.222', '192.68.19.333'] + + distanceOne = lbrynet.dht.distance.Distance(baseIp)(ipTestList[0]) + distanceTwo = lbrynet.dht.distance.Distance(baseIp)(ipTestList[1]) + + self.failIf(distanceOne > distanceTwo, '%s should be closer to the base ip %s than %s' % + (ipTestList[0], baseIp, ipTestList[1])) + + def testAddContact(self): + """ Tests if a contact can be added and retrieved correctly """ + # Create the contact + h = hashlib.sha384() + h.update('node2') + contactID = h.digest() + contact = lbrynet.dht.contact.Contact(contactID, '127.0.0.1', 91824, self.protocol) + # Now add it... + self.routingTable.addContact(contact) + # ...and request the closest nodes to it (will retrieve it) + closestNodes = self.routingTable.findCloseNodes(contactID, lbrynet.dht.constants.k) + self.failUnlessEqual(len(closestNodes), 1, 'Wrong amount of contacts returned; expected 1,' + ' got %d' % len(closestNodes)) + self.failUnless(contact in closestNodes, 'Added contact not found by issueing ' + '_findCloseNodes()') + + def testGetContact(self): + """ Tests if a specific existing contact can be retrieved correctly """ + h = hashlib.sha384() + h.update('node2') + contactID = h.digest() + contact = lbrynet.dht.contact.Contact(contactID, '127.0.0.1', 91824, self.protocol) + # Now add it... + self.routingTable.addContact(contact) + # ...and get it again + sameContact = self.routingTable.getContact(contactID) + self.failUnlessEqual(contact, sameContact, 'getContact() should return the same contact') + + def testAddParentNodeAsContact(self): + """ + Tests the routing table's behaviour when attempting to add its parent node as a contact + """ + + # Create a contact with the same ID as the local node's ID + contact = lbrynet.dht.contact.Contact(self.nodeID, '127.0.0.1', 91824, self.protocol) + # Now try to add it + self.routingTable.addContact(contact) + # ...and request the closest nodes to it using FIND_NODE + closestNodes = self.routingTable.findCloseNodes(self.nodeID, lbrynet.dht.constants.k) + self.failIf(contact in closestNodes, 'Node added itself as a contact') + + def testRemoveContact(self): + """ Tests contact removal """ + # Create the contact + h = hashlib.sha384() + h.update('node2') + contactID = h.digest() + contact = lbrynet.dht.contact.Contact(contactID, '127.0.0.1', 91824, self.protocol) + # Now add it... + self.routingTable.addContact(contact) + # Verify addition + self.failUnlessEqual(len(self.routingTable._buckets[0]), 1, 'Contact not added properly') + # Now remove it + self.routingTable.removeContact(contact.id) + self.failUnlessEqual(len(self.routingTable._buckets[0]), 0, 'Contact not removed properly') + + def testSplitBucket(self): + """ Tests if the the routing table correctly dynamically splits k-buckets """ + self.failUnlessEqual(self.routingTable._buckets[0].rangeMax, 2**384, + 'Initial k-bucket range should be 0 <= range < 2**384') + # Add k contacts + for i in range(lbrynet.dht.constants.k): + h = hashlib.sha384() + h.update('remote node %d' % i) + nodeID = h.digest() + contact = lbrynet.dht.contact.Contact(nodeID, '127.0.0.1', 91824, self.protocol) + self.routingTable.addContact(contact) + self.failUnlessEqual(len(self.routingTable._buckets), 1, + 'Only k nodes have been added; the first k-bucket should now ' + 'be full, but should not yet be split') + # Now add 1 more contact + h = hashlib.sha384() + h.update('yet another remote node') + nodeID = h.digest() + contact = lbrynet.dht.contact.Contact(nodeID, '127.0.0.1', 91824, self.protocol) + self.routingTable.addContact(contact) + self.failUnlessEqual(len(self.routingTable._buckets), 2, + 'k+1 nodes have been added; the first k-bucket should have been ' + 'split into two new buckets') + self.failIfEqual(self.routingTable._buckets[0].rangeMax, 2**384, + 'K-bucket was split, but its range was not properly adjusted') + self.failUnlessEqual(self.routingTable._buckets[1].rangeMax, 2**384, + 'K-bucket was split, but the second (new) bucket\'s ' + 'max range was not set properly') + self.failUnlessEqual(self.routingTable._buckets[0].rangeMax, + self.routingTable._buckets[1].rangeMin, + 'K-bucket was split, but the min/max ranges were ' + 'not divided properly') + + def testFullBucketNoSplit(self): + """ + Test that a bucket is not split if it full, but does not cover the range + containing the parent node's ID + """ + self.routingTable._parentNodeID = 49 * 'a' + # more than 384 bits; this will not be in the range of _any_ k-bucket + # Add k contacts + for i in range(lbrynet.dht.constants.k): + h = hashlib.sha384() + h.update('remote node %d' % i) + nodeID = h.digest() + contact = lbrynet.dht.contact.Contact(nodeID, '127.0.0.1', 91824, self.protocol) + self.routingTable.addContact(contact) + self.failUnlessEqual(len(self.routingTable._buckets), 1, 'Only k nodes have been added; ' + 'the first k-bucket should now be ' + 'full, and there should not be ' + 'more than 1 bucket') + self.failUnlessEqual(len(self.routingTable._buckets[0]._contacts), lbrynet.dht.constants.k, + 'Bucket should have k contacts; expected %d got %d' % + (lbrynet.dht.constants.k, + len(self.routingTable._buckets[0]._contacts))) + # Now add 1 more contact + h = hashlib.sha384() + h.update('yet another remote node') + nodeID = h.digest() + contact = lbrynet.dht.contact.Contact(nodeID, '127.0.0.1', 91824, self.protocol) + self.routingTable.addContact(contact) + self.failUnlessEqual(len(self.routingTable._buckets), 1, + 'There should not be more than 1 bucket, since the bucket ' + 'should not have been split (parent node ID not in range)') + self.failUnlessEqual(len(self.routingTable._buckets[0]._contacts), + lbrynet.dht.constants.k, 'Bucket should have k contacts; ' + 'expected %d got %d' % + (lbrynet.dht.constants.k, + len(self.routingTable._buckets[0]._contacts))) + self.failIf(contact in self.routingTable._buckets[0]._contacts, + 'New contact should have been discarded (since RPC is faked in this test)') From 67ef8be7b7eefcfb09769768a64163ce1a6d75ff Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Mon, 23 Oct 2017 13:13:06 -0400 Subject: [PATCH 02/52] convert node manage function to a looping call --- lbrynet/dht/node.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index cf0cc8e37..a4f2acf3c 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -11,8 +11,7 @@ import hashlib import operator import struct import time - -from twisted.internet import defer, error, reactor, threads, task +from twisted.internet import defer, error, reactor, task import constants import routingtable @@ -86,8 +85,8 @@ class Node(object): # network (add callbacks to this deferred if scheduling such # operations before the node has finished joining the network) self._joinDeferred = None - self.next_refresh_call = None self.change_token_lc = task.LoopingCall(self.change_token) + self.refresh_node_lc = task.LoopingCall(self._refreshNode) # Create k-buckets (for storing contacts) if routingTableClass is None: self._routingTable = routingtable.OptimizedTreeRoutingTable(self.node_id) @@ -123,10 +122,9 @@ class Node(object): self._listeningPort.stopListening() def stop(self): - # cancel callLaters: - if self.next_refresh_call is not None: - self.next_refresh_call.cancel() - self.next_refresh_call = None + # stop LoopingCalls: + if self.refresh_node_lc.running: + self.refresh_node_lc.stop() if self.change_token_lc.running: self.change_token_lc.stop() if self._listeningPort is not None: @@ -182,9 +180,11 @@ class Node(object): # Initiate the Kademlia joining sequence - perform a search for this node's own ID self._joinDeferred = self._iterativeFind(self.node_id, bootstrapContacts) - - result = yield self._joinDeferred - defer.returnValue(result) + # #TODO: Refresh all k-buckets further away than this node's closest neighbour + # Start refreshing k-buckets periodically, if necessary + self.hash_watcher.tick() + yield self._joinDeferred + self.refresh_node_lc.start(constants.checkRefreshInterval) @property def contacts(self): @@ -629,13 +629,14 @@ class Node(object): result = yield outerDf defer.returnValue(result) + @defer.inlineCallbacks def _refreshNode(self): """ Periodically called to perform k-bucket refreshes and data replication/republishing as necessary """ - df = self._refreshRoutingTable() - df.addCallback(self._removeExpiredPeers) - df.addCallback(self._scheduleNextNodeRefresh) + yield self._refreshRoutingTable() + self._dataStore.removeExpiredPeers() + defer.returnValue(None) def _refreshRoutingTable(self): nodeIDs = self._routingTable.getRefreshList(0, False) @@ -654,9 +655,6 @@ class Node(object): searchForNextNodeID() return outerDf - def _scheduleNextNodeRefresh(self, *args): - self.next_refresh_call = reactor.callLater(constants.checkRefreshInterval, - self._refreshNode) # args put here because _refreshRoutingTable does outerDF.callback(None) def _removeExpiredPeers(self, *args): From 446c3a88dc30244abc508fa4b092213a7555188e Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Mon, 23 Oct 2017 15:36:50 -0400 Subject: [PATCH 03/52] refactor DHTHashAnnouncer and iterativeAnnounceHaveBlob -use looping call for running manage function rather than a scheduled callLater -track announce speed -retry store requests that failed up to 3 times -return a dict of {blob_hash: [storing_node_id]} results from _announce_hashes _refreshRoutingTable inline cb refactor -add and use DeferredLockContextManager -don't trap errback from iterativeFindNode in iterativeAnnounceHaveBlob --- lbrynet/core/server/DHTHashAnnouncer.py | 146 +++++++++++++----- lbrynet/core/utils.py | 11 ++ lbrynet/dht/node.py | 104 +++++++------ .../unit/core/server/test_DHTHashAnnouncer.py | 26 +++- 4 files changed, 190 insertions(+), 97 deletions(-) diff --git a/lbrynet/core/server/DHTHashAnnouncer.py b/lbrynet/core/server/DHTHashAnnouncer.py index 8bef7b177..877b0538c 100644 --- a/lbrynet/core/server/DHTHashAnnouncer.py +++ b/lbrynet/core/server/DHTHashAnnouncer.py @@ -2,8 +2,9 @@ import binascii import collections import logging import time +import datetime -from twisted.internet import defer +from twisted.internet import defer, task from lbrynet.core import utils log = logging.getLogger(__name__) @@ -14,6 +15,8 @@ class DHTHashAnnouncer(object): CONCURRENT_ANNOUNCERS = 5 """This class announces to the DHT that this peer has certain blobs""" + STORE_RETRIES = 3 + def __init__(self, dht_node, peer_port): self.dht_node = dht_node self.peer_port = peer_port @@ -21,17 +24,42 @@ class DHTHashAnnouncer(object): self.next_manage_call = None self.hash_queue = collections.deque() self._concurrent_announcers = 0 + self._manage_call_lc = task.LoopingCall(self.manage_lc) + self._lock = utils.DeferredLockContextManager(defer.DeferredLock()) + self._last_checked = time.time(), self.CONCURRENT_ANNOUNCERS + self._retries = {} + self._total = None def run_manage_loop(self): + log.info("Starting hash announcer") + if not self._manage_call_lc.running: + self._manage_call_lc.start(self.ANNOUNCE_CHECK_INTERVAL) + + def manage_lc(self): + last_time, last_hashes = self._last_checked + hashes = len(self.hash_queue) + if hashes: + t, h = time.time() - last_time, last_hashes - hashes + blobs_per_second = float(h) / float(t) + if blobs_per_second > 0: + estimated_time_remaining = int(float(hashes) / blobs_per_second) + remaining = str(datetime.timedelta(seconds=estimated_time_remaining)) + else: + remaining = "unknown" + log.info("Announcing blobs: %i blobs left to announce, %i%s complete, " + "est time remaining: %s", hashes + self._concurrent_announcers, + 100 - int(100.0 * float(hashes + self._concurrent_announcers) / + float(self._total)), "%", remaining) + self._last_checked = t + last_time, hashes + else: + self._total = 0 if self.peer_port is not None: - self._announce_available_hashes() - self.next_manage_call = utils.call_later(self.ANNOUNCE_CHECK_INTERVAL, self.run_manage_loop) + return self._announce_available_hashes() def stop(self): log.info("Stopping DHT hash announcer.") - if self.next_manage_call is not None: - self.next_manage_call.cancel() - self.next_manage_call = None + if self._manage_call_lc.running: + self._manage_call_lc.stop() def add_supplier(self, supplier): self.suppliers.append(supplier) @@ -45,60 +73,101 @@ class DHTHashAnnouncer(object): def hash_queue_size(self): return len(self.hash_queue) + @defer.inlineCallbacks def _announce_available_hashes(self): log.debug('Announcing available hashes') - ds = [] for supplier in self.suppliers: - d = supplier.hashes_to_announce() - d.addCallback(self._announce_hashes) - ds.append(d) - dl = defer.DeferredList(ds) - return dl + hashes = yield supplier.hashes_to_announce() + yield self._announce_hashes(hashes) + @defer.inlineCallbacks def _announce_hashes(self, hashes, immediate=False): if not hashes: - return - log.debug('Announcing %s hashes', len(hashes)) + defer.returnValue(None) + if not self.dht_node.can_store: + log.warning("Client only DHT node cannot store, skipping announce") + defer.returnValue(None) + log.info('Announcing %s hashes', len(hashes)) # TODO: add a timeit decorator start = time.time() - ds = [] - for h in hashes: - announce_deferred = defer.Deferred() - ds.append(announce_deferred) - if immediate: - self.hash_queue.appendleft((h, announce_deferred)) - else: - self.hash_queue.append((h, announce_deferred)) + ds = [] + with self._lock: + for h in hashes: + announce_deferred = defer.Deferred() + if immediate: + self.hash_queue.appendleft((h, announce_deferred)) + else: + self.hash_queue.append((h, announce_deferred)) + if not self._total: + self._total = len(hashes) + log.debug('There are now %s hashes remaining to be announced', self.hash_queue_size()) - def announce(): + @defer.inlineCallbacks + def do_store(blob_hash, announce_d): + if announce_d.called: + defer.returnValue(announce_deferred.result) + try: + store_nodes = yield self.dht_node.announceHaveBlob(binascii.unhexlify(blob_hash)) + if not store_nodes: + retries = self._retries.get(blob_hash, 0) + retries += 1 + self._retries[blob_hash] = retries + if retries <= self.STORE_RETRIES: + log.debug("No nodes stored %s, retrying", blob_hash) + result = yield do_store(blob_hash, announce_d) + else: + log.warning("No nodes stored %s", blob_hash) + else: + result = store_nodes + if not announce_d.called: + announce_d.callback(result) + defer.returnValue(result) + except Exception as err: + if not announce_d.called: + announce_d.errback(err) + raise err + + @defer.inlineCallbacks + def announce(progress=None): + progress = progress or {} if len(self.hash_queue): - h, announce_deferred = self.hash_queue.popleft() - log.debug('Announcing blob %s to dht', h) - d = self.dht_node.announceHaveBlob(binascii.unhexlify(h)) - d.chainDeferred(announce_deferred) - d.addBoth(lambda _: utils.call_later(0, announce)) + with self._lock: + h, announce_deferred = self.hash_queue.popleft() + log.debug('Announcing blob %s to dht', h[:16]) + stored_to_nodes = yield do_store(h, announce_deferred) + progress[h] = stored_to_nodes + log.debug("Stored %s to %i peers (hashes announced by this announcer: %i)", + h.encode('hex')[:16], + len(stored_to_nodes), len(progress)) + + yield announce(progress) else: - self._concurrent_announcers -= 1 + with self._lock: + self._concurrent_announcers -= 1 + defer.returnValue(progress) for i in range(self._concurrent_announcers, self.CONCURRENT_ANNOUNCERS): self._concurrent_announcers += 1 - announce() - d = defer.DeferredList(ds) - d.addCallback(lambda _: log.debug('Took %s seconds to announce %s hashes', - time.time() - start, len(hashes))) - return d + ds.append(announce()) + announcer_results = yield defer.DeferredList(ds) + stored_to = {} + for _, announced_to in announcer_results: + stored_to.update(announced_to) + log.info('Took %s seconds to announce %s hashes', time.time() - start, len(hashes)) + defer.returnValue(stored_to) class DHTHashSupplier(object): # 1 hour is the min time hash will be reannounced - MIN_HASH_REANNOUNCE_TIME = 60*60 + MIN_HASH_REANNOUNCE_TIME = 60 * 60 # conservative assumption of the time it takes to announce # a single hash SINGLE_HASH_ANNOUNCE_DURATION = 5 """Classes derived from this class give hashes to a hash announcer""" + def __init__(self, announcer): if announcer is not None: announcer.add_supplier(self) @@ -107,7 +176,6 @@ class DHTHashSupplier(object): def hashes_to_announce(self): pass - def get_next_announce_time(self, num_hashes_to_announce=1): """ Hash reannounce time is set to current time + MIN_HASH_REANNOUNCE_TIME, @@ -121,9 +189,7 @@ class DHTHashSupplier(object): Returns: timestamp for next announce time """ - queue_size = self.hash_announcer.hash_queue_size()+num_hashes_to_announce + queue_size = self.hash_announcer.hash_queue_size() + num_hashes_to_announce reannounce = max(self.MIN_HASH_REANNOUNCE_TIME, - queue_size*self.SINGLE_HASH_ANNOUNCE_DURATION) + queue_size * self.SINGLE_HASH_ANNOUNCE_DURATION) return time.time() + reannounce - - diff --git a/lbrynet/core/utils.py b/lbrynet/core/utils.py index ae67c9885..ce0d433f2 100644 --- a/lbrynet/core/utils.py +++ b/lbrynet/core/utils.py @@ -148,6 +148,17 @@ def json_dumps_pretty(obj, **kwargs): return json.dumps(obj, sort_keys=True, indent=2, separators=(',', ': '), **kwargs) +class DeferredLockContextManager(object): + def __init__(self, lock): + self._lock = lock + + def __enter__(self): + yield self._lock.aquire() + + def __exit__(self, exc_type, exc_val, exc_tb): + yield self._lock.release() + + @defer.inlineCallbacks def DeferredDict(d, consumeErrors=False): keys = [] diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index a4f2acf3c..bbf75471c 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -117,10 +117,17 @@ class Node(object): self.peerPort = peerPort self.hash_watcher = HashWatcher() + # will be used later + self._can_store = True + def __del__(self): if self._listeningPort is not None: self._listeningPort.stopListening() + @property + def can_store(self): + return self._can_store is True + def stop(self): # stop LoopingCalls: if self.refresh_node_lc.running: @@ -252,20 +259,7 @@ class Node(object): def iterativeAnnounceHaveBlob(self, blob_hash, value): known_nodes = {} - def log_error(err, n): - if err.check(protocol.TimeoutError): - log.debug( - "Timeout while storing blob_hash %s at %s", - binascii.hexlify(blob_hash), n) - else: - log.error( - "Unexpected error while storing blob_hash %s at %s: %s", - binascii.hexlify(blob_hash), n, err.getErrorMessage()) - - def log_success(res): - log.debug("Response to store request: %s", str(res)) - return res - + @defer.inlineCallbacks def announce_to_peer(responseTuple): """ @type responseMsg: kademlia.msgtypes.ResponseMessage """ # The "raw response" tuple contains the response message, @@ -274,40 +268,65 @@ class Node(object): originAddress = responseTuple[1] # tuple: (ip adress, udp port) # Make sure the responding node is valid, and abort the operation if it isn't if not responseMsg.nodeID in known_nodes: - return responseMsg.nodeID - + log.warning("Responding node was not expected") + defer.returnValue(responseMsg.nodeID) n = known_nodes[responseMsg.nodeID] result = responseMsg.response + announced = False if 'token' in result: value['token'] = result['token'] - d = n.store(blob_hash, value, self.node_id, 0) - d.addCallback(log_success) - d.addErrback(log_error, n) + try: + res = yield n.store(blob_hash, value, self.node_id) + log.debug("Response to store request: %s", str(res)) + announced = True + except protocol.TimeoutError: + log.debug("Timeout while storing blob_hash %s at %s", + blob_hash.encode('hex')[:16], n.id.encode('hex')) + except Exception as err: + log.error("Unexpected error while storing blob_hash %s at %s: %s", + blob_hash.encode('hex')[:16], n.id.encode('hex'), err) else: - d = defer.succeed(False) - return d + log.warning("missing token") + defer.returnValue(announced) + @defer.inlineCallbacks def requestPeers(contacts): if self.externalIP is not None and len(contacts) >= constants.k: is_closer = Distance(blob_hash).is_closer(self.node_id, contacts[-1].id) if is_closer: contacts.pop() - self.store(blob_hash, value, self_store=True, originalPublisherID=self.node_id) + yield self.store(blob_hash, value, originalPublisherID=self.node_id, + self_store=True) elif self.externalIP is not None: - self.store(blob_hash, value, self_store=True, originalPublisherID=self.node_id) - ds = [] + yield self.store(blob_hash, value, originalPublisherID=self.node_id, + self_store=True) + else: + raise Exception("Cannot determine external IP: %s" % self.externalIP) + + contacted = [] for contact in contacts: known_nodes[contact.id] = contact rpcMethod = getattr(contact, "findValue") - df = rpcMethod(blob_hash, rawResponse=True) - df.addCallback(announce_to_peer) - df.addErrback(log_error, contact) - ds.append(df) - return defer.DeferredList(ds) + try: + response = yield rpcMethod(blob_hash, rawResponse=True) + stored = yield announce_to_peer(response) + if stored: + contacted.append(contact) + except protocol.TimeoutError: + log.debug("Timeout while storing blob_hash %s at %s", + binascii.hexlify(blob_hash), contact) + except Exception as err: + log.error("Unexpected error while storing blob_hash %s at %s: %s", + binascii.hexlify(blob_hash), contact, err) + log.debug("Stored %s to %i of %i attempted peers", blob_hash.encode('hex')[:16], + len(contacted), len(contacts)) + + contacted_node_ids = [c.id.encode('hex') for c in contacts] + defer.returnValue(contacted_node_ids) d = self.iterativeFindNode(blob_hash) - d.addCallbacks(requestPeers) + d.addCallback(requestPeers) return d def change_token(self): @@ -638,28 +657,13 @@ class Node(object): self._dataStore.removeExpiredPeers() defer.returnValue(None) + @defer.inlineCallbacks def _refreshRoutingTable(self): nodeIDs = self._routingTable.getRefreshList(0, False) - outerDf = defer.Deferred() - - def searchForNextNodeID(dfResult=None): - if len(nodeIDs) > 0: - searchID = nodeIDs.pop() - df = self.iterativeFindNode(searchID) - df.addCallback(searchForNextNodeID) - else: - # If this is reached, we have finished refreshing the routing table - outerDf.callback(None) - - # Start the refreshing cycle - searchForNextNodeID() - return outerDf - - - # args put here because _refreshRoutingTable does outerDF.callback(None) - def _removeExpiredPeers(self, *args): - df = threads.deferToThread(self._dataStore.removeExpiredPeers) - return df + while nodeIDs: + searchID = nodeIDs.pop() + yield self.iterativeFindNode(searchID) + defer.returnValue(None) # This was originally a set of nested methods in _iterativeFind diff --git a/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py b/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py index 60021ffc9..0802cb731 100644 --- a/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py +++ b/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py @@ -1,21 +1,30 @@ from twisted.trial import unittest -from twisted.internet import defer, task +from twisted.internet import defer, task, reactor from lbrynet.core import utils from lbrynet.tests.util import random_lbry_hash +from lbrynet.core.server.DHTHashAnnouncer import DHTHashAnnouncer + class MocDHTNode(object): def __init__(self): + self.can_store = True self.blobs_announced = 0 + @defer.inlineCallbacks def announceHaveBlob(self, blob): self.blobs_announced += 1 - return defer.succeed(True) + d = defer.Deferred(None) + reactor.callLater(1, d.callback, {blob: ["ab" * 48]}) + result = yield d + defer.returnValue(result) + class MocSupplier(object): def __init__(self, blobs_to_announce): self.blobs_to_announce = blobs_to_announce self.announced = False + def hashes_to_announce(self): if not self.announced: self.announced = True @@ -23,8 +32,8 @@ class MocSupplier(object): else: return defer.succeed([]) -class DHTHashAnnouncerTest(unittest.TestCase): +class DHTHashAnnouncerTest(unittest.TestCase): def setUp(self): self.num_blobs = 10 self.blobs_to_announce = [] @@ -33,23 +42,26 @@ class DHTHashAnnouncerTest(unittest.TestCase): self.clock = task.Clock() self.dht_node = MocDHTNode() utils.call_later = self.clock.callLater - from lbrynet.core.server.DHTHashAnnouncer import DHTHashAnnouncer self.announcer = DHTHashAnnouncer(self.dht_node, peer_port=3333) self.supplier = MocSupplier(self.blobs_to_announce) self.announcer.add_supplier(self.supplier) + @defer.inlineCallbacks def test_basic(self): - self.announcer._announce_available_hashes() + d = self.announcer._announce_available_hashes() self.assertEqual(self.announcer.hash_queue_size(), self.announcer.CONCURRENT_ANNOUNCERS) self.clock.advance(1) + yield d self.assertEqual(self.dht_node.blobs_announced, self.num_blobs) self.assertEqual(self.announcer.hash_queue_size(), 0) + @defer.inlineCallbacks def test_immediate_announce(self): # Test that immediate announce puts a hash at the front of the queue - self.announcer._announce_available_hashes() + d = self.announcer._announce_available_hashes() + self.assertEqual(self.announcer.hash_queue_size(), self.announcer.CONCURRENT_ANNOUNCERS) blob_hash = random_lbry_hash() self.announcer.immediate_announce([blob_hash]) self.assertEqual(self.announcer.hash_queue_size(), self.announcer.CONCURRENT_ANNOUNCERS+1) self.assertEqual(blob_hash, self.announcer.hash_queue[0][0]) - + yield d From ad6a2bef7fda8a4c9e558a67ccaf42db02e18c14 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Mon, 23 Oct 2017 15:37:02 -0400 Subject: [PATCH 04/52] handle error from old clients with a broken ping command --- lbrynet/dht/protocol.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lbrynet/dht/protocol.py b/lbrynet/dht/protocol.py index b2d3e657a..5656f29bb 100644 --- a/lbrynet/dht/protocol.py +++ b/lbrynet/dht/protocol.py @@ -253,8 +253,18 @@ class KademliaProtocol(protocol.DatagramProtocol): else: exception_type = UnknownRemoteException remoteException = exception_type(message.response) - log.error("Remote exception (%s): %s", address, remoteException) - df.errback(remoteException) + # this error is returned by nodes that can be contacted but have an old + # and broken version of the ping command, if they return it the node can + # be contacted, so we'll treat it as a successful ping + old_ping_error = "ping() got an unexpected keyword argument '_rpcNodeContact'" + if isinstance(remoteException, TypeError) and \ + remoteException.message == old_ping_error: + log.debug("old pong error") + df.callback('pong') + else: + log.error("DHT RECV REMOTE EXCEPTION FROM %s:%i: %s", address[0], + address[1], remoteException) + df.errback(remoteException) else: # We got a result from the RPC df.callback(message.response) From 0f3385e4dcbed93907578adc1fe48d2bb47bd4ce Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Mon, 6 Nov 2017 13:17:38 -0500 Subject: [PATCH 05/52] make the single hash announce duration adjustable in DHTHashSupplier --- lbrynet/core/server/DHTHashAnnouncer.py | 17 +++++++++++++++-- .../unit/core/server/test_DHTHashAnnouncer.py | 2 ++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/lbrynet/core/server/DHTHashAnnouncer.py b/lbrynet/core/server/DHTHashAnnouncer.py index 877b0538c..d00fa50a3 100644 --- a/lbrynet/core/server/DHTHashAnnouncer.py +++ b/lbrynet/core/server/DHTHashAnnouncer.py @@ -156,6 +156,9 @@ class DHTHashAnnouncer(object): for _, announced_to in announcer_results: stored_to.update(announced_to) log.info('Took %s seconds to announce %s hashes', time.time() - start, len(hashes)) + seconds_per_blob = (time.time() - start) / len(hashes) + for supplier in self.suppliers: + supplier.set_single_hash_announce_duration(seconds_per_blob) defer.returnValue(stored_to) @@ -164,7 +167,7 @@ class DHTHashSupplier(object): MIN_HASH_REANNOUNCE_TIME = 60 * 60 # conservative assumption of the time it takes to announce # a single hash - SINGLE_HASH_ANNOUNCE_DURATION = 5 + DEFAULT_SINGLE_HASH_ANNOUNCE_DURATION = 1 """Classes derived from this class give hashes to a hash announcer""" @@ -172,10 +175,20 @@ class DHTHashSupplier(object): if announcer is not None: announcer.add_supplier(self) self.hash_announcer = announcer + self.single_hash_announce_duration = self.DEFAULT_SINGLE_HASH_ANNOUNCE_DURATION def hashes_to_announce(self): pass + def set_single_hash_announce_duration(self, seconds): + """ + Set the duration it takes to announce a single hash + in seconds, cannot be less than the default single + hash announce duration + """ + seconds = max(seconds, self.DEFAULT_SINGLE_HASH_ANNOUNCE_DURATION) + self.single_hash_announce_duration = seconds + def get_next_announce_time(self, num_hashes_to_announce=1): """ Hash reannounce time is set to current time + MIN_HASH_REANNOUNCE_TIME, @@ -191,5 +204,5 @@ class DHTHashSupplier(object): """ queue_size = self.hash_announcer.hash_queue_size() + num_hashes_to_announce reannounce = max(self.MIN_HASH_REANNOUNCE_TIME, - queue_size * self.SINGLE_HASH_ANNOUNCE_DURATION) + queue_size * self.single_hash_announce_duration) return time.time() + reannounce diff --git a/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py b/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py index 0802cb731..e2c7aee9c 100644 --- a/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py +++ b/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py @@ -32,6 +32,8 @@ class MocSupplier(object): else: return defer.succeed([]) + def set_single_hash_announce_duration(self, seconds): + pass class DHTHashAnnouncerTest(unittest.TestCase): def setUp(self): From 60c72618715f1bc6a9c755c58ce60141d71b6a75 Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Wed, 8 Nov 2017 22:50:40 -0500 Subject: [PATCH 06/52] add single_hash_announce_duration as a field that gets return for API call status for dht status --- lbrynet/daemon/Daemon.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index 348481c74..6ee6b8fcf 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -1042,11 +1042,12 @@ class Daemon(AuthJSONRPCServer): 'dht_status': { 'kbps_received': current kbps receiving, 'kbps_sent': current kdps being sent, - 'total_bytes_sent': total bytes sent, - 'total_bytes_received': total bytes received, - 'queries_received': number of queries received per second, - 'queries_sent': number of queries sent per second, - 'recent_contacts': count of recently contacted peers, + 'total_bytes_sent': total bytes sent + 'total_bytes_received': total bytes received + 'queries_received': number of queries received per second + 'queries_sent': number of queries sent per second + 'recent_contacts': count of recently contacted peers + 'single_hash_announce_duration': avg. seconds it takes to announce a blob 'unique_contacts': count of unique peers }, } @@ -1097,6 +1098,8 @@ class Daemon(AuthJSONRPCServer): } if dht_status: response['dht_status'] = self.session.dht_node.get_bandwidth_stats() + response['dht_status'].update({'single_hash_announce_duration': + self.session.blob_manager.single_hash_announce_duration}) defer.returnValue(response) def jsonrpc_version(self): From 75b977dff93df6b32f7b800a73d5cf79c7885483 Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Mon, 6 Nov 2017 19:48:47 -0500 Subject: [PATCH 07/52] we just have one supplier not a list of suppliers --- lbrynet/core/server/DHTHashAnnouncer.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lbrynet/core/server/DHTHashAnnouncer.py b/lbrynet/core/server/DHTHashAnnouncer.py index d00fa50a3..a1989bc88 100644 --- a/lbrynet/core/server/DHTHashAnnouncer.py +++ b/lbrynet/core/server/DHTHashAnnouncer.py @@ -20,7 +20,7 @@ class DHTHashAnnouncer(object): def __init__(self, dht_node, peer_port): self.dht_node = dht_node self.peer_port = peer_port - self.suppliers = [] + self.supplier = None self.next_manage_call = None self.hash_queue = collections.deque() self._concurrent_announcers = 0 @@ -62,7 +62,7 @@ class DHTHashAnnouncer(object): self._manage_call_lc.stop() def add_supplier(self, supplier): - self.suppliers.append(supplier) + self.supplier = supplier def immediate_announce(self, blob_hashes): if self.peer_port is not None: @@ -76,8 +76,8 @@ class DHTHashAnnouncer(object): @defer.inlineCallbacks def _announce_available_hashes(self): log.debug('Announcing available hashes') - for supplier in self.suppliers: - hashes = yield supplier.hashes_to_announce() + if self.supplier: + hashes = yield self.supplier.hashes_to_announce() yield self._announce_hashes(hashes) @defer.inlineCallbacks @@ -155,10 +155,10 @@ class DHTHashAnnouncer(object): stored_to = {} for _, announced_to in announcer_results: stored_to.update(announced_to) + log.info('Took %s seconds to announce %s hashes', time.time() - start, len(hashes)) seconds_per_blob = (time.time() - start) / len(hashes) - for supplier in self.suppliers: - supplier.set_single_hash_announce_duration(seconds_per_blob) + self.supplier.set_single_hash_announce_duration(seconds_per_blob) defer.returnValue(stored_to) From 0425c95b681d6d09c1eb72e0ef433f6f8bde8d44 Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Thu, 9 Nov 2017 14:47:55 -0500 Subject: [PATCH 08/52] No need for clock now in test for DHTHashAnnouncer --- lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py b/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py index e2c7aee9c..1f5b502ab 100644 --- a/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py +++ b/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py @@ -1,7 +1,6 @@ from twisted.trial import unittest -from twisted.internet import defer, task, reactor +from twisted.internet import defer, reactor -from lbrynet.core import utils from lbrynet.tests.util import random_lbry_hash from lbrynet.core.server.DHTHashAnnouncer import DHTHashAnnouncer @@ -41,9 +40,7 @@ class DHTHashAnnouncerTest(unittest.TestCase): self.blobs_to_announce = [] for i in range(0, self.num_blobs): self.blobs_to_announce.append(random_lbry_hash()) - self.clock = task.Clock() self.dht_node = MocDHTNode() - utils.call_later = self.clock.callLater self.announcer = DHTHashAnnouncer(self.dht_node, peer_port=3333) self.supplier = MocSupplier(self.blobs_to_announce) self.announcer.add_supplier(self.supplier) @@ -52,7 +49,6 @@ class DHTHashAnnouncerTest(unittest.TestCase): def test_basic(self): d = self.announcer._announce_available_hashes() self.assertEqual(self.announcer.hash_queue_size(), self.announcer.CONCURRENT_ANNOUNCERS) - self.clock.advance(1) yield d self.assertEqual(self.dht_node.blobs_announced, self.num_blobs) self.assertEqual(self.announcer.hash_queue_size(), 0) From 4cb461601e32ecf9cc248c0c4be0246fbce5ce9d Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Thu, 9 Nov 2017 14:53:29 -0500 Subject: [PATCH 09/52] result must be set here, otherwise it will not be defined when used later. Add test for it --- lbrynet/core/server/DHTHashAnnouncer.py | 1 + .../unit/core/server/test_DHTHashAnnouncer.py | 20 ++++++++++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/lbrynet/core/server/DHTHashAnnouncer.py b/lbrynet/core/server/DHTHashAnnouncer.py index a1989bc88..073c60540 100644 --- a/lbrynet/core/server/DHTHashAnnouncer.py +++ b/lbrynet/core/server/DHTHashAnnouncer.py @@ -118,6 +118,7 @@ class DHTHashAnnouncer(object): log.debug("No nodes stored %s, retrying", blob_hash) result = yield do_store(blob_hash, announce_d) else: + result = {} log.warning("No nodes stored %s", blob_hash) else: result = store_nodes diff --git a/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py b/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py index 1f5b502ab..bc72fed68 100644 --- a/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py +++ b/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py @@ -4,17 +4,24 @@ from twisted.internet import defer, reactor from lbrynet.tests.util import random_lbry_hash from lbrynet.core.server.DHTHashAnnouncer import DHTHashAnnouncer - class MocDHTNode(object): - def __init__(self): + def __init__(self, announce_will_fail=False): + # if announce_will_fail is True, + # announceHaveBlob will return empty dict self.can_store = True self.blobs_announced = 0 + self.announce_will_fail = announce_will_fail @defer.inlineCallbacks def announceHaveBlob(self, blob): + if self.announce_will_fail: + return_val = {} + else: + return_val = {blob:["ab"*48]} + self.blobs_announced += 1 d = defer.Deferred(None) - reactor.callLater(1, d.callback, {blob: ["ab" * 48]}) + reactor.callLater(1, d.callback, return_val) result = yield d defer.returnValue(result) @@ -45,6 +52,13 @@ class DHTHashAnnouncerTest(unittest.TestCase): self.supplier = MocSupplier(self.blobs_to_announce) self.announcer.add_supplier(self.supplier) + @defer.inlineCallbacks + def test_announce_fail(self): + # test what happens when node.announceHaveBlob() returns empty dict + self.dht_node.announce_will_fail = True + d = yield self.announcer._announce_available_hashes() + yield d + @defer.inlineCallbacks def test_basic(self): d = self.announcer._announce_available_hashes() From 9088d152b5da845a982b7eb385d3a3b5ddafb798 Mon Sep 17 00:00:00 2001 From: Kay Kurokawa Date: Mon, 6 Nov 2017 20:10:35 -0500 Subject: [PATCH 10/52] better to keey track of retry count in function instead of unbounded dictionary --- lbrynet/core/server/DHTHashAnnouncer.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/lbrynet/core/server/DHTHashAnnouncer.py b/lbrynet/core/server/DHTHashAnnouncer.py index 073c60540..d029f1f15 100644 --- a/lbrynet/core/server/DHTHashAnnouncer.py +++ b/lbrynet/core/server/DHTHashAnnouncer.py @@ -27,7 +27,6 @@ class DHTHashAnnouncer(object): self._manage_call_lc = task.LoopingCall(self.manage_lc) self._lock = utils.DeferredLockContextManager(defer.DeferredLock()) self._last_checked = time.time(), self.CONCURRENT_ANNOUNCERS - self._retries = {} self._total = None def run_manage_loop(self): @@ -105,18 +104,16 @@ class DHTHashAnnouncer(object): log.debug('There are now %s hashes remaining to be announced', self.hash_queue_size()) @defer.inlineCallbacks - def do_store(blob_hash, announce_d): + def do_store(blob_hash, announce_d, retry_count=0): if announce_d.called: defer.returnValue(announce_deferred.result) try: store_nodes = yield self.dht_node.announceHaveBlob(binascii.unhexlify(blob_hash)) if not store_nodes: - retries = self._retries.get(blob_hash, 0) - retries += 1 - self._retries[blob_hash] = retries - if retries <= self.STORE_RETRIES: + retry_count += 1 + if retry_count <= self.STORE_RETRIES: log.debug("No nodes stored %s, retrying", blob_hash) - result = yield do_store(blob_hash, announce_d) + result = yield do_store(blob_hash, announce_d, retry_count) else: result = {} log.warning("No nodes stored %s", blob_hash) From e6cc9b0de1f6d64acafb06fcb990cc97a62c5a1f Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 14 Feb 2018 18:20:37 -0500 Subject: [PATCH 11/52] changelog --- CHANGELOG.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d69eac53..f248614ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,10 +15,37 @@ at anytime. ### Fixed * * + * incorrectly raised download cancelled error for already verified blob files + * infinite loop where reflector client keeps trying to send failing blobs, which may be failing because they are invalid and thus will never be successfully received + * docstring bugs for `stream_availability`, `channel_import`, and `blob_announce` + * regression in `stream_availability` due to error in it's docstring + * fixed the inconsistencies in API and CLI docstrings + * `blob_announce` error when announcing a single blob + * `blob_list` error when looking up blobs by stream or sd hash + * issue#1107 whereing claiming a channel with the exact amount present in wallet would give out proper error + * + * + * + * improper parsing of arguments to CLI settings_set (https://github.com/lbryio/lbry/issues/930) + * unnecessarily verbose exchange rate error (https://github.com/lbryio/lbry/issues/984) + * value error due to a race condition when saving to the claim cache (https://github.com/lbryio/lbry/issues/1013) + * being unable to re-download updated content (https://github.com/lbryio/lbry/issues/951) + * sending error messages for failed api requests + * file manager startup being slow when handling thousands of files + * handling decryption error for blobs encrypted with an invalid key + * handling stream with no data blob (https://github.com/lbryio/lbry/issues/905) + * fetching the external ip + * `blob_list` returning an error with --uri parameter and incorrectly returning `[]` for streams where blobs are known (https://github.com/lbryio/lbry/issues/895) + * `get` failing with a non-useful error message when given a uri for a channel claim + * exception checking in several wallet unit tests + * daemon not erring properly for non-numeric values being passed to the `bid` parameter for the `publish` method + * incorrect `blob_num` for the stream terminator blob, which would result in creating invalid streams. Such invalid streams are detected on startup and are automatically removed (https://github.com/lbryio/lbry/issues/1124) + * handling error from dht clients with old `ping` method ### Deprecated * * + * `single_hash_announce_duration` field to `status` response when provided the `dht_status` argument ### Changed * @@ -27,6 +54,17 @@ at anytime. ### Added * * + * `blob_reflect` command to send specific blobs to a reflector server + * unit test for docopt + * + * scripts to autogenerate documentation + * now updating new channel also takes into consideration the original bid amount, so now channel could be updated for wallet balance + the original bid amount + * forward-compaitibility for upcoming DHT bencoding changes + * + * several internal dht functions to use inlineCallbacks + * blob announcement to be retried up to three times if `store` is unsuccessful + * `DHTHashAnnouncer` and `Node` manage functions to use `LoopingCall`s instead of scheduling with `callLater`. + * `store` kademlia rpc method to block on the call finishing and to return storing peer information ### Removed * From 54a152fa8e8b177e02687fd2e32a122d41f2b3c3 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 14 Feb 2018 18:28:28 -0500 Subject: [PATCH 12/52] fix merge conflict in dht tests --- lbrynet/tests/dht/test_routing_table.py | 191 -------------------- lbrynet/tests/unit/dht/test_routingtable.py | 14 +- 2 files changed, 5 insertions(+), 200 deletions(-) delete mode 100644 lbrynet/tests/dht/test_routing_table.py diff --git a/lbrynet/tests/dht/test_routing_table.py b/lbrynet/tests/dht/test_routing_table.py deleted file mode 100644 index c93bbc19b..000000000 --- a/lbrynet/tests/dht/test_routing_table.py +++ /dev/null @@ -1,191 +0,0 @@ - -#!/usr/bin/env python -# -# This library is free software, distributed under the terms of -# the GNU Lesser General Public License Version 3, or any later version. -# See the COPYING file included in this archive - -import hashlib -import unittest - -import lbrynet.dht.constants -import lbrynet.dht.routingtable -import lbrynet.dht.contact -import lbrynet.dht.node -import lbrynet.dht.distance - - -class FakeRPCProtocol(object): - """ Fake RPC protocol; allows lbrynet.dht.contact.Contact objects to "send" RPCs """ - def sendRPC(self, *args, **kwargs): - return FakeDeferred() - - -class FakeDeferred(object): - """ Fake Twisted Deferred object; allows the routing table to add callbacks that do nothing """ - def addCallback(self, *args, **kwargs): - return - - def addErrback(self, *args, **kwargs): - return - - def addCallbacks(self, *args, **kwargs): - return - - -class TreeRoutingTableTest(unittest.TestCase): - """ Test case for the RoutingTable class """ - def setUp(self): - h = hashlib.sha384() - h.update('node1') - self.nodeID = h.digest() - self.protocol = FakeRPCProtocol() - self.routingTable = lbrynet.dht.routingtable.TreeRoutingTable(self.nodeID) - - def testDistance(self): - """ Test to see if distance method returns correct result""" - - # testList holds a couple 3-tuple (variable1, variable2, result) - basicTestList = [('123456789', '123456789', 0L), ('12345', '98765', 34527773184L)] - - for test in basicTestList: - result = lbrynet.dht.distance.Distance(test[0])(test[1]) - self.failIf(result != test[2], 'Result of _distance() should be %s but %s returned' % - (test[2], result)) - - baseIp = '146.64.19.111' - ipTestList = ['146.64.29.222', '192.68.19.333'] - - distanceOne = lbrynet.dht.distance.Distance(baseIp)(ipTestList[0]) - distanceTwo = lbrynet.dht.distance.Distance(baseIp)(ipTestList[1]) - - self.failIf(distanceOne > distanceTwo, '%s should be closer to the base ip %s than %s' % - (ipTestList[0], baseIp, ipTestList[1])) - - def testAddContact(self): - """ Tests if a contact can be added and retrieved correctly """ - # Create the contact - h = hashlib.sha384() - h.update('node2') - contactID = h.digest() - contact = lbrynet.dht.contact.Contact(contactID, '127.0.0.1', 91824, self.protocol) - # Now add it... - self.routingTable.addContact(contact) - # ...and request the closest nodes to it (will retrieve it) - closestNodes = self.routingTable.findCloseNodes(contactID, lbrynet.dht.constants.k) - self.failUnlessEqual(len(closestNodes), 1, 'Wrong amount of contacts returned; expected 1,' - ' got %d' % len(closestNodes)) - self.failUnless(contact in closestNodes, 'Added contact not found by issueing ' - '_findCloseNodes()') - - def testGetContact(self): - """ Tests if a specific existing contact can be retrieved correctly """ - h = hashlib.sha384() - h.update('node2') - contactID = h.digest() - contact = lbrynet.dht.contact.Contact(contactID, '127.0.0.1', 91824, self.protocol) - # Now add it... - self.routingTable.addContact(contact) - # ...and get it again - sameContact = self.routingTable.getContact(contactID) - self.failUnlessEqual(contact, sameContact, 'getContact() should return the same contact') - - def testAddParentNodeAsContact(self): - """ - Tests the routing table's behaviour when attempting to add its parent node as a contact - """ - - # Create a contact with the same ID as the local node's ID - contact = lbrynet.dht.contact.Contact(self.nodeID, '127.0.0.1', 91824, self.protocol) - # Now try to add it - self.routingTable.addContact(contact) - # ...and request the closest nodes to it using FIND_NODE - closestNodes = self.routingTable.findCloseNodes(self.nodeID, lbrynet.dht.constants.k) - self.failIf(contact in closestNodes, 'Node added itself as a contact') - - def testRemoveContact(self): - """ Tests contact removal """ - # Create the contact - h = hashlib.sha384() - h.update('node2') - contactID = h.digest() - contact = lbrynet.dht.contact.Contact(contactID, '127.0.0.1', 91824, self.protocol) - # Now add it... - self.routingTable.addContact(contact) - # Verify addition - self.failUnlessEqual(len(self.routingTable._buckets[0]), 1, 'Contact not added properly') - # Now remove it - self.routingTable.removeContact(contact.id) - self.failUnlessEqual(len(self.routingTable._buckets[0]), 0, 'Contact not removed properly') - - def testSplitBucket(self): - """ Tests if the the routing table correctly dynamically splits k-buckets """ - self.failUnlessEqual(self.routingTable._buckets[0].rangeMax, 2**384, - 'Initial k-bucket range should be 0 <= range < 2**384') - # Add k contacts - for i in range(lbrynet.dht.constants.k): - h = hashlib.sha384() - h.update('remote node %d' % i) - nodeID = h.digest() - contact = lbrynet.dht.contact.Contact(nodeID, '127.0.0.1', 91824, self.protocol) - self.routingTable.addContact(contact) - self.failUnlessEqual(len(self.routingTable._buckets), 1, - 'Only k nodes have been added; the first k-bucket should now ' - 'be full, but should not yet be split') - # Now add 1 more contact - h = hashlib.sha384() - h.update('yet another remote node') - nodeID = h.digest() - contact = lbrynet.dht.contact.Contact(nodeID, '127.0.0.1', 91824, self.protocol) - self.routingTable.addContact(contact) - self.failUnlessEqual(len(self.routingTable._buckets), 2, - 'k+1 nodes have been added; the first k-bucket should have been ' - 'split into two new buckets') - self.failIfEqual(self.routingTable._buckets[0].rangeMax, 2**384, - 'K-bucket was split, but its range was not properly adjusted') - self.failUnlessEqual(self.routingTable._buckets[1].rangeMax, 2**384, - 'K-bucket was split, but the second (new) bucket\'s ' - 'max range was not set properly') - self.failUnlessEqual(self.routingTable._buckets[0].rangeMax, - self.routingTable._buckets[1].rangeMin, - 'K-bucket was split, but the min/max ranges were ' - 'not divided properly') - - def testFullBucketNoSplit(self): - """ - Test that a bucket is not split if it full, but does not cover the range - containing the parent node's ID - """ - self.routingTable._parentNodeID = 49 * 'a' - # more than 384 bits; this will not be in the range of _any_ k-bucket - # Add k contacts - for i in range(lbrynet.dht.constants.k): - h = hashlib.sha384() - h.update('remote node %d' % i) - nodeID = h.digest() - contact = lbrynet.dht.contact.Contact(nodeID, '127.0.0.1', 91824, self.protocol) - self.routingTable.addContact(contact) - self.failUnlessEqual(len(self.routingTable._buckets), 1, 'Only k nodes have been added; ' - 'the first k-bucket should now be ' - 'full, and there should not be ' - 'more than 1 bucket') - self.failUnlessEqual(len(self.routingTable._buckets[0]._contacts), lbrynet.dht.constants.k, - 'Bucket should have k contacts; expected %d got %d' % - (lbrynet.dht.constants.k, - len(self.routingTable._buckets[0]._contacts))) - # Now add 1 more contact - h = hashlib.sha384() - h.update('yet another remote node') - nodeID = h.digest() - contact = lbrynet.dht.contact.Contact(nodeID, '127.0.0.1', 91824, self.protocol) - self.routingTable.addContact(contact) - self.failUnlessEqual(len(self.routingTable._buckets), 1, - 'There should not be more than 1 bucket, since the bucket ' - 'should not have been split (parent node ID not in range)') - self.failUnlessEqual(len(self.routingTable._buckets[0]._contacts), - lbrynet.dht.constants.k, 'Bucket should have k contacts; ' - 'expected %d got %d' % - (lbrynet.dht.constants.k, - len(self.routingTable._buckets[0]._contacts))) - self.failIf(contact in self.routingTable._buckets[0]._contacts, - 'New contact should have been discarded (since RPC is faked in this test)') diff --git a/lbrynet/tests/unit/dht/test_routingtable.py b/lbrynet/tests/unit/dht/test_routingtable.py index d5eec110f..8c0907509 100644 --- a/lbrynet/tests/unit/dht/test_routingtable.py +++ b/lbrynet/tests/unit/dht/test_routingtable.py @@ -1,12 +1,12 @@ import hashlib import unittest -#from lbrynet.dht import contact, routingtable, constants - import lbrynet.dht.constants import lbrynet.dht.routingtable import lbrynet.dht.contact import lbrynet.dht.node +import lbrynet.dht.distance + class FakeRPCProtocol(object): """ Fake RPC protocol; allows lbrynet.dht.contact.Contact objects to "send" RPCs """ @@ -42,15 +42,15 @@ class TreeRoutingTableTest(unittest.TestCase): basicTestList = [('123456789', '123456789', 0L), ('12345', '98765', 34527773184L)] for test in basicTestList: - result = lbrynet.dht.node.Distance(test[0])(test[1]) + result = lbrynet.dht.distance.Distance(test[0])(test[1]) self.failIf(result != test[2], 'Result of _distance() should be %s but %s returned' % (test[2], result)) baseIp = '146.64.19.111' ipTestList = ['146.64.29.222', '192.68.19.333'] - distanceOne = lbrynet.dht.node.Distance(baseIp)(ipTestList[0]) - distanceTwo = lbrynet.dht.node.Distance(baseIp)(ipTestList[1]) + distanceOne = lbrynet.dht.distance.Distance(baseIp)(ipTestList[0]) + distanceTwo = lbrynet.dht.distance.Distance(baseIp)(ipTestList[1]) self.failIf(distanceOne > distanceTwo, '%s should be closer to the base ip %s than %s' % (ipTestList[0], baseIp, ipTestList[1])) @@ -184,10 +184,6 @@ class TreeRoutingTableTest(unittest.TestCase): 'New contact should have been discarded (since RPC is faked in this test)') - - - - class KeyErrorFixedTest(unittest.TestCase): """ Basic tests case for boolean operators on the Contact class """ From 3296c0fb3dd09e939aa7525c9b99921a3192d18e Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Thu, 15 Feb 2018 16:37:43 -0500 Subject: [PATCH 13/52] move dht related classes to lbrynet.dht --- lbrynet/core/BlobManager.py | 2 +- lbrynet/core/Session.py | 12 +++++------- .../DHTHashAnnouncer.py => dht/hashannouncer.py} | 0 lbrynet/dht/node.py | 4 ++-- .../client/DHTPeerFinder.py => dht/peerfinder.py} | 0 lbrynet/{core/PeerManager.py => dht/peermanager.py} | 0 lbrynet/dht/routingtable.py | 2 +- lbrynet/tests/functional/test_misc.py | 2 +- lbrynet/tests/functional/test_reflector.py | 4 ++-- lbrynet/tests/functional/test_streamify.py | 2 +- .../tests/unit/core/client/test_ConnectionManager.py | 2 +- .../tests/unit/core/server/test_DHTHashAnnouncer.py | 2 +- .../lbryfilemanager/test_EncryptedFileCreator.py | 4 ++-- 13 files changed, 17 insertions(+), 19 deletions(-) rename lbrynet/{core/server/DHTHashAnnouncer.py => dht/hashannouncer.py} (100%) rename lbrynet/{core/client/DHTPeerFinder.py => dht/peerfinder.py} (100%) rename lbrynet/{core/PeerManager.py => dht/peermanager.py} (100%) diff --git a/lbrynet/core/BlobManager.py b/lbrynet/core/BlobManager.py index 520e6bd0f..d0a37f403 100644 --- a/lbrynet/core/BlobManager.py +++ b/lbrynet/core/BlobManager.py @@ -5,7 +5,7 @@ from twisted.internet import threads, defer, reactor, task from lbrynet import conf from lbrynet.blob.blob_file import BlobFile from lbrynet.blob.creator import BlobFileCreator -from lbrynet.core.server.DHTHashAnnouncer import DHTHashSupplier +from lbrynet.dht.hashannouncer import DHTHashSupplier log = logging.getLogger(__name__) diff --git a/lbrynet/core/Session.py b/lbrynet/core/Session.py index 2f8cf42b2..cb82161bc 100644 --- a/lbrynet/core/Session.py +++ b/lbrynet/core/Session.py @@ -1,17 +1,15 @@ import logging import miniupnpc +from twisted.internet import threads, defer from lbrynet.core.BlobManager import DiskBlobManager -from lbrynet.dht import node +from lbrynet.dht import node, peerfinder, peermanager from lbrynet.database.storage import SQLiteStorage -from lbrynet.core.PeerManager import PeerManager from lbrynet.core.RateLimiter import RateLimiter -from lbrynet.core.client.DHTPeerFinder import DHTPeerFinder from lbrynet.core.HashAnnouncer import DummyHashAnnouncer -from lbrynet.core.server.DHTHashAnnouncer import DHTHashAnnouncer +from lbrynet.dht.hashannouncer import DHTHashAnnouncer from lbrynet.core.utils import generate_id from lbrynet.core.PaymentRateManager import BasePaymentRateManager, NegotiatedPaymentRateManager from lbrynet.core.BlobAvailability import BlobAvailabilityTracker -from twisted.internet import threads, defer log = logging.getLogger(__name__) @@ -152,7 +150,7 @@ class Session(object): self.wallet = PTCWallet(self.db_dir) if self.peer_manager is None: - self.peer_manager = PeerManager() + self.peer_manager = peermanager.PeerManager() if self.use_upnp is True: d = self._try_upnp() @@ -296,7 +294,7 @@ class Session(object): externalIP=self.external_ip, peerPort=self.peer_port ) - self.peer_finder = DHTPeerFinder(self.dht_node, self.peer_manager) + self.peer_finder = peerfinder.DHTPeerFinder(self.dht_node, self.peer_manager) if self.hash_announcer is None: self.hash_announcer = DHTHashAnnouncer(self.dht_node, self.peer_port) diff --git a/lbrynet/core/server/DHTHashAnnouncer.py b/lbrynet/dht/hashannouncer.py similarity index 100% rename from lbrynet/core/server/DHTHashAnnouncer.py rename to lbrynet/dht/hashannouncer.py diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index bbf75471c..13843ef51 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -278,10 +278,10 @@ class Node(object): value['token'] = result['token'] try: res = yield n.store(blob_hash, value, self.node_id) - log.debug("Response to store request: %s", str(res)) + log.info("Response to store request: %s", str(res)) announced = True except protocol.TimeoutError: - log.debug("Timeout while storing blob_hash %s at %s", + log.info("Timeout while storing blob_hash %s at %s", blob_hash.encode('hex')[:16], n.id.encode('hex')) except Exception as err: log.error("Unexpected error while storing blob_hash %s at %s: %s", diff --git a/lbrynet/core/client/DHTPeerFinder.py b/lbrynet/dht/peerfinder.py similarity index 100% rename from lbrynet/core/client/DHTPeerFinder.py rename to lbrynet/dht/peerfinder.py diff --git a/lbrynet/core/PeerManager.py b/lbrynet/dht/peermanager.py similarity index 100% rename from lbrynet/core/PeerManager.py rename to lbrynet/dht/peermanager.py diff --git a/lbrynet/dht/routingtable.py b/lbrynet/dht/routingtable.py index 1c73c5360..863f37770 100644 --- a/lbrynet/dht/routingtable.py +++ b/lbrynet/dht/routingtable.py @@ -20,7 +20,7 @@ log = logging.getLogger(__name__) class TreeRoutingTable(object): """ This class implements a routing table used by a Node class. - The Kademlia routing table is a binary tree whose leaves are k-buckets, + The Kademlia routing table is a binary tree whFose leaves are k-buckets, where each k-bucket contains nodes with some common prefix of their IDs. This prefix is the k-bucket's position in the binary tree; it therefore covers some range of ID values, and together all of the k-buckets cover diff --git a/lbrynet/tests/functional/test_misc.py b/lbrynet/tests/functional/test_misc.py index dffb100ec..327976c5e 100644 --- a/lbrynet/tests/functional/test_misc.py +++ b/lbrynet/tests/functional/test_misc.py @@ -23,7 +23,7 @@ from twisted.trial.unittest import TestCase from twisted.python.failure import Failure from lbrynet.dht.node import Node -from lbrynet.core.PeerManager import PeerManager +from lbrynet.dht.peermanager import PeerManager from lbrynet.core.RateLimiter import DummyRateLimiter, RateLimiter from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory from lbrynet.core.server.ServerProtocol import ServerProtocolFactory diff --git a/lbrynet/tests/functional/test_reflector.py b/lbrynet/tests/functional/test_reflector.py index 41d24902a..0250607d4 100644 --- a/lbrynet/tests/functional/test_reflector.py +++ b/lbrynet/tests/functional/test_reflector.py @@ -5,7 +5,7 @@ from lbrynet import conf from lbrynet.core.StreamDescriptor import get_sd_info from lbrynet import reflector from lbrynet.core import BlobManager -from lbrynet.core import PeerManager +from lbrynet.dht import peermanager from lbrynet.core import Session from lbrynet.core import StreamDescriptor from lbrynet.lbry_file.client import EncryptedFileOptions @@ -26,7 +26,7 @@ class TestReflector(unittest.TestCase): self.port = None self.addCleanup(self.take_down_env) wallet = mocks.Wallet() - peer_manager = PeerManager.PeerManager() + peer_manager = peermanager.PeerManager() peer_finder = mocks.PeerFinder(5553, peer_manager, 2) hash_announcer = mocks.Announcer() sd_identifier = StreamDescriptor.StreamDescriptorIdentifier() diff --git a/lbrynet/tests/functional/test_streamify.py b/lbrynet/tests/functional/test_streamify.py index f8fd633f2..6d87d382a 100644 --- a/lbrynet/tests/functional/test_streamify.py +++ b/lbrynet/tests/functional/test_streamify.py @@ -13,7 +13,7 @@ from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier from lbrynet.file_manager.EncryptedFileCreator import create_lbry_file from lbrynet.lbry_file.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier from lbrynet.core.StreamDescriptor import get_sd_info -from lbrynet.core.PeerManager import PeerManager +from lbrynet.dht.peermanager import PeerManager from lbrynet.core.RateLimiter import DummyRateLimiter from lbrynet.tests import mocks diff --git a/lbrynet/tests/unit/core/client/test_ConnectionManager.py b/lbrynet/tests/unit/core/client/test_ConnectionManager.py index 107afa997..57ed1e534 100644 --- a/lbrynet/tests/unit/core/client/test_ConnectionManager.py +++ b/lbrynet/tests/unit/core/client/test_ConnectionManager.py @@ -3,7 +3,7 @@ from lbrynet.core.server.ServerProtocol import ServerProtocol from lbrynet.core.client.ClientProtocol import ClientProtocol from lbrynet.core.RateLimiter import RateLimiter from lbrynet.core.Peer import Peer -from lbrynet.core.PeerManager import PeerManager +from lbrynet.dht.peermanager import PeerManager from lbrynet.core.Error import NoResponseError from twisted.trial import unittest diff --git a/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py b/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py index bc72fed68..3e16362fa 100644 --- a/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py +++ b/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py @@ -2,7 +2,7 @@ from twisted.trial import unittest from twisted.internet import defer, reactor from lbrynet.tests.util import random_lbry_hash -from lbrynet.core.server.DHTHashAnnouncer import DHTHashAnnouncer +from lbrynet.dht.hashannouncer import DHTHashAnnouncer class MocDHTNode(object): def __init__(self, announce_will_fail=False): diff --git a/lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py b/lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py index 05c0a8feb..a02e8d78c 100644 --- a/lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py +++ b/lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py @@ -8,7 +8,7 @@ from lbrynet.database.storage import SQLiteStorage from lbrynet.core.StreamDescriptor import get_sd_info, BlobStreamDescriptorReader from lbrynet.core import BlobManager from lbrynet.core import Session -from lbrynet.core.server import DHTHashAnnouncer +from lbrynet.dht import hashannouncer from lbrynet.file_manager import EncryptedFileCreator from lbrynet.file_manager import EncryptedFileManager from lbrynet.tests import mocks @@ -33,7 +33,7 @@ class CreateEncryptedFileTest(unittest.TestCase): self.session = mock.Mock(spec=Session.Session)(None, None) self.session.payment_rate_manager.min_blob_data_payment_rate = 0 - hash_announcer = DHTHashAnnouncer.DHTHashAnnouncer(None, None) + hash_announcer = hashannouncer.DHTHashAnnouncer(None, None) self.blob_manager = BlobManager.DiskBlobManager( hash_announcer, self.tmp_blob_dir, SQLiteStorage(self.tmp_db_dir)) self.session.blob_manager = self.blob_manager From e30ea50ef47f1198a6c5217a432a513d74242917 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Thu, 15 Feb 2018 16:49:00 -0500 Subject: [PATCH 14/52] more --- lbrynet/core/HashAnnouncer.py | 18 ----------------- lbrynet/core/PeerFinder.py | 19 ------------------ lbrynet/core/Session.py | 8 +++----- lbrynet/core/SinglePeerDownloader.py | 5 +++-- lbrynet/dht/hashannouncer.py | 22 ++++++++++++++++++++- lbrynet/dht/peerfinder.py | 18 ++++++++++++++++- lbrynet/tests/unit/core/test_BlobManager.py | 2 +- scripts/download_blob_from_peer.py | 2 +- scripts/encrypt_blob.py | 3 +-- scripts/reseed_file.py | 4 ++-- 10 files changed, 49 insertions(+), 52 deletions(-) delete mode 100644 lbrynet/core/HashAnnouncer.py delete mode 100644 lbrynet/core/PeerFinder.py diff --git a/lbrynet/core/HashAnnouncer.py b/lbrynet/core/HashAnnouncer.py deleted file mode 100644 index 5453eed15..000000000 --- a/lbrynet/core/HashAnnouncer.py +++ /dev/null @@ -1,18 +0,0 @@ -class DummyHashAnnouncer(object): - def __init__(self, *args): - pass - - def run_manage_loop(self): - pass - - def stop(self): - pass - - def add_supplier(self, *args): - pass - - def hash_queue_size(self): - return 0 - - def immediate_announce(self, *args): - pass diff --git a/lbrynet/core/PeerFinder.py b/lbrynet/core/PeerFinder.py deleted file mode 100644 index 3f2339de2..000000000 --- a/lbrynet/core/PeerFinder.py +++ /dev/null @@ -1,19 +0,0 @@ -from twisted.internet import defer - - -class DummyPeerFinder(object): - """This class finds peers which have announced to the DHT that they have certain blobs""" - def __init__(self): - pass - - def run_manage_loop(self): - pass - - def stop(self): - pass - - def find_peers_for_blob(self, blob_hash): - return defer.succeed([]) - - def get_most_popular_hashes(self, num_to_return): - return [] diff --git a/lbrynet/core/Session.py b/lbrynet/core/Session.py index cb82161bc..288a7d7d8 100644 --- a/lbrynet/core/Session.py +++ b/lbrynet/core/Session.py @@ -2,11 +2,9 @@ import logging import miniupnpc from twisted.internet import threads, defer from lbrynet.core.BlobManager import DiskBlobManager -from lbrynet.dht import node, peerfinder, peermanager +from lbrynet.dht import node, peerfinder, peermanager, hashannouncer from lbrynet.database.storage import SQLiteStorage from lbrynet.core.RateLimiter import RateLimiter -from lbrynet.core.HashAnnouncer import DummyHashAnnouncer -from lbrynet.dht.hashannouncer import DHTHashAnnouncer from lbrynet.core.utils import generate_id from lbrynet.core.PaymentRateManager import BasePaymentRateManager, NegotiatedPaymentRateManager from lbrynet.core.BlobAvailability import BlobAvailabilityTracker @@ -162,7 +160,7 @@ class Session(object): else: if self.hash_announcer is None and self.peer_port is not None: log.warning("The server has no way to advertise its available blobs.") - self.hash_announcer = DummyHashAnnouncer() + self.hash_announcer = hashannouncer.DummyHashAnnouncer() d.addCallback(lambda _: self._setup_other_components()) return d @@ -296,7 +294,7 @@ class Session(object): ) self.peer_finder = peerfinder.DHTPeerFinder(self.dht_node, self.peer_manager) if self.hash_announcer is None: - self.hash_announcer = DHTHashAnnouncer(self.dht_node, self.peer_port) + self.hash_announcer = hashannouncer.DHTHashAnnouncer(self.dht_node, self.peer_port) self.dht_node.startNetwork() diff --git a/lbrynet/core/SinglePeerDownloader.py b/lbrynet/core/SinglePeerDownloader.py index 85ade1bb6..9073e980b 100644 --- a/lbrynet/core/SinglePeerDownloader.py +++ b/lbrynet/core/SinglePeerDownloader.py @@ -6,13 +6,14 @@ from twisted.internet import defer, threads, reactor from lbrynet.blob import BlobFile from lbrynet.core.BlobManager import DiskBlobManager -from lbrynet.core.HashAnnouncer import DummyHashAnnouncer from lbrynet.core.RateLimiter import DummyRateLimiter from lbrynet.core.PaymentRateManager import OnlyFreePaymentsManager -from lbrynet.core.PeerFinder import DummyPeerFinder from lbrynet.core.client.BlobRequester import BlobRequester from lbrynet.core.client.StandaloneBlobDownloader import StandaloneBlobDownloader from lbrynet.core.client.ConnectionManager import ConnectionManager +from lbrynet.dht.hashannouncer import DummyHashAnnouncer +from lbrynet.dht.peerfinder import DummyPeerFinder + log = logging.getLogger(__name__) diff --git a/lbrynet/dht/hashannouncer.py b/lbrynet/dht/hashannouncer.py index d029f1f15..baf7a0667 100644 --- a/lbrynet/dht/hashannouncer.py +++ b/lbrynet/dht/hashannouncer.py @@ -10,7 +10,27 @@ from lbrynet.core import utils log = logging.getLogger(__name__) -class DHTHashAnnouncer(object): +class DummyHashAnnouncer(object): + def __init__(self): + pass + + def run_manage_loop(self): + pass + + def stop(self): + pass + + def add_supplier(self, supplier): + pass + + def hash_queue_size(self): + return 0 + + def immediate_announce(self, blob_hashes): + pass + + +class DHTHashAnnouncer(DummyHashAnnouncer): ANNOUNCE_CHECK_INTERVAL = 60 CONCURRENT_ANNOUNCERS = 5 diff --git a/lbrynet/dht/peerfinder.py b/lbrynet/dht/peerfinder.py index b4f94097d..9e4dd167d 100644 --- a/lbrynet/dht/peerfinder.py +++ b/lbrynet/dht/peerfinder.py @@ -10,7 +10,23 @@ from lbrynet.core.utils import short_hash log = logging.getLogger(__name__) -class DHTPeerFinder(object): +class DummyPeerFinder(object): + """This class finds peers which have announced to the DHT that they have certain blobs""" + + def run_manage_loop(self): + pass + + def stop(self): + pass + + def find_peers_for_blob(self, blob_hash): + return defer.succeed([]) + + def get_most_popular_hashes(self, num_to_return): + return [] + + +class DHTPeerFinder(DummyPeerFinder): """This class finds peers which have announced to the DHT that they have certain blobs""" implements(IPeerFinder) diff --git a/lbrynet/tests/unit/core/test_BlobManager.py b/lbrynet/tests/unit/core/test_BlobManager.py index 3f513f623..5bc118f92 100644 --- a/lbrynet/tests/unit/core/test_BlobManager.py +++ b/lbrynet/tests/unit/core/test_BlobManager.py @@ -8,7 +8,7 @@ from twisted.internet import defer, threads from lbrynet.tests.util import random_lbry_hash from lbrynet.core.BlobManager import DiskBlobManager -from lbrynet.core.HashAnnouncer import DummyHashAnnouncer +from lbrynet.dht.hashannouncer import DummyHashAnnouncer from lbrynet.database.storage import SQLiteStorage from lbrynet.core.Peer import Peer from lbrynet import conf diff --git a/scripts/download_blob_from_peer.py b/scripts/download_blob_from_peer.py index c5263d29d..dc688956d 100644 --- a/scripts/download_blob_from_peer.py +++ b/scripts/download_blob_from_peer.py @@ -14,7 +14,7 @@ from lbrynet.core import log_support, Wallet, Peer from lbrynet.core.SinglePeerDownloader import SinglePeerDownloader from lbrynet.core.StreamDescriptor import BlobStreamDescriptorReader from lbrynet.core.BlobManager import DiskBlobManager -from lbrynet.core.HashAnnouncer import DummyHashAnnouncer +from lbrynet.dht.hashannouncer import DummyHashAnnouncer log = logging.getLogger() diff --git a/scripts/encrypt_blob.py b/scripts/encrypt_blob.py index 3d3552f48..2b83dce3e 100644 --- a/scripts/encrypt_blob.py +++ b/scripts/encrypt_blob.py @@ -5,12 +5,11 @@ import sys from twisted.internet import defer from twisted.internet import reactor -from twisted.protocols import basic from twisted.web.client import FileBodyProducer from lbrynet import conf from lbrynet.core import log_support -from lbrynet.core.HashAnnouncer import DummyHashAnnouncer +from lbrynet.dht.hashannouncer import DummyHashAnnouncer from lbrynet.core.BlobManager import DiskBlobManager from lbrynet.cryptstream.CryptStreamCreator import CryptStreamCreator diff --git a/scripts/reseed_file.py b/scripts/reseed_file.py index 108ff2f00..0068ce5c8 100644 --- a/scripts/reseed_file.py +++ b/scripts/reseed_file.py @@ -17,7 +17,7 @@ from twisted.protocols import basic from lbrynet import conf from lbrynet.core import BlobManager -from lbrynet.core import HashAnnouncer +from lbrynet.dht import hashannouncer from lbrynet.core import log_support from lbrynet.cryptstream import CryptStreamCreator @@ -52,7 +52,7 @@ def reseed_file(input_file, sd_blob): sd_blob = SdBlob.new_instance(sd_blob) db_dir = conf.settings['data_dir'] blobfile_dir = os.path.join(db_dir, "blobfiles") - announcer = HashAnnouncer.DummyHashAnnouncer() + announcer = hashannouncer.DummyHashAnnouncer() blob_manager = BlobManager.DiskBlobManager(announcer, blobfile_dir, db_dir) yield blob_manager.setup() creator = CryptStreamCreator.CryptStreamCreator( From efaa97216f8c854f377906e6150a36162f6e9a97 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Thu, 15 Feb 2018 17:30:14 -0500 Subject: [PATCH 15/52] move dht node setup back into node class --- lbrynet/core/Session.py | 47 +++------------------------ lbrynet/dht/node.py | 72 ++++++++++++++++++++--------------------- 2 files changed, 39 insertions(+), 80 deletions(-) diff --git a/lbrynet/core/Session.py b/lbrynet/core/Session.py index 288a7d7d8..603917e7a 100644 --- a/lbrynet/core/Session.py +++ b/lbrynet/core/Session.py @@ -247,59 +247,20 @@ class Session(object): d.addErrback(upnp_failed) return d - # the callback, if any, will be invoked once the joining procedure - # has terminated - def join_dht(self, cb=None): - from twisted.internet import reactor - - def join_resolved_addresses(result): - addresses = [] - for success, value in result: - if success is True: - addresses.append(value) - return addresses - - @defer.inlineCallbacks - def join_network(knownNodes): - log.debug("join DHT using known nodes: " + str(knownNodes)) - result = yield self.dht_node.joinNetwork(knownNodes) - defer.returnValue(result) - - ds = [] - for host, port in self.known_dht_nodes: - d = reactor.resolve(host) - d.addCallback(lambda h: (h, port)) # match host to port - ds.append(d) - - dl = defer.DeferredList(ds) - dl.addCallback(join_resolved_addresses) - dl.addCallback(join_network) - if cb: - dl.addCallback(cb) - - return dl - + @defer.inlineCallbacks def _setup_dht(self): log.info("Starting DHT") - def start_dht(join_network_result): - self.hash_announcer.run_manage_loop() - return True - self.dht_node = self.dht_node_class( udpPort=self.dht_node_port, node_id=self.node_id, externalIP=self.external_ip, peerPort=self.peer_port ) - self.peer_finder = peerfinder.DHTPeerFinder(self.dht_node, self.peer_manager) - if self.hash_announcer is None: - self.hash_announcer = hashannouncer.DHTHashAnnouncer(self.dht_node, self.peer_port) - self.dht_node.startNetwork() - - # pass start_dht() as callback to start the remaining components after joining the DHT - return self.join_dht(start_dht) + yield self.dht_node.joinNetwork(self.known_dht_nodes) + self.peer_finder = self.dht_node.peer_finder + self.hash_announcer = self.dht_node.hash_announcer def _setup_other_components(self): log.debug("Setting up the rest of the components") diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index 13843ef51..489a16957 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -18,6 +18,9 @@ import routingtable import datastore import protocol +from peermanager import PeerManager +from hashannouncer import DHTHashAnnouncer +from peerfinder import DHTPeerFinder from contact import Contact from hashwatcher import HashWatcher from distance import Distance @@ -120,7 +123,12 @@ class Node(object): # will be used later self._can_store = True + self.peer_manager = PeerManager() + self.peer_finder = DHTPeerFinder(self, self.peer_manager) + self.hash_announcer = DHTHashAnnouncer(self, self.port) + def __del__(self): + log.warning("unclean shutdown of the dht node") if self._listeningPort is not None: self._listeningPort.stopListening() @@ -138,60 +146,50 @@ class Node(object): self._listeningPort.stopListening() self.hash_watcher.stop() - def startNetwork(self): - """ Causes the Node to start all the underlying components needed for the DHT - to work. This should be called before any other DHT operations. - """ - log.info("Starting DHT underlying components") - - # Prepare the underlying Kademlia protocol - if self.port is not None: - try: - self._listeningPort = reactor.listenUDP(self.port, self._protocol) - except error.CannotListenError as e: - import traceback - log.error("Couldn't bind to port %d. %s", self.port, traceback.format_exc()) - raise ValueError("%s lbrynet may already be running." % str(e)) - - # Start the token looping call - self.change_token_lc.start(constants.tokenSecretChangeInterval) - # #TODO: Refresh all k-buckets further away than this node's closest neighbour - # Start refreshing k-buckets periodically, if necessary - self.next_refresh_call = reactor.callLater(constants.checkRefreshInterval, - self._refreshNode) - self.hash_watcher.tick() - @defer.inlineCallbacks - def joinNetwork(self, knownNodeAddresses=None): + def joinNetwork(self, known_node_addresses=None): """ Causes the Node to attempt to join the DHT network by contacting the known DHT nodes. This can be called multiple times if the previous attempt has failed or if the Node has lost all the contacts. - @param knownNodeAddresses: A sequence of tuples containing IP address + @param known_node_addresses: A sequence of tuples containing IP address information for existing nodes on the Kademlia network, in the format: C{(, (udp port>)} - @type knownNodeAddresses: tuple + @type known_node_addresses: list """ + + try: + self._listeningPort = reactor.listenUDP(self.port, self._protocol) + log.info("DHT node listening on %i", self.port) + except error.CannotListenError as e: + import traceback + log.error("Couldn't bind to port %d. %s", self.port, traceback.format_exc()) + raise ValueError("%s lbrynet may already be running." % str(e)) + + known_node_addresses = known_node_addresses or [] + bootstrap_contacts = [] + for node_address, port in known_node_addresses: + host = yield reactor.resolve(node_address) + # Create temporary contact information for the list of addresses of known nodes + contact = Contact(self._generateID(), host, port, self._protocol) + bootstrap_contacts.append(contact) + log.info("Attempting to join the DHT network") - # IGNORE:E1101 - # Create temporary contact information for the list of addresses of known nodes - if knownNodeAddresses != None: - bootstrapContacts = [] - for address, port in knownNodeAddresses: - contact = Contact(self._generateID(), address, port, self._protocol) - bootstrapContacts.append(contact) - else: - bootstrapContacts = None - # Initiate the Kademlia joining sequence - perform a search for this node's own ID - self._joinDeferred = self._iterativeFind(self.node_id, bootstrapContacts) + self._joinDeferred = self._iterativeFind(self.node_id, bootstrap_contacts) # #TODO: Refresh all k-buckets further away than this node's closest neighbour # Start refreshing k-buckets periodically, if necessary self.hash_watcher.tick() yield self._joinDeferred + + self.change_token_lc.start(constants.tokenSecretChangeInterval) self.refresh_node_lc.start(constants.checkRefreshInterval) + self.peer_finder.run_manage_loop() + self.hash_announcer.run_manage_loop() + + #TODO: re-attempt joining the network if it fails @property def contacts(self): From 6666468640a9bb2798c86490a3ff3294feed2191 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 20 Feb 2018 13:30:56 -0500 Subject: [PATCH 16/52] add reactor arguments to Node -adds reactor (clock) and reactor functions listenUDP, callLater, and resolve as arguments to Node.__init__ -set the reactor clock on LoopingCalls to make them easily testable -convert callLater manage loops to LoopingCalls --- lbrynet/dht/hashannouncer.py | 13 +++++------ lbrynet/dht/hashwatcher.py | 20 ++++++++--------- lbrynet/dht/node.py | 42 +++++++++++++++++++++++++----------- lbrynet/dht/peerfinder.py | 4 ++-- lbrynet/dht/protocol.py | 9 ++++---- lbrynet/dht/routingtable.py | 15 +++++++------ 6 files changed, 62 insertions(+), 41 deletions(-) diff --git a/lbrynet/dht/hashannouncer.py b/lbrynet/dht/hashannouncer.py index baf7a0667..5338a9e7c 100644 --- a/lbrynet/dht/hashannouncer.py +++ b/lbrynet/dht/hashannouncer.py @@ -45,8 +45,9 @@ class DHTHashAnnouncer(DummyHashAnnouncer): self.hash_queue = collections.deque() self._concurrent_announcers = 0 self._manage_call_lc = task.LoopingCall(self.manage_lc) + self._manage_call_lc.clock = dht_node.clock self._lock = utils.DeferredLockContextManager(defer.DeferredLock()) - self._last_checked = time.time(), self.CONCURRENT_ANNOUNCERS + self._last_checked = dht_node.clock.seconds(), self.CONCURRENT_ANNOUNCERS self._total = None def run_manage_loop(self): @@ -58,7 +59,7 @@ class DHTHashAnnouncer(DummyHashAnnouncer): last_time, last_hashes = self._last_checked hashes = len(self.hash_queue) if hashes: - t, h = time.time() - last_time, last_hashes - hashes + t, h = self.dht_node.clock.seconds() - last_time, last_hashes - hashes blobs_per_second = float(h) / float(t) if blobs_per_second > 0: estimated_time_remaining = int(float(hashes) / blobs_per_second) @@ -108,7 +109,7 @@ class DHTHashAnnouncer(DummyHashAnnouncer): defer.returnValue(None) log.info('Announcing %s hashes', len(hashes)) # TODO: add a timeit decorator - start = time.time() + start = self.dht_node.clock.seconds() ds = [] with self._lock: @@ -174,9 +175,9 @@ class DHTHashAnnouncer(DummyHashAnnouncer): for _, announced_to in announcer_results: stored_to.update(announced_to) - log.info('Took %s seconds to announce %s hashes', time.time() - start, len(hashes)) - seconds_per_blob = (time.time() - start) / len(hashes) - self.supplier.set_single_hash_announce_duration(seconds_per_blob) + log.info('Took %s seconds to announce %s hashes', self.dht_node.clock.seconds() - start, len(hashes)) + seconds_per_blob = (self.dht_node.clock.seconds() - start) / len(hashes) + self.set_single_hash_announce_duration(seconds_per_blob) defer.returnValue(stored_to) diff --git a/lbrynet/dht/hashwatcher.py b/lbrynet/dht/hashwatcher.py index 3f9699de2..80aa30b6a 100644 --- a/lbrynet/dht/hashwatcher.py +++ b/lbrynet/dht/hashwatcher.py @@ -1,24 +1,22 @@ from collections import Counter import datetime +from twisted.internet import task, threads class HashWatcher(object): - def __init__(self): + def __init__(self, clock=None): + if not clock: + from twisted.internet import reactor as clock self.ttl = 600 self.hashes = [] - self.next_tick = None + self.lc = task.LoopingCall(self._remove_old_hashes) + self.lc.clock = clock - def tick(self): - - from twisted.internet import reactor - - self._remove_old_hashes() - self.next_tick = reactor.callLater(10, self.tick) + def start(self): + return self.lc.start(10) def stop(self): - if self.next_tick is not None: - self.next_tick.cancel() - self.next_tick = None + return self.lc.stop() def add_requested_hash(self, hashsum, contact): from_ip = contact.compact_ip diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index 489a16957..c5cc27678 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -11,7 +11,7 @@ import hashlib import operator import struct import time -from twisted.internet import defer, error, reactor, task +from twisted.internet import defer, error, task import constants import routingtable @@ -52,9 +52,11 @@ class Node(object): application is performed via this class (or a subclass). """ - def __init__(self, node_id=None, udpPort=4000, dataStore=None, + def __init__(self, hash_announcer=None, node_id=None, udpPort=4000, dataStore=None, routingTableClass=None, networkProtocol=None, - externalIP=None, peerPort=None): + externalIP=None, peerPort=None, listenUDP=None, + callLater=None, resolve=None, clock=None, peer_finder=None, + peer_manager=None): """ @param dataStore: The data store to use. This must be class inheriting from the C{DataStore} interface (or providing the @@ -79,6 +81,17 @@ class Node(object): @param externalIP: the IP at which this node can be contacted @param peerPort: the port at which this node announces it has a blob for """ + + if not listenUDP or not resolve or not callLater or not clock: + from twisted.internet import reactor + listenUDP = listenUDP or reactor.listenUDP + resolve = resolve or reactor.resolve + callLater = callLater or reactor.callLater + clock = clock or reactor + self.reactor_resolve = resolve + self.reactor_listenUDP = listenUDP + self.reactor_callLater = callLater + self.clock = clock self.node_id = node_id or self._generateID() self.port = udpPort self._listeningPort = None # object implementing Twisted @@ -89,12 +102,14 @@ class Node(object): # operations before the node has finished joining the network) self._joinDeferred = None self.change_token_lc = task.LoopingCall(self.change_token) + self.change_token_lc.clock = self.clock self.refresh_node_lc = task.LoopingCall(self._refreshNode) + self.refresh_node_lc.clock = self.clock # Create k-buckets (for storing contacts) if routingTableClass is None: - self._routingTable = routingtable.OptimizedTreeRoutingTable(self.node_id) + self._routingTable = routingtable.OptimizedTreeRoutingTable(self.node_id, self.clock.seconds) else: - self._routingTable = routingTableClass(self.node_id) + self._routingTable = routingTableClass(self.node_id, self.clock.seconds) # Initialize this node's network access mechanisms if networkProtocol is None: @@ -118,7 +133,7 @@ class Node(object): self._routingTable.addContact(contact) self.externalIP = externalIP self.peerPort = peerPort - self.hash_watcher = HashWatcher() + self.hash_watcher = HashWatcher(self.clock) # will be used later self._can_store = True @@ -136,15 +151,18 @@ class Node(object): def can_store(self): return self._can_store is True + @defer.inlineCallbacks def stop(self): + yield self.hash_announcer.stop() # stop LoopingCalls: if self.refresh_node_lc.running: - self.refresh_node_lc.stop() + yield self.refresh_node_lc.stop() if self.change_token_lc.running: - self.change_token_lc.stop() + yield self.change_token_lc.stop() if self._listeningPort is not None: - self._listeningPort.stopListening() - self.hash_watcher.stop() + yield self._listeningPort.stopListening() + if self.hash_watcher.lc.running: + yield self.hash_watcher.stop() @defer.inlineCallbacks def joinNetwork(self, known_node_addresses=None): @@ -183,10 +201,10 @@ class Node(object): # Start refreshing k-buckets periodically, if necessary self.hash_watcher.tick() yield self._joinDeferred + self.hash_watcher.start() self.change_token_lc.start(constants.tokenSecretChangeInterval) self.refresh_node_lc.start(constants.checkRefreshInterval) - self.peer_finder.run_manage_loop() self.hash_announcer.run_manage_loop() #TODO: re-attempt joining the network if it fails @@ -828,7 +846,7 @@ class _IterativeFindHelper(object): if self._should_lookup_active_calls(): # Schedule the next iteration if there are any active # calls (Kademlia uses loose parallelism) - call = reactor.callLater(constants.iterativeLookupDelay, self.searchIteration) + call = self.node.reactor_callLater(constants.iterativeLookupDelay, self.searchIteration) self.pending_iteration_calls.append(call) # Check for a quick contact response that made an update to the shortList elif prevShortlistLength < len(self.shortlist): diff --git a/lbrynet/dht/peerfinder.py b/lbrynet/dht/peerfinder.py index 9e4dd167d..afbbddd6b 100644 --- a/lbrynet/dht/peerfinder.py +++ b/lbrynet/dht/peerfinder.py @@ -2,7 +2,7 @@ import binascii import logging from zope.interface import implements -from twisted.internet import defer, reactor +from twisted.internet import defer from lbrynet.interfaces import IPeerFinder from lbrynet.core.utils import short_hash @@ -63,7 +63,7 @@ class DHTPeerFinder(DummyPeerFinder): finished_deferred = self.dht_node.getPeersForBlob(bin_hash) if timeout is not None: - reactor.callLater(timeout, _trigger_timeout) + self.dht_node.reactor_callLater(timeout, _trigger_timeout) try: peer_list = yield finished_deferred diff --git a/lbrynet/dht/protocol.py b/lbrynet/dht/protocol.py index 5656f29bb..d22f9dfa9 100644 --- a/lbrynet/dht/protocol.py +++ b/lbrynet/dht/protocol.py @@ -3,7 +3,7 @@ import time import socket import errno -from twisted.internet import protocol, defer, error, reactor, task +from twisted.internet import protocol, defer, error, task import constants import encoding @@ -47,6 +47,7 @@ class KademliaProtocol(protocol.DatagramProtocol): self._total_bytes_tx = 0 self._total_bytes_rx = 0 self._bandwidth_stats_update_lc = task.LoopingCall(self._update_bandwidth_stats) + self._bandwidth_stats_update_lc.clock = self._node.clock def _update_bandwidth_stats(self): recent_rx_history = {} @@ -168,7 +169,7 @@ class KademliaProtocol(protocol.DatagramProtocol): df._rpcRawResponse = True # Set the RPC timeout timer - timeoutCall = reactor.callLater(constants.rpcTimeout, self._msgTimeout, msg.id) + timeoutCall = self._node.reactor_callLater(constants.rpcTimeout, self._msgTimeout, msg.id) # Transmit the data self._send(encodedMsg, msg.id, (contact.address, contact.port)) self._sentMessages[msg.id] = (contact.id, df, timeoutCall, method, args) @@ -331,7 +332,7 @@ class KademliaProtocol(protocol.DatagramProtocol): """Schedule the sending of the next UDP packet """ delay = self._delay() key = object() - delayed_call = reactor.callLater(delay, self._write_and_remove, key, txData, address) + delayed_call = self._node.reactor_callLater(delay, self._write_and_remove, key, txData, address) self._call_later_list[key] = delayed_call def _write_and_remove(self, key, txData, address): @@ -428,7 +429,7 @@ class KademliaProtocol(protocol.DatagramProtocol): # See if any progress has been made; if not, kill the message if self._hasProgressBeenMade(messageID): # Reset the RPC timeout timer - timeoutCall = reactor.callLater(constants.rpcTimeout, self._msgTimeout, messageID) + timeoutCall = self._node.reactor_callLater(constants.rpcTimeout, self._msgTimeout, messageID) self._sentMessages[messageID] = (remoteContactID, df, timeoutCall, method, args) else: # No progress has been made diff --git a/lbrynet/dht/routingtable.py b/lbrynet/dht/routingtable.py index 863f37770..02f8e9686 100644 --- a/lbrynet/dht/routingtable.py +++ b/lbrynet/dht/routingtable.py @@ -5,7 +5,6 @@ # The docstrings in this module contain epytext markup; API documentation # may be created by processing this file with epydoc: http://epydoc.sf.net -import time import random from zope.interface import implements import constants @@ -34,7 +33,7 @@ class TreeRoutingTable(object): """ implements(IRoutingTable) - def __init__(self, parentNodeID): + def __init__(self, parentNodeID, getTime=None): """ @param parentNodeID: The n-bit node ID of the node to which this routing table belongs @@ -43,6 +42,9 @@ class TreeRoutingTable(object): # Create the initial (single) k-bucket covering the range of the entire n-bit ID space self._buckets = [kbucket.KBucket(rangeMin=0, rangeMax=2 ** constants.key_bits)] self._parentNodeID = parentNodeID + if not getTime: + from time import time as getTime + self._getTime = getTime def addContact(self, contact): """ Add the given contact to the correct k-bucket; if it already @@ -194,7 +196,7 @@ class TreeRoutingTable(object): bucketIndex = startIndex refreshIDs = [] for bucket in self._buckets[startIndex:]: - if force or (int(time.time()) - bucket.lastAccessed >= constants.refreshTimeout): + if force or (int(self._getTime()) - bucket.lastAccessed >= constants.refreshTimeout): searchID = self._randomIDInBucketRange(bucketIndex) refreshIDs.append(searchID) bucketIndex += 1 @@ -221,7 +223,7 @@ class TreeRoutingTable(object): @type key: str """ bucketIndex = self._kbucketIndex(key) - self._buckets[bucketIndex].lastAccessed = int(time.time()) + self._buckets[bucketIndex].lastAccessed = int(self._getTime()) def _kbucketIndex(self, key): """ Calculate the index of the k-bucket which is responsible for the @@ -289,8 +291,8 @@ class OptimizedTreeRoutingTable(TreeRoutingTable): of the 13-page version of the Kademlia paper. """ - def __init__(self, parentNodeID): - TreeRoutingTable.__init__(self, parentNodeID) + def __init__(self, parentNodeID, getTime=None): + TreeRoutingTable.__init__(self, parentNodeID, getTime) # Cache containing nodes eligible to replace stale k-bucket entries self._replacementCache = {} @@ -301,6 +303,7 @@ class OptimizedTreeRoutingTable(TreeRoutingTable): @param contact: The contact to add to this node's k-buckets @type contact: kademlia.contact.Contact """ + if contact.id == self._parentNodeID: return From e6caedac913d9bbb3d0fb3236ea3b51623ad56cc Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 20 Feb 2018 13:33:59 -0500 Subject: [PATCH 17/52] remove DHTHashSupplier class, move former functions into DHTHashAnnouncer --- lbrynet/core/BlobManager.py | 16 +++++----- lbrynet/dht/hashannouncer.py | 61 ++++++++++++++++++------------------ lbrynet/dht/node.py | 6 ++-- 3 files changed, 40 insertions(+), 43 deletions(-) diff --git a/lbrynet/core/BlobManager.py b/lbrynet/core/BlobManager.py index d0a37f403..8db8b4d05 100644 --- a/lbrynet/core/BlobManager.py +++ b/lbrynet/core/BlobManager.py @@ -1,16 +1,15 @@ import logging import os from sqlite3 import IntegrityError -from twisted.internet import threads, defer, reactor, task +from twisted.internet import threads, defer, task from lbrynet import conf from lbrynet.blob.blob_file import BlobFile from lbrynet.blob.creator import BlobFileCreator -from lbrynet.dht.hashannouncer import DHTHashSupplier log = logging.getLogger(__name__) -class DiskBlobManager(DHTHashSupplier): +class DiskBlobManager(object): def __init__(self, hash_announcer, blob_dir, storage): """ @@ -18,8 +17,7 @@ class DiskBlobManager(DHTHashSupplier): blob_dir - directory where blobs are stored db_dir - directory where sqlite database of blob information is stored """ - - DHTHashSupplier.__init__(self, hash_announcer) + self.hash_announcer = hash_announcer self.storage = storage self.announce_head_blobs_only = conf.settings['announce_head_blobs_only'] self.blob_dir = blob_dir @@ -70,14 +68,14 @@ class DiskBlobManager(DHTHashSupplier): @defer.inlineCallbacks def blob_completed(self, blob, next_announce_time=None, should_announce=True): if next_announce_time is None: - next_announce_time = self.get_next_announce_time() + next_announce_time = self.hash_announcer.get_next_announce_time() yield self.storage.add_completed_blob( blob.blob_hash, blob.length, next_announce_time, should_announce ) # we announce all blobs immediately, if announce_head_blob_only is False # otherwise, announce only if marked as should_announce if not self.announce_head_blobs_only or should_announce: - reactor.callLater(0, self.immediate_announce, [blob.blob_hash]) + self.immediate_announce([blob.blob_hash]) def completed_blobs(self, blobhashes_to_check): return self._completed_blobs(blobhashes_to_check) @@ -93,7 +91,7 @@ class DiskBlobManager(DHTHashSupplier): blob = self.blobs[blob_hash] if blob.get_is_verified(): return self.storage.set_should_announce( - blob_hash, self.get_next_announce_time(), should_announce + blob_hash, self.hash_announcer.get_next_announce_time(), should_announce ) return defer.succeed(False) @@ -110,7 +108,7 @@ class DiskBlobManager(DHTHashSupplier): raise Exception("Blob has a length of 0") new_blob = BlobFile(self.blob_dir, blob_creator.blob_hash, blob_creator.length) self.blobs[blob_creator.blob_hash] = new_blob - next_announce_time = self.get_next_announce_time() + next_announce_time = self.hash_announcer.get_next_announce_time() return self.blob_completed(new_blob, next_announce_time, should_announce) def immediate_announce_all_blobs(self): diff --git a/lbrynet/dht/hashannouncer.py b/lbrynet/dht/hashannouncer.py index 5338a9e7c..a1533947a 100644 --- a/lbrynet/dht/hashannouncer.py +++ b/lbrynet/dht/hashannouncer.py @@ -20,27 +20,32 @@ class DummyHashAnnouncer(object): def stop(self): pass - def add_supplier(self, supplier): - pass - def hash_queue_size(self): return 0 def immediate_announce(self, blob_hashes): pass + def get_next_announce_time(self): + return 0 + class DHTHashAnnouncer(DummyHashAnnouncer): ANNOUNCE_CHECK_INTERVAL = 60 CONCURRENT_ANNOUNCERS = 5 + # 1 hour is the min time hash will be reannounced + MIN_HASH_REANNOUNCE_TIME = 60 * 60 + # conservative assumption of the time it takes to announce + # a single hash + DEFAULT_SINGLE_HASH_ANNOUNCE_DURATION = 1 + """This class announces to the DHT that this peer has certain blobs""" STORE_RETRIES = 3 - def __init__(self, dht_node, peer_port): + def __init__(self, dht_node): self.dht_node = dht_node - self.peer_port = peer_port - self.supplier = None + self.peer_port = dht_node.peerPort self.next_manage_call = None self.hash_queue = collections.deque() self._concurrent_announcers = 0 @@ -49,6 +54,8 @@ class DHTHashAnnouncer(DummyHashAnnouncer): self._lock = utils.DeferredLockContextManager(defer.DeferredLock()) self._last_checked = dht_node.clock.seconds(), self.CONCURRENT_ANNOUNCERS self._total = None + self.single_hash_announce_duration = self.DEFAULT_SINGLE_HASH_ANNOUNCE_DURATION + self._hashes_to_announce = [] def run_manage_loop(self): log.info("Starting hash announcer") @@ -79,10 +86,7 @@ class DHTHashAnnouncer(DummyHashAnnouncer): def stop(self): log.info("Stopping DHT hash announcer.") if self._manage_call_lc.running: - self._manage_call_lc.stop() - - def add_supplier(self, supplier): - self.supplier = supplier + return self._manage_call_lc.stop() def immediate_announce(self, blob_hashes): if self.peer_port is not None: @@ -96,9 +100,8 @@ class DHTHashAnnouncer(DummyHashAnnouncer): @defer.inlineCallbacks def _announce_available_hashes(self): log.debug('Announcing available hashes') - if self.supplier: - hashes = yield self.supplier.hashes_to_announce() - yield self._announce_hashes(hashes) + hashes = yield self.hashes_to_announce() + yield self._announce_hashes(hashes) @defer.inlineCallbacks def _announce_hashes(self, hashes, immediate=False): @@ -180,24 +183,20 @@ class DHTHashAnnouncer(DummyHashAnnouncer): self.set_single_hash_announce_duration(seconds_per_blob) defer.returnValue(stored_to) + @defer.inlineCallbacks + def add_hashes_to_announce(self, blob_hashes): + yield self._lock._lock.acquire() + self._hashes_to_announce.extend(blob_hashes) + yield self._lock._lock.release() -class DHTHashSupplier(object): - # 1 hour is the min time hash will be reannounced - MIN_HASH_REANNOUNCE_TIME = 60 * 60 - # conservative assumption of the time it takes to announce - # a single hash - DEFAULT_SINGLE_HASH_ANNOUNCE_DURATION = 1 - - """Classes derived from this class give hashes to a hash announcer""" - - def __init__(self, announcer): - if announcer is not None: - announcer.add_supplier(self) - self.hash_announcer = announcer - self.single_hash_announce_duration = self.DEFAULT_SINGLE_HASH_ANNOUNCE_DURATION - + @defer.inlineCallbacks def hashes_to_announce(self): - pass + hashes_to_announce = [] + yield self._lock._lock.acquire() + while self._hashes_to_announce: + hashes_to_announce.append(self._hashes_to_announce.pop()) + yield self._lock._lock.release() + defer.returnValue(hashes_to_announce) def set_single_hash_announce_duration(self, seconds): """ @@ -221,7 +220,7 @@ class DHTHashSupplier(object): Returns: timestamp for next announce time """ - queue_size = self.hash_announcer.hash_queue_size() + num_hashes_to_announce + queue_size = self.hash_queue_size() + num_hashes_to_announce reannounce = max(self.MIN_HASH_REANNOUNCE_TIME, queue_size * self.single_hash_announce_duration) - return time.time() + reannounce + return self.dht_node.clock.seconds() + reannounce diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index c5cc27678..3150a0226 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -138,9 +138,9 @@ class Node(object): # will be used later self._can_store = True - self.peer_manager = PeerManager() - self.peer_finder = DHTPeerFinder(self, self.peer_manager) - self.hash_announcer = DHTHashAnnouncer(self, self.port) + self.peer_manager = peer_manager or PeerManager() + self.peer_finder = peer_finder or DHTPeerFinder(self, self.peer_manager) + self.hash_announcer = hash_announcer or DHTHashAnnouncer(self) def __del__(self): log.warning("unclean shutdown of the dht node") From 04e76443c699dcc6056cb6e74c2fef4f84f1f0e1 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 20 Feb 2018 13:35:03 -0500 Subject: [PATCH 18/52] move dht node component setup from Session into Node --- lbrynet/core/Session.py | 47 +++++++++-------------------------------- 1 file changed, 10 insertions(+), 37 deletions(-) diff --git a/lbrynet/core/Session.py b/lbrynet/core/Session.py index 603917e7a..436b46c52 100644 --- a/lbrynet/core/Session.py +++ b/lbrynet/core/Session.py @@ -2,7 +2,7 @@ import logging import miniupnpc from twisted.internet import threads, defer from lbrynet.core.BlobManager import DiskBlobManager -from lbrynet.dht import node, peerfinder, peermanager, hashannouncer +from lbrynet.dht import node, peermanager, hashannouncer from lbrynet.database.storage import SQLiteStorage from lbrynet.core.RateLimiter import RateLimiter from lbrynet.core.utils import generate_id @@ -97,38 +97,26 @@ class Session(object): """ self.db_dir = db_dir - self.node_id = node_id - self.peer_manager = peer_manager - + self.peer_finder = peer_finder + self.hash_announcer = hash_announcer self.dht_node_port = dht_node_port self.known_dht_nodes = known_dht_nodes if self.known_dht_nodes is None: self.known_dht_nodes = [] - self.peer_finder = peer_finder - self.hash_announcer = hash_announcer - self.blob_dir = blob_dir self.blob_manager = blob_manager - self.blob_tracker = None self.blob_tracker_class = blob_tracker_class or BlobAvailabilityTracker - self.peer_port = peer_port - self.use_upnp = use_upnp - self.rate_limiter = rate_limiter - self.external_ip = external_ip - self.upnp_redirects = [] - self.wallet = wallet self.dht_node_class = dht_node_class self.dht_node = None - self.base_payment_rate_manager = BasePaymentRateManager(blob_data_payment_rate) self.payment_rate_manager = None self.payment_rate_manager_class = payment_rate_manager_class or NegotiatedPaymentRateManager @@ -147,21 +135,11 @@ class Session(object): from lbrynet.core.PTCWallet import PTCWallet self.wallet = PTCWallet(self.db_dir) - if self.peer_manager is None: - self.peer_manager = peermanager.PeerManager() - if self.use_upnp is True: d = self._try_upnp() else: d = defer.succeed(True) - - if self.peer_finder is None: - d.addCallback(lambda _: self._setup_dht()) - else: - if self.hash_announcer is None and self.peer_port is not None: - log.warning("The server has no way to advertise its available blobs.") - self.hash_announcer = hashannouncer.DummyHashAnnouncer() - + d.addCallback(lambda _: self._setup_dht()) d.addCallback(lambda _: self._setup_other_components()) return d @@ -175,10 +153,6 @@ class Session(object): ds.append(defer.maybeDeferred(self.dht_node.stop)) if self.rate_limiter is not None: ds.append(defer.maybeDeferred(self.rate_limiter.stop)) - if self.peer_finder is not None: - ds.append(defer.maybeDeferred(self.peer_finder.stop)) - if self.hash_announcer is not None: - ds.append(defer.maybeDeferred(self.hash_announcer.stop)) if self.wallet is not None: ds.append(defer.maybeDeferred(self.wallet.stop)) if self.blob_manager is not None: @@ -250,17 +224,16 @@ class Session(object): @defer.inlineCallbacks def _setup_dht(self): log.info("Starting DHT") - self.dht_node = self.dht_node_class( + self.hash_announcer, udpPort=self.dht_node_port, node_id=self.node_id, externalIP=self.external_ip, - peerPort=self.peer_port + peerPort=self.peer_port, + peer_manager=self.peer_manager, + peer_finder=self.peer_finder, ) - yield self.dht_node.joinNetwork(self.known_dht_nodes) - self.peer_finder = self.dht_node.peer_finder - self.hash_announcer = self.dht_node.hash_announcer def _setup_other_components(self): log.debug("Setting up the rest of the components") @@ -274,12 +247,12 @@ class Session(object): "TempBlobManager is no longer supported, specify BlobManager or db_dir") else: self.blob_manager = DiskBlobManager( - self.hash_announcer, self.blob_dir, self.storage + self.dht_node.hash_announcer, self.blob_dir, self.storage ) if self.blob_tracker is None: self.blob_tracker = self.blob_tracker_class( - self.blob_manager, self.peer_finder, self.dht_node + self.blob_manager, self.dht_node.peer_finder, self.dht_node ) if self.payment_rate_manager is None: self.payment_rate_manager = self.payment_rate_manager_class( From 43896c8d17afb01daede406968974029566d6d01 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 20 Feb 2018 13:37:02 -0500 Subject: [PATCH 19/52] refactor joinNetwork into smaller functions -try to re-join network if no contacts are known --- lbrynet/dht/node.py | 67 ++++++++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 25 deletions(-) diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index 3150a0226..73e639688 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -100,7 +100,7 @@ class Node(object): # information from the DHT as soon as the node is part of the # network (add callbacks to this deferred if scheduling such # operations before the node has finished joining the network) - self._joinDeferred = None + self._joinDeferred = defer.Deferred(None) self.change_token_lc = task.LoopingCall(self.change_token) self.change_token_lc.clock = self.clock self.refresh_node_lc = task.LoopingCall(self._refreshNode) @@ -164,6 +164,45 @@ class Node(object): if self.hash_watcher.lc.running: yield self.hash_watcher.stop() + def start_listening(self): + try: + self._listeningPort = self.reactor_listenUDP(self.port, self._protocol) + log.info("DHT node listening on %i", self.port) + except error.CannotListenError as e: + import traceback + log.error("Couldn't bind to port %d. %s", self.port, traceback.format_exc()) + raise ValueError("%s lbrynet may already be running." % str(e)) + + def bootstrap_join(self, known_node_addresses, finished_d): + @defer.inlineCallbacks + def _resolve_seeds(): + bootstrap_contacts = [] + for node_address, port in known_node_addresses: + host = yield self.reactor_resolve(node_address) + # Create temporary contact information for the list of addresses of known nodes + contact = Contact(self._generateID(), host, port, self._protocol) + bootstrap_contacts.append(contact) + if not bootstrap_contacts: + if not self.hasContacts(): + log.warning("No known contacts!") + else: + log.info("found contacts") + bootstrap_contacts = self.contacts + defer.returnValue(bootstrap_contacts) + + def _rerun(bootstrap_contacts): + if not bootstrap_contacts: + log.info("Failed to join the dht, re-attempting in 60 seconds") + self.reactor_callLater(60, self.bootstrap_join, known_node_addresses, finished_d) + elif not finished_d.called: + finished_d.callback(bootstrap_contacts) + + log.info("Attempting to join the DHT network") + d = _resolve_seeds() + # Initiate the Kademlia joining sequence - perform a search for this node's own ID + d.addCallback(lambda contacts: self._iterativeFind(self.node_id, contacts)) + d.addCallback(_rerun) + @defer.inlineCallbacks def joinNetwork(self, known_node_addresses=None): """ Causes the Node to attempt to join the DHT network by contacting the @@ -177,38 +216,16 @@ class Node(object): @type known_node_addresses: list """ - try: - self._listeningPort = reactor.listenUDP(self.port, self._protocol) - log.info("DHT node listening on %i", self.port) - except error.CannotListenError as e: - import traceback - log.error("Couldn't bind to port %d. %s", self.port, traceback.format_exc()) - raise ValueError("%s lbrynet may already be running." % str(e)) - - known_node_addresses = known_node_addresses or [] - bootstrap_contacts = [] - for node_address, port in known_node_addresses: - host = yield reactor.resolve(node_address) - # Create temporary contact information for the list of addresses of known nodes - contact = Contact(self._generateID(), host, port, self._protocol) - bootstrap_contacts.append(contact) - - log.info("Attempting to join the DHT network") - - # Initiate the Kademlia joining sequence - perform a search for this node's own ID - self._joinDeferred = self._iterativeFind(self.node_id, bootstrap_contacts) + self.start_listening() # #TODO: Refresh all k-buckets further away than this node's closest neighbour # Start refreshing k-buckets periodically, if necessary - self.hash_watcher.tick() + self.bootstrap_join(known_node_addresses or [], self._joinDeferred) yield self._joinDeferred self.hash_watcher.start() - self.change_token_lc.start(constants.tokenSecretChangeInterval) self.refresh_node_lc.start(constants.checkRefreshInterval) self.hash_announcer.run_manage_loop() - #TODO: re-attempt joining the network if it fails - @property def contacts(self): def _inner(): From 16fcc3f5c1b59d797e36d81e5322fc8f2e8bcb65 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 20 Feb 2018 13:39:14 -0500 Subject: [PATCH 20/52] findValue inlinecallbacks refactor --- lbrynet/dht/node.py | 41 +++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index 73e639688..8b992eacd 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -418,33 +418,26 @@ class Node(object): to the specified key @rtype: twisted.internet.defer.Deferred """ - # Prepare a callback for this operation - outerDf = defer.Deferred() - - def checkResult(result): - if isinstance(result, dict): - # We have found the value; now see who was the closest contact without it... - # ...and store the key/value pair - outerDf.callback(result) - else: - # The value wasn't found, but a list of contacts was returned - # Now, see if we have the value (it might seem wasteful to search on the network - # first, but it ensures that all values are properly propagated through the - # network - if self._dataStore.hasPeersForBlob(key): - # Ok, we have the value locally, so use that - peers = self._dataStore.getPeersForBlob(key) - # Send this value to the closest node without it - outerDf.callback({key: peers}) - else: - # Ok, value does not exist in DHT at all - outerDf.callback(result) # Execute the search iterative_find_result = yield self._iterativeFind(key, rpc='findValue') - checkResult(iterative_find_result) - result = yield outerDf - defer.returnValue(result) + if isinstance(iterative_find_result, dict): + # We have found the value; now see who was the closest contact without it... + # ...and store the key/value pair + defer.returnValue(iterative_find_result) + else: + # The value wasn't found, but a list of contacts was returned + # Now, see if we have the value (it might seem wasteful to search on the network + # first, but it ensures that all values are properly propagated through the + # network + if self._dataStore.hasPeersForBlob(key): + # Ok, we have the value locally, so use that + # Send this value to the closest node without it + peers = self._dataStore.getPeersForBlob(key) + defer.returnValue({key: peers}) + else: + # Ok, value does not exist in DHT at all + defer.returnValue(iterative_find_result) def addContact(self, contact): """ Add/update the given contact; simple wrapper for the same method From bdba26322466aeac2343ada43ffb8b331a49acb5 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 20 Feb 2018 13:40:02 -0500 Subject: [PATCH 21/52] catch TimeoutError in _IterativeFindHelper --- lbrynet/dht/node.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index 8b992eacd..1a7985c66 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -17,6 +17,7 @@ import constants import routingtable import datastore import protocol +from error import TimeoutError from peermanager import PeerManager from hashannouncer import DHTHashAnnouncer @@ -799,7 +800,7 @@ class _IterativeFindHelper(object): def removeFromShortlist(self, failure, deadContactID): """ @type failure: twisted.python.failure.Failure """ - failure.trap(protocol.TimeoutError) + failure.trap(TimeoutError, defer.CancelledError, TypeError) if len(deadContactID) != constants.key_bits / 8: raise ValueError("invalid lbry id") if deadContactID in self.shortlist: From df78f7ff9fb66c44a2aaa7a3dcd042ac857a0681 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 20 Feb 2018 13:41:54 -0500 Subject: [PATCH 22/52] add response assertion to announce_to_peer --- lbrynet/dht/node.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index 1a7985c66..eb595b130 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -304,22 +304,22 @@ class Node(object): if not responseMsg.nodeID in known_nodes: log.warning("Responding node was not expected") defer.returnValue(responseMsg.nodeID) - n = known_nodes[responseMsg.nodeID] + remote_node = known_nodes[responseMsg.nodeID] result = responseMsg.response announced = False if 'token' in result: value['token'] = result['token'] try: - res = yield n.store(blob_hash, value, self.node_id) - log.info("Response to store request: %s", str(res)) + res = yield remote_node.store(blob_hash, value) + assert res == "OK", "unexpected response: {}".format(res) announced = True except protocol.TimeoutError: log.info("Timeout while storing blob_hash %s at %s", - blob_hash.encode('hex')[:16], n.id.encode('hex')) + blob_hash.encode('hex')[:16], remote_node.id.encode('hex')) except Exception as err: log.error("Unexpected error while storing blob_hash %s at %s: %s", - blob_hash.encode('hex')[:16], n.id.encode('hex'), err) + blob_hash.encode('hex')[:16], remote_node.id.encode('hex'), err) else: log.warning("missing token") defer.returnValue(announced) From b4bc5e2110efbe6797661987bec78eab91ce2083 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 20 Feb 2018 13:42:49 -0500 Subject: [PATCH 23/52] cancel callLater on error or timeout --- lbrynet/dht/protocol.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lbrynet/dht/protocol.py b/lbrynet/dht/protocol.py index d22f9dfa9..a21585017 100644 --- a/lbrynet/dht/protocol.py +++ b/lbrynet/dht/protocol.py @@ -173,6 +173,13 @@ class KademliaProtocol(protocol.DatagramProtocol): # Transmit the data self._send(encodedMsg, msg.id, (contact.address, contact.port)) self._sentMessages[msg.id] = (contact.id, df, timeoutCall, method, args) + + def cancel(err): + if timeoutCall.cancelled or timeoutCall.called: + return err + timeoutCall.cancel() + + df.addErrback(cancel) return df def startProtocol(self): @@ -336,7 +343,8 @@ class KademliaProtocol(protocol.DatagramProtocol): self._call_later_list[key] = delayed_call def _write_and_remove(self, key, txData, address): - del self._call_later_list[key] + if key in self._call_later_list: + del self._call_later_list[key] if self.transport: try: self.transport.write(txData, address) From 87c69742cd8b47a6dd6cb3d639d917b283309f16 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 20 Feb 2018 13:43:36 -0500 Subject: [PATCH 24/52] log packet encoding errors and warn if the transport is not connected --- lbrynet/dht/protocol.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lbrynet/dht/protocol.py b/lbrynet/dht/protocol.py index a21585017..4dc2ba176 100644 --- a/lbrynet/dht/protocol.py +++ b/lbrynet/dht/protocol.py @@ -214,8 +214,9 @@ class KademliaProtocol(protocol.DatagramProtocol): try: msgPrimitive = self._encoder.decode(datagram) message = self._translator.fromPrimitive(msgPrimitive) - except (encoding.DecodeError, ValueError): + except (encoding.DecodeError, ValueError) as err: # We received some rubbish here + log.exception("Decode error: %s", err) return except (IndexError, KeyError): log.warning("Couldn't decode dht datagram from %s", address) @@ -359,6 +360,8 @@ class KademliaProtocol(protocol.DatagramProtocol): else: log.error("DHT socket error: %s (%i)", err.message, err.errno) raise err + else: + log.warning("transport not connected!") def _sendResponse(self, contact, rpcID, response): """ Send a RPC response to the specified contact From 2e30ce9ae5af5b5600a0be16a5642b235aa9df85 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 20 Feb 2018 13:46:17 -0500 Subject: [PATCH 25/52] add mock udp transport layer --- lbrynet/tests/mocks.py | 102 +++++++++++++++++++++++++++++++++++++++-- lbrynet/tests/util.py | 35 ++++++++++++++ 2 files changed, 132 insertions(+), 5 deletions(-) diff --git a/lbrynet/tests/mocks.py b/lbrynet/tests/mocks.py index 1d719548b..70539ed83 100644 --- a/lbrynet/tests/mocks.py +++ b/lbrynet/tests/mocks.py @@ -1,12 +1,15 @@ +import struct import io from Crypto.PublicKey import RSA -from twisted.internet import defer +from twisted.internet import defer, threads, error from lbrynet.core import PTCWallet from lbrynet.core import BlobAvailability +from lbrynet.core.utils import generate_id from lbrynet.daemon import ExchangeRateManager as ERM from lbrynet import conf +from util import debug_kademlia_packet KB = 2**10 @@ -18,15 +21,18 @@ class FakeLBRYFile(object): self.stream_hash = stream_hash self.file_name = 'fake_lbry_file' + class Node(object): - def __init__(self, *args, **kwargs): - pass + def __init__(self, hash_announcer, peer_finder=None, peer_manager=None, **kwargs): + self.hash_announcer = hash_announcer + self.peer_finder = peer_finder + self.peer_manager = peer_manager def joinNetwork(self, *args): - pass + return defer.succeed(True) def stop(self): - pass + return defer.succeed(None) class FakeNetwork(object): @@ -164,6 +170,9 @@ class Announcer(object): def stop(self): pass + def get_next_announce_time(self): + return 0 + class GenFile(io.RawIOBase): def __init__(self, size, pattern): @@ -313,4 +322,87 @@ def mock_conf_settings(obj, settings={}): obj.addCleanup(_reset_settings) +MOCK_DHT_NODES = [ + "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", + "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF", +] +MOCK_DHT_SEED_DNS = { # these map to mock nodes 0, 1, and 2 + "lbrynet1.lbry.io": "10.42.42.1", + "lbrynet2.lbry.io": "10.42.42.2", + "lbrynet3.lbry.io": "10.42.42.3", +} + + +def resolve(name, timeout=(1, 3, 11, 45)): + if name not in MOCK_DHT_SEED_DNS: + return defer.fail(error.DNSLookupError(name)) + return defer.succeed(MOCK_DHT_SEED_DNS[name]) + + +class MockUDPTransport(object): + def __init__(self, address, port, max_packet_size, protocol): + self.address = address + self.port = port + self.max_packet_size = max_packet_size + self._node = protocol._node + + def write(self, data, address): + dest = MockNetwork.protocols[address][0] + debug_kademlia_packet(data, (self.address, self.port), address, self._node) + dest.datagramReceived(data, (self.address, self.port)) + + +class MockUDPPort(object): + def __init__(self, protocol): + self.protocol = protocol + + def startListening(self, reason=None): + return self.protocol.startProtocol() + + def stopListening(self, reason=None): + return self.protocol.stopProtocol() + + +class MockNetwork(object): + protocols = {} # (interface, port): (protocol, max_packet_size) + + @classmethod + def add_peer(cls, port, protocol, interface, maxPacketSize): + interface = protocol._node.externalIP + protocol.transport = MockUDPTransport(interface, port, maxPacketSize, protocol) + cls.protocols[(interface, port)] = (protocol, maxPacketSize) + + +def listenUDP(port, protocol, interface='', maxPacketSize=8192): + MockNetwork.add_peer(port, protocol, interface, maxPacketSize) + return MockUDPPort(protocol) + + +def address_generator(address=(10, 42, 42, 1)): + def increment(addr): + value = struct.unpack("I", "".join([chr(x) for x in list(addr)[::-1]]))[0] + 1 + new_addr = [] + for i in range(4): + new_addr.append(value % 256) + value >>= 8 + return tuple(new_addr[::-1]) + + while True: + yield "{}.{}.{}.{}".format(*address) + address = increment(address) + + +def mock_node_generator(count=None, mock_node_ids=MOCK_DHT_NODES): + if mock_node_ids is None: + mock_node_ids = MOCK_DHT_NODES + + for num, node_ip in enumerate(address_generator()): + if count and num >= count: + break + if num >= len(mock_node_ids): + node_id = generate_id().encode('hex') + else: + node_id = mock_node_ids[num] + yield (node_id, node_ip) diff --git a/lbrynet/tests/util.py b/lbrynet/tests/util.py index 43cb007ea..cc4c7da78 100644 --- a/lbrynet/tests/util.py +++ b/lbrynet/tests/util.py @@ -5,24 +5,36 @@ import os import tempfile import shutil import mock +import logging +from lbrynet.dht.encoding import Bencode +from lbrynet.dht.error import DecodeError +from lbrynet.dht.msgformat import DefaultFormat +from lbrynet.dht.msgtypes import ResponseMessage, RequestMessage, ErrorMessage +_encode = Bencode() +_datagram_formatter = DefaultFormat() DEFAULT_TIMESTAMP = datetime.datetime(2016, 1, 1) DEFAULT_ISO_TIME = time.mktime(DEFAULT_TIMESTAMP.timetuple()) +log = logging.getLogger("lbrynet.tests.util") + def mk_db_and_blob_dir(): db_dir = tempfile.mkdtemp() blob_dir = tempfile.mkdtemp() return db_dir, blob_dir + def rm_db_and_blob_dir(db_dir, blob_dir): shutil.rmtree(db_dir, ignore_errors=True) shutil.rmtree(blob_dir, ignore_errors=True) + def random_lbry_hash(): return binascii.b2a_hex(os.urandom(48)) + def resetTime(test_case, timestamp=DEFAULT_TIMESTAMP): iso_time = time.mktime(timestamp.timetuple()) patcher = mock.patch('time.time') @@ -37,5 +49,28 @@ def resetTime(test_case, timestamp=DEFAULT_TIMESTAMP): patcher.start().return_value = timestamp test_case.addCleanup(patcher.stop) + def is_android(): return 'ANDROID_ARGUMENT' in os.environ # detect Android using the Kivy way + + +def debug_kademlia_packet(data, source, destination, node): + if log.level != logging.DEBUG: + return + try: + packet = _datagram_formatter.fromPrimitive(_encode.decode(data)) + if isinstance(packet, RequestMessage): + log.debug("request %s --> %s %s (node time %s)", source[0], destination[0], packet.request, + node.clock.seconds()) + elif isinstance(packet, ResponseMessage): + if isinstance(packet.response, (str, unicode)): + log.debug("response %s <-- %s %s (node time %s)", destination[0], source[0], packet.response, + node.clock.seconds()) + else: + log.debug("response %s <-- %s %i contacts (node time %s)", destination[0], source[0], + len(packet.response), node.clock.seconds()) + elif isinstance(packet, ErrorMessage): + log.error("error %s <-- %s %s (node time %s)", destination[0], source[0], packet.exceptionType, + node.clock.seconds()) + except DecodeError: + log.exception("decode error %s --> %s (node time %s)", source[0], destination[0], node.clock.seconds()) From 0ab5dd28bcfd876fa5858015d7ba941dee478b91 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 20 Feb 2018 13:46:46 -0500 Subject: [PATCH 26/52] update tests --- lbrynet/tests/functional/test_reflector.py | 9 ++++++-- lbrynet/tests/functional/test_streamify.py | 4 ++-- .../unit/core/server/test_DHTHashAnnouncer.py | 23 ++++--------------- lbrynet/tests/unit/dht/test_node.py | 2 +- .../test_EncryptedFileCreator.py | 8 +++---- 5 files changed, 19 insertions(+), 27 deletions(-) diff --git a/lbrynet/tests/functional/test_reflector.py b/lbrynet/tests/functional/test_reflector.py index 0250607d4..4ba01dd4b 100644 --- a/lbrynet/tests/functional/test_reflector.py +++ b/lbrynet/tests/functional/test_reflector.py @@ -15,6 +15,8 @@ from lbrynet.file_manager import EncryptedFileManager from lbrynet.tests import mocks from lbrynet.tests.util import mk_db_and_blob_dir, rm_db_and_blob_dir +from lbrynet.tests.mocks import Node + class TestReflector(unittest.TestCase): def setUp(self): @@ -61,7 +63,8 @@ class TestReflector(unittest.TestCase): use_upnp=False, wallet=wallet, blob_tracker_class=mocks.BlobAvailabilityTracker, - external_ip="127.0.0.1" + external_ip="127.0.0.1", + dht_node_class=Node ) self.lbry_file_manager = EncryptedFileManager.EncryptedFileManager(self.session, @@ -80,7 +83,8 @@ class TestReflector(unittest.TestCase): use_upnp=False, wallet=wallet, blob_tracker_class=mocks.BlobAvailabilityTracker, - external_ip="127.0.0.1" + external_ip="127.0.0.1", + dht_node_class=Node ) self.server_blob_manager = BlobManager.DiskBlobManager(hash_announcer, @@ -364,6 +368,7 @@ class TestReflector(unittest.TestCase): d.addCallback(lambda _: verify_stream_on_reflector()) return d + def iv_generator(): iv = 0 while True: diff --git a/lbrynet/tests/functional/test_streamify.py b/lbrynet/tests/functional/test_streamify.py index 6d87d382a..ed0f82c9f 100644 --- a/lbrynet/tests/functional/test_streamify.py +++ b/lbrynet/tests/functional/test_streamify.py @@ -67,7 +67,7 @@ class TestStreamify(TestCase): blob_dir=self.blob_dir, peer_port=5553, use_upnp=False, rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker, - is_generous=self.is_generous, external_ip="127.0.0.1" + is_generous=self.is_generous, external_ip="127.0.0.1", dht_node_class=mocks.Node ) self.lbry_file_manager = EncryptedFileManager(self.session, sd_identifier) @@ -112,7 +112,7 @@ class TestStreamify(TestCase): self.session = Session( conf.ADJUSTABLE_SETTINGS['data_rate'][1], db_dir=self.db_dir, node_id="abcd", peer_finder=peer_finder, hash_announcer=hash_announcer, - blob_dir=self.blob_dir, peer_port=5553, + blob_dir=self.blob_dir, peer_port=5553, dht_node_class=mocks.Node, use_upnp=False, rate_limiter=rate_limiter, wallet=wallet, blob_tracker_class=DummyBlobAvailabilityTracker, external_ip="127.0.0.1" ) diff --git a/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py b/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py index 3e16362fa..de06f342f 100644 --- a/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py +++ b/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py @@ -26,31 +26,18 @@ class MocDHTNode(object): defer.returnValue(result) -class MocSupplier(object): - def __init__(self, blobs_to_announce): - self.blobs_to_announce = blobs_to_announce - self.announced = False - - def hashes_to_announce(self): - if not self.announced: - self.announced = True - return defer.succeed(self.blobs_to_announce) - else: - return defer.succeed([]) - - def set_single_hash_announce_duration(self, seconds): - pass - class DHTHashAnnouncerTest(unittest.TestCase): + @defer.inlineCallbacks def setUp(self): self.num_blobs = 10 self.blobs_to_announce = [] for i in range(0, self.num_blobs): self.blobs_to_announce.append(random_lbry_hash()) self.dht_node = MocDHTNode() - self.announcer = DHTHashAnnouncer(self.dht_node, peer_port=3333) - self.supplier = MocSupplier(self.blobs_to_announce) - self.announcer.add_supplier(self.supplier) + self.dht_node.peerPort = 3333 + self.dht_node.clock = reactor + self.announcer = DHTHashAnnouncer(self.dht_node) + yield self.announcer.add_hashes_to_announce(self.blobs_to_announce) @defer.inlineCallbacks def test_announce_fail(self): diff --git a/lbrynet/tests/unit/dht/test_node.py b/lbrynet/tests/unit/dht/test_node.py index 9ae3eb3a9..e89811b48 100644 --- a/lbrynet/tests/unit/dht/test_node.py +++ b/lbrynet/tests/unit/dht/test_node.py @@ -198,7 +198,7 @@ class NodeLookupTest(unittest.TestCase): h = hashlib.sha384() h.update('node1') node_id = str(h.digest()) - self.node = lbrynet.dht.node.Node(node_id, 4000, None, None, self._protocol) + self.node = lbrynet.dht.node.Node(None, node_id=node_id, udpPort=4000, networkProtocol=self._protocol) self.updPort = 81173 self.contactsAmount = 80 # Reinitialise the routing table diff --git a/lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py b/lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py index a02e8d78c..0fc769121 100644 --- a/lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py +++ b/lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py @@ -2,16 +2,18 @@ from Crypto.Cipher import AES import mock from twisted.trial import unittest -from twisted.internet import defer +from twisted.internet import defer, reactor from lbrynet.database.storage import SQLiteStorage from lbrynet.core.StreamDescriptor import get_sd_info, BlobStreamDescriptorReader from lbrynet.core import BlobManager from lbrynet.core import Session from lbrynet.dht import hashannouncer +from lbrynet.dht.node import Node from lbrynet.file_manager import EncryptedFileCreator from lbrynet.file_manager import EncryptedFileManager from lbrynet.tests import mocks +from time import time from lbrynet.tests.util import mk_db_and_blob_dir, rm_db_and_blob_dir MB = 2**20 @@ -32,10 +34,8 @@ class CreateEncryptedFileTest(unittest.TestCase): self.session = mock.Mock(spec=Session.Session)(None, None) self.session.payment_rate_manager.min_blob_data_payment_rate = 0 - - hash_announcer = hashannouncer.DHTHashAnnouncer(None, None) self.blob_manager = BlobManager.DiskBlobManager( - hash_announcer, self.tmp_blob_dir, SQLiteStorage(self.tmp_db_dir)) + hashannouncer.DummyHashAnnouncer(), self.tmp_blob_dir, SQLiteStorage(self.tmp_db_dir)) self.session.blob_manager = self.blob_manager self.session.storage = self.session.blob_manager.storage self.file_manager = EncryptedFileManager.EncryptedFileManager(self.session, object()) From 1eff35ce76db346b8163e3a19888c5c842d22a1a Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 20 Feb 2018 13:47:14 -0500 Subject: [PATCH 27/52] update dht unit tests to use task.Clock --- lbrynet/tests/unit/dht/test_protocol.py | 109 +++++++++++++++--------- 1 file changed, 70 insertions(+), 39 deletions(-) diff --git a/lbrynet/tests/unit/dht/test_protocol.py b/lbrynet/tests/unit/dht/test_protocol.py index d616451d9..d10a5a5e7 100644 --- a/lbrynet/tests/unit/dht/test_protocol.py +++ b/lbrynet/tests/unit/dht/test_protocol.py @@ -1,33 +1,54 @@ import time import unittest -import twisted.internet.selectreactor - +from twisted.internet.task import Clock +from twisted.internet import defer import lbrynet.dht.protocol import lbrynet.dht.contact import lbrynet.dht.constants import lbrynet.dht.msgtypes from lbrynet.dht.error import TimeoutError from lbrynet.dht.node import Node, rpcmethod +from lbrynet.tests.mocks import listenUDP, resolve + +import logging + +log = logging.getLogger() class KademliaProtocolTest(unittest.TestCase): """ Test case for the Protocol class """ - def setUp(self): - del lbrynet.dht.protocol.reactor - lbrynet.dht.protocol.reactor = twisted.internet.selectreactor.SelectReactor() - self.node = Node(node_id='1' * 48, udpPort=9182, externalIP="127.0.0.1") - self.protocol = lbrynet.dht.protocol.KademliaProtocol(self.node) + udpPort = 9182 + def setUp(self): + self._reactor = Clock() + self.node = Node(node_id='1' * 48, udpPort=self.udpPort, externalIP="127.0.0.1", listenUDP=listenUDP, + resolve=resolve, clock=self._reactor, callLater=self._reactor.callLater) + + def tearDown(self): + del self._reactor + + @defer.inlineCallbacks def testReactor(self): """ Tests if the reactor can start/stop the protocol correctly """ - lbrynet.dht.protocol.reactor.listenUDP(0, self.protocol) - lbrynet.dht.protocol.reactor.callLater(0, lbrynet.dht.protocol.reactor.stop) - lbrynet.dht.protocol.reactor.run() + + d = defer.Deferred() + self._reactor.callLater(1, d.callback, True) + self._reactor.advance(1) + result = yield d + self.assertTrue(result) def testRPCTimeout(self): """ Tests if a RPC message sent to a dead remote node times out correctly """ + dead_node = Node(node_id='2' * 48, udpPort=self.udpPort, externalIP="127.0.0.2", listenUDP=listenUDP, + resolve=resolve, clock=self._reactor, callLater=self._reactor.callLater) + dead_node.start_listening() + dead_node.stop() + self._reactor.pump([1 for _ in range(10)]) + dead_contact = lbrynet.dht.contact.Contact('2' * 48, '127.0.0.2', 9182, self.node._protocol) + self.node.addContact(dead_contact) + @rpcmethod def fake_ping(*args, **kwargs): time.sleep(lbrynet.dht.constants.rpcTimeout + 1) @@ -38,19 +59,18 @@ class KademliaProtocolTest(unittest.TestCase): real_attempts = lbrynet.dht.constants.rpcAttempts lbrynet.dht.constants.rpcAttempts = 1 lbrynet.dht.constants.rpcTimeout = 1 + self.node.ping = fake_ping - deadContact = lbrynet.dht.contact.Contact('2' * 48, '127.0.0.1', 9182, self.protocol) - self.node.addContact(deadContact) # Make sure the contact was added - self.failIf(deadContact not in self.node.contacts, + self.failIf(dead_contact not in self.node.contacts, 'Contact not added to fake node (error in test code)') - lbrynet.dht.protocol.reactor.listenUDP(9182, self.protocol) + self.node.start_listening() # Run the PING RPC (which should raise a timeout error) - df = self.protocol.sendRPC(deadContact, 'ping', {}) + df = self.node._protocol.sendRPC(dead_contact, 'ping', {}) def check_timeout(err): - self.assertEqual(type(err), TimeoutError) + self.assertEqual(err.type, TimeoutError) df.addErrback(check_timeout) @@ -61,20 +81,24 @@ class KademliaProtocolTest(unittest.TestCase): # See if the contact was removed due to the timeout def check_removed_contact(): - self.failIf(deadContact in self.node.contacts, + self.failIf(dead_contact in self.node.contacts, 'Contact was not removed after RPC timeout; check exception types.') df.addCallback(lambda _: reset_values()) # Stop the reactor if a result arrives (timeout or not) - df.addBoth(lambda _: lbrynet.dht.protocol.reactor.stop()) df.addCallback(lambda _: check_removed_contact()) - lbrynet.dht.protocol.reactor.run() + self._reactor.pump([1 for _ in range(20)]) def testRPCRequest(self): """ Tests if a valid RPC request is executed and responded to correctly """ - remoteContact = lbrynet.dht.contact.Contact('2' * 48, '127.0.0.1', 9182, self.protocol) + + remote_node = Node(node_id='2' * 48, udpPort=self.udpPort, externalIP="127.0.0.2", listenUDP=listenUDP, + resolve=resolve, clock=self._reactor, callLater=self._reactor.callLater) + remote_node.start_listening() + remoteContact = lbrynet.dht.contact.Contact('2' * 48, '127.0.0.2', 9182, self.node._protocol) self.node.addContact(remoteContact) + self.error = None def handleError(f): @@ -87,16 +111,18 @@ class KademliaProtocolTest(unittest.TestCase): % (expectedResult, result) # Publish the "local" node on the network - lbrynet.dht.protocol.reactor.listenUDP(9182, self.protocol) + self.node.start_listening() # Simulate the RPC df = remoteContact.ping() df.addCallback(handleResult) df.addErrback(handleError) - df.addBoth(lambda _: lbrynet.dht.protocol.reactor.stop()) - lbrynet.dht.protocol.reactor.run() + + for _ in range(10): + self._reactor.advance(1) + self.failIf(self.error, self.error) # The list of sent RPC messages should be empty at this stage - self.failUnlessEqual(len(self.protocol._sentMessages), 0, + self.failUnlessEqual(len(self.node._protocol._sentMessages), 0, 'The protocol is still waiting for a RPC result, ' 'but the transaction is already done!') @@ -105,8 +131,12 @@ class KademliaProtocolTest(unittest.TestCase): Verifies that a RPC request for an existing but unpublished method is denied, and that the associated (remote) exception gets raised locally """ - remoteContact = lbrynet.dht.contact.Contact('2' * 48, '127.0.0.1', 9182, self.protocol) - self.node.addContact(remoteContact) + remote_node = Node(node_id='2' * 48, udpPort=self.udpPort, externalIP="127.0.0.2", listenUDP=listenUDP, + resolve=resolve, clock=self._reactor, callLater=self._reactor.callLater) + remote_node.start_listening() + remote_contact = lbrynet.dht.contact.Contact('2' * 48, '127.0.0.2', 9182, self.node._protocol) + self.node.addContact(remote_contact) + self.error = None def handleError(f): @@ -123,24 +153,26 @@ class KademliaProtocolTest(unittest.TestCase): self.error = 'The remote method executed successfully, returning: "%s"; ' \ 'this RPC should not have been allowed.' % result - # Publish the "local" node on the network - lbrynet.dht.protocol.reactor.listenUDP(9182, self.protocol) + self.node.start_listening() + self._reactor.pump([1 for _ in range(10)]) # Simulate the RPC - df = remoteContact.not_a_rpc_function() + df = remote_contact.not_a_rpc_function() df.addCallback(handleResult) df.addErrback(handleError) - df.addBoth(lambda _: lbrynet.dht.protocol.reactor.stop()) - lbrynet.dht.protocol.reactor.run() + self._reactor.pump([1 for _ in range(10)]) self.failIf(self.error, self.error) # The list of sent RPC messages should be empty at this stage - self.failUnlessEqual(len(self.protocol._sentMessages), 0, + self.failUnlessEqual(len(self.node._protocol._sentMessages), 0, 'The protocol is still waiting for a RPC result, ' 'but the transaction is already done!') def testRPCRequestArgs(self): """ Tests if an RPC requiring arguments is executed correctly """ - remoteContact = lbrynet.dht.contact.Contact('2' * 48, '127.0.0.1', 9182, self.protocol) - self.node.addContact(remoteContact) + remote_node = Node(node_id='2' * 48, udpPort=self.udpPort, externalIP="127.0.0.2", listenUDP=listenUDP, + resolve=resolve, clock=self._reactor, callLater=self._reactor.callLater) + remote_node.start_listening() + remote_contact = lbrynet.dht.contact.Contact('2' * 48, '127.0.0.2', 9182, self.node._protocol) + self.node.addContact(remote_contact) self.error = None def handleError(f): @@ -153,15 +185,14 @@ class KademliaProtocolTest(unittest.TestCase): (expectedResult, result) # Publish the "local" node on the network - lbrynet.dht.protocol.reactor.listenUDP(9182, self.protocol) + self.node.start_listening() # Simulate the RPC - df = remoteContact.ping() + df = remote_contact.ping() df.addCallback(handleResult) df.addErrback(handleError) - df.addBoth(lambda _: lbrynet.dht.protocol.reactor.stop()) - lbrynet.dht.protocol.reactor.run() + self._reactor.pump([1 for _ in range(10)]) self.failIf(self.error, self.error) # The list of sent RPC messages should be empty at this stage - self.failUnlessEqual(len(self.protocol._sentMessages), 0, + self.failUnlessEqual(len(self.node._protocol._sentMessages), 0, 'The protocol is still waiting for a RPC result, ' 'but the transaction is already done!') From d2a6dd3ed3af4e3d3f12c6edb762c53c951cd090 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 20 Feb 2018 13:47:36 -0500 Subject: [PATCH 28/52] add dht functional tests --- lbrynet/tests/functional/test_dht.py | 263 +++++++++++++++++++++++++++ 1 file changed, 263 insertions(+) create mode 100644 lbrynet/tests/functional/test_dht.py diff --git a/lbrynet/tests/functional/test_dht.py b/lbrynet/tests/functional/test_dht.py new file mode 100644 index 000000000..142bffc8b --- /dev/null +++ b/lbrynet/tests/functional/test_dht.py @@ -0,0 +1,263 @@ +import time +import logging +from twisted.trial import unittest +from twisted.internet import defer, threads, task +from lbrynet.dht.node import Node +from lbrynet.tests import mocks +from lbrynet.core.utils import generate_id + +log = logging.getLogger("lbrynet.tests.util") +# log.addHandler(logging.StreamHandler()) +# log.setLevel(logging.DEBUG) + + +class TestKademliaBase(unittest.TestCase): + timeout = 300.0 # timeout for each test + network_size = 0 # plus lbrynet1, lbrynet2, and lbrynet3 seed nodes + node_ids = None + seed_dns = mocks.MOCK_DHT_SEED_DNS + + def _add_next_node(self): + node_id, node_ip = self.mock_node_generator.next() + node = Node(node_id=node_id.decode('hex'), udpPort=4444, peerPort=3333, externalIP=node_ip, + resolve=mocks.resolve, listenUDP=mocks.listenUDP, callLater=self.clock.callLater, clock=self.clock) + self.nodes.append(node) + return node + + @defer.inlineCallbacks + def add_node(self): + node = self._add_next_node() + yield node.joinNetwork( + [ + ("lbrynet1.lbry.io", self._seeds[0].port), + ("lbrynet2.lbry.io", self._seeds[1].port), + ("lbrynet3.lbry.io", self._seeds[2].port), + ] + ) + defer.returnValue(node) + + def get_node(self, node_id): + for node in self.nodes: + if node.node_id == node_id: + return node + raise KeyError(node_id) + + @defer.inlineCallbacks + def pop_node(self): + node = self.nodes.pop() + yield node.stop() + + def pump_clock(self, n, step=0.01): + """ + :param n: seconds to run the reactor for + :param step: reactor tick rate (in seconds) + """ + for _ in range(n * 100): + self.clock.advance(step) + + def run_reactor(self, seconds, *deferreds): + dl = [threads.deferToThread(self.pump_clock, seconds)] + for d in deferreds: + dl.append(d) + return defer.DeferredList(dl) + + @defer.inlineCallbacks + def setUp(self): + self.nodes = [] + self._seeds = [] + self.clock = task.Clock() + self.mock_node_generator = mocks.mock_node_generator(mock_node_ids=self.node_ids) + + join_dl = [] + for seed_dns in self.seed_dns: + other_seeds = list(self.seed_dns.keys()) + other_seeds.remove(seed_dns) + + self._add_next_node() + seed = self.nodes.pop() + self._seeds.append(seed) + join_dl.append( + seed.joinNetwork([(other_seed_dns, 4444) for other_seed_dns in other_seeds]) + ) + + if self.network_size: + for _ in range(self.network_size): + join_dl.append(self.add_node()) + yield self.run_reactor(1, *tuple(join_dl)) + self.verify_all_nodes_are_routable() + + @defer.inlineCallbacks + def tearDown(self): + dl = [] + while self.nodes: + dl.append(self.pop_node()) # stop all of the nodes + while self._seeds: + dl.append(self._seeds.pop().stop()) # and the seeds + yield defer.DeferredList(dl) + + def verify_all_nodes_are_routable(self): + routable = set() + node_addresses = {node.externalIP for node in self.nodes} + node_addresses = node_addresses.union({node.externalIP for node in self._seeds}) + for node in self._seeds: + contact_addresses = {contact.address for contact in node.contacts} + routable.update(contact_addresses) + for node in self.nodes: + contact_addresses = {contact.address for contact in node.contacts} + routable.update(contact_addresses) + self.assertSetEqual(routable, node_addresses) + + +class TestKademliaBootstrap(TestKademliaBase): + """ + Test initializing the network / connecting the seed nodes + """ + + def test_bootstrap_network(self): # simulates the real network, which has three seeds + self.assertEqual(len(self._seeds[0].contacts), 2) + self.assertEqual(len(self._seeds[1].contacts), 2) + self.assertEqual(len(self._seeds[2].contacts), 2) + + self.assertSetEqual( + {self._seeds[0].contacts[0].address, self._seeds[0].contacts[1].address}, + {self._seeds[1].externalIP, self._seeds[2].externalIP} + ) + + self.assertSetEqual( + {self._seeds[1].contacts[0].address, self._seeds[1].contacts[1].address}, + {self._seeds[0].externalIP, self._seeds[2].externalIP} + ) + + self.assertSetEqual( + {self._seeds[2].contacts[0].address, self._seeds[2].contacts[1].address}, + {self._seeds[0].externalIP, self._seeds[1].externalIP} + ) + + def test_all_nodes_are_pingable(self): + def _ping_cb(result): + self.assertEqual(result, "pong") + + dl = [] + for seed in self._seeds: + self.assertEqual(len(seed.contacts), 2) + for contact in seed.contacts: + d = contact.ping() + d.addCallback(_ping_cb) + dl.append(d) + self.run_reactor(1, *dl) + + +class TestKademliaBootstrapSixteenSeeds(TestKademliaBase): + node_ids = [ + '000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111', + '222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222', + '333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333', + '444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444', + '555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555', + '666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666666', + '777777777777777777777777777777777777777777777777777777777777777777777777777777777777777777777777', + '888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888', + '999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999', + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + 'cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', + 'dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd', + 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', + 'ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff' + ] + + @defer.inlineCallbacks + def setUp(self): + self.seed_dns.update( + { + "lbrynet4.lbry.io": "10.42.42.4", + "lbrynet5.lbry.io": "10.42.42.5", + "lbrynet6.lbry.io": "10.42.42.6", + "lbrynet7.lbry.io": "10.42.42.7", + "lbrynet8.lbry.io": "10.42.42.8", + "lbrynet9.lbry.io": "10.42.42.9", + "lbrynet10.lbry.io": "10.42.42.10", + "lbrynet11.lbry.io": "10.42.42.11", + "lbrynet12.lbry.io": "10.42.42.12", + "lbrynet13.lbry.io": "10.42.42.13", + "lbrynet14.lbry.io": "10.42.42.14", + "lbrynet15.lbry.io": "10.42.42.15", + "lbrynet16.lbry.io": "10.42.42.16", + } + ) + yield TestKademliaBase.setUp(self) + + @defer.inlineCallbacks + def tearDown(self): + yield TestKademliaBase.tearDown(self) + del self.seed_dns['lbrynet4.lbry.io'] + del self.seed_dns['lbrynet5.lbry.io'] + del self.seed_dns['lbrynet6.lbry.io'] + del self.seed_dns['lbrynet7.lbry.io'] + del self.seed_dns['lbrynet8.lbry.io'] + del self.seed_dns['lbrynet9.lbry.io'] + del self.seed_dns['lbrynet10.lbry.io'] + del self.seed_dns['lbrynet11.lbry.io'] + del self.seed_dns['lbrynet12.lbry.io'] + del self.seed_dns['lbrynet13.lbry.io'] + del self.seed_dns['lbrynet14.lbry.io'] + del self.seed_dns['lbrynet15.lbry.io'] + del self.seed_dns['lbrynet16.lbry.io'] + + def test_bootstrap_network(self): + pass + + +class Test250NodeNetwork(TestKademliaBase): + network_size = 250 + + def test_setup_network_and_verify_connectivity(self): + pass + + def update_network(self): + import random + dl = [] + announced_blobs = [] + + for node in self.nodes: # random events + if random.randint(0, 10000) < 75 and announced_blobs: # get peers for a blob + log.info('find blob') + blob_hash = random.choice(announced_blobs) + dl.append(node.getPeersForBlob(blob_hash)) + if random.randint(0, 10000) < 25: # announce a blob + log.info('announce blob') + blob_hash = generate_id() + announced_blobs.append((blob_hash, node.node_id)) + dl.append(node.announceHaveBlob(blob_hash)) + + random.shuffle(self.nodes) + + # kill nodes + while random.randint(0, 100) > 95: + dl.append(self.pop_node()) + log.info('pop node') + + # add nodes + while random.randint(0, 100) > 95: + dl.append(self.add_node()) + log.info('add node') + return tuple(dl), announced_blobs + + @defer.inlineCallbacks + def _test_simulate_network(self): + total_blobs = [] + for i in range(100): + d, blobs = self.update_network() + total_blobs.extend(blobs) + self.run_reactor(1, *d) + yield threads.deferToThread(time.sleep, 0.1) + routable = set() + node_addresses = {node.externalIP for node in self.nodes} + for node in self.nodes: + contact_addresses = {contact.address for contact in node.contacts} + routable.update(contact_addresses) + log.warning("difference: %i", len(node_addresses.difference(routable))) + log.info("blobs %i", len(total_blobs)) + log.info("step %i, %i nodes", i, len(self.nodes)) + self.pump_clock(100) From 88970cb0a86e8b9cc900388856e293ac9b1043b7 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 21 Feb 2018 14:53:12 -0500 Subject: [PATCH 29/52] move peer manager class to lbrynet.core --- .../peermanager.py => core/PeerManager.py} | 0 lbrynet/core/Session.py | 13 ++++++++----- lbrynet/daemon/Daemon.py | 2 +- lbrynet/daemon/DaemonServer.py | 1 + lbrynet/dht/node.py | 19 +++++++++---------- lbrynet/tests/functional/test_misc.py | 2 +- lbrynet/tests/functional/test_reflector.py | 5 ++--- lbrynet/tests/functional/test_streamify.py | 2 +- .../core/client/test_ConnectionManager.py | 2 +- 9 files changed, 24 insertions(+), 22 deletions(-) rename lbrynet/{dht/peermanager.py => core/PeerManager.py} (100%) diff --git a/lbrynet/dht/peermanager.py b/lbrynet/core/PeerManager.py similarity index 100% rename from lbrynet/dht/peermanager.py rename to lbrynet/core/PeerManager.py diff --git a/lbrynet/core/Session.py b/lbrynet/core/Session.py index 436b46c52..b8ef9ebcb 100644 --- a/lbrynet/core/Session.py +++ b/lbrynet/core/Session.py @@ -2,7 +2,7 @@ import logging import miniupnpc from twisted.internet import threads, defer from lbrynet.core.BlobManager import DiskBlobManager -from lbrynet.dht import node, peermanager, hashannouncer +from lbrynet.dht import node from lbrynet.database.storage import SQLiteStorage from lbrynet.core.RateLimiter import RateLimiter from lbrynet.core.utils import generate_id @@ -122,6 +122,7 @@ class Session(object): self.payment_rate_manager_class = payment_rate_manager_class or NegotiatedPaymentRateManager self.is_generous = is_generous self.storage = storage or SQLiteStorage(self.db_dir) + self._join_dht_deferred = None def setup(self): """Create the blob directory and database if necessary, start all desired services""" @@ -221,9 +222,7 @@ class Session(object): d.addErrback(upnp_failed) return d - @defer.inlineCallbacks - def _setup_dht(self): - log.info("Starting DHT") + def _setup_dht(self): # does not block startup, the dht will re-attempt if necessary self.dht_node = self.dht_node_class( self.hash_announcer, udpPort=self.dht_node_port, @@ -233,7 +232,11 @@ class Session(object): peer_manager=self.peer_manager, peer_finder=self.peer_finder, ) - yield self.dht_node.joinNetwork(self.known_dht_nodes) + self.peer_manager = self.dht_node.peer_manager + self.peer_finder = self.dht_node.peer_finder + self.hash_announcer = self.dht_node.hash_announcer + self._join_dht_deferred = self.dht_node.joinNetwork(self.known_dht_nodes) + self._join_dht_deferred.addCallback(lambda _: log.info("Joined the dht")) def _setup_other_components(self): log.debug("Setting up the rest of the components") diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index 6ee6b8fcf..652fb370e 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -313,7 +313,7 @@ class Daemon(AuthJSONRPCServer): self.session.peer_manager) try: - log.info("Daemon bound to port: %d", self.peer_port) + log.info("Peer protocol listening on TCP %d", self.peer_port) self.lbry_server_port = reactor.listenTCP(self.peer_port, server_factory) except error.CannotListenError as e: import traceback diff --git a/lbrynet/daemon/DaemonServer.py b/lbrynet/daemon/DaemonServer.py index 588f5a936..e8c00606b 100644 --- a/lbrynet/daemon/DaemonServer.py +++ b/lbrynet/daemon/DaemonServer.py @@ -39,6 +39,7 @@ class DaemonServer(object): try: self.server_port = reactor.listenTCP( conf.settings['api_port'], lbrynet_server, interface=conf.settings['api_host']) + log.info("lbrynet API listening on TCP %s:%i", conf.settings['api_host'], conf.settings['api_port']) except error.CannotListenError: log.info('Daemon already running, exiting app') raise diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index eb595b130..c7e18af77 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -11,23 +11,23 @@ import hashlib import operator import struct import time +import logging from twisted.internet import defer, error, task +from lbrynet.core.utils import generate_id +from lbrynet.core.PeerManager import PeerManager + import constants import routingtable import datastore import protocol from error import TimeoutError - -from peermanager import PeerManager from hashannouncer import DHTHashAnnouncer from peerfinder import DHTPeerFinder from contact import Contact from hashwatcher import HashWatcher from distance import Distance -import logging -from lbrynet.core.utils import generate_id log = logging.getLogger(__name__) @@ -168,7 +168,6 @@ class Node(object): def start_listening(self): try: self._listeningPort = self.reactor_listenUDP(self.port, self._protocol) - log.info("DHT node listening on %i", self.port) except error.CannotListenError as e: import traceback log.error("Couldn't bind to port %d. %s", self.port, traceback.format_exc()) @@ -191,12 +190,12 @@ class Node(object): bootstrap_contacts = self.contacts defer.returnValue(bootstrap_contacts) - def _rerun(bootstrap_contacts): - if not bootstrap_contacts: - log.info("Failed to join the dht, re-attempting in 60 seconds") - self.reactor_callLater(60, self.bootstrap_join, known_node_addresses, finished_d) + def _rerun(closest_nodes): + if not closest_nodes: + log.info("Failed to join the dht, re-attempting in 30 seconds") + self.reactor_callLater(30, self.bootstrap_join, known_node_addresses, finished_d) elif not finished_d.called: - finished_d.callback(bootstrap_contacts) + finished_d.callback(closest_nodes) log.info("Attempting to join the DHT network") d = _resolve_seeds() diff --git a/lbrynet/tests/functional/test_misc.py b/lbrynet/tests/functional/test_misc.py index 327976c5e..dffb100ec 100644 --- a/lbrynet/tests/functional/test_misc.py +++ b/lbrynet/tests/functional/test_misc.py @@ -23,7 +23,7 @@ from twisted.trial.unittest import TestCase from twisted.python.failure import Failure from lbrynet.dht.node import Node -from lbrynet.dht.peermanager import PeerManager +from lbrynet.core.PeerManager import PeerManager from lbrynet.core.RateLimiter import DummyRateLimiter, RateLimiter from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory from lbrynet.core.server.ServerProtocol import ServerProtocolFactory diff --git a/lbrynet/tests/functional/test_reflector.py b/lbrynet/tests/functional/test_reflector.py index 4ba01dd4b..09342d3bd 100644 --- a/lbrynet/tests/functional/test_reflector.py +++ b/lbrynet/tests/functional/test_reflector.py @@ -4,8 +4,7 @@ from twisted.trial import unittest from lbrynet import conf from lbrynet.core.StreamDescriptor import get_sd_info from lbrynet import reflector -from lbrynet.core import BlobManager -from lbrynet.dht import peermanager +from lbrynet.core import BlobManager, PeerManager from lbrynet.core import Session from lbrynet.core import StreamDescriptor from lbrynet.lbry_file.client import EncryptedFileOptions @@ -28,7 +27,7 @@ class TestReflector(unittest.TestCase): self.port = None self.addCleanup(self.take_down_env) wallet = mocks.Wallet() - peer_manager = peermanager.PeerManager() + peer_manager = PeerManager.PeerManager() peer_finder = mocks.PeerFinder(5553, peer_manager, 2) hash_announcer = mocks.Announcer() sd_identifier = StreamDescriptor.StreamDescriptorIdentifier() diff --git a/lbrynet/tests/functional/test_streamify.py b/lbrynet/tests/functional/test_streamify.py index ed0f82c9f..c84630272 100644 --- a/lbrynet/tests/functional/test_streamify.py +++ b/lbrynet/tests/functional/test_streamify.py @@ -13,7 +13,7 @@ from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier from lbrynet.file_manager.EncryptedFileCreator import create_lbry_file from lbrynet.lbry_file.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier from lbrynet.core.StreamDescriptor import get_sd_info -from lbrynet.dht.peermanager import PeerManager +from lbrynet.core.PeerManager import PeerManager from lbrynet.core.RateLimiter import DummyRateLimiter from lbrynet.tests import mocks diff --git a/lbrynet/tests/unit/core/client/test_ConnectionManager.py b/lbrynet/tests/unit/core/client/test_ConnectionManager.py index 57ed1e534..107afa997 100644 --- a/lbrynet/tests/unit/core/client/test_ConnectionManager.py +++ b/lbrynet/tests/unit/core/client/test_ConnectionManager.py @@ -3,7 +3,7 @@ from lbrynet.core.server.ServerProtocol import ServerProtocol from lbrynet.core.client.ClientProtocol import ClientProtocol from lbrynet.core.RateLimiter import RateLimiter from lbrynet.core.Peer import Peer -from lbrynet.dht.peermanager import PeerManager +from lbrynet.core.PeerManager import PeerManager from lbrynet.core.Error import NoResponseError from twisted.trial import unittest From 5628d0825b59d100f020fa109289d4f0e7823fa9 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Thu, 22 Feb 2018 11:29:10 -0500 Subject: [PATCH 30/52] add CallLaterManager --- lbrynet/core/call_later_manager.py | 63 +++++++++++++++++++++++++ lbrynet/dht/hashannouncer.py | 1 - lbrynet/dht/node.py | 8 ++-- lbrynet/dht/protocol.py | 33 ++++--------- lbrynet/tests/unit/dht/test_protocol.py | 6 ++- 5 files changed, 80 insertions(+), 31 deletions(-) create mode 100644 lbrynet/core/call_later_manager.py diff --git a/lbrynet/core/call_later_manager.py b/lbrynet/core/call_later_manager.py new file mode 100644 index 000000000..d3f5d3c2d --- /dev/null +++ b/lbrynet/core/call_later_manager.py @@ -0,0 +1,63 @@ +class CallLaterManager(object): + _callLater = None + _pendingCallLaters = [] + + @classmethod + def _cancel(cls, call_later): + """ + :param call_later: DelayedCall + :return: (callable) canceller function + """ + + def cancel(reason=None): + """ + :param reason: reason for cancellation, this is returned after cancelling the DelayedCall + :return: reason + """ + + if call_later.active(): + call_later.cancel() + cls._pendingCallLaters.remove(call_later) + return reason + return cancel + + @classmethod + def stop(cls): + """ + Cancel any callLaters that are still running + """ + + from twisted.internet import defer + while cls._pendingCallLaters: + canceller = cls._cancel(cls._pendingCallLaters[0]) + try: + canceller() + except (defer.CancelledError, defer.AlreadyCalledError): + pass + + @classmethod + def call_later(cls, when, what, *args, **kwargs): + """ + Schedule a call later and get a canceller callback function + + :param when: (float) delay in seconds + :param what: (callable) + :param args: (*tuple) args to be passed to the callable + :param kwargs: (**dict) kwargs to be passed to the callable + + :return: (tuple) twisted.internet.base.DelayedCall object, canceller function + """ + + call_later = cls._callLater(when, what, *args, **kwargs) + canceller = cls._cancel(call_later) + cls._pendingCallLaters.append(call_later) + return call_later, canceller + + @classmethod + def setup(cls, callLater): + """ + Setup the callLater function to use, supports the real reactor as well as task.Clock + + :param callLater: (IReactorTime.callLater) + """ + cls._callLater = callLater diff --git a/lbrynet/dht/hashannouncer.py b/lbrynet/dht/hashannouncer.py index a1533947a..069c4294f 100644 --- a/lbrynet/dht/hashannouncer.py +++ b/lbrynet/dht/hashannouncer.py @@ -1,7 +1,6 @@ import binascii import collections import logging -import time import datetime from twisted.internet import defer, task diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index c7e18af77..b576cf9a2 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -15,6 +15,7 @@ import logging from twisted.internet import defer, error, task from lbrynet.core.utils import generate_id +from lbrynet.core.call_later_manager import CallLaterManager from lbrynet.core.PeerManager import PeerManager import constants @@ -89,10 +90,11 @@ class Node(object): resolve = resolve or reactor.resolve callLater = callLater or reactor.callLater clock = clock or reactor + self.clock = clock + CallLaterManager.setup(callLater) self.reactor_resolve = resolve self.reactor_listenUDP = listenUDP - self.reactor_callLater = callLater - self.clock = clock + self.reactor_callLater = CallLaterManager.call_later self.node_id = node_id or self._generateID() self.port = udpPort self._listeningPort = None # object implementing Twisted @@ -856,7 +858,7 @@ class _IterativeFindHelper(object): if self._should_lookup_active_calls(): # Schedule the next iteration if there are any active # calls (Kademlia uses loose parallelism) - call = self.node.reactor_callLater(constants.iterativeLookupDelay, self.searchIteration) + call, _ = self.node.reactor_callLater(constants.iterativeLookupDelay, self.searchIteration) self.pending_iteration_calls.append(call) # Check for a quick contact response that made an update to the shortList elif prevShortlistLength < len(self.shortlist): diff --git a/lbrynet/dht/protocol.py b/lbrynet/dht/protocol.py index 4dc2ba176..ce45f56c1 100644 --- a/lbrynet/dht/protocol.py +++ b/lbrynet/dht/protocol.py @@ -3,7 +3,8 @@ import time import socket import errno -from twisted.internet import protocol, defer, error, task +from twisted.internet import protocol, defer, task +from lbrynet.core.call_later_manager import CallLaterManager import constants import encoding @@ -169,17 +170,11 @@ class KademliaProtocol(protocol.DatagramProtocol): df._rpcRawResponse = True # Set the RPC timeout timer - timeoutCall = self._node.reactor_callLater(constants.rpcTimeout, self._msgTimeout, msg.id) + timeoutCall, cancelTimeout = self._node.reactor_callLater(constants.rpcTimeout, self._msgTimeout, msg.id) # Transmit the data self._send(encodedMsg, msg.id, (contact.address, contact.port)) self._sentMessages[msg.id] = (contact.id, df, timeoutCall, method, args) - - def cancel(err): - if timeoutCall.cancelled or timeoutCall.called: - return err - timeoutCall.cancel() - - df.addErrback(cancel) + df.addErrback(cancelTimeout) return df def startProtocol(self): @@ -340,12 +335,9 @@ class KademliaProtocol(protocol.DatagramProtocol): """Schedule the sending of the next UDP packet """ delay = self._delay() key = object() - delayed_call = self._node.reactor_callLater(delay, self._write_and_remove, key, txData, address) - self._call_later_list[key] = delayed_call + delayed_call, _ = self._node.reactor_callLater(delay, self._write_and_remove, key, txData, address) def _write_and_remove(self, key, txData, address): - if key in self._call_later_list: - del self._call_later_list[key] if self.transport: try: self.transport.write(txData, address) @@ -440,7 +432,7 @@ class KademliaProtocol(protocol.DatagramProtocol): # See if any progress has been made; if not, kill the message if self._hasProgressBeenMade(messageID): # Reset the RPC timeout timer - timeoutCall = self._node.reactor_callLater(constants.rpcTimeout, self._msgTimeout, messageID) + timeoutCall, _ = self._node.reactor_callLater(constants.rpcTimeout, self._msgTimeout, messageID) self._sentMessages[messageID] = (remoteContactID, df, timeoutCall, method, args) else: # No progress has been made @@ -469,15 +461,6 @@ class KademliaProtocol(protocol.DatagramProtocol): if self._bandwidth_stats_update_lc.running: self._bandwidth_stats_update_lc.stop() - for delayed_call in self._call_later_list.values(): - try: - delayed_call.cancel() - except (error.AlreadyCalled, error.AlreadyCancelled): - log.debug('Attempted to cancel a DelayedCall that was not active') - except Exception: - log.exception('Failed to cancel a DelayedCall') - # not sure why this is needed, but taking this out sometimes causes - # exceptions.AttributeError: 'Port' object has no attribute 'socket' - # to happen on shutdown - # reactor.iterate() + CallLaterManager.stop() + log.info('DHT stopped') diff --git a/lbrynet/tests/unit/dht/test_protocol.py b/lbrynet/tests/unit/dht/test_protocol.py index d10a5a5e7..0b48cf115 100644 --- a/lbrynet/tests/unit/dht/test_protocol.py +++ b/lbrynet/tests/unit/dht/test_protocol.py @@ -1,7 +1,7 @@ import time import unittest from twisted.internet.task import Clock -from twisted.internet import defer +from twisted.internet import defer, threads import lbrynet.dht.protocol import lbrynet.dht.contact import lbrynet.dht.constants @@ -9,6 +9,7 @@ import lbrynet.dht.msgtypes from lbrynet.dht.error import TimeoutError from lbrynet.dht.node import Node, rpcmethod from lbrynet.tests.mocks import listenUDP, resolve +from lbrynet.core.call_later_manager import CallLaterManager import logging @@ -22,10 +23,12 @@ class KademliaProtocolTest(unittest.TestCase): def setUp(self): self._reactor = Clock() + CallLaterManager.setup(self._reactor.callLater) self.node = Node(node_id='1' * 48, udpPort=self.udpPort, externalIP="127.0.0.1", listenUDP=listenUDP, resolve=resolve, clock=self._reactor, callLater=self._reactor.callLater) def tearDown(self): + CallLaterManager.stop() del self._reactor @defer.inlineCallbacks @@ -40,7 +43,6 @@ class KademliaProtocolTest(unittest.TestCase): def testRPCTimeout(self): """ Tests if a RPC message sent to a dead remote node times out correctly """ - dead_node = Node(node_id='2' * 48, udpPort=self.udpPort, externalIP="127.0.0.2", listenUDP=listenUDP, resolve=resolve, clock=self._reactor, callLater=self._reactor.callLater) dead_node.start_listening() From fd7a771f66ff8d887be25ed5790f55bf6f401c43 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Thu, 22 Feb 2018 11:29:41 -0500 Subject: [PATCH 31/52] add sanity check to CreateEncryptedFileTest --- lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py b/lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py index 0fc769121..46ef3b721 100644 --- a/lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py +++ b/lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py @@ -74,6 +74,7 @@ class CreateEncryptedFileTest(unittest.TestCase): # this comes from the database, the blobs returned are sorted sd_info = yield get_sd_info(self.session.storage, lbry_file.stream_hash, include_blobs=True) self.assertDictEqual(sd_info, sd_file_info) + self.assertListEqual(sd_info['blobs'], sd_file_info['blobs']) self.assertEqual(sd_info['stream_hash'], expected_stream_hash) self.assertEqual(len(sd_info['blobs']), 3) self.assertNotEqual(sd_info['blobs'][0]['length'], 0) From 1db44d5fb616ecb489d002b9b2606f8f595580aa Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Mon, 26 Feb 2018 11:52:44 -0500 Subject: [PATCH 32/52] rename variable --- lbrynet/tests/mocks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lbrynet/tests/mocks.py b/lbrynet/tests/mocks.py index 70539ed83..9f8133724 100644 --- a/lbrynet/tests/mocks.py +++ b/lbrynet/tests/mocks.py @@ -349,7 +349,7 @@ class MockUDPTransport(object): self._node = protocol._node def write(self, data, address): - dest = MockNetwork.protocols[address][0] + dest = MockNetwork.peers[address][0] debug_kademlia_packet(data, (self.address, self.port), address, self._node) dest.datagramReceived(data, (self.address, self.port)) @@ -366,13 +366,13 @@ class MockUDPPort(object): class MockNetwork(object): - protocols = {} # (interface, port): (protocol, max_packet_size) + peers = {} # (interface, port): (protocol, max_packet_size) @classmethod def add_peer(cls, port, protocol, interface, maxPacketSize): interface = protocol._node.externalIP protocol.transport = MockUDPTransport(interface, port, maxPacketSize, protocol) - cls.protocols[(interface, port)] = (protocol, maxPacketSize) + cls.peers[(interface, port)] = (protocol, maxPacketSize) def listenUDP(port, protocol, interface='', maxPacketSize=8192): From ebe5dd0e6862f399c7e7fedb59ccd60fce783959 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Mon, 26 Feb 2018 11:53:10 -0500 Subject: [PATCH 33/52] better ping test --- lbrynet/tests/functional/test_dht.py | 46 +++++++++++++++++++++------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/lbrynet/tests/functional/test_dht.py b/lbrynet/tests/functional/test_dht.py index 142bffc8b..ac8572193 100644 --- a/lbrynet/tests/functional/test_dht.py +++ b/lbrynet/tests/functional/test_dht.py @@ -107,6 +107,37 @@ class TestKademliaBase(unittest.TestCase): routable.update(contact_addresses) self.assertSetEqual(routable, node_addresses) + @defer.inlineCallbacks + def verify_all_nodes_are_pingable(self): + ping_replies = {} + ping_dl = [] + contacted = set() + + def _ping_cb(result, node, replies): + replies[node] = result + + for node in self._seeds: + contact_addresses = set() + for contact in node.contacts: + contact_addresses.add(contact.address) + d = contact.ping() + d.addCallback(_ping_cb, contact.address, ping_replies) + contacted.add(contact.address) + ping_dl.append(d) + for node in self.nodes: + contact_addresses = set() + for contact in node.contacts: + contact_addresses.add(contact.address) + d = contact.ping() + d.addCallback(_ping_cb, contact.address, ping_replies) + contacted.add(contact.address) + ping_dl.append(d) + self.run_reactor(2, *ping_dl) + yield threads.deferToThread(time.sleep, 0.1) + node_addresses = {node.externalIP for node in self.nodes}.union({seed.externalIP for seed in self._seeds}) + self.assertSetEqual(node_addresses, contacted) + self.assertDictEqual(ping_replies, {node: "pong" for node in contacted}) + class TestKademliaBootstrap(TestKademliaBase): """ @@ -134,17 +165,7 @@ class TestKademliaBootstrap(TestKademliaBase): ) def test_all_nodes_are_pingable(self): - def _ping_cb(result): - self.assertEqual(result, "pong") - - dl = [] - for seed in self._seeds: - self.assertEqual(len(seed.contacts), 2) - for contact in seed.contacts: - d = contact.ping() - d.addCallback(_ping_cb) - dl.append(d) - self.run_reactor(1, *dl) + return self.verify_all_nodes_are_pingable() class TestKademliaBootstrapSixteenSeeds(TestKademliaBase): @@ -208,6 +229,9 @@ class TestKademliaBootstrapSixteenSeeds(TestKademliaBase): def test_bootstrap_network(self): pass + def test_all_nodes_are_pingable(self): + return self.verify_all_nodes_are_pingable() + class Test250NodeNetwork(TestKademliaBase): network_size = 250 From 339e666f387d8c2252ed4c27f8f3a6fa18974359 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Thu, 1 Mar 2018 16:19:29 -0500 Subject: [PATCH 34/52] + Wallet.wait_for_tx_in_wallet --- lbrynet/core/Wallet.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lbrynet/core/Wallet.py b/lbrynet/core/Wallet.py index adc01bd97..4d6a7ea3f 100644 --- a/lbrynet/core/Wallet.py +++ b/lbrynet/core/Wallet.py @@ -623,6 +623,9 @@ class Wallet(object): d = self._get_transaction(txid) return d + def wait_for_tx_in_wallet(self, txid): + return self._wait_for_tx_in_wallet(txid) + def get_balance(self): return self.wallet_balance - self.total_reserved_points - sum(self.queued_payments.values()) @@ -681,6 +684,9 @@ class Wallet(object): def _get_transaction(self, txid): return defer.fail(NotImplementedError()) + def _wait_for_tx_in_wallet(self, txid): + return defer.fail(NotImplementedError()) + def _update_balance(self): return defer.fail(NotImplementedError()) @@ -1067,6 +1073,9 @@ class LBRYumWallet(Wallet): def _get_transaction(self, txid): return self._run_cmd_as_defer_to_thread("gettransaction", txid) + def _wait_for_tx_in_wallet(self, txid): + return self._run_cmd_as_defer_to_thread("waitfortxinwallet", txid) + def get_name_claims(self): return self._run_cmd_as_defer_succeed('getnameclaims') From cb09be5336ec33c7ee8764532236418f6f3f2b6f Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Mon, 5 Mar 2018 13:14:19 -0500 Subject: [PATCH 35/52] remove unused stuff --- lbrynet/dht/node.py | 27 --------------------------- scripts/dht_scripts.py | 26 ++++++++++++++++++++++++-- 2 files changed, 24 insertions(+), 29 deletions(-) diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index b576cf9a2..12d28cdd5 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -236,39 +236,12 @@ class Node(object): yield contact return list(_inner()) - def printContacts(self, *args): - print '\n\nNODE CONTACTS\n===============' - for i in range(len(self._routingTable._buckets)): - print "bucket %i" % i - for contact in self._routingTable._buckets[i]._contacts: - print " %s:%i" % (contact.address, contact.port) - print '==================================' - def hasContacts(self): for bucket in self._routingTable._buckets: if bucket._contacts: return True return False - def getApproximateTotalDHTNodes(self): - # get the deepest bucket and the number of contacts in that bucket and multiply it - # by the number of equivalently deep buckets in the whole DHT to get a really bad - # estimate! - bucket = self._routingTable._buckets[self._routingTable._kbucketIndex(self.node_id)] - num_in_bucket = len(bucket._contacts) - factor = (2 ** constants.key_bits) / (bucket.rangeMax - bucket.rangeMin) - return num_in_bucket * factor - - def getApproximateTotalHashes(self): - # Divide the number of hashes we know about by k to get a really, really, really - # bad estimate of the average number of hashes per node, then multiply by the - # approximate number of nodes to get a horrendous estimate of the total number - # of hashes in the DHT - num_in_data_store = len(self._dataStore._dict) - if num_in_data_store == 0: - return 0 - return num_in_data_store * self.getApproximateTotalDHTNodes() / 8 - def announceHaveBlob(self, key): return self.iterativeAnnounceHaveBlob(key, {'port': self.peerPort, 'lbryid': self.node_id}) diff --git a/scripts/dht_scripts.py b/scripts/dht_scripts.py index b3a5cafe0..0aec28c57 100644 --- a/scripts/dht_scripts.py +++ b/scripts/dht_scripts.py @@ -73,11 +73,33 @@ def connect(port=None): yield reactor.stop() +def getApproximateTotalDHTNodes(node): + from lbrynet.dht import constants + # get the deepest bucket and the number of contacts in that bucket and multiply it + # by the number of equivalently deep buckets in the whole DHT to get a really bad + # estimate! + bucket = node._routingTable._buckets[node._routingTable._kbucketIndex(node.node_id)] + num_in_bucket = len(bucket._contacts) + factor = (2 ** constants.key_bits) / (bucket.rangeMax - bucket.rangeMin) + return num_in_bucket * factor + + +def getApproximateTotalHashes(node): + # Divide the number of hashes we know about by k to get a really, really, really + # bad estimate of the average number of hashes per node, then multiply by the + # approximate number of nodes to get a horrendous estimate of the total number + # of hashes in the DHT + num_in_data_store = len(node._dataStore._dict) + if num_in_data_store == 0: + return 0 + return num_in_data_store * getApproximateTotalDHTNodes(node) / 8 + + @defer.inlineCallbacks def find(node): try: - log.info("Approximate number of nodes in DHT: %s", str(node.getApproximateTotalDHTNodes())) - log.info("Approximate number of blobs in DHT: %s", str(node.getApproximateTotalHashes())) + log.info("Approximate number of nodes in DHT: %s", str(getApproximateTotalDHTNodes(node))) + log.info("Approximate number of blobs in DHT: %s", str(getApproximateTotalHashes(node))) h = "578f5e82da7db97bfe0677826d452cc0c65406a8e986c9caa126af4ecdbf4913daad2f7f5d1fb0ffec17d0bf8f187f5a" peersFake = yield node.getPeersForBlob(h.decode("hex")) From 5013426e02b5521113bf4febd3c825d48e28bd27 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Mon, 5 Mar 2018 13:14:47 -0500 Subject: [PATCH 36/52] logging and docstring --- lbrynet/dht/node.py | 5 +++++ lbrynet/dht/protocol.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index 12d28cdd5..09842ed99 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -176,6 +176,11 @@ class Node(object): raise ValueError("%s lbrynet may already be running." % str(e)) def bootstrap_join(self, known_node_addresses, finished_d): + """ + Attempt to join the dht, retry every 30 seconds if unsuccessful + :param known_node_addresses: [(str, int)] list of hostnames and ports for known dht seed nodes + :param finished_d: (defer.Deferred) called when join succeeds + """ @defer.inlineCallbacks def _resolve_seeds(): bootstrap_contacts = [] diff --git a/lbrynet/dht/protocol.py b/lbrynet/dht/protocol.py index ce45f56c1..2c624dead 100644 --- a/lbrynet/dht/protocol.py +++ b/lbrynet/dht/protocol.py @@ -211,7 +211,7 @@ class KademliaProtocol(protocol.DatagramProtocol): message = self._translator.fromPrimitive(msgPrimitive) except (encoding.DecodeError, ValueError) as err: # We received some rubbish here - log.exception("Decode error: %s", err) + log.exception("Error decoding datagram from %s:%i - %s", address[0], address[1], err) return except (IndexError, KeyError): log.warning("Couldn't decode dht datagram from %s", address) From 4eab77fa1068551bc35cec10efc955bbf4815106 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Mon, 5 Mar 2018 13:15:07 -0500 Subject: [PATCH 37/52] safe start_listening --- lbrynet/dht/node.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index 09842ed99..00ec2924b 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -168,12 +168,15 @@ class Node(object): yield self.hash_watcher.stop() def start_listening(self): - try: - self._listeningPort = self.reactor_listenUDP(self.port, self._protocol) - except error.CannotListenError as e: - import traceback - log.error("Couldn't bind to port %d. %s", self.port, traceback.format_exc()) - raise ValueError("%s lbrynet may already be running." % str(e)) + if not self._listeningPort: + try: + self._listeningPort = self.reactor_listenUDP(self.port, self._protocol) + except error.CannotListenError as e: + import traceback + log.error("Couldn't bind to port %d. %s", self.port, traceback.format_exc()) + raise ValueError("%s lbrynet may already be running." % str(e)) + else: + log.warning("Already bound to port %d", self._listeningPort.port) def bootstrap_join(self, known_node_addresses, finished_d): """ From a96d827c0f0ee754e174874e62f2f1f2fcbeb561 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Mon, 5 Mar 2018 13:28:59 -0500 Subject: [PATCH 38/52] use reactor time in Delay --- lbrynet/dht/delay.py | 11 +++++------ lbrynet/dht/protocol.py | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lbrynet/dht/delay.py b/lbrynet/dht/delay.py index 9610a73f8..7ef26fcc6 100644 --- a/lbrynet/dht/delay.py +++ b/lbrynet/dht/delay.py @@ -1,18 +1,17 @@ -import time - - class Delay(object): maxToSendDelay = 10 ** -3 # 0.05 minToSendDelay = 10 ** -5 # 0.01 - def __init__(self, start=0): + def __init__(self, start=0, getTime=None): self._next = start + if not getTime: + from time import time as getTime + self._getTime = getTime # TODO: explain why this logic is like it is. And add tests that # show that it actually does what it needs to do. def __call__(self): - ts = time.time() - delay = 0 + ts = self._getTime() if ts >= self._next: delay = self.minToSendDelay self._next = ts + self.minToSendDelay diff --git a/lbrynet/dht/protocol.py b/lbrynet/dht/protocol.py index 2c624dead..505ce4202 100644 --- a/lbrynet/dht/protocol.py +++ b/lbrynet/dht/protocol.py @@ -29,7 +29,7 @@ class KademliaProtocol(protocol.DatagramProtocol): self._sentMessages = {} self._partialMessages = {} self._partialMessagesProgress = {} - self._delay = Delay() + self._delay = Delay(0, self._node.clock.seconds) # keep track of outstanding writes so that they # can be cancelled on shutdown self._call_later_list = {} From 866f220d9bf3b5fc06422cac8adc51368ac3b5c7 Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Wed, 14 Mar 2018 18:12:40 -0400 Subject: [PATCH 39/52] removed PTCWallet --- .travis.yml | 4 +- lbrynet/core/PTCWallet.py | 330 -------------------------------------- lbrynet/tests/mocks.py | 84 +++++++++- requirements_testing.txt | 0 4 files changed, 82 insertions(+), 336 deletions(-) delete mode 100644 lbrynet/core/PTCWallet.py create mode 100644 requirements_testing.txt diff --git a/.travis.yml b/.travis.yml index 6e839bccd..062d88407 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,10 +38,8 @@ install: - pip install . script: - - pip install cython - - pip install mock pylint unqlite + - pip install mock pylint - pylint lbrynet - PYTHONPATH=. trial lbrynet.tests - - python -m unittest discover lbrynet/tests/integration -v - rvm install ruby-2.3.1 - rvm use 2.3.1 && gem install danger --version '~> 4.0' && danger diff --git a/lbrynet/core/PTCWallet.py b/lbrynet/core/PTCWallet.py deleted file mode 100644 index beb203674..000000000 --- a/lbrynet/core/PTCWallet.py +++ /dev/null @@ -1,330 +0,0 @@ -from collections import defaultdict -import logging -import os -import unqlite -import time -from Crypto.Hash import SHA512 -from Crypto.PublicKey import RSA -from lbrynet.core.client.ClientRequest import ClientRequest -from lbrynet.core.Error import RequestCanceledError -from lbrynet.interfaces import IRequestCreator, IQueryHandlerFactory, IQueryHandler, IWallet -from lbrynet.pointtraderclient import pointtraderclient -from twisted.internet import defer, threads -from zope.interface import implements -from twisted.python.failure import Failure -from lbrynet.core.Wallet import ReservedPoints - - -log = logging.getLogger(__name__) - - -class PTCWallet(object): - """This class sends payments to peers and also ensures that expected payments are received. - This class is only intended to be used for testing.""" - implements(IWallet) - - def __init__(self, db_dir): - self.db_dir = db_dir - self.db = None - self.private_key = None - self.encoded_public_key = None - self.peer_pub_keys = {} - self.queued_payments = defaultdict(int) - self.expected_payments = defaultdict(list) - self.received_payments = defaultdict(list) - self.next_manage_call = None - self.payment_check_window = 3 * 60 # 3 minutes - self.new_payments_expected_time = time.time() - self.payment_check_window - self.known_transactions = [] - self.total_reserved_points = 0.0 - self.wallet_balance = 0.0 - - def manage(self): - """Send payments, ensure expected payments are received""" - - from twisted.internet import reactor - - if time.time() < self.new_payments_expected_time + self.payment_check_window: - d1 = self._get_new_payments() - else: - d1 = defer.succeed(None) - d1.addCallback(lambda _: self._check_good_standing()) - d2 = self._send_queued_points() - self.next_manage_call = reactor.callLater(60, self.manage) - dl = defer.DeferredList([d1, d2]) - dl.addCallback(lambda _: self.get_balance()) - - def set_balance(balance): - self.wallet_balance = balance - - dl.addCallback(set_balance) - return dl - - def stop(self): - if self.next_manage_call is not None: - self.next_manage_call.cancel() - self.next_manage_call = None - d = self.manage() - self.next_manage_call.cancel() - self.next_manage_call = None - self.db = None - return d - - def start(self): - - def save_key(success, private_key): - if success is True: - self._save_private_key(private_key.exportKey()) - return True - return False - - def register_private_key(private_key): - self.private_key = private_key - self.encoded_public_key = self.private_key.publickey().exportKey() - d_r = pointtraderclient.register_new_account(private_key) - d_r.addCallback(save_key, private_key) - return d_r - - def ensure_private_key_exists(encoded_private_key): - if encoded_private_key is not None: - self.private_key = RSA.importKey(encoded_private_key) - self.encoded_public_key = self.private_key.publickey().exportKey() - return True - else: - create_d = threads.deferToThread(RSA.generate, 4096) - create_d.addCallback(register_private_key) - return create_d - - def start_manage(): - self.manage() - return True - d = self._open_db() - d.addCallback(lambda _: self._get_wallet_private_key()) - d.addCallback(ensure_private_key_exists) - d.addCallback(lambda _: start_manage()) - return d - - def get_info_exchanger(self): - return PointTraderKeyExchanger(self) - - def get_wallet_info_query_handler_factory(self): - return PointTraderKeyQueryHandlerFactory(self) - - def reserve_points(self, peer, amount): - """Ensure a certain amount of points are available to be sent as - payment, before the service is rendered - - @param peer: The peer to which the payment will ultimately be sent - - @param amount: The amount of points to reserve - - @return: A ReservedPoints object which is given to send_points - once the service has been rendered - - """ - if self.wallet_balance >= self.total_reserved_points + amount: - self.total_reserved_points += amount - return ReservedPoints(peer, amount) - return None - - def cancel_point_reservation(self, reserved_points): - """ - Return all of the points that were reserved previously for some ReservedPoints object - - @param reserved_points: ReservedPoints previously returned by reserve_points - - @return: None - """ - self.total_reserved_points -= reserved_points.amount - - def send_points(self, reserved_points, amount): - """ - Schedule a payment to be sent to a peer - - @param reserved_points: ReservedPoints object previously returned by reserve_points - - @param amount: amount of points to actually send, must be less than or equal to the - amount reserved in reserved_points - - @return: Deferred which fires when the payment has been scheduled - """ - self.queued_payments[reserved_points.identifier] += amount - # make any unused points available - self.total_reserved_points -= reserved_points.amount - amount - reserved_points.identifier.update_stats('points_sent', amount) - d = defer.succeed(True) - return d - - def _send_queued_points(self): - ds = [] - for peer, points in self.queued_payments.items(): - if peer in self.peer_pub_keys: - d = pointtraderclient.send_points( - self.private_key, self.peer_pub_keys[peer], points) - self.wallet_balance -= points - self.total_reserved_points -= points - ds.append(d) - del self.queued_payments[peer] - else: - log.warning("Don't have a payment address for peer %s. Can't send %s points.", - str(peer), str(points)) - return defer.DeferredList(ds) - - def get_balance(self): - """Return the balance of this wallet""" - d = pointtraderclient.get_balance(self.private_key) - return d - - def add_expected_payment(self, peer, amount): - """Increase the number of points expected to be paid by a peer""" - self.expected_payments[peer].append((amount, time.time())) - self.new_payments_expected_time = time.time() - peer.update_stats('expected_points', amount) - - def set_public_key_for_peer(self, peer, pub_key): - self.peer_pub_keys[peer] = pub_key - - def _get_new_payments(self): - - def add_new_transactions(transactions): - for transaction in transactions: - if transaction[1] == self.encoded_public_key: - t_hash = SHA512.new() - t_hash.update(transaction[0]) - t_hash.update(transaction[1]) - t_hash.update(str(transaction[2])) - t_hash.update(transaction[3]) - if t_hash.hexdigest() not in self.known_transactions: - self.known_transactions.append(t_hash.hexdigest()) - self._add_received_payment(transaction[0], transaction[2]) - - d = pointtraderclient.get_recent_transactions(self.private_key) - d.addCallback(add_new_transactions) - return d - - def _add_received_payment(self, encoded_other_public_key, amount): - self.received_payments[encoded_other_public_key].append((amount, time.time())) - - def _check_good_standing(self): - for peer, expected_payments in self.expected_payments.iteritems(): - expected_cutoff = time.time() - 90 - min_expected_balance = sum([a[0] for a in expected_payments if a[1] < expected_cutoff]) - received_balance = 0 - if self.peer_pub_keys[peer] in self.received_payments: - received_balance = sum([ - a[0] for a in self.received_payments[self.peer_pub_keys[peer]]]) - if min_expected_balance > received_balance: - log.warning( - "Account in bad standing: %s (pub_key: %s), expected amount = %s, " - "received_amount = %s", - peer, self.peer_pub_keys[peer], min_expected_balance, received_balance) - - def _open_db(self): - def open_db(): - self.db = unqlite.UnQLite(os.path.join(self.db_dir, "ptcwallet.db")) - return threads.deferToThread(open_db) - - def _save_private_key(self, private_key): - def save_key(): - self.db['private_key'] = private_key - return threads.deferToThread(save_key) - - def _get_wallet_private_key(self): - def get_key(): - if 'private_key' in self.db: - return self.db['private_key'] - return None - return threads.deferToThread(get_key) - - -class PointTraderKeyExchanger(object): - implements([IRequestCreator]) - - def __init__(self, wallet): - self.wallet = wallet - self._protocols = [] - - ######### IRequestCreator ######### - - def send_next_request(self, peer, protocol): - if not protocol in self._protocols: - r = ClientRequest({'public_key': self.wallet.encoded_public_key}, - 'public_key') - d = protocol.add_request(r) - d.addCallback(self._handle_exchange_response, peer, r, protocol) - d.addErrback(self._request_failed, peer) - self._protocols.append(protocol) - return defer.succeed(True) - else: - return defer.succeed(False) - - ######### internal calls ######### - - def _handle_exchange_response(self, response_dict, peer, request, protocol): - assert request.response_identifier in response_dict, \ - "Expected %s in dict but did not get it" % request.response_identifier - assert protocol in self._protocols, "Responding protocol is not in our list of protocols" - peer_pub_key = response_dict[request.response_identifier] - self.wallet.set_public_key_for_peer(peer, peer_pub_key) - return True - - def _request_failed(self, err, peer): - if not err.check(RequestCanceledError): - log.warning("A peer failed to send a valid public key response. Error: %s, peer: %s", - err.getErrorMessage(), str(peer)) - return err - - -class PointTraderKeyQueryHandlerFactory(object): - implements(IQueryHandlerFactory) - - def __init__(self, wallet): - self.wallet = wallet - - ######### IQueryHandlerFactory ######### - - def build_query_handler(self): - q_h = PointTraderKeyQueryHandler(self.wallet) - return q_h - - def get_primary_query_identifier(self): - return 'public_key' - - def get_description(self): - return ("Point Trader Address - an address for receiving payments on the " - "point trader testing network") - - -class PointTraderKeyQueryHandler(object): - implements(IQueryHandler) - - def __init__(self, wallet): - self.wallet = wallet - self.query_identifiers = ['public_key'] - self.public_key = None - self.peer = None - - ######### IQueryHandler ######### - - def register_with_request_handler(self, request_handler, peer): - self.peer = peer - request_handler.register_query_handler(self, self.query_identifiers) - - def handle_queries(self, queries): - if self.query_identifiers[0] in queries: - new_encoded_pub_key = queries[self.query_identifiers[0]] - try: - RSA.importKey(new_encoded_pub_key) - except (ValueError, TypeError, IndexError): - log.warning("Client sent an invalid public key.") - return defer.fail(Failure(ValueError("Client sent an invalid public key"))) - self.public_key = new_encoded_pub_key - self.wallet.set_public_key_for_peer(self.peer, self.public_key) - log.debug("Received the client's public key: %s", str(self.public_key)) - fields = {'public_key': self.wallet.encoded_public_key} - return defer.succeed(fields) - if self.public_key is None: - log.warning("Expected a public key, but did not receive one") - return defer.fail(Failure(ValueError("Expected but did not receive a public key"))) - else: - return defer.succeed({}) diff --git a/lbrynet/tests/mocks.py b/lbrynet/tests/mocks.py index 9f8133724..2661f4a6e 100644 --- a/lbrynet/tests/mocks.py +++ b/lbrynet/tests/mocks.py @@ -3,8 +3,10 @@ import io from Crypto.PublicKey import RSA from twisted.internet import defer, threads, error +from twisted.python.failure import Failure -from lbrynet.core import PTCWallet +from lbrynet.core.client.ClientRequest import ClientRequest +from lbrynet.core.Error import RequestCanceledError from lbrynet.core import BlobAvailability from lbrynet.core.utils import generate_id from lbrynet.daemon import ExchangeRateManager as ERM @@ -75,6 +77,82 @@ class ExchangeRateManager(ERM.ExchangeRateManager): feed.market, rates[feed.market]['spot'], rates[feed.market]['ts']) +class PointTraderKeyExchanger: + + def __init__(self, wallet): + self.wallet = wallet + self._protocols = [] + + def send_next_request(self, peer, protocol): + if not protocol in self._protocols: + r = ClientRequest({'public_key': self.wallet.encoded_public_key}, + 'public_key') + d = protocol.add_request(r) + d.addCallback(self._handle_exchange_response, peer, r, protocol) + d.addErrback(self._request_failed, peer) + self._protocols.append(protocol) + return defer.succeed(True) + else: + return defer.succeed(False) + + def _handle_exchange_response(self, response_dict, peer, request, protocol): + assert request.response_identifier in response_dict, \ + "Expected %s in dict but did not get it" % request.response_identifier + assert protocol in self._protocols, "Responding protocol is not in our list of protocols" + peer_pub_key = response_dict[request.response_identifier] + self.wallet.set_public_key_for_peer(peer, peer_pub_key) + return True + + def _request_failed(self, err, peer): + if not err.check(RequestCanceledError): + return err + + +class PointTraderKeyQueryHandlerFactory: + + def __init__(self, wallet): + self.wallet = wallet + + def build_query_handler(self): + q_h = PointTraderKeyQueryHandler(self.wallet) + return q_h + + def get_primary_query_identifier(self): + return 'public_key' + + def get_description(self): + return ("Point Trader Address - an address for receiving payments on the " + "point trader testing network") + + +class PointTraderKeyQueryHandler: + + def __init__(self, wallet): + self.wallet = wallet + self.query_identifiers = ['public_key'] + self.public_key = None + self.peer = None + + def register_with_request_handler(self, request_handler, peer): + self.peer = peer + request_handler.register_query_handler(self, self.query_identifiers) + + def handle_queries(self, queries): + if self.query_identifiers[0] in queries: + new_encoded_pub_key = queries[self.query_identifiers[0]] + try: + RSA.importKey(new_encoded_pub_key) + except (ValueError, TypeError, IndexError): + return defer.fail(Failure(ValueError("Client sent an invalid public key"))) + self.public_key = new_encoded_pub_key + self.wallet.set_public_key_for_peer(self.peer, self.public_key) + fields = {'public_key': self.wallet.encoded_public_key} + return defer.succeed(fields) + if self.public_key is None: + return defer.fail(Failure(ValueError("Expected but did not receive a public key"))) + else: + return defer.succeed({}) + class Wallet(object): def __init__(self): @@ -100,10 +178,10 @@ class Wallet(object): return defer.succeed(True) def get_info_exchanger(self): - return PTCWallet.PointTraderKeyExchanger(self) + return PointTraderKeyExchanger(self) def get_wallet_info_query_handler_factory(self): - return PTCWallet.PointTraderKeyQueryHandlerFactory(self) + return PointTraderKeyQueryHandlerFactory(self) def reserve_points(self, *args): return True diff --git a/requirements_testing.txt b/requirements_testing.txt new file mode 100644 index 000000000..e69de29bb From 267f50474bd8c958082929b66d02f71359f82fcb Mon Sep 17 00:00:00 2001 From: Lex Berezhny Date: Wed, 14 Mar 2018 18:29:50 -0400 Subject: [PATCH 40/52] removing more references to PTCWallet --- lbrynet/core/Session.py | 4 ---- lbrynet/daemon/Daemon.py | 4 ---- 2 files changed, 8 deletions(-) diff --git a/lbrynet/core/Session.py b/lbrynet/core/Session.py index b8ef9ebcb..8ac992ec4 100644 --- a/lbrynet/core/Session.py +++ b/lbrynet/core/Session.py @@ -132,10 +132,6 @@ class Session(object): if self.node_id is None: self.node_id = generate_id() - if self.wallet is None: - from lbrynet.core.PTCWallet import PTCWallet - self.wallet = PTCWallet(self.db_dir) - if self.use_upnp is True: d = self._try_upnp() else: diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index 652fb370e..b276aefb6 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -547,10 +547,6 @@ class Daemon(AuthJSONRPCServer): config['lbryum_path'] = conf.settings['lbryum_wallet_dir'] wallet = LBRYumWallet(self.storage, config) return defer.succeed(wallet) - elif self.wallet_type == PTC_WALLET: - log.info("Using PTC wallet") - from lbrynet.core.PTCWallet import PTCWallet - return defer.succeed(PTCWallet(self.db_dir)) else: raise ValueError('Wallet Type {} is not valid'.format(self.wallet_type)) From 333d70860bb53246c2f0e2561ec1fcf2baa47a08 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 27 Mar 2018 14:58:29 -0400 Subject: [PATCH 41/52] add last_announced_time to blob table --- lbrynet/daemon/Daemon.py | 2 +- lbrynet/database/migrator/dbmigrator.py | 8 +++--- lbrynet/database/migrator/migrate6to7.py | 31 ++++++++++++++++++++++++ lbrynet/database/storage.py | 7 +++--- 4 files changed, 39 insertions(+), 9 deletions(-) create mode 100644 lbrynet/database/migrator/migrate6to7.py diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index b276aefb6..3e90e4111 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -198,7 +198,7 @@ class Daemon(AuthJSONRPCServer): self.connected_to_internet = True self.connection_status_code = None self.platform = None - self.current_db_revision = 6 + self.current_db_revision = 7 self.db_revision_file = conf.settings.get_db_revision_filename() self.session = None self._session_id = conf.settings.get_session_id() diff --git a/lbrynet/database/migrator/dbmigrator.py b/lbrynet/database/migrator/dbmigrator.py index e5f141f3a..a4057db38 100644 --- a/lbrynet/database/migrator/dbmigrator.py +++ b/lbrynet/database/migrator/dbmigrator.py @@ -6,22 +6,20 @@ def migrate_db(db_dir, start, end): while current < end: if current == 1: from lbrynet.database.migrator.migrate1to2 import do_migration - do_migration(db_dir) elif current == 2: from lbrynet.database.migrator.migrate2to3 import do_migration - do_migration(db_dir) elif current == 3: from lbrynet.database.migrator.migrate3to4 import do_migration - do_migration(db_dir) elif current == 4: from lbrynet.database.migrator.migrate4to5 import do_migration - do_migration(db_dir) elif current == 5: from lbrynet.database.migrator.migrate5to6 import do_migration - do_migration(db_dir) + elif current == 6: + from lbrynet.database.migrator.migrate6to7 import do_migration else: raise Exception("DB migration of version {} to {} is not available".format(current, current+1)) + do_migration(db_dir) current += 1 return None diff --git a/lbrynet/database/migrator/migrate6to7.py b/lbrynet/database/migrator/migrate6to7.py new file mode 100644 index 000000000..d1778a83e --- /dev/null +++ b/lbrynet/database/migrator/migrate6to7.py @@ -0,0 +1,31 @@ +import sqlite3 +import os +import logging +from lbrynet.database.storage import SQLiteStorage + +log = logging.getLogger(__name__) + + +def run_operation(db): + def _decorate(fn): + def _wrapper(*args): + cursor = db.cursor() + try: + result = fn(cursor, *args) + db.commit() + return result + except sqlite3.IntegrityError: + db.rollback() + raise + return _wrapper + return _decorate + + +def do_migration(db_dir): + db_path = os.path.join(db_dir, "lbrynet.sqlite") + connection = sqlite3.connect(db_path) + cursor = connection.cursor() + cursor.executescript("alter table blob add last_announced_time integer;") + cursor.execute("update blob set next_announce_time=0") + connection.commit() + connection.close() diff --git a/lbrynet/database/storage.py b/lbrynet/database/storage.py index a77a8dae8..88216475d 100644 --- a/lbrynet/database/storage.py +++ b/lbrynet/database/storage.py @@ -124,7 +124,8 @@ class SQLiteStorage(object): blob_length integer not null, next_announce_time integer not null, should_announce integer not null default 0, - status text not null + status text not null, + last_announced_time integer ); create table if not exists stream ( @@ -250,8 +251,8 @@ class SQLiteStorage(object): status = yield self.get_blob_status(blob_hash) if status is None: status = "pending" - yield self.db.runOperation("insert into blob values (?, ?, ?, ?, ?)", - (blob_hash, length, 0, 0, status)) + yield self.db.runOperation("insert into blob values (?, ?, ?, ?, ?, ?)", + (blob_hash, length, 0, 0, status, 0)) defer.returnValue(status) def should_announce(self, blob_hash): From a8025b02c68d5883fafc69c4d53eafa33aef163a Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 27 Mar 2018 14:59:39 -0400 Subject: [PATCH 42/52] log invalid vs missing token --- lbrynet/dht/node.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index 00ec2924b..c7dbdf119 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -515,18 +515,20 @@ class Node(object): else: raise TypeError, 'No contact info available' - if ((self_store is False) and - ('token' not in value or not self.verify_token(value['token'], compact_ip))): - raise ValueError('Invalid or missing token') + if not self_store: + if 'token' not in value: + raise ValueError("Missing token") + if not self.verify_token(value['token'], compact_ip): + raise ValueError("Invalid token") if 'port' in value: port = int(value['port']) if 0 <= port <= 65536: compact_port = str(struct.pack('>H', port)) else: - raise TypeError, 'Invalid port' + raise TypeError('Invalid port') else: - raise TypeError, 'No port available' + raise TypeError('No port available') if 'lbryid' in value: if len(value['lbryid']) != constants.key_bits / 8: @@ -535,7 +537,7 @@ class Node(object): else: compact_address = compact_ip + compact_port + value['lbryid'] else: - raise TypeError, 'No lbryid given' + raise TypeError('No lbryid given') now = int(time.time()) originallyPublished = now # - age From 14f9bb7b820ad46a16442764cf11dd5a607d2c04 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 27 Mar 2018 15:13:01 -0400 Subject: [PATCH 43/52] log EWOULDBLOCK --- lbrynet/dht/protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/dht/protocol.py b/lbrynet/dht/protocol.py index 505ce4202..7e018c68a 100644 --- a/lbrynet/dht/protocol.py +++ b/lbrynet/dht/protocol.py @@ -345,7 +345,7 @@ class KademliaProtocol(protocol.DatagramProtocol): if err.errno == errno.EWOULDBLOCK: # i'm scared this may swallow important errors, but i get a million of these # on Linux and it doesnt seem to affect anything -grin - log.debug("Can't send data to dht: EWOULDBLOCK") + log.warning("Can't send data to dht: EWOULDBLOCK") elif err.errno == errno.ENETUNREACH: # this should probably try to retransmit when the network connection is back log.error("Network is unreachable") From ea0ea704a2e072645d62516b8f97ea89514fa88c Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 27 Mar 2018 15:04:47 -0400 Subject: [PATCH 44/52] refactor iterativeAnnounceHaveBlob -change to only self_store if the number of contacts to store to is less than k and we are the closest node to the hash --- lbrynet/dht/node.py | 109 ++++++++++++++++++-------------------------- 1 file changed, 44 insertions(+), 65 deletions(-) diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index c7dbdf119..0adc4f7b6 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -272,78 +272,57 @@ class Node(object): def get_bandwidth_stats(self): return self._protocol.bandwidth_stats + @defer.inlineCallbacks def iterativeAnnounceHaveBlob(self, blob_hash, value): known_nodes = {} - - @defer.inlineCallbacks - def announce_to_peer(responseTuple): - """ @type responseMsg: kademlia.msgtypes.ResponseMessage """ - # The "raw response" tuple contains the response message, - # and the originating address info - responseMsg = responseTuple[0] - originAddress = responseTuple[1] # tuple: (ip adress, udp port) - # Make sure the responding node is valid, and abort the operation if it isn't - if not responseMsg.nodeID in known_nodes: - log.warning("Responding node was not expected") - defer.returnValue(responseMsg.nodeID) - remote_node = known_nodes[responseMsg.nodeID] - - result = responseMsg.response - announced = False - if 'token' in result: - value['token'] = result['token'] - try: - res = yield remote_node.store(blob_hash, value) - assert res == "OK", "unexpected response: {}".format(res) - announced = True - except protocol.TimeoutError: - log.info("Timeout while storing blob_hash %s at %s", - blob_hash.encode('hex')[:16], remote_node.id.encode('hex')) - except Exception as err: - log.error("Unexpected error while storing blob_hash %s at %s: %s", - blob_hash.encode('hex')[:16], remote_node.id.encode('hex'), err) - else: - log.warning("missing token") - defer.returnValue(announced) - - @defer.inlineCallbacks - def requestPeers(contacts): - if self.externalIP is not None and len(contacts) >= constants.k: - is_closer = Distance(blob_hash).is_closer(self.node_id, contacts[-1].id) - if is_closer: - contacts.pop() - yield self.store(blob_hash, value, originalPublisherID=self.node_id, - self_store=True) - elif self.externalIP is not None: + contacts = yield self.iterativeFindNode(blob_hash) + # store locally if we're the closest node and there are less than k contacts to try storing to + if self.externalIP is not None and contacts and len(contacts) < constants.k: + is_closer = Distance(blob_hash).is_closer(self.node_id, contacts[-1].id) + if is_closer: + contacts.pop() yield self.store(blob_hash, value, originalPublisherID=self.node_id, self_store=True) - else: - raise Exception("Cannot determine external IP: %s" % self.externalIP) + elif self.externalIP is not None: + pass + else: + raise Exception("Cannot determine external IP: %s" % self.externalIP) - contacted = [] - for contact in contacts: - known_nodes[contact.id] = contact - rpcMethod = getattr(contact, "findValue") - try: - response = yield rpcMethod(blob_hash, rawResponse=True) - stored = yield announce_to_peer(response) - if stored: - contacted.append(contact) - except protocol.TimeoutError: - log.debug("Timeout while storing blob_hash %s at %s", - binascii.hexlify(blob_hash), contact) - except Exception as err: - log.error("Unexpected error while storing blob_hash %s at %s: %s", - binascii.hexlify(blob_hash), contact, err) - log.debug("Stored %s to %i of %i attempted peers", blob_hash.encode('hex')[:16], - len(contacted), len(contacts)) + contacted = [] - contacted_node_ids = [c.id.encode('hex') for c in contacts] - defer.returnValue(contacted_node_ids) + @defer.inlineCallbacks + def announce_to_contact(contact): + known_nodes[contact.id] = contact + try: + responseMsg, originAddress = yield contact.findValue(blob_hash, rawResponse=True) + if responseMsg.nodeID != contact.id: + raise Exception("node id mismatch") + value['token'] = responseMsg.response['token'] + res = yield contact.store(blob_hash, value) + if res != "OK": + raise ValueError(res) + contacted.append(contact) + log.debug("Stored %s to %s (%s)", blob_hash.encode('hex'), contact.id.encode('hex'), originAddress[0]) + except protocol.TimeoutError: + log.debug("Timeout while storing blob_hash %s at %s", + blob_hash.encode('hex')[:16], contact.id.encode('hex')) + except ValueError as err: + log.error("Unexpected response: %s" % err.message) + except Exception as err: + log.error("Unexpected error while storing blob_hash %s at %s: %s", + binascii.hexlify(blob_hash), contact, err) - d = self.iterativeFindNode(blob_hash) - d.addCallback(requestPeers) - return d + dl = [] + for c in contacts: + dl.append(announce_to_contact(c)) + + yield defer.DeferredList(dl) + + log.debug("Stored %s to %i of %i attempted peers", blob_hash.encode('hex')[:16], + len(contacted), len(contacts)) + + contacted_node_ids = [c.id.encode('hex') for c in contacted] + defer.returnValue(contacted_node_ids) def change_token(self): self.old_token_secret = self.token_secret From c5bf64cf0aa5d2b2e926db2510341b1c3ec7d811 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 27 Mar 2018 15:12:44 -0400 Subject: [PATCH 45/52] refactor DHTHashAnnouncer -remove hash_announcer from Node and DiskBlobManager -remove announcement related functions from DiskBlobManager -update SQLiteStorage to store announcement times and provide blob hashes needing to be announced -use dataExpireTimeout from lbrynet.dht.constants for re-announce timing -use DeferredSemaphore for concurrent blob announcement --- lbrynet/core/BlobManager.py | 42 +---- lbrynet/core/Session.py | 19 +- lbrynet/core/SinglePeerDownloader.py | 4 +- lbrynet/database/storage.py | 43 ++--- lbrynet/dht/hashannouncer.py | 259 ++++++--------------------- lbrynet/dht/node.py | 13 +- 6 files changed, 86 insertions(+), 294 deletions(-) diff --git a/lbrynet/core/BlobManager.py b/lbrynet/core/BlobManager.py index 8db8b4d05..e2ccbfb04 100644 --- a/lbrynet/core/BlobManager.py +++ b/lbrynet/core/BlobManager.py @@ -10,16 +10,14 @@ log = logging.getLogger(__name__) class DiskBlobManager(object): - def __init__(self, hash_announcer, blob_dir, storage): + def __init__(self, blob_dir, storage): + """ + This class stores blobs on the hard disk - """ - This class stores blobs on the hard disk, blob_dir - directory where blobs are stored - db_dir - directory where sqlite database of blob information is stored + storage - SQLiteStorage object """ - self.hash_announcer = hash_announcer self.storage = storage - self.announce_head_blobs_only = conf.settings['announce_head_blobs_only'] self.blob_dir = blob_dir self.blob_creator_type = BlobFileCreator # TODO: consider using an LRU for blobs as there could potentially @@ -28,7 +26,7 @@ class DiskBlobManager(object): self.blob_hashes_to_delete = {} # {blob_hash: being_deleted (True/False)} self.check_should_announce_lc = None - if conf.settings['run_reflector_server']: + if conf.settings['run_reflector_server']: # TODO: move this looping call to SQLiteStorage self.check_should_announce_lc = task.LoopingCall(self.storage.verify_will_announce_all_head_and_sd_blobs) def setup(self): @@ -60,40 +58,20 @@ class DiskBlobManager(object): self.blobs[blob_hash] = blob return defer.succeed(blob) - def immediate_announce(self, blob_hashes): - if self.hash_announcer: - return self.hash_announcer.immediate_announce(blob_hashes) - raise Exception("Hash announcer not set") - @defer.inlineCallbacks def blob_completed(self, blob, next_announce_time=None, should_announce=True): - if next_announce_time is None: - next_announce_time = self.hash_announcer.get_next_announce_time() yield self.storage.add_completed_blob( blob.blob_hash, blob.length, next_announce_time, should_announce ) - # we announce all blobs immediately, if announce_head_blob_only is False - # otherwise, announce only if marked as should_announce - if not self.announce_head_blobs_only or should_announce: - self.immediate_announce([blob.blob_hash]) def completed_blobs(self, blobhashes_to_check): return self._completed_blobs(blobhashes_to_check) - def hashes_to_announce(self): - return self.storage.get_blobs_to_announce(self.hash_announcer) - def count_should_announce_blobs(self): return self.storage.count_should_announce_blobs() def set_should_announce(self, blob_hash, should_announce): - if blob_hash in self.blobs: - blob = self.blobs[blob_hash] - if blob.get_is_verified(): - return self.storage.set_should_announce( - blob_hash, self.hash_announcer.get_next_announce_time(), should_announce - ) - return defer.succeed(False) + return self.storage.set_should_announce(blob_hash, should_announce) def get_should_announce(self, blob_hash): return self.storage.should_announce(blob_hash) @@ -108,13 +86,7 @@ class DiskBlobManager(object): raise Exception("Blob has a length of 0") new_blob = BlobFile(self.blob_dir, blob_creator.blob_hash, blob_creator.length) self.blobs[blob_creator.blob_hash] = new_blob - next_announce_time = self.hash_announcer.get_next_announce_time() - return self.blob_completed(new_blob, next_announce_time, should_announce) - - def immediate_announce_all_blobs(self): - d = self._get_all_verified_blob_hashes() - d.addCallback(self.immediate_announce) - return d + return self.blob_completed(new_blob, should_announce) def get_all_verified_blobs(self): d = self._get_all_verified_blob_hashes() diff --git a/lbrynet/core/Session.py b/lbrynet/core/Session.py index 8ac992ec4..f65a331fe 100644 --- a/lbrynet/core/Session.py +++ b/lbrynet/core/Session.py @@ -2,7 +2,7 @@ import logging import miniupnpc from twisted.internet import threads, defer from lbrynet.core.BlobManager import DiskBlobManager -from lbrynet.dht import node +from lbrynet.dht import node, hashannouncer from lbrynet.database.storage import SQLiteStorage from lbrynet.core.RateLimiter import RateLimiter from lbrynet.core.utils import generate_id @@ -136,6 +136,7 @@ class Session(object): d = self._try_upnp() else: d = defer.succeed(True) + d.addCallback(lambda _: self.storage.setup()) d.addCallback(lambda _: self._setup_dht()) d.addCallback(lambda _: self._setup_other_components()) return d @@ -144,6 +145,8 @@ class Session(object): """Stop all services""" log.info('Stopping session.') ds = [] + if self.hash_announcer: + self.hash_announcer.stop() if self.blob_tracker is not None: ds.append(defer.maybeDeferred(self.blob_tracker.stop)) if self.dht_node is not None: @@ -220,19 +223,20 @@ class Session(object): def _setup_dht(self): # does not block startup, the dht will re-attempt if necessary self.dht_node = self.dht_node_class( - self.hash_announcer, - udpPort=self.dht_node_port, node_id=self.node_id, + udpPort=self.dht_node_port, externalIP=self.external_ip, peerPort=self.peer_port, peer_manager=self.peer_manager, peer_finder=self.peer_finder, ) + if not self.hash_announcer: + self.hash_announcer = hashannouncer.DHTHashAnnouncer(self.dht_node, self.storage) self.peer_manager = self.dht_node.peer_manager self.peer_finder = self.dht_node.peer_finder - self.hash_announcer = self.dht_node.hash_announcer self._join_dht_deferred = self.dht_node.joinNetwork(self.known_dht_nodes) self._join_dht_deferred.addCallback(lambda _: log.info("Joined the dht")) + self._join_dht_deferred.addCallback(lambda _: self.hash_announcer.start()) def _setup_other_components(self): log.debug("Setting up the rest of the components") @@ -245,9 +249,7 @@ class Session(object): raise Exception( "TempBlobManager is no longer supported, specify BlobManager or db_dir") else: - self.blob_manager = DiskBlobManager( - self.dht_node.hash_announcer, self.blob_dir, self.storage - ) + self.blob_manager = DiskBlobManager(self.blob_dir, self.storage) if self.blob_tracker is None: self.blob_tracker = self.blob_tracker_class( @@ -259,8 +261,7 @@ class Session(object): ) self.rate_limiter.start() - d = self.storage.setup() - d.addCallback(lambda _: self.blob_manager.setup()) + d = self.blob_manager.setup() d.addCallback(lambda _: self.wallet.start()) d.addCallback(lambda _: self.blob_tracker.start()) return d diff --git a/lbrynet/core/SinglePeerDownloader.py b/lbrynet/core/SinglePeerDownloader.py index 9073e980b..904927080 100644 --- a/lbrynet/core/SinglePeerDownloader.py +++ b/lbrynet/core/SinglePeerDownloader.py @@ -11,7 +11,6 @@ from lbrynet.core.PaymentRateManager import OnlyFreePaymentsManager from lbrynet.core.client.BlobRequester import BlobRequester from lbrynet.core.client.StandaloneBlobDownloader import StandaloneBlobDownloader from lbrynet.core.client.ConnectionManager import ConnectionManager -from lbrynet.dht.hashannouncer import DummyHashAnnouncer from lbrynet.dht.peerfinder import DummyPeerFinder @@ -61,7 +60,6 @@ class SingleBlobDownloadManager(object): class SinglePeerDownloader(object): def __init__(self): self._payment_rate_manager = OnlyFreePaymentsManager() - self._announcer = DummyHashAnnouncer() self._rate_limiter = DummyRateLimiter() self._wallet = None self._blob_manager = None @@ -98,7 +96,7 @@ class SinglePeerDownloader(object): @defer.inlineCallbacks def download_temp_blob_from_peer(self, peer, timeout, blob_hash): tmp_dir = yield threads.deferToThread(tempfile.mkdtemp) - tmp_blob_manager = DiskBlobManager(self._announcer, tmp_dir, tmp_dir) + tmp_blob_manager = DiskBlobManager(tmp_dir, tmp_dir) try: result = yield self.download_blob_from_peer(peer, timeout, blob_hash, tmp_blob_manager) finally: diff --git a/lbrynet/database/storage.py b/lbrynet/database/storage.py index 88216475d..4e6a9c669 100644 --- a/lbrynet/database/storage.py +++ b/lbrynet/database/storage.py @@ -1,6 +1,5 @@ import logging import os -import time import sqlite3 import traceback from decimal import Decimal @@ -11,6 +10,7 @@ from lbryschema.claim import ClaimDict from lbryschema.decode import smart_decode from lbrynet import conf from lbrynet.cryptstream.CryptBlob import CryptBlobInfo +from lbrynet.dht.constants import dataExpireTimeout from lbryum.constants import COIN log = logging.getLogger(__name__) @@ -49,26 +49,6 @@ def open_file_for_writing(download_directory, suggested_file_name): return threads.deferToThread(_open_file_for_writing, download_directory, suggested_file_name) -def get_next_announce_time(hash_announcer, num_hashes_to_announce=1, min_reannounce_time=60*60, - single_announce_duration=5): - """ - Hash reannounce time is set to current time + MIN_HASH_REANNOUNCE_TIME, - unless we are announcing a lot of hashes at once which could cause the - the announce queue to pile up. To prevent pile up, reannounce - only after a conservative estimate of when it will finish - to announce all the hashes. - - Args: - num_hashes_to_announce: number of hashes that will be added to the queue - Returns: - timestamp for next announce time - """ - queue_size = hash_announcer.hash_queue_size() + num_hashes_to_announce - reannounce = max(min_reannounce_time, - queue_size * single_announce_duration) - return time.time() + reannounce - - def rerun_if_locked(f): max_attempts = 3 @@ -186,6 +166,7 @@ class SQLiteStorage(object): log.info("connecting to database: %s", self._db_path) self.db = SqliteConnection(self._db_path) self.db.set_reactor(reactor) + self.clock = reactor # used to refresh the claim attributes on a ManagedEncryptedFileDownloader when a # change to the associated content claim occurs. these are added by the file manager @@ -270,9 +251,15 @@ class SQLiteStorage(object): "select blob_hash from blob where should_announce=1 and status='finished'" ) - def get_blobs_to_announce(self, hash_announcer): + def update_last_announced_blob(self, blob_hash, last_announced): + return self.db.runOperation( + "update blob set next_announce_time=?, last_announced_time=? where blob_hash=?", + (int(last_announced + (dataExpireTimeout / 2)), int(last_announced), blob_hash) + ) + + def get_blobs_to_announce(self): def get_and_update(transaction): - timestamp = time.time() + timestamp = self.clock.seconds() if conf.settings['announce_head_blobs_only']: r = transaction.execute( "select blob_hash from blob " @@ -284,16 +271,8 @@ class SQLiteStorage(object): "select blob_hash from blob where blob_hash is not null " "and next_announce_time 0: - estimated_time_remaining = int(float(hashes) / blobs_per_second) - remaining = str(datetime.timedelta(seconds=estimated_time_remaining)) - else: - remaining = "unknown" - log.info("Announcing blobs: %i blobs left to announce, %i%s complete, " - "est time remaining: %s", hashes + self._concurrent_announcers, - 100 - int(100.0 * float(hashes + self._concurrent_announcers) / - float(self._total)), "%", remaining) - self._last_checked = t + last_time, hashes - else: - self._total = 0 - if self.peer_port is not None: - return self._announce_available_hashes() + def start(self): + self._manage_lc.start(30) def stop(self): - log.info("Stopping DHT hash announcer.") - if self._manage_call_lc.running: - return self._manage_call_lc.stop() + if self._manage_lc.running: + self._manage_lc.stop() - def immediate_announce(self, blob_hashes): - if self.peer_port is not None: - return self._announce_hashes(blob_hashes, immediate=True) + @defer.inlineCallbacks + def do_store(self, blob_hash): + storing_node_ids = yield self.dht_node.announceHaveBlob(binascii.unhexlify(blob_hash)) + now = self.clock.seconds() + if storing_node_ids: + result = (now, storing_node_ids) + yield self.storage.update_last_announced_blob(blob_hash, now) + log.debug("Stored %s to %i peers", blob_hash[:16], len(storing_node_ids)) else: - return defer.succeed(False) + result = (None, []) + self.hash_queue.remove(blob_hash) + defer.returnValue(result) - def hash_queue_size(self): - return len(self.hash_queue) + def _show_announce_progress(self, size, start): + queue_size = len(self.hash_queue) + average_blobs_per_second = float(size - queue_size) / (self.clock.seconds() - start) + log.info("Announced %i/%i blobs, %f blobs per second", size - queue_size, size, average_blobs_per_second) @defer.inlineCallbacks - def _announce_available_hashes(self): - log.debug('Announcing available hashes') - hashes = yield self.hashes_to_announce() - yield self._announce_hashes(hashes) + def immediate_announce(self, blob_hashes): + blob_hashes = [b for b in blob_hashes if b not in self.hash_queue] + self.hash_queue.extend(blob_hashes) + + log.info("Announcing %i blobs", len(self.hash_queue)) + start = self.clock.seconds() + progress_lc = task.LoopingCall(self._show_announce_progress, len(self.hash_queue), start) + progress_lc.start(60, now=False) + s = defer.DeferredSemaphore(self.concurrent_announcers) + results = yield utils.DeferredDict({blob_hash: s.run(self.do_store, blob_hash) for blob_hash in blob_hashes}) + now = self.clock.seconds() + + progress_lc.stop() + + announced_to = [blob_hash for blob_hash in results if results[blob_hash][0]] + if len(announced_to) != len(results): + log.debug("Failed to announce %i blobs", len(results) - len(announced_to)) + if announced_to: + log.info('Took %s seconds to announce %i of %i attempted hashes (%f hashes per second)', + now - start, len(blob_hashes), len(announced_to), + int(float(len(blob_hashes)) / float(now - start))) + defer.returnValue(results) @defer.inlineCallbacks - def _announce_hashes(self, hashes, immediate=False): - if not hashes: - defer.returnValue(None) - if not self.dht_node.can_store: - log.warning("Client only DHT node cannot store, skipping announce") - defer.returnValue(None) - log.info('Announcing %s hashes', len(hashes)) - # TODO: add a timeit decorator - start = self.dht_node.clock.seconds() - - ds = [] - with self._lock: - for h in hashes: - announce_deferred = defer.Deferred() - if immediate: - self.hash_queue.appendleft((h, announce_deferred)) - else: - self.hash_queue.append((h, announce_deferred)) - if not self._total: - self._total = len(hashes) - - log.debug('There are now %s hashes remaining to be announced', self.hash_queue_size()) - - @defer.inlineCallbacks - def do_store(blob_hash, announce_d, retry_count=0): - if announce_d.called: - defer.returnValue(announce_deferred.result) - try: - store_nodes = yield self.dht_node.announceHaveBlob(binascii.unhexlify(blob_hash)) - if not store_nodes: - retry_count += 1 - if retry_count <= self.STORE_RETRIES: - log.debug("No nodes stored %s, retrying", blob_hash) - result = yield do_store(blob_hash, announce_d, retry_count) - else: - result = {} - log.warning("No nodes stored %s", blob_hash) - else: - result = store_nodes - if not announce_d.called: - announce_d.callback(result) - defer.returnValue(result) - except Exception as err: - if not announce_d.called: - announce_d.errback(err) - raise err - - @defer.inlineCallbacks - def announce(progress=None): - progress = progress or {} - if len(self.hash_queue): - with self._lock: - h, announce_deferred = self.hash_queue.popleft() - log.debug('Announcing blob %s to dht', h[:16]) - stored_to_nodes = yield do_store(h, announce_deferred) - progress[h] = stored_to_nodes - log.debug("Stored %s to %i peers (hashes announced by this announcer: %i)", - h.encode('hex')[:16], - len(stored_to_nodes), len(progress)) - - yield announce(progress) - else: - with self._lock: - self._concurrent_announcers -= 1 - defer.returnValue(progress) - - for i in range(self._concurrent_announcers, self.CONCURRENT_ANNOUNCERS): - self._concurrent_announcers += 1 - ds.append(announce()) - announcer_results = yield defer.DeferredList(ds) - stored_to = {} - for _, announced_to in announcer_results: - stored_to.update(announced_to) - - log.info('Took %s seconds to announce %s hashes', self.dht_node.clock.seconds() - start, len(hashes)) - seconds_per_blob = (self.dht_node.clock.seconds() - start) / len(hashes) - self.set_single_hash_announce_duration(seconds_per_blob) - defer.returnValue(stored_to) - - @defer.inlineCallbacks - def add_hashes_to_announce(self, blob_hashes): - yield self._lock._lock.acquire() - self._hashes_to_announce.extend(blob_hashes) - yield self._lock._lock.release() - - @defer.inlineCallbacks - def hashes_to_announce(self): - hashes_to_announce = [] - yield self._lock._lock.acquire() - while self._hashes_to_announce: - hashes_to_announce.append(self._hashes_to_announce.pop()) - yield self._lock._lock.release() - defer.returnValue(hashes_to_announce) - - def set_single_hash_announce_duration(self, seconds): - """ - Set the duration it takes to announce a single hash - in seconds, cannot be less than the default single - hash announce duration - """ - seconds = max(seconds, self.DEFAULT_SINGLE_HASH_ANNOUNCE_DURATION) - self.single_hash_announce_duration = seconds - - def get_next_announce_time(self, num_hashes_to_announce=1): - """ - Hash reannounce time is set to current time + MIN_HASH_REANNOUNCE_TIME, - unless we are announcing a lot of hashes at once which could cause the - the announce queue to pile up. To prevent pile up, reannounce - only after a conservative estimate of when it will finish - to announce all the hashes. - - Args: - num_hashes_to_announce: number of hashes that will be added to the queue - Returns: - timestamp for next announce time - """ - queue_size = self.hash_queue_size() + num_hashes_to_announce - reannounce = max(self.MIN_HASH_REANNOUNCE_TIME, - queue_size * self.single_hash_announce_duration) - return self.dht_node.clock.seconds() + reannounce + def manage(self): + need_reannouncement = yield self.storage.get_blobs_to_announce() + if need_reannouncement: + yield self.immediate_announce(need_reannouncement) + else: + log.debug("Nothing to announce") diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index 0adc4f7b6..c77fce861 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -54,7 +54,7 @@ class Node(object): application is performed via this class (or a subclass). """ - def __init__(self, hash_announcer=None, node_id=None, udpPort=4000, dataStore=None, + def __init__(self, node_id=None, udpPort=4000, dataStore=None, routingTableClass=None, networkProtocol=None, externalIP=None, peerPort=None, listenUDP=None, callLater=None, resolve=None, clock=None, peer_finder=None, @@ -108,6 +108,7 @@ class Node(object): self.change_token_lc.clock = self.clock self.refresh_node_lc = task.LoopingCall(self._refreshNode) self.refresh_node_lc.clock = self.clock + # Create k-buckets (for storing contacts) if routingTableClass is None: self._routingTable = routingtable.OptimizedTreeRoutingTable(self.node_id, self.clock.seconds) @@ -138,25 +139,16 @@ class Node(object): self.peerPort = peerPort self.hash_watcher = HashWatcher(self.clock) - # will be used later - self._can_store = True - self.peer_manager = peer_manager or PeerManager() self.peer_finder = peer_finder or DHTPeerFinder(self, self.peer_manager) - self.hash_announcer = hash_announcer or DHTHashAnnouncer(self) def __del__(self): log.warning("unclean shutdown of the dht node") if self._listeningPort is not None: self._listeningPort.stopListening() - @property - def can_store(self): - return self._can_store is True - @defer.inlineCallbacks def stop(self): - yield self.hash_announcer.stop() # stop LoopingCalls: if self.refresh_node_lc.running: yield self.refresh_node_lc.stop() @@ -234,7 +226,6 @@ class Node(object): self.hash_watcher.start() self.change_token_lc.start(constants.tokenSecretChangeInterval) self.refresh_node_lc.start(constants.checkRefreshInterval) - self.hash_announcer.run_manage_loop() @property def contacts(self): From 43d3f7c08775d518c25dc59f310044d04d437799 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 27 Mar 2018 16:07:55 -0400 Subject: [PATCH 46/52] add concurrent_announcers to config --- lbrynet/conf.py | 3 +++ lbrynet/dht/hashannouncer.py | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index cbeaac87f..b68d04b80 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -40,6 +40,8 @@ ANDROID = 4 KB = 2 ** 10 MB = 2 ** 20 +DEFAULT_CONCURRENT_ANNOUNCERS = 25 + DEFAULT_DHT_NODES = [ ('lbrynet1.lbry.io', 4444), ('lbrynet2.lbry.io', 4444), @@ -263,6 +265,7 @@ ADJUSTABLE_SETTINGS = { 'download_timeout': (int, 180), 'is_generous_host': (bool, True), 'announce_head_blobs_only': (bool, True), + 'concurrent_announcers': (int, DEFAULT_CONCURRENT_ANNOUNCERS), 'known_dht_nodes': (list, DEFAULT_DHT_NODES, server_list), 'lbryum_wallet_dir': (str, default_lbryum_dir), 'max_connections_per_stream': (int, 5), diff --git a/lbrynet/dht/hashannouncer.py b/lbrynet/dht/hashannouncer.py index 957725e83..3a4953ac6 100644 --- a/lbrynet/dht/hashannouncer.py +++ b/lbrynet/dht/hashannouncer.py @@ -3,18 +3,19 @@ import logging from twisted.internet import defer, task from lbrynet.core import utils +from lbrynet import conf log = logging.getLogger(__name__) class DHTHashAnnouncer(object): - def __init__(self, dht_node, storage, concurrent_announcers=25): + def __init__(self, dht_node, storage, concurrent_announcers=None): self.dht_node = dht_node self.storage = storage self.clock = dht_node.clock self.peer_port = dht_node.peerPort self.hash_queue = [] - self.concurrent_announcers = concurrent_announcers + self.concurrent_announcers = concurrent_announcers or conf.settings['concurrent_announcers'] self._manage_lc = task.LoopingCall(self.manage) self._manage_lc.clock = self.clock From a41bbd5e27023186f3d8d873ca3a2d6f7d6ebb96 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 27 Mar 2018 17:35:31 -0400 Subject: [PATCH 47/52] pylint and tests --- lbrynet/core/BlobManager.py | 5 +- lbrynet/daemon/Daemon.py | 2 +- lbrynet/database/migrator/migrate6to7.py | 19 ------- lbrynet/database/storage.py | 1 + lbrynet/dht/hashwatcher.py | 2 +- lbrynet/dht/node.py | 8 ++- lbrynet/tests/functional/test_dht.py | 15 +----- lbrynet/tests/functional/test_reflector.py | 12 ++--- lbrynet/tests/mocks.py | 12 ++--- .../unit/core/server/test_DHTHashAnnouncer.py | 52 ++++++++++++------- lbrynet/tests/unit/core/test_BlobManager.py | 4 +- .../tests/unit/database/test_SQLiteStorage.py | 2 +- lbrynet/tests/unit/dht/test_node.py | 2 +- lbrynet/tests/unit/dht/test_protocol.py | 2 +- .../test_EncryptedFileCreator.py | 8 +-- lbrynet/tests/util.py | 4 +- 16 files changed, 66 insertions(+), 84 deletions(-) diff --git a/lbrynet/core/BlobManager.py b/lbrynet/core/BlobManager.py index e2ccbfb04..a5fe1e8ed 100644 --- a/lbrynet/core/BlobManager.py +++ b/lbrynet/core/BlobManager.py @@ -59,7 +59,7 @@ class DiskBlobManager(object): return defer.succeed(blob) @defer.inlineCallbacks - def blob_completed(self, blob, next_announce_time=None, should_announce=True): + def blob_completed(self, blob, should_announce=False, next_announce_time=None): yield self.storage.add_completed_blob( blob.blob_hash, blob.length, next_announce_time, should_announce ) @@ -71,7 +71,8 @@ class DiskBlobManager(object): return self.storage.count_should_announce_blobs() def set_should_announce(self, blob_hash, should_announce): - return self.storage.set_should_announce(blob_hash, should_announce) + now = self.storage.clock.seconds() + return self.storage.set_should_announce(blob_hash, now, should_announce) def get_should_announce(self, blob_hash): return self.storage.should_announce(blob_hash) diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index 3e90e4111..97c08dc18 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -25,7 +25,7 @@ from lbryschema.decode import smart_decode from lbrynet.core.system_info import get_lbrynet_version from lbrynet.database.storage import SQLiteStorage from lbrynet import conf -from lbrynet.conf import LBRYCRD_WALLET, LBRYUM_WALLET, PTC_WALLET +from lbrynet.conf import LBRYCRD_WALLET, LBRYUM_WALLET from lbrynet.reflector import reupload from lbrynet.reflector import ServerFactory as reflector_server_factory from lbrynet.core.log_support import configure_loggly_handler diff --git a/lbrynet/database/migrator/migrate6to7.py b/lbrynet/database/migrator/migrate6to7.py index d1778a83e..536afc256 100644 --- a/lbrynet/database/migrator/migrate6to7.py +++ b/lbrynet/database/migrator/migrate6to7.py @@ -1,24 +1,5 @@ import sqlite3 import os -import logging -from lbrynet.database.storage import SQLiteStorage - -log = logging.getLogger(__name__) - - -def run_operation(db): - def _decorate(fn): - def _wrapper(*args): - cursor = db.cursor() - try: - result = fn(cursor, *args) - db.commit() - return result - except sqlite3.IntegrityError: - db.rollback() - raise - return _wrapper - return _decorate def do_migration(db_dir): diff --git a/lbrynet/database/storage.py b/lbrynet/database/storage.py index 4e6a9c669..61166f148 100644 --- a/lbrynet/database/storage.py +++ b/lbrynet/database/storage.py @@ -211,6 +211,7 @@ class SQLiteStorage(object): ) def set_should_announce(self, blob_hash, next_announce_time, should_announce): + next_announce_time = next_announce_time or 0 should_announce = 1 if should_announce else 0 return self.db.runOperation( "update blob set next_announce_time=?, should_announce=? where blob_hash=?", diff --git a/lbrynet/dht/hashwatcher.py b/lbrynet/dht/hashwatcher.py index 80aa30b6a..37f8218fd 100644 --- a/lbrynet/dht/hashwatcher.py +++ b/lbrynet/dht/hashwatcher.py @@ -1,6 +1,6 @@ from collections import Counter import datetime -from twisted.internet import task, threads +from twisted.internet import task class HashWatcher(object): diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index c77fce861..8aedb6ebe 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -23,7 +23,6 @@ import routingtable import datastore import protocol from error import TimeoutError -from hashannouncer import DHTHashAnnouncer from peerfinder import DHTPeerFinder from contact import Contact from hashwatcher import HashWatcher @@ -242,7 +241,12 @@ class Node(object): return False def announceHaveBlob(self, key): - return self.iterativeAnnounceHaveBlob(key, {'port': self.peerPort, 'lbryid': self.node_id}) + return self.iterativeAnnounceHaveBlob( + key, { + 'port': self.peerPort, + 'lbryid': self.node_id, + } + ) @defer.inlineCallbacks def getPeersForBlob(self, blob_hash): diff --git a/lbrynet/tests/functional/test_dht.py b/lbrynet/tests/functional/test_dht.py index ac8572193..692185880 100644 --- a/lbrynet/tests/functional/test_dht.py +++ b/lbrynet/tests/functional/test_dht.py @@ -212,24 +212,11 @@ class TestKademliaBootstrapSixteenSeeds(TestKademliaBase): @defer.inlineCallbacks def tearDown(self): yield TestKademliaBase.tearDown(self) - del self.seed_dns['lbrynet4.lbry.io'] - del self.seed_dns['lbrynet5.lbry.io'] - del self.seed_dns['lbrynet6.lbry.io'] - del self.seed_dns['lbrynet7.lbry.io'] - del self.seed_dns['lbrynet8.lbry.io'] - del self.seed_dns['lbrynet9.lbry.io'] - del self.seed_dns['lbrynet10.lbry.io'] - del self.seed_dns['lbrynet11.lbry.io'] - del self.seed_dns['lbrynet12.lbry.io'] - del self.seed_dns['lbrynet13.lbry.io'] - del self.seed_dns['lbrynet14.lbry.io'] - del self.seed_dns['lbrynet15.lbry.io'] - del self.seed_dns['lbrynet16.lbry.io'] def test_bootstrap_network(self): pass - def test_all_nodes_are_pingable(self): + def _test_all_nodes_are_pingable(self): return self.verify_all_nodes_are_pingable() diff --git a/lbrynet/tests/functional/test_reflector.py b/lbrynet/tests/functional/test_reflector.py index 09342d3bd..9cebda795 100644 --- a/lbrynet/tests/functional/test_reflector.py +++ b/lbrynet/tests/functional/test_reflector.py @@ -29,7 +29,6 @@ class TestReflector(unittest.TestCase): wallet = mocks.Wallet() peer_manager = PeerManager.PeerManager() peer_finder = mocks.PeerFinder(5553, peer_manager, 2) - hash_announcer = mocks.Announcer() sd_identifier = StreamDescriptor.StreamDescriptorIdentifier() self.expected_blobs = [ @@ -56,14 +55,14 @@ class TestReflector(unittest.TestCase): db_dir=self.db_dir, node_id="abcd", peer_finder=peer_finder, - hash_announcer=hash_announcer, blob_dir=self.blob_dir, peer_port=5553, use_upnp=False, wallet=wallet, blob_tracker_class=mocks.BlobAvailabilityTracker, external_ip="127.0.0.1", - dht_node_class=Node + dht_node_class=Node, + hash_announcer=mocks.Announcer() ) self.lbry_file_manager = EncryptedFileManager.EncryptedFileManager(self.session, @@ -76,18 +75,17 @@ class TestReflector(unittest.TestCase): db_dir=self.server_db_dir, node_id="abcd", peer_finder=peer_finder, - hash_announcer=hash_announcer, blob_dir=self.server_blob_dir, peer_port=5553, use_upnp=False, wallet=wallet, blob_tracker_class=mocks.BlobAvailabilityTracker, external_ip="127.0.0.1", - dht_node_class=Node + dht_node_class=Node, + hash_announcer=mocks.Announcer() ) - self.server_blob_manager = BlobManager.DiskBlobManager(hash_announcer, - self.server_blob_dir, + self.server_blob_manager = BlobManager.DiskBlobManager(self.server_blob_dir, self.server_session.storage) self.server_lbry_file_manager = EncryptedFileManager.EncryptedFileManager( diff --git a/lbrynet/tests/mocks.py b/lbrynet/tests/mocks.py index 2661f4a6e..db2b55019 100644 --- a/lbrynet/tests/mocks.py +++ b/lbrynet/tests/mocks.py @@ -2,7 +2,7 @@ import struct import io from Crypto.PublicKey import RSA -from twisted.internet import defer, threads, error +from twisted.internet import defer, error from twisted.python.failure import Failure from lbrynet.core.client.ClientRequest import ClientRequest @@ -25,10 +25,10 @@ class FakeLBRYFile(object): class Node(object): - def __init__(self, hash_announcer, peer_finder=None, peer_manager=None, **kwargs): - self.hash_announcer = hash_announcer + def __init__(self, peer_finder=None, peer_manager=None, **kwargs): self.peer_finder = peer_finder self.peer_manager = peer_manager + self.peerPort = 3333 def joinNetwork(self, *args): return defer.succeed(True) @@ -77,7 +77,7 @@ class ExchangeRateManager(ERM.ExchangeRateManager): feed.market, rates[feed.market]['spot'], rates[feed.market]['ts']) -class PointTraderKeyExchanger: +class PointTraderKeyExchanger(object): def __init__(self, wallet): self.wallet = wallet @@ -108,7 +108,7 @@ class PointTraderKeyExchanger: return err -class PointTraderKeyQueryHandlerFactory: +class PointTraderKeyQueryHandlerFactory(object): def __init__(self, wallet): self.wallet = wallet @@ -125,7 +125,7 @@ class PointTraderKeyQueryHandlerFactory: "point trader testing network") -class PointTraderKeyQueryHandler: +class PointTraderKeyQueryHandler(object): def __init__(self, wallet): self.wallet = wallet diff --git a/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py b/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py index de06f342f..0d3999c0b 100644 --- a/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py +++ b/lbrynet/tests/unit/core/server/test_DHTHashAnnouncer.py @@ -1,34 +1,40 @@ +import tempfile +import shutil from twisted.trial import unittest -from twisted.internet import defer, reactor +from twisted.internet import defer, reactor, threads from lbrynet.tests.util import random_lbry_hash from lbrynet.dht.hashannouncer import DHTHashAnnouncer +from lbrynet.core.call_later_manager import CallLaterManager +from lbrynet.database.storage import SQLiteStorage + class MocDHTNode(object): def __init__(self, announce_will_fail=False): # if announce_will_fail is True, # announceHaveBlob will return empty dict - self.can_store = True + self.call_later_manager = CallLaterManager + self.call_later_manager.setup(reactor.callLater) self.blobs_announced = 0 self.announce_will_fail = announce_will_fail - @defer.inlineCallbacks def announceHaveBlob(self, blob): if self.announce_will_fail: return_val = {} else: - return_val = {blob:["ab"*48]} + return_val = {blob: ["ab"*48]} self.blobs_announced += 1 - d = defer.Deferred(None) - reactor.callLater(1, d.callback, return_val) - result = yield d - defer.returnValue(result) + d = defer.Deferred() + self.call_later_manager.call_later(1, d.callback, return_val) + return d class DHTHashAnnouncerTest(unittest.TestCase): @defer.inlineCallbacks def setUp(self): + from lbrynet.conf import initialize_settings + initialize_settings() self.num_blobs = 10 self.blobs_to_announce = [] for i in range(0, self.num_blobs): @@ -36,31 +42,41 @@ class DHTHashAnnouncerTest(unittest.TestCase): self.dht_node = MocDHTNode() self.dht_node.peerPort = 3333 self.dht_node.clock = reactor - self.announcer = DHTHashAnnouncer(self.dht_node) - yield self.announcer.add_hashes_to_announce(self.blobs_to_announce) + self.db_dir = tempfile.mkdtemp() + self.storage = SQLiteStorage(self.db_dir) + yield self.storage.setup() + self.announcer = DHTHashAnnouncer(self.dht_node, self.storage, 10) + for blob_hash in self.blobs_to_announce: + yield self.storage.add_completed_blob(blob_hash, 100, 0, 1) + + @defer.inlineCallbacks + def tearDown(self): + self.dht_node.call_later_manager.stop() + yield self.storage.stop() + yield threads.deferToThread(shutil.rmtree, self.db_dir) @defer.inlineCallbacks def test_announce_fail(self): # test what happens when node.announceHaveBlob() returns empty dict self.dht_node.announce_will_fail = True - d = yield self.announcer._announce_available_hashes() + d = yield self.announcer.manage() yield d @defer.inlineCallbacks def test_basic(self): - d = self.announcer._announce_available_hashes() - self.assertEqual(self.announcer.hash_queue_size(), self.announcer.CONCURRENT_ANNOUNCERS) + d = self.announcer.immediate_announce(self.blobs_to_announce) + self.assertEqual(len(self.announcer.hash_queue), self.num_blobs) yield d self.assertEqual(self.dht_node.blobs_announced, self.num_blobs) - self.assertEqual(self.announcer.hash_queue_size(), 0) + self.assertEqual(len(self.announcer.hash_queue), 0) @defer.inlineCallbacks def test_immediate_announce(self): # Test that immediate announce puts a hash at the front of the queue - d = self.announcer._announce_available_hashes() - self.assertEqual(self.announcer.hash_queue_size(), self.announcer.CONCURRENT_ANNOUNCERS) + d = self.announcer.immediate_announce(self.blobs_to_announce) + self.assertEqual(len(self.announcer.hash_queue), self.num_blobs) blob_hash = random_lbry_hash() self.announcer.immediate_announce([blob_hash]) - self.assertEqual(self.announcer.hash_queue_size(), self.announcer.CONCURRENT_ANNOUNCERS+1) - self.assertEqual(blob_hash, self.announcer.hash_queue[0][0]) + self.assertEqual(len(self.announcer.hash_queue), self.num_blobs+1) + self.assertEqual(blob_hash, self.announcer.hash_queue[-1]) yield d diff --git a/lbrynet/tests/unit/core/test_BlobManager.py b/lbrynet/tests/unit/core/test_BlobManager.py index 5bc118f92..48b6df982 100644 --- a/lbrynet/tests/unit/core/test_BlobManager.py +++ b/lbrynet/tests/unit/core/test_BlobManager.py @@ -8,7 +8,6 @@ from twisted.internet import defer, threads from lbrynet.tests.util import random_lbry_hash from lbrynet.core.BlobManager import DiskBlobManager -from lbrynet.dht.hashannouncer import DummyHashAnnouncer from lbrynet.database.storage import SQLiteStorage from lbrynet.core.Peer import Peer from lbrynet import conf @@ -21,8 +20,7 @@ class BlobManagerTest(unittest.TestCase): conf.initialize_settings() self.blob_dir = tempfile.mkdtemp() self.db_dir = tempfile.mkdtemp() - hash_announcer = DummyHashAnnouncer() - self.bm = DiskBlobManager(hash_announcer, self.blob_dir, SQLiteStorage(self.db_dir)) + self.bm = DiskBlobManager(self.blob_dir, SQLiteStorage(self.db_dir)) self.peer = Peer('somehost', 22) yield self.bm.storage.setup() diff --git a/lbrynet/tests/unit/database/test_SQLiteStorage.py b/lbrynet/tests/unit/database/test_SQLiteStorage.py index dbf1b7c54..7cd69c3ff 100644 --- a/lbrynet/tests/unit/database/test_SQLiteStorage.py +++ b/lbrynet/tests/unit/database/test_SQLiteStorage.py @@ -195,7 +195,7 @@ class StreamStorageTests(StorageTest): should_announce_count = yield self.storage.count_should_announce_blobs() self.assertEqual(should_announce_count, 2) - should_announce_hashes = yield self.storage.get_blobs_to_announce(FakeAnnouncer()) + should_announce_hashes = yield self.storage.get_blobs_to_announce() self.assertSetEqual(set(should_announce_hashes), {sd_hash, blob1}) stream_hashes = yield self.storage.get_all_streams() diff --git a/lbrynet/tests/unit/dht/test_node.py b/lbrynet/tests/unit/dht/test_node.py index e89811b48..ab73ba3e8 100644 --- a/lbrynet/tests/unit/dht/test_node.py +++ b/lbrynet/tests/unit/dht/test_node.py @@ -198,7 +198,7 @@ class NodeLookupTest(unittest.TestCase): h = hashlib.sha384() h.update('node1') node_id = str(h.digest()) - self.node = lbrynet.dht.node.Node(None, node_id=node_id, udpPort=4000, networkProtocol=self._protocol) + self.node = lbrynet.dht.node.Node(node_id=node_id, udpPort=4000, networkProtocol=self._protocol) self.updPort = 81173 self.contactsAmount = 80 # Reinitialise the routing table diff --git a/lbrynet/tests/unit/dht/test_protocol.py b/lbrynet/tests/unit/dht/test_protocol.py index 0b48cf115..af636b631 100644 --- a/lbrynet/tests/unit/dht/test_protocol.py +++ b/lbrynet/tests/unit/dht/test_protocol.py @@ -1,7 +1,7 @@ import time import unittest from twisted.internet.task import Clock -from twisted.internet import defer, threads +from twisted.internet import defer import lbrynet.dht.protocol import lbrynet.dht.contact import lbrynet.dht.constants diff --git a/lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py b/lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py index 46ef3b721..07ad7e87f 100644 --- a/lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py +++ b/lbrynet/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py @@ -2,18 +2,15 @@ from Crypto.Cipher import AES import mock from twisted.trial import unittest -from twisted.internet import defer, reactor +from twisted.internet import defer from lbrynet.database.storage import SQLiteStorage from lbrynet.core.StreamDescriptor import get_sd_info, BlobStreamDescriptorReader from lbrynet.core import BlobManager from lbrynet.core import Session -from lbrynet.dht import hashannouncer -from lbrynet.dht.node import Node from lbrynet.file_manager import EncryptedFileCreator from lbrynet.file_manager import EncryptedFileManager from lbrynet.tests import mocks -from time import time from lbrynet.tests.util import mk_db_and_blob_dir, rm_db_and_blob_dir MB = 2**20 @@ -34,8 +31,7 @@ class CreateEncryptedFileTest(unittest.TestCase): self.session = mock.Mock(spec=Session.Session)(None, None) self.session.payment_rate_manager.min_blob_data_payment_rate = 0 - self.blob_manager = BlobManager.DiskBlobManager( - hashannouncer.DummyHashAnnouncer(), self.tmp_blob_dir, SQLiteStorage(self.tmp_db_dir)) + self.blob_manager = BlobManager.DiskBlobManager(self.tmp_blob_dir, SQLiteStorage(self.tmp_db_dir)) self.session.blob_manager = self.blob_manager self.session.storage = self.session.blob_manager.storage self.file_manager = EncryptedFileManager.EncryptedFileManager(self.session, object()) diff --git a/lbrynet/tests/util.py b/lbrynet/tests/util.py index cc4c7da78..e6ad2005c 100644 --- a/lbrynet/tests/util.py +++ b/lbrynet/tests/util.py @@ -67,8 +67,8 @@ def debug_kademlia_packet(data, source, destination, node): log.debug("response %s <-- %s %s (node time %s)", destination[0], source[0], packet.response, node.clock.seconds()) else: - log.debug("response %s <-- %s %i contacts (node time %s)", destination[0], source[0], - len(packet.response), node.clock.seconds()) + log.debug("response %s <-- %s %i contacts (node time %s)", destination[0], source[0], + len(packet.response), node.clock.seconds()) elif isinstance(packet, ErrorMessage): log.error("error %s <-- %s %s (node time %s)", destination[0], source[0], packet.exceptionType, node.clock.seconds()) From 5bab6f7d39d795d648ebc5b734030e57815516fc Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 27 Mar 2018 20:56:34 -0400 Subject: [PATCH 48/52] remove bandwidth stats --- lbrynet/daemon/Daemon.py | 20 +----- lbrynet/dht/node.py | 3 - lbrynet/dht/protocol.py | 132 +-------------------------------------- 3 files changed, 2 insertions(+), 153 deletions(-) diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index 97c08dc18..bb13f3416 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -993,7 +993,7 @@ class Daemon(AuthJSONRPCServer): ############################################################################ @defer.inlineCallbacks - def jsonrpc_status(self, session_status=False, dht_status=False): + def jsonrpc_status(self, session_status=False): """ Get daemon status @@ -1002,7 +1002,6 @@ class Daemon(AuthJSONRPCServer): Options: --session_status : (bool) include session status in results - --dht_status : (bool) include dht network and peer status Returns: (dict) lbrynet-daemon status @@ -1033,19 +1032,6 @@ class Daemon(AuthJSONRPCServer): 'announce_queue_size': number of blobs currently queued to be announced 'should_announce_blobs': number of blobs that should be announced } - - If given the dht status option: - 'dht_status': { - 'kbps_received': current kbps receiving, - 'kbps_sent': current kdps being sent, - 'total_bytes_sent': total bytes sent - 'total_bytes_received': total bytes received - 'queries_received': number of queries received per second - 'queries_sent': number of queries sent per second - 'recent_contacts': count of recently contacted peers - 'single_hash_announce_duration': avg. seconds it takes to announce a blob - 'unique_contacts': count of unique peers - }, } """ @@ -1092,10 +1078,6 @@ class Daemon(AuthJSONRPCServer): 'announce_queue_size': announce_queue_size, 'should_announce_blobs': should_announce_blobs, } - if dht_status: - response['dht_status'] = self.session.dht_node.get_bandwidth_stats() - response['dht_status'].update({'single_hash_announce_duration': - self.session.blob_manager.single_hash_announce_duration}) defer.returnValue(response) def jsonrpc_version(self): diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index 8aedb6ebe..0e5c980ab 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -264,9 +264,6 @@ class Node(object): def get_most_popular_hashes(self, num_to_return): return self.hash_watcher.most_popular_hashes(num_to_return) - def get_bandwidth_stats(self): - return self._protocol.bandwidth_stats - @defer.inlineCallbacks def iterativeAnnounceHaveBlob(self, blob_hash, value): known_nodes = {} diff --git a/lbrynet/dht/protocol.py b/lbrynet/dht/protocol.py index 7e018c68a..629bb070c 100644 --- a/lbrynet/dht/protocol.py +++ b/lbrynet/dht/protocol.py @@ -1,9 +1,8 @@ import logging -import time import socket import errno -from twisted.internet import protocol, defer, task +from twisted.internet import protocol, defer from lbrynet.core.call_later_manager import CallLaterManager import constants @@ -30,104 +29,6 @@ class KademliaProtocol(protocol.DatagramProtocol): self._partialMessages = {} self._partialMessagesProgress = {} self._delay = Delay(0, self._node.clock.seconds) - # keep track of outstanding writes so that they - # can be cancelled on shutdown - self._call_later_list = {} - - # keep track of bandwidth usage by peer - self._history_rx = {} - self._history_tx = {} - self._bytes_rx = {} - self._bytes_tx = {} - self._unique_contacts = [] - self._queries_rx_per_second = 0 - self._queries_tx_per_second = 0 - self._kbps_tx = 0 - self._kbps_rx = 0 - self._recent_contact_count = 0 - self._total_bytes_tx = 0 - self._total_bytes_rx = 0 - self._bandwidth_stats_update_lc = task.LoopingCall(self._update_bandwidth_stats) - self._bandwidth_stats_update_lc.clock = self._node.clock - - def _update_bandwidth_stats(self): - recent_rx_history = {} - now = time.time() - for address, history in self._history_rx.iteritems(): - recent_rx_history[address] = [(s, t) for (s, t) in history if now - t < 1.0] - qps_rx = sum(len(v) for (k, v) in recent_rx_history.iteritems()) - bps_rx = sum(sum([x[0] for x in v]) for (k, v) in recent_rx_history.iteritems()) - kbps_rx = round(float(bps_rx) / 1024.0, 2) - - recent_tx_history = {} - now = time.time() - for address, history in self._history_tx.iteritems(): - recent_tx_history[address] = [(s, t) for (s, t) in history if now - t < 1.0] - qps_tx = sum(len(v) for (k, v) in recent_tx_history.iteritems()) - bps_tx = sum(sum([x[0] for x in v]) for (k, v) in recent_tx_history.iteritems()) - kbps_tx = round(float(bps_tx) / 1024.0, 2) - - recent_contacts = [] - for k, v in recent_rx_history.iteritems(): - if v: - recent_contacts.append(k) - for k, v in recent_tx_history.iteritems(): - if v and k not in recent_contacts: - recent_contacts.append(k) - - self._queries_rx_per_second = qps_rx - self._queries_tx_per_second = qps_tx - self._kbps_tx = kbps_tx - self._kbps_rx = kbps_rx - self._recent_contact_count = len(recent_contacts) - self._total_bytes_tx = sum(v for (k, v) in self._bytes_tx.iteritems()) - self._total_bytes_rx = sum(v for (k, v) in self._bytes_rx.iteritems()) - - @property - def unique_contacts(self): - return self._unique_contacts - - @property - def queries_rx_per_second(self): - return self._queries_rx_per_second - - @property - def queries_tx_per_second(self): - return self._queries_tx_per_second - - @property - def kbps_tx(self): - return self._kbps_tx - - @property - def kbps_rx(self): - return self._kbps_rx - - @property - def recent_contact_count(self): - return self._recent_contact_count - - @property - def total_bytes_tx(self): - return self._total_bytes_tx - - @property - def total_bytes_rx(self): - return self._total_bytes_rx - - @property - def bandwidth_stats(self): - response = { - "kbps_received": self.kbps_rx, - "kbps_sent": self.kbps_tx, - "total_bytes_sent": self.total_bytes_tx, - "total_bytes_received": self.total_bytes_rx, - "queries_received": self.queries_rx_per_second, - "queries_sent": self.queries_tx_per_second, - "recent_contacts": self.recent_contact_count, - "unique_contacts": len(self.unique_contacts) - } - return response def sendRPC(self, contact, method, args, rawResponse=False): """ Sends an RPC to the specified contact @@ -179,8 +80,6 @@ class KademliaProtocol(protocol.DatagramProtocol): def startProtocol(self): log.info("DHT listening on UDP %i", self._node.port) - if not self._bandwidth_stats_update_lc.running: - self._bandwidth_stats_update_lc.start(1) def datagramReceived(self, datagram, address): """ Handles and parses incoming RPC messages (and responses) @@ -219,18 +118,6 @@ class KademliaProtocol(protocol.DatagramProtocol): remoteContact = Contact(message.nodeID, address[0], address[1], self) - now = time.time() - contact_history = self._history_rx.get(address, []) - if len(contact_history) > 1000: - contact_history = [x for x in contact_history if now - x[1] < 1.0] - contact_history.append((len(datagram), time.time())) - self._history_rx[address] = contact_history - bytes_rx = self._bytes_rx.get(address, 0) - bytes_rx += len(datagram) - self._bytes_rx[address] = bytes_rx - if address not in self.unique_contacts: - self._unique_contacts.append(address) - # Refresh the remote node's details in the local node's k-buckets self._node.addContact(remoteContact) if isinstance(message, msgtypes.RequestMessage): @@ -295,18 +182,6 @@ class KademliaProtocol(protocol.DatagramProtocol): class (see C{kademlia.msgformat} and C{kademlia.encoding}). """ - now = time.time() - contact_history = self._history_tx.get(address, []) - if len(contact_history) > 1000: - contact_history = [x for x in contact_history if now - x[1] < 1.0] - contact_history.append((len(data), time.time())) - self._history_tx[address] = contact_history - bytes_tx = self._bytes_tx.get(address, 0) - bytes_tx += len(data) - self._bytes_tx[address] = bytes_tx - if address not in self.unique_contacts: - self._unique_contacts.append(address) - if len(data) > self.msgSizeLimit: # We have to spread the data over multiple UDP datagrams, # and provide sequencing information @@ -457,10 +332,5 @@ class KademliaProtocol(protocol.DatagramProtocol): Will only be called once, after all ports are disconnected. """ log.info('Stopping DHT') - - if self._bandwidth_stats_update_lc.running: - self._bandwidth_stats_update_lc.stop() - CallLaterManager.stop() - log.info('DHT stopped') From eabf4a0e408c54df9582698276da1c55eab2e234 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 27 Mar 2018 21:08:01 -0400 Subject: [PATCH 49/52] remove delay from udp write --- lbrynet/dht/delay.py | 21 --------------------- lbrynet/dht/protocol.py | 8 ++------ 2 files changed, 2 insertions(+), 27 deletions(-) delete mode 100644 lbrynet/dht/delay.py diff --git a/lbrynet/dht/delay.py b/lbrynet/dht/delay.py deleted file mode 100644 index 7ef26fcc6..000000000 --- a/lbrynet/dht/delay.py +++ /dev/null @@ -1,21 +0,0 @@ -class Delay(object): - maxToSendDelay = 10 ** -3 # 0.05 - minToSendDelay = 10 ** -5 # 0.01 - - def __init__(self, start=0, getTime=None): - self._next = start - if not getTime: - from time import time as getTime - self._getTime = getTime - - # TODO: explain why this logic is like it is. And add tests that - # show that it actually does what it needs to do. - def __call__(self): - ts = self._getTime() - if ts >= self._next: - delay = self.minToSendDelay - self._next = ts + self.minToSendDelay - else: - delay = (self._next - ts) + self.maxToSendDelay - self._next += self.maxToSendDelay - return delay diff --git a/lbrynet/dht/protocol.py b/lbrynet/dht/protocol.py index 629bb070c..f9e68b515 100644 --- a/lbrynet/dht/protocol.py +++ b/lbrynet/dht/protocol.py @@ -11,7 +11,6 @@ import msgtypes import msgformat from contact import Contact from error import BUILTIN_EXCEPTIONS, UnknownRemoteException, TimeoutError -from delay import Delay log = logging.getLogger(__name__) @@ -28,7 +27,6 @@ class KademliaProtocol(protocol.DatagramProtocol): self._sentMessages = {} self._partialMessages = {} self._partialMessagesProgress = {} - self._delay = Delay(0, self._node.clock.seconds) def sendRPC(self, contact, method, args, rawResponse=False): """ Sends an RPC to the specified contact @@ -208,11 +206,9 @@ class KademliaProtocol(protocol.DatagramProtocol): def _scheduleSendNext(self, txData, address): """Schedule the sending of the next UDP packet """ - delay = self._delay() - key = object() - delayed_call, _ = self._node.reactor_callLater(delay, self._write_and_remove, key, txData, address) + delayed_call, _ = self._node.reactor_callLater(0, self._write, txData, address) - def _write_and_remove(self, key, txData, address): + def _write(self, txData, address): if self.transport: try: self.transport.write(txData, address) From dff1fd3fe9b3693e4891b5447b8a7417f26f0a40 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 27 Mar 2018 21:22:53 -0400 Subject: [PATCH 50/52] logging, raise default concurrent announcers --- lbrynet/conf.py | 2 +- lbrynet/dht/hashannouncer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index b68d04b80..3edee1437 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -40,7 +40,7 @@ ANDROID = 4 KB = 2 ** 10 MB = 2 ** 20 -DEFAULT_CONCURRENT_ANNOUNCERS = 25 +DEFAULT_CONCURRENT_ANNOUNCERS = 100 DEFAULT_DHT_NODES = [ ('lbrynet1.lbry.io', 4444), diff --git a/lbrynet/dht/hashannouncer.py b/lbrynet/dht/hashannouncer.py index 3a4953ac6..cd5f8cb68 100644 --- a/lbrynet/dht/hashannouncer.py +++ b/lbrynet/dht/hashannouncer.py @@ -64,7 +64,7 @@ class DHTHashAnnouncer(object): log.debug("Failed to announce %i blobs", len(results) - len(announced_to)) if announced_to: log.info('Took %s seconds to announce %i of %i attempted hashes (%f hashes per second)', - now - start, len(blob_hashes), len(announced_to), + now - start, len(announced_to), len(blob_hashes), int(float(len(blob_hashes)) / float(now - start))) defer.returnValue(results) From 492858596e882a386cd4c17aa0c0db4605b582b6 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 28 Mar 2018 18:47:37 -0400 Subject: [PATCH 51/52] add single_announce column to blob table -remove deprecated blob_announce_all function -remove announce_all parameter to blob_announce -change blob_announce to be asynchronous --- lbrynet/daemon/Daemon.py | 47 +++++++----------------- lbrynet/database/migrator/migrate6to7.py | 1 + lbrynet/database/storage.py | 27 +++++++++++--- lbrynet/dht/hashannouncer.py | 3 +- 4 files changed, 37 insertions(+), 41 deletions(-) diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index bb13f3416..364cc1f79 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -2900,17 +2900,15 @@ class Daemon(AuthJSONRPCServer): return d @defer.inlineCallbacks - def jsonrpc_blob_announce(self, blob_hash=None, stream_hash=None, sd_hash=None, announce_all=None): + def jsonrpc_blob_announce(self, blob_hash=None, stream_hash=None, sd_hash=None): """ Announce blobs to the DHT Usage: blob_announce [ | --blob_hash=] [ | --stream_hash=] | [ | --sd_hash=] - [--announce_all] Options: - --announce_all : (bool) announce all the blobs possessed by user --blob_hash= : (str) announce a blob, specified by blob_hash --stream_hash= : (str) announce all blobs associated with stream_hash @@ -2921,41 +2919,22 @@ class Daemon(AuthJSONRPCServer): (bool) true if successful """ - if announce_all: - yield self.session.blob_manager.immediate_announce_all_blobs() + blob_hashes = [] + if blob_hash: + blob_hashes.append(blob_hash) + elif stream_hash or sd_hash: + if sd_hash and stream_hash: + raise Exception("either the sd hash or the stream hash should be provided, not both") + if sd_hash: + stream_hash = yield self.storage.get_stream_hash_for_sd_hash(sd_hash) + blobs = yield self.storage.get_blobs_for_stream(stream_hash, only_completed=True) + blob_hashes.extend(blob.blob_hash for blob in blobs if blob.blob_hash is not None) else: - blob_hashes = [] - if blob_hash: - blob_hashes.append(blob_hash) - elif stream_hash or sd_hash: - if sd_hash and stream_hash: - raise Exception("either the sd hash or the stream hash should be provided, not both") - if sd_hash: - stream_hash = yield self.storage.get_stream_hash_for_sd_hash(sd_hash) - blobs = yield self.storage.get_blobs_for_stream(stream_hash, only_completed=True) - blob_hashes.extend([blob.blob_hash for blob in blobs if blob.blob_hash is not None]) - else: - raise Exception('single argument must be specified') - yield self.session.blob_manager.immediate_announce(blob_hashes) + raise Exception('single argument must be specified') + yield self.storage.should_single_announce_blobs(blob_hashes, immediate=True) response = yield self._render_response(True) defer.returnValue(response) - @AuthJSONRPCServer.deprecated("blob_announce") - def jsonrpc_blob_announce_all(self): - """ - Announce all blobs to the DHT - - Usage: - blob_announce_all - - Options: - None - - Returns: - (str) Success/fail message - """ - return self.jsonrpc_blob_announce(announce_all=True) - @defer.inlineCallbacks def jsonrpc_file_reflect(self, **kwargs): """ diff --git a/lbrynet/database/migrator/migrate6to7.py b/lbrynet/database/migrator/migrate6to7.py index 536afc256..eff68eca0 100644 --- a/lbrynet/database/migrator/migrate6to7.py +++ b/lbrynet/database/migrator/migrate6to7.py @@ -7,6 +7,7 @@ def do_migration(db_dir): connection = sqlite3.connect(db_path) cursor = connection.cursor() cursor.executescript("alter table blob add last_announced_time integer;") + cursor.executescript("alter table blob add single_announce integer;") cursor.execute("update blob set next_announce_time=0") connection.commit() connection.close() diff --git a/lbrynet/database/storage.py b/lbrynet/database/storage.py index 61166f148..e3bdd649c 100644 --- a/lbrynet/database/storage.py +++ b/lbrynet/database/storage.py @@ -105,7 +105,8 @@ class SQLiteStorage(object): next_announce_time integer not null, should_announce integer not null default 0, status text not null, - last_announced_time integer + last_announced_time integer, + single_announce integer ); create table if not exists stream ( @@ -233,8 +234,8 @@ class SQLiteStorage(object): status = yield self.get_blob_status(blob_hash) if status is None: status = "pending" - yield self.db.runOperation("insert into blob values (?, ?, ?, ?, ?, ?)", - (blob_hash, length, 0, 0, status, 0)) + yield self.db.runOperation("insert into blob values (?, ?, ?, ?, ?, ?, ?)", + (blob_hash, length, 0, 0, status, 0, 0)) defer.returnValue(status) def should_announce(self, blob_hash): @@ -254,17 +255,33 @@ class SQLiteStorage(object): def update_last_announced_blob(self, blob_hash, last_announced): return self.db.runOperation( - "update blob set next_announce_time=?, last_announced_time=? where blob_hash=?", + "update blob set next_announce_time=?, last_announced_time=?, single_announce=0 where blob_hash=?", (int(last_announced + (dataExpireTimeout / 2)), int(last_announced), blob_hash) ) + def should_single_announce_blobs(self, blob_hashes, immediate=False): + def set_single_announce(transaction): + now = self.clock.seconds() + for blob_hash in blob_hashes: + if immediate: + transaction.execute( + "update blob set single_announce=1, next_announce_time=? " + "where blob_hash=? and status='finished'", (int(now), blob_hash) + ) + else: + transaction.execute( + "update blob set single_announce=1 where blob_hash=? and status='finished'", (blob_hash, ) + ) + return self.db.runInteraction(set_single_announce) + def get_blobs_to_announce(self): def get_and_update(transaction): timestamp = self.clock.seconds() if conf.settings['announce_head_blobs_only']: r = transaction.execute( "select blob_hash from blob " - "where blob_hash is not null and should_announce=1 and next_announce_time Date: Wed, 28 Mar 2018 19:25:55 -0400 Subject: [PATCH 52/52] changelog --- CHANGELOG.md | 67 ++++++++++++++++------------------------------------ 1 file changed, 21 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f248614ec..bbbd467e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,62 +13,37 @@ at anytime. * ### Fixed - * - * - * incorrectly raised download cancelled error for already verified blob files - * infinite loop where reflector client keeps trying to send failing blobs, which may be failing because they are invalid and thus will never be successfully received - * docstring bugs for `stream_availability`, `channel_import`, and `blob_announce` - * regression in `stream_availability` due to error in it's docstring - * fixed the inconsistencies in API and CLI docstrings - * `blob_announce` error when announcing a single blob - * `blob_list` error when looking up blobs by stream or sd hash - * issue#1107 whereing claiming a channel with the exact amount present in wallet would give out proper error - * - * - * - * improper parsing of arguments to CLI settings_set (https://github.com/lbryio/lbry/issues/930) - * unnecessarily verbose exchange rate error (https://github.com/lbryio/lbry/issues/984) - * value error due to a race condition when saving to the claim cache (https://github.com/lbryio/lbry/issues/1013) - * being unable to re-download updated content (https://github.com/lbryio/lbry/issues/951) - * sending error messages for failed api requests - * file manager startup being slow when handling thousands of files - * handling decryption error for blobs encrypted with an invalid key - * handling stream with no data blob (https://github.com/lbryio/lbry/issues/905) - * fetching the external ip - * `blob_list` returning an error with --uri parameter and incorrectly returning `[]` for streams where blobs are known (https://github.com/lbryio/lbry/issues/895) - * `get` failing with a non-useful error message when given a uri for a channel claim - * exception checking in several wallet unit tests - * daemon not erring properly for non-numeric values being passed to the `bid` parameter for the `publish` method - * incorrect `blob_num` for the stream terminator blob, which would result in creating invalid streams. Such invalid streams are detected on startup and are automatically removed (https://github.com/lbryio/lbry/issues/1124) * handling error from dht clients with old `ping` method + * blobs not being re-announced if no peers successfully stored, now failed announcements are re-queued ### Deprecated * - * - * `single_hash_announce_duration` field to `status` response when provided the `dht_status` argument - + * + ### Changed - * - * - -### Added - * - * - * `blob_reflect` command to send specific blobs to a reflector server - * unit test for docopt - * - * scripts to autogenerate documentation - * now updating new channel also takes into consideration the original bid amount, so now channel could be updated for wallet balance + the original bid amount - * forward-compaitibility for upcoming DHT bencoding changes - * * several internal dht functions to use inlineCallbacks - * blob announcement to be retried up to three times if `store` is unsuccessful * `DHTHashAnnouncer` and `Node` manage functions to use `LoopingCall`s instead of scheduling with `callLater`. * `store` kademlia rpc method to block on the call finishing and to return storing peer information + * refactored `DHTHashAnnouncer` to longer use locks, use a `DeferredSemaphore` to limit concurrent announcers + * decoupled `DiskBlobManager` from `DHTHashAnnouncer` + * blob hashes to announce to be controlled by`SQLiteStorage` + * kademlia protocol to not delay writes to the UDP socket + * `reactor` and `callLater`, `listenUDP`, and `resolve` functions to be configurable (to allow easier testing) + * calls to get the current time to use `reactor.seconds` (to control callLater and LoopingCall timing in tests) + * `blob_announce` to queue the blob announcement but not block on it + * blob completion to not `callLater` an immediate announce, let `SQLiteStorage` and the `DHTHashAnnouncer` handle it + * raise the default number of concurrent blob announcers to 100 + * dht logging to be more verbose with errors and warnings + * added `single_announce` and `last_announced_time` columns to the `blob` table in sqlite + +### Added + * virtual kademlia network and mock udp transport for dht integration tests + * integration tests for bootstrapping the dht + * configurable `concurrent_announcers` setting ### Removed - * - * + * `announce_all` argument from `blob_announce` + * old `blob_announce_all` command ## [0.19.2] - 2018-03-28