From 502e2227b57396958c8b46ff313e2f47244961e6 Mon Sep 17 00:00:00 2001 From: Alex Grintsvayg Date: Fri, 31 Mar 2017 14:24:21 -0400 Subject: [PATCH 1/4] fix KeyError bug in dht --- lbrynet/dht/routingtable.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lbrynet/dht/routingtable.py b/lbrynet/dht/routingtable.py index ed09fb40d..a4702e7a1 100644 --- a/lbrynet/dht/routingtable.py +++ b/lbrynet/dht/routingtable.py @@ -402,15 +402,15 @@ class OptimizedTreeRoutingTable(TreeRoutingTable): # Put the new contact in our replacement cache for the # corresponding k-bucket (or update it's position if # it exists already) - if not self._replacementCache.has_key(bucketIndex): + if bucketIndex not in self._replacementCache: self._replacementCache[bucketIndex] = [] if contact in self._replacementCache[bucketIndex]: self._replacementCache[bucketIndex].remove(contact) # TODO: Using k to limit the size of the contact - # replacement cache - maybe define a seperate value for + # replacement cache - maybe define a separate value for # this in constants.py? - elif len(self._replacementCache) >= constants.k: - self._replacementCache.pop(0) + elif len(self._replacementCache[bucketIndex]) >= constants.k: + self._replacementCache[bucketIndex].pop(0) self._replacementCache[bucketIndex].append(contact) def removeContact(self, contactID): @@ -428,8 +428,8 @@ class OptimizedTreeRoutingTable(TreeRoutingTable): contact.failedRPCs += 1 if contact.failedRPCs >= 5: self._buckets[bucketIndex].removeContact(contactID) - # Replace this stale contact with one from our replacemnent cache, if we have any - if self._replacementCache.has_key(bucketIndex): + # Replace this stale contact with one from our replacement cache, if we have any + if bucketIndex in self._replacementCache: if len(self._replacementCache[bucketIndex]) > 0: self._buckets[bucketIndex].addContact( self._replacementCache[bucketIndex].pop()) From bee67117c8f526c71860c98c2328ff18814a5c18 Mon Sep 17 00:00:00 2001 From: Alex Grintsvayg Date: Fri, 31 Mar 2017 14:52:56 -0400 Subject: [PATCH 2/4] changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 085572216..7f7b28b4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ at anytime. ### Fixed * Removed update_metadata function that could cause update problems - * + * Fix DHT contact bug * ## [0.9.2rc3] - 2017-03-29 From 99e4f9b00b3f207b681f9e02afa90127c3c9de0b Mon Sep 17 00:00:00 2001 From: Alex Grintsvayg Date: Tue, 4 Apr 2017 15:00:33 -0400 Subject: [PATCH 3/4] we dont always want to encode key here. bucket.keyInRange() already does it when necessary --- lbrynet/dht/routingtable.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lbrynet/dht/routingtable.py b/lbrynet/dht/routingtable.py index a4702e7a1..3aa493119 100644 --- a/lbrynet/dht/routingtable.py +++ b/lbrynet/dht/routingtable.py @@ -308,10 +308,9 @@ class TreeRoutingTable(RoutingTable): @return: The index of the k-bucket responsible for the specified key @rtype: int """ - valKey = long(key.encode('hex'), 16) i = 0 for bucket in self._buckets: - if bucket.keyInRange(valKey): + if bucket.keyInRange(key): return i else: i += 1 From 22f57f6490826b99e35b80d85837c95921cd6c97 Mon Sep 17 00:00:00 2001 From: Alex Grintsvayg Date: Tue, 4 Apr 2017 15:00:48 -0400 Subject: [PATCH 4/4] add test to reproduce keyerror --- tests/unit/dht/test_routingtable.py | 59 +++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 tests/unit/dht/test_routingtable.py diff --git a/tests/unit/dht/test_routingtable.py b/tests/unit/dht/test_routingtable.py new file mode 100644 index 000000000..36d184d8c --- /dev/null +++ b/tests/unit/dht/test_routingtable.py @@ -0,0 +1,59 @@ +import unittest + +from lbrynet.dht import contact, routingtable, constants + + +class KeyErrorFixedTest(unittest.TestCase): + """ Basic tests case for boolean operators on the Contact class """ + + def setUp(self): + own_id = (2 ** constants.key_bits) - 1 + # carefully chosen own_id. here's the logic + # we want a bunch of buckets (k+1, to be exact), and we want to make sure own_id + # is not in bucket 0. so we put own_id at the end so we can keep splitting by adding to the + # end + + self.table = routingtable.OptimizedTreeRoutingTable(own_id) + + def fill_bucket(self, bucket_min): + bucket_size = constants.k + for i in range(bucket_min, bucket_min + bucket_size): + self.table.addContact(contact.Contact(long(i), '127.0.0.1', 9999, None)) + + def overflow_bucket(self, bucket_min): + bucket_size = constants.k + self.fill_bucket(bucket_min) + self.table.addContact( + contact.Contact(long(bucket_min + bucket_size + 1), '127.0.0.1', 9999, None)) + + def testKeyError(self): + + # find middle, so we know where bucket will split + bucket_middle = self.table._buckets[0].rangeMax / 2 + + # fill last bucket + self.fill_bucket(self.table._buckets[0].rangeMax - constants.k - 1) + # -1 in previous line because own_id is in last bucket + + # fill/overflow 7 more buckets + bucket_start = 0 + for i in range(0, constants.k): + self.overflow_bucket(bucket_start) + bucket_start += bucket_middle / (2 ** i) + + # replacement cache now has k-1 entries. + # adding one more contact to bucket 0 used to cause a KeyError, but it should work + self.table.addContact(contact.Contact(long(constants.k + 2), '127.0.0.1', 9999, None)) + + # import math + # print "" + # for i, bucket in enumerate(self.table._buckets): + # print "Bucket " + str(i) + " (2 ** " + str( + # math.log(bucket.rangeMin, 2) if bucket.rangeMin > 0 else 0) + " <= x < 2 ** "+str( + # math.log(bucket.rangeMax, 2)) + ")" + # for c in bucket.getContacts(): + # print " contact " + str(c.id) + # for key, bucket in self.table._replacementCache.iteritems(): + # print "Replacement Cache for Bucket " + str(key) + # for c in bucket: + # print " contact " + str(c.id)