Madiator2011/lbry-sdk
1
0
Fork 0
forked from LBRYCommunity/lbry-sdk
Code Pull requests Activity

add dht tests

Browse source
This commit is contained in:
Jack Robison 2017-10-10 13:31:18 -04:00
parent e9fd8eb096
commit c7acb31614
No known key found for this signature in database
GPG key ID: 284699E7404E3CFF
4 changed files with 233 additions and 270 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

0
lbrynet/tests/dht/__init__.py Normal file
View file

203
lbrynet/tests/dht/testNode.py → lbrynet/tests/dht/test_node.py
View file

@ -8,10 +8,13 @@ import hashlib
import unittest import unittest
import struct import struct
from twisted.internet import protocol, defer, selectreactor
from lbrynet.dht.msgtypes import ResponseMessage
import lbrynet.dht.node import lbrynet.dht.node
import lbrynet.dht.constants import lbrynet.dht.constants
import lbrynet.dht.datastore import lbrynet.dht.datastore
class NodeIDTest(unittest.TestCase): class NodeIDTest(unittest.TestCase):
""" Test case for the Node class's ID """ """ Test case for the Node class's ID """
def setUp(self): def setUp(self):
@ -19,66 +22,70 @@ class NodeIDTest(unittest.TestCase):
def testAutoCreatedID(self): def testAutoCreatedID(self):
""" Tests if a new node has a valid node ID """ """ Tests if a new node has a valid node ID """
self.failUnlessEqual(type(self.node.id), str, 'Node does not have a valid ID') self.failUnlessEqual(type(self.node.node_id), str, 'Node does not have a valid ID')
self.failUnlessEqual(len(self.node.id), 20, 'Node ID length is incorrect! Expected 160 bits, got %d bits.' % (len(self.node.id)*8)) self.failUnlessEqual(len(self.node.node_id), 48, 'Node ID length is incorrect! '
'Expected 384 bits, got %d bits.' %
(len(self.node.node_id) * 8))
def testUniqueness(self): def testUniqueness(self):
""" Tests the uniqueness of the values created by the NodeID generator """ Tests the uniqueness of the values created by the NodeID generator """
"""
generatedIDs = [] generatedIDs = []
for i in range(100): for i in range(100):
newID = self.node._generateID() newID = self.node._generateID()
# ugly uniqueness test # ugly uniqueness test
self.failIf(newID in generatedIDs, 'Generated ID #%d not unique!' % (i+1)) self.failIf(newID in generatedIDs, 'Generated ID #%d not unique!' % (i+1))
generatedIDs.append(newID) generatedIDs.append(newID)
def testKeyLength(self): def testKeyLength(self):
""" Tests the key Node ID key length """ """ Tests the key Node ID key length """
for i in range(20): for i in range(20):
id = self.node._generateID() id = self.node._generateID()
# Key length: 20 bytes == 160 bits # Key length: 20 bytes == 160 bits
self.failUnlessEqual(len(id), 20, 'Length of generated ID is incorrect! Expected 160 bits, got %d bits.' % (len(id)*8)) self.failUnlessEqual(len(id), 48,
'Length of generated ID is incorrect! Expected 384 bits, '
'got %d bits.' % (len(id)*8))
class NodeDataTest(unittest.TestCase): class NodeDataTest(unittest.TestCase):
""" Test case for the Node class's data-related functions """ """ Test case for the Node class's data-related functions """
def setUp(self): def setUp(self):
import lbrynet.dht.contact import lbrynet.dht.contact
h = hashlib.sha1() h = hashlib.sha384()
h.update('test') h.update('test')
self.node = lbrynet.dht.node.Node() self.node = lbrynet.dht.node.Node()
self.contact = lbrynet.dht.contact.Contact(h.digest(), '127.0.0.1', 12345, self.node._protocol) self.contact = lbrynet.dht.contact.Contact(h.digest(), '127.0.0.1', 12345,
self.node._protocol)
self.token = self.node.make_token(self.contact.compact_ip()) self.token = self.node.make_token(self.contact.compact_ip())
self.cases = [] self.cases = []
for i in xrange(5): for i in xrange(5):
h.update(str(i)) h.update(str(i))
self.cases.append((h.digest(), 5000+2*i)) self.cases.append((h.digest(), 5000+2*i))
self.cases.append((h.digest(), 5001+2*i)) self.cases.append((h.digest(), 5001+2*i))
<<<<<<< Updated upstream
#(('a', 'hello there\nthis is a test'),
# ('aMuchLongerKeyThanAnyOfThePreviousOnes', 'some data'))
=======
>>>>>>> Stashed changes
def testStore(self): def testStore(self):
def check_val_in_result(r, peer_info):
self.failUnless
""" Tests if the node can store (and privately retrieve) some data """ """ Tests if the node can store (and privately retrieve) some data """
for key, value in self.cases: for key, value in self.cases:
self.node.store(key, {'port': value, 'bbid': self.contact.id, 'token': self.token}, self.contact.id, _rpcNodeContact=self.contact) request = {
'port': value,
'lbryid': self.contact.id,
'token': self.token
}
self.node.store(key, request, self.contact.id, _rpcNodeContact=self.contact)
for key, value in self.cases: for key, value in self.cases:
expected_result = self.contact.compact_ip() + str(struct.pack('>H', value)) + self.contact.id expected_result = self.contact.compact_ip() + str(struct.pack('>H', value)) + \
self.failUnless(self.node._dataStore.hasPeersForBlob(key), 'Stored key not found in node\'s DataStore: "%s"' % key) self.contact.id
self.failUnless(expected_result in self.node._dataStore.getPeersForBlob(key), 'Stored val not found in node\'s DataStore: key:"%s" port:"%s" %s' % (key, value, self.node._dataStore.getPeersForBlob(key))) self.failUnless(self.node._dataStore.hasPeersForBlob(key),
'Stored key not found in node\'s DataStore: "%s"' % key)
self.failUnless(expected_result in self.node._dataStore.getPeersForBlob(key),
'Stored val not found in node\'s DataStore: key:"%s" port:"%s" %s'
% (key, value, self.node._dataStore.getPeersForBlob(key)))
class NodeContactTest(unittest.TestCase): class NodeContactTest(unittest.TestCase):
""" Test case for the Node class's contact management-related functions """ """ Test case for the Node class's contact management-related functions """
def setUp(self): def setUp(self):
self.node = lbrynet.dht.node.Node() self.node = lbrynet.dht.node.Node()
def testAddContact(self): def testAddContact(self):
""" Tests if a contact can be added and retrieved correctly """ """ Tests if a contact can be added and retrieved correctly """
import lbrynet.dht.contact import lbrynet.dht.contact
@ -91,67 +98,55 @@ class NodeContactTest(unittest.TestCase):
self.node.addContact(contact) self.node.addContact(contact)
# ...and request the closest nodes to it using FIND_NODE # ...and request the closest nodes to it using FIND_NODE
closestNodes = self.node._routingTable.findCloseNodes(contactID, lbrynet.dht.constants.k) closestNodes = self.node._routingTable.findCloseNodes(contactID, lbrynet.dht.constants.k)
self.failUnlessEqual(len(closestNodes), 1, 'Wrong amount of contacts returned; expected 1, got %d' % len(closestNodes)) self.failUnlessEqual(len(closestNodes), 1, 'Wrong amount of contacts returned; '
self.failUnless(contact in closestNodes, 'Added contact not found by issueing _findCloseNodes()') 'expected 1, got %d' % len(closestNodes))
self.failUnless(contact in closestNodes, 'Added contact not found by issueing '
'_findCloseNodes()')
def testAddSelfAsContact(self): def testAddSelfAsContact(self):
""" Tests the node's behaviour when attempting to add itself as a contact """ """ Tests the node's behaviour when attempting to add itself as a contact """
import lbrynet.dht.contact import lbrynet.dht.contact
# Create a contact with the same ID as the local node's ID # Create a contact with the same ID as the local node's ID
contact = lbrynet.dht.contact.Contact(self.node.id, '127.0.0.1', 91824, None) contact = lbrynet.dht.contact.Contact(self.node.node_id, '127.0.0.1', 91824, None)
# Now try to add it # Now try to add it
self.node.addContact(contact) self.node.addContact(contact)
# ...and request the closest nodes to it using FIND_NODE # ...and request the closest nodes to it using FIND_NODE
closestNodes = self.node._routingTable.findCloseNodes(self.node.id, lbrynet.dht.constants.k) closestNodes = self.node._routingTable.findCloseNodes(self.node.node_id,
lbrynet.dht.constants.k)
self.failIf(contact in closestNodes, 'Node added itself as a contact') self.failIf(contact in closestNodes, 'Node added itself as a contact')
<<<<<<< Updated upstream
# """ Test case for the Node class's iterative node lookup algorithm """
# """ Ugly brute-force test to see if the iterative node lookup algorithm runs without failing """
=======
>>>>>>> Stashed changes
"""Some scaffolding for the NodeLookupTest class. Allows isolated
node testing by simulating remote node responses"""
from twisted.internet import protocol, defer, selectreactor
from lbrynet.dht.msgtypes import ResponseMessage
class FakeRPCProtocol(protocol.DatagramProtocol): class FakeRPCProtocol(protocol.DatagramProtocol):
def __init__(self): def __init__(self):
self.reactor = selectreactor.SelectReactor() self.reactor = selectreactor.SelectReactor()
self.testResponse = None self.testResponse = None
self.network = None self.network = None
def createNetwork(self, contactNetwork): def createNetwork(self, contactNetwork):
""" set up a list of contacts together with their closest contacts """
@param contactNetwork: a sequence of tuples, each containing a contact together with its closest set up a list of contacts together with their closest contacts
contacts: C{(<contact>, <closest contact 1, ...,closest contact n>)} @param contactNetwork: a sequence of tuples, each containing a contact together with its
""" closest contacts: C{(<contact>, <closest contact 1, ...,closest contact n>)}
self.network = contactNetwork """
self.network = contactNetwork
""" Fake RPC protocol; allows entangled.kademlia.contact.Contact objects to "send" RPCs """
def sendRPC(self, contact, method, args, rawResponse=False): def sendRPC(self, contact, method, args, rawResponse=False):
""" Fake RPC protocol; allows entangled.kademlia.contact.Contact objects to "send" RPCs"""
if method == "findNode":
if method == "findNode":
# get the specific contacts closest contacts # get the specific contacts closest contacts
closestContacts = [] closestContacts = []
closestContactsList = []
for contactTuple in self.network: for contactTuple in self.network:
if contact == contactTuple[0]: if contact == contactTuple[0]:
# get the list of closest contacts for this contact # get the list of closest contacts for this contact
closestContactsList = contactTuple[1] closestContactsList = contactTuple[1]
# Pack the closest contacts into a ResponseMessage
# Pack the closest contacts into a ResponseMessage
for closeContact in closestContactsList: for closeContact in closestContactsList:
closestContacts.append((closeContact.id, closeContact.address, closeContact.port)) closestContacts.append((closeContact.id, closeContact.address, closeContact.port))
message = ResponseMessage("rpcId", contact.id, closestContacts) message = ResponseMessage("rpcId", contact.id, closestContacts)
df = defer.Deferred() df = defer.Deferred()
df.callback((message,(contact.address, contact.port))) df.callback((message, (contact.address, contact.port)))
return df return df
elif method == "findValue": elif method == "findValue":
for contactTuple in self.network: for contactTuple in self.network:
@ -160,12 +155,10 @@ class FakeRPCProtocol(protocol.DatagramProtocol):
dataDict = contactTuple[2] dataDict = contactTuple[2]
dataKey = dataDict.keys()[0] dataKey = dataDict.keys()[0]
data = dataDict.get(dataKey) data = dataDict.get(dataKey)
# Check if this contact has the requested value # Check if this contact has the requested value
if dataKey == args[0]: if dataKey == args[0]:
# Return the data value # Return the data value
response = dataDict response = dataDict
print "data found at contact: " + contact.id print "data found at contact: " + contact.id
else: else:
# Return the closest contact to the requested data key # Return the closest contact to the requested data key
@ -173,62 +166,52 @@ class FakeRPCProtocol(protocol.DatagramProtocol):
closeContacts = contactTuple[1] closeContacts = contactTuple[1]
closestContacts = [] closestContacts = []
for closeContact in closeContacts: for closeContact in closeContacts:
closestContacts.append((closeContact.id, closeContact.address, closeContact.port)) closestContacts.append((closeContact.id, closeContact.address,
closeContact.port))
response = closestContacts response = closestContacts
# Create the response message # Create the response message
message = ResponseMessage("rpcId", contact.id, response) message = ResponseMessage("rpcId", contact.id, response)
df = defer.Deferred() df = defer.Deferred()
df.callback((message,(contact.address, contact.port))) df.callback((message, (contact.address, contact.port)))
return df return df
def _send(self, data, rpcID, address): def _send(self, data, rpcID, address):
""" fake sending data """ """ fake sending data """
class NodeLookupTest(unittest.TestCase): class NodeLookupTest(unittest.TestCase):
""" Test case for the Node class's iterativeFind node lookup algorithm """ """ Test case for the Node class's iterativeFind node lookup algorithm """
def setUp(self): def setUp(self):
# create a fake protocol to imitate communication with other nodes # create a fake protocol to imitate communication with other nodes
self._protocol = FakeRPCProtocol() self._protocol = FakeRPCProtocol()
# Note: The reactor is never started for this test. All deferred calls run sequentially,
# Note: The reactor is never started for this test. All deferred calls run sequentially,
# since there is no asynchronous network communication # since there is no asynchronous network communication
# create the node to be tested in isolation # create the node to be tested in isolation
self.node = lbrynet.dht.node.Node(None, 4000, None, None, self._protocol) self.node = lbrynet.dht.node.Node('12345678901234567800', 4000, None, None, self._protocol)
self.updPort = 81173 self.updPort = 81173
<<<<<<< Updated upstream
# create a dummy reactor
=======
>>>>>>> Stashed changes
self.contactsAmount = 80 self.contactsAmount = 80
# set the node ID manually for testing
self.node.id = '12345678901234567800'
# Reinitialise the routing table # Reinitialise the routing table
self.node._routingTable = lbrynet.dht.routingtable.OptimizedTreeRoutingTable(self.node.id) self.node._routingTable = lbrynet.dht.routingtable.OptimizedTreeRoutingTable(
self.node.node_id)
# create 160 bit node ID's for test purposes # create 160 bit node ID's for test purposes
self.testNodeIDs = [] self.testNodeIDs = []
idNum = int(self.node.id) idNum = int(self.node.node_id)
for i in range(self.contactsAmount): for i in range(self.contactsAmount):
# create the testNodeIDs in ascending order, away from the actual node ID, with regards to the distance metric # create the testNodeIDs in ascending order, away from the actual node ID,
# with regards to the distance metric
self.testNodeIDs.append(idNum + i + 1) self.testNodeIDs.append(idNum + i + 1)
# generate contacts # generate contacts
self.contacts = [] self.contacts = []
for i in range(self.contactsAmount): for i in range(self.contactsAmount):
contact = lbrynet.dht.contact.Contact(str(self.testNodeIDs[i]), "127.0.0.1", self.updPort + i + 1, self._protocol) contact = lbrynet.dht.contact.Contact(str(self.testNodeIDs[i]), "127.0.0.1",
self.updPort + i + 1, self._protocol)
self.contacts.append(contact) self.contacts.append(contact)
# create the network of contacts in format: (contact, closest contacts) # create the network of contacts in format: (contact, closest contacts)
contactNetwork = ((self.contacts[0], self.contacts[8:15]), contactNetwork = ((self.contacts[0], self.contacts[8:15]),
(self.contacts[1], self.contacts[16:23]), (self.contacts[1], self.contacts[16:23]),
(self.contacts[2], self.contacts[24:31]), (self.contacts[2], self.contacts[24:31]),
@ -254,43 +237,27 @@ class NodeLookupTest(unittest.TestCase):
contacts_with_datastores = [] contacts_with_datastores = []
for contact_tuple in contactNetwork: for contact_tuple in contactNetwork:
contacts_with_datastores.append((contact_tuple[0], contact_tuple[1], lbrynet.dht.datastore.DictDataStore())) contacts_with_datastores.append((contact_tuple[0], contact_tuple[1],
lbrynet.dht.datastore.DictDataStore()))
self._protocol.createNetwork(contacts_with_datastores) self._protocol.createNetwork(contacts_with_datastores)
def testNodeBootStrap(self): def testNodeBootStrap(self):
""" Test bootstrap with the closest possible contacts """ """ Test bootstrap with the closest possible contacts """
df = self.node._iterativeFind(self.node.id, self.contacts[0:8]) df = self.node._iterativeFind(self.node.node_id, self.contacts[0:8])
# Set the expected result # Set the expected result
expectedResult = [] expectedResult = []
for item in self.contacts[0:6]: for item in self.contacts[0:6]:
expectedResult.append(item.id) expectedResult.append(item.id)
# Get the result from the deferred # Get the result from the deferred
activeContacts = df.result activeContacts = df.result
# Check the length of the active contacts # Check the length of the active contacts
self.failUnlessEqual(activeContacts.__len__(), expectedResult.__len__(), \ self.failUnlessEqual(activeContacts.__len__(), expectedResult.__len__(),
"More active contacts should exist, there should be %d contacts" %expectedResult.__len__()) "More active contacts should exist, there should be %d "
"contacts" % expectedResult.__len__())
# Check that the received active contacts are the same as the input contacts # Check that the received active contacts are the same as the input contacts
self.failUnlessEqual(activeContacts, expectedResult, \ self.failUnlessEqual(activeContacts, expectedResult,
"Active should only contain the closest possible contacts which were used as input for the boostrap") "Active should only contain the closest possible contacts"
" which were used as input for the boostrap")
def suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(NodeIDTest))
suite.addTest(unittest.makeSuite(NodeDataTest))
suite.addTest(unittest.makeSuite(NodeContactTest))
suite.addTest(unittest.makeSuite(NodeLookupTest))
return suite
if __name__ == '__main__':
# If this module is executed from the commandline, run all its tests
unittest.TextTestRunner().run(suite())

181
lbrynet/tests/dht/testProtocol.py → lbrynet/tests/dht/test_protocol.py
View file

@ -1,88 +1,22 @@
#!/usr/bin/env python
#
# This library is free software, distributed under the terms of
# the GNU Lesser General Public License Version 3, or any later version.
# See the COPYING file included in this archive
import time import time
import unittest import unittest
from twisted.internet import defer
from twisted.python import failure
import twisted.internet.selectreactor import twisted.internet.selectreactor
from twisted.internet.protocol import DatagramProtocol
import lbrynet.dht.protocol import lbrynet.dht.protocol
import lbrynet.dht.contact import lbrynet.dht.contact
import lbrynet.dht.constants import lbrynet.dht.constants
import lbrynet.dht.msgtypes import lbrynet.dht.msgtypes
from lbrynet.dht.node import rpcmethod from lbrynet.dht.error import TimeoutError
from lbrynet.dht.node import Node, rpcmethod
class FakeNode(object):
""" A fake node object implementing some RPC and non-RPC methods to
test the Kademlia protocol's behaviour
"""
def __init__(self, id):
self.id = id
self.contacts = []
@rpcmethod
def ping(self):
return 'pong'
def pingNoRPC(self):
return 'pong'
@rpcmethod
def echo(self, value):
return value
def addContact(self, contact):
self.contacts.append(contact)
def removeContact(self, contact):
self.contacts.remove(contact)
def indirectPingContact(self, protocol, contact):
""" Pings the given contact (using the specified KademliaProtocol
object, not the direct Contact API), and removes the contact
on a timeout """
df = protocol.sendRPC(contact, 'ping', {})
def handleError(f):
if f.check(lbrynet.dht.protocol.TimeoutError):
self.removeContact(contact)
return f
else:
# This is some other error
return f
df.addErrback(handleError)
return df
class ClientDatagramProtocol(lbrynet.dht.protocol.KademliaProtocol):
data = ''
msgID = ''
destination = ('127.0.0.1', 9182)
def __init__(self):
lbrynet.dht.protocol.KademliaProtocol.__init__(self, None)
def startProtocol(self):
self.sendDatagram()
def sendDatagram(self):
if len(self.data):
self._send(self.data, self.msgID, self.destination)
class KademliaProtocolTest(unittest.TestCase): class KademliaProtocolTest(unittest.TestCase):
""" Test case for the Protocol class """ """ Test case for the Protocol class """
def setUp(self): def setUp(self):
del lbrynet.dht.protocol.reactor del lbrynet.dht.protocol.reactor
lbrynet.dht.protocol.reactor = twisted.internet.selectreactor.SelectReactor() lbrynet.dht.protocol.reactor = twisted.internet.selectreactor.SelectReactor()
self.node = FakeNode('node1') self.node = Node(node_id='node1', udpPort=9182, externalIP="127.0.0.1")
self.protocol = lbrynet.dht.protocol.KademliaProtocol(self.node) self.protocol = lbrynet.dht.protocol.KademliaProtocol(self.node)
def testReactor(self): def testReactor(self):
@ -93,36 +27,66 @@ class KademliaProtocolTest(unittest.TestCase):
def testRPCTimeout(self): def testRPCTimeout(self):
""" Tests if a RPC message sent to a dead remote node times out correctly """ """ Tests if a RPC message sent to a dead remote node times out correctly """
@rpcmethod
def fake_ping(*args, **kwargs):
time.sleep(lbrynet.dht.constants.rpcTimeout + 1)
return 'pong'
real_ping = self.node.ping
real_timeout = lbrynet.dht.constants.rpcTimeout
real_attempts = lbrynet.dht.constants.rpcAttempts
lbrynet.dht.constants.rpcAttempts = 1
lbrynet.dht.constants.rpcTimeout = 1
self.node.ping = fake_ping
deadContact = lbrynet.dht.contact.Contact('node2', '127.0.0.1', 9182, self.protocol) deadContact = lbrynet.dht.contact.Contact('node2', '127.0.0.1', 9182, self.protocol)
self.node.addContact(deadContact) self.node.addContact(deadContact)
# Make sure the contact was added # Make sure the contact was added
self.failIf(deadContact not in self.node.contacts, 'Contact not added to fake node (error in test code)') self.failIf(deadContact not in self.node.contacts,
# Set the timeout to 0 for testing 'Contact not added to fake node (error in test code)')
tempTimeout = lbrynet.dht.constants.rpcTimeout lbrynet.dht.protocol.reactor.listenUDP(9182, self.protocol)
lbrynet.dht.constants.rpcTimeout = 0
lbrynet.dht.protocol.reactor.listenUDP(0, self.protocol) # Run the PING RPC (which should raise a timeout error)
# Run the PING RPC (which should timeout) df = self.protocol.sendRPC(deadContact, 'ping', {})
df = self.node.indirectPingContact(self.protocol, deadContact)
def check_timeout(err):
self.assertEqual(type(err), TimeoutError)
df.addErrback(check_timeout)
def reset_values():
self.node.ping = real_ping
lbrynet.dht.constants.rpcTimeout = real_timeout
lbrynet.dht.constants.rpcAttempts = real_attempts
# See if the contact was removed due to the timeout
def check_removed_contact():
self.failIf(deadContact in self.node.contacts,
'Contact was not removed after RPC timeout; check exception types.')
df.addCallback(lambda _: reset_values())
# Stop the reactor if a result arrives (timeout or not) # Stop the reactor if a result arrives (timeout or not)
df.addBoth(lambda _: lbrynet.dht.protocol.reactor.stop()) df.addBoth(lambda _: lbrynet.dht.protocol.reactor.stop())
df.addCallback(lambda _: check_removed_contact())
lbrynet.dht.protocol.reactor.run() lbrynet.dht.protocol.reactor.run()
# See if the contact was removed due to the timeout
self.failIf(deadContact in self.node.contacts, 'Contact was not removed after RPC timeout; check exception types.')
# Restore the global timeout
lbrynet.dht.constants.rpcTimeout = tempTimeout
def testRPCRequest(self): def testRPCRequest(self):
""" Tests if a valid RPC request is executed and responded to correctly """ """ Tests if a valid RPC request is executed and responded to correctly """
remoteContact = lbrynet.dht.contact.Contact('node2', '127.0.0.1', 9182, self.protocol) remoteContact = lbrynet.dht.contact.Contact('node2', '127.0.0.1', 9182, self.protocol)
self.node.addContact(remoteContact) self.node.addContact(remoteContact)
self.error = None self.error = None
def handleError(f): def handleError(f):
self.error = 'An RPC error occurred: %s' % f.getErrorMessage() self.error = 'An RPC error occurred: %s' % f.getErrorMessage()
def handleResult(result): def handleResult(result):
expectedResult = 'pong' expectedResult = 'pong'
if result != expectedResult: if result != expectedResult:
self.error = 'Result from RPC is incorrect; expected "%s", got "%s"' % (expectedResult, result) self.error = 'Result from RPC is incorrect; expected "%s", got "%s"' \
# Publish the "local" node on the network % (expectedResult, result)
# Publish the "local" node on the network
lbrynet.dht.protocol.reactor.listenUDP(9182, self.protocol) lbrynet.dht.protocol.reactor.listenUDP(9182, self.protocol)
# Simulate the RPC # Simulate the RPC
df = remoteContact.ping() df = remoteContact.ping()
@ -132,17 +96,19 @@ class KademliaProtocolTest(unittest.TestCase):
lbrynet.dht.protocol.reactor.run() lbrynet.dht.protocol.reactor.run()
self.failIf(self.error, self.error) self.failIf(self.error, self.error)
# The list of sent RPC messages should be empty at this stage # The list of sent RPC messages should be empty at this stage
self.failUnlessEqual(len(self.protocol._sentMessages), 0, 'The protocol is still waiting for a RPC result, but the transaction is already done!') self.failUnlessEqual(len(self.protocol._sentMessages), 0,
'The protocol is still waiting for a RPC result, '
'but the transaction is already done!')
def testRPCAccess(self): def testRPCAccess(self):
""" Tests invalid RPC requests """ Tests invalid RPC requests
Verifies that a RPC request for an existing but unpublished Verifies that a RPC request for an existing but unpublished
method is denied, and that the associated (remote) exception gets method is denied, and that the associated (remote) exception gets
raised locally """ raised locally """
remoteContact = lbrynet.dht.contact.Contact('node2', '127.0.0.1', 9182, self.protocol) remoteContact = lbrynet.dht.contact.Contact('node2', '127.0.0.1', 9182, self.protocol)
self.node.addContact(remoteContact) self.node.addContact(remoteContact)
self.error = None self.error = None
def handleError(f): def handleError(f):
try: try:
f.raiseException() f.raiseException()
@ -150,11 +116,14 @@ class KademliaProtocolTest(unittest.TestCase):
# This is the expected outcome since the remote node did not publish the method # This is the expected outcome since the remote node did not publish the method
self.error = None self.error = None
except Exception, e: except Exception, e:
self.error = 'The remote method failed, but the wrong exception was raised; expected AttributeError, got %s' % type(e) self.error = 'The remote method failed, but the wrong exception was raised; ' \
'expected AttributeError, got %s' % type(e)
def handleResult(result): def handleResult(result):
self.error = 'The remote method executed successfully, returning: "%s"; this RPC should not have been allowed.' % result self.error = 'The remote method executed successfully, returning: "%s"; ' \
# Publish the "local" node on the network 'this RPC should not have been allowed.' % result
# Publish the "local" node on the network
lbrynet.dht.protocol.reactor.listenUDP(9182, self.protocol) lbrynet.dht.protocol.reactor.listenUDP(9182, self.protocol)
# Simulate the RPC # Simulate the RPC
df = remoteContact.pingNoRPC() df = remoteContact.pingNoRPC()
@ -164,37 +133,35 @@ class KademliaProtocolTest(unittest.TestCase):
lbrynet.dht.protocol.reactor.run() lbrynet.dht.protocol.reactor.run()
self.failIf(self.error, self.error) self.failIf(self.error, self.error)
# The list of sent RPC messages should be empty at this stage # The list of sent RPC messages should be empty at this stage
self.failUnlessEqual(len(self.protocol._sentMessages), 0, 'The protocol is still waiting for a RPC result, but the transaction is already done!') self.failUnlessEqual(len(self.protocol._sentMessages), 0,
'The protocol is still waiting for a RPC result, '
'but the transaction is already done!')
def testRPCRequestArgs(self): def testRPCRequestArgs(self):
""" Tests if an RPC requiring arguments is executed correctly """ """ Tests if an RPC requiring arguments is executed correctly """
remoteContact = lbrynet.dht.contact.Contact('node2', '127.0.0.1', 9182, self.protocol) remoteContact = lbrynet.dht.contact.Contact('node2', '127.0.0.1', 9182, self.protocol)
self.node.addContact(remoteContact) self.node.addContact(remoteContact)
self.error = None self.error = None
def handleError(f): def handleError(f):
self.error = 'An RPC error occurred: %s' % f.getErrorMessage() self.error = 'An RPC error occurred: %s' % f.getErrorMessage()
def handleResult(result): def handleResult(result):
expectedResult = 'This should be returned.' expectedResult = 'pong'
if result != 'This should be returned.': if result != expectedResult:
self.error = 'Result from RPC is incorrect; expected "%s", got "%s"' % (expectedResult, result) self.error = 'Result from RPC is incorrect; expected "%s", got "%s"' % \
# Publish the "local" node on the network (expectedResult, result)
# Publish the "local" node on the network
lbrynet.dht.protocol.reactor.listenUDP(9182, self.protocol) lbrynet.dht.protocol.reactor.listenUDP(9182, self.protocol)
# Simulate the RPC # Simulate the RPC
df = remoteContact.echo('This should be returned.') df = remoteContact.ping()
df.addCallback(handleResult) df.addCallback(handleResult)
df.addErrback(handleError) df.addErrback(handleError)
df.addBoth(lambda _: lbrynet.dht.protocol.reactor.stop()) df.addBoth(lambda _: lbrynet.dht.protocol.reactor.stop())
lbrynet.dht.protocol.reactor.run() lbrynet.dht.protocol.reactor.run()
self.failIf(self.error, self.error) self.failIf(self.error, self.error)
# The list of sent RPC messages should be empty at this stage # The list of sent RPC messages should be empty at this stage
self.failUnlessEqual(len(self.protocol._sentMessages), 0, 'The protocol is still waiting for a RPC result, but the transaction is already done!') self.failUnlessEqual(len(self.protocol._sentMessages), 0,
'The protocol is still waiting for a RPC result, '
'but the transaction is already done!')
def suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(KademliaProtocolTest))
return suite
if __name__ == '__main__':
# If this module is executed from the commandline, run all its tests
unittest.TextTestRunner().run(suite())

119
lbrynet/tests/dht/testRoutingTable.py → lbrynet/tests/dht/test_routing_table.py
View file

@ -10,6 +10,8 @@ import unittest
import lbrynet.dht.constants import lbrynet.dht.constants
import lbrynet.dht.routingtable import lbrynet.dht.routingtable
import lbrynet.dht.contact import lbrynet.dht.contact
import lbrynet.dht.node
class FakeRPCProtocol(object): class FakeRPCProtocol(object):
""" Fake RPC protocol; allows lbrynet.dht.contact.Contact objects to "send" RPCs """ """ Fake RPC protocol; allows lbrynet.dht.contact.Contact objects to "send" RPCs """
@ -21,6 +23,7 @@ class FakeDeferred(object):
""" Fake Twisted Deferred object; allows the routing table to add callbacks that do nothing """ """ Fake Twisted Deferred object; allows the routing table to add callbacks that do nothing """
def addCallback(self, *args, **kwargs): def addCallback(self, *args, **kwargs):
return return
def addErrback(self, *args, **kwargs): def addErrback(self, *args, **kwargs):
return return
@ -28,34 +31,36 @@ class FakeDeferred(object):
class TreeRoutingTableTest(unittest.TestCase): class TreeRoutingTableTest(unittest.TestCase):
""" Test case for the RoutingTable class """ """ Test case for the RoutingTable class """
def setUp(self): def setUp(self):
h = hashlib.sha1() h = hashlib.sha384()
h.update('node1') h.update('node1')
self.nodeID = h.digest() self.nodeID = h.digest()
self.protocol = FakeRPCProtocol() self.protocol = FakeRPCProtocol()
self.routingTable = lbrynet.dht.routingtable.TreeRoutingTable(self.nodeID) self.routingTable = lbrynet.dht.routingtable.TreeRoutingTable(self.nodeID)
def testDistance(self): def testDistance(self):
""" Test to see if distance method returns correct result""" """ Test to see if distance method returns correct result"""
# testList holds a couple 3-tuple (variable1, variable2, result) # testList holds a couple 3-tuple (variable1, variable2, result)
basicTestList = [('123456789','123456789', 0L), ('12345', '98765', 34527773184L)] basicTestList = [('123456789', '123456789', 0L), ('12345', '98765', 34527773184L)]
for test in basicTestList: for test in basicTestList:
result = self.routingTable.distance(test[0], test[1]) result = lbrynet.dht.node.Distance(test[0])(test[1])
self.failIf(result != test[2], 'Result of _distance() should be %s but %s returned' % (test[2], result)) self.failIf(result != test[2], 'Result of _distance() should be %s but %s returned' %
(test[2], result))
baseIp = '146.64.19.111' baseIp = '146.64.19.111'
ipTestList = ['146.64.29.222', '192.68.19.333'] ipTestList = ['146.64.29.222', '192.68.19.333']
distanceOne = self.routingTable.distance(baseIp, ipTestList[0]) distanceOne = lbrynet.dht.node.Distance(baseIp)(ipTestList[0])
distanceTwo = self.routingTable.distance(baseIp, ipTestList[1]) distanceTwo = lbrynet.dht.node.Distance(baseIp)(ipTestList[1])
self.failIf(distanceOne > distanceTwo, '%s should be closer to the base ip %s than %s' %
(ipTestList[0], baseIp, ipTestList[1]))
self.failIf(distanceOne > distanceTwo, '%s should be closer to the base ip %s than %s' % (ipTestList[0], baseIp, ipTestList[1]))
def testAddContact(self): def testAddContact(self):
""" Tests if a contact can be added and retrieved correctly """ """ Tests if a contact can be added and retrieved correctly """
# Create the contact # Create the contact
h = hashlib.sha1() h = hashlib.sha384()
h.update('node2') h.update('node2')
contactID = h.digest() contactID = h.digest()
contact = lbrynet.dht.contact.Contact(contactID, '127.0.0.1', 91824, self.protocol) contact = lbrynet.dht.contact.Contact(contactID, '127.0.0.1', 91824, self.protocol)
@ -63,12 +68,14 @@ class TreeRoutingTableTest(unittest.TestCase):
self.routingTable.addContact(contact) self.routingTable.addContact(contact)
# ...and request the closest nodes to it (will retrieve it) # ...and request the closest nodes to it (will retrieve it)
closestNodes = self.routingTable.findCloseNodes(contactID, lbrynet.dht.constants.k) closestNodes = self.routingTable.findCloseNodes(contactID, lbrynet.dht.constants.k)
self.failUnlessEqual(len(closestNodes), 1, 'Wrong amount of contacts returned; expected 1, got %d' % len(closestNodes)) self.failUnlessEqual(len(closestNodes), 1, 'Wrong amount of contacts returned; expected 1,'
self.failUnless(contact in closestNodes, 'Added contact not found by issueing _findCloseNodes()') ' got %d' % len(closestNodes))
self.failUnless(contact in closestNodes, 'Added contact not found by issueing '
'_findCloseNodes()')
def testGetContact(self): def testGetContact(self):
""" Tests if a specific existing contact can be retrieved correctly """ """ Tests if a specific existing contact can be retrieved correctly """
h = hashlib.sha1() h = hashlib.sha384()
h.update('node2') h.update('node2')
contactID = h.digest() contactID = h.digest()
contact = lbrynet.dht.contact.Contact(contactID, '127.0.0.1', 91824, self.protocol) contact = lbrynet.dht.contact.Contact(contactID, '127.0.0.1', 91824, self.protocol)
@ -77,9 +84,12 @@ class TreeRoutingTableTest(unittest.TestCase):
# ...and get it again # ...and get it again
sameContact = self.routingTable.getContact(contactID) sameContact = self.routingTable.getContact(contactID)
self.failUnlessEqual(contact, sameContact, 'getContact() should return the same contact') self.failUnlessEqual(contact, sameContact, 'getContact() should return the same contact')
def testAddParentNodeAsContact(self): def testAddParentNodeAsContact(self):
""" Tests the routing table's behaviour when attempting to add its parent node as a contact """ """
Tests the routing table's behaviour when attempting to add its parent node as a contact
"""
# Create a contact with the same ID as the local node's ID # Create a contact with the same ID as the local node's ID
contact = lbrynet.dht.contact.Contact(self.nodeID, '127.0.0.1', 91824, self.protocol) contact = lbrynet.dht.contact.Contact(self.nodeID, '127.0.0.1', 91824, self.protocol)
# Now try to add it # Now try to add it
@ -87,11 +97,11 @@ class TreeRoutingTableTest(unittest.TestCase):
# ...and request the closest nodes to it using FIND_NODE # ...and request the closest nodes to it using FIND_NODE
closestNodes = self.routingTable.findCloseNodes(self.nodeID, lbrynet.dht.constants.k) closestNodes = self.routingTable.findCloseNodes(self.nodeID, lbrynet.dht.constants.k)
self.failIf(contact in closestNodes, 'Node added itself as a contact') self.failIf(contact in closestNodes, 'Node added itself as a contact')
def testRemoveContact(self): def testRemoveContact(self):
""" Tests contact removal """ """ Tests contact removal """
# Create the contact # Create the contact
h = hashlib.sha1() h = hashlib.sha384()
h.update('node2') h.update('node2')
contactID = h.digest() contactID = h.digest()
contact = lbrynet.dht.contact.Contact(contactID, '127.0.0.1', 91824, self.protocol) contact = lbrynet.dht.contact.Contact(contactID, '127.0.0.1', 91824, self.protocol)
@ -105,54 +115,73 @@ class TreeRoutingTableTest(unittest.TestCase):
def testSplitBucket(self): def testSplitBucket(self):
""" Tests if the the routing table correctly dynamically splits k-buckets """ """ Tests if the the routing table correctly dynamically splits k-buckets """
self.failUnlessEqual(self.routingTable._buckets[0].rangeMax, 2**160, 'Initial k-bucket range should be 0 <= range < 2**160') self.failUnlessEqual(self.routingTable._buckets[0].rangeMax, 2**384,
'Initial k-bucket range should be 0 <= range < 2**384')
# Add k contacts # Add k contacts
for i in range(lbrynet.dht.constants.k): for i in range(lbrynet.dht.constants.k):
h = hashlib.sha1() h = hashlib.sha384()
h.update('remote node %d' % i) h.update('remote node %d' % i)
nodeID = h.digest() nodeID = h.digest()
contact = lbrynet.dht.contact.Contact(nodeID, '127.0.0.1', 91824, self.protocol) contact = lbrynet.dht.contact.Contact(nodeID, '127.0.0.1', 91824, self.protocol)
self.routingTable.addContact(contact) self.routingTable.addContact(contact)
self.failUnlessEqual(len(self.routingTable._buckets), 1, 'Only k nodes have been added; the first k-bucket should now be full, but should not yet be split') self.failUnlessEqual(len(self.routingTable._buckets), 1,
'Only k nodes have been added; the first k-bucket should now '
'be full, but should not yet be split')
# Now add 1 more contact # Now add 1 more contact
h = hashlib.sha1() h = hashlib.sha384()
h.update('yet another remote node') h.update('yet another remote node')
nodeID = h.digest() nodeID = h.digest()
contact = lbrynet.dht.contact.Contact(nodeID, '127.0.0.1', 91824, self.protocol) contact = lbrynet.dht.contact.Contact(nodeID, '127.0.0.1', 91824, self.protocol)
self.routingTable.addContact(contact) self.routingTable.addContact(contact)
self.failUnlessEqual(len(self.routingTable._buckets), 2, 'k+1 nodes have been added; the first k-bucket should have been split into two new buckets') self.failUnlessEqual(len(self.routingTable._buckets), 2,
self.failIfEqual(self.routingTable._buckets[0].rangeMax, 2**160, 'K-bucket was split, but its range was not properly adjusted') 'k+1 nodes have been added; the first k-bucket should have been '
self.failUnlessEqual(self.routingTable._buckets[1].rangeMax, 2**160, 'K-bucket was split, but the second (new) bucket\'s max range was not set properly') 'split into two new buckets')
self.failUnlessEqual(self.routingTable._buckets[0].rangeMax, self.routingTable._buckets[1].rangeMin, 'K-bucket was split, but the min/max ranges were not divided properly') self.failIfEqual(self.routingTable._buckets[0].rangeMax, 2**384,
'K-bucket was split, but its range was not properly adjusted')
self.failUnlessEqual(self.routingTable._buckets[1].rangeMax, 2**384,
'K-bucket was split, but the second (new) bucket\'s '
'max range was not set properly')
self.failUnlessEqual(self.routingTable._buckets[0].rangeMax,
self.routingTable._buckets[1].rangeMin,
'K-bucket was split, but the min/max ranges were '
'not divided properly')
def testFullBucketNoSplit(self): def testFullBucketNoSplit(self):
""" Test that a bucket is not split if it full, but does not cover the range containing the parent node's ID """ """
self.routingTable._parentNodeID = 21*'a' # more than 160 bits; this will not be in the range of _any_ k-bucket Test that a bucket is not split if it full, but does not cover the range
containing the parent node's ID
"""
self.routingTable._parentNodeID = 49 * 'a'
# more than 384 bits; this will not be in the range of _any_ k-bucket
# Add k contacts # Add k contacts
for i in range(lbrynet.dht.constants.k): for i in range(lbrynet.dht.constants.k):
h = hashlib.sha1() h = hashlib.sha384()
h.update('remote node %d' % i) h.update('remote node %d' % i)
nodeID = h.digest() nodeID = h.digest()
contact = lbrynet.dht.contact.Contact(nodeID, '127.0.0.1', 91824, self.protocol) contact = lbrynet.dht.contact.Contact(nodeID, '127.0.0.1', 91824, self.protocol)
self.routingTable.addContact(contact) self.routingTable.addContact(contact)
self.failUnlessEqual(len(self.routingTable._buckets), 1, 'Only k nodes have been added; the first k-bucket should now be full, and there should not be more than 1 bucket') self.failUnlessEqual(len(self.routingTable._buckets), 1, 'Only k nodes have been added; '
self.failUnlessEqual(len(self.routingTable._buckets[0]._contacts), lbrynet.dht.constants.k, 'Bucket should have k contacts; expected %d got %d' % (lbrynet.dht.constants.k, len(self.routingTable._buckets[0]._contacts))) 'the first k-bucket should now be '
'full, and there should not be '
'more than 1 bucket')
self.failUnlessEqual(len(self.routingTable._buckets[0]._contacts), lbrynet.dht.constants.k,
'Bucket should have k contacts; expected %d got %d' %
(lbrynet.dht.constants.k,
len(self.routingTable._buckets[0]._contacts)))
# Now add 1 more contact # Now add 1 more contact
h = hashlib.sha1() h = hashlib.sha384()
h.update('yet another remote node') h.update('yet another remote node')
nodeID = h.digest() nodeID = h.digest()
contact = lbrynet.dht.contact.Contact(nodeID, '127.0.0.1', 91824, self.protocol) contact = lbrynet.dht.contact.Contact(nodeID, '127.0.0.1', 91824, self.protocol)
self.routingTable.addContact(contact) self.routingTable.addContact(contact)
self.failUnlessEqual(len(self.routingTable._buckets), 1, 'There should not be more than 1 bucket, since the bucket should not have been split (parent node ID not in range)') self.failUnlessEqual(len(self.routingTable._buckets), 1,
self.failUnlessEqual(len(self.routingTable._buckets[0]._contacts), lbrynet.dht.constants.k, 'Bucket should have k contacts; expected %d got %d' % (lbrynet.dht.constants.k, len(self.routingTable._buckets[0]._contacts))) 'There should not be more than 1 bucket, since the bucket '
self.failIf(contact in self.routingTable._buckets[0]._contacts, 'New contact should have been discarded (since RPC is faked in this test)') 'should not have been split (parent node ID not in range)')
self.failUnlessEqual(len(self.routingTable._buckets[0]._contacts),
lbrynet.dht.constants.k, 'Bucket should have k contacts; '
'expected %d got %d' %
(lbrynet.dht.constants.k,
len(self.routingTable._buckets[0]._contacts)))
self.failIf(contact in self.routingTable._buckets[0]._contacts,
'New contact should have been discarded (since RPC is faked in this test)')
def suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TreeRoutingTableTest))
return suite
if __name__ == '__main__':
# If this module is executed from the commandline, run all its tests
unittest.TextTestRunner().run(suite())