diff --git a/scripts/create_network.py b/scripts/create_network.py deleted file mode 100644 index 52726c4e1..000000000 --- a/scripts/create_network.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python -# -# This library is free software, distributed under the terms of -# the GNU Lesser General Public License Version 3, or any later version. -# See the COPYING file included in this archive -# - -# Thanks to Paul Cannon for IP-address resolution functions (taken from aspn.activestate.com) - -import argparse -import os -import sys -import time -import signal - -amount = 0 - - -def destroyNetwork(nodes): - print 'Destroying Kademlia network' - i = 0 - for node in nodes: - i += 1 - hashAmount = i * 50 / amount - hashbar = '#' * hashAmount - output = '\r[%-50s] %d/%d' % (hashbar, i, amount) - sys.stdout.write(output) - time.sleep(0.15) - os.kill(node, signal.SIGTERM) - print - - -def main(): - parser = argparse.ArgumentParser(description="Launch a network of dht nodes") - - parser.add_argument("amount_of_nodes", - help="The number of nodes to create", - type=int) - parser.add_argument( - "--nic_ip_address", - help=("The network interface on which these nodes will listen for connections " - "from each other and from other nodes. If omitted, an attempt will be " - "made to automatically determine the system's IP address, but this may " - "result in the nodes being reachable only from this system")) - - args = parser.parse_args() - - global amount - amount = args.amount_of_nodes - if args.nic_ip_address: - ipAddress = args.nic_ip_address - else: - import socket - ipAddress = socket.gethostbyname(socket.gethostname()) - print 'Network interface IP address omitted; using %s' % ipAddress - - startPort = 4000 - port = startPort + 1 - nodes = [] - print 'Creating Kademlia network' - try: - node = os.spawnlp( - os.P_NOWAIT, 'lbrynet-launch-node', 'lbrynet-launch-node', str(startPort)) - nodes.append(node) - for i in range(amount - 1): - time.sleep(0.15) - hashAmount = i * 50 / amount - hashbar = '#' * hashAmount - output = '\r[%-50s] %d/%d' % (hashbar, i, amount) - sys.stdout.write(output) - node = os.spawnlp( - os.P_NOWAIT, 'lbrynet-launch-node', 'lbrynet-launch-node', str(port), - ipAddress, str(startPort)) - nodes.append(node) - port += 1 - except KeyboardInterrupt: - '\nNetwork creation cancelled.' - destroyNetwork(nodes) - sys.exit(1) - - print '\n\n---------------\nNetwork running\n---------------\n' - try: - while 1: - time.sleep(1) - except KeyboardInterrupt: - pass - finally: - destroyNetwork(nodes) - - -if __name__ == '__main__': - main() diff --git a/scripts/dht_scripts.py b/scripts/dht_scripts.py deleted file mode 100644 index 0aec28c57..000000000 --- a/scripts/dht_scripts.py +++ /dev/null @@ -1,129 +0,0 @@ -from lbrynet.core import log_support - -import logging.handlers -import sys -import traceback - -from lbrynet.dht.node import Node - -from twisted.internet import reactor, defer -from lbrynet.core.utils import generate_id - - -log = logging.getLogger(__name__) - - -def print_usage(): - print "Usage:\n%s UDP_PORT KNOWN_NODE_IP KNOWN_NODE_PORT HASH" - - -@defer.inlineCallbacks -def join_network(udp_port, known_nodes): - lbryid = generate_id() - - log.info('Creating node') - node = Node(udpPort=udp_port, node_id=lbryid) - - log.info('Joining network') - yield node.joinNetwork(known_nodes) - - defer.returnValue(node) - - -@defer.inlineCallbacks -def get_hosts(node, h): - log.info("Looking up %s", h) - hosts = yield node.getPeersForBlob(h.decode("hex")) - log.info("Hosts returned from the DHT: %s", hosts) - - -@defer.inlineCallbacks -def announce_hash(node, h): - results = yield node.announceHaveBlob(h, 34567) - for success, result in results: - if success: - log.info("Succeeded: %s", str(result)) - else: - log.info("Failed: %s", str(result.getErrorMessage())) - - -# def get_args(): -# if len(sys.argv) < 5: -# print_usage() -# sys.exit(1) -# udp_port = int(sys.argv[1]) -# known_nodes = [(sys.argv[2], int(sys.argv[3]))] -# h = binascii.unhexlify(sys.argv[4]) -# return udp_port, known_nodes, h - - -@defer.inlineCallbacks -def connect(port=None): - try: - if port is None: - raise Exception("need a port") - known_nodes = [('54.236.227.82', 4444)] # lbrynet1 - node = yield join_network(port, known_nodes) - log.info("joined") - reactor.callLater(3, find, node) - except Exception: - log.error("CAUGHT EXCEPTION") - traceback.print_exc() - log.info("Stopping reactor") - yield reactor.stop() - - -def getApproximateTotalDHTNodes(node): - from lbrynet.dht import constants - # get the deepest bucket and the number of contacts in that bucket and multiply it - # by the number of equivalently deep buckets in the whole DHT to get a really bad - # estimate! - bucket = node._routingTable._buckets[node._routingTable._kbucketIndex(node.node_id)] - num_in_bucket = len(bucket._contacts) - factor = (2 ** constants.key_bits) / (bucket.rangeMax - bucket.rangeMin) - return num_in_bucket * factor - - -def getApproximateTotalHashes(node): - # Divide the number of hashes we know about by k to get a really, really, really - # bad estimate of the average number of hashes per node, then multiply by the - # approximate number of nodes to get a horrendous estimate of the total number - # of hashes in the DHT - num_in_data_store = len(node._dataStore._dict) - if num_in_data_store == 0: - return 0 - return num_in_data_store * getApproximateTotalDHTNodes(node) / 8 - - -@defer.inlineCallbacks -def find(node): - try: - log.info("Approximate number of nodes in DHT: %s", str(getApproximateTotalDHTNodes(node))) - log.info("Approximate number of blobs in DHT: %s", str(getApproximateTotalHashes(node))) - - h = "578f5e82da7db97bfe0677826d452cc0c65406a8e986c9caa126af4ecdbf4913daad2f7f5d1fb0ffec17d0bf8f187f5a" - peersFake = yield node.getPeersForBlob(h.decode("hex")) - print peersFake - peers = yield node.getPeersForBlob(h.decode("hex")) - print peers - - # yield get_hosts(node, h) - except Exception: - log.error("CAUGHT EXCEPTION") - traceback.print_exc() - - log.info("Stopping reactor") - yield reactor.stop() - - - -def main(): - log_support.configure_console(level='DEBUG') - log_support.configure_twisted() - reactor.callLater(0, connect, port=10001) - log.info("Running reactor") - reactor.run() - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/scripts/dhttest.py b/scripts/dhttest.py deleted file mode 100644 index fe0a0af7f..000000000 --- a/scripts/dhttest.py +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env python -# -# This is a basic single-node example of how to use the Entangled -# DHT. It creates a Node and (optionally) joins an existing DHT. It -# then does a Kademlia store and find, and then it deletes the stored -# value (non-Kademlia method). -# -# No tuple space functionality is demonstrated by this script. -# -# To test it properly, start a multi-node Kademlia DHT with the "create_network.py" -# script and point this node to that, e.g.: -# $python create_network.py 10 127.0.0.1 -# -# $python basic_example.py 5000 127.0.0.1 4000 -# -# This library is free software, distributed under the terms of -# the GNU Lesser General Public License Version 3, or any later version. -# See the COPYING file included in this archive -# - -# Thanks to Paul Cannon for IP-address resolution functions (taken from aspn.activestate.com) - - -import binascii -import random -import twisted.internet.reactor -from lbrynet.dht.node import Node -from lbrynet.core.cryptoutils import get_lbry_hash_obj - -# The Entangled DHT node; instantiated in the main() method -node = None - -# The key to use for this example when storing/retrieving data -h = get_lbry_hash_obj() -h.update("key") -KEY = h.digest() -# The value to store -VALUE = random.randint(10000, 20000) - -lbryid = KEY - - -def storeValue(key, value): - """ Stores the specified value in the DHT using the specified key """ - global node - print '\nStoring value; Key: %s, Value: %s' % (key, value) - # Store the value in the DHT. This method returns a Twisted - # Deferred result, which we then add callbacks to - deferredResult = node.announceHaveHash(key, value) - # Add our callback; this method is called when the operation completes... - deferredResult.addCallback(storeValueCallback) - # ...and for error handling, add an "error callback" as well. - # - # For this example script, I use a generic error handler; usually - # you would need something more specific - deferredResult.addErrback(genericErrorCallback) - - -def storeValueCallback(*args, **kwargs): - """ Callback function that is invoked when the storeValue() operation succeeds """ - print 'Value has been stored in the DHT' - # Now that the value has been stored, schedule that the value is read again after 2.5 seconds - print 'Scheduling retrieval in 2.5 seconds' - twisted.internet.reactor.callLater(2.5, getValue) - - -def genericErrorCallback(failure): - """ Callback function that is invoked if an error occurs during any of the DHT operations """ - print 'An error has occurred:', failure.getErrorMessage() - twisted.internet.reactor.callLater(0, stop) - - -def getValue(): - """ Retrieves the value of the specified key (KEY) from the DHT """ - global node, KEY - # Get the value for the specified key (immediately returns a Twisted deferred result) - print ('\nRetrieving value from DHT for key "%s"' % - binascii.unhexlify("f7d9dc4de674eaa2c5a022eb95bc0d33ec2e75c6")) - deferredResult = node.iterativeFindValue( - binascii.unhexlify("f7d9dc4de674eaa2c5a022eb95bc0d33ec2e75c6")) - # Add a callback to this result; this will be called as soon as the operation has completed - deferredResult.addCallback(getValueCallback) - # As before, add the generic error callback - deferredResult.addErrback(genericErrorCallback) - - -def getValueCallback(result): - """ Callback function that is invoked when the getValue() operation succeeds """ - # Check if the key was found (result is a dict of format {key: - # value}) or not (in which case a list of "closest" Kademlia - # contacts would be returned instead") - print "Got the value" - print result - - # Either way, schedule a "delete" operation for the key - print 'Scheduling shutdown in 2.5 seconds' - twisted.internet.reactor.callLater(2.5, stop) - - -def stop(): - """ Stops the Twisted reactor, and thus the script """ - print '\nStopping Kademlia node and terminating script' - twisted.internet.reactor.stop() - - -if __name__ == '__main__': - import sys - - if len(sys.argv) < 2: - print 'Usage:\n%s UDP_PORT [KNOWN_NODE_IP KNOWN_NODE_PORT]' % sys.argv[0] - print 'or:\n%s UDP_PORT [FILE_WITH_KNOWN_NODES]' % sys.argv[0] - print - print 'If a file is specified, it should containg one IP address and UDP port' - print 'per line, seperated by a space.' - sys.exit(1) - try: - int(sys.argv[1]) - except ValueError: - print '\nUDP_PORT must be an integer value.\n' - print 'Usage:\n%s UDP_PORT [KNOWN_NODE_IP KNOWN_NODE_PORT]' % sys.argv[0] - print 'or:\n%s UDP_PORT [FILE_WITH_KNOWN_NODES]' % sys.argv[0] - print - print 'If a file is specified, it should contain one IP address and UDP port' - print 'per line, seperated by a space.' - sys.exit(1) - - if len(sys.argv) == 4: - knownNodes = [(sys.argv[2], int(sys.argv[3]))] - elif len(sys.argv) == 3: - knownNodes = [] - f = open(sys.argv[2], 'r') - lines = f.readlines() - f.close() - for line in lines: - ipAddress, udpPort = line.split() - knownNodes.append((ipAddress, int(udpPort))) - else: - knownNodes = None - print '\nNOTE: You have not specified any remote DHT node(s) to connect to' - print 'It will thus not be aware of any existing DHT, but will still function as' - print ' a self-contained DHT (until another node contacts it).' - print 'Run this script without any arguments for info.\n' - - # Set up SQLite-based data store (you could use an in-memory store instead, for example) - # - # Create the Entangled node. It extends the functionality of a - # basic Kademlia node (but is fully backwards-compatible with a - # Kademlia-only network) - # - # If you wish to have a pure Kademlia network, use the - # entangled.kademlia.node.Node class instead - print 'Creating Node' - node = Node(udpPort=int(sys.argv[1]), node_id=lbryid) - - # Schedule the node to join the Kademlia/Entangled DHT - node.joinNetwork(knownNodes) - # Schedule the "storeValue() call to be invoked after 2.5 seconds, - # using KEY and VALUE as arguments - twisted.internet.reactor.callLater(2.5, getValue) - # Start the Twisted reactor - this fires up all networking, and - # allows the scheduled join operation to take place - print 'Twisted reactor started (script will commence in 2.5 seconds)' - twisted.internet.reactor.run() diff --git a/scripts/node_rpc_cli.py b/scripts/node_rpc_cli.py deleted file mode 100644 index 43ffa6daf..000000000 --- a/scripts/node_rpc_cli.py +++ /dev/null @@ -1,41 +0,0 @@ -""" -CLI for sending rpc commands to a DHT node -""" - -import argparse - -from twisted.internet import reactor -from txjsonrpc.web.jsonrpc import Proxy - - -def print_value(value): - print value - - -def print_error(err): - print err.getErrorMessage() - - -def shut_down(): - reactor.stop() - - -def main(): - parser = argparse.ArgumentParser(description="Send an rpc command to a dht node") - parser.add_argument("rpc_command", - help="The rpc command to send to the dht node") - parser.add_argument("--node_host", - help="The host of the node to connect to", - default="127.0.0.1") - parser.add_argument("--node_port", - help="The port of the node to connect to", - default="8888") - - args = parser.parse_args() - connect_string = 'http://%s:%s' % (args.node_host, args.node_port) - proxy = Proxy(connect_string) - - d = proxy.callRemote(args.rpc_command) - d.addCallbacks(print_value, print_error) - d.addBoth(lambda _: shut_down()) - reactor.run() diff --git a/scripts/rpc_node.py b/scripts/rpc_node.py deleted file mode 100644 index 40d69b8e7..000000000 --- a/scripts/rpc_node.py +++ /dev/null @@ -1,214 +0,0 @@ -import logging -import requests -import miniupnpc -import argparse -from copy import deepcopy -from twisted.internet import reactor, defer -from twisted.web import resource -from twisted.web.server import Site - -from lbrynet import conf -from lbrynet.core.log_support import configure_console -from lbrynet.dht.error import TimeoutError -conf.initialize_settings() - -log = logging.getLogger("dht tool") -configure_console() -log.setLevel(logging.INFO) - -from lbrynet.dht.node import Node -from lbrynet.dht.contact import Contact -from lbrynet.daemon.auth.server import AuthJSONRPCServer -from lbrynet.core.utils import generate_id - -def get_external_ip_and_setup_upnp(): - try: - u = miniupnpc.UPnP() - u.discoverdelay = 200 - u.discover() - u.selectigd() - - if u.getspecificportmapping(4444, "UDP"): - u.deleteportmapping(4444, "UDP") - log.info("Removed UPnP redirect for UDP 4444.") - u.addportmapping(4444, 'UDP', u.lanaddr, 4444, 'LBRY DHT port', '') - log.info("got external ip from upnp") - return u.externalipaddress() - except Exception: - log.exception("derp") - r = requests.get('https://api.ipify.org', {'format': 'json'}) - log.info("got external ip from ipify.org") - return r.json()['ip'] - - -class NodeRPC(AuthJSONRPCServer): - def __init__(self, lbryid, seeds, node_port, rpc_port): - AuthJSONRPCServer.__init__(self, False) - self.root = None - self.port = None - self.seeds = seeds - self.node_port = node_port - self.rpc_port = rpc_port - if lbryid: - lbryid = lbryid.decode('hex') - else: - lbryid = generate_id() - self.node_id = lbryid - self.external_ip = get_external_ip_and_setup_upnp() - self.node_port = node_port - - @defer.inlineCallbacks - def setup(self): - self.node = Node(node_id=self.node_id, udpPort=self.node_port, - externalIP=self.external_ip) - hosts = [] - for hostname, hostport in self.seeds: - host_ip = yield reactor.resolve(hostname) - hosts.append((host_ip, hostport)) - log.info("connecting to dht") - yield self.node.joinNetwork(tuple(hosts)) - log.info("connected to dht") - if not self.announced_startup: - self.announced_startup = True - self.start_api() - log.info("lbry id: %s (%i bytes)", self.node.node_id.encode('hex'), len(self.node.node_id)) - - def start_api(self): - root = resource.Resource() - root.putChild('', self) - self.port = reactor.listenTCP(self.rpc_port, Site(root), interface='localhost') - log.info("started jsonrpc server") - - @defer.inlineCallbacks - def jsonrpc_node_id_set(self, node_id): - old_id = self.node.node_id - self.node.stop() - del self.node - self.node_id = node_id.decode('hex') - yield self.setup() - msg = "changed dht id from %s to %s" % (old_id.encode('hex'), - self.node.node_id.encode('hex')) - defer.returnValue(msg) - - def jsonrpc_node_id_get(self): - return self._render_response(self.node.node_id.encode('hex')) - - @defer.inlineCallbacks - def jsonrpc_peer_find(self, node_id): - node_id = node_id.decode('hex') - contact = yield self.node.findContact(node_id) - result = None - if contact: - result = (contact.address, contact.port) - defer.returnValue(result) - - @defer.inlineCallbacks - def jsonrpc_peer_list_for_blob(self, blob_hash): - peers = yield self.node.getPeersForBlob(blob_hash.decode('hex')) - defer.returnValue(peers) - - @defer.inlineCallbacks - def jsonrpc_ping(self, node_id): - contact_host = yield self.jsonrpc_peer_find(node_id=node_id) - if not contact_host: - defer.returnValue("failed to find node") - contact_ip, contact_port = contact_host - contact = Contact(node_id.decode('hex'), contact_ip, contact_port, self.node._protocol) - try: - result = yield contact.ping() - except TimeoutError: - self.node.removeContact(contact.id) - self.node._dataStore.removePeer(contact.id) - result = {'error': 'timeout'} - defer.returnValue(result) - - def get_routing_table(self): - result = {} - data_store = deepcopy(self.node._dataStore._dict) - datastore_len = len(data_store) - hosts = {} - missing_contacts = [] - if datastore_len: - for k, v in data_store.iteritems(): - for value, lastPublished, originallyPublished, originalPublisherID in v: - try: - contact = self.node._routingTable.getContact(originalPublisherID) - except ValueError: - if originalPublisherID.encode('hex') not in missing_contacts: - missing_contacts.append(originalPublisherID.encode('hex')) - continue - if contact in hosts: - blobs = hosts[contact] - else: - blobs = [] - blobs.append(k.encode('hex')) - hosts[contact] = blobs - - contact_set = [] - blob_hashes = [] - result['buckets'] = {} - - for i in range(len(self.node._routingTable._buckets)): - for contact in self.node._routingTable._buckets[i]._contacts: - contacts = result['buckets'].get(i, []) - if contact in hosts: - blobs = hosts[contact] - del hosts[contact] - else: - blobs = [] - host = { - "address": contact.address, - "id": contact.id.encode("hex"), - "blobs": blobs, - } - for blob_hash in blobs: - if blob_hash not in blob_hashes: - blob_hashes.append(blob_hash) - contacts.append(host) - result['buckets'][i] = contacts - contact_set.append(contact.id.encode("hex")) - if hosts: - result['datastore extra'] = [ - { - "id": host.id.encode('hex'), - "blobs": hosts[host], - } - for host in hosts] - result['missing contacts'] = missing_contacts - result['contacts'] = contact_set - result['blob hashes'] = blob_hashes - result['node id'] = self.node_id.encode('hex') - return result - - def jsonrpc_routing_table_get(self): - return self._render_response(self.get_routing_table()) - - -def main(): - parser = argparse.ArgumentParser(description="Launch a dht node which responds to rpc commands") - parser.add_argument("--node_port", - help=("The UDP port on which the node will listen for connections " - "from other dht nodes"), - type=int, default=4444) - parser.add_argument("--rpc_port", - help="The TCP port on which the node will listen for rpc commands", - type=int, default=5280) - parser.add_argument("--bootstrap_host", - help="The IP of a DHT node to be used to bootstrap into the network", - default='lbrynet1.lbry.io') - parser.add_argument("--node_id", - help="The IP of a DHT node to be used to bootstrap into the network", - default=None) - parser.add_argument("--bootstrap_port", - help="The port of a DHT node to be used to bootstrap into the network", - default=4444, type=int) - - args = parser.parse_args() - seeds = [(args.bootstrap_host, args.bootstrap_port)] - server = NodeRPC(args.node_id, seeds, args.node_port, args.rpc_port) - reactor.addSystemEventTrigger('after', 'startup', server.setup) - reactor.run() - - -if __name__ == "__main__": - main() diff --git a/scripts/simple_dht_node.py b/scripts/simple_dht_node.py deleted file mode 100755 index cb8bc727e..000000000 --- a/scripts/simple_dht_node.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python - -from lbrynet.core import log_support -import logging.handlers -import sys -import time -from pprint import pprint - -from twisted.internet import defer, reactor -from lbrynet.dht.node import Node -import lbrynet.dht.constants -import lbrynet.dht.datastore -from lbrynet.tests.util import random_lbry_hash - -log = logging.getLogger(__name__) - - -@defer.inlineCallbacks -def run(): - nodeid = "9648996b4bef3ff41176668a0577f86aba7f1ea2996edd18f9c42430802c8085331345c5f0c44a7f352e2ba8ae59aaaa".decode("hex") - node = Node(node_id=nodeid, externalIP='127.0.0.1', udpPort=21999, peerPort=1234) - node.startNetwork() - yield node.joinNetwork([("127.0.0.1", 21001)]) - - print "" - print "" - print "" - print "" - print "" - print "" - - yield node.announceHaveBlob("2bb150cb996b4bef3ff41176648a0577f86abb7f1ea2996edd18f9c42430802c8085331345c5f0c44a7f352e2ba8ae59".decode("hex")) - - log.info("Shutting down...") - reactor.callLater(1, reactor.stop) - - -def main(): - log_support.configure_console(level='DEBUG') - log_support.configure_twisted() - reactor.callLater(0, run) - log.info("Running reactor") - reactor.run() - - -if __name__ == '__main__': - sys.exit(main())