Madiator2011/lbry-sdk
1
0
Fork 0
forked from LBRYCommunity/lbry-sdk
Code Pull requests Activity

remove unused scripts

Browse source
This commit is contained in:
Jack Robison 2018-05-29 16:15:12 -04:00
parent 9582b7fcf5
commit 51b42da1c5
No known key found for this signature in database
GPG key ID: DF25C68FE0239BB2
6 changed files with 0 additions and 686 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

92
scripts/create_network.py
View file

@ -1,92 +0,0 @@
#!/usr/bin/env python
#
# This library is free software, distributed under the terms of
# the GNU Lesser General Public License Version 3, or any later version.
# See the COPYING file included in this archive
#
# Thanks to Paul Cannon for IP-address resolution functions (taken from aspn.activestate.com)
import argparse
import os
import sys
import time
import signal
amount = 0
def destroyNetwork(nodes):
print 'Destroying Kademlia network'
i = 0
for node in nodes:
i += 1
hashAmount = i * 50 / amount
hashbar = '#' * hashAmount
output = '\r[%-50s] %d/%d' % (hashbar, i, amount)
sys.stdout.write(output)
time.sleep(0.15)
os.kill(node, signal.SIGTERM)
print
def main():
parser = argparse.ArgumentParser(description="Launch a network of dht nodes")
parser.add_argument("amount_of_nodes",
help="The number of nodes to create",
type=int)
parser.add_argument(
"--nic_ip_address",
help=("The network interface on which these nodes will listen for connections "
"from each other and from other nodes. If omitted, an attempt will be "
"made to automatically determine the system's IP address, but this may "
"result in the nodes being reachable only from this system"))
args = parser.parse_args()
global amount
amount = args.amount_of_nodes
if args.nic_ip_address:
ipAddress = args.nic_ip_address
else:
import socket
ipAddress = socket.gethostbyname(socket.gethostname())
print 'Network interface IP address omitted; using %s' % ipAddress
startPort = 4000
port = startPort + 1
nodes = []
print 'Creating Kademlia network'
try:
node = os.spawnlp(
os.P_NOWAIT, 'lbrynet-launch-node', 'lbrynet-launch-node', str(startPort))
nodes.append(node)
for i in range(amount - 1):
time.sleep(0.15)
hashAmount = i * 50 / amount
hashbar = '#' * hashAmount
output = '\r[%-50s] %d/%d' % (hashbar, i, amount)
sys.stdout.write(output)
node = os.spawnlp(
os.P_NOWAIT, 'lbrynet-launch-node', 'lbrynet-launch-node', str(port),
ipAddress, str(startPort))
nodes.append(node)
port += 1
except KeyboardInterrupt:
'\nNetwork creation cancelled.'
destroyNetwork(nodes)
sys.exit(1)
print '\n\n---------------\nNetwork running\n---------------\n'
try:
while 1:
time.sleep(1)
except KeyboardInterrupt:
pass
finally:
destroyNetwork(nodes)
if __name__ == '__main__':
main()

129
scripts/dht_scripts.py
View file

@ -1,129 +0,0 @@
from lbrynet.core import log_support
import logging.handlers
import sys
import traceback
from lbrynet.dht.node import Node
from twisted.internet import reactor, defer
from lbrynet.core.utils import generate_id
log = logging.getLogger(__name__)
def print_usage():
print "Usage:\n%s UDP_PORT KNOWN_NODE_IP KNOWN_NODE_PORT HASH"
@defer.inlineCallbacks
def join_network(udp_port, known_nodes):
lbryid = generate_id()
log.info('Creating node')
node = Node(udpPort=udp_port, node_id=lbryid)
log.info('Joining network')
yield node.joinNetwork(known_nodes)
defer.returnValue(node)
@defer.inlineCallbacks
def get_hosts(node, h):
log.info("Looking up %s", h)
hosts = yield node.getPeersForBlob(h.decode("hex"))
log.info("Hosts returned from the DHT: %s", hosts)
@defer.inlineCallbacks
def announce_hash(node, h):
results = yield node.announceHaveBlob(h, 34567)
for success, result in results:
if success:
log.info("Succeeded: %s", str(result))
else:
log.info("Failed: %s", str(result.getErrorMessage()))
# def get_args():
# if len(sys.argv) < 5:
# print_usage()
# sys.exit(1)
# udp_port = int(sys.argv[1])
# known_nodes = [(sys.argv[2], int(sys.argv[3]))]
# h = binascii.unhexlify(sys.argv[4])
# return udp_port, known_nodes, h
@defer.inlineCallbacks
def connect(port=None):
try:
if port is None:
raise Exception("need a port")
known_nodes = [('54.236.227.82', 4444)] # lbrynet1
node = yield join_network(port, known_nodes)
log.info("joined")
reactor.callLater(3, find, node)
except Exception:
log.error("CAUGHT EXCEPTION")
traceback.print_exc()
log.info("Stopping reactor")
yield reactor.stop()
def getApproximateTotalDHTNodes(node):
from lbrynet.dht import constants
# get the deepest bucket and the number of contacts in that bucket and multiply it
# by the number of equivalently deep buckets in the whole DHT to get a really bad
# estimate!
bucket = node._routingTable._buckets[node._routingTable._kbucketIndex(node.node_id)]
num_in_bucket = len(bucket._contacts)
factor = (2 ** constants.key_bits) / (bucket.rangeMax - bucket.rangeMin)
return num_in_bucket * factor
def getApproximateTotalHashes(node):
# Divide the number of hashes we know about by k to get a really, really, really
# bad estimate of the average number of hashes per node, then multiply by the
# approximate number of nodes to get a horrendous estimate of the total number
# of hashes in the DHT
num_in_data_store = len(node._dataStore._dict)
if num_in_data_store == 0:
return 0
return num_in_data_store * getApproximateTotalDHTNodes(node) / 8
@defer.inlineCallbacks
def find(node):
try:
log.info("Approximate number of nodes in DHT: %s", str(getApproximateTotalDHTNodes(node)))
log.info("Approximate number of blobs in DHT: %s", str(getApproximateTotalHashes(node)))
h = "578f5e82da7db97bfe0677826d452cc0c65406a8e986c9caa126af4ecdbf4913daad2f7f5d1fb0ffec17d0bf8f187f5a"
peersFake = yield node.getPeersForBlob(h.decode("hex"))
print peersFake
peers = yield node.getPeersForBlob(h.decode("hex"))
print peers
# yield get_hosts(node, h)
except Exception:
log.error("CAUGHT EXCEPTION")
traceback.print_exc()
log.info("Stopping reactor")
yield reactor.stop()
def main():
log_support.configure_console(level='DEBUG')
log_support.configure_twisted()
reactor.callLater(0, connect, port=10001)
log.info("Running reactor")
reactor.run()
if __name__ == '__main__':
sys.exit(main())

163
scripts/dhttest.py
View file

@ -1,163 +0,0 @@
#!/usr/bin/env python
#
# This is a basic single-node example of how to use the Entangled
# DHT. It creates a Node and (optionally) joins an existing DHT. It
# then does a Kademlia store and find, and then it deletes the stored
# value (non-Kademlia method).
#
# No tuple space functionality is demonstrated by this script.
#
# To test it properly, start a multi-node Kademlia DHT with the "create_network.py"
# script and point this node to that, e.g.:
# $python create_network.py 10 127.0.0.1
#
# $python basic_example.py 5000 127.0.0.1 4000
#
# This library is free software, distributed under the terms of
# the GNU Lesser General Public License Version 3, or any later version.
# See the COPYING file included in this archive
#
# Thanks to Paul Cannon for IP-address resolution functions (taken from aspn.activestate.com)
import binascii
import random
import twisted.internet.reactor
from lbrynet.dht.node import Node
from lbrynet.core.cryptoutils import get_lbry_hash_obj
# The Entangled DHT node; instantiated in the main() method
node = None
# The key to use for this example when storing/retrieving data
h = get_lbry_hash_obj()
h.update("key")
KEY = h.digest()
# The value to store
VALUE = random.randint(10000, 20000)
lbryid = KEY
def storeValue(key, value):
""" Stores the specified value in the DHT using the specified key """
global node
print '\nStoring value; Key: %s, Value: %s' % (key, value)
# Store the value in the DHT. This method returns a Twisted
# Deferred result, which we then add callbacks to
deferredResult = node.announceHaveHash(key, value)
# Add our callback; this method is called when the operation completes...
deferredResult.addCallback(storeValueCallback)
# ...and for error handling, add an "error callback" as well.
#
# For this example script, I use a generic error handler; usually
# you would need something more specific
deferredResult.addErrback(genericErrorCallback)
def storeValueCallback(*args, **kwargs):
""" Callback function that is invoked when the storeValue() operation succeeds """
print 'Value has been stored in the DHT'
# Now that the value has been stored, schedule that the value is read again after 2.5 seconds
print 'Scheduling retrieval in 2.5 seconds'
twisted.internet.reactor.callLater(2.5, getValue)
def genericErrorCallback(failure):
""" Callback function that is invoked if an error occurs during any of the DHT operations """
print 'An error has occurred:', failure.getErrorMessage()
twisted.internet.reactor.callLater(0, stop)
def getValue():
""" Retrieves the value of the specified key (KEY) from the DHT """
global node, KEY
# Get the value for the specified key (immediately returns a Twisted deferred result)
print ('\nRetrieving value from DHT for key "%s"' %
binascii.unhexlify("f7d9dc4de674eaa2c5a022eb95bc0d33ec2e75c6"))
deferredResult = node.iterativeFindValue(
binascii.unhexlify("f7d9dc4de674eaa2c5a022eb95bc0d33ec2e75c6"))
# Add a callback to this result; this will be called as soon as the operation has completed
deferredResult.addCallback(getValueCallback)
# As before, add the generic error callback
deferredResult.addErrback(genericErrorCallback)
def getValueCallback(result):
""" Callback function that is invoked when the getValue() operation succeeds """
# Check if the key was found (result is a dict of format {key:
# value}) or not (in which case a list of "closest" Kademlia
# contacts would be returned instead")
print "Got the value"
print result
# Either way, schedule a "delete" operation for the key
print 'Scheduling shutdown in 2.5 seconds'
twisted.internet.reactor.callLater(2.5, stop)
def stop():
""" Stops the Twisted reactor, and thus the script """
print '\nStopping Kademlia node and terminating script'
twisted.internet.reactor.stop()
if __name__ == '__main__':
import sys
if len(sys.argv) < 2:
print 'Usage:\n%s UDP_PORT [KNOWN_NODE_IP KNOWN_NODE_PORT]' % sys.argv[0]
print 'or:\n%s UDP_PORT [FILE_WITH_KNOWN_NODES]' % sys.argv[0]
print
print 'If a file is specified, it should containg one IP address and UDP port'
print 'per line, seperated by a space.'
sys.exit(1)
try:
int(sys.argv[1])
except ValueError:
print '\nUDP_PORT must be an integer value.\n'
print 'Usage:\n%s UDP_PORT [KNOWN_NODE_IP KNOWN_NODE_PORT]' % sys.argv[0]
print 'or:\n%s UDP_PORT [FILE_WITH_KNOWN_NODES]' % sys.argv[0]
print
print 'If a file is specified, it should contain one IP address and UDP port'
print 'per line, seperated by a space.'
sys.exit(1)
if len(sys.argv) == 4:
knownNodes = [(sys.argv[2], int(sys.argv[3]))]
elif len(sys.argv) == 3:
knownNodes = []
f = open(sys.argv[2], 'r')
lines = f.readlines()
f.close()
for line in lines:
ipAddress, udpPort = line.split()
knownNodes.append((ipAddress, int(udpPort)))
else:
knownNodes = None
print '\nNOTE: You have not specified any remote DHT node(s) to connect to'
print 'It will thus not be aware of any existing DHT, but will still function as'
print ' a self-contained DHT (until another node contacts it).'
print 'Run this script without any arguments for info.\n'
# Set up SQLite-based data store (you could use an in-memory store instead, for example)
#
# Create the Entangled node. It extends the functionality of a
# basic Kademlia node (but is fully backwards-compatible with a
# Kademlia-only network)
#
# If you wish to have a pure Kademlia network, use the
# entangled.kademlia.node.Node class instead
print 'Creating Node'
node = Node(udpPort=int(sys.argv[1]), node_id=lbryid)
# Schedule the node to join the Kademlia/Entangled DHT
node.joinNetwork(knownNodes)
# Schedule the "storeValue() call to be invoked after 2.5 seconds,
# using KEY and VALUE as arguments
twisted.internet.reactor.callLater(2.5, getValue)
# Start the Twisted reactor - this fires up all networking, and
# allows the scheduled join operation to take place
print 'Twisted reactor started (script will commence in 2.5 seconds)'
twisted.internet.reactor.run()

41
scripts/node_rpc_cli.py
View file

@ -1,41 +0,0 @@
"""
CLI for sending rpc commands to a DHT node
"""
import argparse
from twisted.internet import reactor
from txjsonrpc.web.jsonrpc import Proxy
def print_value(value):
print value
def print_error(err):
print err.getErrorMessage()
def shut_down():
reactor.stop()
def main():
parser = argparse.ArgumentParser(description="Send an rpc command to a dht node")
parser.add_argument("rpc_command",
help="The rpc command to send to the dht node")
parser.add_argument("--node_host",
help="The host of the node to connect to",
default="127.0.0.1")
parser.add_argument("--node_port",
help="The port of the node to connect to",
default="8888")
args = parser.parse_args()
connect_string = 'http://%s:%s' % (args.node_host, args.node_port)
proxy = Proxy(connect_string)
d = proxy.callRemote(args.rpc_command)
d.addCallbacks(print_value, print_error)
d.addBoth(lambda _: shut_down())
reactor.run()

214
scripts/rpc_node.py
View file

@ -1,214 +0,0 @@
import logging
import requests
import miniupnpc
import argparse
from copy import deepcopy
from twisted.internet import reactor, defer
from twisted.web import resource
from twisted.web.server import Site
from lbrynet import conf
from lbrynet.core.log_support import configure_console
from lbrynet.dht.error import TimeoutError
conf.initialize_settings()
log = logging.getLogger("dht tool")
configure_console()
log.setLevel(logging.INFO)
from lbrynet.dht.node import Node
from lbrynet.dht.contact import Contact
from lbrynet.daemon.auth.server import AuthJSONRPCServer
from lbrynet.core.utils import generate_id
def get_external_ip_and_setup_upnp():
try:
u = miniupnpc.UPnP()
u.discoverdelay = 200
u.discover()
u.selectigd()
if u.getspecificportmapping(4444, "UDP"):
u.deleteportmapping(4444, "UDP")
log.info("Removed UPnP redirect for UDP 4444.")
u.addportmapping(4444, 'UDP', u.lanaddr, 4444, 'LBRY DHT port', '')
log.info("got external ip from upnp")
return u.externalipaddress()
except Exception:
log.exception("derp")
r = requests.get('https://api.ipify.org', {'format': 'json'})
log.info("got external ip from ipify.org")
return r.json()['ip']
class NodeRPC(AuthJSONRPCServer):
def __init__(self, lbryid, seeds, node_port, rpc_port):
AuthJSONRPCServer.__init__(self, False)
self.root = None
self.port = None
self.seeds = seeds
self.node_port = node_port
self.rpc_port = rpc_port
if lbryid:
lbryid = lbryid.decode('hex')
else:
lbryid = generate_id()
self.node_id = lbryid
self.external_ip = get_external_ip_and_setup_upnp()
self.node_port = node_port
@defer.inlineCallbacks
def setup(self):
self.node = Node(node_id=self.node_id, udpPort=self.node_port,
externalIP=self.external_ip)
hosts = []
for hostname, hostport in self.seeds:
host_ip = yield reactor.resolve(hostname)
hosts.append((host_ip, hostport))
log.info("connecting to dht")
yield self.node.joinNetwork(tuple(hosts))
log.info("connected to dht")
if not self.announced_startup:
self.announced_startup = True
self.start_api()
log.info("lbry id: %s (%i bytes)", self.node.node_id.encode('hex'), len(self.node.node_id))
def start_api(self):
root = resource.Resource()
root.putChild('', self)
self.port = reactor.listenTCP(self.rpc_port, Site(root), interface='localhost')
log.info("started jsonrpc server")
@defer.inlineCallbacks
def jsonrpc_node_id_set(self, node_id):
old_id = self.node.node_id
self.node.stop()
del self.node
self.node_id = node_id.decode('hex')
yield self.setup()
msg = "changed dht id from %s to %s" % (old_id.encode('hex'),
self.node.node_id.encode('hex'))
defer.returnValue(msg)
def jsonrpc_node_id_get(self):
return self._render_response(self.node.node_id.encode('hex'))
@defer.inlineCallbacks
def jsonrpc_peer_find(self, node_id):
node_id = node_id.decode('hex')
contact = yield self.node.findContact(node_id)
result = None
if contact:
result = (contact.address, contact.port)
defer.returnValue(result)
@defer.inlineCallbacks
def jsonrpc_peer_list_for_blob(self, blob_hash):
peers = yield self.node.getPeersForBlob(blob_hash.decode('hex'))
defer.returnValue(peers)
@defer.inlineCallbacks
def jsonrpc_ping(self, node_id):
contact_host = yield self.jsonrpc_peer_find(node_id=node_id)
if not contact_host:
defer.returnValue("failed to find node")
contact_ip, contact_port = contact_host
contact = Contact(node_id.decode('hex'), contact_ip, contact_port, self.node._protocol)
try:
result = yield contact.ping()
except TimeoutError:
self.node.removeContact(contact.id)
self.node._dataStore.removePeer(contact.id)
result = {'error': 'timeout'}
defer.returnValue(result)
def get_routing_table(self):
result = {}
data_store = deepcopy(self.node._dataStore._dict)
datastore_len = len(data_store)
hosts = {}
missing_contacts = []
if datastore_len:
for k, v in data_store.iteritems():
for value, lastPublished, originallyPublished, originalPublisherID in v:
try:
contact = self.node._routingTable.getContact(originalPublisherID)
except ValueError:
if originalPublisherID.encode('hex') not in missing_contacts:
missing_contacts.append(originalPublisherID.encode('hex'))
continue
if contact in hosts:
blobs = hosts[contact]
else:
blobs = []
blobs.append(k.encode('hex'))
hosts[contact] = blobs
contact_set = []
blob_hashes = []
result['buckets'] = {}
for i in range(len(self.node._routingTable._buckets)):
for contact in self.node._routingTable._buckets[i]._contacts:
contacts = result['buckets'].get(i, [])
if contact in hosts:
blobs = hosts[contact]
del hosts[contact]
else:
blobs = []
host = {
"address": contact.address,
"id": contact.id.encode("hex"),
"blobs": blobs,
}
for blob_hash in blobs:
if blob_hash not in blob_hashes:
blob_hashes.append(blob_hash)
contacts.append(host)
result['buckets'][i] = contacts
contact_set.append(contact.id.encode("hex"))
if hosts:
result['datastore extra'] = [
{
"id": host.id.encode('hex'),
"blobs": hosts[host],
}
for host in hosts]
result['missing contacts'] = missing_contacts
result['contacts'] = contact_set
result['blob hashes'] = blob_hashes
result['node id'] = self.node_id.encode('hex')
return result
def jsonrpc_routing_table_get(self):
return self._render_response(self.get_routing_table())
def main():
parser = argparse.ArgumentParser(description="Launch a dht node which responds to rpc commands")
parser.add_argument("--node_port",
help=("The UDP port on which the node will listen for connections "
"from other dht nodes"),
type=int, default=4444)
parser.add_argument("--rpc_port",
help="The TCP port on which the node will listen for rpc commands",
type=int, default=5280)
parser.add_argument("--bootstrap_host",
help="The IP of a DHT node to be used to bootstrap into the network",
default='lbrynet1.lbry.io')
parser.add_argument("--node_id",
help="The IP of a DHT node to be used to bootstrap into the network",
default=None)
parser.add_argument("--bootstrap_port",
help="The port of a DHT node to be used to bootstrap into the network",
default=4444, type=int)
args = parser.parse_args()
seeds = [(args.bootstrap_host, args.bootstrap_port)]
server = NodeRPC(args.node_id, seeds, args.node_port, args.rpc_port)
reactor.addSystemEventTrigger('after', 'startup', server.setup)
reactor.run()
if __name__ == "__main__":
main()

47
scripts/simple_dht_node.py
View file

@ -1,47 +0,0 @@
#!/usr/bin/env python
from lbrynet.core import log_support
import logging.handlers
import sys
import time
from pprint import pprint
from twisted.internet import defer, reactor
from lbrynet.dht.node import Node
import lbrynet.dht.constants
import lbrynet.dht.datastore
from lbrynet.tests.util import random_lbry_hash
log = logging.getLogger(__name__)
@defer.inlineCallbacks
def run():
nodeid = "9648996b4bef3ff41176668a0577f86aba7f1ea2996edd18f9c42430802c8085331345c5f0c44a7f352e2ba8ae59aaaa".decode("hex")
node = Node(node_id=nodeid, externalIP='127.0.0.1', udpPort=21999, peerPort=1234)
node.startNetwork()
yield node.joinNetwork([("127.0.0.1", 21001)])
print ""
print ""
print ""
print ""
print ""
print ""
yield node.announceHaveBlob("2bb150cb996b4bef3ff41176648a0577f86abb7f1ea2996edd18f9c42430802c8085331345c5f0c44a7f352e2ba8ae59".decode("hex"))
log.info("Shutting down...")
reactor.callLater(1, reactor.stop)
def main():
log_support.configure_console(level='DEBUG')
log_support.configure_twisted()
reactor.callLater(0, run)
log.info("Running reactor")
reactor.run()
if __name__ == '__main__':
sys.exit(main())