From b6f9b606838c31ae09c173dee46950abff0f3b99 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Mon, 29 Oct 2018 13:33:24 -0400 Subject: [PATCH 01/16] remove commit in version for source installs --- lbrynet/p2p/system_info.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/lbrynet/p2p/system_info.py b/lbrynet/p2p/system_info.py index f8d0f42de..1c1e3c83a 100644 --- a/lbrynet/p2p/system_info.py +++ b/lbrynet/p2p/system_info.py @@ -1,29 +1,17 @@ import platform import json -import subprocess import os from six.moves.urllib import request from six.moves.urllib.error import URLError from lbrynet.schema import __version__ as schema_version from lbrynet import build_type, __version__ as lbrynet_version -from lbrynet.extras.daemon.conf import ROOT_DIR import logging.handlers log = logging.getLogger(__name__) def get_lbrynet_version() -> str: - if build_type.BUILD == "dev": - try: - with open(os.devnull, 'w') as devnull: - git_dir = ROOT_DIR + '/.git' - return subprocess.check_output( - ['git', '--git-dir='+git_dir, 'describe', '--dirty', '--always'], - stderr=devnull - ).decode().strip().lstrip('v') - except (subprocess.CalledProcessError, OSError): - log.debug("failed to get version from git") return lbrynet_version From 2e921437d87fb5b004e606172ce21472de0e98a8 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Mon, 29 Oct 2018 13:41:14 -0400 Subject: [PATCH 02/16] fix external ip setup fixes https://github.com/lbryio/lbry/issues/1561 --- lbrynet/extras/daemon/Components.py | 47 ++++++++++++++++++----------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/lbrynet/extras/daemon/Components.py b/lbrynet/extras/daemon/Components.py index eada5c554..6ddc42e4a 100644 --- a/lbrynet/extras/daemon/Components.py +++ b/lbrynet/extras/daemon/Components.py @@ -424,12 +424,18 @@ class DHTComponent(Component): node_id = CS.get_node_id() if node_id is None: node_id = generate_id() + external_ip = self.upnp_component.external_ip + if not external_ip: + log.warning("UPnP component failed to get external ip") + external_ip = CS.get_external_ip() + if not external_ip: + log.warning("failed to get external ip") self.dht_node = node.Node( node_id=node_id, udpPort=GCS('dht_node_port'), externalUDPPort=self.external_udp_port, - externalIP=self.upnp_component.external_ip, + externalIP=external_ip, peerPort=self.external_peer_port ) @@ -710,33 +716,39 @@ class UPnPComponent(Component): log.info("found upnp gateway: %s", self.upnp.gateway.manufacturer_string) except Exception as err: log.warning("upnp discovery failed: %s", err) - return + self.upnp = None # update the external ip - try: - external_ip = yield from_future(self.upnp.get_external_ip()) - if external_ip == "0.0.0.0": - log.warning("upnp doesn't know the external ip address (returned 0.0.0.0), using fallback") - external_ip = CS.get_external_ip() - if self.external_ip and self.external_ip != external_ip: - log.info("external ip changed from %s to %s", self.external_ip, external_ip) - elif not self.external_ip: - log.info("got external ip: %s", external_ip) - self.external_ip = external_ip - except (asyncio.TimeoutError, UPnPError): - pass - - if not self.upnp_redirects: # setup missing redirects + external_ip = None + if self.upnp: try: + external_ip = yield from_future(self.upnp.get_external_ip()) + if external_ip != "0.0.0.0": + log.info("got external ip from UPnP: %s", external_ip) + except (asyncio.TimeoutError, UPnPError): + pass + + if external_ip == "0.0.0.0" or not external_ip: + log.warning("unable to get external ip from UPnP, checking lbry.io fallback") + external_ip = CS.get_external_ip() + if self.external_ip and self.external_ip != external_ip: + log.info("external ip changed from %s to %s", self.external_ip, external_ip) + self.external_ip = external_ip + assert self.external_ip is not None # TODO: handle going/starting offline + + if not self.upnp_redirects and self.upnp: # setup missing redirects + try: + log.info("add UPnP port mappings") upnp_redirects = yield DeferredDict({ "UDP": from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")), "TCP": from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")) }) + log.info("set up redirects: %s", upnp_redirects) self.upnp_redirects.update(upnp_redirects) except (asyncio.TimeoutError, UPnPError): self.upnp = None return self._maintain_redirects() - else: # check existing redirects are still active + elif self.upnp: # check existing redirects are still active found = set() mappings = yield from_future(self.upnp.get_redirects()) for mapping in mappings: @@ -767,6 +779,7 @@ class UPnPComponent(Component): @defer.inlineCallbacks def start(self): + log.info("detecting external ip") if not self.use_upnp: self.external_ip = CS.get_external_ip() return From 0bf8416d9f82ee81b36ed446e2aa00ea23c6afd0 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 30 Oct 2018 13:38:12 -0400 Subject: [PATCH 03/16] fix components_to_skip setting --- lbrynet/extras/daemon/Daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lbrynet/extras/daemon/Daemon.py b/lbrynet/extras/daemon/Daemon.py index ef22fd3b6..735166ebf 100644 --- a/lbrynet/extras/daemon/Daemon.py +++ b/lbrynet/extras/daemon/Daemon.py @@ -220,7 +220,7 @@ class Daemon(AuthJSONRPCServer): } def __init__(self, analytics_manager=None, component_manager=None): - to_skip = list(conf.settings['components_to_skip']) + to_skip = conf.settings['components_to_skip'] if 'reflector' not in to_skip and not conf.settings['run_reflector_server']: to_skip.append('reflector') looping_calls = { From 253912b52e38e0a2c1640274ea7d9837fcd5e0d5 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Tue, 30 Oct 2018 13:41:38 -0400 Subject: [PATCH 04/16] fix skipping dht, peer_protocol_server, and hash_announcer components -move PeerFinder and PeerManager initialization to the ComponentManager -remove dht component requirement from all but the hash_announcer component. This allows running the file manager component without either of the server components. --- lbrynet/dht/node.py | 14 +-- lbrynet/dht/peerfinder.py | 36 +++--- lbrynet/extras/daemon/ComponentManager.py | 13 +- lbrynet/extras/daemon/Components.py | 119 ++++++++---------- lbrynet/extras/daemon/Daemon.py | 18 ++- tests/integration/cli/test_cli.py | 8 +- .../unit/components/test_Component_Manager.py | 19 ++- tests/unit/lbrynet_daemon/test_Daemon.py | 4 +- 8 files changed, 104 insertions(+), 127 deletions(-) diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index 287cae8fb..64c1a64ab 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -13,7 +13,6 @@ from . import constants from . import routingtable from . import datastore from . import protocol -from .peerfinder import DHTPeerFinder from .contact import ContactManager from .iterativefind import iterativeFind @@ -83,8 +82,8 @@ class Node(MockKademliaHelper): def __init__(self, node_id=None, udpPort=4000, dataStore=None, routingTableClass=None, networkProtocol=None, externalIP=None, peerPort=3333, listenUDP=None, - callLater=None, resolve=None, clock=None, peer_finder=None, - peer_manager=None, interface='', externalUDPPort=None): + callLater=None, resolve=None, clock=None, + interface='', externalUDPPort=None): """ @param dataStore: The data store to use. This must be class inheriting from the C{DataStore} interface (or providing the @@ -124,20 +123,13 @@ class Node(MockKademliaHelper): else: self._routingTable = routingTableClass(self.node_id, self.clock.seconds) - # Initialize this node's network access mechanisms - if networkProtocol is None: - self._protocol = protocol.KademliaProtocol(self) - else: - self._protocol = networkProtocol - # Initialize the data storage mechanism used by this node + self._protocol = networkProtocol or protocol.KademliaProtocol(self) self.token_secret = self._generateID() self.old_token_secret = None self.externalIP = externalIP self.peerPort = peerPort self.externalUDPPort = externalUDPPort or self.port self._dataStore = dataStore or datastore.DictDataStore(self.clock.seconds) - self.peer_manager = peer_manager or PeerManager() - self.peer_finder = peer_finder or DHTPeerFinder(self, self.peer_manager) self._join_deferred = None #def __del__(self): diff --git a/lbrynet/dht/peerfinder.py b/lbrynet/dht/peerfinder.py index 7d5eea159..edabb9ee7 100644 --- a/lbrynet/dht/peerfinder.py +++ b/lbrynet/dht/peerfinder.py @@ -18,13 +18,12 @@ class DHTPeerFinder(DummyPeerFinder): """This class finds peers which have announced to the DHT that they have certain blobs""" #implements(IPeerFinder) - def __init__(self, dht_node, peer_manager): + def __init__(self, component_manager): """ - dht_node - an instance of dht.Node class - peer_manager - an instance of PeerManager class + component_manager - an instance of ComponentManager """ - self.dht_node = dht_node - self.peer_manager = peer_manager + self.component_manager = component_manager + self.peer_manager = component_manager.peer_manager self.peers = {} self._ongoing_searchs = {} @@ -39,19 +38,30 @@ class DHTPeerFinder(DummyPeerFinder): Returns: list of peers for the blob """ - self.peers.setdefault(blob_hash, {(self.dht_node.externalIP, self.dht_node.peerPort,)}) + if "dht" in self.component_manager.skip_components: + return defer.succeed([]) + if not self.component_manager.all_components_running("dht"): + return defer.succeed([]) + dht_node = self.component_manager.get_component("dht") + + self.peers.setdefault(blob_hash, {(dht_node.externalIP, dht_node.peerPort,)}) if not blob_hash in self._ongoing_searchs or self._ongoing_searchs[blob_hash].called: - self._ongoing_searchs[blob_hash] = self._execute_peer_search(blob_hash, timeout) - peers = set(self._filter_self(blob_hash) if filter_self else self.peers[blob_hash]) + self._ongoing_searchs[blob_hash] = self._execute_peer_search(dht_node, blob_hash, timeout) + + def _filter_self(blob_hash): + my_host, my_port = dht_node.externalIP, dht_node.peerPort + return {(host, port) for host, port in self.peers[blob_hash] if (host, port) != (my_host, my_port)} + + peers = set(_filter_self(blob_hash) if filter_self else self.peers[blob_hash]) return defer.succeed([self.peer_manager.get_peer(*peer) for peer in peers]) @defer.inlineCallbacks - def _execute_peer_search(self, blob_hash, timeout): + def _execute_peer_search(self, dht_node, blob_hash, timeout): bin_hash = binascii.unhexlify(blob_hash) - finished_deferred = self.dht_node.iterativeFindValue(bin_hash, exclude=self.peers[blob_hash]) + finished_deferred = dht_node.iterativeFindValue(bin_hash, exclude=self.peers[blob_hash]) timeout = timeout or conf.settings['peer_search_timeout'] if timeout: - finished_deferred.addTimeout(timeout, self.dht_node.clock) + finished_deferred.addTimeout(timeout, dht_node.clock) try: peer_list = yield finished_deferred self.peers[blob_hash].update({(host, port) for _, host, port in peer_list}) @@ -59,7 +69,3 @@ class DHTPeerFinder(DummyPeerFinder): log.debug("DHT timed out while looking peers for blob %s after %s seconds", blob_hash, timeout) finally: del self._ongoing_searchs[blob_hash] - - def _filter_self(self, blob_hash): - my_host, my_port = self.dht_node.externalIP, self.dht_node.peerPort - return {(host, port) for host, port in self.peers[blob_hash] if (host, port) != (my_host, my_port)} diff --git a/lbrynet/extras/daemon/ComponentManager.py b/lbrynet/extras/daemon/ComponentManager.py index 087a446ee..59f1304aa 100644 --- a/lbrynet/extras/daemon/ComponentManager.py +++ b/lbrynet/extras/daemon/ComponentManager.py @@ -1,7 +1,8 @@ import logging from twisted.internet import defer - from lbrynet.p2p.Error import ComponentStartConditionNotMet +from lbrynet.core.PeerManager import PeerManager +from lbrynet.dht.peerfinder import DHTPeerFinder log = logging.getLogger(__name__) @@ -33,13 +34,15 @@ class RequiredCondition(metaclass=RequiredConditionType): class ComponentManager: default_component_classes = {} - def __init__(self, reactor=None, analytics_manager=None, skip_components=None, **override_components): + def __init__(self, reactor=None, analytics_manager=None, skip_components=None, + peer_manager=None, peer_finder=None, **override_components): self.skip_components = skip_components or [] - self.reactor = reactor self.component_classes = {} self.components = set() self.analytics_manager = analytics_manager + self.peer_manager = peer_manager or PeerManager() + self.peer_finder = peer_finder or DHTPeerFinder(self) for component_name, component_class in self.default_component_classes.items(): if component_name in override_components: @@ -114,10 +117,10 @@ class ComponentManager: :return: (defer.Deferred) """ - for component_name, cb in callbacks.items(): if component_name not in self.component_classes: - raise NameError("unknown component: %s" % component_name) + if component_name not in self.skip_components: + raise NameError("unknown component: %s" % component_name) if not callable(cb): raise ValueError("%s is not callable" % cb) diff --git a/lbrynet/extras/daemon/Components.py b/lbrynet/extras/daemon/Components.py index 6ddc42e4a..b1c936429 100644 --- a/lbrynet/extras/daemon/Components.py +++ b/lbrynet/extras/daemon/Components.py @@ -42,7 +42,6 @@ HEADERS_COMPONENT = "blockchain_headers" WALLET_COMPONENT = "wallet" DHT_COMPONENT = "dht" HASH_ANNOUNCER_COMPONENT = "hash_announcer" -STREAM_IDENTIFIER_COMPONENT = "stream_identifier" FILE_MANAGER_COMPONENT = "file_manager" PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server" REFLECTOR_COMPONENT = "reflector" @@ -366,7 +365,7 @@ class WalletComponent(Component): class BlobComponent(Component): component_name = BLOB_COMPONENT - depends_on = [DATABASE_COMPONENT, DHT_COMPONENT] + depends_on = [DATABASE_COMPONENT] def __init__(self, component_manager): super().__init__(component_manager) @@ -378,8 +377,12 @@ class BlobComponent(Component): def start(self): storage = self.component_manager.get_component(DATABASE_COMPONENT) - dht_node = self.component_manager.get_component(DHT_COMPONENT) - self.blob_manager = DiskBlobManager(CS.get_blobfiles_dir(), storage, dht_node._dataStore) + datastore = None + if DHT_COMPONENT not in self.component_manager.skip_components: + dht_node = self.component_manager.get_component(DHT_COMPONENT) + if dht_node: + datastore = dht_node._dataStore + self.blob_manager = DiskBlobManager(CS.get_blobfiles_dir(), storage, datastore) return self.blob_manager.setup() def stop(self): @@ -439,11 +442,7 @@ class DHTComponent(Component): peerPort=self.external_peer_port ) - self.dht_node.start_listening() - yield self.dht_node._protocol._listening - d = self.dht_node.joinNetwork(GCS('known_dht_nodes')) - d.addCallback(lambda _: self.dht_node.start_looping_calls()) - d.addCallback(lambda _: log.info("Joined the dht")) + yield self.dht_node.start(GCS('known_dht_nodes')) log.info("Started the dht") @defer.inlineCallbacks @@ -500,41 +499,6 @@ class RateLimiterComponent(Component): return defer.succeed(None) -class StreamIdentifierComponent(Component): - component_name = STREAM_IDENTIFIER_COMPONENT - depends_on = [DHT_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT] - - def __init__(self, component_manager): - super().__init__(component_manager) - self.sd_identifier = StreamDescriptorIdentifier() - - @property - def component(self): - return self.sd_identifier - - @defer.inlineCallbacks - def start(self): - dht_node = self.component_manager.get_component(DHT_COMPONENT) - rate_limiter = self.component_manager.get_component(RATE_LIMITER_COMPONENT) - blob_manager = self.component_manager.get_component(BLOB_COMPONENT) - storage = self.component_manager.get_component(DATABASE_COMPONENT) - wallet = self.component_manager.get_component(WALLET_COMPONENT) - - add_lbry_file_to_sd_identifier(self.sd_identifier) - file_saver_factory = EncryptedFileSaverFactory( - dht_node.peer_finder, - rate_limiter, - blob_manager, - storage, - wallet, - GCS('download_directory') - ) - yield self.sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory) - - def stop(self): - pass - - class PaymentRateComponent(Component): component_name = PAYMENT_RATE_COMPONENT @@ -555,8 +519,8 @@ class PaymentRateComponent(Component): class FileManagerComponent(Component): component_name = FILE_MANAGER_COMPONENT - depends_on = [DHT_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT, - STREAM_IDENTIFIER_COMPONENT, PAYMENT_RATE_COMPONENT] + depends_on = [RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT, + PAYMENT_RATE_COMPONENT] def __init__(self, component_manager): super().__init__(component_manager) @@ -575,15 +539,26 @@ class FileManagerComponent(Component): @defer.inlineCallbacks def start(self): - dht_node = self.component_manager.get_component(DHT_COMPONENT) rate_limiter = self.component_manager.get_component(RATE_LIMITER_COMPONENT) blob_manager = self.component_manager.get_component(BLOB_COMPONENT) storage = self.component_manager.get_component(DATABASE_COMPONENT) wallet = self.component_manager.get_component(WALLET_COMPONENT) - sd_identifier = self.component_manager.get_component(STREAM_IDENTIFIER_COMPONENT) + + sd_identifier = StreamDescriptorIdentifier() + add_lbry_file_to_sd_identifier(sd_identifier) + file_saver_factory = EncryptedFileSaverFactory( + self.component_manager.peer_finder, + rate_limiter, + blob_manager, + storage, + wallet, + GCS('download_directory') + ) + yield sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory) + payment_rate_manager = self.component_manager.get_component(PAYMENT_RATE_COMPONENT) log.info('Starting the file manager') - self.file_manager = EncryptedFileManager(dht_node.peer_finder, rate_limiter, blob_manager, wallet, + self.file_manager = EncryptedFileManager(self.component_manager.peer_finder, rate_limiter, blob_manager, wallet, payment_rate_manager, storage, sd_identifier) yield self.file_manager.setup() log.info('Done setting up file manager') @@ -595,7 +570,7 @@ class FileManagerComponent(Component): class PeerProtocolServerComponent(Component): component_name = PEER_PROTOCOL_SERVER_COMPONENT - depends_on = [UPNP_COMPONENT, DHT_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, + depends_on = [UPNP_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, PAYMENT_RATE_COMPONENT] def __init__(self, component_manager): @@ -624,7 +599,7 @@ class PeerProtocolServerComponent(Component): } server_factory = ServerProtocolFactory( self.component_manager.get_component(RATE_LIMITER_COMPONENT), query_handlers, - self.component_manager.get_component(DHT_COMPONENT).peer_manager + self.component_manager.peer_manager ) try: @@ -648,7 +623,7 @@ class PeerProtocolServerComponent(Component): class ReflectorComponent(Component): component_name = REFLECTOR_COMPONENT - depends_on = [DHT_COMPONENT, BLOB_COMPONENT, FILE_MANAGER_COMPONENT] + depends_on = [BLOB_COMPONENT, FILE_MANAGER_COMPONENT] def __init__(self, component_manager): super().__init__(component_manager) @@ -662,10 +637,9 @@ class ReflectorComponent(Component): @defer.inlineCallbacks def start(self): log.info("Starting reflector server") - dht_node = self.component_manager.get_component(DHT_COMPONENT) blob_manager = self.component_manager.get_component(BLOB_COMPONENT) file_manager = self.component_manager.get_component(FILE_MANAGER_COMPONENT) - reflector_factory = reflector_server_factory(dht_node.peer_manager, blob_manager, file_manager) + reflector_factory = reflector_server_factory(self.component_manager.peer_manager, blob_manager, file_manager) try: self.reflector_server = yield reactor.listenTCP(self.reflector_server_port, reflector_factory) log.info('Started reflector on port %s', self.reflector_server_port) @@ -701,10 +675,12 @@ class UPnPComponent(Component): @defer.inlineCallbacks def _setup_redirects(self): - upnp_redirects = yield DeferredDict({ - "UDP": from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")), - "TCP": from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")) - }) + d = {} + if PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components: + d["TCP"] = from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")) + if DHT_COMPONENT not in self.component_manager.skip_components: + d["UDP"] = from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")) + upnp_redirects = yield DeferredDict(d) self.upnp_redirects.update(upnp_redirects) @defer.inlineCallbacks @@ -739,10 +715,12 @@ class UPnPComponent(Component): if not self.upnp_redirects and self.upnp: # setup missing redirects try: log.info("add UPnP port mappings") - upnp_redirects = yield DeferredDict({ - "UDP": from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")), - "TCP": from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")) - }) + d = {} + if PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components: + d["TCP"] = from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")) + if DHT_COMPONENT not in self.component_manager.skip_components: + d["UDP"] = from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")) + upnp_redirects = yield DeferredDict(d) log.info("set up redirects: %s", upnp_redirects) self.upnp_redirects.update(upnp_redirects) except (asyncio.TimeoutError, UPnPError): @@ -756,7 +734,7 @@ class UPnPComponent(Component): if proto in self.upnp_redirects and mapping['NewExternalPort'] == self.upnp_redirects[proto]: if mapping['NewInternalClient'] == self.upnp.lan_address: found.add(proto) - if 'UDP' not in found: + if 'UDP' not in found and DHT_COMPONENT not in self.component_manager.skip_components: try: udp_port = yield from_future( self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port") @@ -765,7 +743,7 @@ class UPnPComponent(Component): log.info("refreshed upnp redirect for dht port: %i", udp_port) except (asyncio.TimeoutError, UPnPError): del self.upnp_redirects['UDP'] - if 'TCP' not in found: + if 'TCP' not in found and PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components: try: tcp_port = yield from_future( self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port") @@ -774,8 +752,11 @@ class UPnPComponent(Component): log.info("refreshed upnp redirect for peer port: %i", tcp_port) except (asyncio.TimeoutError, UPnPError): del self.upnp_redirects['TCP'] - if 'TCP' in self.upnp_redirects and 'UDP' in self.upnp_redirects: - log.debug("upnp redirects are still active") + if ('TCP' in self.upnp_redirects + and PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components) and ( + 'UDP' in self.upnp_redirects and DHT_COMPONENT not in self.component_manager.skip_components): + if self.upnp_redirects: + log.debug("upnp redirects are still active") @defer.inlineCallbacks def start(self): @@ -786,11 +767,13 @@ class UPnPComponent(Component): success = False yield self._maintain_redirects() if self.upnp: - if not self.upnp_redirects: + if not self.upnp_redirects and not all([x in self.component_manager.skip_components for x in + (DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT)]): log.error("failed to setup upnp, debugging infomation: %s", self.upnp.zipped_debugging_info) else: success = True - log.debug("set up upnp port redirects for gateway: %s", self.upnp.gateway.manufacturer_string) + if self.upnp_redirects: + log.debug("set up upnp port redirects for gateway: %s", self.upnp.gateway.manufacturer_string) else: log.error("failed to setup upnp") self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, self.get_status()) diff --git a/lbrynet/extras/daemon/Daemon.py b/lbrynet/extras/daemon/Daemon.py index 735166ebf..8e8c13fd4 100644 --- a/lbrynet/extras/daemon/Daemon.py +++ b/lbrynet/extras/daemon/Daemon.py @@ -210,7 +210,6 @@ class Daemon(AuthJSONRPCServer): DATABASE_COMPONENT: "storage", DHT_COMPONENT: "dht_node", WALLET_COMPONENT: "wallet_manager", - STREAM_IDENTIFIER_COMPONENT: "sd_identifier", FILE_MANAGER_COMPONENT: "file_manager", EXCHANGE_RATE_MANAGER_COMPONENT: "exchange_rate_manager", PAYMENT_RATE_COMPONENT: "payment_rate_manager", @@ -241,7 +240,6 @@ class Daemon(AuthJSONRPCServer): self.storage = None self.dht_node = None self.wallet_manager: LbryWalletManager = None - self.sd_identifier = None self.file_manager = None self.exchange_rate_manager = None self.payment_rate_manager = None @@ -305,7 +303,7 @@ class Daemon(AuthJSONRPCServer): rate_manager = rate_manager or self.payment_rate_manager timeout = timeout or 30 downloader = StandaloneBlobDownloader( - blob_hash, self.blob_manager, self.dht_node.peer_finder, self.rate_limiter, + blob_hash, self.blob_manager, self.component_manager.peer_finder, self.rate_limiter, rate_manager, self.wallet_manager, timeout ) return downloader.download() @@ -372,8 +370,8 @@ class Daemon(AuthJSONRPCServer): self.analytics_manager.send_download_started(download_id, name, claim_dict) self.analytics_manager.send_new_download_start(download_id, name, claim_dict) self.streams[sd_hash] = GetStream( - self.sd_identifier, self.wallet_manager, self.exchange_rate_manager, self.blob_manager, - self.dht_node.peer_finder, self.rate_limiter, self.payment_rate_manager, self.storage, + self.file_manager.sd_identifier, self.wallet_manager, self.exchange_rate_manager, self.blob_manager, + self.component_manager.peer_finder, self.rate_limiter, self.payment_rate_manager, self.storage, conf.settings['max_key_fee'], conf.settings['disable_max_key_fee'], conf.settings['data_rate'], timeout ) @@ -432,7 +430,7 @@ class Daemon(AuthJSONRPCServer): if blob: return self.blob_manager.get_blob(blob[0]) return download_sd_blob( - sd_hash.decode(), self.blob_manager, self.dht_node.peer_finder, self.rate_limiter, + sd_hash.decode(), self.blob_manager, self.component_manager.peer_finder, self.rate_limiter, self.payment_rate_manager, self.wallet_manager, timeout=conf.settings['peer_search_timeout'], download_mirrors=conf.settings['download_mirrors'] ) @@ -819,7 +817,7 @@ class Daemon(AuthJSONRPCServer): 'ip': (str) remote ip, if available, 'lbrynet_version': (str) lbrynet_version, 'lbryum_version': (str) lbryum_version, - 'lbrynet.schema_version': (str) lbrynet.schema_version, + 'lbryschema_version': (str) lbryschema_version, 'os_release': (str) os release string 'os_system': (str) os name 'platform': (str) platform string @@ -1776,8 +1774,8 @@ class Daemon(AuthJSONRPCServer): results[resolved_uri] = resolved[resolved_uri] return results - @requires(STREAM_IDENTIFIER_COMPONENT, WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT, - DHT_COMPONENT, RATE_LIMITER_COMPONENT, PAYMENT_RATE_COMPONENT, DATABASE_COMPONENT, + @requires(WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT, + RATE_LIMITER_COMPONENT, PAYMENT_RATE_COMPONENT, DATABASE_COMPONENT, conditions=[WALLET_IS_UNLOCKED]) async def jsonrpc_get(self, uri, file_name=None, timeout=None): """ @@ -1962,7 +1960,7 @@ class Daemon(AuthJSONRPCServer): response = yield self._render_response(result) defer.returnValue(response) - @requires(STREAM_IDENTIFIER_COMPONENT, WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT, + @requires(WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT, DHT_COMPONENT, RATE_LIMITER_COMPONENT, PAYMENT_RATE_COMPONENT, DATABASE_COMPONENT, conditions=[WALLET_IS_UNLOCKED]) def jsonrpc_stream_cost_estimate(self, uri, size=None): diff --git a/tests/integration/cli/test_cli.py b/tests/integration/cli/test_cli.py index 855411cf0..cd652b2f7 100644 --- a/tests/integration/cli/test_cli.py +++ b/tests/integration/cli/test_cli.py @@ -3,10 +3,10 @@ from twisted.trial import unittest from io import StringIO from twisted.internet import defer -from lbrynet.extras.daemon import conf -from lbrynet.extras import cli +from lbrynet import conf +from lbrynet import cli from lbrynet.extras.daemon.Components import DATABASE_COMPONENT, BLOB_COMPONENT, HEADERS_COMPONENT, WALLET_COMPONENT, \ - DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, STREAM_IDENTIFIER_COMPONENT, FILE_MANAGER_COMPONENT, \ + DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, \ PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, \ RATE_LIMITER_COMPONENT, PAYMENT_RATE_COMPONENT from lbrynet.extras.daemon.Daemon import Daemon @@ -35,7 +35,7 @@ class CLIIntegrationTest(unittest.TestCase): def setUp(self): skip = [ DATABASE_COMPONENT, BLOB_COMPONENT, HEADERS_COMPONENT, WALLET_COMPONENT, - DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, STREAM_IDENTIFIER_COMPONENT, FILE_MANAGER_COMPONENT, + DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, RATE_LIMITER_COMPONENT, PAYMENT_RATE_COMPONENT ] diff --git a/tests/unit/components/test_Component_Manager.py b/tests/unit/components/test_Component_Manager.py index 15feeb78b..782711f43 100644 --- a/tests/unit/components/test_Component_Manager.py +++ b/tests/unit/components/test_Component_Manager.py @@ -2,7 +2,7 @@ from twisted.internet.task import Clock from twisted.trial import unittest from lbrynet.extras.daemon.ComponentManager import ComponentManager -from lbrynet.extras.daemon.Components import DATABASE_COMPONENT, DHT_COMPONENT, STREAM_IDENTIFIER_COMPONENT +from lbrynet.extras.daemon.Components import DATABASE_COMPONENT, DHT_COMPONENT from lbrynet.extras.daemon.Components import HASH_ANNOUNCER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT from lbrynet.extras.daemon.Components import PEER_PROTOCOL_SERVER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT from lbrynet.extras.daemon.Components import RATE_LIMITER_COMPONENT, HEADERS_COMPONENT, PAYMENT_RATE_COMPONENT @@ -13,6 +13,7 @@ from tests import mocks class TestComponentManager(unittest.TestCase): def setUp(self): mocks.mock_conf_settings(self) + self.default_components_sort = [ [ Components.HeadersComponent, @@ -23,19 +24,14 @@ class TestComponentManager(unittest.TestCase): Components.UPnPComponent ], [ + Components.BlobComponent, Components.DHTComponent, Components.WalletComponent ], [ - Components.BlobComponent, - Components.HashAnnouncerComponent - ], - [ - Components.PeerProtocolServerComponent, - Components.StreamIdentifierComponent - ], - [ - Components.FileManagerComponent + Components.FileManagerComponent, + Components.HashAnnouncerComponent, + Components.PeerProtocolServerComponent ], [ Components.ReflectorComponent @@ -48,7 +44,6 @@ class TestComponentManager(unittest.TestCase): def test_sort_components(self): stages = self.component_manager.sort_components() - for stage_list, sorted_stage_list in zip(stages, self.default_components_sort): self.assertEqual([type(stage) for stage in stage_list], sorted_stage_list) @@ -101,7 +96,7 @@ class TestComponentManagerProperStart(unittest.TestCase): self.reactor = Clock() mocks.mock_conf_settings(self) self.component_manager = ComponentManager( - skip_components=[DATABASE_COMPONENT, DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, STREAM_IDENTIFIER_COMPONENT, + skip_components=[DATABASE_COMPONENT, DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT, HEADERS_COMPONENT, PAYMENT_RATE_COMPONENT, RATE_LIMITER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT], diff --git a/tests/unit/lbrynet_daemon/test_Daemon.py b/tests/unit/lbrynet_daemon/test_Daemon.py index fed651391..eed2d60a8 100644 --- a/tests/unit/lbrynet_daemon/test_Daemon.py +++ b/tests/unit/lbrynet_daemon/test_Daemon.py @@ -12,7 +12,7 @@ from lbrynet.schema.decode import smart_decode from lbrynet.extras.daemon import conf from lbrynet.extras.daemon.storage import SQLiteStorage from lbrynet.extras.daemon.ComponentManager import ComponentManager -from lbrynet.extras.daemon.Components import DATABASE_COMPONENT, DHT_COMPONENT, WALLET_COMPONENT, STREAM_IDENTIFIER_COMPONENT +from lbrynet.extras.daemon.Components import DATABASE_COMPONENT, DHT_COMPONENT, WALLET_COMPONENT from lbrynet.extras.daemon.Components import f2d from lbrynet.extras.daemon.Components import HASH_ANNOUNCER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT, BLOB_COMPONENT from lbrynet.extras.daemon.Components import PEER_PROTOCOL_SERVER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT @@ -45,7 +45,7 @@ def get_test_daemon(data_rate=None, generous=True, with_fee=False): component_manager = ComponentManager( skip_components=[DATABASE_COMPONENT, DHT_COMPONENT, WALLET_COMPONENT, UPNP_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, HASH_ANNOUNCER_COMPONENT, - STREAM_IDENTIFIER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT, + EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT, HEADERS_COMPONENT, RATE_LIMITER_COMPONENT], file_manager=FakeFileManager ) From 488b9767345ce90847f0915d575666b982cae36b Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 7 Nov 2018 09:54:09 -0500 Subject: [PATCH 05/16] move HashAnnouncer.py to extras --- lbrynet/extras/daemon/Components.py | 5 +++-- .../{dht/hashannouncer.py => extras/daemon/HashAnnouncer.py} | 0 tests/unit/dht/test_hash_announcer.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) rename lbrynet/{dht/hashannouncer.py => extras/daemon/HashAnnouncer.py} (100%) diff --git a/lbrynet/extras/daemon/Components.py b/lbrynet/extras/daemon/Components.py index b1c936429..8642e34d9 100644 --- a/lbrynet/extras/daemon/Components.py +++ b/lbrynet/extras/daemon/Components.py @@ -24,7 +24,8 @@ from lbrynet.p2p.server.ServerProtocol import ServerProtocolFactory from .Component import Component from .ExchangeRateManager import ExchangeRateManager from lbrynet.extras.daemon.storage import SQLiteStorage -from lbrynet.dht import node, hashannouncer +from lbrynet.dht import node +from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer from lbrynet.blob.EncryptedFileManager import EncryptedFileManager from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaverFactory from lbrynet.blob.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier @@ -466,7 +467,7 @@ class HashAnnouncerComponent(Component): def start(self): storage = self.component_manager.get_component(DATABASE_COMPONENT) dht_node = self.component_manager.get_component(DHT_COMPONENT) - self.hash_announcer = hashannouncer.DHTHashAnnouncer(dht_node, storage) + self.hash_announcer = DHTHashAnnouncer(dht_node, storage) yield self.hash_announcer.start() @defer.inlineCallbacks diff --git a/lbrynet/dht/hashannouncer.py b/lbrynet/extras/daemon/HashAnnouncer.py similarity index 100% rename from lbrynet/dht/hashannouncer.py rename to lbrynet/extras/daemon/HashAnnouncer.py diff --git a/tests/unit/dht/test_hash_announcer.py b/tests/unit/dht/test_hash_announcer.py index 7919a01e1..5aeb54943 100644 --- a/tests/unit/dht/test_hash_announcer.py +++ b/tests/unit/dht/test_hash_announcer.py @@ -1,7 +1,7 @@ from twisted.trial import unittest from twisted.internet import defer, task from lbrynet.p2p import utils -from lbrynet.dht.hashannouncer import DHTHashAnnouncer +from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer from tests.util import random_lbry_hash from tests.mocks import mock_conf_settings From a6b643bb341c4866264482b8da279f09cdc678f8 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 7 Nov 2018 15:15:05 -0500 Subject: [PATCH 06/16] move files --- lbrynet/blob/EncryptedFileCreator.py | 2 +- lbrynet/blob/EncryptedFileDownloader.py | 4 ++-- lbrynet/blob/EncryptedFileManager.py | 4 ++-- lbrynet/blob/blob_file.py | 4 ++-- lbrynet/blob/client/CryptStreamDownloader.py | 8 ++++--- .../blob/client/EncryptedFileDownloader.py | 2 +- lbrynet/blob/creator.py | 2 +- lbrynet/blob/writer.py | 2 +- lbrynet/{extras/daemon => }/conf.py | 2 +- lbrynet/{p2p => }/cryptoutils.py | 0 lbrynet/{p2p => dht}/call_later_manager.py | 0 lbrynet/dht/contact.py | 2 +- lbrynet/dht/distance.py | 2 +- lbrynet/dht/msgtypes.py | 2 +- lbrynet/dht/node.py | 5 ++--- lbrynet/extras/cli.py | 2 +- lbrynet/extras/daemon/ComponentManager.py | 4 ++-- lbrynet/extras/daemon/Components.py | 18 +++++++-------- lbrynet/extras/daemon/Daemon.py | 13 ++++------- lbrynet/extras/daemon/DaemonConsole.py | 6 ++--- lbrynet/extras/daemon/DaemonControl.py | 5 ++--- lbrynet/extras/daemon/Downloader.py | 4 ++-- lbrynet/extras/daemon/HashAnnouncer.py | 3 +-- .../daemon/PeerFinder.py} | 2 +- lbrynet/{p2p => extras/daemon}/PeerManager.py | 0 lbrynet/extras/daemon/analytics.py | 4 ++-- lbrynet/extras/daemon/auth/client.py | 2 +- lbrynet/extras/daemon/auth/factory.py | 2 +- lbrynet/extras/daemon/auth/keyring.py | 2 +- lbrynet/extras/daemon/auth/server.py | 7 +++--- lbrynet/extras/daemon/migrator/migrate5to6.py | 2 +- lbrynet/extras/daemon/storage.py | 3 +-- .../{p2p => extras}/looping_call_manager.py | 0 lbrynet/extras/reflector/reupload.py | 2 +- lbrynet/extras/reflector/server/server.py | 2 +- lbrynet/{p2p => }/log_support.py | 8 +++---- lbrynet/p2p/HTTPBlobDownloader.py | 2 +- lbrynet/p2p/PaymentRateManager.py | 2 +- lbrynet/p2p/Peer.py | 2 +- lbrynet/p2p/PriceModel.py | 3 +-- lbrynet/p2p/SinglePeerDownloader.py | 2 +- lbrynet/p2p/Strategy.py | 3 +-- lbrynet/p2p/StreamDescriptor.py | 2 +- lbrynet/p2p/client/ClientProtocol.py | 3 +-- lbrynet/p2p/client/ConnectionManager.py | 3 +-- .../p2p/client/StandaloneBlobDownloader.py | 2 +- lbrynet/{p2p => }/system_info.py | 0 lbrynet/{p2p => }/utils.py | 3 ++- lbrynet/{extras/daemon => }/winpaths.py | 0 scripts/encrypt_blob.py | 2 +- tests/functional/test_misc.py | 6 ++--- tests/functional/test_reflector.py | 5 +++-- tests/functional/test_streamify.py | 2 +- tests/integration/cli/test_cli.py | 2 +- tests/integration/wallet/test_commands.py | 2 +- tests/mocks.py | 4 ++-- tests/{util.py => test_utils.py} | 5 +++-- .../core/client/test_ConnectionManager.py | 15 ++++++------- tests/unit/core/test_BlobManager.py | 6 ++--- tests/unit/core/test_HTTPBlobDownloader.py | 2 +- tests/unit/core/test_HashBlob.py | 2 +- tests/unit/core/test_Wallet.py | 3 --- tests/unit/core/test_utils.py | 2 +- tests/unit/database/test_SQLiteStorage.py | 4 ++-- tests/unit/dht/test_contact.py | 2 +- tests/unit/dht/test_hash_announcer.py | 4 ++-- tests/unit/dht/test_kbucket.py | 2 +- tests/unit/dht/test_node.py | 2 +- tests/unit/dht/test_routingtable.py | 2 +- .../test_EncryptedFileCreator.py | 4 ++-- tests/unit/lbrynet_daemon/auth/test_server.py | 2 +- tests/unit/lbrynet_daemon/test_Daemon.py | 22 +++++++------------ tests/unit/lbrynet_daemon/test_Downloader.py | 2 +- .../test_ExchangeRateManager.py | 16 +++++++------- .../lbrynet_daemon/test_claims_comparator.py | 2 +- tests/unit/test_cli.py | 2 +- tests/unit/test_conf.py | 2 +- 77 files changed, 131 insertions(+), 154 deletions(-) rename lbrynet/{extras/daemon => }/conf.py (99%) rename lbrynet/{p2p => }/cryptoutils.py (100%) rename lbrynet/{p2p => dht}/call_later_manager.py (100%) rename lbrynet/{dht/peerfinder.py => extras/daemon/PeerFinder.py} (98%) rename lbrynet/{p2p => extras/daemon}/PeerManager.py (100%) rename lbrynet/{p2p => extras}/looping_call_manager.py (100%) rename lbrynet/{p2p => }/log_support.py (99%) rename lbrynet/{p2p => }/system_info.py (100%) rename lbrynet/{p2p => }/utils.py (99%) rename lbrynet/{extras/daemon => }/winpaths.py (100%) rename tests/{util.py => test_utils.py} (91%) diff --git a/lbrynet/blob/EncryptedFileCreator.py b/lbrynet/blob/EncryptedFileCreator.py index 7d1fec30a..d8ee1868d 100644 --- a/lbrynet/blob/EncryptedFileCreator.py +++ b/lbrynet/blob/EncryptedFileCreator.py @@ -2,8 +2,8 @@ Utilities for turning plain files into LBRY Files. """ -import logging import os +import logging from binascii import hexlify from twisted.internet import defer diff --git a/lbrynet/blob/EncryptedFileDownloader.py b/lbrynet/blob/EncryptedFileDownloader.py index 12664a0c6..108bb2f80 100644 --- a/lbrynet/blob/EncryptedFileDownloader.py +++ b/lbrynet/blob/EncryptedFileDownloader.py @@ -5,10 +5,10 @@ import logging from binascii import hexlify, unhexlify from twisted.internet import defer -from lbrynet.extras.daemon import conf +from lbrynet import conf from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager from lbrynet.p2p.HTTPBlobDownloader import HTTPBlobDownloader -from lbrynet.p2p.utils import short_hash +from lbrynet.utils import short_hash from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaver from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileDownloader from lbrynet.blob.EncryptedFileStatusReport import EncryptedFileStatusReport diff --git a/lbrynet/blob/EncryptedFileManager.py b/lbrynet/blob/EncryptedFileManager.py index f7d138db3..902ecae88 100644 --- a/lbrynet/blob/EncryptedFileManager.py +++ b/lbrynet/blob/EncryptedFileManager.py @@ -7,14 +7,14 @@ from binascii import hexlify, unhexlify from twisted.internet import defer, task, reactor from twisted.python.failure import Failure +from lbrynet import conf from lbrynet.extras.reflector.reupload import reflect_file from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloader from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamType, get_sd_info from lbrynet.blob.client.CryptStreamDownloader import AlreadyStoppedError from lbrynet.blob.client.CryptStreamDownloader import CurrentlyStoppingError -from lbrynet.p2p.utils import safe_start_looping_call, safe_stop_looping_call -from lbrynet.extras.daemon import conf +from lbrynet.utils import safe_start_looping_call, safe_stop_looping_call log = logging.getLogger(__name__) diff --git a/lbrynet/blob/blob_file.py b/lbrynet/blob/blob_file.py index 0046b6c5a..5f9173969 100644 --- a/lbrynet/blob/blob_file.py +++ b/lbrynet/blob/blob_file.py @@ -1,10 +1,10 @@ -import logging import os +import logging from twisted.internet import defer, threads from twisted.web.client import FileBodyProducer from twisted.python.failure import Failure from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError, InvalidBlobHashError -from lbrynet.p2p.utils import is_valid_blobhash +from lbrynet.utils import is_valid_blobhash from lbrynet.blob.writer import HashBlobWriter from lbrynet.blob.reader import HashBlobReader diff --git a/lbrynet/blob/client/CryptStreamDownloader.py b/lbrynet/blob/client/CryptStreamDownloader.py index ffc0844cf..985e399b7 100644 --- a/lbrynet/blob/client/CryptStreamDownloader.py +++ b/lbrynet/blob/client/CryptStreamDownloader.py @@ -1,12 +1,14 @@ -from binascii import unhexlify import logging +from binascii import unhexlify +from twisted.internet import defer +from twisted.python.failure import Failure + from lbrynet.p2p.client.BlobRequester import BlobRequester from lbrynet.p2p.client.ConnectionManager import ConnectionManager from lbrynet.p2p.client.DownloadManager import DownloadManager from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager from lbrynet.blob.client.CryptBlobHandler import CryptBlobHandler -from twisted.internet import defer -from twisted.python.failure import Failure + log = logging.getLogger(__name__) diff --git a/lbrynet/blob/client/EncryptedFileDownloader.py b/lbrynet/blob/client/EncryptedFileDownloader.py index 735ea3bf4..48e7b6565 100644 --- a/lbrynet/blob/client/EncryptedFileDownloader.py +++ b/lbrynet/blob/client/EncryptedFileDownloader.py @@ -2,13 +2,13 @@ import os import logging import traceback from binascii import hexlify, unhexlify +from twisted.internet import defer, threads from lbrynet.p2p.StreamDescriptor import save_sd_info from lbrynet.blob.client.CryptStreamDownloader import CryptStreamDownloader from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager from lbrynet.p2p.Error import FileOpenError from lbrynet.blob.client.EncryptedFileMetadataHandler import EncryptedFileMetadataHandler -from twisted.internet import defer, threads log = logging.getLogger(__name__) diff --git a/lbrynet/blob/creator.py b/lbrynet/blob/creator.py index 04b42961b..af9dc213f 100644 --- a/lbrynet/blob/creator.py +++ b/lbrynet/blob/creator.py @@ -3,7 +3,7 @@ import logging from io import BytesIO from twisted.internet import defer from twisted.web.client import FileBodyProducer -from lbrynet.p2p.cryptoutils import get_lbry_hash_obj +from lbrynet.cryptoutils import get_lbry_hash_obj log = logging.getLogger(__name__) diff --git a/lbrynet/blob/writer.py b/lbrynet/blob/writer.py index 5332d3533..c66e3b037 100644 --- a/lbrynet/blob/writer.py +++ b/lbrynet/blob/writer.py @@ -2,7 +2,7 @@ import logging from io import BytesIO from twisted.python.failure import Failure from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError -from lbrynet.p2p.cryptoutils import get_lbry_hash_obj +from lbrynet.cryptoutils import get_lbry_hash_obj log = logging.getLogger(__name__) diff --git a/lbrynet/extras/daemon/conf.py b/lbrynet/conf.py similarity index 99% rename from lbrynet/extras/daemon/conf.py rename to lbrynet/conf.py index 311ba64cc..72ba2b700 100644 --- a/lbrynet/extras/daemon/conf.py +++ b/lbrynet/conf.py @@ -7,7 +7,7 @@ import sys import yaml import envparse from appdirs import user_data_dir, user_config_dir -from lbrynet.p2p import utils +from lbrynet import utils from lbrynet.p2p.Error import InvalidCurrencyError, NoSuchDirectoryError from lbrynet.androidhelpers.paths import ( android_internal_storage_dir, diff --git a/lbrynet/p2p/cryptoutils.py b/lbrynet/cryptoutils.py similarity index 100% rename from lbrynet/p2p/cryptoutils.py rename to lbrynet/cryptoutils.py diff --git a/lbrynet/p2p/call_later_manager.py b/lbrynet/dht/call_later_manager.py similarity index 100% rename from lbrynet/p2p/call_later_manager.py rename to lbrynet/dht/call_later_manager.py diff --git a/lbrynet/dht/contact.py b/lbrynet/dht/contact.py index 99c84532b..afd2b4718 100644 --- a/lbrynet/dht/contact.py +++ b/lbrynet/dht/contact.py @@ -1,7 +1,7 @@ import ipaddress from binascii import hexlify from functools import reduce -from lbrynet.dht import constants +from . import constants def is_valid_ipv4(address): diff --git a/lbrynet/dht/distance.py b/lbrynet/dht/distance.py index 2c1099535..94de80433 100644 --- a/lbrynet/dht/distance.py +++ b/lbrynet/dht/distance.py @@ -1,4 +1,4 @@ -from lbrynet.dht import constants +from . import constants class Distance: diff --git a/lbrynet/dht/msgtypes.py b/lbrynet/dht/msgtypes.py index 8bf8f3227..59f9440bd 100644 --- a/lbrynet/dht/msgtypes.py +++ b/lbrynet/dht/msgtypes.py @@ -7,7 +7,7 @@ # The docstrings in this module contain epytext markup; API documentation # may be created by processing this file with epydoc: http://epydoc.sf.net -from lbrynet.p2p.utils import generate_id +from lbrynet.utils import generate_id from . import constants diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index 64c1a64ab..eb929803a 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -5,9 +5,8 @@ from functools import reduce from twisted.internet import defer, error, task -from lbrynet.p2p.utils import generate_id, DeferredDict -from lbrynet.p2p.call_later_manager import CallLaterManager -from lbrynet.p2p.PeerManager import PeerManager +from lbrynet.utils import generate_id, DeferredDict +from .call_later_manager import CallLaterManager from .error import TimeoutError from . import constants from . import routingtable diff --git a/lbrynet/extras/cli.py b/lbrynet/extras/cli.py index 3ebaf7b8a..3fa0dce11 100644 --- a/lbrynet/extras/cli.py +++ b/lbrynet/extras/cli.py @@ -24,7 +24,7 @@ from lbrynet.extras.daemon.Daemon import Daemon from lbrynet.extras.daemon.DaemonControl import start as daemon_main from lbrynet.extras.daemon.DaemonConsole import main as daemon_console from lbrynet.extras.daemon.auth.client import LBRYAPIClient -from lbrynet.p2p.system_info import get_platform +from lbrynet.system_info import get_platform async def execute_command(method, params, conf_path=None): diff --git a/lbrynet/extras/daemon/ComponentManager.py b/lbrynet/extras/daemon/ComponentManager.py index 59f1304aa..3ccf5889b 100644 --- a/lbrynet/extras/daemon/ComponentManager.py +++ b/lbrynet/extras/daemon/ComponentManager.py @@ -1,8 +1,8 @@ import logging from twisted.internet import defer from lbrynet.p2p.Error import ComponentStartConditionNotMet -from lbrynet.core.PeerManager import PeerManager -from lbrynet.dht.peerfinder import DHTPeerFinder +from .PeerManager import PeerManager +from .PeerFinder import DHTPeerFinder log = logging.getLogger(__name__) diff --git a/lbrynet/extras/daemon/Components.py b/lbrynet/extras/daemon/Components.py index 8642e34d9..8ba078eef 100644 --- a/lbrynet/extras/daemon/Components.py +++ b/lbrynet/extras/daemon/Components.py @@ -11,8 +11,8 @@ import lbrynet.schema from aioupnp import __version__ as aioupnp_version from aioupnp.upnp import UPnP from aioupnp.fault import UPnPError -from lbrynet.extras.daemon import conf -from lbrynet.p2p.utils import DeferredDict +from lbrynet import conf, system_info +from lbrynet.utils import DeferredDict, generate_id from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager from lbrynet.p2p.RateLimiter import RateLimiter from lbrynet.p2p.BlobManager import DiskBlobManager @@ -23,16 +23,15 @@ from lbrynet.p2p.server.BlobRequestHandler import BlobRequestHandlerFactory from lbrynet.p2p.server.ServerProtocol import ServerProtocolFactory from .Component import Component from .ExchangeRateManager import ExchangeRateManager -from lbrynet.extras.daemon.storage import SQLiteStorage -from lbrynet.dht import node -from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer +from .storage import SQLiteStorage +from .HashAnnouncer import DHTHashAnnouncer + +from lbrynet.dht.node import Node from lbrynet.blob.EncryptedFileManager import EncryptedFileManager from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaverFactory from lbrynet.blob.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier from lbrynet.extras.reflector import ServerFactory as reflector_server_factory -from lbrynet.p2p.utils import generate_id - log = logging.getLogger(__name__) # settings must be initialized before this file is imported @@ -95,8 +94,7 @@ class ConfigSettings: @staticmethod def get_external_ip(): - from lbrynet.p2p.system_info import get_platform - platform = get_platform(get_ip=True) + platform = system_info.get_platform(get_ip=True) return platform['ip'] @@ -435,7 +433,7 @@ class DHTComponent(Component): if not external_ip: log.warning("failed to get external ip") - self.dht_node = node.Node( + self.dht_node = Node( node_id=node_id, udpPort=GCS('dht_node_port'), externalUDPPort=self.external_udp_port, diff --git a/lbrynet/extras/daemon/Daemon.py b/lbrynet/extras/daemon/Daemon.py index 8e8c13fd4..00b4c747c 100644 --- a/lbrynet/extras/daemon/Daemon.py +++ b/lbrynet/extras/daemon/Daemon.py @@ -16,6 +16,8 @@ from twisted.python.failure import Failure from torba.client.baseaccount import SingleKey, HierarchicalDeterministic +from lbrynet import conf, utils, system_info +from lbrynet.system_info import get_lbrynet_version from lbrynet.schema.claim import ClaimDict from lbrynet.schema.uri import parse_lbry_uri from lbrynet.schema.error import URIParseError, DecodeError @@ -23,19 +25,15 @@ from lbrynet.schema.validator import validate_claim_id from lbrynet.schema.address import decode_address from lbrynet.schema.decode import smart_decode -# TODO: importing this when internet is disabled raises a socket.gaierror -from lbrynet.p2p.system_info import get_lbrynet_version -from lbrynet.extras.daemon import conf from lbrynet.extras.reflector import reupload from .Components import d2f, f2d from .Components import WALLET_COMPONENT, DATABASE_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT -from .Components import STREAM_IDENTIFIER_COMPONENT, FILE_MANAGER_COMPONENT, RATE_LIMITER_COMPONENT +from .Components import FILE_MANAGER_COMPONENT, RATE_LIMITER_COMPONENT from .Components import EXCHANGE_RATE_MANAGER_COMPONENT, PAYMENT_RATE_COMPONENT, UPNP_COMPONENT from .ComponentManager import RequiredCondition from .Downloader import GetStream from .Publisher import Publisher from .auth.server import AuthJSONRPCServer -from lbrynet.p2p import utils, system_info from lbrynet.p2p.StreamDescriptor import download_sd_blob from lbrynet.p2p.Error import InsufficientFundsError, UnknownNameError from lbrynet.p2p.Error import DownloadDataTimeout, DownloadSDTimeout @@ -448,7 +446,7 @@ class Daemon(AuthJSONRPCServer): Get total stream size in bytes from a sd blob """ - d = self.sd_identifier.get_metadata_for_sd_blob(sd_blob) + d = self.file_manager.sd_identifier.get_metadata_for_sd_blob(sd_blob) d.addCallback(lambda metadata: metadata.validator.info_to_show()) d.addCallback(lambda info: int(dict(info)['stream_size'])) return d @@ -466,9 +464,7 @@ class Daemon(AuthJSONRPCServer): """ Calculate estimated LBC cost for a stream given its size in bytes """ - cost = self._get_est_cost_from_stream_size(size) - resolved = await self.wallet_manager.resolve(uri) if uri in resolved and 'claim' in resolved[uri]: @@ -533,7 +529,6 @@ class Daemon(AuthJSONRPCServer): sd blob will be downloaded to determine the stream size """ - if size is not None: return self.get_est_cost_using_known_size(uri, size) return self.get_est_cost_from_uri(uri) diff --git a/lbrynet/extras/daemon/DaemonConsole.py b/lbrynet/extras/daemon/DaemonConsole.py index 5a7d6caa0..c7e82c6ac 100644 --- a/lbrynet/extras/daemon/DaemonConsole.py +++ b/lbrynet/extras/daemon/DaemonConsole.py @@ -6,9 +6,9 @@ import logging.handlers from twisted.internet import defer, reactor, threads from aiohttp import client_exceptions -from lbrynet.extras.daemon import analytics, conf -from lbrynet.p2p import utils -from lbrynet.p2p import log_support +from lbrynet.extras.daemon import analytics +from lbrynet import utils, conf +from lbrynet import log_support from .auth.client import LBRYAPIClient from .Daemon import Daemon diff --git a/lbrynet/extras/daemon/DaemonControl.py b/lbrynet/extras/daemon/DaemonControl.py index 09fa07cf8..1cb0a8a4a 100644 --- a/lbrynet/extras/daemon/DaemonControl.py +++ b/lbrynet/extras/daemon/DaemonControl.py @@ -7,15 +7,14 @@ if 'win' in sys.platform: import certifi os.environ['SSL_CERT_FILE'] = certifi.where() -from lbrynet.p2p import log_support +from lbrynet import log_support import argparse import logging.handlers from twisted.internet import reactor -from lbrynet.extras.daemon import conf -from lbrynet.p2p import utils, system_info +from lbrynet import utils, system_info, conf from .Daemon import Daemon log = logging.getLogger(__name__) diff --git a/lbrynet/extras/daemon/Downloader.py b/lbrynet/extras/daemon/Downloader.py index fe036b3cd..5c73c49e7 100644 --- a/lbrynet/extras/daemon/Downloader.py +++ b/lbrynet/extras/daemon/Downloader.py @@ -4,14 +4,14 @@ from twisted.internet import defer from twisted.internet.task import LoopingCall from .Components import f2d +from lbrynet import conf from lbrynet.schema.fee import Fee from lbrynet.p2p.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed, InvalidStreamDescriptorError from lbrynet.p2p.Error import DownloadDataTimeout, DownloadCanceledError, DownloadSDTimeout -from lbrynet.p2p.utils import safe_start_looping_call, safe_stop_looping_call +from lbrynet.utils import safe_start_looping_call, safe_stop_looping_call from lbrynet.p2p.StreamDescriptor import download_sd_blob from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory -from lbrynet.extras.daemon import conf from torba.client.constants import COIN from lbrynet.extras.wallet.dewies import dewies_to_lbc diff --git a/lbrynet/extras/daemon/HashAnnouncer.py b/lbrynet/extras/daemon/HashAnnouncer.py index d1cc26d85..819f890fa 100644 --- a/lbrynet/extras/daemon/HashAnnouncer.py +++ b/lbrynet/extras/daemon/HashAnnouncer.py @@ -2,8 +2,7 @@ import binascii import logging from twisted.internet import defer, task -from lbrynet.p2p import utils -from lbrynet.extras.daemon import conf +from lbrynet import utils, conf log = logging.getLogger(__name__) diff --git a/lbrynet/dht/peerfinder.py b/lbrynet/extras/daemon/PeerFinder.py similarity index 98% rename from lbrynet/dht/peerfinder.py rename to lbrynet/extras/daemon/PeerFinder.py index edabb9ee7..2934b13f5 100644 --- a/lbrynet/dht/peerfinder.py +++ b/lbrynet/extras/daemon/PeerFinder.py @@ -2,7 +2,7 @@ import binascii import logging from twisted.internet import defer -from lbrynet.extras.daemon import conf +from lbrynet import conf log = logging.getLogger(__name__) diff --git a/lbrynet/p2p/PeerManager.py b/lbrynet/extras/daemon/PeerManager.py similarity index 100% rename from lbrynet/p2p/PeerManager.py rename to lbrynet/extras/daemon/PeerManager.py diff --git a/lbrynet/extras/daemon/analytics.py b/lbrynet/extras/daemon/analytics.py index 2fe911b2f..907d524f6 100644 --- a/lbrynet/extras/daemon/analytics.py +++ b/lbrynet/extras/daemon/analytics.py @@ -4,8 +4,8 @@ import logging import treq from twisted.internet import defer, task -from lbrynet.extras.daemon import conf -from lbrynet.p2p import looping_call_manager, utils, system_info +from lbrynet import conf, utils, system_info +from lbrynet.extras import looping_call_manager # Things We Track SERVER_STARTUP = 'Server Startup' diff --git a/lbrynet/extras/daemon/auth/client.py b/lbrynet/extras/daemon/auth/client.py index b34e61e50..cb9c657f9 100644 --- a/lbrynet/extras/daemon/auth/client.py +++ b/lbrynet/extras/daemon/auth/client.py @@ -3,7 +3,7 @@ import aiohttp import logging from urllib.parse import urlparse -from lbrynet.extras.daemon import conf +from lbrynet import conf from .keyring import Keyring, APIKey log = logging.getLogger(__name__) diff --git a/lbrynet/extras/daemon/auth/factory.py b/lbrynet/extras/daemon/auth/factory.py index e9bb29375..549b9bd68 100644 --- a/lbrynet/extras/daemon/auth/factory.py +++ b/lbrynet/extras/daemon/auth/factory.py @@ -3,7 +3,7 @@ import logging from twisted.web import server, guard, resource from twisted.cred import portal -from lbrynet.extras.daemon import conf +from lbrynet import conf from .auth import PasswordChecker, HttpPasswordRealm from ..auth.keyring import Keyring diff --git a/lbrynet/extras/daemon/auth/keyring.py b/lbrynet/extras/daemon/auth/keyring.py index 01ac81f16..91d9290a1 100644 --- a/lbrynet/extras/daemon/auth/keyring.py +++ b/lbrynet/extras/daemon/auth/keyring.py @@ -13,7 +13,7 @@ from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives import serialization from twisted.internet import ssl import logging -from lbrynet.extras.daemon import conf +from lbrynet import conf log = logging.getLogger(__name__) diff --git a/lbrynet/extras/daemon/auth/server.py b/lbrynet/extras/daemon/auth/server.py index 3e94231db..93d0e8b26 100644 --- a/lbrynet/extras/daemon/auth/server.py +++ b/lbrynet/extras/daemon/auth/server.py @@ -13,12 +13,11 @@ from twisted.python.failure import Failure from twisted.internet.error import ConnectionDone, ConnectionLost from txjsonrpc import jsonrpclib from traceback import format_exc - -from lbrynet.extras.daemon import analytics, conf +from lbrynet import conf, utils +from lbrynet.extras.daemon import analytics from lbrynet.p2p.Error import InvalidAuthenticationToken -from lbrynet.p2p import utils from lbrynet.p2p.Error import ComponentsNotStarted, ComponentStartConditionNotMet -from lbrynet.p2p.looping_call_manager import LoopingCallManager +from lbrynet.extras.looping_call_manager import LoopingCallManager from lbrynet.extras.daemon.ComponentManager import ComponentManager from .keyring import APIKey, Keyring from .undecorated import undecorated diff --git a/lbrynet/extras/daemon/migrator/migrate5to6.py b/lbrynet/extras/daemon/migrator/migrate5to6.py index 367039ab7..017aa965f 100644 --- a/lbrynet/extras/daemon/migrator/migrate5to6.py +++ b/lbrynet/extras/daemon/migrator/migrate5to6.py @@ -3,7 +3,7 @@ import os import json import logging from lbrynet.schema.decode import smart_decode -from lbrynet.extras.daemon import conf +from lbrynet import conf log = logging.getLogger(__name__) diff --git a/lbrynet/extras/daemon/storage.py b/lbrynet/extras/daemon/storage.py index 48742034b..724aabeef 100644 --- a/lbrynet/extras/daemon/storage.py +++ b/lbrynet/extras/daemon/storage.py @@ -6,10 +6,9 @@ from binascii import hexlify, unhexlify from decimal import Decimal from twisted.internet import defer, task, threads from twisted.enterprise import adbapi - +from lbrynet import conf from lbrynet.schema.claim import ClaimDict from lbrynet.schema.decode import smart_decode -from lbrynet.extras.daemon import conf from lbrynet.blob.CryptBlob import CryptBlobInfo from lbrynet.dht.constants import dataExpireTimeout from torba.client.constants import COIN diff --git a/lbrynet/p2p/looping_call_manager.py b/lbrynet/extras/looping_call_manager.py similarity index 100% rename from lbrynet/p2p/looping_call_manager.py rename to lbrynet/extras/looping_call_manager.py diff --git a/lbrynet/extras/reflector/reupload.py b/lbrynet/extras/reflector/reupload.py index 5eca291d5..a2b1548e3 100644 --- a/lbrynet/extras/reflector/reupload.py +++ b/lbrynet/extras/reflector/reupload.py @@ -1,7 +1,7 @@ import random from twisted.internet import reactor, defer -from lbrynet.extras.daemon import conf +from lbrynet import conf from . import ClientFactory, BlobClientFactory diff --git a/lbrynet/extras/reflector/server/server.py b/lbrynet/extras/reflector/server/server.py index 14bf2f889..c1d5f9ac1 100644 --- a/lbrynet/extras/reflector/server/server.py +++ b/lbrynet/extras/reflector/server/server.py @@ -3,7 +3,7 @@ import json from twisted.python import failure from twisted.internet import error, defer from twisted.internet.protocol import Protocol, ServerFactory -from lbrynet.p2p.utils import is_valid_blobhash +from lbrynet.utils import is_valid_blobhash from lbrynet.p2p.Error import DownloadCanceledError, InvalidBlobHashError from lbrynet.p2p.StreamDescriptor import BlobStreamDescriptorReader from lbrynet.p2p.StreamDescriptor import save_sd_info diff --git a/lbrynet/p2p/log_support.py b/lbrynet/log_support.py similarity index 99% rename from lbrynet/p2p/log_support.py rename to lbrynet/log_support.py index 1eed6659a..72cacc2fb 100644 --- a/lbrynet/p2p/log_support.py +++ b/lbrynet/log_support.py @@ -3,14 +3,11 @@ import logging import logging.handlers import sys import traceback - import treq -from twisted.internet import defer import twisted.python.log - +from twisted.internet import defer from lbrynet import __version__ as lbrynet_version, build_type -from lbrynet.extras.daemon import conf -from lbrynet.p2p import utils +from lbrynet import utils, conf class HTTPSHandler(logging.Handler): @@ -214,6 +211,7 @@ def configure_logging(file_name, console, verbose=None): verbose: a list of loggers to set to debug level. See `convert_verbose` for more details. """ + logger = logging.getLoggerClass() verbose = convert_verbose(verbose) configure_twisted() configure_file_handler(file_name) diff --git a/lbrynet/p2p/HTTPBlobDownloader.py b/lbrynet/p2p/HTTPBlobDownloader.py index 8619737fa..62e685f48 100644 --- a/lbrynet/p2p/HTTPBlobDownloader.py +++ b/lbrynet/p2p/HTTPBlobDownloader.py @@ -4,7 +4,7 @@ from twisted.internet import defer, task from twisted.internet.error import ConnectingCancelledError from twisted.web._newclient import ResponseNeverReceived import treq -from lbrynet.p2p.utils import DeferredDict +from lbrynet.utils import DeferredDict from lbrynet.p2p.Error import DownloadCanceledError log = logging.getLogger(__name__) diff --git a/lbrynet/p2p/PaymentRateManager.py b/lbrynet/p2p/PaymentRateManager.py index fcf7bbd54..372204365 100644 --- a/lbrynet/p2p/PaymentRateManager.py +++ b/lbrynet/p2p/PaymentRateManager.py @@ -1,5 +1,5 @@ from lbrynet.p2p.Strategy import get_default_strategy, OnlyFreeStrategy -from lbrynet.extras.daemon import conf +from lbrynet import conf from decimal import Decimal diff --git a/lbrynet/p2p/Peer.py b/lbrynet/p2p/Peer.py index f06ce806d..bd52e0da3 100644 --- a/lbrynet/p2p/Peer.py +++ b/lbrynet/p2p/Peer.py @@ -1,6 +1,6 @@ import datetime from collections import defaultdict -from lbrynet.p2p import utils +from lbrynet import utils # Do not create this object except through PeerManager class Peer: diff --git a/lbrynet/p2p/PriceModel.py b/lbrynet/p2p/PriceModel.py index 14d7f0c34..4d065d47a 100644 --- a/lbrynet/p2p/PriceModel.py +++ b/lbrynet/p2p/PriceModel.py @@ -1,6 +1,5 @@ from decimal import Decimal - -from lbrynet.extras.daemon import conf +from lbrynet import conf def get_default_price_model(blob_tracker, base_price, **kwargs): diff --git a/lbrynet/p2p/SinglePeerDownloader.py b/lbrynet/p2p/SinglePeerDownloader.py index ab1eee31f..7936caa85 100644 --- a/lbrynet/p2p/SinglePeerDownloader.py +++ b/lbrynet/p2p/SinglePeerDownloader.py @@ -12,7 +12,7 @@ from lbrynet.p2p.client.BlobRequester import BlobRequester from lbrynet.p2p.client.StandaloneBlobDownloader import StandaloneBlobDownloader from lbrynet.p2p.client.ConnectionManager import ConnectionManager from lbrynet.extras.daemon.storage import SQLiteStorage -from lbrynet.dht.peerfinder import DummyPeerFinder +from lbrynet.extras.daemon.PeerFinder import DummyPeerFinder log = logging.getLogger(__name__) diff --git a/lbrynet/p2p/Strategy.py b/lbrynet/p2p/Strategy.py index 3477985ca..da478fb67 100644 --- a/lbrynet/p2p/Strategy.py +++ b/lbrynet/p2p/Strategy.py @@ -1,5 +1,5 @@ from decimal import Decimal -from lbrynet.extras.daemon import conf +from lbrynet import conf from lbrynet.p2p.Offer import Offer from lbrynet.p2p.PriceModel import MeanAvailabilityWeightedPrice, ZeroPrice @@ -133,7 +133,6 @@ class BasicAvailabilityWeightedStrategy(Strategy): class OnlyFreeStrategy(Strategy): - def __init__(self, *args, **kwargs): price_model = ZeroPrice() super().__init__(price_model, 0.0, 0.0, True) diff --git a/lbrynet/p2p/StreamDescriptor.py b/lbrynet/p2p/StreamDescriptor.py index 9e8849dcb..534d02db6 100644 --- a/lbrynet/p2p/StreamDescriptor.py +++ b/lbrynet/p2p/StreamDescriptor.py @@ -5,7 +5,7 @@ import json import logging from twisted.internet import threads, defer -from lbrynet.p2p.cryptoutils import get_lbry_hash_obj +from lbrynet.cryptoutils import get_lbry_hash_obj from lbrynet.p2p.client.StandaloneBlobDownloader import StandaloneBlobDownloader from lbrynet.p2p.Error import UnknownStreamTypeError, InvalidStreamDescriptorError from lbrynet.p2p.HTTPBlobDownloader import HTTPBlobDownloader diff --git a/lbrynet/p2p/client/ClientProtocol.py b/lbrynet/p2p/client/ClientProtocol.py index 20568dc97..0f8167b79 100644 --- a/lbrynet/p2p/client/ClientProtocol.py +++ b/lbrynet/p2p/client/ClientProtocol.py @@ -5,8 +5,7 @@ from twisted.internet import error, defer from twisted.internet.protocol import Protocol, ClientFactory from twisted.protocols.policies import TimeoutMixin from twisted.python import failure -from lbrynet.extras.daemon import conf -from lbrynet.p2p import utils +from lbrynet import conf, utils from lbrynet.p2p.Error import ConnectionClosedBeforeResponseError, NoResponseError from lbrynet.p2p.Error import DownloadCanceledError, MisbehavingPeerError from lbrynet.p2p.Error import RequestCanceledError diff --git a/lbrynet/p2p/client/ConnectionManager.py b/lbrynet/p2p/client/ConnectionManager.py index f56fb0382..af299d167 100644 --- a/lbrynet/p2p/client/ConnectionManager.py +++ b/lbrynet/p2p/client/ConnectionManager.py @@ -1,10 +1,9 @@ import random import logging from twisted.internet import defer, reactor -from lbrynet.extras.daemon import conf +from lbrynet import utils, conf from lbrynet.p2p.client.ClientProtocol import ClientProtocolFactory from lbrynet.p2p.Error import InsufficientFundsError -from lbrynet.p2p import utils log = logging.getLogger(__name__) diff --git a/lbrynet/p2p/client/StandaloneBlobDownloader.py b/lbrynet/p2p/client/StandaloneBlobDownloader.py index 08aa0b386..6715f10b3 100644 --- a/lbrynet/p2p/client/StandaloneBlobDownloader.py +++ b/lbrynet/p2p/client/StandaloneBlobDownloader.py @@ -4,7 +4,7 @@ from lbrynet.p2p.client.BlobRequester import BlobRequester from lbrynet.p2p.client.ConnectionManager import ConnectionManager from lbrynet.p2p.client.DownloadManager import DownloadManager from lbrynet.p2p.Error import InvalidBlobHashError, DownloadSDTimeout -from lbrynet.p2p.utils import is_valid_blobhash, safe_start_looping_call, safe_stop_looping_call +from lbrynet.utils import is_valid_blobhash, safe_start_looping_call, safe_stop_looping_call from twisted.python.failure import Failure from twisted.internet import defer from twisted.internet.task import LoopingCall diff --git a/lbrynet/p2p/system_info.py b/lbrynet/system_info.py similarity index 100% rename from lbrynet/p2p/system_info.py rename to lbrynet/system_info.py diff --git a/lbrynet/p2p/utils.py b/lbrynet/utils.py similarity index 99% rename from lbrynet/p2p/utils.py rename to lbrynet/utils.py index e7a459edf..52a33e581 100644 --- a/lbrynet/p2p/utils.py +++ b/lbrynet/utils.py @@ -12,7 +12,7 @@ import pkg_resources from twisted.python.failure import Failure from twisted.internet import defer from lbrynet.schema.claim import ClaimDict -from lbrynet.p2p.cryptoutils import get_lbry_hash_obj +from lbrynet.cryptoutils import get_lbry_hash_obj log = logging.getLogger(__name__) @@ -52,6 +52,7 @@ def call_later(delay, func, *args, **kwargs): from twisted.internet import reactor return reactor.callLater(delay, func, *args, **kwargs) + def safe_start_looping_call(looping_call, interval_sec): if not looping_call.running: looping_call.start(interval_sec) diff --git a/lbrynet/extras/daemon/winpaths.py b/lbrynet/winpaths.py similarity index 100% rename from lbrynet/extras/daemon/winpaths.py rename to lbrynet/winpaths.py diff --git a/scripts/encrypt_blob.py b/scripts/encrypt_blob.py index 2b83dce3e..3993ffaeb 100644 --- a/scripts/encrypt_blob.py +++ b/scripts/encrypt_blob.py @@ -9,7 +9,7 @@ from twisted.web.client import FileBodyProducer from lbrynet import conf from lbrynet.core import log_support -from lbrynet.dht.hashannouncer import DummyHashAnnouncer +from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer from lbrynet.core.BlobManager import DiskBlobManager from lbrynet.cryptstream.CryptStreamCreator import CryptStreamCreator diff --git a/tests/functional/test_misc.py b/tests/functional/test_misc.py index 0cbead0c7..606c2b372 100644 --- a/tests/functional/test_misc.py +++ b/tests/functional/test_misc.py @@ -2,13 +2,13 @@ import os from hashlib import md5 from twisted.internet import defer, reactor from twisted.trial import unittest -from lbrynet.extras.daemon import conf +from lbrynet import conf from lbrynet.p2p.server.BlobAvailabilityHandler import BlobAvailabilityHandlerFactory from lbrynet.p2p.StreamDescriptor import StreamDescriptorIdentifier from lbrynet.p2p.StreamDescriptor import download_sd_blob from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager from lbrynet.p2p.BlobManager import DiskBlobManager -from lbrynet.p2p.PeerManager import PeerManager +from lbrynet.extras.daemon.PeerManager import PeerManager from lbrynet.p2p.RateLimiter import RateLimiter from lbrynet.p2p.server.BlobRequestHandler import BlobRequestHandlerFactory from lbrynet.p2p.server.ServerProtocol import ServerProtocolFactory @@ -18,7 +18,7 @@ from lbrynet.blob.EncryptedFileManager import EncryptedFileManager from lbrynet.blob.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier from tests import mocks -from tests.util import mk_db_and_blob_dir, rm_db_and_blob_dir +from tests.test_utils import mk_db_and_blob_dir, rm_db_and_blob_dir FakeNode = mocks.Node FakeWallet = mocks.Wallet diff --git a/tests/functional/test_reflector.py b/tests/functional/test_reflector.py index c6b4128ac..bac25c3a4 100644 --- a/tests/functional/test_reflector.py +++ b/tests/functional/test_reflector.py @@ -5,7 +5,8 @@ from twisted.internet import defer, error from twisted.trial import unittest from lbrynet.p2p.StreamDescriptor import get_sd_info from lbrynet.extras import reflector -from lbrynet.p2p import BlobManager, PeerManager +from lbrynet.extras.daemon.PeerManager import PeerManager +from lbrynet.p2p import BlobManager from lbrynet.p2p import StreamDescriptor from lbrynet.blob import EncryptedFileCreator from lbrynet.blob.EncryptedFileManager import EncryptedFileManager @@ -13,7 +14,7 @@ from lbrynet.p2p.RateLimiter import DummyRateLimiter from lbrynet.extras.daemon.storage import SQLiteStorage from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager from tests import mocks -from tests.util import mk_db_and_blob_dir, rm_db_and_blob_dir +from tests.test_utils import mk_db_and_blob_dir, rm_db_and_blob_dir class TestReflector(unittest.TestCase): diff --git a/tests/functional/test_streamify.py b/tests/functional/test_streamify.py index a044624b7..52008aeb9 100644 --- a/tests/functional/test_streamify.py +++ b/tests/functional/test_streamify.py @@ -7,8 +7,8 @@ from twisted.internet import defer, threads from lbrynet.p2p.StreamDescriptor import StreamDescriptorIdentifier from lbrynet.p2p.BlobManager import DiskBlobManager from lbrynet.p2p.StreamDescriptor import get_sd_info -from lbrynet.p2p.PeerManager import PeerManager from lbrynet.p2p.RateLimiter import DummyRateLimiter +from lbrynet.extras.daemon.PeerManager import PeerManager from lbrynet.extras.daemon.storage import SQLiteStorage from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager from lbrynet.blob.EncryptedFileCreator import create_lbry_file diff --git a/tests/integration/cli/test_cli.py b/tests/integration/cli/test_cli.py index cd652b2f7..d2861ec87 100644 --- a/tests/integration/cli/test_cli.py +++ b/tests/integration/cli/test_cli.py @@ -4,7 +4,7 @@ from io import StringIO from twisted.internet import defer from lbrynet import conf -from lbrynet import cli +from lbrynet.extras import cli from lbrynet.extras.daemon.Components import DATABASE_COMPONENT, BLOB_COMPONENT, HEADERS_COMPONENT, WALLET_COMPONENT, \ DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, \ PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, \ diff --git a/tests/integration/wallet/test_commands.py b/tests/integration/wallet/test_commands.py index 65e095515..0bdc2cf3c 100644 --- a/tests/integration/wallet/test_commands.py +++ b/tests/integration/wallet/test_commands.py @@ -15,7 +15,7 @@ from torba.testcase import IntegrationTestCase as BaseIntegrationTestCase import lbrynet.schema lbrynet.schema.BLOCKCHAIN_NAME = 'lbrycrd_regtest' -from lbrynet.extras.daemon import conf as lbry_conf +from lbrynet import conf as lbry_conf from lbrynet.dht.node import Node from lbrynet.extras.daemon.Daemon import Daemon from lbrynet.extras.wallet import LbryWalletManager diff --git a/tests/mocks.py b/tests/mocks.py index 8e9be2294..47ec2e583 100644 --- a/tests/mocks.py +++ b/tests/mocks.py @@ -7,13 +7,13 @@ from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives import serialization from twisted.internet import defer from twisted.python.failure import Failure - +from lbrynet import conf from lbrynet.p2p.client.ClientRequest import ClientRequest from lbrynet.p2p.Error import RequestCanceledError from lbrynet.p2p import BlobAvailability from lbrynet.blob.EncryptedFileManager import EncryptedFileManager from lbrynet.dht.node import Node as RealNode -from lbrynet.extras.daemon import ExchangeRateManager as ERM, conf +from lbrynet.extras.daemon import ExchangeRateManager as ERM KB = 2**10 PUBLIC_EXPONENT = 65537 # http://www.daemonology.net/blog/2009-06-11-cryptographic-right-answers.html diff --git a/tests/util.py b/tests/test_utils.py similarity index 91% rename from tests/util.py rename to tests/test_utils.py index 8b65b7033..c03de4745 100644 --- a/tests/util.py +++ b/tests/test_utils.py @@ -32,13 +32,14 @@ def resetTime(test_case, timestamp=DEFAULT_TIMESTAMP): patcher.start().return_value = iso_time test_case.addCleanup(patcher.stop) - patcher = mock.patch('lbrynet.p2p.utils.now') + patcher = mock.patch('lbrynet.utils.now') patcher.start().return_value = timestamp test_case.addCleanup(patcher.stop) - patcher = mock.patch('lbrynet.p2p.utils.utcnow') + patcher = mock.patch('lbrynet.utils.utcnow') patcher.start().return_value = timestamp test_case.addCleanup(patcher.stop) + def is_android(): return 'ANDROID_ARGUMENT' in os.environ # detect Android using the Kivy way diff --git a/tests/unit/core/client/test_ConnectionManager.py b/tests/unit/core/client/test_ConnectionManager.py index b8c0b301d..7a9d95069 100644 --- a/tests/unit/core/client/test_ConnectionManager.py +++ b/tests/unit/core/client/test_ConnectionManager.py @@ -1,17 +1,16 @@ +from twisted.trial.unittest import TestCase +from twisted.internet import defer, reactor, task +from twisted.internet.task import deferLater +from twisted.internet.protocol import ServerFactory + +from lbrynet import conf, utils from lbrynet.p2p.client.ClientRequest import ClientRequest from lbrynet.p2p.server.ServerProtocol import ServerProtocol from lbrynet.p2p.client.ClientProtocol import ClientProtocol from lbrynet.p2p.RateLimiter import RateLimiter from lbrynet.p2p.Peer import Peer -from lbrynet.p2p.PeerManager import PeerManager from lbrynet.p2p.Error import NoResponseError - -from twisted.trial.unittest import TestCase -from twisted.internet import defer, reactor, task -from twisted.internet.task import deferLater -from twisted.internet.protocol import ServerFactory -from lbrynet.extras.daemon import conf -from lbrynet.p2p import utils +from lbrynet.extras.daemon.PeerManager import PeerManager PEER_PORT = 5551 LOCAL_HOST = '127.0.0.1' diff --git a/tests/unit/core/test_BlobManager.py b/tests/unit/core/test_BlobManager.py index 590cb6225..70338ff1f 100644 --- a/tests/unit/core/test_BlobManager.py +++ b/tests/unit/core/test_BlobManager.py @@ -6,12 +6,12 @@ import string from twisted.trial import unittest from twisted.internet import defer -from tests.util import random_lbry_hash +from tests.test_utils import random_lbry_hash from lbrynet.p2p.BlobManager import DiskBlobManager from lbrynet.extras.daemon.storage import SQLiteStorage from lbrynet.p2p.Peer import Peer -from lbrynet.extras.daemon import conf -from lbrynet.p2p.cryptoutils import get_lbry_hash_obj +from lbrynet import conf +from lbrynet.cryptoutils import get_lbry_hash_obj class BlobManagerTest(unittest.TestCase): diff --git a/tests/unit/core/test_HTTPBlobDownloader.py b/tests/unit/core/test_HTTPBlobDownloader.py index 40fb98452..5112e38aa 100644 --- a/tests/unit/core/test_HTTPBlobDownloader.py +++ b/tests/unit/core/test_HTTPBlobDownloader.py @@ -5,7 +5,7 @@ from twisted.internet import defer from lbrynet.blob import BlobFile from lbrynet.p2p.HTTPBlobDownloader import HTTPBlobDownloader -from tests.util import mk_db_and_blob_dir, rm_db_and_blob_dir +from tests.test_utils import mk_db_and_blob_dir, rm_db_and_blob_dir class HTTPBlobDownloaderTest(unittest.TestCase): diff --git a/tests/unit/core/test_HashBlob.py b/tests/unit/core/test_HashBlob.py index 5e7ab86ff..d7c5003c6 100644 --- a/tests/unit/core/test_HashBlob.py +++ b/tests/unit/core/test_HashBlob.py @@ -1,7 +1,7 @@ from lbrynet.blob import BlobFile from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError -from tests.util import mk_db_and_blob_dir, rm_db_and_blob_dir, random_lbry_hash +from tests.test_utils import mk_db_and_blob_dir, rm_db_and_blob_dir, random_lbry_hash from twisted.trial import unittest from twisted.internet import defer diff --git a/tests/unit/core/test_Wallet.py b/tests/unit/core/test_Wallet.py index 3083be6f4..d5fa5e193 100644 --- a/tests/unit/core/test_Wallet.py +++ b/tests/unit/core/test_Wallet.py @@ -5,9 +5,6 @@ from decimal import Decimal from twisted.trial import unittest from twisted.internet import defer from lbrynet.p2p.Error import InsufficientFundsError -#from lbrynet.p2p.Wallet import LBRYumWallet, ReservedPoints -#from lbryum.commands import Commands -#from lbryum.simple_config import SimpleConfig from lbrynet.schema.claim import ClaimDict test_metadata = { diff --git a/tests/unit/core/test_utils.py b/tests/unit/core/test_utils.py index c4f1289e2..bfb5f8b86 100644 --- a/tests/unit/core/test_utils.py +++ b/tests/unit/core/test_utils.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from lbrynet.p2p import utils +from lbrynet import utils from twisted.trial import unittest diff --git a/tests/unit/database/test_SQLiteStorage.py b/tests/unit/database/test_SQLiteStorage.py index 318a05a8e..e8794bbb7 100644 --- a/tests/unit/database/test_SQLiteStorage.py +++ b/tests/unit/database/test_SQLiteStorage.py @@ -5,10 +5,10 @@ import logging from copy import deepcopy from twisted.internet import defer from twisted.trial import unittest -from lbrynet.extras.daemon import conf +from lbrynet import conf from lbrynet.extras.daemon.storage import SQLiteStorage, open_file_for_writing from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloader -from tests.util import random_lbry_hash +from tests.test_utils import random_lbry_hash log = logging.getLogger() diff --git a/tests/unit/dht/test_contact.py b/tests/unit/dht/test_contact.py index 3543854d9..2721a8fe7 100644 --- a/tests/unit/dht/test_contact.py +++ b/tests/unit/dht/test_contact.py @@ -1,7 +1,7 @@ from binascii import hexlify from twisted.internet import task from twisted.trial import unittest -from lbrynet.p2p.utils import generate_id +from lbrynet.utils import generate_id from lbrynet.dht.contact import ContactManager from lbrynet.dht import constants diff --git a/tests/unit/dht/test_hash_announcer.py b/tests/unit/dht/test_hash_announcer.py index 5aeb54943..5d7179712 100644 --- a/tests/unit/dht/test_hash_announcer.py +++ b/tests/unit/dht/test_hash_announcer.py @@ -1,8 +1,8 @@ from twisted.trial import unittest from twisted.internet import defer, task -from lbrynet.p2p import utils +from lbrynet import utils from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer -from tests.util import random_lbry_hash +from tests.test_utils import random_lbry_hash from tests.mocks import mock_conf_settings diff --git a/tests/unit/dht/test_kbucket.py b/tests/unit/dht/test_kbucket.py index b746afa7b..360dee32d 100644 --- a/tests/unit/dht/test_kbucket.py +++ b/tests/unit/dht/test_kbucket.py @@ -6,7 +6,7 @@ from twisted.trial import unittest import struct -from lbrynet.p2p.utils import generate_id +from lbrynet.utils import generate_id from lbrynet.dht import kbucket from lbrynet.dht.contact import ContactManager from lbrynet.dht import constants diff --git a/tests/unit/dht/test_node.py b/tests/unit/dht/test_node.py index a82cdb803..983975235 100644 --- a/tests/unit/dht/test_node.py +++ b/tests/unit/dht/test_node.py @@ -5,7 +5,7 @@ from twisted.trial import unittest from twisted.internet import defer from lbrynet.dht.node import Node from lbrynet.dht import constants -from lbrynet.p2p.utils import generate_id +from lbrynet.utils import generate_id class NodeIDTest(unittest.TestCase): diff --git a/tests/unit/dht/test_routingtable.py b/tests/unit/dht/test_routingtable.py index 37131b53f..1c7985cc8 100644 --- a/tests/unit/dht/test_routingtable.py +++ b/tests/unit/dht/test_routingtable.py @@ -6,7 +6,7 @@ from lbrynet.dht import constants from lbrynet.dht.routingtable import TreeRoutingTable from lbrynet.dht.contact import ContactManager from lbrynet.dht.distance import Distance -from lbrynet.p2p.utils import generate_id +from lbrynet.utils import generate_id class FakeRPCProtocol: diff --git a/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py b/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py index 8266f29bf..36d6db299 100644 --- a/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py +++ b/tests/unit/lbryfilemanager/test_EncryptedFileCreator.py @@ -3,10 +3,10 @@ from twisted.trial import unittest from twisted.internet import defer from cryptography.hazmat.primitives.ciphers.algorithms import AES +from lbrynet.extras.daemon.PeerManager import PeerManager from lbrynet.p2p.StreamDescriptor import get_sd_info, BlobStreamDescriptorReader from lbrynet.p2p.StreamDescriptor import StreamDescriptorIdentifier from lbrynet.p2p.BlobManager import DiskBlobManager -from lbrynet.p2p.PeerManager import PeerManager from lbrynet.p2p.RateLimiter import DummyRateLimiter from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager from lbrynet.extras.daemon.storage import SQLiteStorage @@ -14,7 +14,7 @@ from lbrynet.blob import EncryptedFileCreator from lbrynet.blob.EncryptedFileManager import EncryptedFileManager from lbrynet.p2p.StreamDescriptor import JSONBytesEncoder from tests import mocks -from tests.util import mk_db_and_blob_dir, rm_db_and_blob_dir +from tests.test_utils import mk_db_and_blob_dir, rm_db_and_blob_dir FakeNode = mocks.Node diff --git a/tests/unit/lbrynet_daemon/auth/test_server.py b/tests/unit/lbrynet_daemon/auth/test_server.py index 69b1a12be..849a2634c 100644 --- a/tests/unit/lbrynet_daemon/auth/test_server.py +++ b/tests/unit/lbrynet_daemon/auth/test_server.py @@ -1,7 +1,7 @@ import mock from twisted.internet import reactor from twisted.trial import unittest -from lbrynet.extras.daemon import conf +from lbrynet import conf from lbrynet.extras.daemon.auth import server from tests.mocks import mock_conf_settings diff --git a/tests/unit/lbrynet_daemon/test_Daemon.py b/tests/unit/lbrynet_daemon/test_Daemon.py index eed2d60a8..fd8b6d5ce 100644 --- a/tests/unit/lbrynet_daemon/test_Daemon.py +++ b/tests/unit/lbrynet_daemon/test_Daemon.py @@ -7,9 +7,8 @@ from twisted.internet import defer from twisted.trial import unittest from faker import Faker - +from lbrynet import conf from lbrynet.schema.decode import smart_decode -from lbrynet.extras.daemon import conf from lbrynet.extras.daemon.storage import SQLiteStorage from lbrynet.extras.daemon.ComponentManager import ComponentManager from lbrynet.extras.daemon.Components import DATABASE_COMPONENT, DHT_COMPONENT, WALLET_COMPONENT @@ -23,24 +22,19 @@ from lbrynet.extras.wallet import LbryWalletManager from torba.client.wallet import Wallet from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager -from tests import util +from tests import test_utils from tests.mocks import mock_conf_settings, FakeNetwork, FakeFileManager from tests.mocks import ExchangeRateManager as DummyExchangeRateManager from tests.mocks import BTCLBCFeed, USDBTCFeed -from tests.util import is_android - - -import logging -logging.getLogger("lbryum").setLevel(logging.WARNING) +from tests.test_utils import is_android def get_test_daemon(data_rate=None, generous=True, with_fee=False): if data_rate is None: data_rate = conf.ADJUSTABLE_SETTINGS['data_rate'][1] - rates = { - 'BTCLBC': {'spot': 3.0, 'ts': util.DEFAULT_ISO_TIME + 1}, - 'USDBTC': {'spot': 2.0, 'ts': util.DEFAULT_ISO_TIME + 2} + 'BTCLBC': {'spot': 3.0, 'ts': test_utils.DEFAULT_ISO_TIME + 1}, + 'USDBTC': {'spot': 2.0, 'ts': test_utils.DEFAULT_ISO_TIME + 2} } component_manager = ComponentManager( skip_components=[DATABASE_COMPONENT, DHT_COMPONENT, WALLET_COMPONENT, UPNP_COMPONENT, @@ -89,7 +83,7 @@ class TestCostEst(unittest.TestCase): def setUp(self): mock_conf_settings(self) - util.resetTime(self) + test_utils.resetTime(self) @defer.inlineCallbacks def test_fee_and_generous_data(self): @@ -134,7 +128,7 @@ class TestJsonRpc(unittest.TestCase): return None mock_conf_settings(self) - util.resetTime(self) + test_utils.resetTime(self) self.test_daemon = get_test_daemon() self.test_daemon.wallet_manager.is_first_run = False self.test_daemon.wallet_manager.get_best_blockhash = noop @@ -156,7 +150,7 @@ class TestFileListSorting(unittest.TestCase): def setUp(self): mock_conf_settings(self) - util.resetTime(self) + test_utils.resetTime(self) self.faker = Faker('en_US') self.faker.seed(129) # contains 3 same points paid (5.9) self.test_daemon = get_test_daemon() diff --git a/tests/unit/lbrynet_daemon/test_Downloader.py b/tests/unit/lbrynet_daemon/test_Downloader.py index f930d2d5e..aaf600b5e 100644 --- a/tests/unit/lbrynet_daemon/test_Downloader.py +++ b/tests/unit/lbrynet_daemon/test_Downloader.py @@ -12,7 +12,7 @@ from lbrynet.p2p.RateLimiter import DummyRateLimiter from lbrynet.extras.daemon import Downloader from lbrynet.extras.daemon import ExchangeRateManager from lbrynet.extras.daemon.storage import SQLiteStorage -from lbrynet.dht.peerfinder import DummyPeerFinder +from lbrynet.extras.daemon.PeerFinder import DummyPeerFinder from lbrynet.blob.EncryptedFileStatusReport import EncryptedFileStatusReport from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloader from lbrynet.extras.wallet import LbryWalletManager diff --git a/tests/unit/lbrynet_daemon/test_ExchangeRateManager.py b/tests/unit/lbrynet_daemon/test_ExchangeRateManager.py index b3f2f1b3e..651d3c35e 100644 --- a/tests/unit/lbrynet_daemon/test_ExchangeRateManager.py +++ b/tests/unit/lbrynet_daemon/test_ExchangeRateManager.py @@ -3,7 +3,7 @@ from lbrynet.extras.daemon import ExchangeRateManager from lbrynet.p2p.Error import InvalidExchangeRateResponse from twisted.trial import unittest from twisted.internet import defer -from tests import util +from tests import test_utils from tests.mocks import ExchangeRateManager as DummyExchangeRateManager from tests.mocks import BTCLBCFeed, USDBTCFeed @@ -32,18 +32,18 @@ class FeeFormatTest(unittest.TestCase): class ExchangeRateTest(unittest.TestCase): def setUp(self): - util.resetTime(self) + test_utils.resetTime(self) def test_invalid_rates(self): with self.assertRaises(ValueError): - ExchangeRateManager.ExchangeRate('USDBTC', 0, util.DEFAULT_ISO_TIME) + ExchangeRateManager.ExchangeRate('USDBTC', 0, test_utils.DEFAULT_ISO_TIME) with self.assertRaises(ValueError): - ExchangeRateManager.ExchangeRate('USDBTC', -1, util.DEFAULT_ISO_TIME) + ExchangeRateManager.ExchangeRate('USDBTC', -1, test_utils.DEFAULT_ISO_TIME) class FeeTest(unittest.TestCase): def setUp(self): - util.resetTime(self) + test_utils.resetTime(self) def test_fee_converts_to_lbc(self): fee = Fee({ @@ -53,8 +53,8 @@ class FeeTest(unittest.TestCase): }) rates = { - 'BTCLBC': {'spot': 3.0, 'ts': util.DEFAULT_ISO_TIME + 1}, - 'USDBTC': {'spot': 2.0, 'ts': util.DEFAULT_ISO_TIME + 2} + 'BTCLBC': {'spot': 3.0, 'ts': test_utils.DEFAULT_ISO_TIME + 1}, + 'USDBTC': {'spot': 2.0, 'ts': test_utils.DEFAULT_ISO_TIME + 2} } market_feeds = [BTCLBCFeed(), USDBTCFeed()] @@ -71,7 +71,7 @@ class FeeTest(unittest.TestCase): }) rates = { - 'BTCLBC': {'spot': 1.0, 'ts': util.DEFAULT_ISO_TIME + 1}, + 'BTCLBC': {'spot': 1.0, 'ts': test_utils.DEFAULT_ISO_TIME + 1}, } market_feeds = [BTCLBCFeed()] manager = DummyExchangeRateManager(market_feeds, rates) diff --git a/tests/unit/lbrynet_daemon/test_claims_comparator.py b/tests/unit/lbrynet_daemon/test_claims_comparator.py index afd246900..a1d6dd0aa 100644 --- a/tests/unit/lbrynet_daemon/test_claims_comparator.py +++ b/tests/unit/lbrynet_daemon/test_claims_comparator.py @@ -1,4 +1,4 @@ -import unittest +from twisted.trial import unittest from lbrynet.extras.daemon.Daemon import sort_claim_results diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index d4cbf96a1..a3b49047c 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -3,7 +3,7 @@ from io import StringIO from twisted.trial import unittest from lbrynet.extras.cli import normalize_value, main -from lbrynet.p2p.system_info import get_platform +from lbrynet.system_info import get_platform class CLITest(unittest.TestCase): diff --git a/tests/unit/test_conf.py b/tests/unit/test_conf.py index 8958d64d6..5ab939b6d 100644 --- a/tests/unit/test_conf.py +++ b/tests/unit/test_conf.py @@ -4,7 +4,7 @@ import sys import tempfile from unittest import skipIf from twisted.trial import unittest -from lbrynet.extras.daemon import conf +from lbrynet import conf from lbrynet.p2p.Error import InvalidCurrencyError From bc5824b8bc2ce1b1d257157a37543079791af251 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 7 Nov 2018 15:23:08 -0500 Subject: [PATCH 07/16] better setup of custom logger class --- lbrynet/custom_logger.py | 7 +++++-- lbrynet/log_support.py | 3 +++ lbrynet/p2p/__init__.py | 2 -- tests/unit/test_customLogger.py | 14 ++++++-------- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/lbrynet/custom_logger.py b/lbrynet/custom_logger.py index bd608f1cd..8e3499e42 100644 --- a/lbrynet/custom_logger.py +++ b/lbrynet/custom_logger.py @@ -102,5 +102,8 @@ class Logger(logging.Logger): self._log(TRACE, msg, args, **kwargs) -logging.setLoggerClass(Logger) -logging.addLevelName(TRACE, 'TRACE') +def install_logger(): + current = logging.getLoggerClass() + if current is not Logger: + logging.setLoggerClass(Logger) + logging.addLevelName(TRACE, 'TRACE') diff --git a/lbrynet/log_support.py b/lbrynet/log_support.py index 72cacc2fb..06ab68ead 100644 --- a/lbrynet/log_support.py +++ b/lbrynet/log_support.py @@ -8,6 +8,7 @@ import twisted.python.log from twisted.internet import defer from lbrynet import __version__ as lbrynet_version, build_type from lbrynet import utils, conf +from lbrynet.custom_logger import install_logger class HTTPSHandler(logging.Handler): @@ -212,6 +213,8 @@ def configure_logging(file_name, console, verbose=None): See `convert_verbose` for more details. """ logger = logging.getLoggerClass() + if not hasattr(logger, "fail"): + install_logger() verbose = convert_verbose(verbose) configure_twisted() configure_file_handler(file_name) diff --git a/lbrynet/p2p/__init__.py b/lbrynet/p2p/__init__.py index df7d37558..6ac1f3432 100644 --- a/lbrynet/p2p/__init__.py +++ b/lbrynet/p2p/__init__.py @@ -5,5 +5,3 @@ This includes classes for connecting to other peers and downloading blobs from t connections from peers and responding to their requests, managing locally stored blobs, sending and receiving payments, and locating peers in the DHT. """ - -from lbrynet import custom_logger diff --git a/tests/unit/test_customLogger.py b/tests/unit/test_customLogger.py index 051b20185..77ad3b8e4 100644 --- a/tests/unit/test_customLogger.py +++ b/tests/unit/test_customLogger.py @@ -1,16 +1,14 @@ from io import StringIO import logging - import mock -import unittest +from unittest import skipIf from twisted.internet import defer -from twisted import trial - +from twisted.trial import unittest from lbrynet import custom_logger -from tests.util import is_android +from tests.test_utils import is_android -class TestLogger(trial.unittest.TestCase): +class TestLogger(unittest.TestCase): def raiseError(self): raise Exception('terrible things happened') @@ -28,7 +26,7 @@ class TestLogger(trial.unittest.TestCase): handler.setFormatter(logging.Formatter("%(filename)s:%(lineno)d - %(message)s")) self.log.addHandler(handler) - @unittest.skipIf(is_android(), + @skipIf(is_android(), 'Test cannot pass on Android because the tests package is compiled ' 'which results in a different method call stack') def test_can_log_failure(self): @@ -36,7 +34,7 @@ class TestLogger(trial.unittest.TestCase): return self.stream.getvalue().split('\n') # the line number could change if this file gets refactored - expected_first_line = 'test_customLogger.py:20 - My message: terrible things happened' + expected_first_line = 'test_customLogger.py:18 - My message: terrible things happened' # testing the entirety of the message is futile as the # traceback will depend on the system the test is being run on From b98621d7ea0686d9092cd0e7a3013c60d4100c7b Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 7 Nov 2018 15:24:05 -0500 Subject: [PATCH 08/16] avoid name errors --- lbrynet/extras/cli.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lbrynet/extras/cli.py b/lbrynet/extras/cli.py index 3fa0dce11..78721617e 100644 --- a/lbrynet/extras/cli.py +++ b/lbrynet/extras/cli.py @@ -29,17 +29,20 @@ from lbrynet.system_info import get_platform async def execute_command(method, params, conf_path=None): # this check if the daemon is running or not + api = None try: api = await LBRYAPIClient.get_client(conf_path) await api.status() except (ClientConnectorError, ConnectionError): - await api.session.close() + if api: + await api.session.close() print("Could not connect to daemon. Are you sure it's running?") return 1 # this actually executes the method + resp = await api.call(method, params) + try: - resp = await api.call(method, params) await api.session.close() print(json.dumps(resp["result"], indent=2)) except KeyError: From 7e27a6e37d8e8c048ba768e800d83b6510bc456a Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 7 Nov 2018 16:12:11 -0500 Subject: [PATCH 09/16] fix --- lbrynet/__init__.py | 3 +++ lbrynet/log_support.py | 4 ---- tests/functional/dht/test_store.py | 2 +- tests/functional/test_reflector.py | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lbrynet/__init__.py b/lbrynet/__init__.py index d4afab105..58a08780a 100644 --- a/lbrynet/__init__.py +++ b/lbrynet/__init__.py @@ -1,7 +1,10 @@ import logging +from lbrynet.custom_logger import install_logger + __name__ = "lbrynet" __version__ = "0.30.1rc1" version = tuple(__version__.split('.')) +install_logger() logging.getLogger(__name__).addHandler(logging.NullHandler()) diff --git a/lbrynet/log_support.py b/lbrynet/log_support.py index 06ab68ead..2b2803f5b 100644 --- a/lbrynet/log_support.py +++ b/lbrynet/log_support.py @@ -8,7 +8,6 @@ import twisted.python.log from twisted.internet import defer from lbrynet import __version__ as lbrynet_version, build_type from lbrynet import utils, conf -from lbrynet.custom_logger import install_logger class HTTPSHandler(logging.Handler): @@ -212,9 +211,6 @@ def configure_logging(file_name, console, verbose=None): verbose: a list of loggers to set to debug level. See `convert_verbose` for more details. """ - logger = logging.getLoggerClass() - if not hasattr(logger, "fail"): - install_logger() verbose = convert_verbose(verbose) configure_twisted() configure_file_handler(file_name) diff --git a/tests/functional/dht/test_store.py b/tests/functional/dht/test_store.py index f3e81b332..b45eef3d7 100644 --- a/tests/functional/dht/test_store.py +++ b/tests/functional/dht/test_store.py @@ -3,7 +3,7 @@ from binascii import hexlify from twisted.internet import defer from lbrynet.dht import constants -from lbrynet.p2p.utils import generate_id +from lbrynet.utils import generate_id from .dht_test_environment import TestKademliaBase import logging diff --git a/tests/functional/test_reflector.py b/tests/functional/test_reflector.py index bac25c3a4..5b9cfc116 100644 --- a/tests/functional/test_reflector.py +++ b/tests/functional/test_reflector.py @@ -26,7 +26,7 @@ class TestReflector(unittest.TestCase): self.client_db_dir, self.client_blob_dir = mk_db_and_blob_dir() prm = OnlyFreePaymentsManager() wallet = mocks.Wallet() - peer_manager = PeerManager.PeerManager() + peer_manager = PeerManager() peer_finder = mocks.PeerFinder(5553, peer_manager, 2) self.server_storage = SQLiteStorage(self.server_db_dir) self.server_blob_manager = BlobManager.DiskBlobManager(self.server_blob_dir, self.server_storage) From 572777305e888fa0181eb1969e82698d9a5a8434 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 7 Nov 2018 18:18:55 -0500 Subject: [PATCH 10/16] reduce analytics event frequency --- lbrynet/extras/daemon/analytics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lbrynet/extras/daemon/analytics.py b/lbrynet/extras/daemon/analytics.py index 907d524f6..015c1277b 100644 --- a/lbrynet/extras/daemon/analytics.py +++ b/lbrynet/extras/daemon/analytics.py @@ -136,8 +136,8 @@ class Manager: def _get_looping_calls(self): return [ - ('send_heartbeat', self._send_heartbeat, 60), - ('update_tracked_metrics', self._update_tracked_metrics, 300), + ('send_heartbeat', self._send_heartbeat, 300), + ('update_tracked_metrics', self._update_tracked_metrics, 600), ] def _setup_looping_calls(self): From 7b32e2ff7cd348b064b7f6d2434aa8df749370d2 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 7 Nov 2018 18:35:00 -0500 Subject: [PATCH 11/16] don't block the dht component on populating the routing table --- lbrynet/dht/node.py | 9 ++++++--- lbrynet/extras/daemon/Components.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index eb929803a..793e46b76 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -246,7 +246,7 @@ class Node(MockKademliaHelper): yield _iterative_join() @defer.inlineCallbacks - def start(self, known_node_addresses=None): + def start(self, known_node_addresses=None, block_on_join=False): """ Causes the Node to attempt to join the DHT network by contacting the known DHT nodes. This can be called multiple times if the previous attempt has failed or if the Node has lost all the contacts. @@ -261,8 +261,11 @@ class Node(MockKademliaHelper): self.start_listening() yield self._protocol._listening # TODO: Refresh all k-buckets further away than this node's closest neighbour - yield self.joinNetwork(known_node_addresses or []) - self.start_looping_calls() + d = self.joinNetwork(known_node_addresses or []) + d.addCallback(lambda _: self.start_looping_calls()) + d.addCallback(lambda _: log.info("Joined the dht")) + if block_on_join: + yield d def start_looping_calls(self): self.safe_start_looping_call(self._change_token_lc, constants.tokenSecretChangeInterval) diff --git a/lbrynet/extras/daemon/Components.py b/lbrynet/extras/daemon/Components.py index 8ba078eef..15158c3ae 100644 --- a/lbrynet/extras/daemon/Components.py +++ b/lbrynet/extras/daemon/Components.py @@ -441,7 +441,7 @@ class DHTComponent(Component): peerPort=self.external_peer_port ) - yield self.dht_node.start(GCS('known_dht_nodes')) + yield self.dht_node.start(GCS('known_dht_nodes'), block_on_join=False) log.info("Started the dht") @defer.inlineCallbacks From e9efd9f68a150870429f6f6077093506ad8abe84 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Wed, 7 Nov 2018 18:35:32 -0500 Subject: [PATCH 12/16] improve get_external_ip --- lbrynet/extras/cli.py | 2 +- lbrynet/extras/daemon/Components.py | 25 ++++++++++++++++++------- lbrynet/extras/daemon/DaemonControl.py | 2 +- lbrynet/system_info.py | 15 +-------------- tests/unit/test_cli.py | 2 +- 5 files changed, 22 insertions(+), 24 deletions(-) diff --git a/lbrynet/extras/cli.py b/lbrynet/extras/cli.py index 78721617e..91ebdb447 100644 --- a/lbrynet/extras/cli.py +++ b/lbrynet/extras/cli.py @@ -140,7 +140,7 @@ def main(argv=None): elif method in ['version', '--version', '-v']: print("{lbrynet_name} {lbrynet_version}".format( - lbrynet_name=lbrynet_name, **get_platform(get_ip=False) + lbrynet_name=lbrynet_name, **get_platform() )) return 0 diff --git a/lbrynet/extras/daemon/Components.py b/lbrynet/extras/daemon/Components.py index 15158c3ae..cbcf06ed5 100644 --- a/lbrynet/extras/daemon/Components.py +++ b/lbrynet/extras/daemon/Components.py @@ -2,6 +2,7 @@ import os import asyncio import logging import treq +import json import math import binascii from hashlib import sha256 @@ -11,7 +12,7 @@ import lbrynet.schema from aioupnp import __version__ as aioupnp_version from aioupnp.upnp import UPnP from aioupnp.fault import UPnPError -from lbrynet import conf, system_info +from lbrynet import conf from lbrynet.utils import DeferredDict, generate_id from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager from lbrynet.p2p.RateLimiter import RateLimiter @@ -93,9 +94,19 @@ class ConfigSettings: return conf.settings.node_id @staticmethod - def get_external_ip(): - platform = system_info.get_platform(get_ip=True) - return platform['ip'] + @defer.inlineCallbacks + def get_external_ip(): # used if upnp is disabled or non-functioning + try: + buf = [] + response = yield treq.get("https://api.lbry.io/ip") + yield treq.collect(response, buf.append) + parsed = json.loads(b"".join(buf).decode()) + if parsed['success']: + return parsed['data']['ip'] + return + except Exception as err: + return + # Shorthand for common ConfigSettings methods @@ -429,7 +440,7 @@ class DHTComponent(Component): external_ip = self.upnp_component.external_ip if not external_ip: log.warning("UPnP component failed to get external ip") - external_ip = CS.get_external_ip() + external_ip = yield CS.get_external_ip() if not external_ip: log.warning("failed to get external ip") @@ -705,7 +716,7 @@ class UPnPComponent(Component): if external_ip == "0.0.0.0" or not external_ip: log.warning("unable to get external ip from UPnP, checking lbry.io fallback") - external_ip = CS.get_external_ip() + external_ip = yield CS.get_external_ip() if self.external_ip and self.external_ip != external_ip: log.info("external ip changed from %s to %s", self.external_ip, external_ip) self.external_ip = external_ip @@ -761,7 +772,7 @@ class UPnPComponent(Component): def start(self): log.info("detecting external ip") if not self.use_upnp: - self.external_ip = CS.get_external_ip() + self.external_ip = yield CS.get_external_ip() return success = False yield self._maintain_redirects() diff --git a/lbrynet/extras/daemon/DaemonControl.py b/lbrynet/extras/daemon/DaemonControl.py index 1cb0a8a4a..8542c86c4 100644 --- a/lbrynet/extras/daemon/DaemonControl.py +++ b/lbrynet/extras/daemon/DaemonControl.py @@ -53,7 +53,7 @@ def start(argv=None, conf_path=None): conf.settings.update({'use_auth_http': args.useauth}, data_types=(conf.TYPE_CLI,)) if args.version: - version = system_info.get_platform(get_ip=False) + version = system_info.get_platform() version['installation_id'] = conf.settings.installation_id print(utils.json_dumps_pretty(version)) return diff --git a/lbrynet/system_info.py b/lbrynet/system_info.py index 1c1e3c83a..7e42c25b8 100644 --- a/lbrynet/system_info.py +++ b/lbrynet/system_info.py @@ -1,9 +1,6 @@ import platform -import json import os -from six.moves.urllib import request -from six.moves.urllib.error import URLError from lbrynet.schema import __version__ as schema_version from lbrynet import build_type, __version__ as lbrynet_version import logging.handlers @@ -15,7 +12,7 @@ def get_lbrynet_version() -> str: return lbrynet_version -def get_platform(get_ip: bool = True) -> dict: +def get_platform() -> dict: p = { "processor": platform.processor(), "python_version": platform.python_version(), @@ -34,14 +31,4 @@ def get_platform(get_ip: bool = True) -> dict: except ModuleNotFoundError: pass - # TODO: remove this from get_platform and add a get_external_ip function using treq - if get_ip: - try: - response = json.loads(request.urlopen("https://api.lbry.io/ip").read()) - if not response['success']: - raise URLError("failed to get external ip") - p['ip'] = response['data']['ip'] - except (URLError, AssertionError): - p['ip'] = "Could not determine IP" - return p diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index a3b49047c..267dcbc15 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -67,7 +67,7 @@ class CLITest(unittest.TestCase): main(['version']) self.assertEqual( actual_output.getvalue().strip(), - "lbrynet {lbrynet_version}".format(**get_platform(get_ip=False)) + "lbrynet {lbrynet_version}".format(**get_platform()) ) def test_invalid_command(self): From 88095af1cdfce9bebb1f54ae4bc00a95083806f0 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Fri, 9 Nov 2018 13:06:02 -0500 Subject: [PATCH 13/16] move system_info.py into lbrynet.extras --- lbrynet/extras/cli.py | 2 +- lbrynet/extras/daemon/Daemon.py | 6 +++--- lbrynet/extras/daemon/DaemonControl.py | 5 ++--- lbrynet/extras/daemon/analytics.py | 4 ++-- lbrynet/{ => extras}/system_info.py | 6 +----- tests/unit/test_cli.py | 2 +- 6 files changed, 10 insertions(+), 15 deletions(-) rename lbrynet/{ => extras}/system_info.py (88%) diff --git a/lbrynet/extras/cli.py b/lbrynet/extras/cli.py index 91ebdb447..1d45ddaee 100644 --- a/lbrynet/extras/cli.py +++ b/lbrynet/extras/cli.py @@ -24,7 +24,7 @@ from lbrynet.extras.daemon.Daemon import Daemon from lbrynet.extras.daemon.DaemonControl import start as daemon_main from lbrynet.extras.daemon.DaemonConsole import main as daemon_console from lbrynet.extras.daemon.auth.client import LBRYAPIClient -from lbrynet.system_info import get_platform +from lbrynet.extras.system_info import get_platform async def execute_command(method, params, conf_path=None): diff --git a/lbrynet/extras/daemon/Daemon.py b/lbrynet/extras/daemon/Daemon.py index 00b4c747c..ddb771bd0 100644 --- a/lbrynet/extras/daemon/Daemon.py +++ b/lbrynet/extras/daemon/Daemon.py @@ -16,8 +16,8 @@ from twisted.python.failure import Failure from torba.client.baseaccount import SingleKey, HierarchicalDeterministic -from lbrynet import conf, utils, system_info -from lbrynet.system_info import get_lbrynet_version +from lbrynet import conf, utils, __version__ +from lbrynet.extras import system_info from lbrynet.schema.claim import ClaimDict from lbrynet.schema.uri import parse_lbry_uri from lbrynet.schema.error import URIParseError, DecodeError @@ -844,7 +844,7 @@ class Daemon(AuthJSONRPCServer): message, conf.settings.installation_id, platform_name, - get_lbrynet_version() + __version__ ) return self._render_response(True) diff --git a/lbrynet/extras/daemon/DaemonControl.py b/lbrynet/extras/daemon/DaemonControl.py index 8542c86c4..bdaa71dc8 100644 --- a/lbrynet/extras/daemon/DaemonControl.py +++ b/lbrynet/extras/daemon/DaemonControl.py @@ -7,14 +7,13 @@ if 'win' in sys.platform: import certifi os.environ['SSL_CERT_FILE'] = certifi.where() -from lbrynet import log_support import argparse import logging.handlers from twisted.internet import reactor - -from lbrynet import utils, system_info, conf +from lbrynet import utils, conf, log_support +from lbrynet.extras import system_info from .Daemon import Daemon log = logging.getLogger(__name__) diff --git a/lbrynet/extras/daemon/analytics.py b/lbrynet/extras/daemon/analytics.py index 015c1277b..0208d6757 100644 --- a/lbrynet/extras/daemon/analytics.py +++ b/lbrynet/extras/daemon/analytics.py @@ -4,8 +4,8 @@ import logging import treq from twisted.internet import defer, task -from lbrynet import conf, utils, system_info -from lbrynet.extras import looping_call_manager +from lbrynet import conf, utils +from lbrynet.extras import looping_call_manager, system_info # Things We Track SERVER_STARTUP = 'Server Startup' diff --git a/lbrynet/system_info.py b/lbrynet/extras/system_info.py similarity index 88% rename from lbrynet/system_info.py rename to lbrynet/extras/system_info.py index 7e42c25b8..d3552d4f7 100644 --- a/lbrynet/system_info.py +++ b/lbrynet/extras/system_info.py @@ -8,10 +8,6 @@ import logging.handlers log = logging.getLogger(__name__) -def get_lbrynet_version() -> str: - return lbrynet_version - - def get_platform() -> dict: p = { "processor": platform.processor(), @@ -19,7 +15,7 @@ def get_platform() -> dict: "platform": platform.platform(), "os_release": platform.release(), "os_system": platform.system(), - "lbrynet_version": get_lbrynet_version(), + "lbrynet_version": lbrynet_version, "lbryschema_version": schema_version, "build": build_type.BUILD, # CI server sets this during build step } diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 267dcbc15..772b3ae09 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -3,7 +3,7 @@ from io import StringIO from twisted.trial import unittest from lbrynet.extras.cli import normalize_value, main -from lbrynet.system_info import get_platform +from lbrynet.extras.system_info import get_platform class CLITest(unittest.TestCase): From 1b4230b06e06cc59050934c48ddd80a165b7bfb4 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Fri, 9 Nov 2018 13:20:58 -0500 Subject: [PATCH 14/16] move is_valid_blobhash to lbrynet.blob.blob_file --- lbrynet/blob/blob_file.py | 20 ++++++++++++++++++- lbrynet/extras/reflector/server/server.py | 2 +- .../p2p/client/StandaloneBlobDownloader.py | 3 ++- lbrynet/utils.py | 18 ----------------- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/lbrynet/blob/blob_file.py b/lbrynet/blob/blob_file.py index 5f9173969..728962971 100644 --- a/lbrynet/blob/blob_file.py +++ b/lbrynet/blob/blob_file.py @@ -3,8 +3,8 @@ import logging from twisted.internet import defer, threads from twisted.web.client import FileBodyProducer from twisted.python.failure import Failure +from lbrynet.cryptoutils import get_lbry_hash_obj from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError, InvalidBlobHashError -from lbrynet.utils import is_valid_blobhash from lbrynet.blob.writer import HashBlobWriter from lbrynet.blob.reader import HashBlobReader @@ -12,6 +12,24 @@ log = logging.getLogger(__name__) MAX_BLOB_SIZE = 2 * 2 ** 20 +# digest_size is in bytes, and blob hashes are hex encoded +blobhash_length = get_lbry_hash_obj().digest_size * 2 + + +def is_valid_hashcharacter(char): + return char in "0123456789abcdef" + + +def is_valid_blobhash(blobhash): + """Checks whether the blobhash is the correct length and contains only + valid characters (0-9, a-f) + + @param blobhash: string, the blobhash to check + + @return: True/False + """ + return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash) + class BlobFile: """ diff --git a/lbrynet/extras/reflector/server/server.py b/lbrynet/extras/reflector/server/server.py index c1d5f9ac1..f51869e38 100644 --- a/lbrynet/extras/reflector/server/server.py +++ b/lbrynet/extras/reflector/server/server.py @@ -3,7 +3,7 @@ import json from twisted.python import failure from twisted.internet import error, defer from twisted.internet.protocol import Protocol, ServerFactory -from lbrynet.utils import is_valid_blobhash +from lbrynet.blob.blob_file import is_valid_blobhash from lbrynet.p2p.Error import DownloadCanceledError, InvalidBlobHashError from lbrynet.p2p.StreamDescriptor import BlobStreamDescriptorReader from lbrynet.p2p.StreamDescriptor import save_sd_info diff --git a/lbrynet/p2p/client/StandaloneBlobDownloader.py b/lbrynet/p2p/client/StandaloneBlobDownloader.py index 6715f10b3..c9434c98c 100644 --- a/lbrynet/p2p/client/StandaloneBlobDownloader.py +++ b/lbrynet/p2p/client/StandaloneBlobDownloader.py @@ -4,7 +4,8 @@ from lbrynet.p2p.client.BlobRequester import BlobRequester from lbrynet.p2p.client.ConnectionManager import ConnectionManager from lbrynet.p2p.client.DownloadManager import DownloadManager from lbrynet.p2p.Error import InvalidBlobHashError, DownloadSDTimeout -from lbrynet.utils import is_valid_blobhash, safe_start_looping_call, safe_stop_looping_call +from lbrynet.blob.blob_file import is_valid_blobhash +from lbrynet.utils import safe_start_looping_call, safe_stop_looping_call from twisted.python.failure import Failure from twisted.internet import defer from twisted.internet.task import LoopingCall diff --git a/lbrynet/utils.py b/lbrynet/utils.py index 52a33e581..114291102 100644 --- a/lbrynet/utils.py +++ b/lbrynet/utils.py @@ -16,9 +16,6 @@ from lbrynet.cryptoutils import get_lbry_hash_obj log = logging.getLogger(__name__) -# digest_size is in bytes, and blob hashes are hex encoded -blobhash_length = get_lbry_hash_obj().digest_size * 2 - # defining these time functions here allows for easier overriding in testing def now(): @@ -70,21 +67,6 @@ def generate_id(num=None): return h.digest() -def is_valid_hashcharacter(char): - return char in "0123456789abcdef" - - -def is_valid_blobhash(blobhash): - """Checks whether the blobhash is the correct length and contains only - valid characters (0-9, a-f) - - @param blobhash: string, the blobhash to check - - @return: True/False - """ - return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash) - - def version_is_greater_than(a, b): """Returns True if version a is more recent than version b""" return pkg_resources.parse_version(a) > pkg_resources.parse_version(b) From d704336f205b6e95a5fec7086e2aeab468517ab8 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Fri, 9 Nov 2018 14:02:03 -0500 Subject: [PATCH 15/16] absolute imports --- lbrynet/blob/__init__.py | 4 -- lbrynet/blob/client/CryptStreamDownloader.py | 1 - lbrynet/conf.py | 2 +- lbrynet/dht/contact.py | 2 +- lbrynet/dht/datastore.py | 2 +- lbrynet/dht/distance.py | 2 +- lbrynet/dht/encoding.py | 2 +- lbrynet/dht/iterativefind.py | 6 +-- lbrynet/dht/kbucket.py | 6 +-- lbrynet/dht/msgformat.py | 2 +- lbrynet/dht/msgtypes.py | 2 +- lbrynet/dht/node.py | 13 +++--- lbrynet/dht/protocol.py | 8 +--- lbrynet/dht/routingtable.py | 13 +++--- lbrynet/extras/daemon/Component.py | 2 +- lbrynet/extras/daemon/ComponentManager.py | 4 +- lbrynet/extras/daemon/Components.py | 29 +++++++------ lbrynet/extras/daemon/Daemon.py | 41 +++++++++---------- lbrynet/extras/daemon/DaemonConsole.py | 7 ++-- lbrynet/extras/daemon/DaemonControl.py | 2 +- lbrynet/extras/daemon/Downloader.py | 2 +- lbrynet/extras/daemon/__init__.py | 2 +- lbrynet/extras/daemon/auth/auth.py | 2 +- lbrynet/extras/daemon/auth/client.py | 2 +- lbrynet/extras/daemon/auth/factory.py | 4 +- lbrynet/extras/daemon/auth/keyring.py | 3 +- lbrynet/extras/daemon/auth/server.py | 9 ++-- lbrynet/extras/daemon/migrator/migrate5to6.py | 2 +- lbrynet/extras/daemon/storage.py | 3 +- lbrynet/extras/reflector/__init__.py | 3 -- lbrynet/extras/reflector/reupload.py | 7 ++-- lbrynet/extras/system_info.py | 2 +- lbrynet/extras/wallet/__init__.py | 6 +-- lbrynet/extras/wallet/ledger.py | 17 ++++---- lbrynet/extras/wallet/manager.py | 10 ++--- lbrynet/extras/wallet/resolve.py | 3 +- .../extras/wallet/server/block_processor.py | 2 +- lbrynet/extras/wallet/server/coin.py | 2 +- lbrynet/extras/wallet/server/db.py | 2 +- lbrynet/extras/wallet/server/opcodes.py | 2 +- lbrynet/extras/wallet/server/session.py | 4 +- lbrynet/extras/wallet/server/tx.py | 4 +- lbrynet/extras/wallet/transaction.py | 7 ++-- lbrynet/p2p/BlobAvailability.py | 3 +- lbrynet/p2p/HTTPBlobDownloader.py | 5 ++- lbrynet/p2p/PaymentRateManager.py | 4 +- lbrynet/p2p/SinglePeerDownloader.py | 2 +- lbrynet/p2p/StreamDescriptor.py | 6 +-- lbrynet/schema/decode.py | 3 +- tests/functional/test_reflector.py | 16 ++++---- tests/unit/core/test_HTTPBlobDownloader.py | 2 +- tests/unit/core/test_HashBlob.py | 2 +- 52 files changed, 139 insertions(+), 154 deletions(-) diff --git a/lbrynet/blob/__init__.py b/lbrynet/blob/__init__.py index 3c5de8fa9..e69de29bb 100644 --- a/lbrynet/blob/__init__.py +++ b/lbrynet/blob/__init__.py @@ -1,4 +0,0 @@ -from .blob_file import BlobFile -from .creator import BlobFileCreator -from .writer import HashBlobWriter -from .reader import HashBlobReader diff --git a/lbrynet/blob/client/CryptStreamDownloader.py b/lbrynet/blob/client/CryptStreamDownloader.py index 985e399b7..ae14b0e6d 100644 --- a/lbrynet/blob/client/CryptStreamDownloader.py +++ b/lbrynet/blob/client/CryptStreamDownloader.py @@ -10,7 +10,6 @@ from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager from lbrynet.blob.client.CryptBlobHandler import CryptBlobHandler - log = logging.getLogger(__name__) diff --git a/lbrynet/conf.py b/lbrynet/conf.py index 72ba2b700..757aced9b 100644 --- a/lbrynet/conf.py +++ b/lbrynet/conf.py @@ -15,7 +15,7 @@ from lbrynet.androidhelpers.paths import ( ) try: - from .winpaths import get_path, FOLDERID, UserHandle + from lbrynet.winpaths import get_path, FOLDERID, UserHandle except (ImportError, ValueError, NameError): # Android platform: NameError: name 'c_wchar' is not defined pass diff --git a/lbrynet/dht/contact.py b/lbrynet/dht/contact.py index afd2b4718..99c84532b 100644 --- a/lbrynet/dht/contact.py +++ b/lbrynet/dht/contact.py @@ -1,7 +1,7 @@ import ipaddress from binascii import hexlify from functools import reduce -from . import constants +from lbrynet.dht import constants def is_valid_ipv4(address): diff --git a/lbrynet/dht/datastore.py b/lbrynet/dht/datastore.py index 2ae0f393d..12be982f8 100644 --- a/lbrynet/dht/datastore.py +++ b/lbrynet/dht/datastore.py @@ -1,5 +1,5 @@ from collections import UserDict -from . import constants +from lbrynet.dht import constants class DictDataStore(UserDict): diff --git a/lbrynet/dht/distance.py b/lbrynet/dht/distance.py index 94de80433..2c1099535 100644 --- a/lbrynet/dht/distance.py +++ b/lbrynet/dht/distance.py @@ -1,4 +1,4 @@ -from . import constants +from lbrynet.dht import constants class Distance: diff --git a/lbrynet/dht/encoding.py b/lbrynet/dht/encoding.py index f31bd119f..4eb4d0764 100644 --- a/lbrynet/dht/encoding.py +++ b/lbrynet/dht/encoding.py @@ -1,4 +1,4 @@ -from .error import DecodeError +from lbrynet.dht.error import DecodeError def bencode(data): diff --git a/lbrynet/dht/iterativefind.py b/lbrynet/dht/iterativefind.py index 765c548dc..a2be6b09b 100644 --- a/lbrynet/dht/iterativefind.py +++ b/lbrynet/dht/iterativefind.py @@ -1,8 +1,8 @@ import logging from twisted.internet import defer -from .distance import Distance -from .error import TimeoutError -from . import constants +from lbrynet.dht.distance import Distance +from lbrynet.dht.error import TimeoutError +from lbrynet.dht import constants log = logging.getLogger(__name__) diff --git a/lbrynet/dht/kbucket.py b/lbrynet/dht/kbucket.py index 7fffb4ce7..fa3f19d2e 100644 --- a/lbrynet/dht/kbucket.py +++ b/lbrynet/dht/kbucket.py @@ -1,8 +1,8 @@ import logging -from . import constants -from .distance import Distance -from .error import BucketFull +from lbrynet.dht import constants +from lbrynet.dht.distance import Distance +from lbrynet.dht.error import BucketFull log = logging.getLogger(__name__) diff --git a/lbrynet/dht/msgformat.py b/lbrynet/dht/msgformat.py index fc4381d1c..5e4c59d79 100644 --- a/lbrynet/dht/msgformat.py +++ b/lbrynet/dht/msgformat.py @@ -7,7 +7,7 @@ # The docstrings in this module contain epytext markup; API documentation # may be created by processing this file with epydoc: http://epydoc.sf.net -from . import msgtypes +from lbrynet.dht import msgtypes class MessageTranslator: diff --git a/lbrynet/dht/msgtypes.py b/lbrynet/dht/msgtypes.py index 59f9440bd..907c1a8e7 100644 --- a/lbrynet/dht/msgtypes.py +++ b/lbrynet/dht/msgtypes.py @@ -8,7 +8,7 @@ # may be created by processing this file with epydoc: http://epydoc.sf.net from lbrynet.utils import generate_id -from . import constants +from lbrynet.dht import constants class Message: diff --git a/lbrynet/dht/node.py b/lbrynet/dht/node.py index 793e46b76..fc5618d05 100644 --- a/lbrynet/dht/node.py +++ b/lbrynet/dht/node.py @@ -6,14 +6,11 @@ from functools import reduce from twisted.internet import defer, error, task from lbrynet.utils import generate_id, DeferredDict -from .call_later_manager import CallLaterManager -from .error import TimeoutError -from . import constants -from . import routingtable -from . import datastore -from . import protocol -from .contact import ContactManager -from .iterativefind import iterativeFind +from lbrynet.dht.call_later_manager import CallLaterManager +from lbrynet.dht.error import TimeoutError +from lbrynet.dht import constants, routingtable, datastore, protocol +from lbrynet.dht.contact import ContactManager +from lbrynet.dht.iterativefind import iterativeFind log = logging.getLogger(__name__) diff --git a/lbrynet/dht/protocol.py b/lbrynet/dht/protocol.py index 18d55977a..6504baea3 100644 --- a/lbrynet/dht/protocol.py +++ b/lbrynet/dht/protocol.py @@ -3,12 +3,8 @@ import errno from binascii import hexlify from twisted.internet import protocol, defer -from .error import BUILTIN_EXCEPTIONS, UnknownRemoteException, TimeoutError, TransportNotConnected - -from . import constants -from . import encoding -from . import msgtypes -from . import msgformat +from lbrynet.dht import constants, encoding, msgformat, msgtypes +from lbrynet.dht.error import BUILTIN_EXCEPTIONS, UnknownRemoteException, TimeoutError, TransportNotConnected log = logging.getLogger(__name__) diff --git a/lbrynet/dht/routingtable.py b/lbrynet/dht/routingtable.py index 7417e3970..dc8ff8089 100644 --- a/lbrynet/dht/routingtable.py +++ b/lbrynet/dht/routingtable.py @@ -6,14 +6,13 @@ # may be created by processing this file with epydoc: http://epydoc.sf.net import random -from binascii import unhexlify - -from twisted.internet import defer -from . import constants -from . import kbucket -from .error import TimeoutError -from .distance import Distance import logging +from binascii import unhexlify +from twisted.internet import defer + +from lbrynet.dht import constants, kbucket +from lbrynet.dht.error import TimeoutError +from lbrynet.dht.distance import Distance log = logging.getLogger(__name__) diff --git a/lbrynet/extras/daemon/Component.py b/lbrynet/extras/daemon/Component.py index 03f03ddf5..8de53a6ae 100644 --- a/lbrynet/extras/daemon/Component.py +++ b/lbrynet/extras/daemon/Component.py @@ -1,7 +1,7 @@ import logging from twisted.internet import defer from twisted._threads import AlreadyQuit -from .ComponentManager import ComponentManager +from lbrynet.extras.daemon.ComponentManager import ComponentManager log = logging.getLogger(__name__) diff --git a/lbrynet/extras/daemon/ComponentManager.py b/lbrynet/extras/daemon/ComponentManager.py index 3ccf5889b..9af07fd8d 100644 --- a/lbrynet/extras/daemon/ComponentManager.py +++ b/lbrynet/extras/daemon/ComponentManager.py @@ -1,8 +1,8 @@ import logging from twisted.internet import defer from lbrynet.p2p.Error import ComponentStartConditionNotMet -from .PeerManager import PeerManager -from .PeerFinder import DHTPeerFinder +from lbrynet.extras.daemon.PeerManager import PeerManager +from lbrynet.extras.daemon.PeerFinder import DHTPeerFinder log = logging.getLogger(__name__) diff --git a/lbrynet/extras/daemon/Components.py b/lbrynet/extras/daemon/Components.py index cbcf06ed5..9751adb78 100644 --- a/lbrynet/extras/daemon/Components.py +++ b/lbrynet/extras/daemon/Components.py @@ -8,30 +8,33 @@ import binascii from hashlib import sha256 from types import SimpleNamespace from twisted.internet import defer, threads, reactor, error, task -import lbrynet.schema + from aioupnp import __version__ as aioupnp_version from aioupnp.upnp import UPnP from aioupnp.fault import UPnPError + +import lbrynet.schema from lbrynet import conf + +from lbrynet.blob.EncryptedFileManager import EncryptedFileManager +from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaverFactory +from lbrynet.blob.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier +from lbrynet.dht.node import Node +from lbrynet.extras.daemon.Component import Component +from lbrynet.extras.daemon.ExchangeRateManager import ExchangeRateManager +from lbrynet.extras.daemon.storage import SQLiteStorage +from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer +from lbrynet.extras.reflector.server.server import ReflectorServerFactory +from lbrynet.extras.wallet import LbryWalletManager +from lbrynet.extras.wallet import Network from lbrynet.utils import DeferredDict, generate_id from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager from lbrynet.p2p.RateLimiter import RateLimiter from lbrynet.p2p.BlobManager import DiskBlobManager from lbrynet.p2p.StreamDescriptor import StreamDescriptorIdentifier, EncryptedFileStreamType -from lbrynet.extras.wallet import LbryWalletManager -from lbrynet.extras.wallet import Network from lbrynet.p2p.server.BlobRequestHandler import BlobRequestHandlerFactory from lbrynet.p2p.server.ServerProtocol import ServerProtocolFactory -from .Component import Component -from .ExchangeRateManager import ExchangeRateManager -from .storage import SQLiteStorage -from .HashAnnouncer import DHTHashAnnouncer -from lbrynet.dht.node import Node -from lbrynet.blob.EncryptedFileManager import EncryptedFileManager -from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaverFactory -from lbrynet.blob.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier -from lbrynet.extras.reflector import ServerFactory as reflector_server_factory log = logging.getLogger(__name__) @@ -649,7 +652,7 @@ class ReflectorComponent(Component): log.info("Starting reflector server") blob_manager = self.component_manager.get_component(BLOB_COMPONENT) file_manager = self.component_manager.get_component(FILE_MANAGER_COMPONENT) - reflector_factory = reflector_server_factory(self.component_manager.peer_manager, blob_manager, file_manager) + reflector_factory = ReflectorServerFactory(self.component_manager.peer_manager, blob_manager, file_manager) try: self.reflector_server = yield reactor.listenTCP(self.reflector_server_port, reflector_factory) log.info('Started reflector on port %s', self.reflector_server_port) diff --git a/lbrynet/extras/daemon/Daemon.py b/lbrynet/extras/daemon/Daemon.py index ddb771bd0..cdd64458e 100644 --- a/lbrynet/extras/daemon/Daemon.py +++ b/lbrynet/extras/daemon/Daemon.py @@ -17,7 +17,26 @@ from twisted.python.failure import Failure from torba.client.baseaccount import SingleKey, HierarchicalDeterministic from lbrynet import conf, utils, __version__ +from lbrynet.dht.error import TimeoutError from lbrynet.extras import system_info +from lbrynet.extras.reflector import reupload +from lbrynet.extras.daemon.Components import d2f, f2d +from lbrynet.extras.daemon.Components import WALLET_COMPONENT, DATABASE_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT +from lbrynet.extras.daemon.Components import FILE_MANAGER_COMPONENT, RATE_LIMITER_COMPONENT +from lbrynet.extras.daemon.Components import EXCHANGE_RATE_MANAGER_COMPONENT, PAYMENT_RATE_COMPONENT, UPNP_COMPONENT +from lbrynet.extras.daemon.ComponentManager import RequiredCondition +from lbrynet.extras.daemon.Downloader import GetStream +from lbrynet.extras.daemon.Publisher import Publisher +from lbrynet.extras.daemon.auth.server import AuthJSONRPCServer +from lbrynet.extras.wallet import LbryWalletManager +from lbrynet.extras.wallet.account import Account as LBCAccount +from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies +from lbrynet.p2p.StreamDescriptor import download_sd_blob +from lbrynet.p2p.Error import InsufficientFundsError, UnknownNameError, DownloadDataTimeout, DownloadSDTimeout +from lbrynet.p2p.Error import NullFundsError, NegativeFundsError, ResolveError +from lbrynet.p2p.Peer import Peer +from lbrynet.p2p.SinglePeerDownloader import SinglePeerDownloader +from lbrynet.p2p.client.StandaloneBlobDownloader import StandaloneBlobDownloader from lbrynet.schema.claim import ClaimDict from lbrynet.schema.uri import parse_lbry_uri from lbrynet.schema.error import URIParseError, DecodeError @@ -25,28 +44,6 @@ from lbrynet.schema.validator import validate_claim_id from lbrynet.schema.address import decode_address from lbrynet.schema.decode import smart_decode -from lbrynet.extras.reflector import reupload -from .Components import d2f, f2d -from .Components import WALLET_COMPONENT, DATABASE_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT -from .Components import FILE_MANAGER_COMPONENT, RATE_LIMITER_COMPONENT -from .Components import EXCHANGE_RATE_MANAGER_COMPONENT, PAYMENT_RATE_COMPONENT, UPNP_COMPONENT -from .ComponentManager import RequiredCondition -from .Downloader import GetStream -from .Publisher import Publisher -from .auth.server import AuthJSONRPCServer -from lbrynet.p2p.StreamDescriptor import download_sd_blob -from lbrynet.p2p.Error import InsufficientFundsError, UnknownNameError -from lbrynet.p2p.Error import DownloadDataTimeout, DownloadSDTimeout -from lbrynet.p2p.Error import NullFundsError, NegativeFundsError -from lbrynet.p2p.Error import ResolveError -from lbrynet.dht.error import TimeoutError -from lbrynet.p2p.Peer import Peer -from lbrynet.p2p.SinglePeerDownloader import SinglePeerDownloader -from lbrynet.p2p.client.StandaloneBlobDownloader import StandaloneBlobDownloader -from lbrynet.extras.wallet import LbryWalletManager -from lbrynet.extras.wallet.account import Account as LBCAccount -from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies - log = logging.getLogger(__name__) requires = AuthJSONRPCServer.requires diff --git a/lbrynet/extras/daemon/DaemonConsole.py b/lbrynet/extras/daemon/DaemonConsole.py index c7e82c6ac..d3a1c1d67 100644 --- a/lbrynet/extras/daemon/DaemonConsole.py +++ b/lbrynet/extras/daemon/DaemonConsole.py @@ -6,11 +6,10 @@ import logging.handlers from twisted.internet import defer, reactor, threads from aiohttp import client_exceptions +from lbrynet import utils, conf, log_support from lbrynet.extras.daemon import analytics -from lbrynet import utils, conf -from lbrynet import log_support -from .auth.client import LBRYAPIClient -from .Daemon import Daemon +from lbrynet.extras.daemon.auth.client import LBRYAPIClient +from lbrynet.extras.daemon.Daemon import Daemon log = logging.getLogger(__name__) diff --git a/lbrynet/extras/daemon/DaemonControl.py b/lbrynet/extras/daemon/DaemonControl.py index bdaa71dc8..0ad0158fc 100644 --- a/lbrynet/extras/daemon/DaemonControl.py +++ b/lbrynet/extras/daemon/DaemonControl.py @@ -14,7 +14,7 @@ import logging.handlers from twisted.internet import reactor from lbrynet import utils, conf, log_support from lbrynet.extras import system_info -from .Daemon import Daemon +from lbrynet.extras.daemon.Daemon import Daemon log = logging.getLogger(__name__) diff --git a/lbrynet/extras/daemon/Downloader.py b/lbrynet/extras/daemon/Downloader.py index 5c73c49e7..f4badd0ad 100644 --- a/lbrynet/extras/daemon/Downloader.py +++ b/lbrynet/extras/daemon/Downloader.py @@ -3,7 +3,6 @@ import os from twisted.internet import defer from twisted.internet.task import LoopingCall -from .Components import f2d from lbrynet import conf from lbrynet.schema.fee import Fee @@ -14,6 +13,7 @@ from lbrynet.p2p.StreamDescriptor import download_sd_blob from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory from torba.client.constants import COIN from lbrynet.extras.wallet.dewies import dewies_to_lbc +from lbrynet.extras.daemon.Components import f2d INITIALIZING_CODE = 'initializing' DOWNLOAD_METADATA_CODE = 'downloading_metadata' diff --git a/lbrynet/extras/daemon/__init__.py b/lbrynet/extras/daemon/__init__.py index 7d3f2be07..9fb57d0d9 100644 --- a/lbrynet/extras/daemon/__init__.py +++ b/lbrynet/extras/daemon/__init__.py @@ -1 +1 @@ -from . import Components # register Component classes +from lbrynet.extras.daemon import Components # register Component classes diff --git a/lbrynet/extras/daemon/auth/auth.py b/lbrynet/extras/daemon/auth/auth.py index 5fb5cd2e7..8e350589a 100644 --- a/lbrynet/extras/daemon/auth/auth.py +++ b/lbrynet/extras/daemon/auth/auth.py @@ -3,7 +3,7 @@ from zope.interface import implementer from twisted.cred import portal, checkers, credentials, error as cred_error from twisted.internet import defer from twisted.web import resource -from .keyring import Keyring +from lbrynet.extras.daemon.auth.keyring import Keyring log = logging.getLogger(__name__) diff --git a/lbrynet/extras/daemon/auth/client.py b/lbrynet/extras/daemon/auth/client.py index cb9c657f9..50b0def8b 100644 --- a/lbrynet/extras/daemon/auth/client.py +++ b/lbrynet/extras/daemon/auth/client.py @@ -4,7 +4,7 @@ import logging from urllib.parse import urlparse from lbrynet import conf -from .keyring import Keyring, APIKey +from lbrynet.extras.daemon.auth.keyring import Keyring, APIKey log = logging.getLogger(__name__) USER_AGENT = "AuthServiceProxy/0.1" diff --git a/lbrynet/extras/daemon/auth/factory.py b/lbrynet/extras/daemon/auth/factory.py index 549b9bd68..163d49fe0 100644 --- a/lbrynet/extras/daemon/auth/factory.py +++ b/lbrynet/extras/daemon/auth/factory.py @@ -4,8 +4,8 @@ from twisted.web import server, guard, resource from twisted.cred import portal from lbrynet import conf -from .auth import PasswordChecker, HttpPasswordRealm -from ..auth.keyring import Keyring +from lbrynet.extras.daemon.auth.auth import PasswordChecker, HttpPasswordRealm +from lbrynet.extras.daemon.auth.keyring import Keyring log = logging.getLogger(__name__) diff --git a/lbrynet/extras/daemon/auth/keyring.py b/lbrynet/extras/daemon/auth/keyring.py index 91d9290a1..eb4f7011a 100644 --- a/lbrynet/extras/daemon/auth/keyring.py +++ b/lbrynet/extras/daemon/auth/keyring.py @@ -3,6 +3,7 @@ import datetime import hmac import hashlib import base58 +import logging from OpenSSL.crypto import FILETYPE_PEM from ssl import create_default_context, SSLContext from cryptography.hazmat.backends import default_backend @@ -12,7 +13,7 @@ from cryptography.x509.name import NameOID, NameAttribute from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives import serialization from twisted.internet import ssl -import logging + from lbrynet import conf log = logging.getLogger(__name__) diff --git a/lbrynet/extras/daemon/auth/server.py b/lbrynet/extras/daemon/auth/server.py index 93d0e8b26..82f814cf8 100644 --- a/lbrynet/extras/daemon/auth/server.py +++ b/lbrynet/extras/daemon/auth/server.py @@ -4,7 +4,6 @@ from six.moves.urllib import parse as urlparse import json import inspect import signal - from functools import wraps from twisted.web import server from twisted.internet import defer @@ -13,16 +12,18 @@ from twisted.python.failure import Failure from twisted.internet.error import ConnectionDone, ConnectionLost from txjsonrpc import jsonrpclib from traceback import format_exc + from lbrynet import conf, utils from lbrynet.extras.daemon import analytics from lbrynet.p2p.Error import InvalidAuthenticationToken from lbrynet.p2p.Error import ComponentsNotStarted, ComponentStartConditionNotMet from lbrynet.extras.looping_call_manager import LoopingCallManager from lbrynet.extras.daemon.ComponentManager import ComponentManager -from .keyring import APIKey, Keyring -from .undecorated import undecorated -from .factory import AuthJSONRPCResource +from lbrynet.extras.daemon.auth.keyring import APIKey, Keyring +from lbrynet.extras.daemon.auth.undecorated import undecorated +from lbrynet.extras.daemon.auth.factory import AuthJSONRPCResource from lbrynet.extras.daemon.json_response_encoder import JSONResponseEncoder + log = logging.getLogger(__name__) EMPTY_PARAMS = [{}] diff --git a/lbrynet/extras/daemon/migrator/migrate5to6.py b/lbrynet/extras/daemon/migrator/migrate5to6.py index 017aa965f..d8dcacdb5 100644 --- a/lbrynet/extras/daemon/migrator/migrate5to6.py +++ b/lbrynet/extras/daemon/migrator/migrate5to6.py @@ -2,8 +2,8 @@ import sqlite3 import os import json import logging -from lbrynet.schema.decode import smart_decode from lbrynet import conf +from lbrynet.schema.decode import smart_decode log = logging.getLogger(__name__) diff --git a/lbrynet/extras/daemon/storage.py b/lbrynet/extras/daemon/storage.py index 724aabeef..30c2d9330 100644 --- a/lbrynet/extras/daemon/storage.py +++ b/lbrynet/extras/daemon/storage.py @@ -6,12 +6,13 @@ from binascii import hexlify, unhexlify from decimal import Decimal from twisted.internet import defer, task, threads from twisted.enterprise import adbapi +from torba.client.constants import COIN + from lbrynet import conf from lbrynet.schema.claim import ClaimDict from lbrynet.schema.decode import smart_decode from lbrynet.blob.CryptBlob import CryptBlobInfo from lbrynet.dht.constants import dataExpireTimeout -from torba.client.constants import COIN log = logging.getLogger(__name__) diff --git a/lbrynet/extras/reflector/__init__.py b/lbrynet/extras/reflector/__init__.py index d05ef4820..56f9bfdf6 100644 --- a/lbrynet/extras/reflector/__init__.py +++ b/lbrynet/extras/reflector/__init__.py @@ -63,6 +63,3 @@ If the transfer was not successful (False), the blob is re-added to the needed_b Blob requests continue for each of the blobs the client has queued to send, when completed the client disconnects. """ -from .server.server import ReflectorServerFactory as ServerFactory -from .client.client import EncryptedFileReflectorClientFactory as ClientFactory -from .client.blob import BlobReflectorClientFactory as BlobClientFactory diff --git a/lbrynet/extras/reflector/reupload.py b/lbrynet/extras/reflector/reupload.py index a2b1548e3..1a6567d27 100644 --- a/lbrynet/extras/reflector/reupload.py +++ b/lbrynet/extras/reflector/reupload.py @@ -2,7 +2,8 @@ import random from twisted.internet import reactor, defer from lbrynet import conf -from . import ClientFactory, BlobClientFactory +from lbrynet.extras.reflector.client.client import EncryptedFileReflectorClientFactory +from lbrynet.extras.reflector.client.blob import BlobReflectorClientFactory def _is_ip(host): @@ -26,7 +27,7 @@ def resolve(host): @defer.inlineCallbacks def _reflect_stream(blob_manager, stream_hash, sd_hash, reflector_server): reflector_address, reflector_port = reflector_server[0], reflector_server[1] - factory = ClientFactory(blob_manager, stream_hash, sd_hash) + factory = EncryptedFileReflectorClientFactory(blob_manager, stream_hash, sd_hash) ip = yield resolve(reflector_address) yield reactor.connectTCP(ip, reflector_port, factory) result = yield factory.finished_deferred @@ -40,7 +41,7 @@ def _reflect_file(lbry_file, reflector_server): @defer.inlineCallbacks def _reflect_blobs(blob_manager, blob_hashes, reflector_server): reflector_address, reflector_port = reflector_server[0], reflector_server[1] - factory = BlobClientFactory(blob_manager, blob_hashes) + factory = BlobReflectorClientFactory(blob_manager, blob_hashes) ip = yield resolve(reflector_address) yield reactor.connectTCP(ip, reflector_port, factory) result = yield factory.finished_deferred diff --git a/lbrynet/extras/system_info.py b/lbrynet/extras/system_info.py index d3552d4f7..63fa3b065 100644 --- a/lbrynet/extras/system_info.py +++ b/lbrynet/extras/system_info.py @@ -1,9 +1,9 @@ import platform import os +import logging.handlers from lbrynet.schema import __version__ as schema_version from lbrynet import build_type, __version__ as lbrynet_version -import logging.handlers log = logging.getLogger(__name__) diff --git a/lbrynet/extras/wallet/__init__.py b/lbrynet/extras/wallet/__init__.py index 80205e427..b2ee7b8a5 100644 --- a/lbrynet/extras/wallet/__init__.py +++ b/lbrynet/extras/wallet/__init__.py @@ -6,6 +6,6 @@ __node_url__ = ( ) __spvserver__ = 'lbrynet.extras.wallet.server.coin.LBCRegTest' -from .ledger import MainNetLedger, RegTestLedger -from .manager import LbryWalletManager -from .network import Network +from lbrynet.extras.wallet.ledger import MainNetLedger, RegTestLedger +from lbrynet.extras.wallet.manager import LbryWalletManager +from lbrynet.extras.wallet.network import Network diff --git a/lbrynet/extras/wallet/ledger.py b/lbrynet/extras/wallet/ledger.py index dcddad4f7..7245ae8c4 100644 --- a/lbrynet/extras/wallet/ledger.py +++ b/lbrynet/extras/wallet/ledger.py @@ -2,17 +2,16 @@ import asyncio import logging from binascii import unhexlify -from lbrynet.extras.wallet.dewies import dewies_to_lbc +from torba.client.baseledger import BaseLedger from lbrynet.schema.error import URIParseError from lbrynet.schema.uri import parse_lbry_uri -from torba.client.baseledger import BaseLedger - -from .resolve import Resolver -from .account import Account -from .network import Network -from .database import WalletDatabase -from .transaction import Transaction -from .header import Headers, UnvalidatedHeaders +from lbrynet.extras.wallet.dewies import dewies_to_lbc +from lbrynet.extras.wallet.resolve import Resolver +from lbrynet.extras.wallet.account import Account +from lbrynet.extras.wallet.network import Network +from lbrynet.extras.wallet.database import WalletDatabase +from lbrynet.extras.wallet.transaction import Transaction +from lbrynet.extras.wallet.header import Headers, UnvalidatedHeaders log = logging.getLogger(__name__) diff --git a/lbrynet/extras/wallet/manager.py b/lbrynet/extras/wallet/manager.py index 8e11cc1bd..692651e43 100644 --- a/lbrynet/extras/wallet/manager.py +++ b/lbrynet/extras/wallet/manager.py @@ -14,11 +14,11 @@ from torba.client.basemanager import BaseWalletManager from lbrynet.schema.claim import ClaimDict -from .ledger import MainNetLedger -from .account import BaseAccount, generate_certificate -from .transaction import Transaction -from .database import WalletDatabase -from .dewies import dewies_to_lbc +from lbrynet.extras.wallet.ledger import MainNetLedger +from lbrynet.extras.wallet.account import BaseAccount, generate_certificate +from lbrynet.extras.wallet.transaction import Transaction +from lbrynet.extras.wallet.database import WalletDatabase +from lbrynet.extras.wallet.dewies import dewies_to_lbc log = logging.getLogger(__name__) diff --git a/lbrynet/extras/wallet/resolve.py b/lbrynet/extras/wallet/resolve.py index b3195c394..6f5fef78f 100644 --- a/lbrynet/extras/wallet/resolve.py +++ b/lbrynet/extras/wallet/resolve.py @@ -9,8 +9,7 @@ from lbrynet.schema.claim import ClaimDict from lbrynet.schema.decode import smart_decode from lbrynet.schema.error import DecodeError from lbrynet.schema.uri import parse_lbry_uri - -from .claim_proofs import verify_proof, InvalidProofError +from lbrynet.extras.wallet.claim_proofs import verify_proof, InvalidProofError log = logging.getLogger(__name__) diff --git a/lbrynet/extras/wallet/server/block_processor.py b/lbrynet/extras/wallet/server/block_processor.py index 9937f4298..eae699d67 100644 --- a/lbrynet/extras/wallet/server/block_processor.py +++ b/lbrynet/extras/wallet/server/block_processor.py @@ -9,7 +9,7 @@ from lbrynet.schema.proto.claim_pb2 import Claim from lbrynet.schema.uri import parse_lbry_uri from lbrynet.schema.decode import smart_decode -from .model import NameClaim, ClaimInfo, ClaimUpdate, ClaimSupport +from lbrynet.extras.wallet.server.model import NameClaim, ClaimInfo, ClaimUpdate, ClaimSupport class LBRYBlockProcessor(BlockProcessor): diff --git a/lbrynet/extras/wallet/server/coin.py b/lbrynet/extras/wallet/server/coin.py index 7d991565a..de5d8ef1e 100644 --- a/lbrynet/extras/wallet/server/coin.py +++ b/lbrynet/extras/wallet/server/coin.py @@ -6,7 +6,7 @@ from torba.server.hash import hash_to_hex_str, HASHX_LEN from hashlib import sha256 from torba.server.coins import Coin, CoinError -from .opcodes import decode_claim_script, opcodes as lbry_opcodes +from lbrynet.extras.wallet.server.opcodes import decode_claim_script, opcodes as lbry_opcodes class LBC(Coin): diff --git a/lbrynet/extras/wallet/server/db.py b/lbrynet/extras/wallet/server/db.py index 1bcb891bc..de2ad15b2 100644 --- a/lbrynet/extras/wallet/server/db.py +++ b/lbrynet/extras/wallet/server/db.py @@ -6,7 +6,7 @@ from torba.server.hash import hash_to_hex_str from torba.server.db import DB -from .model import ClaimInfo +from lbrynet.extras.wallet.server.model import ClaimInfo class LBRYDB(DB): diff --git a/lbrynet/extras/wallet/server/opcodes.py b/lbrynet/extras/wallet/server/opcodes.py index 2b697a064..8e17f5a25 100644 --- a/lbrynet/extras/wallet/server/opcodes.py +++ b/lbrynet/extras/wallet/server/opcodes.py @@ -1,6 +1,6 @@ import struct from torba.server.enum import Enumeration -from .model import NameClaim, ClaimSupport, ClaimUpdate +from lbrynet.extras.wallet.server.model import NameClaim, ClaimSupport, ClaimUpdate # TODO: Take this to lbryschema (it's also on lbryum and lbryum-server) diff --git a/lbrynet/extras/wallet/server/session.py b/lbrynet/extras/wallet/server/session.py index 037e23817..50313da7e 100644 --- a/lbrynet/extras/wallet/server/session.py +++ b/lbrynet/extras/wallet/server/session.py @@ -8,8 +8,8 @@ from torba.server import util from lbrynet.schema.uri import parse_lbry_uri from lbrynet.schema.error import URIParseError -from .block_processor import LBRYBlockProcessor -from .db import LBRYDB +from lbrynet.extras.wallet.server.block_processor import LBRYBlockProcessor +from lbrynet.extras.wallet.server.db import LBRYDB class LBRYElectrumX(ElectrumX): diff --git a/lbrynet/extras/wallet/server/tx.py b/lbrynet/extras/wallet/server/tx.py index 4d24c4088..eba24d751 100644 --- a/lbrynet/extras/wallet/server/tx.py +++ b/lbrynet/extras/wallet/server/tx.py @@ -1,6 +1,6 @@ from torba.server.tx import Deserializer -from .opcodes import decode_claim_script -from .model import TxClaimOutput, LBRYTx +from lbrynet.extras.wallet.server.opcodes import decode_claim_script +from lbrynet.extras.wallet.server.model import TxClaimOutput, LBRYTx class LBRYDeserializer(Deserializer): diff --git a/lbrynet/extras/wallet/transaction.py b/lbrynet/extras/wallet/transaction.py index 55856d7cf..fea2fff52 100644 --- a/lbrynet/extras/wallet/transaction.py +++ b/lbrynet/extras/wallet/transaction.py @@ -2,13 +2,12 @@ import struct from binascii import hexlify, unhexlify from typing import List, Iterable, Optional -from lbrynet.schema.decode import smart_decode -from .account import Account from torba.client.basetransaction import BaseTransaction, BaseInput, BaseOutput from torba.client.hash import hash160 - +from lbrynet.schema.decode import smart_decode from lbrynet.schema.claim import ClaimDict -from .script import InputScript, OutputScript +from lbrynet.extras.wallet.account import Account +from lbrynet.extras.wallet.script import InputScript, OutputScript class Input(BaseInput): diff --git a/lbrynet/p2p/BlobAvailability.py b/lbrynet/p2p/BlobAvailability.py index 36fb92a27..0ddb7072e 100644 --- a/lbrynet/p2p/BlobAvailability.py +++ b/lbrynet/p2p/BlobAvailability.py @@ -1,10 +1,9 @@ import logging import random import time - +from decimal import Decimal from twisted.internet import defer from twisted.internet.task import LoopingCall -from decimal import Decimal log = logging.getLogger(__name__) diff --git a/lbrynet/p2p/HTTPBlobDownloader.py b/lbrynet/p2p/HTTPBlobDownloader.py index 62e685f48..28e6e2a4b 100644 --- a/lbrynet/p2p/HTTPBlobDownloader.py +++ b/lbrynet/p2p/HTTPBlobDownloader.py @@ -1,9 +1,10 @@ -from random import choice import logging +import treq +from random import choice from twisted.internet import defer, task from twisted.internet.error import ConnectingCancelledError from twisted.web._newclient import ResponseNeverReceived -import treq + from lbrynet.utils import DeferredDict from lbrynet.p2p.Error import DownloadCanceledError diff --git a/lbrynet/p2p/PaymentRateManager.py b/lbrynet/p2p/PaymentRateManager.py index 372204365..7eb8b142c 100644 --- a/lbrynet/p2p/PaymentRateManager.py +++ b/lbrynet/p2p/PaymentRateManager.py @@ -1,6 +1,6 @@ -from lbrynet.p2p.Strategy import get_default_strategy, OnlyFreeStrategy -from lbrynet import conf from decimal import Decimal +from lbrynet import conf +from lbrynet.p2p.Strategy import get_default_strategy, OnlyFreeStrategy class BasePaymentRateManager: diff --git a/lbrynet/p2p/SinglePeerDownloader.py b/lbrynet/p2p/SinglePeerDownloader.py index 7936caa85..682939427 100644 --- a/lbrynet/p2p/SinglePeerDownloader.py +++ b/lbrynet/p2p/SinglePeerDownloader.py @@ -4,7 +4,7 @@ import tempfile from twisted.internet import defer, threads, reactor -from lbrynet.blob import BlobFile +from lbrynet.blob.blob_file import BlobFile from lbrynet.p2p.BlobManager import DiskBlobManager from lbrynet.p2p.RateLimiter import DummyRateLimiter from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager diff --git a/lbrynet/p2p/StreamDescriptor.py b/lbrynet/p2p/StreamDescriptor.py index 534d02db6..9ef1048d3 100644 --- a/lbrynet/p2p/StreamDescriptor.py +++ b/lbrynet/p2p/StreamDescriptor.py @@ -1,10 +1,10 @@ -from binascii import unhexlify import string -from collections import defaultdict import json import logging - +from collections import defaultdict +from binascii import unhexlify from twisted.internet import threads, defer + from lbrynet.cryptoutils import get_lbry_hash_obj from lbrynet.p2p.client.StandaloneBlobDownloader import StandaloneBlobDownloader from lbrynet.p2p.Error import UnknownStreamTypeError, InvalidStreamDescriptorError diff --git a/lbrynet/schema/decode.py b/lbrynet/schema/decode.py index 95aa18a5b..e5be724e4 100644 --- a/lbrynet/schema/decode.py +++ b/lbrynet/schema/decode.py @@ -1,13 +1,12 @@ import json import binascii - +from google.protobuf import json_format # pylint: disable=no-name-in-module import six from lbrynet.schema.error import DecodeError, InvalidAddress from lbrynet.schema.legacy.migrate import migrate as schema_migrator from lbrynet.schema.claim import ClaimDict -from google.protobuf import json_format # pylint: disable=no-name-in-module def migrate_json_claim_value(decoded_json): diff --git a/tests/functional/test_reflector.py b/tests/functional/test_reflector.py index 5b9cfc116..6c2ff7285 100644 --- a/tests/functional/test_reflector.py +++ b/tests/functional/test_reflector.py @@ -4,7 +4,9 @@ from binascii import hexlify from twisted.internet import defer, error from twisted.trial import unittest from lbrynet.p2p.StreamDescriptor import get_sd_info -from lbrynet.extras import reflector +from lbrynet.extras.reflector.server.server import ReflectorServerFactory +from lbrynet.extras.reflector.client.client import EncryptedFileReflectorClientFactory +from lbrynet.extras.reflector.client.blob import BlobReflectorClientFactory from lbrynet.extras.daemon.PeerManager import PeerManager from lbrynet.p2p import BlobManager from lbrynet.p2p import StreamDescriptor @@ -96,7 +98,7 @@ class TestReflector(unittest.TestCase): return d def start_server(): - server_factory = reflector.ServerFactory(peer_manager, self.server_blob_manager, + server_factory = ReflectorServerFactory(peer_manager, self.server_blob_manager, self.server_lbry_file_manager) from twisted.internet import reactor port = 8943 @@ -180,7 +182,7 @@ class TestReflector(unittest.TestCase): return d def send_to_server(): - factory = reflector.ClientFactory(self.client_blob_manager, self.stream_hash, self.sd_hash) + factory = EncryptedFileReflectorClientFactory(self.client_blob_manager, self.stream_hash, self.sd_hash) from twisted.internet import reactor reactor.connectTCP('localhost', self.port, factory) @@ -209,7 +211,7 @@ class TestReflector(unittest.TestCase): return d def send_to_server(blob_hashes_to_send): - factory = reflector.BlobClientFactory( + factory = BlobReflectorClientFactory( self.client_blob_manager, blob_hashes_to_send ) @@ -248,7 +250,7 @@ class TestReflector(unittest.TestCase): return d def send_to_server(blob_hashes_to_send): - factory = reflector.BlobClientFactory( + factory = BlobReflectorClientFactory( self.client_blob_manager, blob_hashes_to_send ) @@ -302,7 +304,7 @@ class TestReflector(unittest.TestCase): return d def send_to_server_as_blobs(blob_hashes_to_send): - factory = reflector.BlobClientFactory( + factory = BlobReflectorClientFactory( self.client_blob_manager, blob_hashes_to_send ) @@ -313,7 +315,7 @@ class TestReflector(unittest.TestCase): return factory.finished_deferred def send_to_server_as_stream(result): - factory = reflector.ClientFactory(self.client_blob_manager, self.stream_hash, self.sd_hash) + factory = EncryptedFileReflectorClientFactory(self.client_blob_manager, self.stream_hash, self.sd_hash) from twisted.internet import reactor reactor.connectTCP('localhost', self.port, factory) diff --git a/tests/unit/core/test_HTTPBlobDownloader.py b/tests/unit/core/test_HTTPBlobDownloader.py index 5112e38aa..3916e6ba8 100644 --- a/tests/unit/core/test_HTTPBlobDownloader.py +++ b/tests/unit/core/test_HTTPBlobDownloader.py @@ -3,7 +3,7 @@ from mock import MagicMock from twisted.trial import unittest from twisted.internet import defer -from lbrynet.blob import BlobFile +from lbrynet.blob.blob_file import BlobFile from lbrynet.p2p.HTTPBlobDownloader import HTTPBlobDownloader from tests.test_utils import mk_db_and_blob_dir, rm_db_and_blob_dir diff --git a/tests/unit/core/test_HashBlob.py b/tests/unit/core/test_HashBlob.py index d7c5003c6..da8b3f5bc 100644 --- a/tests/unit/core/test_HashBlob.py +++ b/tests/unit/core/test_HashBlob.py @@ -1,4 +1,4 @@ -from lbrynet.blob import BlobFile +from lbrynet.blob.blob_file import BlobFile from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError from tests.test_utils import mk_db_and_blob_dir, rm_db_and_blob_dir, random_lbry_hash From 7c8016d80764e656a25118b26df5c2a53b0432f0 Mon Sep 17 00:00:00 2001 From: Jack Robison Date: Fri, 9 Nov 2018 14:14:20 -0500 Subject: [PATCH 16/16] remove accidentally added lbrynet.schema.signature --- lbrynet/schema/signature/flags.py | 41 -------------------------- lbrynet/schema/signature/serializer.py | 11 ------- 2 files changed, 52 deletions(-) delete mode 100644 lbrynet/schema/signature/flags.py delete mode 100644 lbrynet/schema/signature/serializer.py diff --git a/lbrynet/schema/signature/flags.py b/lbrynet/schema/signature/flags.py deleted file mode 100644 index 259be0aff..000000000 --- a/lbrynet/schema/signature/flags.py +++ /dev/null @@ -1,41 +0,0 @@ -class SignatureSerializationFlag: - UNSIGNED = 0 - ''' - Format: - - or (legacy) - - ''' - ECDSA_LEGACY = 1 - ''' - Old claim format, which carried the signature inside the protobuf. Requires serializing back the claim with - signature stripped out for validation. This process requires knowledge on how a claim is serialized, thus requires - old fixed protobuf schema to work. - - Format: - - Curves: NIST256p, NIST384p, SECP256k1 - Signature content: `r` and `s` in each half of the 64 or 96 bytes (depends on curve) - Signed payload: - 1. Claim transaction output address (raw, decoded using base58) - 2. Stripped out claim protobuf serialization (without the signature) - 3. Certificate claim id (binary, not in network byte order) - ''' - ECDSA_SECP256K1 = 2 - ''' - Format: - - Curve: SECP256K1 - Signature content: 64 bytes total, each half represents `r` and `s` - Signed payload: - 1. raw claim name as bytes - 2. Claim transaction output address (raw, decoded using base58) - 3. Binary payload, independent of serialization (everything after the signature last byte) - 4. Certificate claim id, not in network byte order. - - A certificate can be signed as well, but this serialization model is unaware of content type or protobuf format. - ''' - @classmethod - def is_flag_valid(cls, flag): - # todo: use python 3 enum when fully ported, but not worth now as its an extra dependency for py2 - return 0 <= flag <= 2 \ No newline at end of file diff --git a/lbrynet/schema/signature/serializer.py b/lbrynet/schema/signature/serializer.py deleted file mode 100644 index 7ddcb2277..000000000 --- a/lbrynet/schema/signature/serializer.py +++ /dev/null @@ -1,11 +0,0 @@ -import struct -from collections import namedtuple -from .flags import SignatureSerializationFlag - - -class Signature(namedtuple("Signature", "flags signature certificate_id")): - def deserialize(cls, payload): - flag = struct.unpack("