Merge pull request #1604 from lbryio/fix-external-ip

Fix setting the external ip

lbrynet/__init__.py

@@ -1,7 +1,10 @@
 import logging
+from lbrynet.custom_logger import install_logger
+
 
 __name__ = "lbrynet"
 __version__ = "0.30.1rc1"
 version = tuple(__version__.split('.'))
 
+install_logger()
 logging.getLogger(__name__).addHandler(logging.NullHandler())

lbrynet/blob/EncryptedFileCreator.py

@@ -2,8 +2,8 @@
 Utilities for turning plain files into LBRY Files.
 """
 
-import logging
 import os
+import logging
 from binascii import hexlify
 
 from twisted.internet import defer

lbrynet/blob/EncryptedFileDownloader.py

@@ -5,10 +5,10 @@ import logging
 from binascii import hexlify, unhexlify
 
 from twisted.internet import defer
-from lbrynet.extras.daemon import conf
+from lbrynet import conf
 from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
 from lbrynet.p2p.HTTPBlobDownloader import HTTPBlobDownloader
-from lbrynet.p2p.utils import short_hash
+from lbrynet.utils import short_hash
 from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaver
 from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileDownloader
 from lbrynet.blob.EncryptedFileStatusReport import EncryptedFileStatusReport

lbrynet/blob/EncryptedFileManager.py

@@ -7,14 +7,14 @@ from binascii import hexlify, unhexlify
 
 from twisted.internet import defer, task, reactor
 from twisted.python.failure import Failure
+from lbrynet import conf
 from lbrynet.extras.reflector.reupload import reflect_file
 from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloader
 from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
 from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamType, get_sd_info
 from lbrynet.blob.client.CryptStreamDownloader import AlreadyStoppedError
 from lbrynet.blob.client.CryptStreamDownloader import CurrentlyStoppingError
-from lbrynet.p2p.utils import safe_start_looping_call, safe_stop_looping_call
+from lbrynet.utils import safe_start_looping_call, safe_stop_looping_call
-from lbrynet.extras.daemon import conf
 
 log = logging.getLogger(__name__)
 

lbrynet/blob/__init__.py

@@ -1,4 +0,0 @@
-from .blob_file import BlobFile
-from .creator import BlobFileCreator
-from .writer import HashBlobWriter
-from .reader import HashBlobReader

lbrynet/blob/blob_file.py

@@ -1,10 +1,10 @@
-import logging
 import os
+import logging
 from twisted.internet import defer, threads
 from twisted.web.client import FileBodyProducer
 from twisted.python.failure import Failure
+from lbrynet.cryptoutils import get_lbry_hash_obj
 from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError, InvalidBlobHashError
-from lbrynet.p2p.utils import is_valid_blobhash
 from lbrynet.blob.writer import HashBlobWriter
 from lbrynet.blob.reader import HashBlobReader
 
@@ -12,6 +12,24 @@ log = logging.getLogger(__name__)
 
 MAX_BLOB_SIZE = 2 * 2 ** 20
 
+# digest_size is in bytes, and blob hashes are hex encoded
+blobhash_length = get_lbry_hash_obj().digest_size * 2
+
+
+def is_valid_hashcharacter(char):
+    return char in "0123456789abcdef"
+
+
+def is_valid_blobhash(blobhash):
+    """Checks whether the blobhash is the correct length and contains only
+    valid characters (0-9, a-f)
+
+    @param blobhash: string, the blobhash to check
+
+    @return: True/False
+    """
+    return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
+
 
 class BlobFile:
     """

lbrynet/blob/client/CryptStreamDownloader.py

@@ -1,12 +1,13 @@
-from binascii import unhexlify
 import logging
+from binascii import unhexlify
+from twisted.internet import defer
+from twisted.python.failure import Failure
+
 from lbrynet.p2p.client.BlobRequester import BlobRequester
 from lbrynet.p2p.client.ConnectionManager import ConnectionManager
 from lbrynet.p2p.client.DownloadManager import DownloadManager
 from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
 from lbrynet.blob.client.CryptBlobHandler import CryptBlobHandler
-from twisted.internet import defer
-from twisted.python.failure import Failure
 
 
 log = logging.getLogger(__name__)

lbrynet/blob/client/EncryptedFileDownloader.py

@@ -2,13 +2,13 @@ import os
 import logging
 import traceback
 from binascii import hexlify, unhexlify
+from twisted.internet import defer, threads
 
 from lbrynet.p2p.StreamDescriptor import save_sd_info
 from lbrynet.blob.client.CryptStreamDownloader import CryptStreamDownloader
 from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
 from lbrynet.p2p.Error import FileOpenError
 from lbrynet.blob.client.EncryptedFileMetadataHandler import EncryptedFileMetadataHandler
-from twisted.internet import defer, threads
 
 
 log = logging.getLogger(__name__)

lbrynet/blob/creator.py

@@ -3,7 +3,7 @@ import logging
 from io import BytesIO
 from twisted.internet import defer
 from twisted.web.client import FileBodyProducer
-from lbrynet.p2p.cryptoutils import get_lbry_hash_obj
+from lbrynet.cryptoutils import get_lbry_hash_obj
 
 log = logging.getLogger(__name__)
 

lbrynet/blob/writer.py

@@ -2,7 +2,7 @@ import logging
 from io import BytesIO
 from twisted.python.failure import Failure
 from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError
-from lbrynet.p2p.cryptoutils import get_lbry_hash_obj
+from lbrynet.cryptoutils import get_lbry_hash_obj
 
 log = logging.getLogger(__name__)
 

lbrynet/conf.py

@@ -7,7 +7,7 @@ import sys
 import yaml
 import envparse
 from appdirs import user_data_dir, user_config_dir
-from lbrynet.p2p import utils
+from lbrynet import utils
 from lbrynet.p2p.Error import InvalidCurrencyError, NoSuchDirectoryError
 from lbrynet.androidhelpers.paths import (
     android_internal_storage_dir,
@@ -15,7 +15,7 @@ from lbrynet.androidhelpers.paths import (
 )
 
 try:
-    from .winpaths import get_path, FOLDERID, UserHandle
+    from lbrynet.winpaths import get_path, FOLDERID, UserHandle
 except (ImportError, ValueError, NameError):
     # Android platform: NameError: name 'c_wchar' is not defined
     pass

lbrynet/custom_logger.py

@@ -102,5 +102,8 @@ class Logger(logging.Logger):
             self._log(TRACE, msg, args, **kwargs)
 
 
-logging.setLoggerClass(Logger)
+def install_logger():
-logging.addLevelName(TRACE, 'TRACE')
+    current = logging.getLoggerClass()
+    if current is not Logger:
+        logging.setLoggerClass(Logger)
+        logging.addLevelName(TRACE, 'TRACE')

lbrynet/dht/datastore.py

@@ -1,5 +1,5 @@
 from collections import UserDict
-from . import constants
+from lbrynet.dht import constants
 
 
 class DictDataStore(UserDict):

lbrynet/dht/encoding.py

@@ -1,4 +1,4 @@
-from .error import DecodeError
+from lbrynet.dht.error import DecodeError
 
 
 def bencode(data):

lbrynet/dht/iterativefind.py

@@ -1,8 +1,8 @@
 import logging
 from twisted.internet import defer
-from .distance import Distance
+from lbrynet.dht.distance import Distance
-from .error import TimeoutError
+from lbrynet.dht.error import TimeoutError
-from . import constants
+from lbrynet.dht import constants
 
 log = logging.getLogger(__name__)
 

lbrynet/dht/kbucket.py

@@ -1,8 +1,8 @@
 import logging
 
-from . import constants
+from lbrynet.dht import constants
-from .distance import Distance
+from lbrynet.dht.distance import Distance
-from .error import BucketFull
+from lbrynet.dht.error import BucketFull
 
 log = logging.getLogger(__name__)
 

lbrynet/dht/msgformat.py

@@ -7,7 +7,7 @@
 # The docstrings in this module contain epytext markup; API documentation
 # may be created by processing this file with epydoc: http://epydoc.sf.net
 
-from . import msgtypes
+from lbrynet.dht import msgtypes
 
 
 class MessageTranslator:

lbrynet/dht/msgtypes.py

@@ -7,8 +7,8 @@
 # The docstrings in this module contain epytext markup; API documentation
 # may be created by processing this file with epydoc: http://epydoc.sf.net
 
-from lbrynet.p2p.utils import generate_id
+from lbrynet.utils import generate_id
-from . import constants
+from lbrynet.dht import constants
 
 
 class Message:

lbrynet/dht/node.py

@@ -5,17 +5,12 @@ from functools import reduce
 
 from twisted.internet import defer, error, task
 
-from lbrynet.p2p.utils import generate_id, DeferredDict
+from lbrynet.utils import generate_id, DeferredDict
-from lbrynet.p2p.call_later_manager import CallLaterManager
+from lbrynet.dht.call_later_manager import CallLaterManager
-from lbrynet.p2p.PeerManager import PeerManager
+from lbrynet.dht.error import TimeoutError
-from .error import TimeoutError
+from lbrynet.dht import constants, routingtable, datastore, protocol
-from . import constants
+from lbrynet.dht.contact import ContactManager
-from . import routingtable
+from lbrynet.dht.iterativefind import iterativeFind
-from . import datastore
-from . import protocol
-from .peerfinder import DHTPeerFinder
-from .contact import ContactManager
-from .iterativefind import iterativeFind
 
 log = logging.getLogger(__name__)
 
@@ -83,8 +78,8 @@ class Node(MockKademliaHelper):
     def __init__(self, node_id=None, udpPort=4000, dataStore=None,
                  routingTableClass=None, networkProtocol=None,
                  externalIP=None, peerPort=3333, listenUDP=None,
-                 callLater=None, resolve=None, clock=None, peer_finder=None,
+                 callLater=None, resolve=None, clock=None,
-                 peer_manager=None, interface='', externalUDPPort=None):
+                 interface='', externalUDPPort=None):
         """
         @param dataStore: The data store to use. This must be class inheriting
                           from the C{DataStore} interface (or providing the
@@ -124,20 +119,13 @@ class Node(MockKademliaHelper):
         else:
             self._routingTable = routingTableClass(self.node_id, self.clock.seconds)
 
-        # Initialize this node's network access mechanisms
+        self._protocol = networkProtocol or protocol.KademliaProtocol(self)
-        if networkProtocol is None:
-            self._protocol = protocol.KademliaProtocol(self)
-        else:
-            self._protocol = networkProtocol
-        # Initialize the data storage mechanism used by this node
         self.token_secret = self._generateID()
         self.old_token_secret = None
         self.externalIP = externalIP
         self.peerPort = peerPort
         self.externalUDPPort = externalUDPPort or self.port
         self._dataStore = dataStore or datastore.DictDataStore(self.clock.seconds)
-        self.peer_manager = peer_manager or PeerManager()
-        self.peer_finder = peer_finder or DHTPeerFinder(self, self.peer_manager)
         self._join_deferred = None
 
     #def __del__(self):
@@ -255,7 +243,7 @@ class Node(MockKademliaHelper):
         yield _iterative_join()
 
     @defer.inlineCallbacks
-    def start(self, known_node_addresses=None):
+    def start(self, known_node_addresses=None, block_on_join=False):
         """ Causes the Node to attempt to join the DHT network by contacting the
         known DHT nodes. This can be called multiple times if the previous attempt
         has failed or if the Node has lost all the contacts.
@@ -270,8 +258,11 @@ class Node(MockKademliaHelper):
         self.start_listening()
         yield self._protocol._listening
         # TODO: Refresh all k-buckets further away than this node's closest neighbour
-        yield self.joinNetwork(known_node_addresses or [])
+        d = self.joinNetwork(known_node_addresses or [])
-        self.start_looping_calls()
+        d.addCallback(lambda _: self.start_looping_calls())
+        d.addCallback(lambda _: log.info("Joined the dht"))
+        if block_on_join:
+            yield d
 
     def start_looping_calls(self):
         self.safe_start_looping_call(self._change_token_lc, constants.tokenSecretChangeInterval)

lbrynet/dht/protocol.py

@@ -3,12 +3,8 @@ import errno
 from binascii import hexlify
 
 from twisted.internet import protocol, defer
-from .error import BUILTIN_EXCEPTIONS, UnknownRemoteException, TimeoutError, TransportNotConnected
+from lbrynet.dht import constants, encoding, msgformat, msgtypes
-
+from lbrynet.dht.error import BUILTIN_EXCEPTIONS, UnknownRemoteException, TimeoutError, TransportNotConnected
-from . import constants
-from . import encoding
-from . import msgtypes
-from . import msgformat
 
 log = logging.getLogger(__name__)
 

lbrynet/dht/routingtable.py

@@ -6,14 +6,13 @@
 # may be created by processing this file with epydoc: http://epydoc.sf.net
 
 import random
-from binascii import unhexlify
-
-from twisted.internet import defer
-from . import constants
-from . import kbucket
-from .error import TimeoutError
-from .distance import Distance
 import logging
+from binascii import unhexlify
+from twisted.internet import defer
+
+from lbrynet.dht import constants, kbucket
+from lbrynet.dht.error import TimeoutError
+from lbrynet.dht.distance import Distance
 
 log = logging.getLogger(__name__)
 

lbrynet/extras/cli.py

@@ -24,22 +24,25 @@ from lbrynet.extras.daemon.Daemon import Daemon
 from lbrynet.extras.daemon.DaemonControl import start as daemon_main
 from lbrynet.extras.daemon.DaemonConsole import main as daemon_console
 from lbrynet.extras.daemon.auth.client import LBRYAPIClient
-from lbrynet.p2p.system_info import get_platform
+from lbrynet.extras.system_info import get_platform
 
 
 async def execute_command(method, params, conf_path=None):
     # this check if the daemon is running or not
+    api = None
     try:
         api = await LBRYAPIClient.get_client(conf_path)
         await api.status()
     except (ClientConnectorError, ConnectionError):
-        await api.session.close()
+        if api:
+            await api.session.close()
         print("Could not connect to daemon. Are you sure it's running?")
         return 1
 
     # this actually executes the method
+    resp = await api.call(method, params)
+
     try:
-        resp = await api.call(method, params)
         await api.session.close()
         print(json.dumps(resp["result"], indent=2))
     except KeyError:
@@ -137,7 +140,7 @@ def main(argv=None):
 
     elif method in ['version', '--version', '-v']:
         print("{lbrynet_name} {lbrynet_version}".format(
-            lbrynet_name=lbrynet_name, **get_platform(get_ip=False)
+            lbrynet_name=lbrynet_name, **get_platform()
         ))
         return 0
 

lbrynet/extras/daemon/Component.py

@@ -1,7 +1,7 @@
 import logging
 from twisted.internet import defer
 from twisted._threads import AlreadyQuit
-from .ComponentManager import ComponentManager
+from lbrynet.extras.daemon.ComponentManager import ComponentManager
 
 log = logging.getLogger(__name__)
 

lbrynet/extras/daemon/ComponentManager.py

@@ -1,7 +1,8 @@
 import logging
 from twisted.internet import defer
-
 from lbrynet.p2p.Error import ComponentStartConditionNotMet
+from lbrynet.extras.daemon.PeerManager import PeerManager
+from lbrynet.extras.daemon.PeerFinder import DHTPeerFinder
 
 log = logging.getLogger(__name__)
 
@@ -33,13 +34,15 @@ class RequiredCondition(metaclass=RequiredConditionType):
 class ComponentManager:
     default_component_classes = {}
 
-    def __init__(self, reactor=None, analytics_manager=None, skip_components=None, **override_components):
+    def __init__(self, reactor=None, analytics_manager=None, skip_components=None,
+                 peer_manager=None, peer_finder=None, **override_components):
         self.skip_components = skip_components or []
-
         self.reactor = reactor
         self.component_classes = {}
         self.components = set()
         self.analytics_manager = analytics_manager
+        self.peer_manager = peer_manager or PeerManager()
+        self.peer_finder = peer_finder or DHTPeerFinder(self)
 
         for component_name, component_class in self.default_component_classes.items():
             if component_name in override_components:
@@ -114,10 +117,10 @@ class ComponentManager:
 
         :return: (defer.Deferred)
         """
-
         for component_name, cb in callbacks.items():
             if component_name not in self.component_classes:
-                raise NameError("unknown component: %s" % component_name)
+                if component_name not in self.skip_components:
+                    raise NameError("unknown component: %s" % component_name)
             if not callable(cb):
                 raise ValueError("%s is not callable" % cb)
 

lbrynet/extras/daemon/Components.py

@@ -2,35 +2,39 @@ import os
 import asyncio
 import logging
 import treq
+import json
 import math
 import binascii
 from hashlib import sha256
 from types import SimpleNamespace
 from twisted.internet import defer, threads, reactor, error, task
-import lbrynet.schema
+
 from aioupnp import __version__ as aioupnp_version
 from aioupnp.upnp import UPnP
 from aioupnp.fault import UPnPError
-from lbrynet.extras.daemon import conf
+
-from lbrynet.p2p.utils import DeferredDict
+import lbrynet.schema
+from lbrynet import conf
+
+from lbrynet.blob.EncryptedFileManager import EncryptedFileManager
+from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaverFactory
+from lbrynet.blob.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
+from lbrynet.dht.node import Node
+from lbrynet.extras.daemon.Component import Component
+from lbrynet.extras.daemon.ExchangeRateManager import ExchangeRateManager
+from lbrynet.extras.daemon.storage import SQLiteStorage
+from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer
+from lbrynet.extras.reflector.server.server import ReflectorServerFactory
+from lbrynet.extras.wallet import LbryWalletManager
+from lbrynet.extras.wallet import Network
+from lbrynet.utils import DeferredDict, generate_id
 from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager
 from lbrynet.p2p.RateLimiter import RateLimiter
 from lbrynet.p2p.BlobManager import DiskBlobManager
 from lbrynet.p2p.StreamDescriptor import StreamDescriptorIdentifier, EncryptedFileStreamType
-from lbrynet.extras.wallet import LbryWalletManager
-from lbrynet.extras.wallet import Network
 from lbrynet.p2p.server.BlobRequestHandler import BlobRequestHandlerFactory
 from lbrynet.p2p.server.ServerProtocol import ServerProtocolFactory
-from .Component import Component
-from .ExchangeRateManager import ExchangeRateManager
-from lbrynet.extras.daemon.storage import SQLiteStorage
-from lbrynet.dht import node, hashannouncer
-from lbrynet.blob.EncryptedFileManager import EncryptedFileManager
-from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaverFactory
-from lbrynet.blob.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
-from lbrynet.extras.reflector import ServerFactory as reflector_server_factory
 
-from lbrynet.p2p.utils import generate_id
 
 log = logging.getLogger(__name__)
 
@@ -42,7 +46,6 @@ HEADERS_COMPONENT = "blockchain_headers"
 WALLET_COMPONENT = "wallet"
 DHT_COMPONENT = "dht"
 HASH_ANNOUNCER_COMPONENT = "hash_announcer"
-STREAM_IDENTIFIER_COMPONENT = "stream_identifier"
 FILE_MANAGER_COMPONENT = "file_manager"
 PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
 REFLECTOR_COMPONENT = "reflector"
@@ -94,10 +97,19 @@ class ConfigSettings:
         return conf.settings.node_id
 
     @staticmethod
-    def get_external_ip():
+    @defer.inlineCallbacks
-        from lbrynet.p2p.system_info import get_platform
+    def get_external_ip():  # used if upnp is disabled or non-functioning
-        platform = get_platform(get_ip=True)
+        try:
-        return platform['ip']
+            buf = []
+            response = yield treq.get("https://api.lbry.io/ip")
+            yield treq.collect(response, buf.append)
+            parsed = json.loads(b"".join(buf).decode())
+            if parsed['success']:
+                return parsed['data']['ip']
+            return
+        except Exception as err:
+            return
+
 
 
 # Shorthand for common ConfigSettings methods
@@ -366,7 +378,7 @@ class WalletComponent(Component):
 
 class BlobComponent(Component):
     component_name = BLOB_COMPONENT
-    depends_on = [DATABASE_COMPONENT, DHT_COMPONENT]
+    depends_on = [DATABASE_COMPONENT]
 
     def __init__(self, component_manager):
         super().__init__(component_manager)
@@ -378,8 +390,12 @@ class BlobComponent(Component):
 
     def start(self):
         storage = self.component_manager.get_component(DATABASE_COMPONENT)
-        dht_node = self.component_manager.get_component(DHT_COMPONENT)
+        datastore = None
-        self.blob_manager = DiskBlobManager(CS.get_blobfiles_dir(), storage, dht_node._dataStore)
+        if DHT_COMPONENT not in self.component_manager.skip_components:
+            dht_node = self.component_manager.get_component(DHT_COMPONENT)
+            if dht_node:
+                datastore = dht_node._dataStore
+        self.blob_manager = DiskBlobManager(CS.get_blobfiles_dir(), storage, datastore)
         return self.blob_manager.setup()
 
     def stop(self):
@@ -424,20 +440,22 @@ class DHTComponent(Component):
         node_id = CS.get_node_id()
         if node_id is None:
             node_id = generate_id()
+        external_ip = self.upnp_component.external_ip
+        if not external_ip:
+            log.warning("UPnP component failed to get external ip")
+            external_ip = yield CS.get_external_ip()
+            if not external_ip:
+                log.warning("failed to get external ip")
 
-        self.dht_node = node.Node(
+        self.dht_node = Node(
             node_id=node_id,
             udpPort=GCS('dht_node_port'),
             externalUDPPort=self.external_udp_port,
-            externalIP=self.upnp_component.external_ip,
+            externalIP=external_ip,
             peerPort=self.external_peer_port
         )
 
-        self.dht_node.start_listening()
+        yield self.dht_node.start(GCS('known_dht_nodes'), block_on_join=False)
-        yield self.dht_node._protocol._listening
-        d = self.dht_node.joinNetwork(GCS('known_dht_nodes'))
-        d.addCallback(lambda _: self.dht_node.start_looping_calls())
-        d.addCallback(lambda _: log.info("Joined the dht"))
         log.info("Started the dht")
 
     @defer.inlineCallbacks
@@ -461,7 +479,7 @@ class HashAnnouncerComponent(Component):
     def start(self):
         storage = self.component_manager.get_component(DATABASE_COMPONENT)
         dht_node = self.component_manager.get_component(DHT_COMPONENT)
-        self.hash_announcer = hashannouncer.DHTHashAnnouncer(dht_node, storage)
+        self.hash_announcer = DHTHashAnnouncer(dht_node, storage)
         yield self.hash_announcer.start()
 
     @defer.inlineCallbacks
@@ -494,41 +512,6 @@ class RateLimiterComponent(Component):
         return defer.succeed(None)
 
 
-class StreamIdentifierComponent(Component):
-    component_name = STREAM_IDENTIFIER_COMPONENT
-    depends_on = [DHT_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT]
-
-    def __init__(self, component_manager):
-        super().__init__(component_manager)
-        self.sd_identifier = StreamDescriptorIdentifier()
-
-    @property
-    def component(self):
-        return self.sd_identifier
-
-    @defer.inlineCallbacks
-    def start(self):
-        dht_node = self.component_manager.get_component(DHT_COMPONENT)
-        rate_limiter = self.component_manager.get_component(RATE_LIMITER_COMPONENT)
-        blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
-        storage = self.component_manager.get_component(DATABASE_COMPONENT)
-        wallet = self.component_manager.get_component(WALLET_COMPONENT)
-
-        add_lbry_file_to_sd_identifier(self.sd_identifier)
-        file_saver_factory = EncryptedFileSaverFactory(
-            dht_node.peer_finder,
-            rate_limiter,
-            blob_manager,
-            storage,
-            wallet,
-            GCS('download_directory')
-        )
-        yield self.sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory)
-
-    def stop(self):
-        pass
-
-
 class PaymentRateComponent(Component):
     component_name = PAYMENT_RATE_COMPONENT
 
@@ -549,8 +532,8 @@ class PaymentRateComponent(Component):
 
 class FileManagerComponent(Component):
     component_name = FILE_MANAGER_COMPONENT
-    depends_on = [DHT_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT,
+    depends_on = [RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT,
-                  STREAM_IDENTIFIER_COMPONENT, PAYMENT_RATE_COMPONENT]
+                  PAYMENT_RATE_COMPONENT]
 
     def __init__(self, component_manager):
         super().__init__(component_manager)
@@ -569,15 +552,26 @@ class FileManagerComponent(Component):
 
     @defer.inlineCallbacks
     def start(self):
-        dht_node = self.component_manager.get_component(DHT_COMPONENT)
         rate_limiter = self.component_manager.get_component(RATE_LIMITER_COMPONENT)
         blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
         storage = self.component_manager.get_component(DATABASE_COMPONENT)
         wallet = self.component_manager.get_component(WALLET_COMPONENT)
-        sd_identifier = self.component_manager.get_component(STREAM_IDENTIFIER_COMPONENT)
+
+        sd_identifier = StreamDescriptorIdentifier()
+        add_lbry_file_to_sd_identifier(sd_identifier)
+        file_saver_factory = EncryptedFileSaverFactory(
+            self.component_manager.peer_finder,
+            rate_limiter,
+            blob_manager,
+            storage,
+            wallet,
+            GCS('download_directory')
+        )
+        yield sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory)
+
         payment_rate_manager = self.component_manager.get_component(PAYMENT_RATE_COMPONENT)
         log.info('Starting the file manager')
-        self.file_manager = EncryptedFileManager(dht_node.peer_finder, rate_limiter, blob_manager, wallet,
+        self.file_manager = EncryptedFileManager(self.component_manager.peer_finder, rate_limiter, blob_manager, wallet,
                                                  payment_rate_manager, storage, sd_identifier)
         yield self.file_manager.setup()
         log.info('Done setting up file manager')
@@ -589,7 +583,7 @@ class FileManagerComponent(Component):
 
 class PeerProtocolServerComponent(Component):
     component_name = PEER_PROTOCOL_SERVER_COMPONENT
-    depends_on = [UPNP_COMPONENT, DHT_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT,
+    depends_on = [UPNP_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT,
                   PAYMENT_RATE_COMPONENT]
 
     def __init__(self, component_manager):
@@ -618,7 +612,7 @@ class PeerProtocolServerComponent(Component):
         }
         server_factory = ServerProtocolFactory(
             self.component_manager.get_component(RATE_LIMITER_COMPONENT), query_handlers,
-            self.component_manager.get_component(DHT_COMPONENT).peer_manager
+            self.component_manager.peer_manager
         )
 
         try:
@@ -642,7 +636,7 @@ class PeerProtocolServerComponent(Component):
 
 class ReflectorComponent(Component):
     component_name = REFLECTOR_COMPONENT
-    depends_on = [DHT_COMPONENT, BLOB_COMPONENT, FILE_MANAGER_COMPONENT]
+    depends_on = [BLOB_COMPONENT, FILE_MANAGER_COMPONENT]
 
     def __init__(self, component_manager):
         super().__init__(component_manager)
@@ -656,10 +650,9 @@ class ReflectorComponent(Component):
     @defer.inlineCallbacks
     def start(self):
         log.info("Starting reflector server")
-        dht_node = self.component_manager.get_component(DHT_COMPONENT)
         blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
         file_manager = self.component_manager.get_component(FILE_MANAGER_COMPONENT)
-        reflector_factory = reflector_server_factory(dht_node.peer_manager, blob_manager, file_manager)
+        reflector_factory = ReflectorServerFactory(self.component_manager.peer_manager, blob_manager, file_manager)
         try:
             self.reflector_server = yield reactor.listenTCP(self.reflector_server_port, reflector_factory)
             log.info('Started reflector on port %s', self.reflector_server_port)
@@ -695,10 +688,12 @@ class UPnPComponent(Component):
 
     @defer.inlineCallbacks
     def _setup_redirects(self):
-        upnp_redirects = yield DeferredDict({
+        d = {}
-            "UDP": from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")),
+        if PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components:
-            "TCP": from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port"))
+            d["TCP"] = from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port"))
-        })
+        if DHT_COMPONENT not in self.component_manager.skip_components:
+            d["UDP"] = from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port"))
+        upnp_redirects = yield DeferredDict(d)
         self.upnp_redirects.update(upnp_redirects)
 
     @defer.inlineCallbacks
@@ -710,33 +705,41 @@ class UPnPComponent(Component):
                 log.info("found upnp gateway: %s", self.upnp.gateway.manufacturer_string)
             except Exception as err:
                 log.warning("upnp discovery failed: %s", err)
-                return
+                self.upnp = None
 
         # update the external ip
-        try:
+        external_ip = None
-            external_ip = yield from_future(self.upnp.get_external_ip())
+        if self.upnp:
-            if external_ip == "0.0.0.0":
-                log.warning("upnp doesn't know the external ip address (returned 0.0.0.0), using fallback")
-                external_ip = CS.get_external_ip()
-            if self.external_ip and self.external_ip != external_ip:
-                log.info("external ip changed from %s to %s", self.external_ip, external_ip)
-            elif not self.external_ip:
-                log.info("got external ip: %s", external_ip)
-            self.external_ip = external_ip
-        except (asyncio.TimeoutError, UPnPError):
-            pass
-
-        if not self.upnp_redirects:  # setup missing redirects
             try:
-                upnp_redirects = yield DeferredDict({
+                external_ip = yield from_future(self.upnp.get_external_ip())
-                    "UDP": from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")),
+                if external_ip != "0.0.0.0":
-                    "TCP": from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port"))
+                    log.info("got external ip from UPnP: %s", external_ip)
-                })
+            except (asyncio.TimeoutError, UPnPError):
+                pass
+
+        if external_ip == "0.0.0.0" or not external_ip:
+            log.warning("unable to get external ip from UPnP, checking lbry.io fallback")
+            external_ip = yield CS.get_external_ip()
+        if self.external_ip and self.external_ip != external_ip:
+            log.info("external ip changed from %s to %s", self.external_ip, external_ip)
+        self.external_ip = external_ip
+        assert self.external_ip is not None   # TODO: handle going/starting offline
+
+        if not self.upnp_redirects and self.upnp:  # setup missing redirects
+            try:
+                log.info("add UPnP port mappings")
+                d = {}
+                if PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components:
+                    d["TCP"] = from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port"))
+                if DHT_COMPONENT not in self.component_manager.skip_components:
+                    d["UDP"] = from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port"))
+                upnp_redirects = yield DeferredDict(d)
+                log.info("set up redirects: %s", upnp_redirects)
                 self.upnp_redirects.update(upnp_redirects)
             except (asyncio.TimeoutError, UPnPError):
                 self.upnp = None
                 return self._maintain_redirects()
-        else:  # check existing redirects are still active
+        elif self.upnp:  # check existing redirects are still active
             found = set()
             mappings = yield from_future(self.upnp.get_redirects())
             for mapping in mappings:
@@ -744,7 +747,7 @@ class UPnPComponent(Component):
                 if proto in self.upnp_redirects and mapping['NewExternalPort'] == self.upnp_redirects[proto]:
                     if mapping['NewInternalClient'] == self.upnp.lan_address:
                         found.add(proto)
-            if 'UDP' not in found:
+            if 'UDP' not in found and DHT_COMPONENT not in self.component_manager.skip_components:
                 try:
                     udp_port = yield from_future(
                         self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")
@@ -753,7 +756,7 @@ class UPnPComponent(Component):
                     log.info("refreshed upnp redirect for dht port: %i", udp_port)
                 except (asyncio.TimeoutError, UPnPError):
                     del self.upnp_redirects['UDP']
-            if 'TCP' not in found:
+            if 'TCP' not in found and PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components:
                 try:
                     tcp_port = yield from_future(
                         self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")
@@ -762,22 +765,28 @@ class UPnPComponent(Component):
                     log.info("refreshed upnp redirect for peer port: %i", tcp_port)
                 except (asyncio.TimeoutError, UPnPError):
                     del self.upnp_redirects['TCP']
-            if 'TCP' in self.upnp_redirects and 'UDP' in self.upnp_redirects:
+            if ('TCP' in self.upnp_redirects
-                log.debug("upnp redirects are still active")
+                and PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components) and (
+                    'UDP' in self.upnp_redirects and DHT_COMPONENT not in self.component_manager.skip_components):
+                if self.upnp_redirects:
+                    log.debug("upnp redirects are still active")
 
     @defer.inlineCallbacks
     def start(self):
+        log.info("detecting external ip")
         if not self.use_upnp:
-            self.external_ip = CS.get_external_ip()
+            self.external_ip = yield CS.get_external_ip()
             return
         success = False
         yield self._maintain_redirects()
         if self.upnp:
-            if not self.upnp_redirects:
+            if not self.upnp_redirects and not all([x in self.component_manager.skip_components for x in
+                                                    (DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT)]):
                 log.error("failed to setup upnp, debugging infomation: %s", self.upnp.zipped_debugging_info)
             else:
                 success = True
-                log.debug("set up upnp port redirects for gateway: %s", self.upnp.gateway.manufacturer_string)
+                if self.upnp_redirects:
+                    log.debug("set up upnp port redirects for gateway: %s", self.upnp.gateway.manufacturer_string)
         else:
             log.error("failed to setup upnp")
         self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, self.get_status())

lbrynet/extras/daemon/Daemon.py

@@ -16,6 +16,27 @@ from twisted.python.failure import Failure
 
 from torba.client.baseaccount import SingleKey, HierarchicalDeterministic
 
+from lbrynet import conf, utils, __version__
+from lbrynet.dht.error import TimeoutError
+from lbrynet.extras import system_info
+from lbrynet.extras.reflector import reupload
+from lbrynet.extras.daemon.Components import d2f, f2d
+from lbrynet.extras.daemon.Components import WALLET_COMPONENT, DATABASE_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT
+from lbrynet.extras.daemon.Components import FILE_MANAGER_COMPONENT, RATE_LIMITER_COMPONENT
+from lbrynet.extras.daemon.Components import EXCHANGE_RATE_MANAGER_COMPONENT, PAYMENT_RATE_COMPONENT, UPNP_COMPONENT
+from lbrynet.extras.daemon.ComponentManager import RequiredCondition
+from lbrynet.extras.daemon.Downloader import GetStream
+from lbrynet.extras.daemon.Publisher import Publisher
+from lbrynet.extras.daemon.auth.server import AuthJSONRPCServer
+from lbrynet.extras.wallet import LbryWalletManager
+from lbrynet.extras.wallet.account import Account as LBCAccount
+from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies
+from lbrynet.p2p.StreamDescriptor import download_sd_blob
+from lbrynet.p2p.Error import InsufficientFundsError, UnknownNameError, DownloadDataTimeout, DownloadSDTimeout
+from lbrynet.p2p.Error import NullFundsError, NegativeFundsError, ResolveError
+from lbrynet.p2p.Peer import Peer
+from lbrynet.p2p.SinglePeerDownloader import SinglePeerDownloader
+from lbrynet.p2p.client.StandaloneBlobDownloader import StandaloneBlobDownloader
 from lbrynet.schema.claim import ClaimDict
 from lbrynet.schema.uri import parse_lbry_uri
 from lbrynet.schema.error import URIParseError, DecodeError
@@ -23,32 +44,6 @@ from lbrynet.schema.validator import validate_claim_id
 from lbrynet.schema.address import decode_address
 from lbrynet.schema.decode import smart_decode
 
-# TODO: importing this when internet is disabled raises a socket.gaierror
-from lbrynet.p2p.system_info import get_lbrynet_version
-from lbrynet.extras.daemon import conf
-from lbrynet.extras.reflector import reupload
-from .Components import d2f, f2d
-from .Components import WALLET_COMPONENT, DATABASE_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT
-from .Components import STREAM_IDENTIFIER_COMPONENT, FILE_MANAGER_COMPONENT, RATE_LIMITER_COMPONENT
-from .Components import EXCHANGE_RATE_MANAGER_COMPONENT, PAYMENT_RATE_COMPONENT, UPNP_COMPONENT
-from .ComponentManager import RequiredCondition
-from .Downloader import GetStream
-from .Publisher import Publisher
-from .auth.server import AuthJSONRPCServer
-from lbrynet.p2p import utils, system_info
-from lbrynet.p2p.StreamDescriptor import download_sd_blob
-from lbrynet.p2p.Error import InsufficientFundsError, UnknownNameError
-from lbrynet.p2p.Error import DownloadDataTimeout, DownloadSDTimeout
-from lbrynet.p2p.Error import NullFundsError, NegativeFundsError
-from lbrynet.p2p.Error import ResolveError
-from lbrynet.dht.error import TimeoutError
-from lbrynet.p2p.Peer import Peer
-from lbrynet.p2p.SinglePeerDownloader import SinglePeerDownloader
-from lbrynet.p2p.client.StandaloneBlobDownloader import StandaloneBlobDownloader
-from lbrynet.extras.wallet import LbryWalletManager
-from lbrynet.extras.wallet.account import Account as LBCAccount
-from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies
-
 log = logging.getLogger(__name__)
 requires = AuthJSONRPCServer.requires
 
@@ -210,7 +205,6 @@ class Daemon(AuthJSONRPCServer):
         DATABASE_COMPONENT: "storage",
         DHT_COMPONENT: "dht_node",
         WALLET_COMPONENT: "wallet_manager",
-        STREAM_IDENTIFIER_COMPONENT: "sd_identifier",
         FILE_MANAGER_COMPONENT: "file_manager",
         EXCHANGE_RATE_MANAGER_COMPONENT: "exchange_rate_manager",
         PAYMENT_RATE_COMPONENT: "payment_rate_manager",
@@ -220,7 +214,7 @@ class Daemon(AuthJSONRPCServer):
     }
 
     def __init__(self, analytics_manager=None, component_manager=None):
-        to_skip = list(conf.settings['components_to_skip'])
+        to_skip = conf.settings['components_to_skip']
         if 'reflector' not in to_skip and not conf.settings['run_reflector_server']:
             to_skip.append('reflector')
         looping_calls = {
@@ -241,7 +235,6 @@ class Daemon(AuthJSONRPCServer):
         self.storage = None
         self.dht_node = None
         self.wallet_manager: LbryWalletManager = None
-        self.sd_identifier = None
         self.file_manager = None
         self.exchange_rate_manager = None
         self.payment_rate_manager = None
@@ -305,7 +298,7 @@ class Daemon(AuthJSONRPCServer):
         rate_manager = rate_manager or self.payment_rate_manager
         timeout = timeout or 30
         downloader = StandaloneBlobDownloader(
-            blob_hash, self.blob_manager, self.dht_node.peer_finder, self.rate_limiter,
+            blob_hash, self.blob_manager, self.component_manager.peer_finder, self.rate_limiter,
             rate_manager, self.wallet_manager, timeout
         )
         return downloader.download()
@@ -372,8 +365,8 @@ class Daemon(AuthJSONRPCServer):
             self.analytics_manager.send_download_started(download_id, name, claim_dict)
             self.analytics_manager.send_new_download_start(download_id, name, claim_dict)
             self.streams[sd_hash] = GetStream(
-                self.sd_identifier, self.wallet_manager, self.exchange_rate_manager, self.blob_manager,
+                self.file_manager.sd_identifier, self.wallet_manager, self.exchange_rate_manager, self.blob_manager,
-                self.dht_node.peer_finder, self.rate_limiter, self.payment_rate_manager, self.storage,
+                self.component_manager.peer_finder, self.rate_limiter, self.payment_rate_manager, self.storage,
                 conf.settings['max_key_fee'], conf.settings['disable_max_key_fee'], conf.settings['data_rate'],
                 timeout
             )
@@ -432,7 +425,7 @@ class Daemon(AuthJSONRPCServer):
         if blob:
             return self.blob_manager.get_blob(blob[0])
         return download_sd_blob(
-            sd_hash.decode(), self.blob_manager, self.dht_node.peer_finder, self.rate_limiter,
+            sd_hash.decode(), self.blob_manager, self.component_manager.peer_finder, self.rate_limiter,
             self.payment_rate_manager, self.wallet_manager, timeout=conf.settings['peer_search_timeout'],
             download_mirrors=conf.settings['download_mirrors']
         )
@@ -450,7 +443,7 @@ class Daemon(AuthJSONRPCServer):
         Get total stream size in bytes from a sd blob
         """
 
-        d = self.sd_identifier.get_metadata_for_sd_blob(sd_blob)
+        d = self.file_manager.sd_identifier.get_metadata_for_sd_blob(sd_blob)
         d.addCallback(lambda metadata: metadata.validator.info_to_show())
         d.addCallback(lambda info: int(dict(info)['stream_size']))
         return d
@@ -468,9 +461,7 @@ class Daemon(AuthJSONRPCServer):
         """
         Calculate estimated LBC cost for a stream given its size in bytes
         """
-
         cost = self._get_est_cost_from_stream_size(size)
-
         resolved = await self.wallet_manager.resolve(uri)
 
         if uri in resolved and 'claim' in resolved[uri]:
@@ -535,7 +526,6 @@ class Daemon(AuthJSONRPCServer):
         sd blob will be downloaded to determine the stream size
 
         """
-
         if size is not None:
             return self.get_est_cost_using_known_size(uri, size)
         return self.get_est_cost_from_uri(uri)
@@ -819,7 +809,7 @@ class Daemon(AuthJSONRPCServer):
                 'ip': (str) remote ip, if available,
                 'lbrynet_version': (str) lbrynet_version,
                 'lbryum_version': (str) lbryum_version,
-                'lbrynet.schema_version': (str) lbrynet.schema_version,
+                'lbryschema_version': (str) lbryschema_version,
                 'os_release': (str) os release string
                 'os_system': (str) os name
                 'platform': (str) platform string
@@ -851,7 +841,7 @@ class Daemon(AuthJSONRPCServer):
             message,
             conf.settings.installation_id,
             platform_name,
-            get_lbrynet_version()
+            __version__
         )
         return self._render_response(True)
 
@@ -1776,8 +1766,8 @@ class Daemon(AuthJSONRPCServer):
             results[resolved_uri] = resolved[resolved_uri]
         return results
 
-    @requires(STREAM_IDENTIFIER_COMPONENT, WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT,
+    @requires(WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT,
-              DHT_COMPONENT, RATE_LIMITER_COMPONENT, PAYMENT_RATE_COMPONENT, DATABASE_COMPONENT,
+              RATE_LIMITER_COMPONENT, PAYMENT_RATE_COMPONENT, DATABASE_COMPONENT,
               conditions=[WALLET_IS_UNLOCKED])
     async def jsonrpc_get(self, uri, file_name=None, timeout=None):
         """
@@ -1962,7 +1952,7 @@ class Daemon(AuthJSONRPCServer):
         response = yield self._render_response(result)
         defer.returnValue(response)
 
-    @requires(STREAM_IDENTIFIER_COMPONENT, WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT,
+    @requires(WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT,
               DHT_COMPONENT, RATE_LIMITER_COMPONENT, PAYMENT_RATE_COMPONENT, DATABASE_COMPONENT,
               conditions=[WALLET_IS_UNLOCKED])
     def jsonrpc_stream_cost_estimate(self, uri, size=None):

lbrynet/extras/daemon/DaemonConsole.py

@@ -6,11 +6,10 @@ import logging.handlers
 from twisted.internet import defer, reactor, threads
 from aiohttp import client_exceptions
 
-from lbrynet.extras.daemon import analytics, conf
+from lbrynet import utils, conf, log_support
-from lbrynet.p2p import utils
+from lbrynet.extras.daemon import analytics
-from lbrynet.p2p import log_support
+from lbrynet.extras.daemon.auth.client import LBRYAPIClient
-from .auth.client import LBRYAPIClient
+from lbrynet.extras.daemon.Daemon import Daemon
-from .Daemon import Daemon
 
 log = logging.getLogger(__name__)
 

lbrynet/extras/daemon/DaemonControl.py

@@ -7,16 +7,14 @@ if 'win' in sys.platform:
     import certifi
     os.environ['SSL_CERT_FILE'] = certifi.where()
 
-from lbrynet.p2p import log_support
 
 import argparse
 import logging.handlers
 
 from twisted.internet import reactor
-
+from lbrynet import utils, conf, log_support
-from lbrynet.extras.daemon import conf
+from lbrynet.extras import system_info
-from lbrynet.p2p import utils, system_info
+from lbrynet.extras.daemon.Daemon import Daemon
-from .Daemon import Daemon
 
 log = logging.getLogger(__name__)
 
@@ -54,7 +52,7 @@ def start(argv=None, conf_path=None):
         conf.settings.update({'use_auth_http': args.useauth}, data_types=(conf.TYPE_CLI,))
 
     if args.version:
-        version = system_info.get_platform(get_ip=False)
+        version = system_info.get_platform()
         version['installation_id'] = conf.settings.installation_id
         print(utils.json_dumps_pretty(version))
         return

lbrynet/extras/daemon/Downloader.py

@@ -3,17 +3,17 @@ import os
 from twisted.internet import defer
 from twisted.internet.task import LoopingCall
 
-from .Components import f2d
+from lbrynet import conf
 from lbrynet.schema.fee import Fee
 
 from lbrynet.p2p.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed, InvalidStreamDescriptorError
 from lbrynet.p2p.Error import DownloadDataTimeout, DownloadCanceledError, DownloadSDTimeout
-from lbrynet.p2p.utils import safe_start_looping_call, safe_stop_looping_call
+from lbrynet.utils import safe_start_looping_call, safe_stop_looping_call
 from lbrynet.p2p.StreamDescriptor import download_sd_blob
 from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
-from lbrynet.extras.daemon import conf
 from torba.client.constants import COIN
 from lbrynet.extras.wallet.dewies import dewies_to_lbc
+from lbrynet.extras.daemon.Components import f2d
 
 INITIALIZING_CODE = 'initializing'
 DOWNLOAD_METADATA_CODE = 'downloading_metadata'

lbrynet/extras/daemon/HashAnnouncer.py

@@ -2,8 +2,7 @@ import binascii
 import logging
 
 from twisted.internet import defer, task
-from lbrynet.p2p import utils
+from lbrynet import utils, conf
-from lbrynet.extras.daemon import conf
 
 log = logging.getLogger(__name__)
 

lbrynet/extras/daemon/PeerFinder.py

@@ -2,7 +2,7 @@ import binascii
 import logging
 
 from twisted.internet import defer
-from lbrynet.extras.daemon import conf
+from lbrynet import conf
 
 log = logging.getLogger(__name__)
 
@@ -18,13 +18,12 @@ class DHTPeerFinder(DummyPeerFinder):
     """This class finds peers which have announced to the DHT that they have certain blobs"""
     #implements(IPeerFinder)
 
-    def __init__(self, dht_node, peer_manager):
+    def __init__(self, component_manager):
         """
-        dht_node - an instance of dht.Node class
+        component_manager - an instance of ComponentManager
-        peer_manager - an instance of PeerManager class
         """
-        self.dht_node = dht_node
+        self.component_manager = component_manager
-        self.peer_manager = peer_manager
+        self.peer_manager = component_manager.peer_manager
         self.peers = {}
         self._ongoing_searchs = {}
 
@@ -39,19 +38,30 @@ class DHTPeerFinder(DummyPeerFinder):
         Returns:
         list of peers for the blob
         """
-        self.peers.setdefault(blob_hash, {(self.dht_node.externalIP, self.dht_node.peerPort,)})
+        if "dht" in self.component_manager.skip_components:
+            return defer.succeed([])
+        if not self.component_manager.all_components_running("dht"):
+            return defer.succeed([])
+        dht_node = self.component_manager.get_component("dht")
+
+        self.peers.setdefault(blob_hash, {(dht_node.externalIP, dht_node.peerPort,)})
         if not blob_hash in self._ongoing_searchs or self._ongoing_searchs[blob_hash].called:
-            self._ongoing_searchs[blob_hash] = self._execute_peer_search(blob_hash, timeout)
+            self._ongoing_searchs[blob_hash] = self._execute_peer_search(dht_node, blob_hash, timeout)
-        peers = set(self._filter_self(blob_hash) if filter_self else self.peers[blob_hash])
+
+        def _filter_self(blob_hash):
+            my_host, my_port = dht_node.externalIP, dht_node.peerPort
+            return {(host, port) for host, port in self.peers[blob_hash] if (host, port) != (my_host, my_port)}
+
+        peers = set(_filter_self(blob_hash) if filter_self else self.peers[blob_hash])
         return defer.succeed([self.peer_manager.get_peer(*peer) for peer in peers])
 
     @defer.inlineCallbacks
-    def _execute_peer_search(self, blob_hash, timeout):
+    def _execute_peer_search(self, dht_node, blob_hash, timeout):
         bin_hash = binascii.unhexlify(blob_hash)
-        finished_deferred = self.dht_node.iterativeFindValue(bin_hash, exclude=self.peers[blob_hash])
+        finished_deferred = dht_node.iterativeFindValue(bin_hash, exclude=self.peers[blob_hash])
         timeout = timeout or conf.settings['peer_search_timeout']
         if timeout:
-            finished_deferred.addTimeout(timeout, self.dht_node.clock)
+            finished_deferred.addTimeout(timeout, dht_node.clock)
         try:
             peer_list = yield finished_deferred
             self.peers[blob_hash].update({(host, port) for _, host, port in peer_list})
@@ -59,7 +69,3 @@ class DHTPeerFinder(DummyPeerFinder):
             log.debug("DHT timed out while looking peers for blob %s after %s seconds", blob_hash, timeout)
         finally:
             del self._ongoing_searchs[blob_hash]
-
-    def _filter_self(self, blob_hash):
-        my_host, my_port = self.dht_node.externalIP, self.dht_node.peerPort
-        return {(host, port) for host, port in self.peers[blob_hash] if (host, port) != (my_host, my_port)}

lbrynet/extras/daemon/__init__.py

@@ -1 +1 @@
-from . import Components  # register Component classes
+from lbrynet.extras.daemon import Components  # register Component classes

lbrynet/extras/daemon/analytics.py

@@ -4,8 +4,8 @@ import logging
 import treq
 from twisted.internet import defer, task
 
-from lbrynet.extras.daemon import conf
+from lbrynet import conf, utils
-from lbrynet.p2p import looping_call_manager, utils, system_info
+from lbrynet.extras import looping_call_manager, system_info
 
 # Things We Track
 SERVER_STARTUP = 'Server Startup'
@@ -136,8 +136,8 @@ class Manager:
 
     def _get_looping_calls(self):
         return [
-            ('send_heartbeat', self._send_heartbeat, 60),
+            ('send_heartbeat', self._send_heartbeat, 300),
-            ('update_tracked_metrics', self._update_tracked_metrics, 300),
+            ('update_tracked_metrics', self._update_tracked_metrics, 600),
         ]
 
     def _setup_looping_calls(self):

lbrynet/extras/daemon/auth/auth.py

@@ -3,7 +3,7 @@ from zope.interface import implementer
 from twisted.cred import portal, checkers, credentials, error as cred_error
 from twisted.internet import defer
 from twisted.web import resource
-from .keyring import Keyring
+from lbrynet.extras.daemon.auth.keyring import Keyring
 
 log = logging.getLogger(__name__)
 

lbrynet/extras/daemon/auth/client.py

@@ -3,8 +3,8 @@ import aiohttp
 import logging
 from urllib.parse import urlparse
 
-from lbrynet.extras.daemon import conf
+from lbrynet import conf
-from .keyring import Keyring, APIKey
+from lbrynet.extras.daemon.auth.keyring import Keyring, APIKey
 
 log = logging.getLogger(__name__)
 USER_AGENT = "AuthServiceProxy/0.1"

lbrynet/extras/daemon/auth/factory.py

@@ -3,9 +3,9 @@ import logging
 from twisted.web import server, guard, resource
 from twisted.cred import portal
 
-from lbrynet.extras.daemon import conf
+from lbrynet import conf
-from .auth import PasswordChecker, HttpPasswordRealm
+from lbrynet.extras.daemon.auth.auth import PasswordChecker, HttpPasswordRealm
-from ..auth.keyring import Keyring
+from lbrynet.extras.daemon.auth.keyring import Keyring
 
 log = logging.getLogger(__name__)
 

lbrynet/extras/daemon/auth/keyring.py

@@ -3,6 +3,7 @@ import datetime
 import hmac
 import hashlib
 import base58
+import logging
 from OpenSSL.crypto import FILETYPE_PEM
 from ssl import create_default_context, SSLContext
 from cryptography.hazmat.backends import default_backend
@@ -12,8 +13,8 @@ from cryptography.x509.name import NameOID, NameAttribute
 from cryptography.hazmat.primitives.asymmetric import rsa
 from cryptography.hazmat.primitives import serialization
 from twisted.internet import ssl
-import logging
+
-from lbrynet.extras.daemon import conf
+from lbrynet import conf
 
 log = logging.getLogger(__name__)
 

lbrynet/extras/daemon/auth/server.py

@@ -4,7 +4,6 @@ from six.moves.urllib import parse as urlparse
 import json
 import inspect
 import signal
-
 from functools import wraps
 from twisted.web import server
 from twisted.internet import defer
@@ -14,16 +13,17 @@ from twisted.internet.error import ConnectionDone, ConnectionLost
 from txjsonrpc import jsonrpclib
 from traceback import format_exc
 
-from lbrynet.extras.daemon import analytics, conf
+from lbrynet import conf, utils
+from lbrynet.extras.daemon import analytics
 from lbrynet.p2p.Error import InvalidAuthenticationToken
-from lbrynet.p2p import utils
 from lbrynet.p2p.Error import ComponentsNotStarted, ComponentStartConditionNotMet
-from lbrynet.p2p.looping_call_manager import LoopingCallManager
+from lbrynet.extras.looping_call_manager import LoopingCallManager
 from lbrynet.extras.daemon.ComponentManager import ComponentManager
-from .keyring import APIKey, Keyring
+from lbrynet.extras.daemon.auth.keyring import APIKey, Keyring
-from .undecorated import undecorated
+from lbrynet.extras.daemon.auth.undecorated import undecorated
-from .factory import AuthJSONRPCResource
+from lbrynet.extras.daemon.auth.factory import AuthJSONRPCResource
 from lbrynet.extras.daemon.json_response_encoder import JSONResponseEncoder
+
 log = logging.getLogger(__name__)
 
 EMPTY_PARAMS = [{}]

lbrynet/extras/daemon/migrator/migrate5to6.py

@@ -2,8 +2,8 @@ import sqlite3
 import os
 import json
 import logging
+from lbrynet import conf
 from lbrynet.schema.decode import smart_decode
-from lbrynet.extras.daemon import conf
 
 log = logging.getLogger(__name__)
 

lbrynet/extras/daemon/storage.py

@@ -6,13 +6,13 @@ from binascii import hexlify, unhexlify
 from decimal import Decimal
 from twisted.internet import defer, task, threads
 from twisted.enterprise import adbapi
+from torba.client.constants import COIN
 
+from lbrynet import conf
 from lbrynet.schema.claim import ClaimDict
 from lbrynet.schema.decode import smart_decode
-from lbrynet.extras.daemon import conf
 from lbrynet.blob.CryptBlob import CryptBlobInfo
 from lbrynet.dht.constants import dataExpireTimeout
-from torba.client.constants import COIN
 
 log = logging.getLogger(__name__)
 

lbrynet/extras/reflector/__init__.py

@@ -63,6 +63,3 @@ If the transfer was not successful (False), the blob is re-added to the needed_b
 Blob requests continue for each of the blobs the client has queued to send, when completed
 the client disconnects.
 """
-from .server.server import ReflectorServerFactory as ServerFactory
-from .client.client import EncryptedFileReflectorClientFactory as ClientFactory
-from .client.blob import BlobReflectorClientFactory as BlobClientFactory

lbrynet/extras/reflector/reupload.py

@@ -1,8 +1,9 @@
 import random
 
 from twisted.internet import reactor, defer
-from lbrynet.extras.daemon import conf
+from lbrynet import conf
-from . import ClientFactory, BlobClientFactory
+from lbrynet.extras.reflector.client.client import EncryptedFileReflectorClientFactory
+from lbrynet.extras.reflector.client.blob import BlobReflectorClientFactory
 
 
 def _is_ip(host):
@@ -26,7 +27,7 @@ def resolve(host):
 @defer.inlineCallbacks
 def _reflect_stream(blob_manager, stream_hash, sd_hash, reflector_server):
     reflector_address, reflector_port = reflector_server[0], reflector_server[1]
-    factory = ClientFactory(blob_manager, stream_hash, sd_hash)
+    factory = EncryptedFileReflectorClientFactory(blob_manager, stream_hash, sd_hash)
     ip = yield resolve(reflector_address)
     yield reactor.connectTCP(ip, reflector_port, factory)
     result = yield factory.finished_deferred
@@ -40,7 +41,7 @@ def _reflect_file(lbry_file, reflector_server):
 @defer.inlineCallbacks
 def _reflect_blobs(blob_manager, blob_hashes, reflector_server):
     reflector_address, reflector_port = reflector_server[0], reflector_server[1]
-    factory = BlobClientFactory(blob_manager, blob_hashes)
+    factory = BlobReflectorClientFactory(blob_manager, blob_hashes)
     ip = yield resolve(reflector_address)
     yield reactor.connectTCP(ip, reflector_port, factory)
     result = yield factory.finished_deferred

lbrynet/extras/reflector/server/server.py

@@ -3,7 +3,7 @@ import json
 from twisted.python import failure
 from twisted.internet import error, defer
 from twisted.internet.protocol import Protocol, ServerFactory
-from lbrynet.p2p.utils import is_valid_blobhash
+from lbrynet.blob.blob_file import is_valid_blobhash
 from lbrynet.p2p.Error import DownloadCanceledError, InvalidBlobHashError
 from lbrynet.p2p.StreamDescriptor import BlobStreamDescriptorReader
 from lbrynet.p2p.StreamDescriptor import save_sd_info

lbrynet/extras/system_info.py

@@ -0,0 +1,30 @@
+import platform
+import os
+import logging.handlers
+
+from lbrynet.schema import __version__ as schema_version
+from lbrynet import build_type, __version__ as lbrynet_version
+
+log = logging.getLogger(__name__)
+
+
+def get_platform() -> dict:
+    p = {
+        "processor": platform.processor(),
+        "python_version": platform.python_version(),
+        "platform": platform.platform(),
+        "os_release": platform.release(),
+        "os_system": platform.system(),
+        "lbrynet_version": lbrynet_version,
+        "lbryschema_version": schema_version,
+        "build": build_type.BUILD,  # CI server sets this during build step
+    }
+    if p["os_system"] == "Linux":
+        try:
+            import distro
+            p["distro"] = distro.info()
+            p["desktop"] = os.environ.get('XDG_CURRENT_DESKTOP', 'Unknown')
+        except ModuleNotFoundError:
+            pass
+
+    return p

lbrynet/extras/wallet/__init__.py

@@ -6,6 +6,6 @@ __node_url__ = (
 )
 __spvserver__ = 'lbrynet.extras.wallet.server.coin.LBCRegTest'
 
-from .ledger import MainNetLedger, RegTestLedger
+from lbrynet.extras.wallet.ledger import MainNetLedger, RegTestLedger
-from .manager import LbryWalletManager
+from lbrynet.extras.wallet.manager import LbryWalletManager
-from .network import Network
+from lbrynet.extras.wallet.network import Network

lbrynet/extras/wallet/ledger.py

@@ -2,17 +2,16 @@ import asyncio
 import logging
 from binascii import unhexlify
 
-from lbrynet.extras.wallet.dewies import dewies_to_lbc
+from torba.client.baseledger import BaseLedger
 from lbrynet.schema.error import URIParseError
 from lbrynet.schema.uri import parse_lbry_uri
-from torba.client.baseledger import BaseLedger
+from lbrynet.extras.wallet.dewies import dewies_to_lbc
-
+from lbrynet.extras.wallet.resolve import Resolver
-from .resolve import Resolver
+from lbrynet.extras.wallet.account import Account
-from .account import Account
+from lbrynet.extras.wallet.network import Network
-from .network import Network
+from lbrynet.extras.wallet.database import WalletDatabase
-from .database import WalletDatabase
+from lbrynet.extras.wallet.transaction import Transaction
-from .transaction import Transaction
+from lbrynet.extras.wallet.header import Headers, UnvalidatedHeaders
-from .header import Headers, UnvalidatedHeaders
 
 
 log = logging.getLogger(__name__)

lbrynet/extras/wallet/manager.py

@@ -14,11 +14,11 @@ from torba.client.basemanager import BaseWalletManager
 
 from lbrynet.schema.claim import ClaimDict
 
-from .ledger import MainNetLedger
+from lbrynet.extras.wallet.ledger import MainNetLedger
-from .account import BaseAccount, generate_certificate
+from lbrynet.extras.wallet.account import BaseAccount, generate_certificate
-from .transaction import Transaction
+from lbrynet.extras.wallet.transaction import Transaction
-from .database import WalletDatabase
+from lbrynet.extras.wallet.database import WalletDatabase
-from .dewies import dewies_to_lbc
+from lbrynet.extras.wallet.dewies import dewies_to_lbc
 
 
 log = logging.getLogger(__name__)

lbrynet/extras/wallet/resolve.py

@@ -9,8 +9,7 @@ from lbrynet.schema.claim import ClaimDict
 from lbrynet.schema.decode import smart_decode
 from lbrynet.schema.error import DecodeError
 from lbrynet.schema.uri import parse_lbry_uri
-
+from lbrynet.extras.wallet.claim_proofs import verify_proof, InvalidProofError
-from .claim_proofs import verify_proof, InvalidProofError
 log = logging.getLogger(__name__)
 
 

lbrynet/extras/wallet/server/block_processor.py

@@ -9,7 +9,7 @@ from lbrynet.schema.proto.claim_pb2 import Claim
 from lbrynet.schema.uri import parse_lbry_uri
 from lbrynet.schema.decode import smart_decode
 
-from .model import NameClaim, ClaimInfo, ClaimUpdate, ClaimSupport
+from lbrynet.extras.wallet.server.model import NameClaim, ClaimInfo, ClaimUpdate, ClaimSupport
 
 
 class LBRYBlockProcessor(BlockProcessor):

lbrynet/extras/wallet/server/coin.py

@@ -6,7 +6,7 @@ from torba.server.hash import hash_to_hex_str, HASHX_LEN
 from hashlib import sha256
 from torba.server.coins import Coin, CoinError
 
-from .opcodes import decode_claim_script, opcodes as lbry_opcodes
+from lbrynet.extras.wallet.server.opcodes import decode_claim_script, opcodes as lbry_opcodes
 
 
 class LBC(Coin):

lbrynet/extras/wallet/server/db.py

@@ -6,7 +6,7 @@ from torba.server.hash import hash_to_hex_str
 
 from torba.server.db import DB
 
-from .model import ClaimInfo
+from lbrynet.extras.wallet.server.model import ClaimInfo
 
 
 class LBRYDB(DB):

lbrynet/extras/wallet/server/opcodes.py

@@ -1,6 +1,6 @@
 import struct
 from torba.server.enum import Enumeration
-from .model import NameClaim, ClaimSupport, ClaimUpdate
+from lbrynet.extras.wallet.server.model import NameClaim, ClaimSupport, ClaimUpdate
 # TODO: Take this to lbryschema (it's also on lbryum and lbryum-server)
 
 

lbrynet/extras/wallet/server/session.py

@@ -8,8 +8,8 @@ from torba.server import util
 
 from lbrynet.schema.uri import parse_lbry_uri
 from lbrynet.schema.error import URIParseError
-from .block_processor import LBRYBlockProcessor
+from lbrynet.extras.wallet.server.block_processor import LBRYBlockProcessor
-from .db import LBRYDB
+from lbrynet.extras.wallet.server.db import LBRYDB
 
 
 class LBRYElectrumX(ElectrumX):

lbrynet/extras/wallet/server/tx.py

@@ -1,6 +1,6 @@
 from torba.server.tx import Deserializer
-from .opcodes import decode_claim_script
+from lbrynet.extras.wallet.server.opcodes import decode_claim_script
-from .model import TxClaimOutput, LBRYTx
+from lbrynet.extras.wallet.server.model import TxClaimOutput, LBRYTx
 
 
 class LBRYDeserializer(Deserializer):

lbrynet/extras/wallet/transaction.py

@@ -2,13 +2,12 @@ import struct
 from binascii import hexlify, unhexlify
 from typing import List, Iterable, Optional
 
-from lbrynet.schema.decode import smart_decode
-from .account import Account
 from torba.client.basetransaction import BaseTransaction, BaseInput, BaseOutput
 from torba.client.hash import hash160
-
+from lbrynet.schema.decode import smart_decode
 from lbrynet.schema.claim import ClaimDict
-from .script import InputScript, OutputScript
+from lbrynet.extras.wallet.account import Account
+from lbrynet.extras.wallet.script import InputScript, OutputScript
 
 
 class Input(BaseInput):

lbrynet/log_support.py

@@ -3,14 +3,11 @@ import logging
 import logging.handlers
 import sys
 import traceback
-
 import treq
-from twisted.internet import defer
 import twisted.python.log
-
+from twisted.internet import defer
 from lbrynet import __version__ as lbrynet_version, build_type
-from lbrynet.extras.daemon import conf
+from lbrynet import utils, conf
-from lbrynet.p2p import utils
 
 
 class HTTPSHandler(logging.Handler):

lbrynet/p2p/BlobAvailability.py

@@ -1,10 +1,9 @@
 import logging
 import random
 import time
-
+from decimal import Decimal
 from twisted.internet import defer
 from twisted.internet.task import LoopingCall
-from decimal import Decimal
 
 log = logging.getLogger(__name__)
 

lbrynet/p2p/HTTPBlobDownloader.py

@@ -1,10 +1,11 @@
-from random import choice
 import logging
+import treq
+from random import choice
 from twisted.internet import defer, task
 from twisted.internet.error import ConnectingCancelledError
 from twisted.web._newclient import ResponseNeverReceived
-import treq
+
-from lbrynet.p2p.utils import DeferredDict
+from lbrynet.utils import DeferredDict
 from lbrynet.p2p.Error import DownloadCanceledError
 
 log = logging.getLogger(__name__)

lbrynet/p2p/PaymentRateManager.py

@@ -1,6 +1,6 @@
-from lbrynet.p2p.Strategy import get_default_strategy, OnlyFreeStrategy
-from lbrynet.extras.daemon import conf
 from decimal import Decimal
+from lbrynet import conf
+from lbrynet.p2p.Strategy import get_default_strategy, OnlyFreeStrategy
 
 
 class BasePaymentRateManager:

lbrynet/p2p/Peer.py

@@ -1,6 +1,6 @@
 import datetime
 from collections import defaultdict
-from lbrynet.p2p import utils
+from lbrynet import utils
 
 # Do not create this object except through PeerManager
 class Peer:

lbrynet/p2p/PriceModel.py

@@ -1,6 +1,5 @@
 from decimal import Decimal
-
+from lbrynet import conf
-from lbrynet.extras.daemon import conf
 
 
 def get_default_price_model(blob_tracker, base_price, **kwargs):

lbrynet/p2p/SinglePeerDownloader.py

@@ -4,7 +4,7 @@ import tempfile
 
 from twisted.internet import defer, threads, reactor
 
-from lbrynet.blob import BlobFile
+from lbrynet.blob.blob_file import BlobFile
 from lbrynet.p2p.BlobManager import DiskBlobManager
 from lbrynet.p2p.RateLimiter import DummyRateLimiter
 from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager
@@ -12,7 +12,7 @@ from lbrynet.p2p.client.BlobRequester import BlobRequester
 from lbrynet.p2p.client.StandaloneBlobDownloader import StandaloneBlobDownloader
 from lbrynet.p2p.client.ConnectionManager import ConnectionManager
 from lbrynet.extras.daemon.storage import SQLiteStorage
-from lbrynet.dht.peerfinder import DummyPeerFinder
+from lbrynet.extras.daemon.PeerFinder import DummyPeerFinder
 
 
 log = logging.getLogger(__name__)

lbrynet/p2p/Strategy.py

@@ -1,5 +1,5 @@
 from decimal import Decimal
-from lbrynet.extras.daemon import conf
+from lbrynet import conf
 from lbrynet.p2p.Offer import Offer
 from lbrynet.p2p.PriceModel import MeanAvailabilityWeightedPrice, ZeroPrice
 
@@ -133,7 +133,6 @@ class BasicAvailabilityWeightedStrategy(Strategy):
 
 
 class OnlyFreeStrategy(Strategy):
-
     def __init__(self, *args, **kwargs):
         price_model = ZeroPrice()
         super().__init__(price_model, 0.0, 0.0, True)

lbrynet/p2p/StreamDescriptor.py

@@ -1,11 +1,11 @@
-from binascii import unhexlify
 import string
-from collections import defaultdict
 import json
 import logging
-
+from collections import defaultdict
+from binascii import unhexlify
 from twisted.internet import threads, defer
-from lbrynet.p2p.cryptoutils import get_lbry_hash_obj
+
+from lbrynet.cryptoutils import get_lbry_hash_obj
 from lbrynet.p2p.client.StandaloneBlobDownloader import StandaloneBlobDownloader
 from lbrynet.p2p.Error import UnknownStreamTypeError, InvalidStreamDescriptorError
 from lbrynet.p2p.HTTPBlobDownloader import HTTPBlobDownloader

lbrynet/p2p/__init__.py

@@ -5,5 +5,3 @@ This includes classes for connecting to other peers and downloading blobs from t
 connections from peers and responding to their requests, managing locally stored blobs, sending
 and receiving payments, and locating peers in the DHT.
 """
-
-from lbrynet import custom_logger

lbrynet/p2p/client/ClientProtocol.py

@@ -5,8 +5,7 @@ from twisted.internet import error, defer
 from twisted.internet.protocol import Protocol, ClientFactory
 from twisted.protocols.policies import TimeoutMixin
 from twisted.python import failure
-from lbrynet.extras.daemon import conf
+from lbrynet import conf, utils
-from lbrynet.p2p import utils
 from lbrynet.p2p.Error import ConnectionClosedBeforeResponseError, NoResponseError
 from lbrynet.p2p.Error import DownloadCanceledError, MisbehavingPeerError
 from lbrynet.p2p.Error import RequestCanceledError

lbrynet/p2p/client/ConnectionManager.py

@@ -1,10 +1,9 @@
 import random
 import logging
 from twisted.internet import defer, reactor
-from lbrynet.extras.daemon import conf
+from lbrynet import utils, conf
 from lbrynet.p2p.client.ClientProtocol import ClientProtocolFactory
 from lbrynet.p2p.Error import InsufficientFundsError
-from lbrynet.p2p import utils
 
 log = logging.getLogger(__name__)
 

lbrynet/p2p/client/StandaloneBlobDownloader.py

@@ -4,7 +4,8 @@ from lbrynet.p2p.client.BlobRequester import BlobRequester
 from lbrynet.p2p.client.ConnectionManager import ConnectionManager
 from lbrynet.p2p.client.DownloadManager import DownloadManager
 from lbrynet.p2p.Error import InvalidBlobHashError, DownloadSDTimeout
-from lbrynet.p2p.utils import is_valid_blobhash, safe_start_looping_call, safe_stop_looping_call
+from lbrynet.blob.blob_file import is_valid_blobhash
+from lbrynet.utils import safe_start_looping_call, safe_stop_looping_call
 from twisted.python.failure import Failure
 from twisted.internet import defer
 from twisted.internet.task import LoopingCall

lbrynet/p2p/system_info.py

@@ -1,59 +0,0 @@
-import platform
-import json
-import subprocess
-import os
-
-from six.moves.urllib import request
-from six.moves.urllib.error import URLError
-from lbrynet.schema import __version__ as schema_version
-from lbrynet import build_type, __version__ as lbrynet_version
-from lbrynet.extras.daemon.conf import ROOT_DIR
-import logging.handlers
-
-log = logging.getLogger(__name__)
-
-
-def get_lbrynet_version() -> str:
-    if build_type.BUILD == "dev":
-        try:
-            with open(os.devnull, 'w') as devnull:
-                git_dir = ROOT_DIR + '/.git'
-                return subprocess.check_output(
-                    ['git', '--git-dir='+git_dir, 'describe', '--dirty', '--always'],
-                    stderr=devnull
-                ).decode().strip().lstrip('v')
-        except (subprocess.CalledProcessError, OSError):
-            log.debug("failed to get version from git")
-    return lbrynet_version
-
-
-def get_platform(get_ip: bool = True) -> dict:
-    p = {
-        "processor": platform.processor(),
-        "python_version": platform.python_version(),
-        "platform": platform.platform(),
-        "os_release": platform.release(),
-        "os_system": platform.system(),
-        "lbrynet_version": get_lbrynet_version(),
-        "lbryschema_version": schema_version,
-        "build": build_type.BUILD,  # CI server sets this during build step
-    }
-    if p["os_system"] == "Linux":
-        try:
-            import distro
-            p["distro"] = distro.info()
-            p["desktop"] = os.environ.get('XDG_CURRENT_DESKTOP', 'Unknown')
-        except ModuleNotFoundError:
-            pass
-
-    # TODO: remove this from get_platform and add a get_external_ip function using treq
-    if get_ip:
-        try:
-            response = json.loads(request.urlopen("https://api.lbry.io/ip").read())
-            if not response['success']:
-                raise URLError("failed to get external ip")
-            p['ip'] = response['data']['ip']
-        except (URLError, AssertionError):
-            p['ip'] = "Could not determine IP"
-
-    return p

lbrynet/schema/decode.py

@@ -1,13 +1,12 @@
 import json
 import binascii
-
+from google.protobuf import json_format  # pylint: disable=no-name-in-module
 import six
 
 from lbrynet.schema.error import DecodeError, InvalidAddress
 from lbrynet.schema.legacy.migrate import migrate as schema_migrator
 from lbrynet.schema.claim import ClaimDict
 
-from google.protobuf import json_format  # pylint: disable=no-name-in-module
 
 
 def migrate_json_claim_value(decoded_json):

lbrynet/schema/signature/flags.py

@@ -1,41 +0,0 @@
-class SignatureSerializationFlag:
-    UNSIGNED = 0
-    '''
-    Format:
-    <FLAG><CLAIM BINARY>
-    or (legacy)
-    <CLAIM BINARY>
-    '''
-    ECDSA_LEGACY = 1
-    '''
-    Old claim format, which carried the signature inside the protobuf. Requires serializing back the claim with
-    signature stripped out for validation. This process requires knowledge on how a claim is serialized, thus requires
-    old fixed protobuf schema to work.
-    
-    Format:
-    <CLAIM PROTOBUF SERIALIZED>
-    Curves: NIST256p, NIST384p, SECP256k1
-    Signature content: `r` and `s` in each half of the 64 or 96 bytes (depends on curve)
-    Signed payload:
-    1. Claim transaction output address (raw, decoded using base58)
-    2. Stripped out claim protobuf serialization (without the signature)
-    3. Certificate claim id (binary, not in network byte order)
-    '''
-    ECDSA_SECP256K1 = 2
-    '''
-    Format:
-    <FLAG><CERTIFICATE ID><SIGNATURE><BINARY PAYLOAD>
-    Curve: SECP256K1
-    Signature content: 64 bytes total, each half represents `r` and `s`
-    Signed payload:
-    1. raw claim name as bytes
-    2. Claim transaction output address (raw, decoded using base58)
-    3. Binary payload, independent of serialization (everything after the signature last byte)
-    4. Certificate claim id, not in network byte order.
-    
-    A certificate can be signed as well, but this serialization model is unaware of content type or protobuf format.
-    '''
-    @classmethod
-    def is_flag_valid(cls, flag):
-        # todo: use python 3 enum when fully ported, but not worth now as its an extra dependency for py2
-        return 0 <= flag <= 2

lbrynet/schema/signature/serializer.py

@@ -1,11 +0,0 @@
-import struct
-from collections import namedtuple
-from .flags import SignatureSerializationFlag
-
-
-class Signature(namedtuple("Signature", "flags signature certificate_id")):
-    def deserialize(cls, payload):
-        flag = struct.unpack("<b", payload[0])[0]
-        if not SignatureSerializationFlag.is_flag_valid(flag):
-            return Signature(SignatureSerializationFlag.ECDSA_LEGACY, )
-        certificate

lbrynet/utils.py

@@ -12,13 +12,10 @@ import pkg_resources
 from twisted.python.failure import Failure
 from twisted.internet import defer
 from lbrynet.schema.claim import ClaimDict
-from lbrynet.p2p.cryptoutils import get_lbry_hash_obj
+from lbrynet.cryptoutils import get_lbry_hash_obj
 
 log = logging.getLogger(__name__)
 
-# digest_size is in bytes, and blob hashes are hex encoded
-blobhash_length = get_lbry_hash_obj().digest_size * 2
-
 
 # defining these time functions here allows for easier overriding in testing
 def now():
@@ -52,6 +49,7 @@ def call_later(delay, func, *args, **kwargs):
     from twisted.internet import reactor
     return reactor.callLater(delay, func, *args, **kwargs)
 
+
 def safe_start_looping_call(looping_call, interval_sec):
     if not looping_call.running:
         looping_call.start(interval_sec)
@@ -69,21 +67,6 @@ def generate_id(num=None):
     return h.digest()
 
 
-def is_valid_hashcharacter(char):
-    return char in "0123456789abcdef"
-
-
-def is_valid_blobhash(blobhash):
-    """Checks whether the blobhash is the correct length and contains only
-    valid characters (0-9, a-f)
-
-    @param blobhash: string, the blobhash to check
-
-    @return: True/False
-    """
-    return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
-
-
 def version_is_greater_than(a, b):
     """Returns True if version a is more recent than version b"""
     return pkg_resources.parse_version(a) > pkg_resources.parse_version(b)

scripts/encrypt_blob.py

@@ -9,7 +9,7 @@ from twisted.web.client import FileBodyProducer
 
 from lbrynet import conf
 from lbrynet.core import log_support
-from lbrynet.dht.hashannouncer import DummyHashAnnouncer
+from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer
 from lbrynet.core.BlobManager import DiskBlobManager
 from lbrynet.cryptstream.CryptStreamCreator import CryptStreamCreator
 

tests/functional/dht/test_store.py

@@ -3,7 +3,7 @@ from binascii import hexlify
 
 from twisted.internet import defer
 from lbrynet.dht import constants
-from lbrynet.p2p.utils import generate_id
+from lbrynet.utils import generate_id
 from .dht_test_environment import TestKademliaBase
 import logging
 

tests/functional/test_misc.py

@@ -2,13 +2,13 @@ import os
 from hashlib import md5
 from twisted.internet import defer, reactor
 from twisted.trial import unittest
-from lbrynet.extras.daemon import conf
+from lbrynet import conf
 from lbrynet.p2p.server.BlobAvailabilityHandler import BlobAvailabilityHandlerFactory
 from lbrynet.p2p.StreamDescriptor import StreamDescriptorIdentifier
 from lbrynet.p2p.StreamDescriptor import download_sd_blob
 from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager
 from lbrynet.p2p.BlobManager import DiskBlobManager
-from lbrynet.p2p.PeerManager import PeerManager
+from lbrynet.extras.daemon.PeerManager import PeerManager
 from lbrynet.p2p.RateLimiter import RateLimiter
 from lbrynet.p2p.server.BlobRequestHandler import BlobRequestHandlerFactory
 from lbrynet.p2p.server.ServerProtocol import ServerProtocolFactory
@@ -18,7 +18,7 @@ from lbrynet.blob.EncryptedFileManager import EncryptedFileManager
 from lbrynet.blob.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
 
 from tests import mocks
-from tests.util import mk_db_and_blob_dir, rm_db_and_blob_dir
+from tests.test_utils import mk_db_and_blob_dir, rm_db_and_blob_dir
 
 FakeNode = mocks.Node
 FakeWallet = mocks.Wallet

tests/functional/test_reflector.py

@@ -4,8 +4,11 @@ from binascii import hexlify
 from twisted.internet import defer, error
 from twisted.trial import unittest
 from lbrynet.p2p.StreamDescriptor import get_sd_info
-from lbrynet.extras import reflector
+from lbrynet.extras.reflector.server.server import ReflectorServerFactory
-from lbrynet.p2p import BlobManager, PeerManager
+from lbrynet.extras.reflector.client.client import EncryptedFileReflectorClientFactory
+from lbrynet.extras.reflector.client.blob import BlobReflectorClientFactory
+from lbrynet.extras.daemon.PeerManager import PeerManager
+from lbrynet.p2p import BlobManager
 from lbrynet.p2p import StreamDescriptor
 from lbrynet.blob import EncryptedFileCreator
 from lbrynet.blob.EncryptedFileManager import EncryptedFileManager
@@ -13,7 +16,7 @@ from lbrynet.p2p.RateLimiter import DummyRateLimiter
 from lbrynet.extras.daemon.storage import SQLiteStorage
 from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager
 from tests import mocks
-from tests.util import mk_db_and_blob_dir, rm_db_and_blob_dir
+from tests.test_utils import mk_db_and_blob_dir, rm_db_and_blob_dir
 
 
 class TestReflector(unittest.TestCase):
@@ -25,7 +28,7 @@ class TestReflector(unittest.TestCase):
         self.client_db_dir, self.client_blob_dir = mk_db_and_blob_dir()
         prm = OnlyFreePaymentsManager()
         wallet = mocks.Wallet()
-        peer_manager = PeerManager.PeerManager()
+        peer_manager = PeerManager()
         peer_finder = mocks.PeerFinder(5553, peer_manager, 2)
         self.server_storage = SQLiteStorage(self.server_db_dir)
         self.server_blob_manager = BlobManager.DiskBlobManager(self.server_blob_dir, self.server_storage)
@@ -95,7 +98,7 @@ class TestReflector(unittest.TestCase):
             return d
 
         def start_server():
-            server_factory = reflector.ServerFactory(peer_manager, self.server_blob_manager,
+            server_factory = ReflectorServerFactory(peer_manager, self.server_blob_manager,
                                                      self.server_lbry_file_manager)
             from twisted.internet import reactor
             port = 8943
@@ -179,7 +182,7 @@ class TestReflector(unittest.TestCase):
             return d
 
         def send_to_server():
-            factory = reflector.ClientFactory(self.client_blob_manager, self.stream_hash, self.sd_hash)
+            factory = EncryptedFileReflectorClientFactory(self.client_blob_manager, self.stream_hash, self.sd_hash)
 
             from twisted.internet import reactor
             reactor.connectTCP('localhost', self.port, factory)
@@ -208,7 +211,7 @@ class TestReflector(unittest.TestCase):
             return d
 
         def send_to_server(blob_hashes_to_send):
-            factory = reflector.BlobClientFactory(
+            factory = BlobReflectorClientFactory(
                 self.client_blob_manager,
                 blob_hashes_to_send
             )
@@ -247,7 +250,7 @@ class TestReflector(unittest.TestCase):
             return d
 
         def send_to_server(blob_hashes_to_send):
-            factory = reflector.BlobClientFactory(
+            factory = BlobReflectorClientFactory(
                 self.client_blob_manager,
                 blob_hashes_to_send
             )
@@ -301,7 +304,7 @@ class TestReflector(unittest.TestCase):
             return d
 
         def send_to_server_as_blobs(blob_hashes_to_send):
-            factory = reflector.BlobClientFactory(
+            factory = BlobReflectorClientFactory(
                 self.client_blob_manager,
                 blob_hashes_to_send
             )
@@ -312,7 +315,7 @@ class TestReflector(unittest.TestCase):
             return factory.finished_deferred
 
         def send_to_server_as_stream(result):
-            factory = reflector.ClientFactory(self.client_blob_manager, self.stream_hash, self.sd_hash)
+            factory = EncryptedFileReflectorClientFactory(self.client_blob_manager, self.stream_hash, self.sd_hash)
 
             from twisted.internet import reactor
             reactor.connectTCP('localhost', self.port, factory)

tests/functional/test_streamify.py

@@ -7,8 +7,8 @@ from twisted.internet import defer, threads
 from lbrynet.p2p.StreamDescriptor import StreamDescriptorIdentifier
 from lbrynet.p2p.BlobManager import DiskBlobManager
 from lbrynet.p2p.StreamDescriptor import get_sd_info
-from lbrynet.p2p.PeerManager import PeerManager
 from lbrynet.p2p.RateLimiter import DummyRateLimiter
+from lbrynet.extras.daemon.PeerManager import PeerManager
 from lbrynet.extras.daemon.storage import SQLiteStorage
 from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager
 from lbrynet.blob.EncryptedFileCreator import create_lbry_file

tests/integration/cli/test_cli.py

@@ -3,10 +3,10 @@ from twisted.trial import unittest
 from io import StringIO
 from twisted.internet import defer
 
-from lbrynet.extras.daemon import conf
+from lbrynet import conf
 from lbrynet.extras import cli
 from lbrynet.extras.daemon.Components import DATABASE_COMPONENT, BLOB_COMPONENT, HEADERS_COMPONENT, WALLET_COMPONENT, \
-    DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, STREAM_IDENTIFIER_COMPONENT, FILE_MANAGER_COMPONENT, \
+    DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, \
     PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, \
     RATE_LIMITER_COMPONENT, PAYMENT_RATE_COMPONENT
 from lbrynet.extras.daemon.Daemon import Daemon
@@ -35,7 +35,7 @@ class CLIIntegrationTest(unittest.TestCase):
     def setUp(self):
         skip = [
             DATABASE_COMPONENT, BLOB_COMPONENT, HEADERS_COMPONENT, WALLET_COMPONENT,
-            DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, STREAM_IDENTIFIER_COMPONENT, FILE_MANAGER_COMPONENT,
+            DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT,
             PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT,
             RATE_LIMITER_COMPONENT, PAYMENT_RATE_COMPONENT
         ]

tests/integration/wallet/test_commands.py

@@ -15,7 +15,7 @@ from torba.testcase import IntegrationTestCase as BaseIntegrationTestCase
 import lbrynet.schema
 lbrynet.schema.BLOCKCHAIN_NAME = 'lbrycrd_regtest'
 
-from lbrynet.extras.daemon import conf as lbry_conf
+from lbrynet import conf as lbry_conf
 from lbrynet.dht.node import Node
 from lbrynet.extras.daemon.Daemon import Daemon
 from lbrynet.extras.wallet import LbryWalletManager

tests/mocks.py

@@ -7,13 +7,13 @@ from cryptography.hazmat.primitives.asymmetric import rsa
 from cryptography.hazmat.primitives import serialization
 from twisted.internet import defer
 from twisted.python.failure import Failure
-
+from lbrynet import conf
 from lbrynet.p2p.client.ClientRequest import ClientRequest
 from lbrynet.p2p.Error import RequestCanceledError
 from lbrynet.p2p import BlobAvailability
 from lbrynet.blob.EncryptedFileManager import EncryptedFileManager
 from lbrynet.dht.node import Node as RealNode
-from lbrynet.extras.daemon import ExchangeRateManager as ERM, conf
+from lbrynet.extras.daemon import ExchangeRateManager as ERM
 
 KB = 2**10
 PUBLIC_EXPONENT = 65537  # http://www.daemonology.net/blog/2009-06-11-cryptographic-right-answers.html

tests/test_utils.py

@@ -32,13 +32,14 @@ def resetTime(test_case, timestamp=DEFAULT_TIMESTAMP):
     patcher.start().return_value = iso_time
     test_case.addCleanup(patcher.stop)
 
-    patcher = mock.patch('lbrynet.p2p.utils.now')
+    patcher = mock.patch('lbrynet.utils.now')
     patcher.start().return_value = timestamp
     test_case.addCleanup(patcher.stop)
 
-    patcher = mock.patch('lbrynet.p2p.utils.utcnow')
+    patcher = mock.patch('lbrynet.utils.utcnow')
     patcher.start().return_value = timestamp
     test_case.addCleanup(patcher.stop)
 
+
 def is_android():
     return 'ANDROID_ARGUMENT' in os.environ # detect Android using the Kivy way

tests/unit/components/test_Component_Manager.py

@@ -2,7 +2,7 @@ from twisted.internet.task import Clock
 from twisted.trial import unittest
 
 from lbrynet.extras.daemon.ComponentManager import ComponentManager
-from lbrynet.extras.daemon.Components import DATABASE_COMPONENT, DHT_COMPONENT, STREAM_IDENTIFIER_COMPONENT
+from lbrynet.extras.daemon.Components import DATABASE_COMPONENT, DHT_COMPONENT
 from lbrynet.extras.daemon.Components import HASH_ANNOUNCER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT
 from lbrynet.extras.daemon.Components import PEER_PROTOCOL_SERVER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
 from lbrynet.extras.daemon.Components import RATE_LIMITER_COMPONENT, HEADERS_COMPONENT, PAYMENT_RATE_COMPONENT
@@ -13,6 +13,7 @@ from tests import mocks
 class TestComponentManager(unittest.TestCase):
     def setUp(self):
         mocks.mock_conf_settings(self)
+
         self.default_components_sort = [
             [
                 Components.HeadersComponent,
@@ -23,19 +24,14 @@ class TestComponentManager(unittest.TestCase):
                 Components.UPnPComponent
             ],
             [
+                Components.BlobComponent,
                 Components.DHTComponent,
                 Components.WalletComponent
             ],
             [
-                Components.BlobComponent,
+                Components.FileManagerComponent,
-                Components.HashAnnouncerComponent
+                Components.HashAnnouncerComponent,
-            ],
+                Components.PeerProtocolServerComponent
-            [
-                Components.PeerProtocolServerComponent,
-                Components.StreamIdentifierComponent
-            ],
-            [
-                Components.FileManagerComponent
             ],
             [
                 Components.ReflectorComponent
@@ -48,7 +44,6 @@ class TestComponentManager(unittest.TestCase):
 
     def test_sort_components(self):
         stages = self.component_manager.sort_components()
-
         for stage_list, sorted_stage_list in zip(stages, self.default_components_sort):
             self.assertEqual([type(stage) for stage in stage_list], sorted_stage_list)
 
@@ -101,7 +96,7 @@ class TestComponentManagerProperStart(unittest.TestCase):
         self.reactor = Clock()
         mocks.mock_conf_settings(self)
         self.component_manager = ComponentManager(
-            skip_components=[DATABASE_COMPONENT, DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, STREAM_IDENTIFIER_COMPONENT,
+            skip_components=[DATABASE_COMPONENT, DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT,
                              PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT,
                              HEADERS_COMPONENT, PAYMENT_RATE_COMPONENT, RATE_LIMITER_COMPONENT,
                              EXCHANGE_RATE_MANAGER_COMPONENT],

tests/unit/core/client/test_ConnectionManager.py

@@ -1,17 +1,16 @@
+from twisted.trial.unittest import TestCase
+from twisted.internet import defer, reactor, task
+from twisted.internet.task import deferLater
+from twisted.internet.protocol import ServerFactory
+
+from lbrynet import conf, utils
 from lbrynet.p2p.client.ClientRequest import ClientRequest
 from lbrynet.p2p.server.ServerProtocol import ServerProtocol
 from lbrynet.p2p.client.ClientProtocol import ClientProtocol
 from lbrynet.p2p.RateLimiter import RateLimiter
 from lbrynet.p2p.Peer import Peer
-from lbrynet.p2p.PeerManager import PeerManager
 from lbrynet.p2p.Error import NoResponseError
-
+from lbrynet.extras.daemon.PeerManager import PeerManager
-from twisted.trial.unittest import TestCase
-from twisted.internet import defer, reactor, task
-from twisted.internet.task import deferLater
-from twisted.internet.protocol import ServerFactory
-from lbrynet.extras.daemon import conf
-from lbrynet.p2p import utils
 
 PEER_PORT = 5551
 LOCAL_HOST = '127.0.0.1'

tests/unit/core/test_BlobManager.py

@@ -6,12 +6,12 @@ import string
 from twisted.trial import unittest
 from twisted.internet import defer
 
-from tests.util import random_lbry_hash
+from tests.test_utils import random_lbry_hash
 from lbrynet.p2p.BlobManager import DiskBlobManager
 from lbrynet.extras.daemon.storage import SQLiteStorage
 from lbrynet.p2p.Peer import Peer
-from lbrynet.extras.daemon import conf
+from lbrynet import conf
-from lbrynet.p2p.cryptoutils import get_lbry_hash_obj
+from lbrynet.cryptoutils import get_lbry_hash_obj
 
 
 class BlobManagerTest(unittest.TestCase):

tests/unit/core/test_HTTPBlobDownloader.py

@@ -3,9 +3,9 @@ from mock import MagicMock
 from twisted.trial import unittest
 from twisted.internet import defer
 
-from lbrynet.blob import BlobFile
+from lbrynet.blob.blob_file import BlobFile
 from lbrynet.p2p.HTTPBlobDownloader import HTTPBlobDownloader
-from tests.util import mk_db_and_blob_dir, rm_db_and_blob_dir
+from tests.test_utils import mk_db_and_blob_dir, rm_db_and_blob_dir
 
 
 class HTTPBlobDownloaderTest(unittest.TestCase):

tests/unit/core/test_HashBlob.py

@@ -1,7 +1,7 @@
-from lbrynet.blob import BlobFile
+from lbrynet.blob.blob_file import BlobFile
 from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError
 
-from tests.util import mk_db_and_blob_dir, rm_db_and_blob_dir, random_lbry_hash
+from tests.test_utils import mk_db_and_blob_dir, rm_db_and_blob_dir, random_lbry_hash
 from twisted.trial import unittest
 from twisted.internet import defer
 

tests/unit/core/test_Wallet.py

@@ -5,9 +5,6 @@ from decimal import Decimal
 from twisted.trial import unittest
 from twisted.internet import defer
 from lbrynet.p2p.Error import InsufficientFundsError
-#from lbrynet.p2p.Wallet import LBRYumWallet, ReservedPoints
-#from lbryum.commands import Commands
-#from lbryum.simple_config import SimpleConfig
 from lbrynet.schema.claim import ClaimDict
 
 test_metadata = {

tests/unit/core/test_utils.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from lbrynet.p2p import utils
+from lbrynet import utils
 
 from twisted.trial import unittest
 

tests/unit/database/test_SQLiteStorage.py

@@ -5,10 +5,10 @@ import logging
 from copy import deepcopy
 from twisted.internet import defer
 from twisted.trial import unittest
-from lbrynet.extras.daemon import conf
+from lbrynet import conf
 from lbrynet.extras.daemon.storage import SQLiteStorage, open_file_for_writing
 from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloader
-from tests.util import random_lbry_hash
+from tests.test_utils import random_lbry_hash
 
 log = logging.getLogger()
 

tests/unit/dht/test_contact.py

@@ -1,7 +1,7 @@
 from binascii import hexlify
 from twisted.internet import task
 from twisted.trial import unittest
-from lbrynet.p2p.utils import generate_id
+from lbrynet.utils import generate_id
 from lbrynet.dht.contact import ContactManager
 from lbrynet.dht import constants
 

tests/unit/dht/test_hash_announcer.py

@@ -1,8 +1,8 @@
 from twisted.trial import unittest
 from twisted.internet import defer, task
-from lbrynet.p2p import utils
+from lbrynet import utils
-from lbrynet.dht.hashannouncer import DHTHashAnnouncer
+from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer
-from tests.util import random_lbry_hash
+from tests.test_utils import random_lbry_hash
 from tests.mocks import mock_conf_settings
 
 

tests/unit/dht/test_kbucket.py

@@ -6,7 +6,7 @@
 
 from twisted.trial import unittest
 import struct
-from lbrynet.p2p.utils import generate_id
+from lbrynet.utils import generate_id
 from lbrynet.dht import kbucket
 from lbrynet.dht.contact import ContactManager
 from lbrynet.dht import constants

tests/unit/dht/test_node.py

@@ -5,7 +5,7 @@ from twisted.trial import unittest
 from twisted.internet import defer
 from lbrynet.dht.node import Node
 from lbrynet.dht import constants
-from lbrynet.p2p.utils import generate_id
+from lbrynet.utils import generate_id
 
 
 class NodeIDTest(unittest.TestCase):

tests/unit/dht/test_routingtable.py

@@ -6,7 +6,7 @@ from lbrynet.dht import constants
 from lbrynet.dht.routingtable import TreeRoutingTable
 from lbrynet.dht.contact import ContactManager
 from lbrynet.dht.distance import Distance
-from lbrynet.p2p.utils import generate_id
+from lbrynet.utils import generate_id
 
 
 class FakeRPCProtocol: