Madiator2011/lbry-sdk
1
0
Fork 0
forked from LBRYCommunity/lbry-sdk
Code Pull requests Activity

Merge pull request #1604 from lbryio/fix-external-ip

Browse source 
Fix setting the external ip
This commit is contained in:
Lex Berezhny 2018-11-10 20:48:40 -05:00 committed by GitHub
commit 457a56ce04
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
109 changed files with 477 additions and 587 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
lbrynet/__init__.py
View file

@ -1,7 +1,10 @@
import logging
from lbrynet.custom_logger import install_logger
__name__ = "lbrynet"
__version__ = "0.30.1rc1"
version = tuple(__version__.split('.'))
install_logger()
logging.getLogger(__name__).addHandler(logging.NullHandler())

2
lbrynet/blob/EncryptedFileCreator.py
View file

@ -2,8 +2,8 @@
Utilities for turning plain files into LBRY Files.
"""
import logging
import os
import logging
from binascii import hexlify
from twisted.internet import defer

4
lbrynet/blob/EncryptedFileDownloader.py
View file

@ -5,10 +5,10 @@ import logging
from binascii import hexlify, unhexlify
from twisted.internet import defer
from lbrynet.extras.daemon import conf
from lbrynet import conf
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
from lbrynet.p2p.HTTPBlobDownloader import HTTPBlobDownloader
from lbrynet.p2p.utils import short_hash
from lbrynet.utils import short_hash
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaver
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileDownloader
from lbrynet.blob.EncryptedFileStatusReport import EncryptedFileStatusReport

4
lbrynet/blob/EncryptedFileManager.py
View file

@ -7,14 +7,14 @@ from binascii import hexlify, unhexlify
from twisted.internet import defer, task, reactor
from twisted.python.failure import Failure
from lbrynet import conf
from lbrynet.extras.reflector.reupload import reflect_file
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloader
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamType, get_sd_info
from lbrynet.blob.client.CryptStreamDownloader import AlreadyStoppedError
from lbrynet.blob.client.CryptStreamDownloader import CurrentlyStoppingError
from lbrynet.p2p.utils import safe_start_looping_call, safe_stop_looping_call
from lbrynet.extras.daemon import conf
from lbrynet.utils import safe_start_looping_call, safe_stop_looping_call
log = logging.getLogger(__name__)

4
lbrynet/blob/__init__.py
View file

@ -1,4 +0,0 @@
from .blob_file import BlobFile
from .creator import BlobFileCreator
from .writer import HashBlobWriter
from .reader import HashBlobReader

22
lbrynet/blob/blob_file.py
View file

@ -1,10 +1,10 @@
import logging
import os
import logging
from twisted.internet import defer, threads
from twisted.web.client import FileBodyProducer
from twisted.python.failure import Failure
from lbrynet.cryptoutils import get_lbry_hash_obj
from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError, InvalidBlobHashError
from lbrynet.p2p.utils import is_valid_blobhash
from lbrynet.blob.writer import HashBlobWriter
from lbrynet.blob.reader import HashBlobReader
@ -12,6 +12,24 @@ log = logging.getLogger(__name__)
MAX_BLOB_SIZE = 2 * 2 ** 20
# digest_size is in bytes, and blob hashes are hex encoded
blobhash_length = get_lbry_hash_obj().digest_size * 2
def is_valid_hashcharacter(char):
return char in "0123456789abcdef"
def is_valid_blobhash(blobhash):
"""Checks whether the blobhash is the correct length and contains only
valid characters (0-9, a-f)
@param blobhash: string, the blobhash to check
@return: True/False
"""
return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
class BlobFile:
"""

7
lbrynet/blob/client/CryptStreamDownloader.py
View file

@ -1,12 +1,13 @@
from binascii import unhexlify
import logging
from binascii import unhexlify
from twisted.internet import defer
from twisted.python.failure import Failure
from lbrynet.p2p.client.BlobRequester import BlobRequester
from lbrynet.p2p.client.ConnectionManager import ConnectionManager
from lbrynet.p2p.client.DownloadManager import DownloadManager
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
from lbrynet.blob.client.CryptBlobHandler import CryptBlobHandler
from twisted.internet import defer
from twisted.python.failure import Failure
log = logging.getLogger(__name__)

2
lbrynet/blob/client/EncryptedFileDownloader.py
View file

@ -2,13 +2,13 @@ import os
import logging
import traceback
from binascii import hexlify, unhexlify
from twisted.internet import defer, threads
from lbrynet.p2p.StreamDescriptor import save_sd_info
from lbrynet.blob.client.CryptStreamDownloader import CryptStreamDownloader
from lbrynet.p2p.client.StreamProgressManager import FullStreamProgressManager
from lbrynet.p2p.Error import FileOpenError
from lbrynet.blob.client.EncryptedFileMetadataHandler import EncryptedFileMetadataHandler
from twisted.internet import defer, threads
log = logging.getLogger(__name__)

2
lbrynet/blob/creator.py
View file

@ -3,7 +3,7 @@ import logging
from io import BytesIO
from twisted.internet import defer
from twisted.web.client import FileBodyProducer
from lbrynet.p2p.cryptoutils import get_lbry_hash_obj
from lbrynet.cryptoutils import get_lbry_hash_obj
log = logging.getLogger(__name__)

2
lbrynet/blob/writer.py
View file

@ -2,7 +2,7 @@ import logging
from io import BytesIO
from twisted.python.failure import Failure
from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError
from lbrynet.p2p.cryptoutils import get_lbry_hash_obj
from lbrynet.cryptoutils import get_lbry_hash_obj
log = logging.getLogger(__name__)

4
lbrynet/extras/daemon/conf.py → lbrynet/conf.py
View file

@ -7,7 +7,7 @@ import sys
import yaml
import envparse
from appdirs import user_data_dir, user_config_dir
from lbrynet.p2p import utils
from lbrynet import utils
from lbrynet.p2p.Error import InvalidCurrencyError, NoSuchDirectoryError
from lbrynet.androidhelpers.paths import (
android_internal_storage_dir,
@ -15,7 +15,7 @@ from lbrynet.androidhelpers.paths import (
)
try:
from .winpaths import get_path, FOLDERID, UserHandle
from lbrynet.winpaths import get_path, FOLDERID, UserHandle
except (ImportError, ValueError, NameError):
# Android platform: NameError: name 'c_wchar' is not defined
pass

0
lbrynet/p2p/cryptoutils.py → lbrynet/cryptoutils.py
View file

7
lbrynet/custom_logger.py
View file

@ -102,5 +102,8 @@ class Logger(logging.Logger):
self._log(TRACE, msg, args, **kwargs)
logging.setLoggerClass(Logger)
logging.addLevelName(TRACE, 'TRACE')
def install_logger():
current = logging.getLoggerClass()
if current is not Logger:
logging.setLoggerClass(Logger)
logging.addLevelName(TRACE, 'TRACE')

0
lbrynet/p2p/call_later_manager.py → lbrynet/dht/call_later_manager.py
View file

2
lbrynet/dht/datastore.py
View file

@ -1,5 +1,5 @@
from collections import UserDict
from . import constants
from lbrynet.dht import constants
class DictDataStore(UserDict):

2
lbrynet/dht/encoding.py
View file

@ -1,4 +1,4 @@
from .error import DecodeError
from lbrynet.dht.error import DecodeError
def bencode(data):

6
lbrynet/dht/iterativefind.py
View file

@ -1,8 +1,8 @@
import logging
from twisted.internet import defer
from .distance import Distance
from .error import TimeoutError
from . import constants
from lbrynet.dht.distance import Distance
from lbrynet.dht.error import TimeoutError
from lbrynet.dht import constants
log = logging.getLogger(__name__)

6
lbrynet/dht/kbucket.py
View file

@ -1,8 +1,8 @@
import logging
from . import constants
from .distance import Distance
from .error import BucketFull
from lbrynet.dht import constants
from lbrynet.dht.distance import Distance
from lbrynet.dht.error import BucketFull
log = logging.getLogger(__name__)

2
lbrynet/dht/msgformat.py
View file

@ -7,7 +7,7 @@
# The docstrings in this module contain epytext markup; API documentation
# may be created by processing this file with epydoc: http://epydoc.sf.net
from . import msgtypes
from lbrynet.dht import msgtypes
class MessageTranslator:

4
lbrynet/dht/msgtypes.py
View file

@ -7,8 +7,8 @@
# The docstrings in this module contain epytext markup; API documentation
# may be created by processing this file with epydoc: http://epydoc.sf.net
from lbrynet.p2p.utils import generate_id
from . import constants
from lbrynet.utils import generate_id
from lbrynet.dht import constants
class Message:

39
lbrynet/dht/node.py
View file

@ -5,17 +5,12 @@ from functools import reduce
from twisted.internet import defer, error, task
from lbrynet.p2p.utils import generate_id, DeferredDict
from lbrynet.p2p.call_later_manager import CallLaterManager
from lbrynet.p2p.PeerManager import PeerManager
from .error import TimeoutError
from . import constants
from . import routingtable
from . import datastore
from . import protocol
from .peerfinder import DHTPeerFinder
from .contact import ContactManager
from .iterativefind import iterativeFind
from lbrynet.utils import generate_id, DeferredDict
from lbrynet.dht.call_later_manager import CallLaterManager
from lbrynet.dht.error import TimeoutError
from lbrynet.dht import constants, routingtable, datastore, protocol
from lbrynet.dht.contact import ContactManager
from lbrynet.dht.iterativefind import iterativeFind
log = logging.getLogger(__name__)
@ -83,8 +78,8 @@ class Node(MockKademliaHelper):
def __init__(self, node_id=None, udpPort=4000, dataStore=None,
routingTableClass=None, networkProtocol=None,
externalIP=None, peerPort=3333, listenUDP=None,
callLater=None, resolve=None, clock=None, peer_finder=None,
peer_manager=None, interface='', externalUDPPort=None):
callLater=None, resolve=None, clock=None,
interface='', externalUDPPort=None):
"""
@param dataStore: The data store to use. This must be class inheriting
from the C{DataStore} interface (or providing the
@ -124,20 +119,13 @@ class Node(MockKademliaHelper):
else:
self._routingTable = routingTableClass(self.node_id, self.clock.seconds)
# Initialize this node's network access mechanisms
if networkProtocol is None:
self._protocol = protocol.KademliaProtocol(self)
else:
self._protocol = networkProtocol
# Initialize the data storage mechanism used by this node
self._protocol = networkProtocol or protocol.KademliaProtocol(self)
self.token_secret = self._generateID()
self.old_token_secret = None
self.externalIP = externalIP
self.peerPort = peerPort
self.externalUDPPort = externalUDPPort or self.port
self._dataStore = dataStore or datastore.DictDataStore(self.clock.seconds)
self.peer_manager = peer_manager or PeerManager()
self.peer_finder = peer_finder or DHTPeerFinder(self, self.peer_manager)
self._join_deferred = None
#def __del__(self):
@ -255,7 +243,7 @@ class Node(MockKademliaHelper):
yield _iterative_join()
@defer.inlineCallbacks
def start(self, known_node_addresses=None):
def start(self, known_node_addresses=None, block_on_join=False):
""" Causes the Node to attempt to join the DHT network by contacting the
known DHT nodes. This can be called multiple times if the previous attempt
has failed or if the Node has lost all the contacts.
@ -270,8 +258,11 @@ class Node(MockKademliaHelper):
self.start_listening()
yield self._protocol._listening
# TODO: Refresh all k-buckets further away than this node's closest neighbour
yield self.joinNetwork(known_node_addresses or [])
self.start_looping_calls()
d = self.joinNetwork(known_node_addresses or [])
d.addCallback(lambda _: self.start_looping_calls())
d.addCallback(lambda _: log.info("Joined the dht"))
if block_on_join:
yield d
def start_looping_calls(self):
self.safe_start_looping_call(self._change_token_lc, constants.tokenSecretChangeInterval)

8
lbrynet/dht/protocol.py
View file

@ -3,12 +3,8 @@ import errno
from binascii import hexlify
from twisted.internet import protocol, defer
from .error import BUILTIN_EXCEPTIONS, UnknownRemoteException, TimeoutError, TransportNotConnected
from . import constants
from . import encoding
from . import msgtypes
from . import msgformat
from lbrynet.dht import constants, encoding, msgformat, msgtypes
from lbrynet.dht.error import BUILTIN_EXCEPTIONS, UnknownRemoteException, TimeoutError, TransportNotConnected
log = logging.getLogger(__name__)

13
lbrynet/dht/routingtable.py
View file

@ -6,14 +6,13 @@
# may be created by processing this file with epydoc: http://epydoc.sf.net
import random
from binascii import unhexlify
from twisted.internet import defer
from . import constants
from . import kbucket
from .error import TimeoutError
from .distance import Distance
import logging
from binascii import unhexlify
from twisted.internet import defer
from lbrynet.dht import constants, kbucket
from lbrynet.dht.error import TimeoutError
from lbrynet.dht.distance import Distance
log = logging.getLogger(__name__)

11
lbrynet/extras/cli.py
View file

@ -24,22 +24,25 @@ from lbrynet.extras.daemon.Daemon import Daemon
from lbrynet.extras.daemon.DaemonControl import start as daemon_main
from lbrynet.extras.daemon.DaemonConsole import main as daemon_console
from lbrynet.extras.daemon.auth.client import LBRYAPIClient
from lbrynet.p2p.system_info import get_platform
from lbrynet.extras.system_info import get_platform
async def execute_command(method, params, conf_path=None):
# this check if the daemon is running or not
api = None
try:
api = await LBRYAPIClient.get_client(conf_path)
await api.status()
except (ClientConnectorError, ConnectionError):
await api.session.close()
if api:
await api.session.close()
print("Could not connect to daemon. Are you sure it's running?")
return 1
# this actually executes the method
resp = await api.call(method, params)
try:
resp = await api.call(method, params)
await api.session.close()
print(json.dumps(resp["result"], indent=2))
except KeyError:
@ -137,7 +140,7 @@ def main(argv=None):
elif method in ['version', '--version', '-v']:
print("{lbrynet_name} {lbrynet_version}".format(
lbrynet_name=lbrynet_name, **get_platform(get_ip=False)
lbrynet_name=lbrynet_name, **get_platform()
))
return 0

2
lbrynet/extras/daemon/Component.py
View file

@ -1,7 +1,7 @@
import logging
from twisted.internet import defer
from twisted._threads import AlreadyQuit
from .ComponentManager import ComponentManager
from lbrynet.extras.daemon.ComponentManager import ComponentManager
log = logging.getLogger(__name__)

13
lbrynet/extras/daemon/ComponentManager.py
View file

@ -1,7 +1,8 @@
import logging
from twisted.internet import defer
from lbrynet.p2p.Error import ComponentStartConditionNotMet
from lbrynet.extras.daemon.PeerManager import PeerManager
from lbrynet.extras.daemon.PeerFinder import DHTPeerFinder
log = logging.getLogger(__name__)
@ -33,13 +34,15 @@ class RequiredCondition(metaclass=RequiredConditionType):
class ComponentManager:
default_component_classes = {}
def __init__(self, reactor=None, analytics_manager=None, skip_components=None, **override_components):
def __init__(self, reactor=None, analytics_manager=None, skip_components=None,
peer_manager=None, peer_finder=None, **override_components):
self.skip_components = skip_components or []
self.reactor = reactor
self.component_classes = {}
self.components = set()
self.analytics_manager = analytics_manager
self.peer_manager = peer_manager or PeerManager()
self.peer_finder = peer_finder or DHTPeerFinder(self)
for component_name, component_class in self.default_component_classes.items():
if component_name in override_components:
@ -114,10 +117,10 @@ class ComponentManager:
:return: (defer.Deferred)
"""
for component_name, cb in callbacks.items():
if component_name not in self.component_classes:
raise NameError("unknown component: %s" % component_name)
if component_name not in self.skip_components:
raise NameError("unknown component: %s" % component_name)
if not callable(cb):
raise ValueError("%s is not callable" % cb)

221
lbrynet/extras/daemon/Components.py
View file

@ -2,35 +2,39 @@ import os
import asyncio
import logging
import treq
import json
import math
import binascii
from hashlib import sha256
from types import SimpleNamespace
from twisted.internet import defer, threads, reactor, error, task
import lbrynet.schema
from aioupnp import __version__ as aioupnp_version
from aioupnp.upnp import UPnP
from aioupnp.fault import UPnPError
from lbrynet.extras.daemon import conf
from lbrynet.p2p.utils import DeferredDict
import lbrynet.schema
from lbrynet import conf
from lbrynet.blob.EncryptedFileManager import EncryptedFileManager
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaverFactory
from lbrynet.blob.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
from lbrynet.dht.node import Node
from lbrynet.extras.daemon.Component import Component
from lbrynet.extras.daemon.ExchangeRateManager import ExchangeRateManager
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer
from lbrynet.extras.reflector.server.server import ReflectorServerFactory
from lbrynet.extras.wallet import LbryWalletManager
from lbrynet.extras.wallet import Network
from lbrynet.utils import DeferredDict, generate_id
from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager
from lbrynet.p2p.RateLimiter import RateLimiter
from lbrynet.p2p.BlobManager import DiskBlobManager
from lbrynet.p2p.StreamDescriptor import StreamDescriptorIdentifier, EncryptedFileStreamType
from lbrynet.extras.wallet import LbryWalletManager
from lbrynet.extras.wallet import Network
from lbrynet.p2p.server.BlobRequestHandler import BlobRequestHandlerFactory
from lbrynet.p2p.server.ServerProtocol import ServerProtocolFactory
from .Component import Component
from .ExchangeRateManager import ExchangeRateManager
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.dht import node, hashannouncer
from lbrynet.blob.EncryptedFileManager import EncryptedFileManager
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaverFactory
from lbrynet.blob.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
from lbrynet.extras.reflector import ServerFactory as reflector_server_factory
from lbrynet.p2p.utils import generate_id
log = logging.getLogger(__name__)
@ -42,7 +46,6 @@ HEADERS_COMPONENT = "blockchain_headers"
WALLET_COMPONENT = "wallet"
DHT_COMPONENT = "dht"
HASH_ANNOUNCER_COMPONENT = "hash_announcer"
STREAM_IDENTIFIER_COMPONENT = "stream_identifier"
FILE_MANAGER_COMPONENT = "file_manager"
PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
REFLECTOR_COMPONENT = "reflector"
@ -94,10 +97,19 @@ class ConfigSettings:
return conf.settings.node_id
@staticmethod
def get_external_ip():
from lbrynet.p2p.system_info import get_platform
platform = get_platform(get_ip=True)
return platform['ip']
@defer.inlineCallbacks
def get_external_ip(): # used if upnp is disabled or non-functioning
try:
buf = []
response = yield treq.get("https://api.lbry.io/ip")
yield treq.collect(response, buf.append)
parsed = json.loads(b"".join(buf).decode())
if parsed['success']:
return parsed['data']['ip']
return
except Exception as err:
return
# Shorthand for common ConfigSettings methods
@ -366,7 +378,7 @@ class WalletComponent(Component):
class BlobComponent(Component):
component_name = BLOB_COMPONENT
depends_on = [DATABASE_COMPONENT, DHT_COMPONENT]
depends_on = [DATABASE_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
@ -378,8 +390,12 @@ class BlobComponent(Component):
def start(self):
storage = self.component_manager.get_component(DATABASE_COMPONENT)
dht_node = self.component_manager.get_component(DHT_COMPONENT)
self.blob_manager = DiskBlobManager(CS.get_blobfiles_dir(), storage, dht_node._dataStore)
datastore = None
if DHT_COMPONENT not in self.component_manager.skip_components:
dht_node = self.component_manager.get_component(DHT_COMPONENT)
if dht_node:
datastore = dht_node._dataStore
self.blob_manager = DiskBlobManager(CS.get_blobfiles_dir(), storage, datastore)
return self.blob_manager.setup()
def stop(self):
@ -424,20 +440,22 @@ class DHTComponent(Component):
node_id = CS.get_node_id()
if node_id is None:
node_id = generate_id()
external_ip = self.upnp_component.external_ip
if not external_ip:
log.warning("UPnP component failed to get external ip")
external_ip = yield CS.get_external_ip()
if not external_ip:
log.warning("failed to get external ip")
self.dht_node = node.Node(
self.dht_node = Node(
node_id=node_id,
udpPort=GCS('dht_node_port'),
externalUDPPort=self.external_udp_port,
externalIP=self.upnp_component.external_ip,
externalIP=external_ip,
peerPort=self.external_peer_port
)
self.dht_node.start_listening()
yield self.dht_node._protocol._listening
d = self.dht_node.joinNetwork(GCS('known_dht_nodes'))
d.addCallback(lambda _: self.dht_node.start_looping_calls())
d.addCallback(lambda _: log.info("Joined the dht"))
yield self.dht_node.start(GCS('known_dht_nodes'), block_on_join=False)
log.info("Started the dht")
@defer.inlineCallbacks
@ -461,7 +479,7 @@ class HashAnnouncerComponent(Component):
def start(self):
storage = self.component_manager.get_component(DATABASE_COMPONENT)
dht_node = self.component_manager.get_component(DHT_COMPONENT)
self.hash_announcer = hashannouncer.DHTHashAnnouncer(dht_node, storage)
self.hash_announcer = DHTHashAnnouncer(dht_node, storage)
yield self.hash_announcer.start()
@defer.inlineCallbacks
@ -494,41 +512,6 @@ class RateLimiterComponent(Component):
return defer.succeed(None)
class StreamIdentifierComponent(Component):
component_name = STREAM_IDENTIFIER_COMPONENT
depends_on = [DHT_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
self.sd_identifier = StreamDescriptorIdentifier()
@property
def component(self):
return self.sd_identifier
@defer.inlineCallbacks
def start(self):
dht_node = self.component_manager.get_component(DHT_COMPONENT)
rate_limiter = self.component_manager.get_component(RATE_LIMITER_COMPONENT)
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
storage = self.component_manager.get_component(DATABASE_COMPONENT)
wallet = self.component_manager.get_component(WALLET_COMPONENT)
add_lbry_file_to_sd_identifier(self.sd_identifier)
file_saver_factory = EncryptedFileSaverFactory(
dht_node.peer_finder,
rate_limiter,
blob_manager,
storage,
wallet,
GCS('download_directory')
)
yield self.sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory)
def stop(self):
pass
class PaymentRateComponent(Component):
component_name = PAYMENT_RATE_COMPONENT
@ -549,8 +532,8 @@ class PaymentRateComponent(Component):
class FileManagerComponent(Component):
component_name = FILE_MANAGER_COMPONENT
depends_on = [DHT_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT,
STREAM_IDENTIFIER_COMPONENT, PAYMENT_RATE_COMPONENT]
depends_on = [RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT,
PAYMENT_RATE_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
@ -569,15 +552,26 @@ class FileManagerComponent(Component):
@defer.inlineCallbacks
def start(self):
dht_node = self.component_manager.get_component(DHT_COMPONENT)
rate_limiter = self.component_manager.get_component(RATE_LIMITER_COMPONENT)
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
storage = self.component_manager.get_component(DATABASE_COMPONENT)
wallet = self.component_manager.get_component(WALLET_COMPONENT)
sd_identifier = self.component_manager.get_component(STREAM_IDENTIFIER_COMPONENT)
sd_identifier = StreamDescriptorIdentifier()
add_lbry_file_to_sd_identifier(sd_identifier)
file_saver_factory = EncryptedFileSaverFactory(
self.component_manager.peer_finder,
rate_limiter,
blob_manager,
storage,
wallet,
GCS('download_directory')
)
yield sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory)
payment_rate_manager = self.component_manager.get_component(PAYMENT_RATE_COMPONENT)
log.info('Starting the file manager')
self.file_manager = EncryptedFileManager(dht_node.peer_finder, rate_limiter, blob_manager, wallet,
self.file_manager = EncryptedFileManager(self.component_manager.peer_finder, rate_limiter, blob_manager, wallet,
payment_rate_manager, storage, sd_identifier)
yield self.file_manager.setup()
log.info('Done setting up file manager')
@ -589,7 +583,7 @@ class FileManagerComponent(Component):
class PeerProtocolServerComponent(Component):
component_name = PEER_PROTOCOL_SERVER_COMPONENT
depends_on = [UPNP_COMPONENT, DHT_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT,
depends_on = [UPNP_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT,
PAYMENT_RATE_COMPONENT]
def __init__(self, component_manager):
@ -618,7 +612,7 @@ class PeerProtocolServerComponent(Component):
}
server_factory = ServerProtocolFactory(
self.component_manager.get_component(RATE_LIMITER_COMPONENT), query_handlers,
self.component_manager.get_component(DHT_COMPONENT).peer_manager
self.component_manager.peer_manager
)
try:
@ -642,7 +636,7 @@ class PeerProtocolServerComponent(Component):
class ReflectorComponent(Component):
component_name = REFLECTOR_COMPONENT
depends_on = [DHT_COMPONENT, BLOB_COMPONENT, FILE_MANAGER_COMPONENT]
depends_on = [BLOB_COMPONENT, FILE_MANAGER_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
@ -656,10 +650,9 @@ class ReflectorComponent(Component):
@defer.inlineCallbacks
def start(self):
log.info("Starting reflector server")
dht_node = self.component_manager.get_component(DHT_COMPONENT)
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
file_manager = self.component_manager.get_component(FILE_MANAGER_COMPONENT)
reflector_factory = reflector_server_factory(dht_node.peer_manager, blob_manager, file_manager)
reflector_factory = ReflectorServerFactory(self.component_manager.peer_manager, blob_manager, file_manager)
try:
self.reflector_server = yield reactor.listenTCP(self.reflector_server_port, reflector_factory)
log.info('Started reflector on port %s', self.reflector_server_port)
@ -695,10 +688,12 @@ class UPnPComponent(Component):
@defer.inlineCallbacks
def _setup_redirects(self):
upnp_redirects = yield DeferredDict({
"UDP": from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")),
"TCP": from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port"))
})
d = {}
if PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components:
d["TCP"] = from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port"))
if DHT_COMPONENT not in self.component_manager.skip_components:
d["UDP"] = from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port"))
upnp_redirects = yield DeferredDict(d)
self.upnp_redirects.update(upnp_redirects)
@defer.inlineCallbacks
@ -710,33 +705,41 @@ class UPnPComponent(Component):
log.info("found upnp gateway: %s", self.upnp.gateway.manufacturer_string)
except Exception as err:
log.warning("upnp discovery failed: %s", err)
return
self.upnp = None
# update the external ip
try:
external_ip = yield from_future(self.upnp.get_external_ip())
if external_ip == "0.0.0.0":
log.warning("upnp doesn't know the external ip address (returned 0.0.0.0), using fallback")
external_ip = CS.get_external_ip()
if self.external_ip and self.external_ip != external_ip:
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
elif not self.external_ip:
log.info("got external ip: %s", external_ip)
self.external_ip = external_ip
except (asyncio.TimeoutError, UPnPError):
pass
if not self.upnp_redirects: # setup missing redirects
external_ip = None
if self.upnp:
try:
upnp_redirects = yield DeferredDict({
"UDP": from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")),
"TCP": from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port"))
})
external_ip = yield from_future(self.upnp.get_external_ip())
if external_ip != "0.0.0.0":
log.info("got external ip from UPnP: %s", external_ip)
except (asyncio.TimeoutError, UPnPError):
pass
if external_ip == "0.0.0.0" or not external_ip:
log.warning("unable to get external ip from UPnP, checking lbry.io fallback")
external_ip = yield CS.get_external_ip()
if self.external_ip and self.external_ip != external_ip:
log.info("external ip changed from %s to %s", self.external_ip, external_ip)
self.external_ip = external_ip
assert self.external_ip is not None # TODO: handle going/starting offline
if not self.upnp_redirects and self.upnp: # setup missing redirects
try:
log.info("add UPnP port mappings")
d = {}
if PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components:
d["TCP"] = from_future(self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port"))
if DHT_COMPONENT not in self.component_manager.skip_components:
d["UDP"] = from_future(self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port"))
upnp_redirects = yield DeferredDict(d)
log.info("set up redirects: %s", upnp_redirects)
self.upnp_redirects.update(upnp_redirects)
except (asyncio.TimeoutError, UPnPError):
self.upnp = None
return self._maintain_redirects()
else: # check existing redirects are still active
elif self.upnp: # check existing redirects are still active
found = set()
mappings = yield from_future(self.upnp.get_redirects())
for mapping in mappings:
@ -744,7 +747,7 @@ class UPnPComponent(Component):
if proto in self.upnp_redirects and mapping['NewExternalPort'] == self.upnp_redirects[proto]:
if mapping['NewInternalClient'] == self.upnp.lan_address:
found.add(proto)
if 'UDP' not in found:
if 'UDP' not in found and DHT_COMPONENT not in self.component_manager.skip_components:
try:
udp_port = yield from_future(
self.upnp.get_next_mapping(self._int_dht_node_port, "UDP", "LBRY DHT port")
@ -753,7 +756,7 @@ class UPnPComponent(Component):
log.info("refreshed upnp redirect for dht port: %i", udp_port)
except (asyncio.TimeoutError, UPnPError):
del self.upnp_redirects['UDP']
if 'TCP' not in found:
if 'TCP' not in found and PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components:
try:
tcp_port = yield from_future(
self.upnp.get_next_mapping(self._int_peer_port, "TCP", "LBRY peer port")
@ -762,22 +765,28 @@ class UPnPComponent(Component):
log.info("refreshed upnp redirect for peer port: %i", tcp_port)
except (asyncio.TimeoutError, UPnPError):
del self.upnp_redirects['TCP']
if 'TCP' in self.upnp_redirects and 'UDP' in self.upnp_redirects:
log.debug("upnp redirects are still active")
if ('TCP' in self.upnp_redirects
and PEER_PROTOCOL_SERVER_COMPONENT not in self.component_manager.skip_components) and (
'UDP' in self.upnp_redirects and DHT_COMPONENT not in self.component_manager.skip_components):
if self.upnp_redirects:
log.debug("upnp redirects are still active")
@defer.inlineCallbacks
def start(self):
log.info("detecting external ip")
if not self.use_upnp:
self.external_ip = CS.get_external_ip()
self.external_ip = yield CS.get_external_ip()
return
success = False
yield self._maintain_redirects()
if self.upnp:
if not self.upnp_redirects:
if not self.upnp_redirects and not all([x in self.component_manager.skip_components for x in
(DHT_COMPONENT, PEER_PROTOCOL_SERVER_COMPONENT)]):
log.error("failed to setup upnp, debugging infomation: %s", self.upnp.zipped_debugging_info)
else:
success = True
log.debug("set up upnp port redirects for gateway: %s", self.upnp.gateway.manufacturer_string)
if self.upnp_redirects:
log.debug("set up upnp port redirects for gateway: %s", self.upnp.gateway.manufacturer_string)
else:
log.error("failed to setup upnp")
self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, self.get_status())

74
lbrynet/extras/daemon/Daemon.py
View file

@ -16,6 +16,27 @@ from twisted.python.failure import Failure
from torba.client.baseaccount import SingleKey, HierarchicalDeterministic
from lbrynet import conf, utils, __version__
from lbrynet.dht.error import TimeoutError
from lbrynet.extras import system_info
from lbrynet.extras.reflector import reupload
from lbrynet.extras.daemon.Components import d2f, f2d
from lbrynet.extras.daemon.Components import WALLET_COMPONENT, DATABASE_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT
from lbrynet.extras.daemon.Components import FILE_MANAGER_COMPONENT, RATE_LIMITER_COMPONENT
from lbrynet.extras.daemon.Components import EXCHANGE_RATE_MANAGER_COMPONENT, PAYMENT_RATE_COMPONENT, UPNP_COMPONENT
from lbrynet.extras.daemon.ComponentManager import RequiredCondition
from lbrynet.extras.daemon.Downloader import GetStream
from lbrynet.extras.daemon.Publisher import Publisher
from lbrynet.extras.daemon.auth.server import AuthJSONRPCServer
from lbrynet.extras.wallet import LbryWalletManager
from lbrynet.extras.wallet.account import Account as LBCAccount
from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies
from lbrynet.p2p.StreamDescriptor import download_sd_blob
from lbrynet.p2p.Error import InsufficientFundsError, UnknownNameError, DownloadDataTimeout, DownloadSDTimeout
from lbrynet.p2p.Error import NullFundsError, NegativeFundsError, ResolveError
from lbrynet.p2p.Peer import Peer
from lbrynet.p2p.SinglePeerDownloader import SinglePeerDownloader
from lbrynet.p2p.client.StandaloneBlobDownloader import StandaloneBlobDownloader
from lbrynet.schema.claim import ClaimDict
from lbrynet.schema.uri import parse_lbry_uri
from lbrynet.schema.error import URIParseError, DecodeError
@ -23,32 +44,6 @@ from lbrynet.schema.validator import validate_claim_id
from lbrynet.schema.address import decode_address
from lbrynet.schema.decode import smart_decode
# TODO: importing this when internet is disabled raises a socket.gaierror
from lbrynet.p2p.system_info import get_lbrynet_version
from lbrynet.extras.daemon import conf
from lbrynet.extras.reflector import reupload
from .Components import d2f, f2d
from .Components import WALLET_COMPONENT, DATABASE_COMPONENT, DHT_COMPONENT, BLOB_COMPONENT
from .Components import STREAM_IDENTIFIER_COMPONENT, FILE_MANAGER_COMPONENT, RATE_LIMITER_COMPONENT
from .Components import EXCHANGE_RATE_MANAGER_COMPONENT, PAYMENT_RATE_COMPONENT, UPNP_COMPONENT
from .ComponentManager import RequiredCondition
from .Downloader import GetStream
from .Publisher import Publisher
from .auth.server import AuthJSONRPCServer
from lbrynet.p2p import utils, system_info
from lbrynet.p2p.StreamDescriptor import download_sd_blob
from lbrynet.p2p.Error import InsufficientFundsError, UnknownNameError
from lbrynet.p2p.Error import DownloadDataTimeout, DownloadSDTimeout
from lbrynet.p2p.Error import NullFundsError, NegativeFundsError
from lbrynet.p2p.Error import ResolveError
from lbrynet.dht.error import TimeoutError
from lbrynet.p2p.Peer import Peer
from lbrynet.p2p.SinglePeerDownloader import SinglePeerDownloader
from lbrynet.p2p.client.StandaloneBlobDownloader import StandaloneBlobDownloader
from lbrynet.extras.wallet import LbryWalletManager
from lbrynet.extras.wallet.account import Account as LBCAccount
from lbrynet.extras.wallet.dewies import dewies_to_lbc, lbc_to_dewies
log = logging.getLogger(__name__)
requires = AuthJSONRPCServer.requires
@ -210,7 +205,6 @@ class Daemon(AuthJSONRPCServer):
DATABASE_COMPONENT: "storage",
DHT_COMPONENT: "dht_node",
WALLET_COMPONENT: "wallet_manager",
STREAM_IDENTIFIER_COMPONENT: "sd_identifier",
FILE_MANAGER_COMPONENT: "file_manager",
EXCHANGE_RATE_MANAGER_COMPONENT: "exchange_rate_manager",
PAYMENT_RATE_COMPONENT: "payment_rate_manager",
@ -220,7 +214,7 @@ class Daemon(AuthJSONRPCServer):
}
def __init__(self, analytics_manager=None, component_manager=None):
to_skip = list(conf.settings['components_to_skip'])
to_skip = conf.settings['components_to_skip']
if 'reflector' not in to_skip and not conf.settings['run_reflector_server']:
to_skip.append('reflector')
looping_calls = {
@ -241,7 +235,6 @@ class Daemon(AuthJSONRPCServer):
self.storage = None
self.dht_node = None
self.wallet_manager: LbryWalletManager = None
self.sd_identifier = None
self.file_manager = None
self.exchange_rate_manager = None
self.payment_rate_manager = None
@ -305,7 +298,7 @@ class Daemon(AuthJSONRPCServer):
rate_manager = rate_manager or self.payment_rate_manager
timeout = timeout or 30
downloader = StandaloneBlobDownloader(
blob_hash, self.blob_manager, self.dht_node.peer_finder, self.rate_limiter,
blob_hash, self.blob_manager, self.component_manager.peer_finder, self.rate_limiter,
rate_manager, self.wallet_manager, timeout
)
return downloader.download()
@ -372,8 +365,8 @@ class Daemon(AuthJSONRPCServer):
self.analytics_manager.send_download_started(download_id, name, claim_dict)
self.analytics_manager.send_new_download_start(download_id, name, claim_dict)
self.streams[sd_hash] = GetStream(
self.sd_identifier, self.wallet_manager, self.exchange_rate_manager, self.blob_manager,
self.dht_node.peer_finder, self.rate_limiter, self.payment_rate_manager, self.storage,
self.file_manager.sd_identifier, self.wallet_manager, self.exchange_rate_manager, self.blob_manager,
self.component_manager.peer_finder, self.rate_limiter, self.payment_rate_manager, self.storage,
conf.settings['max_key_fee'], conf.settings['disable_max_key_fee'], conf.settings['data_rate'],
timeout
)
@ -432,7 +425,7 @@ class Daemon(AuthJSONRPCServer):
if blob:
return self.blob_manager.get_blob(blob[0])
return download_sd_blob(
sd_hash.decode(), self.blob_manager, self.dht_node.peer_finder, self.rate_limiter,
sd_hash.decode(), self.blob_manager, self.component_manager.peer_finder, self.rate_limiter,
self.payment_rate_manager, self.wallet_manager, timeout=conf.settings['peer_search_timeout'],
download_mirrors=conf.settings['download_mirrors']
)
@ -450,7 +443,7 @@ class Daemon(AuthJSONRPCServer):
Get total stream size in bytes from a sd blob
"""
d = self.sd_identifier.get_metadata_for_sd_blob(sd_blob)
d = self.file_manager.sd_identifier.get_metadata_for_sd_blob(sd_blob)
d.addCallback(lambda metadata: metadata.validator.info_to_show())
d.addCallback(lambda info: int(dict(info)['stream_size']))
return d
@ -468,9 +461,7 @@ class Daemon(AuthJSONRPCServer):
"""
Calculate estimated LBC cost for a stream given its size in bytes
"""
cost = self._get_est_cost_from_stream_size(size)
resolved = await self.wallet_manager.resolve(uri)
if uri in resolved and 'claim' in resolved[uri]:
@ -535,7 +526,6 @@ class Daemon(AuthJSONRPCServer):
sd blob will be downloaded to determine the stream size
"""
if size is not None:
return self.get_est_cost_using_known_size(uri, size)
return self.get_est_cost_from_uri(uri)
@ -819,7 +809,7 @@ class Daemon(AuthJSONRPCServer):
'ip': (str) remote ip, if available,
'lbrynet_version': (str) lbrynet_version,
'lbryum_version': (str) lbryum_version,
'lbrynet.schema_version': (str) lbrynet.schema_version,
'lbryschema_version': (str) lbryschema_version,
'os_release': (str) os release string
'os_system': (str) os name
'platform': (str) platform string
@ -851,7 +841,7 @@ class Daemon(AuthJSONRPCServer):
message,
conf.settings.installation_id,
platform_name,
get_lbrynet_version()
__version__
)
return self._render_response(True)
@ -1776,8 +1766,8 @@ class Daemon(AuthJSONRPCServer):
results[resolved_uri] = resolved[resolved_uri]
return results
@requires(STREAM_IDENTIFIER_COMPONENT, WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT,
DHT_COMPONENT, RATE_LIMITER_COMPONENT, PAYMENT_RATE_COMPONENT, DATABASE_COMPONENT,
@requires(WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT,
RATE_LIMITER_COMPONENT, PAYMENT_RATE_COMPONENT, DATABASE_COMPONENT,
conditions=[WALLET_IS_UNLOCKED])
async def jsonrpc_get(self, uri, file_name=None, timeout=None):
"""
@ -1962,7 +1952,7 @@ class Daemon(AuthJSONRPCServer):
response = yield self._render_response(result)
defer.returnValue(response)
@requires(STREAM_IDENTIFIER_COMPONENT, WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT,
@requires(WALLET_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, BLOB_COMPONENT,
DHT_COMPONENT, RATE_LIMITER_COMPONENT, PAYMENT_RATE_COMPONENT, DATABASE_COMPONENT,
conditions=[WALLET_IS_UNLOCKED])
def jsonrpc_stream_cost_estimate(self, uri, size=None):

9
lbrynet/extras/daemon/DaemonConsole.py
View file

@ -6,11 +6,10 @@ import logging.handlers
from twisted.internet import defer, reactor, threads
from aiohttp import client_exceptions
from lbrynet.extras.daemon import analytics, conf
from lbrynet.p2p import utils
from lbrynet.p2p import log_support
from .auth.client import LBRYAPIClient
from .Daemon import Daemon
from lbrynet import utils, conf, log_support
from lbrynet.extras.daemon import analytics
from lbrynet.extras.daemon.auth.client import LBRYAPIClient
from lbrynet.extras.daemon.Daemon import Daemon
log = logging.getLogger(__name__)

10
lbrynet/extras/daemon/DaemonControl.py
View file

@ -7,16 +7,14 @@ if 'win' in sys.platform:
import certifi
os.environ['SSL_CERT_FILE'] = certifi.where()
from lbrynet.p2p import log_support
import argparse
import logging.handlers
from twisted.internet import reactor
from lbrynet.extras.daemon import conf
from lbrynet.p2p import utils, system_info
from .Daemon import Daemon
from lbrynet import utils, conf, log_support
from lbrynet.extras import system_info
from lbrynet.extras.daemon.Daemon import Daemon
log = logging.getLogger(__name__)
@ -54,7 +52,7 @@ def start(argv=None, conf_path=None):
conf.settings.update({'use_auth_http': args.useauth}, data_types=(conf.TYPE_CLI,))
if args.version:
version = system_info.get_platform(get_ip=False)
version = system_info.get_platform()
version['installation_id'] = conf.settings.installation_id
print(utils.json_dumps_pretty(version))
return

6
lbrynet/extras/daemon/Downloader.py
View file

@ -3,17 +3,17 @@ import os
from twisted.internet import defer
from twisted.internet.task import LoopingCall
from .Components import f2d
from lbrynet import conf
from lbrynet.schema.fee import Fee
from lbrynet.p2p.Error import InsufficientFundsError, KeyFeeAboveMaxAllowed, InvalidStreamDescriptorError
from lbrynet.p2p.Error import DownloadDataTimeout, DownloadCanceledError, DownloadSDTimeout
from lbrynet.p2p.utils import safe_start_looping_call, safe_stop_looping_call
from lbrynet.utils import safe_start_looping_call, safe_stop_looping_call
from lbrynet.p2p.StreamDescriptor import download_sd_blob
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloaderFactory
from lbrynet.extras.daemon import conf
from torba.client.constants import COIN
from lbrynet.extras.wallet.dewies import dewies_to_lbc
from lbrynet.extras.daemon.Components import f2d
INITIALIZING_CODE = 'initializing'
DOWNLOAD_METADATA_CODE = 'downloading_metadata'

3
lbrynet/dht/hashannouncer.py → lbrynet/extras/daemon/HashAnnouncer.py
View file

@ -2,8 +2,7 @@ import binascii
import logging
from twisted.internet import defer, task
from lbrynet.p2p import utils
from lbrynet.extras.daemon import conf
from lbrynet import utils, conf
log = logging.getLogger(__name__)

38
lbrynet/dht/peerfinder.py → lbrynet/extras/daemon/PeerFinder.py
View file

@ -2,7 +2,7 @@ import binascii
import logging
from twisted.internet import defer
from lbrynet.extras.daemon import conf
from lbrynet import conf
log = logging.getLogger(__name__)
@ -18,13 +18,12 @@ class DHTPeerFinder(DummyPeerFinder):
"""This class finds peers which have announced to the DHT that they have certain blobs"""
#implements(IPeerFinder)
def __init__(self, dht_node, peer_manager):
def __init__(self, component_manager):
"""
dht_node - an instance of dht.Node class
peer_manager - an instance of PeerManager class
component_manager - an instance of ComponentManager
"""
self.dht_node = dht_node
self.peer_manager = peer_manager
self.component_manager = component_manager
self.peer_manager = component_manager.peer_manager
self.peers = {}
self._ongoing_searchs = {}
@ -39,19 +38,30 @@ class DHTPeerFinder(DummyPeerFinder):
Returns:
list of peers for the blob
"""
self.peers.setdefault(blob_hash, {(self.dht_node.externalIP, self.dht_node.peerPort,)})
if "dht" in self.component_manager.skip_components:
return defer.succeed([])
if not self.component_manager.all_components_running("dht"):
return defer.succeed([])
dht_node = self.component_manager.get_component("dht")
self.peers.setdefault(blob_hash, {(dht_node.externalIP, dht_node.peerPort,)})
if not blob_hash in self._ongoing_searchs or self._ongoing_searchs[blob_hash].called:
self._ongoing_searchs[blob_hash] = self._execute_peer_search(blob_hash, timeout)
peers = set(self._filter_self(blob_hash) if filter_self else self.peers[blob_hash])
self._ongoing_searchs[blob_hash] = self._execute_peer_search(dht_node, blob_hash, timeout)
def _filter_self(blob_hash):
my_host, my_port = dht_node.externalIP, dht_node.peerPort
return {(host, port) for host, port in self.peers[blob_hash] if (host, port) != (my_host, my_port)}
peers = set(_filter_self(blob_hash) if filter_self else self.peers[blob_hash])
return defer.succeed([self.peer_manager.get_peer(*peer) for peer in peers])
@defer.inlineCallbacks
def _execute_peer_search(self, blob_hash, timeout):
def _execute_peer_search(self, dht_node, blob_hash, timeout):
bin_hash = binascii.unhexlify(blob_hash)
finished_deferred = self.dht_node.iterativeFindValue(bin_hash, exclude=self.peers[blob_hash])
finished_deferred = dht_node.iterativeFindValue(bin_hash, exclude=self.peers[blob_hash])
timeout = timeout or conf.settings['peer_search_timeout']
if timeout:
finished_deferred.addTimeout(timeout, self.dht_node.clock)
finished_deferred.addTimeout(timeout, dht_node.clock)
try:
peer_list = yield finished_deferred
self.peers[blob_hash].update({(host, port) for _, host, port in peer_list})
@ -59,7 +69,3 @@ class DHTPeerFinder(DummyPeerFinder):
log.debug("DHT timed out while looking peers for blob %s after %s seconds", blob_hash, timeout)
finally:
del self._ongoing_searchs[blob_hash]
def _filter_self(self, blob_hash):
my_host, my_port = self.dht_node.externalIP, self.dht_node.peerPort
return {(host, port) for host, port in self.peers[blob_hash] if (host, port) != (my_host, my_port)}

0
lbrynet/p2p/PeerManager.py → lbrynet/extras/daemon/PeerManager.py
View file

2
lbrynet/extras/daemon/__init__.py
View file

@ -1 +1 @@
from . import Components # register Component classes
from lbrynet.extras.daemon import Components # register Component classes

8
lbrynet/extras/daemon/analytics.py
View file

@ -4,8 +4,8 @@ import logging
import treq
from twisted.internet import defer, task
from lbrynet.extras.daemon import conf
from lbrynet.p2p import looping_call_manager, utils, system_info
from lbrynet import conf, utils
from lbrynet.extras import looping_call_manager, system_info
# Things We Track
SERVER_STARTUP = 'Server Startup'
@ -136,8 +136,8 @@ class Manager:
def _get_looping_calls(self):
return [
('send_heartbeat', self._send_heartbeat, 60),
('update_tracked_metrics', self._update_tracked_metrics, 300),
('send_heartbeat', self._send_heartbeat, 300),
('update_tracked_metrics', self._update_tracked_metrics, 600),
]
def _setup_looping_calls(self):

2
lbrynet/extras/daemon/auth/auth.py
View file

@ -3,7 +3,7 @@ from zope.interface import implementer
from twisted.cred import portal, checkers, credentials, error as cred_error
from twisted.internet import defer
from twisted.web import resource
from .keyring import Keyring
from lbrynet.extras.daemon.auth.keyring import Keyring
log = logging.getLogger(__name__)

4
lbrynet/extras/daemon/auth/client.py
View file

@ -3,8 +3,8 @@ import aiohttp
import logging
from urllib.parse import urlparse
from lbrynet.extras.daemon import conf
from .keyring import Keyring, APIKey
from lbrynet import conf
from lbrynet.extras.daemon.auth.keyring import Keyring, APIKey
log = logging.getLogger(__name__)
USER_AGENT = "AuthServiceProxy/0.1"

6
lbrynet/extras/daemon/auth/factory.py
View file

@ -3,9 +3,9 @@ import logging
from twisted.web import server, guard, resource
from twisted.cred import portal
from lbrynet.extras.daemon import conf
from .auth import PasswordChecker, HttpPasswordRealm
from ..auth.keyring import Keyring
from lbrynet import conf
from lbrynet.extras.daemon.auth.auth import PasswordChecker, HttpPasswordRealm
from lbrynet.extras.daemon.auth.keyring import Keyring
log = logging.getLogger(__name__)

5
lbrynet/extras/daemon/auth/keyring.py
View file

@ -3,6 +3,7 @@ import datetime
import hmac
import hashlib
import base58
import logging
from OpenSSL.crypto import FILETYPE_PEM
from ssl import create_default_context, SSLContext
from cryptography.hazmat.backends import default_backend
@ -12,8 +13,8 @@ from cryptography.x509.name import NameOID, NameAttribute
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.hazmat.primitives import serialization
from twisted.internet import ssl
import logging
from lbrynet.extras.daemon import conf
from lbrynet import conf
log = logging.getLogger(__name__)

14
lbrynet/extras/daemon/auth/server.py
View file

@ -4,7 +4,6 @@ from six.moves.urllib import parse as urlparse
import json
import inspect
import signal
from functools import wraps
from twisted.web import server
from twisted.internet import defer
@ -14,16 +13,17 @@ from twisted.internet.error import ConnectionDone, ConnectionLost
from txjsonrpc import jsonrpclib
from traceback import format_exc
from lbrynet.extras.daemon import analytics, conf
from lbrynet import conf, utils
from lbrynet.extras.daemon import analytics
from lbrynet.p2p.Error import InvalidAuthenticationToken
from lbrynet.p2p import utils
from lbrynet.p2p.Error import ComponentsNotStarted, ComponentStartConditionNotMet
from lbrynet.p2p.looping_call_manager import LoopingCallManager
from lbrynet.extras.looping_call_manager import LoopingCallManager
from lbrynet.extras.daemon.ComponentManager import ComponentManager
from .keyring import APIKey, Keyring
from .undecorated import undecorated
from .factory import AuthJSONRPCResource
from lbrynet.extras.daemon.auth.keyring import APIKey, Keyring
from lbrynet.extras.daemon.auth.undecorated import undecorated
from lbrynet.extras.daemon.auth.factory import AuthJSONRPCResource
from lbrynet.extras.daemon.json_response_encoder import JSONResponseEncoder
log = logging.getLogger(__name__)
EMPTY_PARAMS = [{}]

2
lbrynet/extras/daemon/migrator/migrate5to6.py
View file

@ -2,8 +2,8 @@ import sqlite3
import os
import json
import logging
from lbrynet import conf
from lbrynet.schema.decode import smart_decode
from lbrynet.extras.daemon import conf
log = logging.getLogger(__name__)

4
lbrynet/extras/daemon/storage.py
View file

@ -6,13 +6,13 @@ from binascii import hexlify, unhexlify
from decimal import Decimal
from twisted.internet import defer, task, threads
from twisted.enterprise import adbapi
from torba.client.constants import COIN
from lbrynet import conf
from lbrynet.schema.claim import ClaimDict
from lbrynet.schema.decode import smart_decode
from lbrynet.extras.daemon import conf
from lbrynet.blob.CryptBlob import CryptBlobInfo
from lbrynet.dht.constants import dataExpireTimeout
from torba.client.constants import COIN
log = logging.getLogger(__name__)

0
lbrynet/p2p/looping_call_manager.py → lbrynet/extras/looping_call_manager.py
View file

3
lbrynet/extras/reflector/__init__.py
View file

@ -63,6 +63,3 @@ If the transfer was not successful (False), the blob is re-added to the needed_b
Blob requests continue for each of the blobs the client has queued to send, when completed
the client disconnects.
"""
from .server.server import ReflectorServerFactory as ServerFactory
from .client.client import EncryptedFileReflectorClientFactory as ClientFactory
from .client.blob import BlobReflectorClientFactory as BlobClientFactory

9
lbrynet/extras/reflector/reupload.py
View file

@ -1,8 +1,9 @@
import random
from twisted.internet import reactor, defer
from lbrynet.extras.daemon import conf
from . import ClientFactory, BlobClientFactory
from lbrynet import conf
from lbrynet.extras.reflector.client.client import EncryptedFileReflectorClientFactory
from lbrynet.extras.reflector.client.blob import BlobReflectorClientFactory
def _is_ip(host):
@ -26,7 +27,7 @@ def resolve(host):
@defer.inlineCallbacks
def _reflect_stream(blob_manager, stream_hash, sd_hash, reflector_server):
reflector_address, reflector_port = reflector_server[0], reflector_server[1]
factory = ClientFactory(blob_manager, stream_hash, sd_hash)
factory = EncryptedFileReflectorClientFactory(blob_manager, stream_hash, sd_hash)
ip = yield resolve(reflector_address)
yield reactor.connectTCP(ip, reflector_port, factory)
result = yield factory.finished_deferred
@ -40,7 +41,7 @@ def _reflect_file(lbry_file, reflector_server):
@defer.inlineCallbacks
def _reflect_blobs(blob_manager, blob_hashes, reflector_server):
reflector_address, reflector_port = reflector_server[0], reflector_server[1]
factory = BlobClientFactory(blob_manager, blob_hashes)
factory = BlobReflectorClientFactory(blob_manager, blob_hashes)
ip = yield resolve(reflector_address)
yield reactor.connectTCP(ip, reflector_port, factory)
result = yield factory.finished_deferred

2
lbrynet/extras/reflector/server/server.py
View file

@ -3,7 +3,7 @@ import json
from twisted.python import failure
from twisted.internet import error, defer
from twisted.internet.protocol import Protocol, ServerFactory
from lbrynet.p2p.utils import is_valid_blobhash
from lbrynet.blob.blob_file import is_valid_blobhash
from lbrynet.p2p.Error import DownloadCanceledError, InvalidBlobHashError
from lbrynet.p2p.StreamDescriptor import BlobStreamDescriptorReader
from lbrynet.p2p.StreamDescriptor import save_sd_info

30
lbrynet/extras/system_info.py Normal file
View file

@ -0,0 +1,30 @@
import platform
import os
import logging.handlers
from lbrynet.schema import __version__ as schema_version
from lbrynet import build_type, __version__ as lbrynet_version
log = logging.getLogger(__name__)
def get_platform() -> dict:
p = {
"processor": platform.processor(),
"python_version": platform.python_version(),
"platform": platform.platform(),
"os_release": platform.release(),
"os_system": platform.system(),
"lbrynet_version": lbrynet_version,
"lbryschema_version": schema_version,
"build": build_type.BUILD, # CI server sets this during build step
}
if p["os_system"] == "Linux":
try:
import distro
p["distro"] = distro.info()
p["desktop"] = os.environ.get('XDG_CURRENT_DESKTOP', 'Unknown')
except ModuleNotFoundError:
pass
return p

6
lbrynet/extras/wallet/__init__.py
View file

@ -6,6 +6,6 @@ __node_url__ = (
)
__spvserver__ = 'lbrynet.extras.wallet.server.coin.LBCRegTest'
from .ledger import MainNetLedger, RegTestLedger
from .manager import LbryWalletManager
from .network import Network
from lbrynet.extras.wallet.ledger import MainNetLedger, RegTestLedger
from lbrynet.extras.wallet.manager import LbryWalletManager
from lbrynet.extras.wallet.network import Network

17
lbrynet/extras/wallet/ledger.py
View file

@ -2,17 +2,16 @@ import asyncio
import logging
from binascii import unhexlify
from lbrynet.extras.wallet.dewies import dewies_to_lbc
from torba.client.baseledger import BaseLedger
from lbrynet.schema.error import URIParseError
from lbrynet.schema.uri import parse_lbry_uri
from torba.client.baseledger import BaseLedger
from .resolve import Resolver
from .account import Account
from .network import Network
from .database import WalletDatabase
from .transaction import Transaction
from .header import Headers, UnvalidatedHeaders
from lbrynet.extras.wallet.dewies import dewies_to_lbc
from lbrynet.extras.wallet.resolve import Resolver
from lbrynet.extras.wallet.account import Account
from lbrynet.extras.wallet.network import Network
from lbrynet.extras.wallet.database import WalletDatabase
from lbrynet.extras.wallet.transaction import Transaction
from lbrynet.extras.wallet.header import Headers, UnvalidatedHeaders
log = logging.getLogger(__name__)

10
lbrynet/extras/wallet/manager.py
View file

@ -14,11 +14,11 @@ from torba.client.basemanager import BaseWalletManager
from lbrynet.schema.claim import ClaimDict
from .ledger import MainNetLedger
from .account import BaseAccount, generate_certificate
from .transaction import Transaction
from .database import WalletDatabase
from .dewies import dewies_to_lbc
from lbrynet.extras.wallet.ledger import MainNetLedger
from lbrynet.extras.wallet.account import BaseAccount, generate_certificate
from lbrynet.extras.wallet.transaction import Transaction
from lbrynet.extras.wallet.database import WalletDatabase
from lbrynet.extras.wallet.dewies import dewies_to_lbc
log = logging.getLogger(__name__)

3
lbrynet/extras/wallet/resolve.py
View file

@ -9,8 +9,7 @@ from lbrynet.schema.claim import ClaimDict
from lbrynet.schema.decode import smart_decode
from lbrynet.schema.error import DecodeError
from lbrynet.schema.uri import parse_lbry_uri
from .claim_proofs import verify_proof, InvalidProofError
from lbrynet.extras.wallet.claim_proofs import verify_proof, InvalidProofError
log = logging.getLogger(__name__)

2
lbrynet/extras/wallet/server/block_processor.py
View file

@ -9,7 +9,7 @@ from lbrynet.schema.proto.claim_pb2 import Claim
from lbrynet.schema.uri import parse_lbry_uri
from lbrynet.schema.decode import smart_decode
from .model import NameClaim, ClaimInfo, ClaimUpdate, ClaimSupport
from lbrynet.extras.wallet.server.model import NameClaim, ClaimInfo, ClaimUpdate, ClaimSupport
class LBRYBlockProcessor(BlockProcessor):

2
lbrynet/extras/wallet/server/coin.py
View file

@ -6,7 +6,7 @@ from torba.server.hash import hash_to_hex_str, HASHX_LEN
from hashlib import sha256
from torba.server.coins import Coin, CoinError
from .opcodes import decode_claim_script, opcodes as lbry_opcodes
from lbrynet.extras.wallet.server.opcodes import decode_claim_script, opcodes as lbry_opcodes
class LBC(Coin):

2
lbrynet/extras/wallet/server/db.py
View file

@ -6,7 +6,7 @@ from torba.server.hash import hash_to_hex_str
from torba.server.db import DB
from .model import ClaimInfo
from lbrynet.extras.wallet.server.model import ClaimInfo
class LBRYDB(DB):

2
lbrynet/extras/wallet/server/opcodes.py
View file

@ -1,6 +1,6 @@
import struct
from torba.server.enum import Enumeration
from .model import NameClaim, ClaimSupport, ClaimUpdate
from lbrynet.extras.wallet.server.model import NameClaim, ClaimSupport, ClaimUpdate
# TODO: Take this to lbryschema (it's also on lbryum and lbryum-server)

4
lbrynet/extras/wallet/server/session.py
View file

@ -8,8 +8,8 @@ from torba.server import util
from lbrynet.schema.uri import parse_lbry_uri
from lbrynet.schema.error import URIParseError
from .block_processor import LBRYBlockProcessor
from .db import LBRYDB
from lbrynet.extras.wallet.server.block_processor import LBRYBlockProcessor
from lbrynet.extras.wallet.server.db import LBRYDB
class LBRYElectrumX(ElectrumX):

4
lbrynet/extras/wallet/server/tx.py
View file

@ -1,6 +1,6 @@
from torba.server.tx import Deserializer
from .opcodes import decode_claim_script
from .model import TxClaimOutput, LBRYTx
from lbrynet.extras.wallet.server.opcodes import decode_claim_script
from lbrynet.extras.wallet.server.model import TxClaimOutput, LBRYTx
class LBRYDeserializer(Deserializer):

7
lbrynet/extras/wallet/transaction.py
View file

@ -2,13 +2,12 @@ import struct
from binascii import hexlify, unhexlify
from typing import List, Iterable, Optional
from lbrynet.schema.decode import smart_decode
from .account import Account
from torba.client.basetransaction import BaseTransaction, BaseInput, BaseOutput
from torba.client.hash import hash160
from lbrynet.schema.decode import smart_decode
from lbrynet.schema.claim import ClaimDict
from .script import InputScript, OutputScript
from lbrynet.extras.wallet.account import Account
from lbrynet.extras.wallet.script import InputScript, OutputScript
class Input(BaseInput):

7
lbrynet/p2p/log_support.py → lbrynet/log_support.py
View file

@ -3,14 +3,11 @@ import logging
import logging.handlers
import sys
import traceback
import treq
from twisted.internet import defer
import twisted.python.log
from twisted.internet import defer
from lbrynet import __version__ as lbrynet_version, build_type
from lbrynet.extras.daemon import conf
from lbrynet.p2p import utils
from lbrynet import utils, conf
class HTTPSHandler(logging.Handler):

3
lbrynet/p2p/BlobAvailability.py
View file

@ -1,10 +1,9 @@
import logging
import random
import time
from decimal import Decimal
from twisted.internet import defer
from twisted.internet.task import LoopingCall
from decimal import Decimal
log = logging.getLogger(__name__)

7
lbrynet/p2p/HTTPBlobDownloader.py
View file

@ -1,10 +1,11 @@
from random import choice
import logging
import treq
from random import choice
from twisted.internet import defer, task
from twisted.internet.error import ConnectingCancelledError
from twisted.web._newclient import ResponseNeverReceived
import treq
from lbrynet.p2p.utils import DeferredDict
from lbrynet.utils import DeferredDict
from lbrynet.p2p.Error import DownloadCanceledError
log = logging.getLogger(__name__)

4
lbrynet/p2p/PaymentRateManager.py
View file

@ -1,6 +1,6 @@
from lbrynet.p2p.Strategy import get_default_strategy, OnlyFreeStrategy
from lbrynet.extras.daemon import conf
from decimal import Decimal
from lbrynet import conf
from lbrynet.p2p.Strategy import get_default_strategy, OnlyFreeStrategy
class BasePaymentRateManager:

2
lbrynet/p2p/Peer.py
View file

@ -1,6 +1,6 @@
import datetime
from collections import defaultdict
from lbrynet.p2p import utils
from lbrynet import utils
# Do not create this object except through PeerManager
class Peer:

3
lbrynet/p2p/PriceModel.py
View file

@ -1,6 +1,5 @@
from decimal import Decimal
from lbrynet.extras.daemon import conf
from lbrynet import conf
def get_default_price_model(blob_tracker, base_price, **kwargs):

4
lbrynet/p2p/SinglePeerDownloader.py
View file

@ -4,7 +4,7 @@ import tempfile
from twisted.internet import defer, threads, reactor
from lbrynet.blob import BlobFile
from lbrynet.blob.blob_file import BlobFile
from lbrynet.p2p.BlobManager import DiskBlobManager
from lbrynet.p2p.RateLimiter import DummyRateLimiter
from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager
@ -12,7 +12,7 @@ from lbrynet.p2p.client.BlobRequester import BlobRequester
from lbrynet.p2p.client.StandaloneBlobDownloader import StandaloneBlobDownloader
from lbrynet.p2p.client.ConnectionManager import ConnectionManager
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.dht.peerfinder import DummyPeerFinder
from lbrynet.extras.daemon.PeerFinder import DummyPeerFinder
log = logging.getLogger(__name__)

3
lbrynet/p2p/Strategy.py
View file

@ -1,5 +1,5 @@
from decimal import Decimal
from lbrynet.extras.daemon import conf
from lbrynet import conf
from lbrynet.p2p.Offer import Offer
from lbrynet.p2p.PriceModel import MeanAvailabilityWeightedPrice, ZeroPrice
@ -133,7 +133,6 @@ class BasicAvailabilityWeightedStrategy(Strategy):
class OnlyFreeStrategy(Strategy):
def __init__(self, *args, **kwargs):
price_model = ZeroPrice()
super().__init__(price_model, 0.0, 0.0, True)

8
lbrynet/p2p/StreamDescriptor.py
View file

@ -1,11 +1,11 @@
from binascii import unhexlify
import string
from collections import defaultdict
import json
import logging
from collections import defaultdict
from binascii import unhexlify
from twisted.internet import threads, defer
from lbrynet.p2p.cryptoutils import get_lbry_hash_obj
from lbrynet.cryptoutils import get_lbry_hash_obj
from lbrynet.p2p.client.StandaloneBlobDownloader import StandaloneBlobDownloader
from lbrynet.p2p.Error import UnknownStreamTypeError, InvalidStreamDescriptorError
from lbrynet.p2p.HTTPBlobDownloader import HTTPBlobDownloader

2
lbrynet/p2p/__init__.py
View file

@ -5,5 +5,3 @@ This includes classes for connecting to other peers and downloading blobs from t
connections from peers and responding to their requests, managing locally stored blobs, sending
and receiving payments, and locating peers in the DHT.
"""
from lbrynet import custom_logger

3
lbrynet/p2p/client/ClientProtocol.py
View file

@ -5,8 +5,7 @@ from twisted.internet import error, defer
from twisted.internet.protocol import Protocol, ClientFactory
from twisted.protocols.policies import TimeoutMixin
from twisted.python import failure
from lbrynet.extras.daemon import conf
from lbrynet.p2p import utils
from lbrynet import conf, utils
from lbrynet.p2p.Error import ConnectionClosedBeforeResponseError, NoResponseError
from lbrynet.p2p.Error import DownloadCanceledError, MisbehavingPeerError
from lbrynet.p2p.Error import RequestCanceledError

3
lbrynet/p2p/client/ConnectionManager.py
View file

@ -1,10 +1,9 @@
import random
import logging
from twisted.internet import defer, reactor
from lbrynet.extras.daemon import conf
from lbrynet import utils, conf
from lbrynet.p2p.client.ClientProtocol import ClientProtocolFactory
from lbrynet.p2p.Error import InsufficientFundsError
from lbrynet.p2p import utils
log = logging.getLogger(__name__)

3
lbrynet/p2p/client/StandaloneBlobDownloader.py
View file

@ -4,7 +4,8 @@ from lbrynet.p2p.client.BlobRequester import BlobRequester
from lbrynet.p2p.client.ConnectionManager import ConnectionManager
from lbrynet.p2p.client.DownloadManager import DownloadManager
from lbrynet.p2p.Error import InvalidBlobHashError, DownloadSDTimeout
from lbrynet.p2p.utils import is_valid_blobhash, safe_start_looping_call, safe_stop_looping_call
from lbrynet.blob.blob_file import is_valid_blobhash
from lbrynet.utils import safe_start_looping_call, safe_stop_looping_call
from twisted.python.failure import Failure
from twisted.internet import defer
from twisted.internet.task import LoopingCall

59
lbrynet/p2p/system_info.py
View file

@ -1,59 +0,0 @@
import platform
import json
import subprocess
import os
from six.moves.urllib import request
from six.moves.urllib.error import URLError
from lbrynet.schema import __version__ as schema_version
from lbrynet import build_type, __version__ as lbrynet_version
from lbrynet.extras.daemon.conf import ROOT_DIR
import logging.handlers
log = logging.getLogger(__name__)
def get_lbrynet_version() -> str:
if build_type.BUILD == "dev":
try:
with open(os.devnull, 'w') as devnull:
git_dir = ROOT_DIR + '/.git'
return subprocess.check_output(
['git', '--git-dir='+git_dir, 'describe', '--dirty', '--always'],
stderr=devnull
).decode().strip().lstrip('v')
except (subprocess.CalledProcessError, OSError):
log.debug("failed to get version from git")
return lbrynet_version
def get_platform(get_ip: bool = True) -> dict:
p = {
"processor": platform.processor(),
"python_version": platform.python_version(),
"platform": platform.platform(),
"os_release": platform.release(),
"os_system": platform.system(),
"lbrynet_version": get_lbrynet_version(),
"lbryschema_version": schema_version,
"build": build_type.BUILD, # CI server sets this during build step
}
if p["os_system"] == "Linux":
try:
import distro
p["distro"] = distro.info()
p["desktop"] = os.environ.get('XDG_CURRENT_DESKTOP', 'Unknown')
except ModuleNotFoundError:
pass
# TODO: remove this from get_platform and add a get_external_ip function using treq
if get_ip:
try:
response = json.loads(request.urlopen("https://api.lbry.io/ip").read())
if not response['success']:
raise URLError("failed to get external ip")
p['ip'] = response['data']['ip']
except (URLError, AssertionError):
p['ip'] = "Could not determine IP"
return p

3
lbrynet/schema/decode.py
View file

@ -1,13 +1,12 @@
import json
import binascii
from google.protobuf import json_format # pylint: disable=no-name-in-module
import six
from lbrynet.schema.error import DecodeError, InvalidAddress
from lbrynet.schema.legacy.migrate import migrate as schema_migrator
from lbrynet.schema.claim import ClaimDict
from google.protobuf import json_format # pylint: disable=no-name-in-module
def migrate_json_claim_value(decoded_json):

41
lbrynet/schema/signature/flags.py
View file

@ -1,41 +0,0 @@
class SignatureSerializationFlag:
UNSIGNED = 0
'''
Format:
<FLAG><CLAIM BINARY>
or (legacy)
<CLAIM BINARY>
'''
ECDSA_LEGACY = 1
'''
Old claim format, which carried the signature inside the protobuf. Requires serializing back the claim with
signature stripped out for validation. This process requires knowledge on how a claim is serialized, thus requires
old fixed protobuf schema to work.
Format:
<CLAIM PROTOBUF SERIALIZED>
Curves: NIST256p, NIST384p, SECP256k1
Signature content: `r` and `s` in each half of the 64 or 96 bytes (depends on curve)
Signed payload:
1. Claim transaction output address (raw, decoded using base58)
2. Stripped out claim protobuf serialization (without the signature)
3. Certificate claim id (binary, not in network byte order)
'''
ECDSA_SECP256K1 = 2
'''
Format:
<FLAG><CERTIFICATE ID><SIGNATURE><BINARY PAYLOAD>
Curve: SECP256K1
Signature content: 64 bytes total, each half represents `r` and `s`
Signed payload:
1. raw claim name as bytes
2. Claim transaction output address (raw, decoded using base58)
3. Binary payload, independent of serialization (everything after the signature last byte)
4. Certificate claim id, not in network byte order.
A certificate can be signed as well, but this serialization model is unaware of content type or protobuf format.
'''
@classmethod
def is_flag_valid(cls, flag):
# todo: use python 3 enum when fully ported, but not worth now as its an extra dependency for py2
return 0 <= flag <= 2

11
lbrynet/schema/signature/serializer.py
View file

@ -1,11 +0,0 @@
import struct
from collections import namedtuple
from .flags import SignatureSerializationFlag
class Signature(namedtuple("Signature", "flags signature certificate_id")):
def deserialize(cls, payload):
flag = struct.unpack("<b", payload[0])[0]
if not SignatureSerializationFlag.is_flag_valid(flag):
return Signature(SignatureSerializationFlag.ECDSA_LEGACY, )
certificate

21
lbrynet/p2p/utils.py → lbrynet/utils.py
View file

@ -12,13 +12,10 @@ import pkg_resources
from twisted.python.failure import Failure
from twisted.internet import defer
from lbrynet.schema.claim import ClaimDict
from lbrynet.p2p.cryptoutils import get_lbry_hash_obj
from lbrynet.cryptoutils import get_lbry_hash_obj
log = logging.getLogger(__name__)
# digest_size is in bytes, and blob hashes are hex encoded
blobhash_length = get_lbry_hash_obj().digest_size * 2
# defining these time functions here allows for easier overriding in testing
def now():
@ -52,6 +49,7 @@ def call_later(delay, func, *args, **kwargs):
from twisted.internet import reactor
return reactor.callLater(delay, func, *args, **kwargs)
def safe_start_looping_call(looping_call, interval_sec):
if not looping_call.running:
looping_call.start(interval_sec)
@ -69,21 +67,6 @@ def generate_id(num=None):
return h.digest()
def is_valid_hashcharacter(char):
return char in "0123456789abcdef"
def is_valid_blobhash(blobhash):
"""Checks whether the blobhash is the correct length and contains only
valid characters (0-9, a-f)
@param blobhash: string, the blobhash to check
@return: True/False
"""
return len(blobhash) == blobhash_length and all(is_valid_hashcharacter(l) for l in blobhash)
def version_is_greater_than(a, b):
"""Returns True if version a is more recent than version b"""
return pkg_resources.parse_version(a) > pkg_resources.parse_version(b)

0
lbrynet/extras/daemon/winpaths.py → lbrynet/winpaths.py
View file

2
scripts/encrypt_blob.py
View file

@ -9,7 +9,7 @@ from twisted.web.client import FileBodyProducer
from lbrynet import conf
from lbrynet.core import log_support
from lbrynet.dht.hashannouncer import DummyHashAnnouncer
from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer
from lbrynet.core.BlobManager import DiskBlobManager
from lbrynet.cryptstream.CryptStreamCreator import CryptStreamCreator

2
tests/functional/dht/test_store.py
View file

@ -3,7 +3,7 @@ from binascii import hexlify
from twisted.internet import defer
from lbrynet.dht import constants
from lbrynet.p2p.utils import generate_id
from lbrynet.utils import generate_id
from .dht_test_environment import TestKademliaBase
import logging

6
tests/functional/test_misc.py
View file

@ -2,13 +2,13 @@ import os
from hashlib import md5
from twisted.internet import defer, reactor
from twisted.trial import unittest
from lbrynet.extras.daemon import conf
from lbrynet import conf
from lbrynet.p2p.server.BlobAvailabilityHandler import BlobAvailabilityHandlerFactory
from lbrynet.p2p.StreamDescriptor import StreamDescriptorIdentifier
from lbrynet.p2p.StreamDescriptor import download_sd_blob
from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager
from lbrynet.p2p.BlobManager import DiskBlobManager
from lbrynet.p2p.PeerManager import PeerManager
from lbrynet.extras.daemon.PeerManager import PeerManager
from lbrynet.p2p.RateLimiter import RateLimiter
from lbrynet.p2p.server.BlobRequestHandler import BlobRequestHandlerFactory
from lbrynet.p2p.server.ServerProtocol import ServerProtocolFactory
@ -18,7 +18,7 @@ from lbrynet.blob.EncryptedFileManager import EncryptedFileManager
from lbrynet.blob.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
from tests import mocks
from tests.util import mk_db_and_blob_dir, rm_db_and_blob_dir
from tests.test_utils import mk_db_and_blob_dir, rm_db_and_blob_dir
FakeNode = mocks.Node
FakeWallet = mocks.Wallet

23
tests/functional/test_reflector.py
View file

@ -4,8 +4,11 @@ from binascii import hexlify
from twisted.internet import defer, error
from twisted.trial import unittest
from lbrynet.p2p.StreamDescriptor import get_sd_info
from lbrynet.extras import reflector
from lbrynet.p2p import BlobManager, PeerManager
from lbrynet.extras.reflector.server.server import ReflectorServerFactory
from lbrynet.extras.reflector.client.client import EncryptedFileReflectorClientFactory
from lbrynet.extras.reflector.client.blob import BlobReflectorClientFactory
from lbrynet.extras.daemon.PeerManager import PeerManager
from lbrynet.p2p import BlobManager
from lbrynet.p2p import StreamDescriptor
from lbrynet.blob import EncryptedFileCreator
from lbrynet.blob.EncryptedFileManager import EncryptedFileManager
@ -13,7 +16,7 @@ from lbrynet.p2p.RateLimiter import DummyRateLimiter
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager
from tests import mocks
from tests.util import mk_db_and_blob_dir, rm_db_and_blob_dir
from tests.test_utils import mk_db_and_blob_dir, rm_db_and_blob_dir
class TestReflector(unittest.TestCase):
@ -25,7 +28,7 @@ class TestReflector(unittest.TestCase):
self.client_db_dir, self.client_blob_dir = mk_db_and_blob_dir()
prm = OnlyFreePaymentsManager()
wallet = mocks.Wallet()
peer_manager = PeerManager.PeerManager()
peer_manager = PeerManager()
peer_finder = mocks.PeerFinder(5553, peer_manager, 2)
self.server_storage = SQLiteStorage(self.server_db_dir)
self.server_blob_manager = BlobManager.DiskBlobManager(self.server_blob_dir, self.server_storage)
@ -95,7 +98,7 @@ class TestReflector(unittest.TestCase):
return d
def start_server():
server_factory = reflector.ServerFactory(peer_manager, self.server_blob_manager,
server_factory = ReflectorServerFactory(peer_manager, self.server_blob_manager,
self.server_lbry_file_manager)
from twisted.internet import reactor
port = 8943
@ -179,7 +182,7 @@ class TestReflector(unittest.TestCase):
return d
def send_to_server():
factory = reflector.ClientFactory(self.client_blob_manager, self.stream_hash, self.sd_hash)
factory = EncryptedFileReflectorClientFactory(self.client_blob_manager, self.stream_hash, self.sd_hash)
from twisted.internet import reactor
reactor.connectTCP('localhost', self.port, factory)
@ -208,7 +211,7 @@ class TestReflector(unittest.TestCase):
return d
def send_to_server(blob_hashes_to_send):
factory = reflector.BlobClientFactory(
factory = BlobReflectorClientFactory(
self.client_blob_manager,
blob_hashes_to_send
)
@ -247,7 +250,7 @@ class TestReflector(unittest.TestCase):
return d
def send_to_server(blob_hashes_to_send):
factory = reflector.BlobClientFactory(
factory = BlobReflectorClientFactory(
self.client_blob_manager,
blob_hashes_to_send
)
@ -301,7 +304,7 @@ class TestReflector(unittest.TestCase):
return d
def send_to_server_as_blobs(blob_hashes_to_send):
factory = reflector.BlobClientFactory(
factory = BlobReflectorClientFactory(
self.client_blob_manager,
blob_hashes_to_send
)
@ -312,7 +315,7 @@ class TestReflector(unittest.TestCase):
return factory.finished_deferred
def send_to_server_as_stream(result):
factory = reflector.ClientFactory(self.client_blob_manager, self.stream_hash, self.sd_hash)
factory = EncryptedFileReflectorClientFactory(self.client_blob_manager, self.stream_hash, self.sd_hash)
from twisted.internet import reactor
reactor.connectTCP('localhost', self.port, factory)

2
tests/functional/test_streamify.py
View file

@ -7,8 +7,8 @@ from twisted.internet import defer, threads
from lbrynet.p2p.StreamDescriptor import StreamDescriptorIdentifier
from lbrynet.p2p.BlobManager import DiskBlobManager
from lbrynet.p2p.StreamDescriptor import get_sd_info
from lbrynet.p2p.PeerManager import PeerManager
from lbrynet.p2p.RateLimiter import DummyRateLimiter
from lbrynet.extras.daemon.PeerManager import PeerManager
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager
from lbrynet.blob.EncryptedFileCreator import create_lbry_file

6
tests/integration/cli/test_cli.py
View file

@ -3,10 +3,10 @@ from twisted.trial import unittest
from io import StringIO
from twisted.internet import defer
from lbrynet.extras.daemon import conf
from lbrynet import conf
from lbrynet.extras import cli
from lbrynet.extras.daemon.Components import DATABASE_COMPONENT, BLOB_COMPONENT, HEADERS_COMPONENT, WALLET_COMPONENT, \
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, STREAM_IDENTIFIER_COMPONENT, FILE_MANAGER_COMPONENT, \
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT, \
PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT, \
RATE_LIMITER_COMPONENT, PAYMENT_RATE_COMPONENT
from lbrynet.extras.daemon.Daemon import Daemon
@ -35,7 +35,7 @@ class CLIIntegrationTest(unittest.TestCase):
def setUp(self):
skip = [
DATABASE_COMPONENT, BLOB_COMPONENT, HEADERS_COMPONENT, WALLET_COMPONENT,
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, STREAM_IDENTIFIER_COMPONENT, FILE_MANAGER_COMPONENT,
DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, FILE_MANAGER_COMPONENT,
PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT,
RATE_LIMITER_COMPONENT, PAYMENT_RATE_COMPONENT
]

2
tests/integration/wallet/test_commands.py
View file

@ -15,7 +15,7 @@ from torba.testcase import IntegrationTestCase as BaseIntegrationTestCase
import lbrynet.schema
lbrynet.schema.BLOCKCHAIN_NAME = 'lbrycrd_regtest'
from lbrynet.extras.daemon import conf as lbry_conf
from lbrynet import conf as lbry_conf
from lbrynet.dht.node import Node
from lbrynet.extras.daemon.Daemon import Daemon
from lbrynet.extras.wallet import LbryWalletManager

4
tests/mocks.py
View file

@ -7,13 +7,13 @@ from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.hazmat.primitives import serialization
from twisted.internet import defer
from twisted.python.failure import Failure
from lbrynet import conf
from lbrynet.p2p.client.ClientRequest import ClientRequest
from lbrynet.p2p.Error import RequestCanceledError
from lbrynet.p2p import BlobAvailability
from lbrynet.blob.EncryptedFileManager import EncryptedFileManager
from lbrynet.dht.node import Node as RealNode
from lbrynet.extras.daemon import ExchangeRateManager as ERM, conf
from lbrynet.extras.daemon import ExchangeRateManager as ERM
KB = 2**10
PUBLIC_EXPONENT = 65537 # http://www.daemonology.net/blog/2009-06-11-cryptographic-right-answers.html

5
tests/util.py → tests/test_utils.py
View file

@ -32,13 +32,14 @@ def resetTime(test_case, timestamp=DEFAULT_TIMESTAMP):
patcher.start().return_value = iso_time
test_case.addCleanup(patcher.stop)
patcher = mock.patch('lbrynet.p2p.utils.now')
patcher = mock.patch('lbrynet.utils.now')
patcher.start().return_value = timestamp
test_case.addCleanup(patcher.stop)
patcher = mock.patch('lbrynet.p2p.utils.utcnow')
patcher = mock.patch('lbrynet.utils.utcnow')
patcher.start().return_value = timestamp
test_case.addCleanup(patcher.stop)
def is_android():
return 'ANDROID_ARGUMENT' in os.environ # detect Android using the Kivy way

19
tests/unit/components/test_Component_Manager.py
View file

@ -2,7 +2,7 @@ from twisted.internet.task import Clock
from twisted.trial import unittest
from lbrynet.extras.daemon.ComponentManager import ComponentManager
from lbrynet.extras.daemon.Components import DATABASE_COMPONENT, DHT_COMPONENT, STREAM_IDENTIFIER_COMPONENT
from lbrynet.extras.daemon.Components import DATABASE_COMPONENT, DHT_COMPONENT
from lbrynet.extras.daemon.Components import HASH_ANNOUNCER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT
from lbrynet.extras.daemon.Components import PEER_PROTOCOL_SERVER_COMPONENT, EXCHANGE_RATE_MANAGER_COMPONENT
from lbrynet.extras.daemon.Components import RATE_LIMITER_COMPONENT, HEADERS_COMPONENT, PAYMENT_RATE_COMPONENT
@ -13,6 +13,7 @@ from tests import mocks
class TestComponentManager(unittest.TestCase):
def setUp(self):
mocks.mock_conf_settings(self)
self.default_components_sort = [
[
Components.HeadersComponent,
@ -23,19 +24,14 @@ class TestComponentManager(unittest.TestCase):
Components.UPnPComponent
],
[
Components.BlobComponent,
Components.DHTComponent,
Components.WalletComponent
],
[
Components.BlobComponent,
Components.HashAnnouncerComponent
],
[
Components.PeerProtocolServerComponent,
Components.StreamIdentifierComponent
],
[
Components.FileManagerComponent
Components.FileManagerComponent,
Components.HashAnnouncerComponent,
Components.PeerProtocolServerComponent
],
[
Components.ReflectorComponent
@ -48,7 +44,6 @@ class TestComponentManager(unittest.TestCase):
def test_sort_components(self):
stages = self.component_manager.sort_components()
for stage_list, sorted_stage_list in zip(stages, self.default_components_sort):
self.assertEqual([type(stage) for stage in stage_list], sorted_stage_list)
@ -101,7 +96,7 @@ class TestComponentManagerProperStart(unittest.TestCase):
self.reactor = Clock()
mocks.mock_conf_settings(self)
self.component_manager = ComponentManager(
skip_components=[DATABASE_COMPONENT, DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT, STREAM_IDENTIFIER_COMPONENT,
skip_components=[DATABASE_COMPONENT, DHT_COMPONENT, HASH_ANNOUNCER_COMPONENT,
PEER_PROTOCOL_SERVER_COMPONENT, REFLECTOR_COMPONENT, UPNP_COMPONENT,
HEADERS_COMPONENT, PAYMENT_RATE_COMPONENT, RATE_LIMITER_COMPONENT,
EXCHANGE_RATE_MANAGER_COMPONENT],

15
tests/unit/core/client/test_ConnectionManager.py
View file

@ -1,17 +1,16 @@
from twisted.trial.unittest import TestCase
from twisted.internet import defer, reactor, task
from twisted.internet.task import deferLater
from twisted.internet.protocol import ServerFactory
from lbrynet import conf, utils
from lbrynet.p2p.client.ClientRequest import ClientRequest
from lbrynet.p2p.server.ServerProtocol import ServerProtocol
from lbrynet.p2p.client.ClientProtocol import ClientProtocol
from lbrynet.p2p.RateLimiter import RateLimiter
from lbrynet.p2p.Peer import Peer
from lbrynet.p2p.PeerManager import PeerManager
from lbrynet.p2p.Error import NoResponseError
from twisted.trial.unittest import TestCase
from twisted.internet import defer, reactor, task
from twisted.internet.task import deferLater
from twisted.internet.protocol import ServerFactory
from lbrynet.extras.daemon import conf
from lbrynet.p2p import utils
from lbrynet.extras.daemon.PeerManager import PeerManager
PEER_PORT = 5551
LOCAL_HOST = '127.0.0.1'

6
tests/unit/core/test_BlobManager.py
View file

@ -6,12 +6,12 @@ import string
from twisted.trial import unittest
from twisted.internet import defer
from tests.util import random_lbry_hash
from tests.test_utils import random_lbry_hash
from lbrynet.p2p.BlobManager import DiskBlobManager
from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.p2p.Peer import Peer
from lbrynet.extras.daemon import conf
from lbrynet.p2p.cryptoutils import get_lbry_hash_obj
from lbrynet import conf
from lbrynet.cryptoutils import get_lbry_hash_obj
class BlobManagerTest(unittest.TestCase):

4
tests/unit/core/test_HTTPBlobDownloader.py
View file

@ -3,9 +3,9 @@ from mock import MagicMock
from twisted.trial import unittest
from twisted.internet import defer
from lbrynet.blob import BlobFile
from lbrynet.blob.blob_file import BlobFile
from lbrynet.p2p.HTTPBlobDownloader import HTTPBlobDownloader
from tests.util import mk_db_and_blob_dir, rm_db_and_blob_dir
from tests.test_utils import mk_db_and_blob_dir, rm_db_and_blob_dir
class HTTPBlobDownloaderTest(unittest.TestCase):

4
tests/unit/core/test_HashBlob.py
View file

@ -1,7 +1,7 @@
from lbrynet.blob import BlobFile
from lbrynet.blob.blob_file import BlobFile
from lbrynet.p2p.Error import DownloadCanceledError, InvalidDataError
from tests.util import mk_db_and_blob_dir, rm_db_and_blob_dir, random_lbry_hash
from tests.test_utils import mk_db_and_blob_dir, rm_db_and_blob_dir, random_lbry_hash
from twisted.trial import unittest
from twisted.internet import defer

3
tests/unit/core/test_Wallet.py
View file

@ -5,9 +5,6 @@ from decimal import Decimal
from twisted.trial import unittest
from twisted.internet import defer
from lbrynet.p2p.Error import InsufficientFundsError
#from lbrynet.p2p.Wallet import LBRYumWallet, ReservedPoints
#from lbryum.commands import Commands
#from lbryum.simple_config import SimpleConfig
from lbrynet.schema.claim import ClaimDict
test_metadata = {

2
tests/unit/core/test_utils.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
from lbrynet.p2p import utils
from lbrynet import utils
from twisted.trial import unittest

4
tests/unit/database/test_SQLiteStorage.py
View file

@ -5,10 +5,10 @@ import logging
from copy import deepcopy
from twisted.internet import defer
from twisted.trial import unittest
from lbrynet.extras.daemon import conf
from lbrynet import conf
from lbrynet.extras.daemon.storage import SQLiteStorage, open_file_for_writing
from lbrynet.blob.EncryptedFileDownloader import ManagedEncryptedFileDownloader
from tests.util import random_lbry_hash
from tests.test_utils import random_lbry_hash
log = logging.getLogger()

2
tests/unit/dht/test_contact.py
View file

@ -1,7 +1,7 @@
from binascii import hexlify
from twisted.internet import task
from twisted.trial import unittest
from lbrynet.p2p.utils import generate_id
from lbrynet.utils import generate_id
from lbrynet.dht.contact import ContactManager
from lbrynet.dht import constants

6
tests/unit/dht/test_hash_announcer.py
View file

@ -1,8 +1,8 @@
from twisted.trial import unittest
from twisted.internet import defer, task
from lbrynet.p2p import utils
from lbrynet.dht.hashannouncer import DHTHashAnnouncer
from tests.util import random_lbry_hash
from lbrynet import utils
from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer
from tests.test_utils import random_lbry_hash
from tests.mocks import mock_conf_settings

2
tests/unit/dht/test_kbucket.py
View file

@ -6,7 +6,7 @@
from twisted.trial import unittest
import struct
from lbrynet.p2p.utils import generate_id
from lbrynet.utils import generate_id
from lbrynet.dht import kbucket
from lbrynet.dht.contact import ContactManager
from lbrynet.dht import constants

2
tests/unit/dht/test_node.py
View file

@ -5,7 +5,7 @@ from twisted.trial import unittest
from twisted.internet import defer
from lbrynet.dht.node import Node
from lbrynet.dht import constants
from lbrynet.p2p.utils import generate_id
from lbrynet.utils import generate_id
class NodeIDTest(unittest.TestCase):

2
tests/unit/dht/test_routingtable.py
View file

@ -6,7 +6,7 @@ from lbrynet.dht import constants
from lbrynet.dht.routingtable import TreeRoutingTable
from lbrynet.dht.contact import ContactManager
from lbrynet.dht.distance import Distance
from lbrynet.p2p.utils import generate_id
from lbrynet.utils import generate_id
class FakeRPCProtocol:

Some files were not shown because too many files have changed in this diff Show more