Madiator2011/lbry-sdk
1
0
Fork 0
forked from LBRYCommunity/lbry-sdk
Code Pull requests Activity

update daemon and components

Browse source
This commit is contained in:
Jack Robison 2019-01-22 17:44:17 -05:00 committed by Lex Berezhny
parent ea15674e62
commit 494917158c
9 changed files with 590 additions and 1350 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

91
lbrynet/extras/cli.py
View file

@ -2,22 +2,63 @@ import sys
import json import json
import asyncio import asyncio
import argparse import argparse
import logging import typing
import logging.handlers
import aiohttp
from docopt import docopt from docopt import docopt
from textwrap import dedent from textwrap import dedent
import aiohttp from lbrynet import __name__ as lbrynet_name, __version__ as lbrynet_version
from lbrynet.extras.compat import force_asyncioreactor_install
force_asyncioreactor_install()
from lbrynet import log_support, __name__ as lbrynet_name, __version__ as lbrynet_version
from lbrynet.extras.daemon.loggly_handler import get_loggly_handler
from lbrynet.conf import Config, CLIConfig from lbrynet.conf import Config, CLIConfig
from lbrynet.utils import check_connection
from lbrynet.extras.daemon.Daemon import Daemon from lbrynet.extras.daemon.Daemon import Daemon
from lbrynet.extras.daemon.loggly_handler import get_loggly_handler
log = logging.getLogger(lbrynet_name) log = logging.getLogger(lbrynet_name)
log.addHandler(logging.NullHandler()) log.addHandler(logging.NullHandler())
default_formatter = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s")
optional_path_getter_type = typing.Optional[typing.Callable[[], str]]
async def start_daemon(conf: Config, args):
file_handler = logging.handlers.RotatingFileHandler(conf.log_file_path,
maxBytes=2097152, backupCount=5)
file_handler.setFormatter(default_formatter)
log.addHandler(file_handler)
if not args.quiet:
handler = logging.StreamHandler()
handler.setFormatter(default_formatter)
log.addHandler(handler)
# mostly disable third part logging
logging.getLogger('urllib3').setLevel(logging.CRITICAL)
logging.getLogger('BitcoinRPC').setLevel(logging.INFO)
logging.getLogger('aioupnp').setLevel(logging.WARNING)
logging.getLogger('aiohttp').setLevel(logging.CRITICAL)
if args.verbose:
log.setLevel(logging.DEBUG)
else:
log.setLevel(logging.INFO)
if conf.share_usage_data:
loggly_handler = get_loggly_handler()
loggly_handler.setLevel(logging.ERROR)
log.addHandler(loggly_handler)
log.info("Starting lbrynet-daemon from command line")
daemon = Daemon(conf)
try:
await daemon.start_listening()
except (OSError, asyncio.CancelledError):
return 1
try:
await daemon.server.wait_closed()
except (KeyboardInterrupt, asyncio.CancelledError):
await daemon.shutdown()
return 0
def display(data): def display(data):
@ -179,55 +220,27 @@ def main(argv=None):
argv = argv or sys.argv[1:] argv = argv or sys.argv[1:]
parser = get_argument_parser() parser = get_argument_parser()
args, command_args = parser.parse_known_args(argv) args, command_args = parser.parse_known_args(argv)
conf = Config.create_from_arguments(args) conf = Config.create_from_arguments(args)
if args.cli_version: if args.cli_version:
print(f"{lbrynet_name} {lbrynet_version}") print(f"{lbrynet_name} {lbrynet_version}")
return 0 return 0
elif args.command == 'start': elif args.command == 'start':
return asyncio.run(start_daemon(conf, args))
if args.help:
args.start_parser.print_help()
return 0
log_support.configure_logging(conf.log_file_path, not args.quiet, args.verbose)
if conf.share_usage_data:
loggly_handler = get_loggly_handler()
loggly_handler.setLevel(logging.ERROR)
log.addHandler(loggly_handler)
log.debug('Final Settings: %s', conf.settings_dict)
log.info("Starting lbrynet-daemon from command line")
daemon = Daemon(conf)
if check_connection():
from twisted.internet import reactor
reactor._asyncioEventloop.create_task(daemon.start())
reactor.run()
else:
log.info("Not connected to internet, unable to start")
elif args.command is not None: elif args.command is not None:
doc = args.doc doc = args.doc
api_method_name = args.api_method_name api_method_name = args.api_method_name
if args.replaced_by: if args.replaced_by:
print(f"{args.api_method_name} is deprecated, using {args.replaced_by['api_method_name']}.") print(f"{args.api_method_name} is deprecated, using {args.replaced_by['api_method_name']}.")
doc = args.replaced_by['doc'] doc = args.replaced_by['doc']
api_method_name = args.replaced_by['api_method_name'] api_method_name = args.replaced_by['api_method_name']
if args.help: if args.help:
print(doc) print(doc)
return 0
else: else:
parsed = docopt(doc, command_args) parsed = docopt(doc, command_args)
params = set_kwargs(parsed) params = set_kwargs(parsed)
loop = asyncio.get_event_loop() asyncio.run(execute_command(conf, api_method_name, params))
loop.run_until_complete(execute_command(conf, api_method_name, params))
elif args.group is not None: elif args.group is not None:
args.group_parser.print_help() args.group_parser.print_help()

8
lbrynet/extras/daemon/Component.py
View file

@ -1,7 +1,5 @@
import asyncio
import logging import logging
from twisted.internet import defer
from twisted._threads import AlreadyQuit
from lbrynet.conf import Config from lbrynet.conf import Config
from lbrynet.extras.daemon.ComponentManager import ComponentManager from lbrynet.extras.daemon.ComponentManager import ComponentManager
@ -57,7 +55,7 @@ class Component(metaclass=ComponentType):
result = await self.start() result = await self.start()
self._running = True self._running = True
return result return result
except (defer.CancelledError, AlreadyQuit): except asyncio.CancelledError:
pass pass
except Exception as err: except Exception as err:
log.exception("Error setting up %s", self.component_name or self.__class__.__name__) log.exception("Error setting up %s", self.component_name or self.__class__.__name__)
@ -68,7 +66,7 @@ class Component(metaclass=ComponentType):
result = await self.stop() result = await self.stop()
self._running = False self._running = False
return result return result
except (defer.CancelledError, AlreadyQuit): except asyncio.CancelledError:
pass pass
except Exception as err: except Exception as err:
log.exception("Error stopping %s", self.__class__.__name__) log.exception("Error stopping %s", self.__class__.__name__)

26
lbrynet/extras/daemon/ComponentManager.py
View file

@ -1,9 +1,8 @@
import asyncio
import logging import logging
from lbrynet.p2p.Error import ComponentStartConditionNotMet import asyncio
from lbrynet.extras.daemon.PeerManager import PeerManager
from lbrynet.extras.daemon.PeerFinder import DHTPeerFinder
from lbrynet.conf import Config from lbrynet.conf import Config
from lbrynet.error import ComponentStartConditionNotMet
from lbrynet.dht.peer import PeerManager
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -35,16 +34,16 @@ class RequiredCondition(metaclass=RequiredConditionType):
class ComponentManager: class ComponentManager:
default_component_classes = {} default_component_classes = {}
def __init__(self, conf: Config, reactor=None, analytics_manager=None, skip_components=None, def __init__(self, conf: Config, analytics_manager=None, skip_components=None,
peer_manager=None, peer_finder=None, **override_components): peer_manager=None, **override_components):
self.conf = conf self.conf = conf
self.skip_components = skip_components or [] self.skip_components = skip_components or []
self.reactor = reactor self.loop = asyncio.get_event_loop()
self.analytics_manager = analytics_manager
self.component_classes = {} self.component_classes = {}
self.components = set() self.components = set()
self.analytics_manager = analytics_manager self.started = asyncio.Event(loop=self.loop)
self.peer_manager = peer_manager or PeerManager() self.peer_manager = peer_manager or PeerManager(asyncio.get_event_loop_policy().get_event_loop())
self.peer_finder = peer_finder or DHTPeerFinder(self)
for component_name, component_class in self.default_component_classes.items(): for component_name, component_class in self.default_component_classes.items():
if component_name in override_components: if component_name in override_components:
@ -127,7 +126,7 @@ class ComponentManager:
if component.component_name in callbacks: if component.component_name in callbacks:
maybe_coro = callbacks[component.component_name](component) maybe_coro = callbacks[component.component_name](component)
if asyncio.iscoroutine(maybe_coro): if asyncio.iscoroutine(maybe_coro):
asyncio.create_task(maybe_coro) await asyncio.create_task(maybe_coro)
stages = self.sort_components() stages = self.sort_components()
for stage in stages: for stage in stages:
@ -136,12 +135,11 @@ class ComponentManager:
] ]
if needing_start: if needing_start:
await asyncio.wait(needing_start) await asyncio.wait(needing_start)
self.started.set()
async def stop(self): async def stop(self):
""" """
Stop Components in reversed startup order Stop Components in reversed startup order
:return: (defer.Deferred)
""" """
stages = self.sort_components(reverse=True) stages = self.sort_components(reverse=True)
for stage in stages: for stage in stages:
@ -149,7 +147,7 @@ class ComponentManager:
component._stop() for component in stage if component.running component._stop() for component in stage if component.running
] ]
if needing_stop: if needing_stop:
await asyncio.wait(needing_stop) await asyncio.wait(needing_stop, loop=self.loop)
def all_components_running(self, *component_names): def all_components_running(self, *component_names):
""" """

339
lbrynet/extras/daemon/Components.py
View file

@ -2,39 +2,30 @@ import os
import asyncio import asyncio
import aiohttp import aiohttp
import logging import logging
import treq
import math import math
import binascii import binascii
import typing
import socket
from hashlib import sha256 from hashlib import sha256
from types import SimpleNamespace from types import SimpleNamespace
from twisted.internet import defer, reactor, error
from aioupnp import __version__ as aioupnp_version from aioupnp import __version__ as aioupnp_version
from aioupnp.upnp import UPnP from aioupnp.upnp import UPnP
from aioupnp.fault import UPnPError from aioupnp.fault import UPnPError
import lbrynet.schema import lbrynet.schema
from lbrynet.conf import HEADERS_FILE_SHA256_CHECKSUM from lbrynet.conf import HEADERS_FILE_SHA256_CHECKSUM
from lbrynet.extras.compat import d2f
from lbrynet.blob.EncryptedFileManager import EncryptedFileManager
from lbrynet.blob.client.EncryptedFileDownloader import EncryptedFileSaverFactory
from lbrynet.blob.client.EncryptedFileOptions import add_lbry_file_to_sd_identifier
from lbrynet.dht.node import Node from lbrynet.dht.node import Node
from lbrynet.dht.peer import KademliaPeer
from lbrynet.dht.blob_announcer import BlobAnnouncer
from lbrynet.blob.blob_manager import BlobFileManager
from lbrynet.blob_exchange.server import BlobServer
from lbrynet.stream.stream_manager import StreamManager
from lbrynet.extras.daemon.Component import Component from lbrynet.extras.daemon.Component import Component
from lbrynet.extras.daemon.exchange_rate_manager import ExchangeRateManager from lbrynet.extras.daemon.exchange_rate_manager import ExchangeRateManager
from lbrynet.extras.daemon.storage import SQLiteStorage from lbrynet.extras.daemon.storage import SQLiteStorage
from lbrynet.extras.daemon.HashAnnouncer import DHTHashAnnouncer
from lbrynet.extras.reflector.server.server import ReflectorServerFactory
from lbrynet.extras.wallet import LbryWalletManager from lbrynet.extras.wallet import LbryWalletManager
from lbrynet.extras.wallet import Network from lbrynet.extras.wallet import Network
from lbrynet.utils import generate_id
from lbrynet.p2p.PaymentRateManager import OnlyFreePaymentsManager
from lbrynet.p2p.RateLimiter import RateLimiter
from lbrynet.p2p.BlobManager import DiskBlobManager
from lbrynet.p2p.StreamDescriptor import StreamDescriptorIdentifier, EncryptedFileStreamType
from lbrynet.p2p.server.BlobRequestHandler import BlobRequestHandlerFactory
from lbrynet.p2p.server.ServerProtocol import ServerProtocolFactory
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -47,13 +38,10 @@ HEADERS_COMPONENT = "blockchain_headers"
WALLET_COMPONENT = "wallet" WALLET_COMPONENT = "wallet"
DHT_COMPONENT = "dht" DHT_COMPONENT = "dht"
HASH_ANNOUNCER_COMPONENT = "hash_announcer" HASH_ANNOUNCER_COMPONENT = "hash_announcer"
FILE_MANAGER_COMPONENT = "file_manager" STREAM_MANAGER_COMPONENT = "stream_manager"
PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server" PEER_PROTOCOL_SERVER_COMPONENT = "peer_protocol_server"
REFLECTOR_COMPONENT = "reflector"
UPNP_COMPONENT = "upnp" UPNP_COMPONENT = "upnp"
EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager" EXCHANGE_RATE_MANAGER_COMPONENT = "exchange_rate_manager"
RATE_LIMITER_COMPONENT = "rate_limiter"
PAYMENT_RATE_COMPONENT = "payment_rate_manager"
async def gather_dict(tasks: dict): async def gather_dict(tasks: dict):
@ -75,6 +63,14 @@ async def get_external_ip(): # used if upnp is disabled or non-functioning
pass pass
async def resolve_host(loop: asyncio.BaseEventLoop, url: str):
info = await loop.getaddrinfo(
url, 'https',
proto=socket.IPPROTO_TCP,
)
return info[0][4][0]
class DatabaseComponent(Component): class DatabaseComponent(Component):
component_name = DATABASE_COMPONENT component_name = DATABASE_COMPONENT
@ -158,39 +154,41 @@ class HeadersComponent(Component):
'download_progress': self._headers_progress_percent 'download_progress': self._headers_progress_percent
} }
@defer.inlineCallbacks async def fetch_headers_from_s3(self):
def fetch_headers_from_s3(self): def collector(d, h_file):
def collector(data, h_file): h_file.write(d)
h_file.write(data)
local_size = float(h_file.tell()) local_size = float(h_file.tell())
final_size = float(final_size_after_download) final_size = float(final_size_after_download)
self._headers_progress_percent = math.ceil(local_size / final_size * 100) self._headers_progress_percent = math.ceil(local_size / final_size * 100)
local_header_size = self.local_header_file_size() local_header_size = self.local_header_file_size()
resume_header = {"Range": f"bytes={local_header_size}-"} resume_header = {"Range": f"bytes={local_header_size}-"}
response = yield treq.get(HEADERS_URL, headers=resume_header) async with aiohttp.request('get', HEADERS_URL, headers=resume_header) as response:
got_406 = response.code == 406 # our file is bigger got_406 = response.status == 406 # our file is bigger
final_size_after_download = response.length + local_header_size final_size_after_download = response.content_length + local_header_size
if got_406: if got_406:
log.warning("s3 is more out of date than we are")
# should have something to download and a final length divisible by the header size
elif final_size_after_download and not final_size_after_download % HEADER_SIZE:
s3_height = (final_size_after_download / HEADER_SIZE) - 1
local_height = self.local_header_file_height()
if s3_height > local_height:
if local_header_size:
log.info("Resuming download of %i bytes from s3", response.length)
with open(self.headers_file, "a+b") as headers_file:
yield treq.collect(response, lambda d: collector(d, headers_file))
else:
with open(self.headers_file, "wb") as headers_file:
yield treq.collect(response, lambda d: collector(d, headers_file))
log.info("fetched headers from s3 (s3 height: %i), now verifying integrity after download.", s3_height)
self._check_header_file_integrity()
else:
log.warning("s3 is more out of date than we are") log.warning("s3 is more out of date than we are")
else: # should have something to download and a final length divisible by the header size
log.error("invalid size for headers from s3") elif final_size_after_download and not final_size_after_download % HEADER_SIZE:
s3_height = (final_size_after_download / HEADER_SIZE) - 1
local_height = self.local_header_file_height()
if s3_height > local_height:
data = await response.read()
if local_header_size:
log.info("Resuming download of %i bytes from s3", response.content_length)
with open(self.headers_file, "a+b") as headers_file:
collector(data, headers_file)
else:
with open(self.headers_file, "wb") as headers_file:
collector(data, headers_file)
log.info("fetched headers from s3 (s3 height: %i), now verifying integrity after download.",
s3_height)
self._check_header_file_integrity()
else:
log.warning("s3 is more out of date than we are")
else:
log.error("invalid size for headers from s3")
def local_header_file_height(self): def local_header_file_height(self):
return max((self.local_header_file_size() / HEADER_SIZE) - 1, 0) return max((self.local_header_file_size() / HEADER_SIZE) - 1, 0)
@ -259,7 +257,7 @@ class HeadersComponent(Component):
self._downloading_headers = await self.should_download_headers_from_s3() self._downloading_headers = await self.should_download_headers_from_s3()
if self._downloading_headers: if self._downloading_headers:
try: try:
await d2f(self.fetch_headers_from_s3()) await self.fetch_headers_from_s3()
except Exception as err: except Exception as err:
log.error("failed to fetch headers from s3: %s", err) log.error("failed to fetch headers from s3: %s", err)
finally: finally:
@ -313,29 +311,32 @@ class BlobComponent(Component):
def __init__(self, component_manager): def __init__(self, component_manager):
super().__init__(component_manager) super().__init__(component_manager)
self.blob_manager = None self.blob_manager: BlobFileManager = None
@property @property
def component(self): def component(self) -> typing.Optional[BlobFileManager]:
return self.blob_manager return self.blob_manager
def start(self): async def start(self):
storage = self.component_manager.get_component(DATABASE_COMPONENT) storage = self.component_manager.get_component(DATABASE_COMPONENT)
datastore = None data_store = None
if DHT_COMPONENT not in self.component_manager.skip_components: if DHT_COMPONENT not in self.component_manager.skip_components:
dht_node = self.component_manager.get_component(DHT_COMPONENT) dht_node: Node = self.component_manager.get_component(DHT_COMPONENT)
if dht_node: if dht_node:
datastore = dht_node._dataStore data_store = dht_node.protocol.data_store
self.blob_manager = DiskBlobManager(os.path.join(self.conf.data_dir, "blobfiles"), storage, datastore) self.blob_manager = BlobFileManager(asyncio.get_event_loop(), os.path.join(self.conf.data_dir, "blobfiles"),
return self.blob_manager.setup() storage, data_store)
return await self.blob_manager.setup()
def stop(self): async def stop(self):
return self.blob_manager.stop() while self.blob_manager and self.blob_manager.blobs:
_, blob = self.blob_manager.blobs.popitem()
await blob.close()
async def get_status(self): async def get_status(self):
count = 0 count = 0
if self.blob_manager: if self.blob_manager:
count = await self.blob_manager.storage.count_finished_blobs() count = len(self.blob_manager.completed_blob_hashes)
return {'finished_blobs': count} return {'finished_blobs': count}
@ -345,28 +346,27 @@ class DHTComponent(Component):
def __init__(self, component_manager): def __init__(self, component_manager):
super().__init__(component_manager) super().__init__(component_manager)
self.dht_node = None self.dht_node: Node = None
self.upnp_component = None self.upnp_component = None
self.external_udp_port = None self.external_udp_port = None
self.external_peer_port = None self.external_peer_port = None
@property @property
def component(self): def component(self) -> typing.Optional[Node]:
return self.dht_node return self.dht_node
async def get_status(self): async def get_status(self):
return { return {
'node_id': binascii.hexlify(self.component_manager.daemon.node_id), 'node_id': binascii.hexlify(self.component_manager.daemon.node_id),
'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.contacts) 'peers_in_routing_table': 0 if not self.dht_node else len(self.dht_node.protocol.routing_table.get_peers())
} }
async def start(self): async def start(self):
log.info("start the dht")
self.upnp_component = self.component_manager.get_component(UPNP_COMPONENT) self.upnp_component = self.component_manager.get_component(UPNP_COMPONENT)
self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.peer_port) self.external_peer_port = self.upnp_component.upnp_redirects.get("TCP", self.conf.peer_port)
self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.dht_node_port) self.external_udp_port = self.upnp_component.upnp_redirects.get("UDP", self.conf.dht_node_port)
node_id = self.component_manager.daemon.node_id node_id = self.component_manager.daemon.node_id
if node_id is None:
node_id = generate_id()
external_ip = self.upnp_component.external_ip external_ip = self.upnp_component.external_ip
if not external_ip: if not external_ip:
log.warning("UPnP component failed to get external ip") log.warning("UPnP component failed to get external ip")
@ -375,18 +375,21 @@ class DHTComponent(Component):
log.warning("failed to get external ip") log.warning("failed to get external ip")
self.dht_node = Node( self.dht_node = Node(
asyncio.get_event_loop(),
self.component_manager.peer_manager,
node_id=node_id, node_id=node_id,
udpPort=self.conf.dht_node_port, internal_udp_port=self.conf.dht_node_port,
externalUDPPort=self.external_udp_port, udp_port=self.external_udp_port,
externalIP=external_ip, external_ip=external_ip,
peerPort=self.external_peer_port peer_port=self.external_peer_port
)
self.dht_node.start(
interface='0.0.0.0', known_node_urls=self.conf.known_dht_nodes
) )
await d2f(self.dht_node.start(self.conf.known_dht_nodes, block_on_join=False))
log.info("Started the dht") log.info("Started the dht")
def stop(self): async def stop(self):
return d2f(self.dht_node.stop()) self.dht_node.stop()
class HashAnnouncerComponent(Component): class HashAnnouncerComponent(Component):
@ -395,195 +398,96 @@ class HashAnnouncerComponent(Component):
def __init__(self, component_manager): def __init__(self, component_manager):
super().__init__(component_manager) super().__init__(component_manager)
self.hash_announcer = None self.hash_announcer: BlobAnnouncer = None
@property @property
def component(self): def component(self) -> typing.Optional[BlobAnnouncer]:
return self.hash_announcer return self.hash_announcer
async def start(self): async def start(self):
storage = self.component_manager.get_component(DATABASE_COMPONENT) storage = self.component_manager.get_component(DATABASE_COMPONENT)
dht_node = self.component_manager.get_component(DHT_COMPONENT) dht_node = self.component_manager.get_component(DHT_COMPONENT)
self.hash_announcer = DHTHashAnnouncer(self.conf, dht_node, storage) self.hash_announcer = BlobAnnouncer(asyncio.get_event_loop(), dht_node, storage)
self.hash_announcer.start() self.hash_announcer.start(self.conf.concurrent_announcers)
log.info("Started blob announcer")
def stop(self): async def stop(self):
self.hash_announcer.stop() self.hash_announcer.stop()
log.info("Stopped blob announcer")
async def get_status(self): async def get_status(self):
return { return {
'announce_queue_size': 0 if not self.hash_announcer else len(self.hash_announcer.hash_queue) 'announce_queue_size': 0 if not self.hash_announcer else len(self.hash_announcer.announce_queue)
} }
class RateLimiterComponent(Component): class StreamManagerComponent(Component):
component_name = RATE_LIMITER_COMPONENT component_name = STREAM_MANAGER_COMPONENT
depends_on = [BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT, DHT_COMPONENT]
def __init__(self, component_manager): def __init__(self, component_manager):
super().__init__(component_manager) super().__init__(component_manager)
self.rate_limiter = RateLimiter() self.stream_manager: StreamManager = None
@property @property
def component(self): def component(self) -> typing.Optional[StreamManager]:
return self.rate_limiter return self.stream_manager
async def start(self):
self.rate_limiter.start()
async def stop(self):
self.rate_limiter.stop()
class PaymentRateComponent(Component):
component_name = PAYMENT_RATE_COMPONENT
def __init__(self, component_manager):
super().__init__(component_manager)
self.payment_rate_manager = OnlyFreePaymentsManager()
@property
def component(self):
return self.payment_rate_manager
async def start(self):
pass
async def stop(self):
pass
class FileManagerComponent(Component):
component_name = FILE_MANAGER_COMPONENT
depends_on = [RATE_LIMITER_COMPONENT, BLOB_COMPONENT, DATABASE_COMPONENT, WALLET_COMPONENT,
PAYMENT_RATE_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
self.file_manager = None
@property
def component(self):
return self.file_manager
async def get_status(self): async def get_status(self):
if not self.file_manager: if not self.stream_manager:
return return
return { return {
'managed_files': len(self.file_manager.lbry_files) 'managed_files': len(self.stream_manager.streams)
} }
def start(self): async def start(self):
rate_limiter = self.component_manager.get_component(RATE_LIMITER_COMPONENT)
blob_manager = self.component_manager.get_component(BLOB_COMPONENT) blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
storage = self.component_manager.get_component(DATABASE_COMPONENT) storage = self.component_manager.get_component(DATABASE_COMPONENT)
wallet = self.component_manager.get_component(WALLET_COMPONENT) wallet = self.component_manager.get_component(WALLET_COMPONENT)
node = self.component_manager.get_component(DHT_COMPONENT)
sd_identifier = StreamDescriptorIdentifier()
add_lbry_file_to_sd_identifier(sd_identifier)
file_saver_factory = EncryptedFileSaverFactory(
self.conf,
self.component_manager.peer_finder,
rate_limiter,
blob_manager,
storage,
wallet,
self.conf.download_dir
)
sd_identifier.add_stream_downloader_factory(EncryptedFileStreamType, file_saver_factory)
payment_rate_manager = self.component_manager.get_component(PAYMENT_RATE_COMPONENT)
log.info('Starting the file manager') log.info('Starting the file manager')
self.file_manager = EncryptedFileManager( loop = asyncio.get_event_loop()
self.conf, self.component_manager.peer_finder, rate_limiter, self.stream_manager = StreamManager(
blob_manager, wallet, payment_rate_manager, storage, sd_identifier loop, blob_manager, wallet, storage, node, self.conf.blob_download_timeout,
self.conf.peer_connect_timeout, [
KademliaPeer(loop, address=(await resolve_host(loop, url)), tcp_port=port + 1)
for url, port in self.conf.reflector_servers
]
) )
return self.file_manager.setup() await self.stream_manager.start()
log.info('Done setting up file manager')
def stop(self): async def stop(self):
return d2f(self.file_manager.stop()) await self.stream_manager.stop()
class PeerProtocolServerComponent(Component): class PeerProtocolServerComponent(Component):
component_name = PEER_PROTOCOL_SERVER_COMPONENT component_name = PEER_PROTOCOL_SERVER_COMPONENT
depends_on = [UPNP_COMPONENT, RATE_LIMITER_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT, depends_on = [UPNP_COMPONENT, BLOB_COMPONENT, WALLET_COMPONENT]
PAYMENT_RATE_COMPONENT]
def __init__(self, component_manager): def __init__(self, component_manager):
super().__init__(component_manager) super().__init__(component_manager)
self.lbry_server_port = None self.blob_server: BlobServer = None
@property @property
def component(self): def component(self) -> typing.Optional[BlobServer]:
return self.lbry_server_port return self.blob_server
async def start(self): async def start(self):
wallet = self.component_manager.get_component(WALLET_COMPONENT) log.info("start blob server")
upnp = self.component_manager.get_component(UPNP_COMPONENT) upnp = self.component_manager.get_component(UPNP_COMPONENT)
peer_port = self.conf.peer_port blob_manager: BlobFileManager = self.component_manager.get_component(BLOB_COMPONENT)
query_handlers = { wallet: LbryWalletManager = self.component_manager.get_component(WALLET_COMPONENT)
handler.get_primary_query_identifier(): handler for handler in [ peer_port = upnp.upnp_redirects.get("TCP", self.conf.settings["peer_port"])
BlobRequestHandlerFactory( address = await wallet.get_unused_address()
self.component_manager.get_component(BLOB_COMPONENT), self.blob_server = BlobServer(asyncio.get_event_loop(), blob_manager, address)
wallet, self.blob_server.start_server(peer_port, interface='0.0.0.0')
self.component_manager.get_component(PAYMENT_RATE_COMPONENT), await self.blob_server.started_listening.wait()
self.component_manager.analytics_manager
),
wallet.get_wallet_info_query_handler_factory(),
]
}
server_factory = ServerProtocolFactory(
self.component_manager.get_component(RATE_LIMITER_COMPONENT), query_handlers,
self.component_manager.peer_manager
)
try:
log.info("Peer protocol listening on TCP %i (ext port %i)", peer_port,
upnp.upnp_redirects.get("TCP", peer_port))
self.lbry_server_port = reactor.listenTCP(peer_port, server_factory)
except error.CannotListenError as e:
import traceback
log.error("Couldn't bind to port %d. Visit lbry.io/faq/how-to-change-port for"
" more details.", peer_port)
log.error("%s", traceback.format_exc())
raise ValueError("%s lbrynet may already be running on your computer." % str(e))
async def stop(self): async def stop(self):
if self.lbry_server_port is not None: if self.blob_server:
self.lbry_server_port, old_port = None, self.lbry_server_port self.blob_server.stop_server()
log.info('Stop listening on port %s', old_port.port)
await d2f(old_port.stopListening())
class ReflectorComponent(Component):
component_name = REFLECTOR_COMPONENT
depends_on = [BLOB_COMPONENT, FILE_MANAGER_COMPONENT]
def __init__(self, component_manager):
super().__init__(component_manager)
self.reflector_server_port = self.conf.reflector_port
self.reflector_server = None
@property
def component(self):
return self.reflector_server
async def start(self):
log.info("Starting reflector server")
blob_manager = self.component_manager.get_component(BLOB_COMPONENT)
file_manager = self.component_manager.get_component(FILE_MANAGER_COMPONENT)
reflector_factory = ReflectorServerFactory(self.component_manager.peer_manager, blob_manager, file_manager)
try:
self.reflector_server = await d2f(reactor.listenTCP(self.reflector_server_port, reflector_factory))
log.info('Started reflector on port %s', self.reflector_server_port)
except error.CannotListenError as e:
log.exception("Couldn't bind reflector to port %d", self.reflector_server_port)
raise ValueError(f"{e} lbrynet may already be running on your computer.")
async def stop(self):
if self.reflector_server is not None:
log.info("Stopping reflector server")
self.reflector_server, p = None, self.reflector_server
await d2f(p.stopListening())
class UPnPComponent(Component): class UPnPComponent(Component):
@ -600,7 +504,7 @@ class UPnPComponent(Component):
self._maintain_redirects_task = None self._maintain_redirects_task = None
@property @property
def component(self): def component(self) -> 'UPnPComponent':
return self return self
async def _repeatedly_maintain_redirects(self, now=True): async def _repeatedly_maintain_redirects(self, now=True):
@ -696,7 +600,8 @@ class UPnPComponent(Component):
log.debug("set up upnp port redirects for gateway: %s", self.upnp.gateway.manufacturer_string) log.debug("set up upnp port redirects for gateway: %s", self.upnp.gateway.manufacturer_string)
else: else:
log.error("failed to setup upnp") log.error("failed to setup upnp")
await self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, await self.get_status()) if self.component_manager.analytics_manager:
self.component_manager.analytics_manager.send_upnp_setup_success_fail(success, await self.get_status())
self._maintain_redirects_task = asyncio.create_task(self._repeatedly_maintain_redirects(now=False)) self._maintain_redirects_task = asyncio.create_task(self._repeatedly_maintain_redirects(now=False))
async def stop(self): async def stop(self):
@ -704,7 +609,7 @@ class UPnPComponent(Component):
await asyncio.wait([ await asyncio.wait([
self.upnp.delete_port_mapping(port, protocol) for protocol, port in self.upnp_redirects.items() self.upnp.delete_port_mapping(port, protocol) for protocol, port in self.upnp_redirects.items()
]) ])
if self._maintain_redirects_task is not None and not self._maintain_redirects_task.done(): if self._maintain_redirects_task and not self._maintain_redirects_task.done():
self._maintain_redirects_task.cancel() self._maintain_redirects_task.cancel()
async def get_status(self): async def get_status(self):
@ -726,7 +631,7 @@ class ExchangeRateManagerComponent(Component):
self.exchange_rate_manager = ExchangeRateManager() self.exchange_rate_manager = ExchangeRateManager()
@property @property
def component(self): def component(self) -> ExchangeRateManager:
return self.exchange_rate_manager return self.exchange_rate_manager
async def start(self): async def start(self):

1259
lbrynet/extras/daemon/Daemon.py
View file

File diff suppressed because it is too large Load diff

22
lbrynet/extras/daemon/analytics.py
View file

@ -160,28 +160,6 @@ class Manager:
async def _send_heartbeat(self): async def _send_heartbeat(self):
await self.track(self._event(HEARTBEAT)) await self.track(self._event(HEARTBEAT))
async def _update_tracked_metrics(self):
should_send, value = self.summarize_and_reset(BLOB_BYTES_UPLOADED)
if should_send:
await self.track(self._metric_event(BLOB_BYTES_UPLOADED, value))
def add_observation(self, metric, value):
self._tracked_data[metric].append(value)
def summarize_and_reset(self, metric, op=sum):
"""Apply `op` on the current values for `metric`.
This operation also resets the metric.
Returns:
a tuple (should_send, value)
"""
try:
values = self._tracked_data.pop(metric)
return True, op(values)
except KeyError:
return False, None
def _event(self, event, event_properties=None): def _event(self, event, event_properties=None):
return { return {
'userId': 'lbry', 'userId': 'lbry',

58
lbrynet/extras/daemon/client.py Normal file
View file

@ -0,0 +1,58 @@
from lbrynet import conf
import aiohttp
import logging
from urllib.parse import urlparse
log = logging.getLogger(__name__)
USER_AGENT = "AuthServiceProxy/0.1"
TWISTED_SECURE_SESSION = "TWISTED_SECURE_SESSION"
TWISTED_SESSION = "TWISTED_SESSION"
LBRY_SECRET = "LBRY_SECRET"
HTTP_TIMEOUT = 30
class JSONRPCException(Exception):
def __init__(self, rpc_error):
super().__init__()
self.error = rpc_error
class UnAuthAPIClient:
def __init__(self, host, port, session):
self.host = host
self.port = port
self.session = session
def __getattr__(self, method):
async def f(*args, **kwargs):
return await self.call(method, [args, kwargs])
return f
@classmethod
async def from_url(cls, url):
url_fragment = urlparse(url)
host = url_fragment.hostname
port = url_fragment.port
connector = aiohttp.TCPConnector()
session = aiohttp.ClientSession(connector=connector)
return cls(host, port, session)
async def call(self, method, params=None):
message = {'method': method, 'params': params}
async with self.session.get(conf.settings.get_api_connection_string(), json=message) as resp:
response_dict = await resp.json()
if 'error' in response_dict:
raise JSONRPCException(response_dict['error'])
else:
return response_dict['result']
class LBRYAPIClient:
@staticmethod
def get_client(conf_path=None):
conf.conf_file = conf_path
if not conf.settings:
conf.initialize_settings()
return UnAuthAPIClient.from_url(conf.settings.get_api_connection_string())

22
lbrynet/extras/daemon/migrator/migrate8to9.py
View file

@ -1,10 +1,9 @@
import sqlite3 import sqlite3
import logging import logging
import os import os
import asyncio
from lbrynet.p2p.Error import InvalidStreamDescriptorError from lbrynet.blob.blob_info import BlobInfo
from lbrynet.p2p.StreamDescriptor import EncryptedFileStreamType, format_sd_info, format_blobs, validate_descriptor from lbrynet.stream.descriptor import StreamDescriptor
from lbrynet.blob.CryptBlob import CryptBlobInfo
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -22,18 +21,13 @@ def do_migration(conf):
"left outer join blob b ON b.blob_hash=s.blob_hash order by s.position").fetchall() "left outer join blob b ON b.blob_hash=s.blob_hash order by s.position").fetchall()
blobs_by_stream = {} blobs_by_stream = {}
for stream_hash, position, iv, blob_hash, blob_length in blobs: for stream_hash, position, iv, blob_hash, blob_length in blobs:
blobs_by_stream.setdefault(stream_hash, []).append(CryptBlobInfo(blob_hash, position, blob_length or 0, iv)) blobs_by_stream.setdefault(stream_hash, []).append(BlobInfo(position, blob_length or 0, iv, blob_hash))
for stream_name, stream_key, suggested_filename, sd_hash, stream_hash in streams: for stream_name, stream_key, suggested_filename, sd_hash, stream_hash in streams:
sd_info = format_sd_info( sd = StreamDescriptor(asyncio.get_event_loop(), blob_dir, stream_name, stream_key, suggested_filename,
EncryptedFileStreamType, stream_name, stream_key, blobs_by_stream[stream_hash], stream_hash, sd_hash)
suggested_filename, stream_hash, format_blobs(blobs_by_stream[stream_hash]) if sd_hash != sd.calculate_sd_hash():
) log.warning("Stream for descriptor %s is invalid, cleaning it up", sd_hash)
try:
validate_descriptor(sd_info)
except InvalidStreamDescriptorError as err:
log.warning("Stream for descriptor %s is invalid (%s), cleaning it up",
sd_hash, err.message)
blob_hashes = [blob.blob_hash for blob in blobs_by_stream[stream_hash]] blob_hashes = [blob.blob_hash for blob in blobs_by_stream[stream_hash]]
delete_stream(cursor, stream_hash, sd_hash, blob_hashes, blob_dir) delete_stream(cursor, stream_hash, sd_hash, blob_hashes, blob_dir)

115
lbrynet/utils.py
View file

@ -5,15 +5,14 @@ import random
import socket import socket
import string import string
import json import json
import traceback import typing
import functools import asyncio
import logging import logging
import pkg_resources import pkg_resources
from twisted.python.failure import Failure
from twisted.internet import defer
from lbrynet.schema.claim import ClaimDict from lbrynet.schema.claim import ClaimDict
from lbrynet.cryptoutils import get_lbry_hash_obj from lbrynet.cryptoutils import get_lbry_hash_obj
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -43,21 +42,6 @@ def datetime_obj(*args, **kwargs):
return datetime.datetime(*args, **kwargs) return datetime.datetime(*args, **kwargs)
def call_later(delay, func, *args, **kwargs):
# Import here to ensure that it gets called after installing a reactor
# see: http://twistedmatrix.com/documents/current/core/howto/choosing-reactor.html
from twisted.internet import reactor
return reactor.callLater(delay, func, *args, **kwargs)
def safe_start_looping_call(looping_call, interval_sec):
if not looping_call.running:
looping_call.start(interval_sec)
def safe_stop_looping_call(looping_call):
if looping_call.running:
looping_call.stop()
def generate_id(num=None): def generate_id(num=None):
h = get_lbry_hash_obj() h = get_lbry_hash_obj()
if num is not None: if num is not None:
@ -139,91 +123,16 @@ def json_dumps_pretty(obj, **kwargs):
return json.dumps(obj, sort_keys=True, indent=2, separators=(',', ': '), **kwargs) return json.dumps(obj, sort_keys=True, indent=2, separators=(',', ': '), **kwargs)
class DeferredLockContextManager: def cancel_task(task: typing.Optional[asyncio.Task]):
def __init__(self, lock): if task and not task.done():
self._lock = lock task.cancel()
def __enter__(self):
yield self._lock.acquire()
def __exit__(self, exc_type, exc_val, exc_tb):
yield self._lock.release()
@defer.inlineCallbacks def cancel_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
def DeferredDict(d, consumeErrors=False): for task in tasks:
keys = [] cancel_task(task)
dl = []
response = {}
for k, v in d.items():
keys.append(k)
dl.append(v)
results = yield defer.DeferredList(dl, consumeErrors=consumeErrors)
for k, (success, result) in zip(keys, results):
if success:
response[k] = result
defer.returnValue(response)
class DeferredProfiler: def drain_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
def __init__(self): while tasks:
self.profile_results = {} cancel_task(tasks.pop())
def add_result(self, fn, start_time, finished_time, stack, success):
self.profile_results[fn].append((start_time, finished_time, stack, success))
def show_profile_results(self, fn):
profile_results = list(self.profile_results[fn])
call_counts = {
caller: [(start, finished, finished - start, success)
for (start, finished, _caller, success) in profile_results
if _caller == caller]
for caller in {result[2] for result in profile_results}
}
log.info("called %s %i times from %i sources\n", fn.__name__, len(profile_results), len(call_counts))
for caller in sorted(list(call_counts.keys()), key=lambda c: len(call_counts[c]), reverse=True):
call_info = call_counts[caller]
times = [r[2] for r in call_info]
own_time = sum(times)
times.sort()
longest = 0 if not times else times[-1]
shortest = 0 if not times else times[0]
log.info(
"%i successes and %i failures\nlongest %f, shortest %f, avg %f\ncaller:\n%s",
len([r for r in call_info if r[3]]),
len([r for r in call_info if not r[3]]),
longest, shortest, own_time / float(len(call_info)), caller
)
def profiled_deferred(self, reactor=None):
if not reactor:
from twisted.internet import reactor
def _cb(result, fn, start, caller_info):
got_error = isinstance(result, (Failure, Exception))
self.add_result(fn, start, reactor.seconds(), caller_info, not got_error)
if got_error:
raise result
else:
return result
def _profiled_deferred(fn):
reactor.addSystemEventTrigger("after", "shutdown", self.show_profile_results, fn)
self.profile_results[fn] = []
@functools.wraps(fn)
def _wrapper(*args, **kwargs):
caller_info = "".join(traceback.format_list(traceback.extract_stack()[-3:-1]))
start = reactor.seconds()
d = defer.maybeDeferred(fn, *args, **kwargs)
d.addBoth(_cb, fn, start, caller_info)
return d
return _wrapper
return _profiled_deferred
_profiler = DeferredProfiler()
profile_deferred = _profiler.profiled_deferred