From 77e19ab1a4d74722eec1b45cf336885aa0196566 Mon Sep 17 00:00:00 2001
From: Jack Robison <jackrobison@lbry.io>
Date: Thu, 31 Dec 2020 14:43:35 -0500
Subject: [PATCH 1/2] prometheus metrics for asyncio loop responsiveness
---
lbry/prometheus.py | 38 +++++++++++++++++++++++++++++++++++++-
1 file changed, 37 insertions(+), 1 deletion(-)
diff --git a/lbry/prometheus.py b/lbry/prometheus.py
index 220ee97bd..3c09c49f6 100644
--- a/lbry/prometheus.py
+++ b/lbry/prometheus.py
@@ -1,14 +1,44 @@
+import time
import logging
+import asyncio
+import asyncio.tasks
from aiohttp import web
from prometheus_client import generate_latest as prom_generate_latest
+from prometheus_client import Counter, Histogram, Gauge
+
+
+PROBES_IN_FLIGHT = Counter("probes_in_flight", "Number of loop probes in flight", namespace='asyncio')
+PROBES_FINISHED = Counter("probes_finished", "Number of finished loop probes", namespace='asyncio')
+PROBE_TIMES = Histogram("probe_times", "Loop probe times", namespace='asyncio')
+TASK_COUNT = Gauge("running_tasks", "Number of running tasks", namespace='asyncio')
+
+
+def get_loop_metrics(delay=1):
+ loop = asyncio.get_event_loop()
+
+ def callback(started):
+ PROBE_TIMES.observe(time.perf_counter() - started - delay)
+ PROBES_FINISHED.inc()
+
+ async def monitor_loop_responsiveness():
+ while True:
+ now = time.perf_counter()
+ loop.call_later(delay, callback, now)
+ PROBES_IN_FLIGHT.inc()
+ TASK_COUNT.set(len(asyncio.tasks._all_tasks))
+ await asyncio.sleep(delay)
+
+ return loop.create_task(monitor_loop_responsiveness())
class PrometheusServer:
def __init__(self, logger=None):
self.runner = None
self.logger = logger or logging.getLogger(__name__)
+ self._monitor_loop_task = None
async def start(self, interface: str, port: int):
+ self.logger.info("start prometheus metrics")
prom_app = web.Application()
prom_app.router.add_get('/metrics', self.handle_metrics_get_request)
self.runner = web.AppRunner(prom_app)
@@ -16,7 +46,10 @@ class PrometheusServer:
metrics_site = web.TCPSite(self.runner, interface, port, shutdown_timeout=.5)
await metrics_site.start()
- self.logger.info('metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2])
+ self.logger.info(
+ 'prometheus metrics server listening on %s:%i', *metrics_site._server.sockets[0].getsockname()[:2]
+ )
+ self._monitor_loop_task = get_loop_metrics()
async def handle_metrics_get_request(self, request: web.Request):
try:
@@ -29,4 +62,7 @@ class PrometheusServer:
raise
async def stop(self):
+ if self._monitor_loop_task and not self._monitor_loop_task.done():
+ self._monitor_loop_task.cancel()
+ self._monitor_loop_task = None
await self.runner.cleanup()
From f309a65cb49d10d09acd88aa0c64fed2249476d9 Mon Sep 17 00:00:00 2001
From: Jack Robison <jackrobison@lbry.io>
Date: Fri, 1 Jan 2021 14:14:08 -0500
Subject: [PATCH 2/2] fix
---
tests/integration/other/test_exchange_rate_manager.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/tests/integration/other/test_exchange_rate_manager.py b/tests/integration/other/test_exchange_rate_manager.py
index 513b636da..c9664f12c 100644
--- a/tests/integration/other/test_exchange_rate_manager.py
+++ b/tests/integration/other/test_exchange_rate_manager.py
@@ -4,9 +4,9 @@ from lbry.extras.daemon.exchange_rate_manager import ExchangeRate, ExchangeRateM
class TestExchangeRateManager(AsyncioTestCase):
-
async def test_exchange_rate_manager(self):
# TODO: re-enable cryptonator.com
+ # TODO: this uses real exchange rate feeds... update to use mocks
manager = ExchangeRateManager(FEEDS)
manager.start()
self.addCleanup(manager.stop)
@@ -18,5 +18,5 @@ class TestExchangeRateManager(AsyncioTestCase):
self.assertTrue(feed.is_online)
self.assertIsInstance(feed.rate, ExchangeRate)
lbc = manager.convert_currency('USD', 'LBC', Decimal('0.01'))
- self.assertGreaterEqual(lbc, 0.1)
+ self.assertGreaterEqual(lbc, 0.01)
self.assertLessEqual(lbc, 10.0)