From a9f6a68952c64625a6ef223483a3eb3ac451a8cf Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 4 Jun 2021 11:54:37 -0300 Subject: [PATCH 1/4] use LRU caches for DHT metrics --- lbry/dht/peer.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lbry/dht/peer.py b/lbry/dht/peer.py index cf30b359e..bc2a60022 100644 --- a/lbry/dht/peer.py +++ b/lbry/dht/peer.py @@ -4,7 +4,7 @@ import logging from binascii import hexlify from dataclasses import dataclass, field from functools import lru_cache -from lbry.utils import is_valid_public_ipv4 as _is_valid_public_ipv4 +from lbry.utils import is_valid_public_ipv4 as _is_valid_public_ipv4, LRUCache from lbry.dht import constants from lbry.dht.serialization.datagram import make_compact_address, make_compact_ip, decode_compact_address @@ -31,12 +31,12 @@ class PeerManager: self._rpc_failures: typing.Dict[ typing.Tuple[str, int], typing.Tuple[typing.Optional[float], typing.Optional[float]] ] = {} - self._last_replied: typing.Dict[typing.Tuple[str, int], float] = {} - self._last_sent: typing.Dict[typing.Tuple[str, int], float] = {} - self._last_requested: typing.Dict[typing.Tuple[str, int], float] = {} - self._node_id_mapping: typing.Dict[typing.Tuple[str, int], bytes] = {} - self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = {} - self._node_tokens: typing.Dict[bytes, (float, bytes)] = {} + self._last_replied: typing.Dict[typing.Tuple[str, int], float] = LRUCache(2048) + self._last_sent: typing.Dict[typing.Tuple[str, int], float] = LRUCache(2048) + self._last_requested: typing.Dict[typing.Tuple[str, int], float] = LRUCache(2048) + self._node_id_mapping: typing.Dict[typing.Tuple[str, int], bytes] = LRUCache(2048) + self._node_id_reverse_mapping: typing.Dict[bytes, typing.Tuple[str, int]] = LRUCache(2048) + self._node_tokens: typing.Dict[bytes, (float, bytes)] = LRUCache(2048) def reset(self): for statistic in (self._rpc_failures, self._last_replied, self._last_sent, self._last_requested): From 00d038c8f3421ca21ca66380821df5aaf12ad07b Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 4 Jun 2021 12:15:47 -0300 Subject: [PATCH 2/4] add default parameter to pop on LRUCache --- lbry/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lbry/utils.py b/lbry/utils.py index e5d624991..763f0f811 100644 --- a/lbry/utils.py +++ b/lbry/utils.py @@ -313,8 +313,8 @@ class LRUCache: def clear(self): self.cache.clear() - def pop(self, key): - return self.cache.pop(key) + def pop(self, key, default=None): + return self.cache.pop(key, default) def __setitem__(self, key, value): return self.set(key, value) From 4e6b4f179b7b9801bc2c781c2be7d0d689c45322 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 4 Jun 2021 12:20:44 -0300 Subject: [PATCH 3/4] add items() to LRUCache --- lbry/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lbry/utils.py b/lbry/utils.py index 763f0f811..c944ed29b 100644 --- a/lbry/utils.py +++ b/lbry/utils.py @@ -310,6 +310,9 @@ class LRUCache: self.cache.popitem(last=False) self.cache[key] = value + def items(self): + return self.cache.items() + def clear(self): self.cache.clear() From fb438dc108a59f61197bf12b49c21fa4e285b690 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 4 Jun 2021 12:34:00 -0300 Subject: [PATCH 4/4] remove the unregister call --- lbry/utils.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lbry/utils.py b/lbry/utils.py index c944ed29b..ee998da41 100644 --- a/lbry/utils.py +++ b/lbry/utils.py @@ -21,7 +21,6 @@ import pkg_resources import certifi import aiohttp from prometheus_client import Counter -from prometheus_client.registry import REGISTRY from lbry.schema.claim import Claim @@ -276,12 +275,6 @@ class LRUCacheWithMetrics: def __del__(self): self.clear() - if self._track_metrics: # needed for tests - try: - REGISTRY.unregister(self.hits) - REGISTRY.unregister(self.misses) - except AttributeError: - pass class LRUCache: