From 56c8ad1221e0764f601928bfcd897b2536ec748e Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 10 Mar 2020 20:50:25 -0300 Subject: [PATCH 1/5] start/stop tracemalloc over api --- lbry/extras/daemon/daemon.py | 24 +++++++++++++++++++ .../integration/other/test_other_commands.py | 6 +++++ 2 files changed, 30 insertions(+) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index c91c72f14..fa2a4f675 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -8,6 +8,7 @@ import inspect import typing import random import hashlib +import tracemalloc from urllib.parse import urlencode, quote from typing import Callable, Optional, List from binascii import hexlify, unhexlify @@ -4531,6 +4532,29 @@ class Daemon(metaclass=JSONRPCServerType): result['node_id'] = hexlify(self.dht_node.protocol.node_id).decode() return result + TRACEMALLOC_DOC = """ + Controls and queries tracemalloc memory tracing tools for troubleshooting. + """ + + def jsonrpc_tracemalloc_set(self, enable: bool): + """ + Enable/disable tracemalloc memory tracing + + Usage: + jsonrpc_tracemalloc_set () + + Options: + --enable : (bool) True enables, False disables + + Returns: + (bool) is it tracing? + """ + if enable: + tracemalloc.start() + else: + tracemalloc.stop() + return tracemalloc.is_tracing() + COMMENT_DOC = """ View, create and abandon comments. """ diff --git a/tests/integration/other/test_other_commands.py b/tests/integration/other/test_other_commands.py index 4ba5b9f10..ba95c6225 100644 --- a/tests/integration/other/test_other_commands.py +++ b/tests/integration/other/test_other_commands.py @@ -38,3 +38,9 @@ class SettingsManagement(CommandTestCase): self.assertTrue(self.daemon.analytics_manager.enabled) self.assertTrue(loggly.enabled) self.daemon.jsonrpc_settings_set('share_usage_data', False) + + +class TroubleshootingCommands(CommandTestCase): + async def test_tracemalloc_commands(self): + self.assertFalse(self.daemon.jsonrpc_tracemalloc_set(False)) + self.assertTrue(self.daemon.jsonrpc_tracemalloc_set(True)) From 8169bf6b97f6edb661ee5ea1e45846ab9d78f67e Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 10 Mar 2020 21:39:42 -0300 Subject: [PATCH 2/5] top objects api --- lbry/extras/daemon/daemon.py | 48 ++++++++++++++++++- .../integration/other/test_other_commands.py | 10 ++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index fa2a4f675..56614b98a 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -1,3 +1,4 @@ +import linecache import os import re import asyncio @@ -4541,10 +4542,10 @@ class Daemon(metaclass=JSONRPCServerType): Enable/disable tracemalloc memory tracing Usage: - jsonrpc_tracemalloc_set () + jsonrpc_tracemalloc_set ( | --enable=) Options: - --enable : (bool) True enables, False disables + --enable= : (bool) True enables, False disables Returns: (bool) is it tracing? @@ -4555,6 +4556,49 @@ class Daemon(metaclass=JSONRPCServerType): tracemalloc.stop() return tracemalloc.is_tracing() + def jsonrpc_tracemalloc_top(self, items: int = 10): + """ + Show most common objects, the place that created them and their size. + + Usage: + jsonrpc_tracemalloc_top [( | --items=)] + + Options: + --items= : (int) maximum items to return, from the most common + + Returns: + (dict) dictionary containing most common objects in memory + { + "line": (str) filename and line number where it was created, + "code": (str) code that created it, + "size": (int) size in bytes, for each "memory block", + "count" (int) number of memory blocks + } + """ + if not tracemalloc.is_tracing(): + raise Exception("Enable tracemalloc first! See 'tracemalloc set' command.") + stats = tracemalloc.take_snapshot().filter_traces(( + tracemalloc.Filter(False, ""), + tracemalloc.Filter(False, ""), + # tracemalloc and linecache here use some memory, but thats not relevant + tracemalloc.Filter(False, tracemalloc.__file__), + tracemalloc.Filter(False, linecache.__file__), + )).statistics('lineno', True) + results = [] + for stat in stats: + frame = stat.traceback[0] + filename = os.sep.join(frame.filename.split(os.sep)[-2:]) + line = linecache.getline(frame.filename, frame.lineno).strip() + results.append({ + "line": f"{filename}:{frame.lineno}", + "code": line, + "size": stat.size, + "count": stat.count + }) + if len(results) == items: + break + return results + COMMENT_DOC = """ View, create and abandon comments. """ diff --git a/tests/integration/other/test_other_commands.py b/tests/integration/other/test_other_commands.py index ba95c6225..deb9bf936 100644 --- a/tests/integration/other/test_other_commands.py +++ b/tests/integration/other/test_other_commands.py @@ -44,3 +44,13 @@ class TroubleshootingCommands(CommandTestCase): async def test_tracemalloc_commands(self): self.assertFalse(self.daemon.jsonrpc_tracemalloc_set(False)) self.assertTrue(self.daemon.jsonrpc_tracemalloc_set(True)) + + class WeirdObject(): + pass + hold_em = [WeirdObject() for _ in range(500)] + self.assertEqual( + [{'code': 'hold_em = [WeirdObject() for _ in range(500)]', + 'count': 502, + 'line': 'other/test_other_commands.py:50', + 'size': 36656}], self.daemon.jsonrpc_tracemalloc_top(items=1) + ) From ec541e20574808e215a0cfae5b4ccf53ecd674c5 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 11 Mar 2020 17:59:56 -0300 Subject: [PATCH 3/5] lint and improve tests --- lbry/extras/daemon/daemon.py | 4 ++-- tests/integration/other/test_other_commands.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index 56614b98a..c63ce39ab 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -4537,7 +4537,7 @@ class Daemon(metaclass=JSONRPCServerType): Controls and queries tracemalloc memory tracing tools for troubleshooting. """ - def jsonrpc_tracemalloc_set(self, enable: bool): + def jsonrpc_tracemalloc_set(self, enable: bool): # pylint: disable=no-self-use """ Enable/disable tracemalloc memory tracing @@ -4556,7 +4556,7 @@ class Daemon(metaclass=JSONRPCServerType): tracemalloc.stop() return tracemalloc.is_tracing() - def jsonrpc_tracemalloc_top(self, items: int = 10): + def jsonrpc_tracemalloc_top(self, items: int = 10): # pylint: disable=no-self-use """ Show most common objects, the place that created them and their size. diff --git a/tests/integration/other/test_other_commands.py b/tests/integration/other/test_other_commands.py index deb9bf936..5d1c74048 100644 --- a/tests/integration/other/test_other_commands.py +++ b/tests/integration/other/test_other_commands.py @@ -48,9 +48,9 @@ class TroubleshootingCommands(CommandTestCase): class WeirdObject(): pass hold_em = [WeirdObject() for _ in range(500)] - self.assertEqual( - [{'code': 'hold_em = [WeirdObject() for _ in range(500)]', - 'count': 502, - 'line': 'other/test_other_commands.py:50', - 'size': 36656}], self.daemon.jsonrpc_tracemalloc_top(items=1) - ) + top = self.daemon.jsonrpc_tracemalloc_top(1) + self.assertEqual(1, len(top)) + self.assertEqual('hold_em = [WeirdObject() for _ in range(500)]', top[0]['code']) + self.assertTrue(top[0]['line'].startswith('other/test_other_commands.py:')) + self.assertGreaterEqual(top[0]['count'], 500) + self.assertGreater(top[0]['size'], 0) # just matters that its a positive integer From d44d5c33040806821ae08bbb522dd61a4c2c4ec5 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 16 Mar 2020 06:39:42 -0300 Subject: [PATCH 4/5] enable/disable instead of set --- lbry/extras/daemon/daemon.py | 29 ++++++++++++++----- .../integration/other/test_other_commands.py | 4 +-- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/lbry/extras/daemon/daemon.py b/lbry/extras/daemon/daemon.py index c63ce39ab..014a69194 100644 --- a/lbry/extras/daemon/daemon.py +++ b/lbry/extras/daemon/daemon.py @@ -4537,23 +4537,36 @@ class Daemon(metaclass=JSONRPCServerType): Controls and queries tracemalloc memory tracing tools for troubleshooting. """ - def jsonrpc_tracemalloc_set(self, enable: bool): # pylint: disable=no-self-use + def jsonrpc_tracemalloc_enable(self): # pylint: disable=no-self-use """ - Enable/disable tracemalloc memory tracing + Enable tracemalloc memory tracing Usage: - jsonrpc_tracemalloc_set ( | --enable=) + jsonrpc_tracemalloc_enable Options: - --enable= : (bool) True enables, False disables + None Returns: (bool) is it tracing? """ - if enable: - tracemalloc.start() - else: - tracemalloc.stop() + tracemalloc.start() + return tracemalloc.is_tracing() + + def jsonrpc_tracemalloc_disable(self): # pylint: disable=no-self-use + """ + Disable tracemalloc memory tracing + + Usage: + jsonrpc_tracemalloc_disable + + Options: + None + + Returns: + (bool) is it tracing? + """ + tracemalloc.stop() return tracemalloc.is_tracing() def jsonrpc_tracemalloc_top(self, items: int = 10): # pylint: disable=no-self-use diff --git a/tests/integration/other/test_other_commands.py b/tests/integration/other/test_other_commands.py index 5d1c74048..1ec10c4ba 100644 --- a/tests/integration/other/test_other_commands.py +++ b/tests/integration/other/test_other_commands.py @@ -42,8 +42,8 @@ class SettingsManagement(CommandTestCase): class TroubleshootingCommands(CommandTestCase): async def test_tracemalloc_commands(self): - self.assertFalse(self.daemon.jsonrpc_tracemalloc_set(False)) - self.assertTrue(self.daemon.jsonrpc_tracemalloc_set(True)) + self.assertFalse(self.daemon.jsonrpc_tracemalloc_disable()) + self.assertTrue(self.daemon.jsonrpc_tracemalloc_enable()) class WeirdObject(): pass From 9561f93594ba1e52997b1ed72f5c3a24ec61c074 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 16 Mar 2020 11:29:16 -0300 Subject: [PATCH 5/5] disable tracemalloc after test finishes so it doesnt slow down everything else --- tests/integration/other/test_other_commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/other/test_other_commands.py b/tests/integration/other/test_other_commands.py index 1ec10c4ba..a96c4effd 100644 --- a/tests/integration/other/test_other_commands.py +++ b/tests/integration/other/test_other_commands.py @@ -42,6 +42,7 @@ class SettingsManagement(CommandTestCase): class TroubleshootingCommands(CommandTestCase): async def test_tracemalloc_commands(self): + self.addCleanup(self.daemon.jsonrpc_tracemalloc_disable) self.assertFalse(self.daemon.jsonrpc_tracemalloc_disable()) self.assertTrue(self.daemon.jsonrpc_tracemalloc_enable())