From 038a5f999f0c406d240ea6f9c5ca2becd5535dee Mon Sep 17 00:00:00 2001 From: Victor Shyba <victor.shyba@gmail.com> Date: Thu, 4 Feb 2021 19:44:18 -0300 Subject: [PATCH] cache encoded headers --- lbry/wallet/server/leveldb.py | 12 ++++++++++++ lbry/wallet/server/session.py | 6 ++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/lbry/wallet/server/leveldb.py b/lbry/wallet/server/leveldb.py index 109ae9a8c..02b2afe65 100644 --- a/lbry/wallet/server/leveldb.py +++ b/lbry/wallet/server/leveldb.py @@ -12,6 +12,7 @@ import asyncio import array import ast +import base64 import os import time import zlib @@ -82,6 +83,7 @@ class LevelDB: self.utxo_db = None self.tx_counts = None self.headers = None + self.encoded_headers = LRUCacheWithMetrics(1 << 21, metric_name='encoded_headers', namespace='wallet_server') self.last_flush = time.time() self.logger.info(f'using {self.env.db_engine} for DB backend') @@ -440,6 +442,16 @@ class LevelDB: raise IndexError(f'height {height:,d} out of range') return header + def encode_headers(self, start_height, count, headers): + key = (start_height, count) + if not self.encoded_headers.get(key): + compressobj = zlib.compressobj(wbits=-15, level=1, memLevel=9) + headers = base64.b64encode(compressobj.compress(headers) + compressobj.flush()).decode() + if start_height % 1000 != 0: + return headers + self.encoded_headers[key] = headers + return self.encoded_headers.get(key) + def read_headers(self, start_height, count) -> typing.Tuple[bytes, int]: """Requires start_height >= 0, count >= 0. Reads as many headers as are available starting at start_height up to count. This diff --git a/lbry/wallet/server/session.py b/lbry/wallet/server/session.py index 8193f6c11..684558e5c 100644 --- a/lbry/wallet/server/session.py +++ b/lbry/wallet/server/session.py @@ -3,7 +3,6 @@ import ssl import math import time import json -import zlib import base64 import codecs import typing @@ -16,7 +15,7 @@ from asyncio import Event, sleep from collections import defaultdict from functools import partial -from binascii import hexlify, unhexlify +from binascii import hexlify from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor from prometheus_client import Counter, Info, Histogram, Gauge @@ -1345,8 +1344,7 @@ class LBRYElectrumX(SessionBase): headers, count = self.db.read_headers(start_height, count) if b64: - compressobj = zlib.compressobj(wbits=-15, level=1, memLevel=9) - headers = base64.b64encode(compressobj.compress(headers) + compressobj.flush()).decode() + headers = self.db.encode_headers(start_height, count, headers) else: headers = headers.hex() result = {