forked from LBRYCommunity/lbry-sdk
238 lines
7.6 KiB
Python
238 lines
7.6 KiB
Python
import os
|
|
import sys
|
|
import time
|
|
from typing import Dict, Any
|
|
from tempfile import TemporaryFile
|
|
|
|
import tqdm
|
|
|
|
from lbry import __version__
|
|
from lbry.service.base import Service
|
|
from lbry.service.full_node import FullNode
|
|
from lbry.service.light_client import LightClient
|
|
|
|
|
|
class RedirectOutput:
|
|
|
|
silence_lines = [
|
|
b'libprotobuf ERROR google/protobuf/wire_format_lite.cc:626',
|
|
]
|
|
|
|
def __init__(self, stream_type: str):
|
|
assert stream_type in ('stderr', 'stdout')
|
|
self.stream_type = stream_type
|
|
self.stream_no = getattr(sys, stream_type).fileno()
|
|
self.last_flush = time.time()
|
|
self.last_read = 0
|
|
|
|
def __enter__(self):
|
|
self.backup = os.dup(self.stream_no)
|
|
setattr(sys, self.stream_type, os.fdopen(self.backup, 'w'))
|
|
self.file = TemporaryFile()
|
|
self.backup = os.dup2(self.file.fileno(), self.stream_no)
|
|
|
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
self.file.close()
|
|
os.dup2(self.backup, self.stream_no)
|
|
os.close(self.backup)
|
|
setattr(sys, self.stream_type, os.fdopen(self.stream_no, 'w'))
|
|
|
|
def capture(self):
|
|
self.__enter__()
|
|
|
|
def release(self):
|
|
self.__exit__(None, None, None)
|
|
|
|
def flush(self, writer, force=False):
|
|
if not force and (time.time() - self.last_flush) < 5:
|
|
return
|
|
self.file.seek(self.last_read)
|
|
for line in self.file.readlines():
|
|
silence = False
|
|
for bad_line in self.silence_lines:
|
|
if bad_line in line:
|
|
silence = True
|
|
break
|
|
if not silence:
|
|
writer(line.decode().rstrip())
|
|
self.last_read = self.file.tell()
|
|
self.last_flush = time.time()
|
|
|
|
|
|
class Console:
|
|
|
|
def __init__(self, service: Service):
|
|
self.service = service
|
|
|
|
def starting(self):
|
|
pass
|
|
|
|
def stopping(self):
|
|
pass
|
|
|
|
|
|
class Basic(Console):
|
|
|
|
def __init__(self, service: Service):
|
|
super().__init__(service)
|
|
self.service.sync.on_progress.listen(self.on_sync_progress)
|
|
|
|
def starting(self):
|
|
conf = self.service.conf
|
|
s = [f'LBRY v{__version__}']
|
|
if isinstance(self.service, FullNode):
|
|
s.append('Full Node')
|
|
elif isinstance(self.service, LightClient):
|
|
s.append('Light Client')
|
|
if conf.processes == -1:
|
|
s.append('Threads Only')
|
|
elif conf.processes == 0:
|
|
s.append(f'{os.cpu_count()} Process(es)')
|
|
else:
|
|
s.append(f'{conf.processes} Process(es)')
|
|
s.append(f'({os.cpu_count()} CPU(s) available)')
|
|
print(' '.join(s))
|
|
|
|
@staticmethod
|
|
def stopping():
|
|
print('bye.')
|
|
|
|
@staticmethod
|
|
def on_sync_progress(event):
|
|
print(event)
|
|
|
|
|
|
class Advanced(Basic):
|
|
|
|
FORMAT = '{l_bar}{bar}| {n_fmt:>8}/{total_fmt:>8} [{elapsed:>7}<{remaining:>8}, {rate_fmt:>17}]'
|
|
|
|
def __init__(self, service: Service):
|
|
super().__init__(service)
|
|
self.bars: Dict[Any, tqdm.tqdm] = {}
|
|
self.is_single_sync_bar = False
|
|
self.single_bar_relative_steps = 0
|
|
self.last_stats = ""
|
|
self.sync_steps = []
|
|
self.block_savers = 0
|
|
self.block_readers = 0
|
|
self.stderr = RedirectOutput('stderr')
|
|
|
|
def starting(self):
|
|
self.stderr.capture()
|
|
super().starting()
|
|
|
|
def stopping(self):
|
|
super().stopping()
|
|
self.stderr.flush(self.bars['sync'].write, True)
|
|
self.stderr.release()
|
|
|
|
def get_or_create_bar(self, name, desc, unit, total, leave=False, bar_format=None, postfix=None, position=None):
|
|
bar = self.bars.get(name)
|
|
if bar is None:
|
|
bar = self.bars[name] = tqdm.tqdm(
|
|
desc=desc, unit=unit, total=total,
|
|
bar_format=bar_format or self.FORMAT, leave=leave,
|
|
postfix=postfix, position=position
|
|
)
|
|
return bar
|
|
|
|
def sync_start(self, d):
|
|
self.bars.clear()
|
|
self.sync_steps = d['sync_steps']
|
|
if d['ending_height']-d['starting_height'] > 0:
|
|
label = f"sync {d['starting_height']:,d}-{d['ending_height']:,d}"
|
|
else:
|
|
label = f"sync {d['ending_height']:,d}"
|
|
self.last_stats = f"{d['txs']:,d} txs, {d['claims']:,d} claims and {d['supports']:,d} supports"
|
|
self.get_or_create_bar(
|
|
"sync", label, "tasks", len(d['sync_steps']), True,
|
|
"{l_bar}{bar}| {postfix[0]:<55}", (self.last_stats,)
|
|
)
|
|
self.get_or_create_bar("read", "├─ blocks read", "blocks", d['blocks'], True)
|
|
self.get_or_create_bar("save", "└─┬ txs saved", "txs", d['txs'], True)
|
|
|
|
def update_progress(self, e, d):
|
|
if e in ('blockchain.sync.block.read', 'blockchain.sync.block.save'):
|
|
self.update_block_bars(e, d)
|
|
else:
|
|
self.update_steps_bar(e, d)
|
|
self.update_other_bars(e, d)
|
|
|
|
def update_steps_bar(self, e, d):
|
|
sync_bar = self.bars['sync']
|
|
if d['step'] == d['total']:
|
|
sync_done = (self.sync_steps.index(e)+1)-sync_bar.last_print_n
|
|
sync_bar.postfix = (f'finished: {e}',)
|
|
sync_bar.update(sync_done)
|
|
|
|
def update_block_bars(self, event, d):
|
|
total_bar = self.bars[event[-4:]]
|
|
if event.endswith("read") and self.block_readers == 0:
|
|
total_bar.unpause()
|
|
if event.endswith("read") and d['step'] == d['total']:
|
|
self.block_readers -= 1
|
|
|
|
bar_name = f"block-{d['block_file']}"
|
|
bar = self.bars.get(bar_name)
|
|
if bar is None:
|
|
self.block_readers += 1
|
|
return self.get_or_create_bar(
|
|
bar_name,
|
|
f" ├─ blk{d['block_file']:05}.dat reading", 'blocks', d['total']
|
|
)
|
|
|
|
if event.endswith("save") and bar.unit == "blocks":
|
|
if self.block_savers == 0:
|
|
total_bar.unpause()
|
|
self.block_savers += 1
|
|
bar.desc = f" ├─ blk{d['block_file']:05}.dat saving"
|
|
bar.unit = "txs"
|
|
bar.reset(d['total'])
|
|
return
|
|
|
|
diff = d['step']-bar.last_print_n
|
|
bar.update(diff)
|
|
if event.endswith("save") and d['step'] == d['total']:
|
|
self.block_savers -= 1
|
|
bar.close()
|
|
|
|
total_bar.update(diff)
|
|
if total_bar.total == total_bar.last_print_n:
|
|
self.update_steps_bar(event, {'step': 1, 'total': 1})
|
|
if total_bar.desc.endswith('txs saved'):
|
|
total_bar.desc = "├─ txs saved"
|
|
total_bar.refresh()
|
|
|
|
def update_other_bars(self, e, d):
|
|
if d['total'] == 0:
|
|
return
|
|
bar = self.bars.get(e)
|
|
if not bar:
|
|
name = (
|
|
' '.join(e.split('.')[-2:])
|
|
.replace('support', 'suprt')
|
|
.replace('channels', 'chanls')
|
|
.replace('signatures', 'sigs')
|
|
)
|
|
bar = self.get_or_create_bar(e, f"├─ {name:>12}", d['unit'], d['total'], True)
|
|
diff = d['step']-bar.last_print_n
|
|
bar.update(diff)
|
|
#if d['step'] == d['total']:
|
|
#bar.close()
|
|
|
|
def sync_complete(self):
|
|
self.bars['sync'].postfix = (self.last_stats,)
|
|
for bar in self.bars.values():
|
|
bar.close()
|
|
|
|
def on_sync_progress(self, event):
|
|
e, d = event['event'], event.get('data', {})
|
|
if e.endswith("sync.start"):
|
|
self.sync_start(d)
|
|
self.stderr.flush(self.bars['sync'].write)
|
|
elif e.endswith("sync.complete"):
|
|
self.stderr.flush(self.bars['sync'].write, True)
|
|
self.sync_complete()
|
|
else:
|
|
self.stderr.flush(self.bars['sync'].write)
|
|
self.update_progress(e, d)
|