lbry-sdk/lbry/console.py

import os
import sys
import time
import itertools
import logging
from typing import Dict, Any
from tempfile import TemporaryFile

from tqdm.std import tqdm, Bar
from tqdm.utils import FormatReplace, _unicode, disp_len, disp_trim, _is_ascii

from lbry import __version__
from lbry.service.base import Service
from lbry.service.full_node import FullNode
from lbry.service.light_client import LightClient


log = logging.getLogger(__name__)


class RedirectOutput:

    silence_lines = [
        b'libprotobuf ERROR google/protobuf/wire_format_lite.cc:626',
    ]

    def __init__(self, stream_type: str):
        assert stream_type in ('stderr', 'stdout')
        self.stream_type = stream_type
        self.stream_no = getattr(sys, stream_type).fileno()
        self.last_flush = time.time()
        self.last_read = 0
        self.backup = None
        self.file = None

    def __enter__(self):
        self.backup = os.dup(self.stream_no)
        setattr(sys, self.stream_type, os.fdopen(self.backup, 'w'))
        self.file = TemporaryFile()
        self.backup = os.dup2(self.file.fileno(), self.stream_no)

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.file.close()
        os.dup2(self.backup, self.stream_no)
        os.close(self.backup)
        setattr(sys, self.stream_type, os.fdopen(self.stream_no, 'w'))

    def capture(self):
        self.__enter__()

    def release(self):
        self.__exit__(None, None, None)

    def flush(self, writer, force=False):
        if not force and (time.time() - self.last_flush) < 5:
            return
        self.file.seek(self.last_read)
        for line in self.file.readlines():
            silence = False
            for bad_line in self.silence_lines:
                if bad_line in line:
                    silence = True
                    break
            if not silence:
                writer(line.decode().rstrip())
        self.last_read = self.file.tell()
        self.last_flush = time.time()


class Console:

    def __init__(self, service: Service):
        self.service = service

    def starting(self):
        pass

    def stopping(self):
        pass


class Basic(Console):

    def __init__(self, service: Service):
        super().__init__(service)
        self.service.sync.on_progress.listen(self.on_sync_progress)
        self.tasks = {}
        logging.basicConfig(level=logging.INFO)

    def starting(self):
        conf = self.service.conf
        s = [f'LBRY v{__version__}']
        if isinstance(self.service, FullNode):
            s.append('Full Node')
        elif isinstance(self.service, LightClient):
            s.append('Light Client')
        if conf.workers == -1:
            s.append('Threads Only')
        else:
            workers = os.cpu_count() if conf.workers == 0 else conf.workers
            s.append(f'{workers} Worker' if workers == 1 else f'{workers} Workers')
        s.append(f'({os.cpu_count()} CPUs available)')
        log.info(' '.join(s))

    def stopping(self):
        log.info('exiting')

    @staticmethod
    def maybe_log_progress(event, done, total, last):
        if done == 0:
            log.info("%s 0%%", event)
            return 0
        elif done == total:
            log.info("%s 100%%", event)
            return 1
        else:
            percent = done/total
            if percent >= 0.25 > last:
                log.info("%s 25%%", event)
                return 0.25
            elif percent >= 0.50 > last:
                log.info("%s 50%%", event)
                return 0.50
            elif percent >= 0.75 > last:
                log.info("%s 75%%", event)
                return 0.75
        return last

    def on_sync_progress(self, event):
        e, data = event["event"], event["data"]
        name, current, total, last = e, data['done'][0], 0, 0
        if not e.endswith("init") and not e.endswith("main") and not  e.endswith("indexes"):
            name = f"{e}#{data['id']}"
        if "total" in data:
            total, last = self.tasks[name] = (data["total"][0], last)
        elif name in self.tasks:
            total, last = self.tasks[name]
        elif total == 0:
            return
        self.tasks[name] = (total, self.maybe_log_progress(name, current, total, last))


class Bar2(Bar):

    def __init__(self, frac, default_len=10, charset=None):
        super().__init__(frac[0], default_len, charset)
        self.frac2 = frac[1]

    def __format__(self, format_spec):
        width = self.default_len
        row1 = (1,)*int(self.frac2 * width * 2)
        row2 = (2,)*int(self.frac * width * 2)
        fill = []
        for one, two, _ in itertools.zip_longest(row1, row2, range(width*2)):
            fill.append((one or 0)+(two or 0))
        bar = []
        for i in range(0, width*2, 2):
            if fill[i] == 1:
                if fill[i+1] == 1:
                    bar.append('▀')
                else:
                    bar.append('▘')
            elif fill[i] == 2:
                if fill[i+1] == 2:
                    bar.append('▄')
                else:
                    bar.append('▖')
            elif fill[i] == 3:
                if fill[i+1] == 1:
                    bar.append('▛')
                elif fill[i+1] == 2:
                    bar.append('▙')
                elif fill[i+1] == 3:
                    bar.append('█')
                else:
                    bar.append('▌')
            else:
                bar.append(' ')
        return ''.join(bar)


class tqdm2(tqdm):  # pylint: disable=invalid-name

    def __init__(self, initial=(0, 0), unit=('it', 'it'), total=(None, None), **kwargs):
        self.n2 = self.last_print_n2 = initial[1]  # pylint: disable=invalid-name
        self.unit2 = unit[1]
        self.total2 = total[1]
        super().__init__(initial=initial[0], unit=unit[0], total=total[0], **kwargs)

    @property
    def format_dict(self):
        d = super().format_dict
        d.update({
            'n2': self.n2,
            'unit2': self.unit2,
            'total2': self.total2,
        })
        return d

    def update(self, n=(1, 1)):
        if self.disable:
            return
        last_last_print_t = self.last_print_t
        self.n2 += n[1]
        super().update(n[0])
        if last_last_print_t != self.last_print_t:
            self.last_print_n2 = self.n2

    @staticmethod
    def format_meter(
        n, total, elapsed, ncols=None, prefix='', ascii=False,  # pylint: disable=redefined-builtin
        unit='it', unit_scale=False, rate=None, bar_format=None,
        postfix=None, unit_divisor=1000, **extra_kwargs
    ):

        # sanity check: total
        if total and n >= (total + 0.5):  # allow float imprecision (#849)
            total = None

        # apply custom scale if necessary
        if unit_scale and unit_scale not in (True, 1):
            if total:
                total *= unit_scale
            n *= unit_scale
            if rate:
                rate *= unit_scale  # by default rate = 1 / self.avg_time
            unit_scale = False

        elapsed_str = tqdm.format_interval(elapsed)

        # if unspecified, attempt to use rate = average speed
        # (we allow manual override since predicting time is an arcane art)
        if rate is None and elapsed:
            rate = n / elapsed
        inv_rate = 1 / rate if rate else None
        format_sizeof = tqdm.format_sizeof
        rate_noinv_fmt = ((format_sizeof(rate) if unit_scale else
                           '{0:5.2f}'.format(rate))
                          if rate else '?') + unit + '/s'
        rate_inv_fmt = ((format_sizeof(inv_rate) if unit_scale else
                         '{0:5.2f}'.format(inv_rate))
                        if inv_rate else '?') + 's/' + unit
        rate_fmt = rate_inv_fmt if inv_rate and inv_rate > 1 else rate_noinv_fmt

        if unit_scale:
            n_fmt = format_sizeof(n, divisor=unit_divisor)
            total_fmt = format_sizeof(total, divisor=unit_divisor) \
                if total is not None else '?'
        else:
            n_fmt = str(n)
            total_fmt = str(total) if total is not None else '?'

        try:
            postfix = ', ' + postfix if postfix else ''
        except TypeError:
            pass

        remaining = (total - n) / rate if rate and total else 0
        remaining_str = tqdm.format_interval(remaining) if rate else '?'

        # format the stats displayed to the left and right sides of the bar
        if prefix:
            # old prefix setup work around
            bool_prefix_colon_already = (prefix[-2:] == ": ")
            l_bar = prefix if bool_prefix_colon_already else prefix + ": "
        else:
            l_bar = ''

        r_bar = '| {0}/{1} [{2}<{3}, {4}{5}]'.format(
            n_fmt, total_fmt, elapsed_str, remaining_str, rate_fmt, postfix)

        # Custom bar formatting
        # Populate a dict with all available progress indicators
        format_dict = dict(
            # slight extension of self.format_dict
            n=n, n_fmt=n_fmt, total=total, total_fmt=total_fmt,
            elapsed=elapsed_str, elapsed_s=elapsed,
            ncols=ncols, desc=prefix or '', unit=unit,
            rate=inv_rate if inv_rate and inv_rate > 1 else rate,
            rate_fmt=rate_fmt, rate_noinv=rate,
            rate_noinv_fmt=rate_noinv_fmt, rate_inv=inv_rate,
            rate_inv_fmt=rate_inv_fmt,
            postfix=postfix, unit_divisor=unit_divisor,
            # plus more useful definitions
            remaining=remaining_str, remaining_s=remaining,
            l_bar=l_bar, r_bar=r_bar,
            **extra_kwargs)

        # total is known: we can predict some stats
        if total:
            n2, total2 = extra_kwargs['n2'], extra_kwargs['total2']  # pylint: disable=invalid-name

            # fractional and percentage progress
            frac = n / total
            frac2 = n2 / total2
            percentage = frac * 100

            l_bar += '{0:3.0f}%|'.format(percentage)

            if ncols == 0:
                return l_bar[:-1] + r_bar[1:]

            format_dict.update(l_bar=l_bar)
            if bar_format:
                format_dict.update(percentage=percentage)

                # auto-remove colon for empty `desc`
                if not prefix:
                    bar_format = bar_format.replace("{desc}: ", '')
            else:
                bar_format = "{l_bar}{bar}{r_bar}"

            full_bar = FormatReplace()
            try:
                nobar = bar_format.format(bar=full_bar, **format_dict)
            except UnicodeEncodeError:
                bar_format = _unicode(bar_format)
                nobar = bar_format.format(bar=full_bar, **format_dict)
            if not full_bar.format_called:
                # no {bar}, we can just format and return
                return nobar

            # Formatting progress bar space available for bar's display
            full_bar = Bar2(
                (frac, frac2),
                max(1, ncols - disp_len(nobar))
                if ncols else 10,
                charset=Bar2.ASCII if ascii is True else ascii or Bar2.UTF)
            if not _is_ascii(full_bar.charset) and _is_ascii(bar_format):
                bar_format = _unicode(bar_format)
            res = bar_format.format(bar=full_bar, **format_dict)
            return disp_trim(res, ncols) if ncols else res

        elif bar_format:
            # user-specified bar_format but no total
            l_bar += '|'
            format_dict.update(l_bar=l_bar, percentage=0)
            full_bar = FormatReplace()
            nobar = bar_format.format(bar=full_bar, **format_dict)
            if not full_bar.format_called:
                return nobar
            full_bar = Bar2(
                (0, 0),
                max(1, ncols - disp_len(nobar))
                if ncols else 10,
                charset=Bar2.BLANK)
            res = bar_format.format(bar=full_bar, **format_dict)
            return disp_trim(res, ncols) if ncols else res
        else:
            # no total: no progressbar, ETA, just progress stats
            return ((prefix + ": ") if prefix else '') + \
                   '{0}{1} [{2}, {3}{4}]'.format(
                       n_fmt, unit, elapsed_str, rate_fmt, postfix)


class Advanced(Basic):

    FORMAT = '{l_bar}{bar}| {n_fmt:>8}/{total_fmt:>8} [{elapsed:>7}<{remaining:>8}, {rate_fmt:>17}]'

    def __init__(self, service: Service):
        super().__init__(service)
        self.bars: Dict[Any, tqdm] = {}
        self.stderr = RedirectOutput('stderr')

    def starting(self):
        self.stderr.capture()
        super().starting()

    def stopping(self):
        for bar in self.bars.values():
            bar.close()
        super().stopping()
        #self.stderr.flush(self.bars['read'].write, True)
        #self.stderr.release()

    def get_or_create_bar(self, name, desc, units, totals, leave=False, bar_format=None, postfix=None, position=None):
        bar = self.bars.get(name)
        if bar is None:
            if len(units) == 2:
                bar = self.bars[name] = tqdm2(
                    desc=desc, unit=units, total=totals,
                    bar_format=bar_format or self.FORMAT, leave=leave,
                    postfix=postfix, position=position
                )
            else:
                bar = self.bars[name] = tqdm(
                    desc=desc, unit=units[0], total=totals[0],
                    bar_format=bar_format or self.FORMAT, leave=leave,
                    postfix=postfix, position=position
                )
        return bar

    def sync_init(self, name, d):
        bar_name = f"{name}#{d['id']}"
        bar = self.bars.get(bar_name)
        if bar is None:
            label = d.get('label', name[-11:])
            self.get_or_create_bar(bar_name, label, d['units'], d['total'], True)
        else:
            if d['done'][0] != -1:
                bar.update(d['done'][0] - bar.n)
            if d['done'][0] == -1 or d['done'][0] == bar.total:
                bar.close()
                self.bars.pop(bar_name)

    def sync_main(self, name, d):
        bar = self.bars.get(name)
        if bar is None:
            label = d.get('label', name[-11:])
            self.get_or_create_bar(name, label, d['units'], d['total'], True)
            #self.last_stats = f"{d['txs']:,d} txs, {d['claims']:,d} claims and {d['supports']:,d} supports"
            #self.get_or_create_bar("read", "├─  blocks read", "blocks", d['blocks'], True)
            #self.get_or_create_bar("save", "└─┬   txs saved", "txs", d['txs'], True)
        else:
            if d['done'] == (-1,)*len(d['done']):
                base_name = name[:name.rindex('.')]
                for child_name, child_bar in self.bars.items():
                    if child_name.startswith(base_name):
                        child_bar.close()
                bar.close()
                self.bars.pop(name)
            else:
                if len(d['done']) == 2:
                    bar.update((d['done'][0]-bar.n, d['done'][1]-bar.n2))
                else:
                    bar.update(d['done'][0]-bar.n)

    def sync_task(self, name, d):
        bar_name = f"{name}#{d['id']}"
        bar = self.bars.get(bar_name)
        if bar is None:
            #assert d['done'][0] == 0
            label = d.get('label', name[-11:])
            self.get_or_create_bar(
                f"{name}#{d['id']}", label, d['units'], d['total'],
                name.split('.')[-1] not in ('insert', 'update', 'file')
            )
        else:
            if d['done'][0] != -1:
                main_bar_name = f"{name[:name.rindex('.')]}.main"
                if len(d['done']) > 1:
                    diff = tuple(a-b for a, b in zip(d['done'], (bar.n, bar.n2)))
                else:
                    diff = d['done'][0] - bar.n
                if main_bar_name != name:
                    main_bar = self.bars.get(main_bar_name)
                    if main_bar and main_bar.unit == bar.unit:
                        main_bar.update(diff)
                bar.update(diff)
            if d['done'][0] == -1 or d['done'][0] == bar.total:
                bar.close()
                self.bars.pop(bar_name)

    def update_other_bars(self, e, d):
        if d['total'] == 0:
            return
        bar = self.bars.get(e)
        if not bar:
            name = (
                ' '.join(e.split('.')[-2:])
                .replace('support', 'suprt')
                .replace('channels', 'chanls')
                .replace('signatures', 'sigs')
            )
            bar = self.get_or_create_bar(e, f"├─ {name:>12}", d['unit'], d['total'], True)
        diff = d['step']-bar.n
        bar.update(diff)
        #if d['step'] == d['total']:
            #bar.close()

    def on_sync_progress(self, event):
        e, d = event['event'], event.get('data', {})
        if e.endswith(".init"):
            self.sync_init(e, d)
        elif e.endswith(".main"):
            self.sync_main(e, d)
        else:
            self.sync_task(e, d)

#        if e.endswith("sync.start"):
#            self.sync_start(d)
#            self.stderr.flush(self.bars['read'].write)
#        elif e.endswith("sync.complete"):
#            self.stderr.flush(self.bars['read'].write, True)
#            self.sync_complete()
#        else:
#            self.stderr.flush(self.bars['read'].write)
#            self.update_progress(e, d)