From 2fa2269cc774352fdf4bb6fcc1a17501540ed81b Mon Sep 17 00:00:00 2001 From: Akinwale Ariwodola <akinwale@gmail.com> Date: Thu, 21 Jun 2018 23:49:22 +0100 Subject: [PATCH] add download progress and is downloading flag to daemon status (#1266) --- CHANGELOG.md | 2 +- lbrynet/core/Wallet.py | 26 ++++++++++++++++++++++++-- lbrynet/daemon/Daemon.py | 21 ++++++++++++++++----- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7334a4dff..7e939707a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,7 @@ at anytime. * ### Added - * + * added blockchain_headers download progress percentage to daemon status call * ### Removed diff --git a/lbrynet/core/Wallet.py b/lbrynet/core/Wallet.py index 0b71ed59d..8aee6a119 100644 --- a/lbrynet/core/Wallet.py +++ b/lbrynet/core/Wallet.py @@ -2,6 +2,7 @@ import os from collections import defaultdict, deque import datetime import logging +import math from decimal import Decimal import treq @@ -86,6 +87,10 @@ class Wallet(object): self.max_expected_payment_time = datetime.timedelta(minutes=3) self.stopped = True + # blockchain_headers progress + self.headers_download = False + self.headers_progress_percent = 0 + self.manage_running = False self._manage_count = 0 self._balance_refresh_time = 3 @@ -105,14 +110,24 @@ class Wallet(object): elif final_size_after_download and not final_size_after_download % HEADER_SIZE: s3_height = (final_size_after_download / HEADER_SIZE) - 1 local_height = self.local_header_file_height() + if s3_height > local_height: + self.headers_download = True + + def collector(data, h_file): + h_file.write(data) + local_size = float(h_file.tell()) + final_size = float(final_size_after_download) + self.headers_progress_percent = math.ceil(local_size / final_size * 100) + self.headers_download = self.headers_progress_percent < 100 + if local_header_size: log.info("Resuming download of %i bytes from s3", response.length) with open(os.path.join(self.config.path, "blockchain_headers"), "a+b") as headers_file: - yield treq.collect(response, headers_file.write) + yield treq.collect(response, lambda d: collector(d, headers_file)) else: with open(os.path.join(self.config.path, "blockchain_headers"), "wb") as headers_file: - yield treq.collect(response, headers_file.write) + yield treq.collect(response, lambda d: collector(d, headers_file)) log.info("fetched headers from s3 (s3 height: %i), now verifying integrity after download.", s3_height) self._check_header_file_integrity() else: @@ -129,6 +144,12 @@ class Wallet(object): return os.stat(headers_path).st_size return 0 + def is_downloading_headers(self): + return self.headers_download + + def get_headers_progress_percent(self): + return self.headers_progress_percent if (self.headers_download or self.headers_progress_percent > 0) else 0 + @defer.inlineCallbacks def get_remote_height(self, server, port): connected = defer.Deferred() @@ -192,6 +213,7 @@ class Wallet(object): try: yield self.fetch_headers_from_s3() except Exception as err: + self.headers_download = False log.error("failed to fetch headers from s3: %s", err) log.info("Starting wallet.") yield self._start() diff --git a/lbrynet/daemon/Daemon.py b/lbrynet/daemon/Daemon.py index db2726208..cc3be2c4f 100644 --- a/lbrynet/daemon/Daemon.py +++ b/lbrynet/daemon/Daemon.py @@ -1,5 +1,6 @@ import binascii import logging.handlers +import math import mimetypes import os import base58 @@ -1053,6 +1054,8 @@ class Daemon(AuthJSONRPCServer): 'blocks': local blockchain height, 'blocks_behind': remote_height - local_height, 'best_blockhash': block hash of most recent block, + 'headers_download_progress': the download progress of the blockchain_headers file, + 'is_downloading_headers': bool, flag to indicate if the blockchain_headers file is being downloaded }, 'wallet_is_encrypted': bool, @@ -1067,12 +1070,18 @@ class Daemon(AuthJSONRPCServer): """ # on startup, the wallet or network won't be available but we still need this call to work - has_wallet = self.session and self.session.wallet and self.session.wallet.network - local_height = self.session.wallet.network.get_local_height() if has_wallet else 0 - remote_height = self.session.wallet.network.get_server_height() if has_wallet else 0 - best_hash = (yield self.session.wallet.get_best_blockhash()) if has_wallet else None - wallet_is_encrypted = has_wallet and self.session.wallet.wallet and \ + has_wallet = self.session and self.session.wallet + downloading_headers = self.session.wallet.is_downloading_headers() if has_wallet else False + has_wallet_with_network = has_wallet and self.session.wallet.network + local_height = self.session.wallet.network.get_local_height() if has_wallet_with_network else 0 + remote_height = self.session.wallet.network.get_server_height() if has_wallet_with_network else 0 + best_hash = (yield self.session.wallet.get_best_blockhash()) if has_wallet_with_network else None + wallet_is_encrypted = has_wallet_with_network and self.session.wallet.wallet and \ self.session.wallet.wallet.use_encryption + headers_progress_percent = self.session.wallet.get_headers_progress_percent() if has_wallet else 0 + headers_download_progress = headers_progress_percent \ + if downloading_headers or headers_progress_percent == 100 \ + else math.ceil((local_height / float(remote_height)) * 100) if remote_height else 0 response = { 'lbry_id': base58.b58encode(self.node_id), @@ -1097,6 +1106,8 @@ class Daemon(AuthJSONRPCServer): 'blocks': local_height, 'blocks_behind': remote_height - local_height, 'best_blockhash': best_hash, + 'headers_download_progress': headers_download_progress, + 'is_downloading_headers': downloading_headers, } } if session_status: