From 5f49827bd27eda5372a72d5796437c5526d0bb6f Mon Sep 17 00:00:00 2001 From: Jack Date: Sun, 17 Apr 2016 19:54:04 -0400 Subject: [PATCH 1/4] fix lagging indicator and improve diagnostic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit -make is_lagging more meaningful - it is set to true after 90 seconds with no progress, this is to stop slow but steady catchups from triggering it -prevent situation where repeated shutdowns before wallet catchup results in never receiving first run credits -fix settings to write new defaults that aren’t already in the configuration file -report log of startup sequence if upload_log set to true -redirect /view?name=lbry to the main UI page. This is to make the ui accessible from a lbry:// link on linux --- lbrynet/__init__.py | 2 +- lbrynet/core/LBRYcrdWallet.py | 14 ++- lbrynet/lbrynet_daemon/LBRYDaemon.py | 149 +++++++++++++++++++-------- 3 files changed, 116 insertions(+), 49 deletions(-) diff --git a/lbrynet/__init__.py b/lbrynet/__init__.py index 993518cab..8b1a36331 100644 --- a/lbrynet/__init__.py +++ b/lbrynet/__init__.py @@ -4,5 +4,5 @@ import logging logging.getLogger(__name__).addHandler(logging.NullHandler()) -version = (0, 2, 1) +version = (0, 2, 2) __version__ = ".".join([str(x) for x in version]) diff --git a/lbrynet/core/LBRYcrdWallet.py b/lbrynet/core/LBRYcrdWallet.py index 895dbfa49..bdf35af6b 100644 --- a/lbrynet/core/LBRYcrdWallet.py +++ b/lbrynet/core/LBRYcrdWallet.py @@ -904,6 +904,7 @@ class LBRYumWallet(LBRYWallet): self._start_check = None self._catch_up_check = None self._caught_up_counter = 0 + self._lag_counter = 0 self.blocks_behind_alert = 0 self.catchup_progress = 0 self.max_behind = 0 @@ -997,8 +998,18 @@ class LBRYumWallet(LBRYWallet): self._catch_up_check.stop() self._catch_up_check = None blockchain_caught_d.callback(True) + elif remote_height != 0: + past_blocks_behind = self.blocks_behind_alert self.blocks_behind_alert = remote_height - local_height + if self.blocks_behind_alert < past_blocks_behind: + self._lag_counter = 0 + self.is_lagging = False + else: + self._lag_counter += 1 + if self._lag_counter >= 900: + self.is_lagging = True + if self.blocks_behind_alert > self.max_behind: self.max_behind = self.blocks_behind_alert self.catchup_progress = int(100 * (self.blocks_behind_alert / (5 + self.max_behind))) @@ -1006,9 +1017,6 @@ class LBRYumWallet(LBRYWallet): alert.info('Catching up to the blockchain...showing blocks left...') if self._caught_up_counter % 30 == 0: alert.info('%d...', (remote_height - local_height)) - alert.info("Catching up: " + str(self.catchup_progress) + "%") - if self._caught_up_counter >= 600: - self.is_lagging = True self._caught_up_counter += 1 diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 95340be0d..ff0bfd87a 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -58,6 +58,13 @@ if not os.path.isdir(log_dir): LOG_FILENAME = os.path.join(log_dir, 'lbrynet-daemon.log') +if os.path.isfile(LOG_FILENAME): + f = open(LOG_FILENAME, 'r') + PREVIOUS_LOG = len(f.read()) + f.close() +else: + PREVIOUS_LOG = 0 + log = logging.getLogger(__name__) handler = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=262144, backupCount=5) log.addHandler(handler) @@ -121,6 +128,10 @@ class LBRYDaemon(jsonrpc.JSONRPC): "lbryum_version: ": lbryum_version, "ui_version": self.ui_version, } + try: + self.platform_info['ip'] = json.load(urlopen('http://jsonip.com'))['ip'] + except: + self.platform_info['ip'] = "Could not determine" if os.name == "nt": from lbrynet.winhelpers.knownpaths import get_path, FOLDERID, UserHandle @@ -185,28 +196,8 @@ class LBRYDaemon(jsonrpc.JSONRPC): 'dht_node_port': 4444, 'use_upnp': True, 'start_lbrycrdd': True, + 'requested_first_run_credits': False } - if os.path.isfile(self.daemon_conf): - #load given settings, set missing settings to defaults - temp_settings = self.default_settings - f = open(self.daemon_conf, "r") - s = json.loads(f.read()) - f.close() - for k in temp_settings.keys(): - if k in s.keys(): - if type(temp_settings[k]) == type(s[k]): - temp_settings[k] = s[k] - self.__dict__[k] = s[k] - f = open(self.daemon_conf, "w") - f.write(json.dumps(temp_settings)) - f.close() - else: - log.info("Writing default settings: " + json.dumps(self.default_settings) + " --> " + str(self.daemon_conf)) - f = open(self.daemon_conf, "w") - f.write(json.dumps(self.default_settings)) - f.close() - for k in self.default_settings.keys(): - self.__dict__[k] = self.default_settings[k] def render(self, request): request.content.seek(0, 0) @@ -290,8 +281,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): return version def _get_lbrynet_version(): - r = urlopen("https://raw.githubusercontent.com/lbryio/lbry/master/lbrynet/__init__.py").read().split( - '\n') + r = urlopen("https://raw.githubusercontent.com/lbryio/lbry/master/lbrynet/__init__.py").read().split('\n') vs = next(i for i in r if 'version =' in i).split("=")[1].replace(" ", "") vt = tuple(int(x) for x in vs[1:-1].split(',')) vr = ".".join([str(x) for x in vt]) @@ -308,10 +298,22 @@ class LBRYDaemon(jsonrpc.JSONRPC): return defer.succeed(None) def _announce_startup(): - self.announced_startup = True - self.startup_status = STARTUP_STAGES[5] - log.info("[" + str(datetime.now()) + "] Started lbrynet-daemon") - return defer.succeed(None) + def _announce(): + self.announced_startup = True + self.startup_status = STARTUP_STAGES[5] + log.info("[" + str(datetime.now()) + "] Started lbrynet-daemon") + + if self.first_run: + d = self._upload_log(name_prefix="fr") + else: + d = self._upload_log(exclude_previous=True, name_prefix="start") + + if float(self.session.wallet.wallet_balance) == 0.0: + d.addCallback(lambda _: self._check_first_run()) + d.addCallback(self._show_first_run_result) + + d.addCallback(lambda _: _announce()) + return d log.info("[" + str(datetime.now()) + "] Starting lbrynet-daemon") @@ -343,13 +345,30 @@ class LBRYDaemon(jsonrpc.JSONRPC): def _load_daemon_conf(): if os.path.isfile(self.daemon_conf): - return json.loads(open(self.daemon_conf, "r").read()) + f = open(self.daemon_conf, "r") + loaded_settings = json.loads(f.read()) + f.close() + missing_settings = {} + for k in self.default_settings.keys(): + if k not in loaded_settings.keys(): + missing_settings[k] = self.default_settings[k] + if missing_settings != {}: + for k in missing_settings.keys(): + log.info("Adding missing setting: " + k + " with default value: " + str(missing_settings[k])) + loaded_settings[k] = missing_settings[k] + f = open(self.daemon_conf, "w") + f.write(json.dumps(loaded_settings)) + f.close() + rsettings = loaded_settings else: log.info("Writing default settings : " + json.dumps(self.default_settings) + " --> " + str(self.daemon_conf)) f = open(self.daemon_conf, "w") f.write(json.dumps(self.default_settings)) f.close() - return self.default_settings + rsettings = self.default_settings + for k in rsettings.keys(): + self.__dict__[k] = rsettings[k] + return rsettings d = _log_platform() d.addCallback(lambda _: _load_daemon_conf()) @@ -434,21 +453,36 @@ class LBRYDaemon(jsonrpc.JSONRPC): dl.addCallback(_set_query_handlers) return dl - def _upload_log(self): + def _upload_log(self, name_prefix=None, exclude_previous=False): + if name_prefix: + name_prefix = name_prefix + "-" + platform.system() + else: + name_prefix = platform.system() + if self.session_settings['upload_log']: LOG_URL = "https://lbry.io/log-upload" - f = open(self.log_file, "r") + if exclude_previous: + f = open(self.log_file, "r") + f.seek(PREVIOUS_LOG) + log_contents = f.read() + f.close() + else: + f = open(self.log_file, "r") + log_contents = f.read() + f.close() t = datetime.now() - log_name = base58.b58encode(self.lbryid)[:20] + "-" + str(t.month) + "-" + str(t.day) + "-" + str(t.year) + "-" + str(t.hour) + "-" + str(t.minute) - params = {'name': log_name, 'log': f.read()} - f.close() + log_name = name_prefix + "-" + base58.b58encode(self.lbryid)[:20] + "-" + str(t.month) + "-" + str(t.day) + "-" + str(t.year) + "-" + str(t.hour) + "-" + str(t.minute) + params = {'name': log_name, 'log': log_contents} + requests.post(LOG_URL, params) return defer.succeed(None) def _shutdown(self): log.info("Closing lbrynet session") - d = self._upload_log() + log.info("Status at time of shutdown: " + self.startup_status[0]) + + d = self._upload_log(name_prefix="close", exclude_previous=False if self.first_run else True) d.addCallback(lambda _: self._stop_server()) d.addErrback(lambda err: log.info("Bad server shutdown: " + err.getTraceback())) if self.session is not None: @@ -638,8 +672,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): dl.addCallback(combine_results) dl.addCallback(create_session) dl.addCallback(lambda _: self.session.setup()) - dl.addCallback(lambda _: self._check_first_run()) - dl.addCallback(self._show_first_run_result) + return dl def _check_first_run(self): @@ -649,18 +682,20 @@ class LBRYDaemon(jsonrpc.JSONRPC): return 0.0 d = self.session.wallet.is_first_run() - - d.addCallback(lambda is_first_run: self._do_first_run() if is_first_run else _set_first_run_false()) + d.addCallback(lambda is_first_run: self._do_first_run() if is_first_run or not self.requested_first_run_credits + else _set_first_run_false()) return d def _do_first_run(self): - self.first_run = True - d = self.session.wallet.get_new_address() - def send_request(url, data): log.info("Requesting first run credits") r = requests.post(url, json=data) if r.status_code == 200: + self.requested_first_run_credits = True + self.session_settings['requested_first_run_credits'] = True + f = open(self.daemon_conf, "w") + f.write(json.dumps(self.session_settings)) + f.close() return r.json()['credits_sent'] return 0.0 @@ -675,7 +710,10 @@ class LBRYDaemon(jsonrpc.JSONRPC): d.addErrback(log_error) return d + self.first_run = True + d = self.session.wallet.get_new_address() d.addCallback(request_credits) + return d def _show_first_run_result(self, credits_received): @@ -1560,6 +1598,24 @@ class LBRYDaemon(jsonrpc.JSONRPC): return _check_version() + def jsonrpc_upload_log(self, p=None): + if p: + if 'name_prefix' in p.keys(): + prefix = p['name_prefix'] + '_api' + else: + prefix = None + if 'exclude_previous' in p.keys: + exclude_previous = p['exclude_previous'] + else: + exclude_previous = True + else: + prefix = "api" + exclude_previous = True + + d = self._upload_log(name_prefix=prefix, exclude_previous=exclude_previous) + d.addCallback(lambda _: self._render_response(True, OK_CODE)) + return d + class LBRYDaemonCommandHandler(object): def __init__(self, command): @@ -1600,9 +1656,12 @@ class LBRYFileRender(resource.Resource): def render_GET(self, request): if 'name' in request.args.keys(): api = jsonrpc.Proxy(API_CONNECTION_STRING) - d = api.callRemote("get", {'name': request.args['name'][0]}) - d.addCallback(lambda results: static.File(results['path']).render_GET(request)) - + if request.args['name'][0] != 'lbry': + d = api.callRemote("get", {'name': request.args['name'][0]}) + d.addCallback(lambda results: static.File(results['path']).render_GET(request)) + else: + request.redirect(UI_ADDRESS) + request.finish() return server.NOT_DONE_YET else: return server.failure From 7b9d84f528f63a4c6d012f70b0c13430dafd6a5b Mon Sep 17 00:00:00 2001 From: Jack Date: Sun, 17 Apr 2016 20:11:10 -0400 Subject: [PATCH 2/4] use master branch for ui in .deb --- packaging/ubuntu/lbry | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/ubuntu/lbry b/packaging/ubuntu/lbry index 0c6edeeeb..a4aed0ff8 100755 --- a/packaging/ubuntu/lbry +++ b/packaging/ubuntu/lbry @@ -17,7 +17,7 @@ urlencode() { DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" if [ -z "$(pgrep lbrynet-daemon)" ]; then echo "running lbrynet-daemon..." - $DIR/lbrynet-daemon --branch=settings-page & + $DIR/lbrynet-daemon & sleep 3 # let the daemon load before connecting fi From 99b3cdd45ddf88cede391c01497ec6d5aee3c5e1 Mon Sep 17 00:00:00 2001 From: Jack Date: Sun, 17 Apr 2016 20:16:35 -0400 Subject: [PATCH 3/4] default on launching ui if daemon already running when command is called -also update deb version --- lbrynet/lbrynet_daemon/LBRYDaemonControl.py | 6 ++++-- packaging/ubuntu/lbry.desktop | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py index 5fc6f98ff..d04133144 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py @@ -74,9 +74,13 @@ def start(): parser.add_argument('--no-launch', dest='launchui', action="store_false") parser.set_defaults(launchui=True) + args = parser.parse_args() + try: JSONRPCProxy.from_url(API_CONNECTION_STRING).is_running() log.info("lbrynet-daemon is already running") + if args.launchui: + webbrowser.open(UI_ADDRESS) return except: pass @@ -88,8 +92,6 @@ def start(): print "JSONRPC API is available at " + API_CONNECTION_STRING print "To quit press ctrl-c or call 'stop' via the API" - args = parser.parse_args() - if args.branch == "HEAD": GIT_CMD_STRING = "git ls-remote https://github.com/lbryio/lbry-web-ui.git | grep %s | cut -f 1" % args.branch DIST_URL = "https://raw.githubusercontent.com/lbryio/lbry-web-ui/master/dist.zip" diff --git a/packaging/ubuntu/lbry.desktop b/packaging/ubuntu/lbry.desktop index f034b97c6..d5c029ec6 100644 --- a/packaging/ubuntu/lbry.desktop +++ b/packaging/ubuntu/lbry.desktop @@ -1,5 +1,5 @@ [Desktop Entry] -Version=0.2.1 +Version=0.2.2 Name=LBRY # Only KDE 4 seems to use GenericName, so we reuse the KDE strings. # From Ubuntu's language-pack-kde-XX-base packages, version 9.04-20090413. From 7a622986d669002293086e73c8c84c9c282927d7 Mon Sep 17 00:00:00 2001 From: Jack Date: Sun, 17 Apr 2016 22:56:51 -0400 Subject: [PATCH 4/4] remove lbryum version from setup.py version number made dependency link not work --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1863dad82..e4a9432f0 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ console_scripts = ['lbrynet-console = lbrynet.lbrynet_console.LBRYConsole:launch requires = ['pycrypto', 'twisted', 'miniupnpc', 'yapsy', 'seccure', 'python-bitcoinrpc==0.1', 'txJSON-RPC', 'requests>=2.4.2', 'unqlite==0.2.0', - 'leveldb', 'lbryum>=2.6.0.1', 'jsonrpc', 'simplejson', 'appdirs', 'six==1.9.0', 'base58'] + 'leveldb', 'lbryum', 'jsonrpc', 'simplejson', 'appdirs', 'six==1.9.0', 'base58'] gui_data_files = ['close2.gif', 'lbry-dark-242x80.gif', 'lbry-dark-icon.xbm', 'lbry-dark-icon.ico', 'drop_down.gif', 'show_options.gif', 'hide_options.gif', 'lbry.conf', 'lbry.png']