From e5e25012ffc7ce2c86940b955208342a7e5a55ff Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 20 Apr 2016 22:02:52 -0400 Subject: [PATCH] move daemon server stuff into its own class --- lbrynet/lbrynet_daemon/LBRYDaemon.py | 87 +------- lbrynet/lbrynet_daemon/LBRYDaemonControl.py | 125 ++++-------- lbrynet/lbrynet_daemon/LBRYDaemonServer.py | 210 ++++++++++++++++++++ 3 files changed, 248 insertions(+), 174 deletions(-) create mode 100644 lbrynet/lbrynet_daemon/LBRYDaemonServer.py diff --git a/lbrynet/lbrynet_daemon/LBRYDaemon.py b/lbrynet/lbrynet_daemon/LBRYDaemon.py index 7d0645404..c30b5b594 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemon.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemon.py @@ -3,17 +3,13 @@ import os import sys import simplejson as json import binascii -import subprocess -import logging import logging.handlers import requests -import base64 import base58 import platform -import json import socket -from twisted.web import server, resource, static +from twisted.web import server from twisted.internet import defer, threads, error, reactor from twisted.internet.task import LoopingCall from txjsonrpc import jsonrpclib @@ -31,7 +27,6 @@ from lbrynet.core.PaymentRateManager import PaymentRateManager from lbrynet.core.server.BlobAvailabilityHandler import BlobAvailabilityHandlerFactory from lbrynet.core.server.BlobRequestHandler import BlobRequestHandlerFactory from lbrynet.core.server.ServerProtocol import ServerProtocolFactory -from lbrynet.lbrynet_console.ControlHandlers import get_time_behind_blockchain from lbrynet.core.Error import UnknownNameError from lbrynet.lbryfile.StreamDescriptor import LBRYFileStreamType from lbrynet.lbryfile.client.LBRYFileDownloader import LBRYFileSaverFactory, LBRYFileOpenerFactory @@ -40,7 +35,8 @@ from lbrynet.lbrynet_daemon.LBRYDownloader import GetStream, FetcherDaemon from lbrynet.lbrynet_daemon.LBRYPublisher import Publisher from lbrynet.core.utils import generate_id from lbrynet.lbrynet_console.LBRYSettings import LBRYSettings -from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, DEFAULT_MAX_SEARCH_RESULTS, KNOWN_DHT_NODES, DEFAULT_MAX_KEY_FEE +from lbrynet.conf import MIN_BLOB_DATA_PAYMENT_RATE, DEFAULT_MAX_SEARCH_RESULTS, KNOWN_DHT_NODES, DEFAULT_MAX_KEY_FEE, \ + DEFAULT_WALLET, API_INTERFACE from lbrynet.conf import API_CONNECTION_STRING, API_PORT, API_ADDRESS, DEFAULT_TIMEOUT, UI_ADDRESS from lbrynet.core.StreamDescriptor import StreamDescriptorIdentifier, download_sd_blob from lbrynet.core.Session import LBRYSession @@ -117,7 +113,7 @@ class LBRYDaemon(jsonrpc.JSONRPC): isLeaf = True - def __init__(self, ui_version_info, wallet_type="lbryum"): + def __init__(self, ui_version_info, wallet_type=DEFAULT_WALLET): jsonrpc.JSONRPC.__init__(self) reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown) @@ -1693,8 +1689,6 @@ class LBRYDaemon(jsonrpc.JSONRPC): true/false, true meaning that there is a new version available """ - - def _check_version(): if (lbrynet_version >= self.git_lbrynet_version) and (lbryum_version >= self.git_lbryum_version): log.info("[" + str(datetime.now()) + "] Up to date") @@ -1738,76 +1732,3 @@ class LBRYDaemon(jsonrpc.JSONRPC): d = self._upload_log(name_prefix=prefix, exclude_previous=exclude_previous, force=force) d.addCallback(lambda _: self._render_response(True, OK_CODE)) return d - - -class LBRYDaemonCommandHandler(object): - def __init__(self, command): - self._api = jsonrpc.Proxy(API_CONNECTION_STRING) - self.command = command - - def run(self, params=None): - if params: - d = self._api.callRemote(self.command, params) - else: - d = self._api.callRemote(self.command) - return d - - -class LBRYindex(resource.Resource): - def __init__(self, ui_dir): - resource.Resource.__init__(self) - self.ui_dir = ui_dir - - isLeaf = False - - def _delayed_render(self, request, results): - request.write(str(results)) - request.finish() - - def getChild(self, name, request): - if name == '': - return self - return resource.Resource.getChild(self, name, request) - - def render_GET(self, request): - return static.File(os.path.join(self.ui_dir, "index.html")).render_GET(request) - - -class LBRYFileRender(resource.Resource): - isLeaf = False - - def render_GET(self, request): - if 'name' in request.args.keys(): - api = jsonrpc.Proxy(API_CONNECTION_STRING) - if request.args['name'][0] != 'lbry': - d = api.callRemote("get", {'name': request.args['name'][0]}) - d.addCallback(lambda results: static.File(results['path']).render_GET(request)) - else: - request.redirect(UI_ADDRESS) - request.finish() - return server.NOT_DONE_YET - else: - return server.failure - -class LBRYBugReport(resource.Resource): - isLeaf = False - - def _delayed_render(self, request, results): - request.write(results) - request.finish() - - def render_GET(self, request): - return '
' \ - '
Please describe the problem you experienced and any information you think might be useful to us. Links to screenshots are great!
' \ - '' \ - '' \ - '
' - - def render_POST(self, request): - msg = request.args["message"][0] - log.info("User submitted error report: " + str(msg)) - api = jsonrpc.Proxy(API_CONNECTION_STRING) - d = api.callRemote("upload_log", {'name_prefix': 'report', 'exclude_previous': False, 'force': True}) - d.addCallback(lambda _: self._delayed_render(request, "Your bug report is greatly appreciated! Click here to return to LBRY")) - - return server.NOT_DONE_YET \ No newline at end of file diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py index 3f7b7e411..0321de4df 100644 --- a/lbrynet/lbrynet_daemon/LBRYDaemonControl.py +++ b/lbrynet/lbrynet_daemon/LBRYDaemonControl.py @@ -1,25 +1,22 @@ import argparse import logging import logging.handlers -import subprocess import os -import shutil import webbrowser import sys import socket +import subprocess +import platform -from StringIO import StringIO -from zipfile import ZipFile -from urllib import urlopen -from datetime import datetime from appdirs import user_data_dir -from twisted.web import server, static +from twisted.web import server from twisted.internet import reactor, defer from jsonrpc.proxy import JSONRPCProxy -from lbrynet.lbrynet_daemon.LBRYDaemon import LBRYDaemon, LBRYindex, LBRYFileRender, LBRYBugReport +from lbrynet.lbrynet_daemon.LBRYDaemonServer import LBRYDaemonServer from lbrynet.conf import API_CONNECTION_STRING, API_INTERFACE, API_ADDRESS, API_PORT, DEFAULT_WALLET, UI_ADDRESS + if sys.platform != "darwin": log_dir = os.path.join(os.path.expanduser("~"), ".lbrynet") else: @@ -34,6 +31,7 @@ handler = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=2097152, b log.addHandler(handler) log.setLevel(logging.INFO) + REMOTE_SERVER = "www.google.com" @@ -46,11 +44,24 @@ def test_internet_connection(): return False +def prompt_for_xcode_if_needed(): + t = subprocess.check_output("git ls-remote https://github.com/lbryio/lbry-web-ui.git | grep HEAD | cut -f 1", shell=True) + if not t: + if platform.system().lower() != "darwin": + print "Please install git" + sys.exit(0) + else: + print "You should have been alerted to install xcode command line tools, please do so and then start lbry" + sys.exit(0) + + def stop(): def _disp_shutdown(): + print "Shutting down lbrynet-daemon from command line" log.info("Shutting down lbrynet-daemon from command line") def _disp_not_running(): + print "Attempt to shut down lbrynet-daemon from command line when daemon isn't running" log.info("Attempt to shut down lbrynet-daemon from command line when daemon isn't running") d = defer.Deferred(None) @@ -67,7 +78,7 @@ def start(): default=DEFAULT_WALLET) parser.add_argument("--ui", help="path to custom UI folder", - default="") + default=None) parser.add_argument("--branch", help="Branch of lbry-web-ui repo to use, defaults on HEAD", default="HEAD") @@ -85,106 +96,38 @@ def start(): try: JSONRPCProxy.from_url(API_CONNECTION_STRING).is_running() log.info("lbrynet-daemon is already running") + if not args.logtoconsole: + print "lbrynet-daemon is already running" if args.launchui: webbrowser.open(UI_ADDRESS) return except: pass + prompt_for_xcode_if_needed() + log.info("Starting lbrynet-daemon from command line") if not args.logtoconsole and not args.quiet: print "Starting lbrynet-daemon from command line" print "To view activity, view the log file here: " + LOG_FILENAME - print "Web UI is available at http://%s:%i" %(API_INTERFACE, API_PORT) + print "Web UI is available at http://%s:%i" % (API_INTERFACE, API_PORT) print "JSONRPC API is available at " + API_CONNECTION_STRING print "To quit press ctrl-c or call 'stop' via the API" - if args.branch == "HEAD": - GIT_CMD_STRING = "git ls-remote https://github.com/lbryio/lbry-web-ui.git | grep %s | cut -f 1" % args.branch - DIST_URL = "https://raw.githubusercontent.com/lbryio/lbry-web-ui/master/dist.zip" - else: - log.info("Using UI branch: " + args.branch) - GIT_CMD_STRING = "git ls-remote https://github.com/lbryio/lbry-web-ui.git | grep refs/heads/%s | cut -f 1" % args.branch - DIST_URL = "https://raw.githubusercontent.com/lbryio/lbry-web-ui/%s/dist.zip" % args.branch - - def getui(ui_dir=None): - if ui_dir: - if os.path.isdir(ui_dir): - log.info("Using user specified UI directory: " + str(ui_dir)) - ui_version_info = "user-specified" - return defer.succeed([ui_dir, ui_version_info]) - else: - log.info("User specified UI directory doesn't exist: " + str(ui_dir)) - - def download_ui(dest_dir, ui_version): - url = urlopen(DIST_URL) - z = ZipFile(StringIO(url.read())) - names = [i for i in z.namelist() if '.DS_Store' not in i and '__MACOSX' not in i] - z.extractall(dest_dir, members=names) - return defer.succeed([dest_dir, ui_version]) - - data_dir = user_data_dir("LBRY") - version_dir = os.path.join(data_dir, "ui_version_history") - - git_version = subprocess.check_output(GIT_CMD_STRING, shell=True) - if not git_version: - log.info("You should have been notified to install xcode command line tools, once it's installed you can start LBRY") - print "You should have been notified to install xcode command line tools, once it's installed you can start LBRY" - sys.exit(0) - - ui_version_info = git_version - - if not os.path.isdir(data_dir): - os.mkdir(data_dir) - - if not os.path.isdir(os.path.join(data_dir, "ui_version_history")): - os.mkdir(version_dir) - - if not os.path.isfile(os.path.join(version_dir, git_version)): - f = open(os.path.join(version_dir, git_version), "w") - version_message = "[" + str(datetime.now()) + "] Updating UI --> " + git_version - f.write(version_message) - f.close() - log.info(version_message) - - if os.path.isdir(os.path.join(data_dir, "lbry-web-ui")): - shutil.rmtree(os.path.join(data_dir, "lbry-web-ui")) - else: - version_message = "[" + str(datetime.now()) + "] UI version " + git_version + " up to date" - log.info(version_message) - - if os.path.isdir(os.path.join(data_dir, "lbry-web-ui")): - return defer.succeed([os.path.join(data_dir, "lbry-web-ui"), ui_version_info]) - else: - return download_ui(os.path.join(data_dir, "lbry-web-ui"), ui_version_info) - - def setupserver(ui_dir, ui_version): - root = LBRYindex(ui_dir) - root.putChild("css", static.File(os.path.join(ui_dir, "css"))) - root.putChild("font", static.File(os.path.join(ui_dir, "font"))) - root.putChild("img", static.File(os.path.join(ui_dir, "img"))) - root.putChild("js", static.File(os.path.join(ui_dir, "js"))) - root.putChild("view", LBRYFileRender()) - root.putChild("report", LBRYBugReport()) - return defer.succeed([root, ui_version]) - - def setupapi(root, wallet, ui_version): - daemon = LBRYDaemon(ui_version, wallet_type=wallet) - root.putChild(API_ADDRESS, daemon) - reactor.listenTCP(API_PORT, server.Site(root), interface=API_INTERFACE) - return daemon.setup() - if test_internet_connection(): - d = getui(args.ui) - d.addCallback(lambda r: setupserver(r[0], r[1])) - d.addCallback(lambda r: setupapi(r[0], args.wallet, r[1])) - if args.launchui: - d.addCallback(lambda _: webbrowser.open(UI_ADDRESS)) + lbry = LBRYDaemonServer() + + d = lbry.start(branch=args.branch, user_specified=args.ui) + d.addCallback(lambda _: webbrowser.open(UI_ADDRESS)) + + reactor.listenTCP(API_PORT, server.Site(lbry.root), interface=API_INTERFACE) reactor.run() + if not args.logtoconsole and not args.quiet: print "\nClosing lbrynet-daemon" else: log.info("Not connected to internet, unable to start") - print "Not connected to internet, unable to start" + if not args.logtoconsole: + print "Not connected to internet, unable to start" return diff --git a/lbrynet/lbrynet_daemon/LBRYDaemonServer.py b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py new file mode 100644 index 000000000..deb23e51d --- /dev/null +++ b/lbrynet/lbrynet_daemon/LBRYDaemonServer.py @@ -0,0 +1,210 @@ +import logging +import subprocess +import os +import shutil +import jsonrpc +import json + +from StringIO import StringIO +from zipfile import ZipFile +from urllib import urlopen +from datetime import datetime +from appdirs import user_data_dir +from twisted.web import server, static, resource +from twisted.internet import defer + +from lbrynet.lbrynet_daemon.LBRYDaemon import LBRYDaemon +from lbrynet.conf import API_CONNECTION_STRING, API_ADDRESS, DEFAULT_WALLET, UI_ADDRESS + +log = logging.getLogger(__name__) + +data_dir = user_data_dir("LBRY") +if not os.path.isdir(data_dir): + os.mkdir(data_dir) +version_dir = os.path.join(data_dir, "ui_version_history") +if not os.path.isdir(version_dir): + os.mkdir(version_dir) + +version_log = logging.getLogger("lbry_version") +version_log.addHandler(logging.FileHandler(os.path.join(version_dir, "lbry_version.log"))) +version_log.setLevel(logging.INFO) + + +class LBRYindex(resource.Resource): + def __init__(self, ui_dir): + resource.Resource.__init__(self) + self.ui_dir = ui_dir + + isLeaf = False + + def _delayed_render(self, request, results): + request.write(str(results)) + request.finish() + + def getChild(self, name, request): + if name == '': + return self + return resource.Resource.getChild(self, name, request) + + def render_GET(self, request): + return static.File(os.path.join(self.ui_dir, "index.html")).render_GET(request) + + +class LBRYFileRender(resource.Resource): + isLeaf = False + + def render_GET(self, request): + if 'name' in request.args.keys(): + api = jsonrpc.Proxy(API_CONNECTION_STRING) + if request.args['name'][0] != 'lbry': + d = api.callRemote("get", {'name': request.args['name'][0]}) + d.addCallback(lambda results: static.File(results['path']).render_GET(request)) + else: + request.redirect(UI_ADDRESS) + request.finish() + return server.NOT_DONE_YET + else: + return server.failure + + +class LBRYBugReport(resource.Resource): + isLeaf = False + + def _delayed_render(self, request, results): + request.write(results) + request.finish() + + def render_GET(self, request): + return '
' \ + '
Please describe the problem you experienced and any information you think might be useful to us. Links to screenshots are great!
' \ + '' \ + '' \ + '
' + + def render_POST(self, request): + msg = request.args["message"][0] + log.info("User submitted error report: " + str(msg)) + api = jsonrpc.Proxy(API_CONNECTION_STRING) + d = api.callRemote("upload_log", {'name_prefix': 'report', 'exclude_previous': False, 'force': True}) + d.addCallback(lambda _: self._delayed_render(request, "Your bug report is greatly appreciated! Click here to return to LBRY")) + + return server.NOT_DONE_YET + + +class LBRYDaemonServer(object): + def __init__(self): + self.data_dir = user_data_dir("LBRY") + if not os.path.isdir(self.data_dir): + os.mkdir(self.data_dir) + self.version_dir = os.path.join(self.data_dir, "ui_version_history") + if not os.path.isdir(self.version_dir): + os.mkdir(self.version_dir) + self.config = os.path.join(self.version_dir, "active.json") + self.ui_dir = os.path.join(self.data_dir, "lbry-web-ui") + self.git_version = None + self._api = None + self.root = None + + if not os.path.isfile(os.path.join(self.config)): + self.loaded_git_version = None + else: + try: + f = open(self.config, "r") + loaded_ui = json.loads(f.read()) + f.close() + self.loaded_git_version = loaded_ui['commit'] + except: + self.loaded_git_version = None + + def setup(self, branch="HEAD", user_specified=None): + self.branch = branch + if user_specified: + if os.path.isdir(user_specified): + log.info("Using user specified UI directory: " + str(user_specified)) + self.branch = "user-specified" + self.loaded_git_version = "user-specified" + self.ui_dir = user_specified + return defer.succeed("user-specified") + else: + log.info("User specified UI directory doesn't exist, using " + branch) + elif branch == "HEAD": + log.info("Using UI branch: " + branch) + self._gitcmd = "git ls-remote https://github.com/lbryio/lbry-web-ui.git | grep %s | cut -f 1" % branch + self._dist_url = "https://raw.githubusercontent.com/lbryio/lbry-web-ui/master/dist.zip" + else: + log.info("Using UI branch: " + branch) + self._gitcmd = "git ls-remote https://github.com/lbryio/lbry-web-ui.git | grep refs/heads/%s | cut -f 1" % branch + self._dist_url = "https://raw.githubusercontent.com/lbryio/lbry-web-ui/%s/dist.zip" % branch + + d = self._up_to_date() + d.addCallback(lambda r: self._download_ui() if not r else self.branch) + return d + + def _up_to_date(self): + def _get_git_info(): + r = subprocess.check_output(self._gitcmd, shell=True) + return defer.succeed(r) + + def _set_git(version): + self.git_version = version + log.info("UI version from git: " + str(self.git_version).replace("\n", "")) + version_log.info("UI version from git: " + str(self.git_version).replace("\n", "")) + + if self.git_version == self.loaded_git_version and os.path.isdir(self.ui_dir): + log.info("UI is up to date") + version_log.info("UI is up to date") + return defer.succeed(True) + else: + log.info("Downloading UI") + version_log.info("Downloading UI") + f = open(self.config, "w") + f.write(json.dumps({'commit': self.git_version, + 'time': str(datetime.now())})) + f.close() + return defer.succeed(False) + + d = _get_git_info() + d.addCallback(_set_git) + return d + + def _download_ui(self): + def _delete_ui_dir(): + if os.path.isdir(self.ui_dir): + if self.loaded_git_version: + version_log.info("Removed ui files for commit " + str(self.loaded_git_version)) + log.info("Removing out of date ui files") + shutil.rmtree(self.ui_dir) + return defer.succeed(None) + + def _dl_ui(): + url = urlopen(self._dist_url) + z = ZipFile(StringIO(url.read())) + names = [i for i in z.namelist() if '.DS_exStore' not in i and '__MACOSX' not in i] + z.extractall(self.ui_dir, members=names) + version_log.info("[" + str(datetime.now()) + "] Updated branch " + self.branch + " commit: " + str(self.loaded_git_version) + " to " + self.git_version) + log.info("Downloaded files for UI commit " + self.git_version) + self.loaded_git_version = self.git_version + return self.branch + + d = _delete_ui_dir() + d.addCallback(lambda _: _dl_ui()) + return d + + def _setup_server(self, ui_ver): + self._api = LBRYDaemon(ui_ver, wallet_type=DEFAULT_WALLET) + self.root = LBRYindex(self.ui_dir) + self.root.putChild("css", static.File(os.path.join(self.ui_dir, "css"))) + self.root.putChild("font", static.File(os.path.join(self.ui_dir, "font"))) + self.root.putChild("img", static.File(os.path.join(self.ui_dir, "img"))) + self.root.putChild("js", static.File(os.path.join(self.ui_dir, "js"))) + self.root.putChild("view", LBRYFileRender()) + self.root.putChild("report", LBRYBugReport()) + self.root.putChild(API_ADDRESS, self._api) + return defer.succeed(True) + + def start(self, branch="HEAD", user_specified=False): + d = self.setup(branch=branch, user_specified=user_specified) + d.addCallback(lambda v: self._setup_server(v)) + d.addCallback(lambda _: self._api.setup()) + + return d