lbry-sdk/lbrynet/lbrynet_daemon/LBRYDaemonServer.py

357 lines
15 KiB
Python
Raw Normal View History

import logging
import subprocess
import os
import shutil
import json
import sys
2016-05-02 10:10:50 +02:00
import mimetypes
from StringIO import StringIO
from zipfile import ZipFile
from urllib import urlopen
from datetime import datetime
from appdirs import user_data_dir
from twisted.web import server, static, resource
2016-05-02 21:58:40 +02:00
from twisted.internet import defer, interfaces, error, reactor, task, threads
from twisted.python.failure import Failure
from txjsonrpc.web import jsonrpc
from zope.interface import implements
from lbrynet.lbrynet_daemon.LBRYDaemon import LBRYDaemon
from lbrynet.conf import API_CONNECTION_STRING, API_ADDRESS, DEFAULT_WALLET, UI_ADDRESS
if sys.platform != "darwin":
data_dir = os.path.join(os.path.expanduser("~"), ".lbrynet")
else:
data_dir = user_data_dir("LBRY")
if not os.path.isdir(data_dir):
os.mkdir(data_dir)
version_dir = os.path.join(data_dir, "ui_version_history")
if not os.path.isdir(version_dir):
os.mkdir(version_dir)
version_log = logging.getLogger("lbry_version")
version_log.addHandler(logging.FileHandler(os.path.join(version_dir, "lbry_version.log")))
version_log.setLevel(logging.INFO)
log = logging.getLogger(__name__)
log.addHandler(logging.FileHandler(os.path.join(data_dir, 'lbrynet-daemon.log')))
log.setLevel(logging.INFO)
class LBRYindex(resource.Resource):
def __init__(self, ui_dir):
resource.Resource.__init__(self)
self.ui_dir = ui_dir
isLeaf = False
def _delayed_render(self, request, results):
request.write(str(results))
request.finish()
def getChild(self, name, request):
if name == '':
return self
return resource.Resource.getChild(self, name, request)
def render_GET(self, request):
return static.File(os.path.join(self.ui_dir, "index.html")).render_GET(request)
class LBRYFileStreamer(object):
"""
Writes downloaded LBRY file to request as the download comes in, pausing and resuming as requested
used for Chrome
"""
implements(interfaces.IPushProducer)
def __init__(self, request, path, start, stop, size):
self._request = request
self._fileObject = file(path)
2016-05-02 10:10:50 +02:00
self._content_type = mimetypes.guess_type(path)[0]
self._stop_pos = size - 1 if stop == '' else int(stop) #chrome and firefox send range requests for "0-"
self._cursor = self._start_pos = int(start)
self._file_size = size
2016-05-02 21:58:40 +02:00
self._depth = 0
2016-05-02 10:10:50 +02:00
self._paused = self._sent_bytes = self._stopped = False
2016-05-02 21:58:40 +02:00
self._delay = 0.25
2016-05-02 10:10:50 +02:00
self._deferred = defer.succeed(None)
self._request.setResponseCode(206)
self._request.setHeader('accept-ranges', 'bytes')
2016-05-02 10:10:50 +02:00
self._request.setHeader('content-type', self._content_type)
self.resumeProducing()
def pauseProducing(self):
self._paused = True
log.info("[" + str(datetime.now()) + "] Pausing producer")
return defer.succeed(None)
def resumeProducing(self):
def _check_for_new_data():
2016-05-02 21:58:40 +02:00
self._depth += 1
self._fileObject.seek(self._start_pos, os.SEEK_END)
readable_bytes = self._fileObject.tell()
self._fileObject.seek(self._cursor)
self._sent_bytes = False
2016-05-02 10:10:50 +02:00
if (readable_bytes > self._cursor) and not (self._stopped or self._paused):
read_length = min(readable_bytes, self._stop_pos) - self._cursor + 1
self._request.setHeader('content-range', 'bytes %s-%s/%s' % (self._cursor, self._cursor + read_length - 1, self._file_size))
self._request.setHeader('content-length', str(read_length))
start_cur = self._cursor
for i in range(read_length):
if self._paused or self._stopped:
break
else:
data = self._fileObject.read(1)
self._request.write(data)
self._cursor += 1
2016-05-02 10:10:50 +02:00
2016-05-02 21:58:40 +02:00
log.info("[" + str(datetime.now()) + "] Wrote range %s-%s/%s, length: %s, readable: %s, depth: %s" %
(start_cur, self._cursor, self._file_size, self._cursor - start_cur, readable_bytes, self._depth))
self._sent_bytes = True
if self._cursor == self._stop_pos + 1:
self.stopProducing()
return defer.succeed(None)
2016-05-02 10:10:50 +02:00
elif self._paused or self._stopped:
return defer.succeed(None)
else:
2016-05-02 21:58:40 +02:00
self._deferred.addCallback(lambda _: threads.deferToThread(reactor.callLater, self._delay, _check_for_new_data))
2016-05-02 10:10:50 +02:00
return defer.succeed(None)
log.info("[" + str(datetime.now()) + "] Resuming producer")
self._paused = False
2016-05-02 10:10:50 +02:00
self._deferred.addCallback(lambda _: _check_for_new_data())
def stopProducing(self):
2016-05-02 10:10:50 +02:00
log.info("[" + str(datetime.now()) + "] Stopping producer")
self._stopped = True
# self._fileObject.close()
2016-05-02 10:10:50 +02:00
self._deferred.addErrback(lambda err: err.trap(defer.CancelledError))
self._deferred.addErrback(lambda err: err.trap(error.ConnectionDone))
self._deferred.cancel()
# self._request.finish()
self._request.unregisterProducer()
2016-05-02 21:58:40 +02:00
return defer.succeed(None)
class HostedLBRYFile(resource.Resource):
def __init__(self, api):
self._api = api
2016-05-02 10:10:50 +02:00
self._producer = None
resource.Resource.__init__(self)
def makeProducer(self, request, stream):
2016-05-02 10:10:50 +02:00
def _save_producer(producer):
self._producer = producer
return defer.succeed(None)
range_header = request.getAllHeaders()['range'].replace('bytes=', '').split('-')
start, stop = int(range_header[0]), range_header[1]
2016-05-02 10:10:50 +02:00
log.info("[" + str(datetime.now()) + "] GET range %s-%s" % (start, stop))
path = os.path.join(self._api.download_directory, stream.file_name)
d = stream.get_total_bytes()
2016-05-02 10:10:50 +02:00
d.addCallback(lambda size: _save_producer(LBRYFileStreamer(request, path, start, stop, size)))
d.addCallback(lambda _: request.registerProducer(self._producer, streaming=True))
# request.notifyFinish().addCallback(lambda _: self._producer.stopProducing())
request.notifyFinish().addErrback(self._responseFailed, d)
return d
def render_GET(self, request):
if 'name' in request.args.keys():
if request.args['name'][0] != 'lbry' and request.args['name'][0] not in self._api.waiting_on.keys():
d = self._api._download_name(request.args['name'][0])
2016-05-02 21:58:40 +02:00
# d.addCallback(lambda stream: self.makeProducer(request, stream))
d.addCallback(lambda stream: static.File(os.path.join(self._api.download_directory,
stream.file_name)).render_GET(request))
2016-05-02 10:10:50 +02:00
elif request.args['name'][0] in self._api.waiting_on.keys():
request.redirect(UI_ADDRESS + "/?watch=" + request.args['name'][0])
request.finish()
else:
request.redirect(UI_ADDRESS)
request.finish()
return server.NOT_DONE_YET
2016-04-23 00:18:17 +02:00
def _responseFailed(self, err, call):
call.addErrback(lambda err: err.trap(error.ConnectionDone))
call.addErrback(lambda err: err.trap(defer.CancelledError))
call.addErrback(lambda err: log.info("Error: " + str(err)))
call.cancel()
2016-04-23 00:18:17 +02:00
class MyLBRYFiles(resource.Resource):
isLeaf = False
def __init__(self):
resource.Resource.__init__(self)
self.files_table = None
def delayed_render(self, request, result):
request.write(result.encode('utf-8'))
request.finish()
def render_GET(self, request):
self.files_table = None
api = jsonrpc.Proxy(API_CONNECTION_STRING)
d = api.callRemote("get_lbry_files", {})
d.addCallback(self._get_table)
d.addCallback(lambda results: self.delayed_render(request, results))
return server.NOT_DONE_YET
def _get_table(self, files):
if not self.files_table:
self.files_table = r'<html><head><title>My LBRY files</title></head><body><table border="1">'
self.files_table += r'<tr>'
self.files_table += r'<td>Stream name</td>'
self.files_table += r'<td>Completed</td>'
self.files_table += r'<td>Toggle</td>'
self.files_table += r'<td>Remove</td>'
self.files_table += r'</tr>'
return self._get_table(files)
if not len(files):
self.files_table += r'</table></body></html>'
return self.files_table
else:
f = files.pop()
self.files_table += r'<tr>'
self.files_table += r'<td>%s</td>' % (f['stream_name'])
self.files_table += r'<td>%s</td>' % (f['completed'])
self.files_table += r'<td>Start</td>' if f['stopped'] else r'<td>Stop</td>'
self.files_table += r'<td>Delete</td>'
self.files_table += r'</tr>'
return self._get_table(files)
class LBRYDaemonServer(object):
def __init__(self):
self.data_dir = user_data_dir("LBRY")
if not os.path.isdir(self.data_dir):
os.mkdir(self.data_dir)
self.version_dir = os.path.join(self.data_dir, "ui_version_history")
if not os.path.isdir(self.version_dir):
os.mkdir(self.version_dir)
self.config = os.path.join(self.version_dir, "active.json")
self.ui_dir = os.path.join(self.data_dir, "lbry-web-ui")
self.git_version = None
self._api = None
self.root = None
if not os.path.isfile(os.path.join(self.config)):
self.loaded_git_version = None
else:
try:
f = open(self.config, "r")
loaded_ui = json.loads(f.read())
f.close()
self.loaded_git_version = loaded_ui['commit']
self.loaded_branch = loaded_ui['branch']
version_log.info("[" + str(datetime.now()) + "] Last used " + self.loaded_branch + " commit " + str(self.loaded_git_version).replace("\n", ""))
except:
self.loaded_git_version = None
self.loaded_branch = None
def setup(self, branch="HEAD", user_specified=None):
self.branch = branch
if user_specified:
if os.path.isdir(user_specified):
log.info("Using user specified UI directory: " + str(user_specified))
self.branch = "user-specified"
self.loaded_git_version = "user-specified"
self.ui_dir = user_specified
return defer.succeed("user-specified")
else:
log.info("User specified UI directory doesn't exist, using " + branch)
else:
log.info("Using UI branch: " + branch)
2016-05-06 21:34:41 +02:00
if branch == "HEAD":
branch = "master"
self._git_url = "https://api.github.com/repos/lbryio/lbry.io/git/refs/heads/%s" % branch
self._dist_url = "https://raw.githubusercontent.com/lbryio/lbry-web-ui/%s/dist.zip" % branch
d = self._up_to_date()
d.addCallback(lambda r: self._download_ui() if not r else self.branch)
return d
def _up_to_date(self):
def _get_git_info():
2016-05-06 21:34:41 +02:00
response = urlopen(self._git_url)
data = json.loads(response.read())
return defer.succeed(data['object']['sha'])
def _set_git(version):
self.git_version = version
version_log.info("[" + str(datetime.now()) + "] UI branch " + self.branch + " has a most recent commit of: " + str(self.git_version).replace("\n", ""))
if self.git_version == self.loaded_git_version and os.path.isdir(self.ui_dir):
version_log.info("[" + str(datetime.now()) + "] local copy of " + self.branch + " is up to date")
return defer.succeed(True)
else:
if self.git_version == self.loaded_git_version:
version_log.info("[" + str(datetime.now()) + "] Can't find ui files, downloading them again")
else:
version_log.info("[" + str(datetime.now()) + "] local copy of " + self.branch + " branch is out of date, updating")
f = open(self.config, "w")
f.write(json.dumps({'commit': self.git_version,
'time': str(datetime.now()),
'branch': self.branch}))
f.close()
return defer.succeed(False)
d = _get_git_info()
d.addCallback(_set_git)
return d
def _download_ui(self):
def _delete_ui_dir():
if os.path.isdir(self.ui_dir):
if self.loaded_git_version:
version_log.info("[" + str(datetime.now()) + "] Removed ui files for commit " + str(self.loaded_git_version).replace("\n", ""))
log.info("Removing out of date ui files")
shutil.rmtree(self.ui_dir)
return defer.succeed(None)
def _dl_ui():
url = urlopen(self._dist_url)
z = ZipFile(StringIO(url.read()))
names = [i for i in z.namelist() if '.DS_exStore' not in i and '__MACOSX' not in i]
z.extractall(self.ui_dir, members=names)
version_log.info("[" + str(datetime.now()) + "] Updated branch " + self.branch + ": " + str(self.loaded_git_version).replace("\n", "") + " --> " + self.git_version.replace("\n", ""))
log.info("Downloaded files for UI commit " + str(self.git_version).replace("\n", ""))
self.loaded_git_version = self.git_version
return self.branch
d = _delete_ui_dir()
d.addCallback(lambda _: _dl_ui())
return d
def _setup_server(self, ui_ver, wallet):
self._api = LBRYDaemon(ui_ver, wallet_type=wallet)
self.root = LBRYindex(self.ui_dir)
self.root.putChild("css", static.File(os.path.join(self.ui_dir, "css")))
self.root.putChild("font", static.File(os.path.join(self.ui_dir, "font")))
self.root.putChild("img", static.File(os.path.join(self.ui_dir, "img")))
self.root.putChild("js", static.File(os.path.join(self.ui_dir, "js")))
self.root.putChild("view", HostedLBRYFile(self._api))
self.root.putChild("files", MyLBRYFiles())
self.root.putChild(API_ADDRESS, self._api)
return defer.succeed(True)
def start(self, branch="HEAD", user_specified=False, wallet=DEFAULT_WALLET):
d = self.setup(branch=branch, user_specified=user_specified)
d.addCallback(lambda v: self._setup_server(v, wallet))
d.addCallback(lambda _: self._api.setup())
return d