Merge pull request #92 from lbryio/file-uploads
Add support for file uploads (for Publish page)
This commit is contained in:
Jack Robison
GitHub
commit
276a7f9d2a
3 changed files with 219 additions and 2 deletions
|
|
@ -175,6 +175,7 @@ class LBRYDaemon(jsonrpc.JSONRPC):
|
|||
self.streams = {}
|
||||
self.known_dht_nodes = KNOWN_DHT_NODES
|
||||
self.first_run_after_update = False
|
||||
self.uploaded_temp_files = []
|
||||
|
||||
if os.name == "nt":
|
||||
from lbrynet.winhelpers.knownpaths import get_path, FOLDERID, UserHandle
|
||||
|
|
@ -713,6 +714,13 @@ class LBRYDaemon(jsonrpc.JSONRPC):
|
|||
else:
|
||||
return defer.succeed(None)
|
||||
|
||||
def _clean_up_temp_files(self):
|
||||
for path in self.uploaded_temp_files:
|
||||
try:
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _shutdown(self):
|
||||
log.info("Closing lbrynet session")
|
||||
log.info("Status at time of shutdown: " + self.startup_status[0])
|
||||
|
|
@ -725,6 +733,8 @@ class LBRYDaemon(jsonrpc.JSONRPC):
|
|||
if self.lbry_ui_manager.update_checker.running:
|
||||
self.lbry_ui_manager.update_checker.stop()
|
||||
|
||||
self._clean_up_temp_files()
|
||||
|
||||
d = self._upload_log(log_type="close", exclude_previous=False if self.first_run else True)
|
||||
d.addCallback(lambda _: self._stop_server())
|
||||
d.addErrback(lambda err: True)
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ from twisted.web import server
|
|||
from twisted.internet import reactor, defer
|
||||
from jsonrpc.proxy import JSONRPCProxy
|
||||
|
||||
from lbrynet.lbrynet_daemon.LBRYDaemonServer import LBRYDaemonServer
|
||||
from lbrynet.lbrynet_daemon.LBRYDaemonServer import LBRYDaemonServer, LBRYDaemonRequest
|
||||
from lbrynet.conf import API_CONNECTION_STRING, API_INTERFACE, API_ADDRESS, API_PORT, \
|
||||
DEFAULT_WALLET, UI_ADDRESS, DEFAULT_UI_BRANCH, LOG_FILE_NAME
|
||||
|
||||
|
|
@ -121,7 +121,9 @@ def start():
|
|||
if args.launchui:
|
||||
d.addCallback(lambda _: webbrowser.open(UI_ADDRESS))
|
||||
|
||||
reactor.listenTCP(API_PORT, server.Site(lbry.root), interface=API_INTERFACE)
|
||||
lbrynet_server = server.Site(lbry.root)
|
||||
lbrynet_server.requestFactory = LBRYDaemonRequest
|
||||
reactor.listenTCP(API_PORT, lbrynet_server, interface=API_INTERFACE)
|
||||
reactor.run()
|
||||
|
||||
if not args.logtoconsole and not args.quiet:
|
||||
|
|
|
|||
|
|
@ -4,6 +4,10 @@ import shutil
|
|||
import json
|
||||
import sys
|
||||
import mimetypes
|
||||
import mimetools
|
||||
import tempfile
|
||||
import time
|
||||
import cgi
|
||||
|
||||
from datetime import datetime
|
||||
from appdirs import user_data_dir
|
||||
|
|
@ -29,6 +33,184 @@ handler = logging.handlers.RotatingFileHandler(lbrynet_log, maxBytes=2097152, ba
|
|||
log.addHandler(handler)
|
||||
log.setLevel(logging.INFO)
|
||||
|
||||
class LBRYDaemonRequest(server.Request):
|
||||
"""
|
||||
For LBRY specific request functionality. Currently just provides
|
||||
handling for large multipart POST requests, taken from here:
|
||||
http://sammitch.ca/2013/07/handling-large-requests-in-twisted/
|
||||
|
||||
For multipart POST requests, this populates self.args with temp
|
||||
file objects instead of strings. Note that these files don't auto-delete
|
||||
on close because we want to be able to move and rename them.
|
||||
|
||||
"""
|
||||
|
||||
# max amount of memory to allow any ~single~ request argument [ie: POSTed file]
|
||||
# note: this value seems to be taken with a grain of salt, memory usage may spike
|
||||
# FAR above this value in some cases.
|
||||
# eg: set the memory limit to 5 MB, write 2 blocks of 4MB, mem usage will
|
||||
# have spiked to 8MB before the data is rolled to disk after the
|
||||
# second write completes.
|
||||
memorylimit = 1024*1024*100
|
||||
|
||||
# enable/disable debug logging
|
||||
do_log = True
|
||||
|
||||
# re-defined only for debug/logging purposes
|
||||
def gotLength(self, length):
|
||||
if self.do_log:
|
||||
print '%f Headers received, Content-Length: %d' % (time.time(), length)
|
||||
server.Request.gotLength(self, length)
|
||||
|
||||
# re-definition of twisted.web.server.Request.requestreceived, the only difference
|
||||
# is that self.parse_multipart() is used rather than cgi.parse_multipart()
|
||||
def requestReceived(self, command, path, version):
|
||||
from twisted.web.http import parse_qs
|
||||
if self.do_log:
|
||||
print '%f Request Received' % time.time()
|
||||
print self.content
|
||||
|
||||
self.content.seek(0,0)
|
||||
self.args = {}
|
||||
self.stack = []
|
||||
|
||||
self.method, self.uri = command, path
|
||||
self.clientproto = version
|
||||
x = self.uri.split(b'?', 1)
|
||||
|
||||
if len(x) == 1:
|
||||
self.path = self.uri
|
||||
else:
|
||||
self.path, argstring = x
|
||||
self.args = parse_qs(argstring, 1)
|
||||
|
||||
# cache the client and server information, we'll need this later to be
|
||||
# serialized and sent with the request so CGIs will work remotely
|
||||
self.client = self.channel.transport.getPeer()
|
||||
self.host = self.channel.transport.getHost()
|
||||
|
||||
# Argument processing
|
||||
args = self.args
|
||||
ctype = self.requestHeaders.getRawHeaders(b'content-type')
|
||||
if ctype is not None:
|
||||
ctype = ctype[0]
|
||||
|
||||
if self.method == b"POST" and ctype:
|
||||
mfd = b'multipart/form-data'
|
||||
key, pdict = cgi.parse_header(ctype)
|
||||
if key == b'application/x-www-form-urlencoded':
|
||||
args.update(parse_qs(self.content.read(), 1))
|
||||
elif key == mfd:
|
||||
try:
|
||||
self.content.seek(0,0)
|
||||
args.update(self.parse_multipart(self.content, pdict))
|
||||
#args.update(cgi.parse_multipart(self.content, pdict))
|
||||
|
||||
except KeyError as e:
|
||||
if e.args[0] == b'content-disposition':
|
||||
# Parse_multipart can't cope with missing
|
||||
# content-dispostion headers in multipart/form-data
|
||||
# parts, so we catch the exception and tell the client
|
||||
# it was a bad request.
|
||||
self.channel.transport.write(
|
||||
b"HTTP/1.1 400 Bad Request\r\n\r\n")
|
||||
self.channel.transport.loseConnection()
|
||||
return
|
||||
raise
|
||||
|
||||
self.content.seek(0, 0)
|
||||
|
||||
self.process()
|
||||
|
||||
# re-definition of cgi.parse_multipart that uses a single temporary file to store
|
||||
# data rather than storing 2 to 3 copies in various lists.
|
||||
def parse_multipart(self, fp, pdict):
|
||||
if self.do_log:
|
||||
print '%f Parsing Multipart data: ' % time.time()
|
||||
rewind = fp.tell() #save cursor
|
||||
fp.seek(0,0) #reset cursor
|
||||
|
||||
boundary = ""
|
||||
if 'boundary' in pdict:
|
||||
boundary = pdict['boundary']
|
||||
if not cgi.valid_boundary(boundary):
|
||||
raise ValueError, ('Invalid boundary in multipart form: %r'
|
||||
% (boundary,))
|
||||
|
||||
nextpart = "--" + boundary
|
||||
lastpart = "--" + boundary + "--"
|
||||
partdict = {}
|
||||
terminator = ""
|
||||
|
||||
while terminator != lastpart:
|
||||
c_bytes = -1
|
||||
|
||||
data = tempfile.NamedTemporaryFile(delete=False)
|
||||
if terminator:
|
||||
# At start of next part. Read headers first.
|
||||
headers = mimetools.Message(fp)
|
||||
clength = headers.getheader('content-length')
|
||||
if clength:
|
||||
try:
|
||||
c_bytes = int(clength)
|
||||
except ValueError:
|
||||
pass
|
||||
if c_bytes > 0:
|
||||
data.write(fp.read(c_bytes))
|
||||
# Read lines until end of part.
|
||||
while 1:
|
||||
line = fp.readline()
|
||||
if not line:
|
||||
terminator = lastpart # End outer loop
|
||||
break
|
||||
if line[:2] == "--":
|
||||
terminator = line.strip()
|
||||
if terminator in (nextpart, lastpart):
|
||||
break
|
||||
data.write(line)
|
||||
# Done with part.
|
||||
if data.tell() == 0:
|
||||
continue
|
||||
if c_bytes < 0:
|
||||
# if a Content-Length header was not supplied with the MIME part
|
||||
# then the trailing line break must be removed.
|
||||
# we have data, read the last 2 bytes
|
||||
rewind = min(2, data.tell())
|
||||
data.seek(-rewind, os.SEEK_END)
|
||||
line = data.read(2)
|
||||
if line[-2:] == "\r\n":
|
||||
data.seek(-2, os.SEEK_END)
|
||||
data.truncate()
|
||||
elif line[-1:] == "\n":
|
||||
data.seek(-1, os.SEEK_END)
|
||||
data.truncate()
|
||||
|
||||
line = headers['content-disposition']
|
||||
if not line:
|
||||
continue
|
||||
key, params = cgi.parse_header(line)
|
||||
if key != 'form-data':
|
||||
continue
|
||||
if 'name' in params:
|
||||
name = params['name']
|
||||
# kludge in the filename
|
||||
if 'filename' in params:
|
||||
fname_index = name + '_filename'
|
||||
if fname_index in partdict:
|
||||
partdict[fname_index].append(params['filename'])
|
||||
else:
|
||||
partdict[fname_index] = [params['filename']]
|
||||
else:
|
||||
# Unnamed parts are not returned at all.
|
||||
continue
|
||||
data.seek(0,0)
|
||||
if name in partdict:
|
||||
partdict[name].append(data)
|
||||
else:
|
||||
partdict[name] = [data]
|
||||
|
||||
fp.seek(rewind) # Restore cursor
|
||||
return partdict
|
||||
|
||||
class LBRYindex(resource.Resource):
|
||||
def __init__(self, ui_dir):
|
||||
|
|
@ -179,12 +361,35 @@ class HostedLBRYFile(resource.Resource):
|
|||
# call.addErrback(lambda err: log.info("Error: " + str(err)))
|
||||
# call.cancel()
|
||||
|
||||
class LBRYFileUpload(resource.Resource):
|
||||
"""
|
||||
Accepts a file sent via the file upload widget in the web UI, saves
|
||||
it into a temporary dir, and responds with a JSON string containing
|
||||
the path of the newly created file.
|
||||
"""
|
||||
|
||||
def __init__(self, api):
|
||||
self._api = api
|
||||
|
||||
def render_POST(self, request):
|
||||
origfilename = request.args['file_filename'][0]
|
||||
uploaded_file = request.args['file'][0] # Temp file created by request
|
||||
|
||||
# Move to a new temporary dir and restore the original file name
|
||||
newdirpath = tempfile.mkdtemp()
|
||||
newpath = os.path.join(newdirpath, origfilename)
|
||||
shutil.move(uploaded_file.name, newpath)
|
||||
self._api.uploaded_temp_files.append(newpath)
|
||||
|
||||
return json.dumps(newpath)
|
||||
|
||||
|
||||
class LBRYDaemonServer(object):
|
||||
def _setup_server(self, wallet):
|
||||
self.root = LBRYindex(os.path.join(os.path.join(data_dir, "lbry-ui"), "active"))
|
||||
self._api = LBRYDaemon(self.root, wallet_type=wallet)
|
||||
self.root.putChild("view", HostedLBRYFile(self._api))
|
||||
self.root.putChild("upload", LBRYFileUpload(self._api))
|
||||
self.root.putChild(API_ADDRESS, self._api)
|
||||
return defer.succeed(True)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue