lbry-sdk/lbrynet/daemon/auth/server.py

import logging
import urlparse
import json
import inspect

from decimal import Decimal
from zope.interface import implements
from twisted.web import server, resource
from twisted.internet import defer
from twisted.python.failure import Failure
from twisted.internet.error import ConnectionDone, ConnectionLost
from txjsonrpc import jsonrpclib
from traceback import format_exc

from lbrynet import conf
from lbrynet.core.Error import InvalidAuthenticationToken
from lbrynet.core import utils
from lbrynet.daemon.auth.util import APIKey, get_auth_message
from lbrynet.daemon.auth.client import LBRY_SECRET
from lbrynet.undecorated import undecorated

log = logging.getLogger(__name__)

EMPTY_PARAMS = [{}]


class JSONRPCError(object):
    # http://www.jsonrpc.org/specification#error_object
    CODE_PARSE_ERROR = -32700  # Invalid JSON. Error while parsing the JSON text.
    CODE_INVALID_REQUEST = -32600  # The JSON sent is not a valid Request object.
    CODE_METHOD_NOT_FOUND = -32601  # The method does not exist / is not available.
    CODE_INVALID_PARAMS = -32602  # Invalid method parameter(s).
    CODE_INTERNAL_ERROR = -32603  # Internal JSON-RPC error (I think this is like a 500?)
    CODE_APPLICATION_ERROR = -32500  # Generic error with our app??
    CODE_AUTHENTICATION_ERROR = -32501  # Authentication failed

    MESSAGES = {
        CODE_PARSE_ERROR: "Parse Error. Data is not valid JSON.",
        CODE_INVALID_REQUEST: "JSON data is not a valid Request",
        CODE_METHOD_NOT_FOUND: "Method Not Found",
        CODE_INVALID_PARAMS: "Invalid Params",
        CODE_INTERNAL_ERROR: "Internal Error",
        CODE_AUTHENTICATION_ERROR: "Authentication Failed",
    }

    HTTP_CODES = {
        CODE_INVALID_REQUEST: 400,
        CODE_PARSE_ERROR: 400,
        CODE_INVALID_PARAMS: 400,
        CODE_METHOD_NOT_FOUND: 404,
        CODE_INTERNAL_ERROR: 500,
        CODE_APPLICATION_ERROR: 500,
        CODE_AUTHENTICATION_ERROR: 401,
    }

    def __init__(self, message, code=CODE_APPLICATION_ERROR, traceback=None, data=None):
        assert isinstance(code, (int, long)), "'code' must be an int"
        assert (data is None or isinstance(data, dict)), "'data' must be None or a dict"
        self.code = code
        if message is None:
            message = self.MESSAGES[code] if code in self.MESSAGES else "Error"
        self.message = message
        self.data = {} if data is None else data
        if traceback is not None:
            self.data['traceback'] = traceback.split("\n")

    def to_dict(self):
        ret = {
            'code': self.code,
            'message': self.message,
        }
        if len(self.data):
            ret['data'] = self.data
        return ret

    @classmethod
    def create_from_exception(cls, exception, code=CODE_APPLICATION_ERROR, traceback=None):
        return cls(exception.message, code=code, traceback=traceback)


def default_decimal(obj):
    if isinstance(obj, Decimal):
        return float(obj)


class UnknownAPIMethodError(Exception):
    pass


class NotAllowedDuringStartupError(Exception):
    pass


def trap(err, *to_trap):
    err.trap(*to_trap)


def jsonrpc_dumps_pretty(obj, **kwargs):
    try:
        id_ = kwargs.pop("id")
    except KeyError:
        id_ = None

    if isinstance(obj, JSONRPCError):
        data = {"jsonrpc": "2.0", "error": obj.to_dict(), "id": id_}
    else:
        data = {"jsonrpc": "2.0", "result": obj, "id": id_}

    return json.dumps(data, cls=jsonrpclib.JSONRPCEncoder, sort_keys=True, indent=2,
                      separators=(',', ': '), **kwargs) + "\n"


class JSONRPCServerType(type):
    def __new__(mcs, name, bases, newattrs):
        klass = type.__new__(mcs, name, bases, newattrs)
        klass.callable_methods = {}
        klass.deprecated_methods = {}
        klass.authorized_functions = []
        klass.queued_methods = []

        for methodname in dir(klass):
            if methodname.startswith("jsonrpc_"):
                method = getattr(klass, methodname)
                if not hasattr(method, '_deprecated'):
                    klass.callable_methods.update({methodname.split("jsonrpc_")[1]: method})
                    if hasattr(method, '_auth_required'):
                        klass.authorized_functions.append(methodname.split("jsonrpc_")[1])
                    if hasattr(method, '_queued'):
                        klass.queued_methods.append(methodname.split("jsonrpc_")[1])
                else:
                    klass.deprecated_methods.update({methodname.split("jsonrpc_")[1]: method})
        return klass


class AuthorizedBase(object):
    __metaclass__ = JSONRPCServerType

    @staticmethod
    def auth_required(f):
        f._auth_required = True
        return f

    @staticmethod
    def queued(f):
        f._queued = True
        return f

    @staticmethod
    def deprecated(new_command=None):
        def _deprecated_wrapper(f):
            f._new_command = new_command
            f._deprecated = True
            return f
        return _deprecated_wrapper

    @staticmethod
    def flags(**kwargs):
        def _flag_wrapper(f):
            f._flags = {}
            for k, v in kwargs.iteritems():
                f._flags[v] = k
            return f
        return _flag_wrapper


class AuthJSONRPCServer(AuthorizedBase):
    """Authorized JSONRPC server used as the base class for the LBRY API

    API methods are named with a leading "jsonrpc_"

    Decorators:

        @AuthJSONRPCServer.auth_required: this requires that the client
            include a valid hmac authentication token in their request

    Attributes:
        allowed_during_startup (list): list of api methods that are
            callable before the server has finished startup

        sessions (dict): dictionary of active session_id:
            lbrynet.lbrynet_daemon.auth.util.APIKey values

        authorized_functions (list): list of api methods that require authentication

        callable_methods (dict): dictionary of api_callable_name: method values

    """
    implements(resource.IResource)

    isLeaf = True
    allowed_during_startup = []

    def __init__(self, use_authentication=None):
        self._call_lock = {}
        self._use_authentication = (
            use_authentication if use_authentication is not None else conf.settings['use_auth_http']
        )
        self.announced_startup = False
        self.sessions = {}

    def setup(self):
        return NotImplementedError()

    def _set_headers(self, request, data, update_secret=False):
        if conf.settings['allowed_origin']:
            request.setHeader("Access-Control-Allow-Origin", conf.settings['allowed_origin'])
        request.setHeader("Content-Type", "application/json")
        request.setHeader("Accept", "application/json-rpc")
        request.setHeader("Content-Length", str(len(data)))
        if update_secret:
            session_id = request.getSession().uid
            request.setHeader(LBRY_SECRET, self.sessions.get(session_id).secret)

    @staticmethod
    def _render_message(request, message):
        request.write(message)
        request.finish()

    def _render_error(self, failure, request, id_):
        if isinstance(failure, JSONRPCError):
            error = failure
        elif isinstance(failure, Failure):
            # maybe failure is JSONRPCError wrapped in a twisted Failure
            error = failure.check(JSONRPCError)
            if error is None:
                # maybe its a twisted Failure with another type of error
                error = JSONRPCError(failure.getErrorMessage(), traceback=failure.getTraceback())
        else:
            # last resort, just cast it as a string
            error = JSONRPCError(str(failure))

        response_content = jsonrpc_dumps_pretty(error, id=id_)

        self._set_headers(request, response_content)
        try:
            request.setResponseCode(JSONRPCError.HTTP_CODES[error.code])
        except KeyError:
            request.setResponseCode(JSONRPCError.HTTP_CODES[JSONRPCError.CODE_INTERNAL_ERROR])
        self._render_message(request, response_content)

    @staticmethod
    def _handle_dropped_request(result, d, function_name):
        if not d.called:
            log.warning("Cancelling dropped api request %s", function_name)
            d.cancel()

    def render(self, request):
        try:
            return self._render(request)
        except BaseException as e:
            log.error(e)
            error = JSONRPCError.create_from_exception(e, traceback=format_exc())
            self._render_error(error, request, None)
            return server.NOT_DONE_YET

    def _render(self, request):
        time_in = utils.now()
        # assert self._check_headers(request), InvalidHeaderError
        session = request.getSession()
        session_id = session.uid
        finished_deferred = request.notifyFinish()

        if self._use_authentication:
            # if this is a new session, send a new secret and set the expiration
            # otherwise, session.touch()
            if self._initialize_session(session_id):
                def expire_session():
                    self._unregister_user_session(session_id)

                session.startCheckingExpiration()
                session.notifyOnExpire(expire_session)
                message = "OK"
                request.setResponseCode(200)
                self._set_headers(request, message, True)
                self._render_message(request, message)
                return server.NOT_DONE_YET
            else:
                session.touch()

        request.content.seek(0, 0)
        content = request.content.read()
        try:
            parsed = jsonrpclib.loads(content)
        except ValueError:
            log.warning("Unable to decode request json")
            self._render_error(JSONRPCError(None, JSONRPCError.CODE_PARSE_ERROR), request, None)
            return server.NOT_DONE_YET

        id_ = None
        try:
            function_name = parsed.get('method')
            is_queued = function_name in self.queued_methods
            args = parsed.get('params', {})
            id_ = parsed.get('id', None)
            token = parsed.pop('hmac', None)
        except AttributeError as err:
            log.warning(err)
            self._render_error(
                JSONRPCError(None, code=JSONRPCError.CODE_INVALID_REQUEST), request, id_
            )
            return server.NOT_DONE_YET

        reply_with_next_secret = False
        if self._use_authentication:
            if function_name in self.authorized_functions:
                try:
                    self._verify_token(session_id, parsed, token)
                except InvalidAuthenticationToken as err:
                    log.warning("API validation failed")
                    self._render_error(
                        JSONRPCError.create_from_exception(
                            err.message, code=JSONRPCError.CODE_AUTHENTICATION_ERROR,
                            traceback=format_exc()
                        ),
                        request, id_
                    )
                    return server.NOT_DONE_YET
                self._update_session_secret(session_id)
                reply_with_next_secret = True

        try:
            function = self._get_jsonrpc_method(function_name)
        except UnknownAPIMethodError as err:
            log.warning('Failed to get function %s: %s', function_name, err)
            self._render_error(
                JSONRPCError(None, JSONRPCError.CODE_METHOD_NOT_FOUND),
                request, id_
            )
            return server.NOT_DONE_YET
        except NotAllowedDuringStartupError:
            log.warning('Function not allowed during startup: %s', function_name)
            self._render_error(
                JSONRPCError("This method is unavailable until the daemon is fully started",
                             code=JSONRPCError.CODE_INVALID_REQUEST),
                request, id_
            )
            return server.NOT_DONE_YET

        if args == EMPTY_PARAMS or args == []:
            args_dict = {}
            _args, _kwargs = (), {}
        elif isinstance(args, dict):
            args_dict = args
        elif len(args) == 1 and isinstance(args[0], dict):
            # TODO: this is for backwards compatibility. Remove this once API and UI are updated
            # TODO: also delete EMPTY_PARAMS then
            args_dict = args[0]
            _args, _kwargs = (), args
        elif isinstance(args, list):
            _args, _kwargs = args, {}
        else:
            # d = defer.maybeDeferred(function, *args)  # if we want to support positional args too
            raise ValueError('Args must be a dict')

        params_error, erroneous_params = self._check_params(function, args_dict)
        if params_error is not None:
            params_error_message = '{} for {} command: {}'.format(
                params_error, function_name, ', '.join(erroneous_params)
            )
            log.warning(params_error_message)
            self._render_error(
                JSONRPCError(params_error_message, code=JSONRPCError.CODE_INVALID_PARAMS),
                request, id_
            )
            return server.NOT_DONE_YET

        if is_queued:
            d_lock = self._call_lock.get(function_name, False)
            if not d_lock:
                d = defer.maybeDeferred(function, self, **args_dict)
                self._call_lock[function_name] = finished_deferred

                def _del_lock(*args):
                    if function_name in self._call_lock:
                        del self._call_lock[function_name]
                    if args:
                        return args

                finished_deferred.addCallback(_del_lock)

            else:
                log.info("queued %s", function_name)
                d = d_lock
                d.addBoth(lambda _: log.info("running %s from queue", function_name))
                d.addCallback(lambda _: defer.maybeDeferred(function, self, **args_dict))
        else:
            d = defer.maybeDeferred(function, self, **args_dict)

        # finished_deferred will callback when the request is finished
        # and errback if something went wrong. If the errback is
        # called, cancel the deferred stack. This is to prevent
        # request.finish() from being called on a closed request.
        finished_deferred.addErrback(self._handle_dropped_request, d, function_name)

        d.addCallback(self._callback_render, request, id_, reply_with_next_secret)
        # TODO: don't trap RuntimeError, which is presently caught to
        # handle deferredLists that won't peacefully cancel, namely
        # get_lbry_files
        d.addErrback(trap, ConnectionDone, ConnectionLost, defer.CancelledError, RuntimeError)
        d.addErrback(log.fail(self._render_error, request, id_),
                     'Failed to process %s', function_name)
        d.addBoth(lambda _: log.debug("%s took %f",
                                      function_name,
                                      (utils.now() - time_in).total_seconds()))
        return server.NOT_DONE_YET

    def _register_user_session(self, session_id):
        """
        Add or update a HMAC secret for a session

        @param session_id:
        @return: secret
        """
        log.info("Register api session")
        token = APIKey.new(seed=session_id)
        self.sessions.update({session_id: token})

    def _unregister_user_session(self, session_id):
        log.info("Unregister API session")
        del self.sessions[session_id]

    def _check_headers(self, request):
        return (
            self._check_header_source(request, 'Origin') and
            self._check_header_source(request, 'Referer'))

    def _check_header_source(self, request, header):
        """Check if the source of the request is allowed based on the header value."""
        source = request.getHeader(header)
        if not self._check_source_of_request(source):
            log.warning("Attempted api call from invalid %s: %s", header, source)
            return False
        return True

    def _check_source_of_request(self, source):
        if source is None:
            return True
        if conf.settings['api_host'] == '0.0.0.0':
            return True
        server, port = self.get_server_port(source)
        return self._check_server_port(server, port)

    def _check_server_port(self, server, port):
        api = (conf.settings['api_host'], conf.settings['api_port'])
        return (server, port) == api or self._is_from_allowed_origin(server, port)

    def _is_from_allowed_origin(self, server, port):
        allowed_origin = conf.settings['allowed_origin']
        if not allowed_origin:
            return False
        if allowed_origin == '*':
            return True
        allowed_server, allowed_port = self.get_server_port(allowed_origin)
        return (allowed_server, allowed_port) == (server, port)

    def get_server_port(self, origin):
        parsed = urlparse.urlparse(origin)
        server_port = parsed.netloc.split(':')
        assert len(server_port) <= 2
        if len(server_port) == 2:
            return server_port[0], int(server_port[1])
        else:
            return server_port[0], 80

    def _verify_method_is_callable(self, function_path):
        if function_path not in self.callable_methods:
            raise UnknownAPIMethodError(function_path)
        if not self.announced_startup:
            if function_path not in self.allowed_during_startup:
                raise NotAllowedDuringStartupError(function_path)

    def _get_jsonrpc_method(self, function_path):
        if function_path in self.deprecated_methods:
            new_command = self.deprecated_methods[function_path]._new_command
            log.warning('API function \"%s\" is deprecated, please update to use \"%s\"',
                        function_path, new_command)
            function_path = new_command
        self._verify_method_is_callable(function_path)
        return self.callable_methods.get(function_path)

    @staticmethod
    def _check_params(function, args_dict):
        argspec = inspect.getargspec(undecorated(function))
        num_optional_params = 0 if argspec.defaults is None else len(argspec.defaults)
        missing_required_params = [
            required_param
            for required_param in argspec.args[1:-num_optional_params]
            if required_param not in args_dict
        ]
        if len(missing_required_params):
            return 'Missing required parameters', missing_required_params

        extraneous_params = [] if argspec.keywords is not None else [
            extra_param
            for extra_param in args_dict
            if extra_param not in argspec.args[1:]
        ]
        if len(extraneous_params):
            return 'Extraneous parameters', extraneous_params

        return None, None

    def _initialize_session(self, session_id):
        if not self.sessions.get(session_id, False):
            self._register_user_session(session_id)
            return True
        return False

    def _verify_token(self, session_id, message, token):
        if token is None:
            raise InvalidAuthenticationToken('Authentication token not found')
        to_auth = get_auth_message(message)
        api_key = self.sessions.get(session_id)
        if not api_key.compare_hmac(to_auth, token):
            raise InvalidAuthenticationToken('Invalid authentication token')

    def _update_session_secret(self, session_id):
        self.sessions.update({session_id: APIKey.new(name=session_id)})

    def _callback_render(self, result, request, id_, auth_required=False):
        try:
            encoded_message = jsonrpc_dumps_pretty(result, id=id_, default=default_decimal)
            request.setResponseCode(200)
            self._set_headers(request, encoded_message, auth_required)
            self._render_message(request, encoded_message)
        except Exception as err:
            log.exception("Failed to render API response: %s", result)
            self._render_error(err, request, id_)

    @staticmethod
    def _render_response(result):
        return defer.succeed(result)