diff --git a/api/server.go b/api/server.go new file mode 100644 index 0000000..5bd3528 --- /dev/null +++ b/api/server.go @@ -0,0 +1,290 @@ +package api + +import ( + "encoding/json" + "net/http" + "reflect" + "strconv" + "strings" + + "github.com/lbryio/lbry.go/errors" + "github.com/lbryio/lbry.go/util" + "github.com/lbryio/lbry.go/validator" + v "github.com/lbryio/ozzo-validation" + + "github.com/spf13/cast" +) + +const authTokenParam = "auth_token" + +// Server HTTP Header Settings. Set on header if exists +// ie. "Content-Type" - "application/json; charset=utf-8" +// ie. "X-Content-Type-Options" - "nosniff" +// ie. "X-Frame-Options" - "deny" +// ie."Content-Security-Policy" - "default-src 'none'" +// ie. "X-XSS-Protection" - "1; mode=block" +// ie. "Server" - "lbry.io" +// ie. "Referrer-Policy" - "same-origin" +// ie. "Strict-Transport-Security" - "max-age=31536000; preload" +// ie. "Access-Control-Allow-Origin" -"" +// ie. "Access-Control-Allow-Methods" - "GET, POST, OPTIONS" +var HeaderSettings map[string]string + +// LogError Allows specific error logging for the server at specific points. +var LogError = func(*http.Request, *Response, error) {} + +// LogInfo Allows for specific logging information. +var LogInfo = func(*http.Request, *Response) {} + +// TraceEnabled Attaches a trace field to the JSON response when enabled. +var TraceEnabled = false + +var errAuthenticationRequired = errors.Base("authentication required") +var errNotAuthenticated = errors.Base("could not authenticate user") +var errForbidden = errors.Base("you are not authorized to perform this action") + +// StatusError represents an error with an associated HTTP status code. +type StatusError struct { + Status int + Err error +} + +// Allows StatusError to satisfy the error interface. +func (se StatusError) Error() string { + return se.Err.Error() +} + +// Response is returned by API handlers +type Response struct { + Status int + Data interface{} + RedirectURL string + Error error +} + +// Handler handles API requests +type Handler func(r *http.Request) Response + +func (h Handler) callHandlerSafely(r *http.Request) (rsp Response) { + defer func() { + if r := recover(); r != nil { + err, ok := r.(error) + if !ok { + err = errors.Err("%v", r) + } + rsp = Response{Error: errors.Wrap(err, 2)} + } + }() + + return h(r) +} + +func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // Set header settings + if HeaderSettings != nil { + //Multiple readers, no writers is okay + for key, value := range HeaderSettings { + w.Header().Set(key, value) + } + } + + // Stop here if its a preflighted OPTIONS request + if r.Method == "OPTIONS" { + return + } + + rsp := h.callHandlerSafely(r) + + if rsp.Status == 0 { + if rsp.Error != nil { + if statusError, ok := rsp.Error.(StatusError); ok { + rsp.Status = statusError.Status + } else if errors.Is(rsp.Error, errAuthenticationRequired) { + rsp.Status = http.StatusUnauthorized + } else if errors.Is(rsp.Error, errNotAuthenticated) || errors.Is(rsp.Error, errForbidden) { + rsp.Status = http.StatusForbidden + } else { + rsp.Status = http.StatusInternalServerError + } + } else if rsp.RedirectURL != "" { + rsp.Status = http.StatusFound + } else { + rsp.Status = http.StatusOK + } + } + + success := rsp.Status < http.StatusBadRequest + + consoleText := r.RemoteAddr + " [" + strconv.Itoa(rsp.Status) + "]: " + r.Method + " " + r.URL.Path + if success { + LogInfo(r, &rsp) + } else { + LogError(r, &rsp, errors.Base(consoleText)) + } + + // redirect + if rsp.Status >= http.StatusMultipleChoices && rsp.Status < http.StatusBadRequest { + http.Redirect(w, r, rsp.RedirectURL, rsp.Status) + return + } else if rsp.RedirectURL != "" { + LogError(r, &rsp, errors.Base("status code "+strconv.Itoa(rsp.Status)+ + " does not indicate a redirect, but RedirectURL is non-empty '"+ + rsp.RedirectURL+"'")) + } + + var errorString *string + if rsp.Error != nil { + errorStringRaw := rsp.Error.Error() + errorString = &errorStringRaw + } + + var trace []string + if TraceEnabled && errors.HasTrace(rsp.Error) { + trace = strings.Split(errors.Trace(rsp.Error), "\n") + for index, element := range trace { + if strings.HasPrefix(element, "\t") { + trace[index] = strings.Replace(element, "\t", " ", 1) + } + } + } + + // http://choly.ca/post/go-json-marshalling/ + jsonResponse, err := json.MarshalIndent(&struct { + Success bool `json:"success"` + Error *string `json:"error"` + Data interface{} `json:"data"` + Trace []string `json:"_trace,omitempty"` + }{ + Success: success, + Error: errorString, + Data: rsp.Data, + Trace: trace, + }, "", " ") + if err != nil { + LogError(r, &rsp, errors.Prefix("Error encoding JSON response: ", err)) + } + + if rsp.Status >= http.StatusInternalServerError { + LogError(r, &rsp, errors.Prefix(r.Method+" "+r.URL.Path+"\n", rsp.Error)) + } + + w.WriteHeader(rsp.Status) + w.Write(jsonResponse) +} + +func FormValues(r *http.Request, params interface{}, validationRules []*v.FieldRules) error { + ref := reflect.ValueOf(params) + if !ref.IsValid() || ref.Kind() != reflect.Ptr || ref.Elem().Kind() != reflect.Struct { + return errors.Err("'params' must be a pointer to a struct") + } + + structType := ref.Elem().Type() + structValue := ref.Elem() + fields := map[string]bool{} + for i := 0; i < structType.NumField(); i++ { + name := structType.Field(i).Name + underscoredName := util.Underscore(name) + value := strings.TrimSpace(r.FormValue(underscoredName)) + + // if param is not set at all, continue + // comes after call to r.FormValue so form values get parsed internally (if they arent already) + if len(r.Form[underscoredName]) == 0 { + continue + } + + fields[underscoredName] = true + isPtr := false + var finalValue reflect.Value + + structField := structValue.FieldByName(name) + structFieldKind := structField.Kind() + if structFieldKind == reflect.Ptr { + isPtr = true + structFieldKind = structField.Type().Elem().Kind() + } + + switch structFieldKind { + case reflect.String: + finalValue = reflect.ValueOf(value) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + if value == "" { + continue + } + castVal, err := cast.ToInt64E(value) + if err != nil { + return errors.Err("%s: must be an integer", underscoredName) + } + switch structFieldKind { + case reflect.Int: + finalValue = reflect.ValueOf(int(castVal)) + case reflect.Int8: + finalValue = reflect.ValueOf(int8(castVal)) + case reflect.Int16: + finalValue = reflect.ValueOf(int16(castVal)) + case reflect.Int32: + finalValue = reflect.ValueOf(int32(castVal)) + case reflect.Int64: + finalValue = reflect.ValueOf(castVal) + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + if value == "" { + continue + } + castVal, err := cast.ToUint64E(value) + if err != nil { + return errors.Err("%s: must be an unsigned integer", underscoredName) + } + switch structFieldKind { + case reflect.Uint: + finalValue = reflect.ValueOf(uint(castVal)) + case reflect.Uint8: + finalValue = reflect.ValueOf(uint8(castVal)) + case reflect.Uint16: + finalValue = reflect.ValueOf(uint16(castVal)) + case reflect.Uint32: + finalValue = reflect.ValueOf(uint32(castVal)) + case reflect.Uint64: + finalValue = reflect.ValueOf(castVal) + } + case reflect.Bool: + if value == "" { + continue + } + if !validator.IsBoolString(value) { + return errors.Err("%s: must be one of the following values: %s", + underscoredName, strings.Join(validator.GetBoolStringValues(), ", ")) + } + finalValue = reflect.ValueOf(validator.IsTruthy(value)) + default: + return errors.Err("field %s is an unsupported type", name) + } + + if isPtr { + if structField.IsNil() { + structField.Set(reflect.New(structField.Type().Elem())) + } + structField.Elem().Set(finalValue) + } else { + structField.Set(finalValue) + } + } + + var extraParams []string + for k := range r.Form { + if _, ok := fields[k]; !ok && k != authTokenParam { //TODO: fix this AUTH_PARAM hack + extraParams = append(extraParams, k) + } + } + if len(extraParams) > 0 { + return errors.Err("Extraneous params: " + strings.Join(extraParams, ", ")) + } + + if len(validationRules) > 0 { + validationErr := v.ValidateStruct(params, validationRules...) + if validationErr != nil { + return errors.Err(validationErr) + } + } + + return nil +} diff --git a/jsonrpc/daemon.go b/jsonrpc/daemon.go index b520172..27bdfba 100644 --- a/jsonrpc/daemon.go +++ b/jsonrpc/daemon.go @@ -21,7 +21,8 @@ import ( const DefaultPort = 5279 type Client struct { - conn *jsonrpc.RPCClient + conn jsonrpc.RPCClient + address string } func NewClient(address string) *Client { @@ -31,7 +32,8 @@ func NewClient(address string) *Client { address = "http://localhost:" + strconv.Itoa(DefaultPort) } - d.conn = jsonrpc.NewRPCClient(address) + d.conn = jsonrpc.NewClient(address) + d.address = address return &d } @@ -106,7 +108,7 @@ func debugParams(params map[string]interface{}) string { func (d *Client) callNoDecode(command string, params map[string]interface{}) (interface{}, error) { log.Debugln("jsonrpc: " + command + " " + debugParams(params)) - r, err := d.conn.CallNamed(command, params) + r, err := d.conn.Call(command, params) if err != nil { return nil, errors.Wrap(err, 0) } @@ -127,7 +129,9 @@ func (d *Client) call(response interface{}, command string, params map[string]in } func (d *Client) SetRPCTimeout(timeout time.Duration) { - d.conn.SetHTTPClient(&http.Client{Timeout: timeout}) + d.conn = jsonrpc.NewClientWithOpts(d.address, &jsonrpc.RPCClientOpts{ + HTTPClient: &http.Client{Timeout: timeout}, + }) } func (d *Client) Commands() (*CommandsResponse, error) { @@ -203,7 +207,6 @@ func (d *Client) PeerList(blobHash string, timeout *uint) (*PeerListResponse, er if err != nil { return nil, err } - castResponse, ok := rawResponse.([]interface{}) if !ok { return nil, errors.Err("invalid peer_list response") @@ -211,7 +214,7 @@ func (d *Client) PeerList(blobHash string, timeout *uint) (*PeerListResponse, er peers := []PeerListResponsePeer{} for _, peer := range castResponse { - t, ok := peer.([]interface{}) + t, ok := peer.(map[string]interface{}) if !ok { return nil, errors.Err("invalid peer_list response") } @@ -220,17 +223,17 @@ func (d *Client) PeerList(blobHash string, timeout *uint) (*PeerListResponse, er return nil, errors.Err("invalid triplet in peer_list response") } - ip, ok := t[0].(string) + ip, ok := t["host"].(string) if !ok { return nil, errors.Err("invalid ip in peer_list response") } - port, ok := t[1].(json.Number) + port, ok := t["port"].(json.Number) if !ok { return nil, errors.Err("invalid port in peer_list response") } - available, ok := t[2].(bool) + nodeid, ok := t["node_id"].(string) if !ok { - return nil, errors.Err("invalid is_available in peer_list response") + return nil, errors.Err("invalid nodeid in peer_list response") } portNum, err := port.Int64() @@ -241,9 +244,9 @@ func (d *Client) PeerList(blobHash string, timeout *uint) (*PeerListResponse, er } peers = append(peers, PeerListResponsePeer{ - IP: ip, - Port: uint(portNum), - IsAvailable: available, + IP: ip, + Port: uint(portNum), + NodeId: nodeid, }) } @@ -269,6 +272,15 @@ func (d *Client) BlobGet(blobHash string, encoding *string, timeout *uint) (*Blo return response, decode(rawResponse, response) } +func (d *Client) StreamAvailability(url string, search_timeout *uint64, blob_timeout *uint64) (*StreamAvailabilityResponse, error) { + response := new(StreamAvailabilityResponse) + return response, d.call(response, "stream_availability", map[string]interface{}{ + "uri": url, + "search_timeout": search_timeout, + "blob_timeout": blob_timeout, + }) +} + func (d *Client) StreamCostEstimate(url string, size *uint64) (*StreamCostEstimateResponse, error) { rawResponse, err := d.callNoDecode("stream_cost_estimate", map[string]interface{}{ "uri": url, diff --git a/jsonrpc/daemon_types.go b/jsonrpc/daemon_types.go index db4f41c..29cf737 100644 --- a/jsonrpc/daemon_types.go +++ b/jsonrpc/daemon_types.go @@ -214,9 +214,9 @@ type ClaimListResponse struct { type ClaimShowResponse Claim type PeerListResponsePeer struct { - IP string - Port uint - IsAvailable bool + IP string `json:"host"` + Port uint `json:"port"` + NodeId string `json:"node_id"` } type PeerListResponse []PeerListResponsePeer @@ -236,6 +236,27 @@ type BlobGetResponse struct { type StreamCostEstimateResponse decimal.Decimal +type BlobAvailability struct { + IsAvailable bool `json:"is_available"` + ReachablePeers []string `json:"reachable_peers"` + UnReachablePeers []string `json:"unreachable_peers"` +} + +type StreamAvailabilityResponse struct { + IsAvailable bool `json:"is_available"` + DidDecode bool `json:"did_decode"` + DidResolve bool `json:"did_resolve"` + IsStream bool `json:"is_stream"` + NumBlobsInStream uint64 `json:"num_blobs_in_stream"` + SDHash string `json:"sd_hash"` + SDBlobAvailability BlobAvailability `json:"sd_blob_availability"` + HeadBlobHash string `json:"head_blob_hash"` + HeadBlobAvailability BlobAvailability `json:"head_blob_availability"` + UseUPNP bool `json:"use_upnp"` + UPNPRedirectIsSet bool `json:"upnp_redirect_is_set"` + Error string `json:"error,omitempty"` +} + type GetResponse File type FileListResponse []File diff --git a/travis/travis.go b/travis/travis.go new file mode 100644 index 0000000..7ad946d --- /dev/null +++ b/travis/travis.go @@ -0,0 +1,112 @@ +package travis + +/* +Copyright 2017 Shapath Neupane (@theshapguy) +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +------------------------------------------------ +Listener - written in Go because it's native web server is much more robust than Python. Plus its fun to write Go! +NOTE: Make sure you are using the right domain for travis [.com] or [.org] +Modified by wilsonk@lbry.io for LBRY internal-apis +*/ + +import ( + "crypto" + "crypto/rsa" + "crypto/sha1" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "net/http" + + "github.com/lbryio/lbry.go/errors" +) + +func publicKey(isPrivateRepo bool) (*rsa.PublicKey, error) { + var response *http.Response + var err error + if !isPrivateRepo{ + response, err = http.Get("https://api.travis-ci.org/config") + }else{ + response, err = http.Get("https://api.travis-ci.com/config") + } + if err != nil { + return nil, errors.Err("cannot fetch travis public key") + } + defer response.Body.Close() + + type configKey struct { + Config struct { + Notifications struct { + Webhook struct { + PublicKey string `json:"public_key"` + } `json:"webhook"` + } `json:"notifications"` + } `json:"config"` + } + + var t configKey + + decoder := json.NewDecoder(response.Body) + err = decoder.Decode(&t) + if err != nil { + return nil, errors.Err("cannot decode travis public key") + } + + keyBlock, _ := pem.Decode([]byte(t.Config.Notifications.Webhook.PublicKey)) + if keyBlock == nil || keyBlock.Type != "PUBLIC KEY" { + return nil, errors.Err("invalid travis public key") + } + + publicKey, err := x509.ParsePKIXPublicKey(keyBlock.Bytes) + if err != nil { + return nil, errors.Err("invalid travis public key") + } + + return publicKey.(*rsa.PublicKey), nil +} + +func payloadDigest(payload string) []byte { + hash := sha1.New() + hash.Write([]byte(payload)) + return hash.Sum(nil) +} + +func ValidateSignature(isPrivateRepo bool,r *http.Request) error { + key, err := publicKey(isPrivateRepo) + if err != nil { + return errors.Err(err) + } + + signature, err := base64.StdEncoding.DecodeString(r.Header.Get("Signature")) + if err != nil { + return errors.Err("cannot decode signature") + } + + payload := payloadDigest(r.FormValue("payload")) + + err = rsa.VerifyPKCS1v15(key, crypto.SHA1, payload, signature) + if err != nil { + if err == rsa.ErrVerification { + return errors.Err("invalid payload signature") + } + return errors.Err(err) + } + + return nil +} + +func NewFromRequest(r *http.Request) (*Webhook, error) { + w := new(Webhook) + + err := json.Unmarshal([]byte(r.FormValue("payload")), w) + if err != nil { + return nil, errors.Err(err) + } + + return w, nil +} diff --git a/travis/webhook.go b/travis/webhook.go new file mode 100644 index 0000000..db998af --- /dev/null +++ b/travis/webhook.go @@ -0,0 +1,66 @@ +package travis + +import "time" + +// https://docs.travis-ci.com/user/notifications/#Webhooks-Delivery-Format + +const ( + statusSuccess = 0 + statusNotSuccess = 1 +) + +type Webhook struct { + ID int `json:"id"` + Number string `json:"number"` + Type string `json:"type"` + State string `json:"state"` + Status int `json:"status"` // status and result are the same + Result int `json:"result"` + StatusMessage string `json:"status_message"` // status_message and result_message are the same + ResultMessage string `json:"result_message"` + StartedAt time.Time `json:"started_at"` + FinishedAt time.Time `json:"finished_at"` + Duration int `json:"duration"` + BuildURL string `json:"build_url"` + CommitID int `json:"commit_id"` + Commit string `json:"commit"` + BaseCommit string `json:"base_commit"` + HeadCommit string `json:"head_commit"` + Branch string `json:"branch"` + Message string `json:"message"` + CompareURL string `json:"compare_url"` + CommittedAt time.Time `json:"committed_at"` + AuthorName string `json:"author_name"` + AuthorEmail string `json:"author_email"` + CommitterName string `json:"committer_name"` + CommitterEmail string `json:"committer_email"` + PullRequest bool `json:"pull_request"` + PullRequestNumber int `json:"pull_request_number"` + PullRequestTitle string `json:"pull_request_title"` + Tag string `json:"tag"` + Repository Repository `json:"repository"` +} + +type Repository struct { + ID int `json:"id"` + Name string `json:"name"` + OwnerName string `json:"owner_name"` + URL string `json:"url"` +} + +// IsMatch make sure the webhook is for you... +func (w Webhook) IsMatch(branch string, repo string, owner string) bool { + return w.Branch == branch && + w.Repository.Name == repo && + w.Repository.OwnerName == owner +} + +func (w Webhook) ShouldDeploy() bool { + // when travis builds a pull request, Branch is the target branch, not the origin branch + // source: https://docs.travis-ci.com/user/environment-variables/#Default-Environment-Variables + return w.Status == statusSuccess && w.Branch == "master" && !w.PullRequest +} + +func (w Webhook) DeploySummary() string { + return w.Commit[:8] + ": " + w.Message +} diff --git a/util/slack.go b/util/slack.go new file mode 100644 index 0000000..c43af80 --- /dev/null +++ b/util/slack.go @@ -0,0 +1,65 @@ +package util + +import ( + "strings" + + "github.com/lbryio/lbry.go/errors" + + "github.com/nlopes/slack" + log "github.com/sirupsen/logrus" +) + +var defaultChannel string +var defaultUsername string +var slackApi *slack.Client + +// InitSlack Initializes a slack client with the given token and sets the default channel. +func InitSlack(token string, channel string, username string) { + slackApi = slack.New(token) + defaultChannel = channel + defaultUsername = username +} + +// SendToSlackUser Sends message to a specific user. +func SendToSlackUser(user, username, message string) error { + if !strings.HasPrefix(user, "@") { + user = "@" + user + } + return sendToSlack(user, username, message) +} + +// SendToSlackChannel Sends message to a specific channel. +func SendToSlackChannel(channel, username, message string) error { + if !strings.HasPrefix(channel, "#") { + channel = "#" + channel + } + return sendToSlack(channel, username, message) +} + +// SendToSlack Sends message to the default channel. +func SendToSlack(message string) error { + + if defaultChannel == "" { + return errors.Err("no default slack channel set") + } + + return sendToSlack(defaultChannel, defaultUsername, message) +} + +func sendToSlack(channel, username, message string) error { + var err error + + if slackApi == nil { + err = errors.Err("no slack token provided") + } else { + log.Debugln("slack: " + channel + ": " + message) + _, _, err = slackApi.PostMessage(channel, message, slack.PostMessageParameters{Username: username}) + } + + if err != nil { + log.Errorln("error sending to slack: " + err.Error()) + return err + } + + return nil +} diff --git a/util/underscore.go b/util/underscore.go new file mode 100644 index 0000000..0edaf65 --- /dev/null +++ b/util/underscore.go @@ -0,0 +1,33 @@ +package util + +// https://github.com/go-pg/pg/blob/7c2d9d39a5cfc18a422c88a9f5f39d8d2cd10030/internal/underscore.go + +func isUpper(c byte) bool { + return c >= 'A' && c <= 'Z' +} + +func isLower(c byte) bool { + return !isUpper(c) +} + +func toLower(c byte) byte { + return c + 32 +} + +// Underscore converts "CamelCasedString" to "camel_cased_string". +func Underscore(s string) string { + r := make([]byte, 0, len(s)) + for i := 0; i < len(s); i++ { + c := s[i] + if isUpper(c) { + if i > 0 && i+1 < len(s) && (isLower(s[i-1]) || isLower(s[i+1])) { + r = append(r, '_', toLower(c)) + } else { + r = append(r, toLower(c)) + } + } else { + r = append(r, c) + } + } + return string(r) +} diff --git a/validator/bool.go b/validator/bool.go new file mode 100644 index 0000000..9daed17 --- /dev/null +++ b/validator/bool.go @@ -0,0 +1,34 @@ +package validator + +var ( + truthyValues = []string{"1", "yes", "y", "true"} + falseyValues = []string{"0", "no", "n", "false"} +) + +// todo: consider using strconv.ParseBool instead + +func IsTruthy(value string) bool { + for _, e := range truthyValues { + if e == value { + return true + } + } + return false +} + +func IsFalsey(value string) bool { + for _, e := range falseyValues { + if e == value { + return true + } + } + return false +} + +func IsBoolString(value string) bool { + return IsTruthy(value) || IsFalsey(value) +} + +func GetBoolStringValues() []string { + return append(truthyValues, falseyValues...) +}