ytsync/sdk/api.go

368 lines
10 KiB
Go
Raw Normal View History

2018-09-05 23:35:59 +02:00
package sdk
import (
"encoding/json"
"fmt"
2018-09-05 23:35:59 +02:00
"io/ioutil"
"net/http"
"net/url"
"regexp"
2018-09-05 23:35:59 +02:00
"strconv"
"strings"
2018-09-05 23:35:59 +02:00
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/null"
2019-02-27 14:38:43 +01:00
"github.com/lbryio/ytsync/util"
2019-02-27 14:38:43 +01:00
log "github.com/sirupsen/logrus"
2018-09-05 23:35:59 +02:00
)
const (
MaxReasonLength = 490
2018-09-05 23:35:59 +02:00
)
type APIConfig struct {
YoutubeAPIKey string
ApiURL string
ApiToken string
HostName string
}
2018-09-26 06:08:18 +02:00
type SyncProperties struct {
2018-09-05 23:35:59 +02:00
SyncFrom int64
SyncUntil int64
YoutubeChannelID string
}
type SyncFlags struct {
StopOnError bool
TakeOverExistingChannel bool
SkipSpaceCheck bool
SyncUpdate bool
SingleRun bool
RemoveDBUnpublished bool
UpgradeMetadata bool
DisableTransfers bool
}
type Fee struct {
Amount string `json:"amount"`
Address string `json:"address"`
Currency string `json:"currency"`
}
2018-09-05 23:35:59 +02:00
type YoutubeChannel struct {
2018-12-24 23:18:13 +01:00
ChannelId string `json:"channel_id"`
TotalVideos uint `json:"total_videos"`
DesiredChannelName string `json:"desired_channel_name"`
Fee *Fee `json:"fee"`
ChannelClaimID string `json:"channel_claim_id"`
TransferState int `json:"transfer_state"`
PublishAddress string `json:"publish_address"`
2019-09-24 20:42:17 +02:00
PublicKey string `json:"public_key"`
2018-09-05 23:35:59 +02:00
}
2018-09-26 06:08:18 +02:00
func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeChannel, error) {
2018-09-05 23:35:59 +02:00
type apiJobsResponse struct {
Success bool `json:"success"`
Error null.String `json:"error"`
Data []YoutubeChannel `json:"data"`
}
endpoint := a.ApiURL + "/yt/jobs"
res, err := http.PostForm(endpoint, url.Values{
2018-09-05 23:35:59 +02:00
"auth_token": {a.ApiToken},
"sync_status": {status},
"min_videos": {strconv.Itoa(1)},
"after": {strconv.Itoa(int(cp.SyncFrom))},
"before": {strconv.Itoa(int(cp.SyncUntil))},
"sync_server": {a.HostName},
"channel_id": {cp.YoutubeChannelID},
})
if err != nil {
return nil, errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
2019-10-25 18:35:09 +02:00
log.Debugln(string(body))
time.Sleep(30 * time.Second)
return a.FetchChannels(status, cp)
}
2018-09-05 23:35:59 +02:00
var response apiJobsResponse
err = json.Unmarshal(body, &response)
2018-09-05 23:35:59 +02:00
if err != nil {
return nil, errors.Err(err)
2018-09-05 23:35:59 +02:00
}
if response.Data == nil {
return nil, errors.Err(response.Error)
}
log.Printf("Fetched channels: %d", len(response.Data))
return response.Data, nil
}
type SyncedVideo struct {
VideoID string `json:"video_id"`
Published bool `json:"published"`
FailureReason string `json:"failure_reason"`
ClaimName string `json:"claim_name"`
ClaimID string `json:"claim_id"`
Size int64 `json:"size"`
MetadataVersion int8 `json:"metadata_version"`
Transferred bool `json:"transferred"`
2018-09-05 23:35:59 +02:00
}
func sanitizeFailureReason(s *string) {
re := regexp.MustCompile("[[:^ascii:]]")
*s = strings.Replace(re.ReplaceAllLiteralString(*s, ""), "\n", " ", -1)
if len(*s) > MaxReasonLength {
*s = (*s)[:MaxReasonLength]
}
}
2019-07-26 05:46:35 +02:00
func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
type apiSetChannelCertResponse struct {
Success bool `json:"success"`
Error null.String `json:"error"`
Data string `json:"data"`
}
2019-07-26 05:46:35 +02:00
endpoint := a.ApiURL + "/yt/channel_cert"
res, err := http.PostForm(endpoint, url.Values{
"channel_claim_id": {channelID},
"channel_cert": {certHex},
"auth_token": {a.ApiToken},
2019-07-26 05:46:35 +02:00
})
if err != nil {
return errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(body)
time.Sleep(30 * time.Second)
return a.SetChannelCert(certHex, channelID)
}
var response apiSetChannelCertResponse
err = json.Unmarshal(body, &response)
2019-07-26 05:46:35 +02:00
if err != nil {
return errors.Err(err)
}
if !response.Error.IsNull() {
return errors.Err(response.Error.String)
2019-07-26 05:46:35 +02:00
}
return nil
}
func (a *APIConfig) SetChannelStatus(channelID string, status string, failureReason string, transferState *int) (map[string]SyncedVideo, map[string]bool, error) {
2018-09-05 23:35:59 +02:00
type apiChannelStatusResponse struct {
Success bool `json:"success"`
Error null.String `json:"error"`
Data []SyncedVideo `json:"data"`
}
endpoint := a.ApiURL + "/yt/channel_status"
sanitizeFailureReason(&failureReason)
params := url.Values{
2018-09-05 23:35:59 +02:00
"channel_id": {channelID},
"sync_server": {a.HostName},
"auth_token": {a.ApiToken},
"sync_status": {status},
"failure_reason": {failureReason},
}
if transferState != nil {
params.Add("transfer_state", strconv.Itoa(*transferState))
}
res, err := http.PostForm(endpoint, params)
if err != nil {
return nil, nil, errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
2019-10-25 18:35:09 +02:00
if res.StatusCode >= http.StatusInternalServerError {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
2019-10-25 18:35:09 +02:00
log.Debugln(string(body))
time.Sleep(30 * time.Second)
return a.SetChannelStatus(channelID, status, failureReason, transferState)
}
2018-09-05 23:35:59 +02:00
var response apiChannelStatusResponse
err = json.Unmarshal(body, &response)
2018-09-05 23:35:59 +02:00
if err != nil {
return nil, nil, errors.Err(err)
2018-09-05 23:35:59 +02:00
}
if !response.Error.IsNull() {
return nil, nil, errors.Err(response.Error.String)
}
if response.Data != nil {
svs := make(map[string]SyncedVideo)
claimNames := make(map[string]bool)
for _, v := range response.Data {
svs[v.VideoID] = v
if v.ClaimName != "" {
claimNames[v.ClaimName] = v.Published
}
2018-09-05 23:35:59 +02:00
}
return svs, claimNames, nil
}
return nil, nil, errors.Err("invalid API response. Status code: %d", res.StatusCode)
}
2018-12-25 01:23:40 +01:00
func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) error {
type apiChannelStatusResponse struct {
Success bool `json:"success"`
Error null.String `json:"error"`
Data string `json:"data"`
}
endpoint := a.ApiURL + "/yt/set_channel_claim_id"
res, err := http.PostForm(endpoint, url.Values{
2018-12-25 01:23:40 +01:00
"channel_id": {channelID},
"auth_token": {a.ApiToken},
"channel_claim_id": {channelClaimID},
})
if err != nil {
return errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
2019-10-25 18:35:09 +02:00
log.Debugln(string(body))
time.Sleep(30 * time.Second)
return a.SetChannelClaimID(channelID, channelClaimID)
}
2018-12-25 01:23:40 +01:00
var response apiChannelStatusResponse
err = json.Unmarshal(body, &response)
2018-12-25 01:23:40 +01:00
if err != nil {
return errors.Err(err)
2018-12-25 01:23:40 +01:00
}
if !response.Error.IsNull() {
return errors.Err(response.Error.String)
}
if response.Data != "ok" {
return errors.Err("Unexpected API response")
}
return nil
}
2018-09-05 23:35:59 +02:00
const (
VideoStatusPublished = "published"
VideoStatusUpgradeFailed = "upgradefailed"
VideoStatusFailed = "failed"
2018-09-05 23:35:59 +02:00
)
func (a *APIConfig) DeleteVideos(videos []string) error {
2019-06-06 16:31:35 +02:00
endpoint := a.ApiURL + "/yt/video_delete"
videoIDs := strings.Join(videos, ",")
vals := url.Values{
2019-06-06 16:31:35 +02:00
"video_ids": {videoIDs},
"auth_token": {a.ApiToken},
}
res, err := http.PostForm(endpoint, vals)
if err != nil {
return errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
2019-10-25 18:35:09 +02:00
log.Debugln(string(body))
time.Sleep(30 * time.Second)
return a.DeleteVideos(videos)
}
2019-06-06 16:31:35 +02:00
var response struct {
Success bool `json:"success"`
Error null.String `json:"error"`
Data null.String `json:"data"`
}
err = json.Unmarshal(body, &response)
if err != nil {
return errors.Err(err)
}
2019-06-06 16:31:35 +02:00
if !response.Error.IsNull() {
return errors.Err(response.Error.String)
}
2019-06-06 16:31:35 +02:00
if !response.Data.IsNull() && response.Data.String == "ok" {
return nil
}
2019-06-06 16:31:35 +02:00
return errors.Err("invalid API response. Status code: %d", res.StatusCode)
}
type VideoStatus struct {
ChannelID string
VideoID string
Status string
ClaimID string
ClaimName string
FailureReason string
Size *int64
MetaDataVersion uint
IsTransferred *bool
}
func (a *APIConfig) MarkVideoStatus(status VideoStatus) error {
2018-09-05 23:35:59 +02:00
endpoint := a.ApiURL + "/yt/video_status"
sanitizeFailureReason(&status.FailureReason)
2018-09-05 23:35:59 +02:00
vals := url.Values{
"youtube_channel_id": {status.ChannelID},
"video_id": {status.VideoID},
"status": {status.Status},
2018-09-05 23:35:59 +02:00
"auth_token": {a.ApiToken},
}
if status.Status == VideoStatusPublished || status.Status == VideoStatusUpgradeFailed {
if status.ClaimID == "" || status.ClaimName == "" {
return errors.Err("claimID (%s) or claimName (%s) missing", status.ClaimID, status.ClaimName)
2018-09-05 23:35:59 +02:00
}
vals.Add("published_at", strconv.FormatInt(time.Now().Unix(), 10))
vals.Add("claim_id", status.ClaimID)
vals.Add("claim_name", status.ClaimName)
if status.MetaDataVersion > 0 {
vals.Add("metadata_version", fmt.Sprintf("%d", status.MetaDataVersion))
2019-06-06 23:25:31 +02:00
}
if status.Size != nil {
vals.Add("size", strconv.FormatInt(*status.Size, 10))
2018-09-05 23:35:59 +02:00
}
}
if status.FailureReason != "" {
vals.Add("failure_reason", status.FailureReason)
}
if status.IsTransferred != nil {
vals.Add("transferred", strconv.FormatBool(*status.IsTransferred))
2018-09-05 23:35:59 +02:00
}
res, err := http.PostForm(endpoint, vals)
if err != nil {
return errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
2019-10-25 18:35:09 +02:00
log.Debugln(string(body))
time.Sleep(30 * time.Second)
return a.MarkVideoStatus(status)
}
2018-09-05 23:35:59 +02:00
var response struct {
Success bool `json:"success"`
Error null.String `json:"error"`
Data null.String `json:"data"`
}
err = json.Unmarshal(body, &response)
2018-09-05 23:35:59 +02:00
if err != nil {
return err
}
if !response.Error.IsNull() {
return errors.Err(response.Error.String)
}
if !response.Data.IsNull() && response.Data.String == "ok" {
return nil
}
return errors.Err("invalid API response. Status code: %d", res.StatusCode)
}