From 64040ea67a50a34f8be53996e3b7a2e272a13d51 Mon Sep 17 00:00:00 2001 From: Mark Beamer Jr Date: Thu, 30 Jul 2020 11:13:19 -0400 Subject: [PATCH] Add calls to internal-apis and add get released date if available. --- downloader/downloader.go | 21 +++++++++++++---- go.mod | 3 +-- go.sum | 4 ---- sdk/api.go | 49 ++++++++++++++++++++++++++++++++++++++++ ytapi/ytapi.go | 2 +- 5 files changed, 68 insertions(+), 11 deletions(-) diff --git a/downloader/downloader.go b/downloader/downloader.go index e233253..0f0293f 100644 --- a/downloader/downloader.go +++ b/downloader/downloader.go @@ -12,6 +12,8 @@ import ( "strings" "time" + "github.com/lbryio/ytsync/v5/sdk" + "github.com/davecgh/go-spew/spew" "github.com/lbryio/ytsync/v5/downloader/ytdl" @@ -38,7 +40,9 @@ func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan) return videoIDs, nil } -func GetVideoInformation(videoID string, stopChan stop.Chan, ip *net.TCPAddr) (*ytdl.YtdlVideo, error) { +const releaseTimeFormat = "2006-01-02, 15:04:05 (MST)" + +func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Chan, ip *net.TCPAddr) (*ytdl.YtdlVideo, error) { args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID} results, err := run(args, false, true, stopChan) if err != nil { @@ -55,7 +59,7 @@ func GetVideoInformation(videoID string, stopChan stop.Chan, ip *net.TCPAddr) (* tries := 0 GetTime: tries++ - t, err := getUploadTime(videoID, ip) + t, err := getUploadTime(config, videoID, ip) if err != nil { //slack(":warning: Upload time error: %v", err) if tries <= maxTries && (errors.Is(err, errNotScraped) || errors.Is(err, errUploadTimeEmpty)) { @@ -146,9 +150,18 @@ func triggerScrape(videoID string, ip *net.TCPAddr) error { //https://caa.iti.gr/caa/api/v4/videos/reports/h-tuxHS5lSM } -func getUploadTime(videoID string, ip *net.TCPAddr) (string, error) { +func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr) (string, error) { //slack("Getting upload time for %s", videoID) - + release, err := config.GetReleasedDate(videoID) + if err != nil { + if release != nil { + const sqlTimeFormat = "2006-01-02 15:04:05" + sqlTime, err := time.ParseInLocation(sqlTimeFormat, release.ReleaseTime, time.UTC) + if err != nil { + return sqlTime.Format(releaseTimeFormat), nil + } + } + } client := getClient(ip) req, err := http.NewRequest(http.MethodGet, "https://caa.iti.gr/get_verificationV3?url=https://www.youtube.com/watch?v="+videoID, nil) if err != nil { diff --git a/go.mod b/go.mod index 558dc66..c915950 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,10 @@ replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203 require ( cloud.google.com/go v0.46.3 // indirect - github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 github.com/Microsoft/go-winio v0.4.14 // indirect github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a github.com/aws/aws-sdk-go v1.25.9 - github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 // indirect + github.com/davecgh/go-spew v1.1.1 github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v1.13.1 github.com/docker/go-connections v0.4.0 // indirect diff --git a/go.sum b/go.sum index 5ba9dde..05ca74d 100644 --- a/go.sum +++ b/go.sum @@ -11,8 +11,6 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:N5Vqww5QISEHsWHOWDEx4PzdIay3Cg0Jp7zItq2ZAro= -github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:GnKXcK+7DYNy/8w2Ex//Uql4IgfaU82Cd5rWKb7ah00= github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= @@ -44,8 +42,6 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:o64h9XF42kVEUuhuer2ehqrlX8rZmvQSU0+Vpj1rF6Q= -github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:Rp8e0DCtEKwXFOC6JPJQVTz8tuGoGvw6Xfexggh/ed0= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= diff --git a/sdk/api.go b/sdk/api.go index bf66b14..f6cb2a6 100644 --- a/sdk/api.go +++ b/sdk/api.go @@ -408,3 +408,52 @@ func (a *APIConfig) VideoState(videoID string) (string, error) { } return "", errors.Err("invalid API response. Status code: %d", res.StatusCode) } + +type VideoRelease struct { + ID uint64 `json:"id"` + YoutubeDataID uint64 `json:"youtube_data_id"` + VideoID string `json:"video_id"` + ReleaseTime string `json:"release_time""` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +func (a *APIConfig) GetReleasedDate(videoID string) (*VideoRelease, error) { + endpoint := a.ApiURL + "/yt/released" + vals := url.Values{ + "video_id": {videoID}, + "auth_token": {a.ApiToken}, + } + + res, err := http.PostForm(endpoint, vals) + if err != nil { + return nil, errors.Err(err) + } + defer res.Body.Close() + body, _ := ioutil.ReadAll(res.Body) + if res.StatusCode == http.StatusNotFound { + return nil, nil + } + if res.StatusCode != http.StatusOK { + util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint) + log.Debugln(string(body)) + time.Sleep(30 * time.Second) + return a.GetReleasedDate(videoID) + } + var response struct { + Success bool `json:"success"` + Error null.String `json:"error"` + Data VideoRelease `json:"data"` + } + err = json.Unmarshal(body, &response) + if err != nil { + return nil, errors.Err(err) + } + if !response.Error.IsNull() { + return nil, errors.Err(response.Error.String) + } + if response.Data.ReleaseTime != "" { + return &response.Data, nil + } + return nil, errors.Err("invalid API response. Status code: %d", res.StatusCode) +} diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go index 1d51532..db8afd8 100644 --- a/ytapi/ytapi.go +++ b/ytapi/ytapi.go @@ -182,7 +182,7 @@ func getVideos(config *sdk.APIConfig, videoIDs []string, stopChan stop.Chan, ipP if state == "published" { continue } - video, err := downloader.GetVideoInformation(videoID, stopChan, nil) + video, err := downloader.GetVideoInformation(config, videoID, stopChan, nil) if err != nil { //ipPool.ReleaseIP(ip) return nil, errors.Err(err)