Add calls to internal-apis and add get released date if available.
This commit is contained in:
Mark Beamer Jr
Alex Grintsvayg
parent
d99e200178
commit
64040ea67a
5 changed files with 68 additions and 11 deletions
|
|
@ -12,6 +12,8 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/lbryio/ytsync/v5/downloader/ytdl"
|
||||
|
||||
|
|
@ -38,7 +40,9 @@ func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan)
|
|||
return videoIDs, nil
|
||||
}
|
||||
|
||||
func GetVideoInformation(videoID string, stopChan stop.Chan, ip *net.TCPAddr) (*ytdl.YtdlVideo, error) {
|
||||
const releaseTimeFormat = "2006-01-02, 15:04:05 (MST)"
|
||||
|
||||
func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Chan, ip *net.TCPAddr) (*ytdl.YtdlVideo, error) {
|
||||
args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID}
|
||||
results, err := run(args, false, true, stopChan)
|
||||
if err != nil {
|
||||
|
|
@ -55,7 +59,7 @@ func GetVideoInformation(videoID string, stopChan stop.Chan, ip *net.TCPAddr) (*
|
|||
tries := 0
|
||||
GetTime:
|
||||
tries++
|
||||
t, err := getUploadTime(videoID, ip)
|
||||
t, err := getUploadTime(config, videoID, ip)
|
||||
if err != nil {
|
||||
//slack(":warning: Upload time error: %v", err)
|
||||
if tries <= maxTries && (errors.Is(err, errNotScraped) || errors.Is(err, errUploadTimeEmpty)) {
|
||||
|
|
@ -146,9 +150,18 @@ func triggerScrape(videoID string, ip *net.TCPAddr) error {
|
|||
//https://caa.iti.gr/caa/api/v4/videos/reports/h-tuxHS5lSM
|
||||
}
|
||||
|
||||
func getUploadTime(videoID string, ip *net.TCPAddr) (string, error) {
|
||||
func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr) (string, error) {
|
||||
//slack("Getting upload time for %s", videoID)
|
||||
|
||||
release, err := config.GetReleasedDate(videoID)
|
||||
if err != nil {
|
||||
if release != nil {
|
||||
const sqlTimeFormat = "2006-01-02 15:04:05"
|
||||
sqlTime, err := time.ParseInLocation(sqlTimeFormat, release.ReleaseTime, time.UTC)
|
||||
if err != nil {
|
||||
return sqlTime.Format(releaseTimeFormat), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
client := getClient(ip)
|
||||
req, err := http.NewRequest(http.MethodGet, "https://caa.iti.gr/get_verificationV3?url=https://www.youtube.com/watch?v="+videoID, nil)
|
||||
if err != nil {
|
||||
|
|
|
|||
3
go.mod
3
go.mod
|
|
@ -4,11 +4,10 @@ replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203
|
|||
|
||||
require (
|
||||
cloud.google.com/go v0.46.3 // indirect
|
||||
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61
|
||||
github.com/Microsoft/go-winio v0.4.14 // indirect
|
||||
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a
|
||||
github.com/aws/aws-sdk-go v1.25.9
|
||||
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1
|
||||
github.com/docker/distribution v2.7.1+incompatible // indirect
|
||||
github.com/docker/docker v1.13.1
|
||||
github.com/docker/go-connections v0.4.0 // indirect
|
||||
|
|
|
|||
4
go.sum
4
go.sum
|
|
@ -11,8 +11,6 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7
|
|||
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:N5Vqww5QISEHsWHOWDEx4PzdIay3Cg0Jp7zItq2ZAro=
|
||||
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:GnKXcK+7DYNy/8w2Ex//Uql4IgfaU82Cd5rWKb7ah00=
|
||||
github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
|
||||
github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
|
||||
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
|
||||
|
|
@ -44,8 +42,6 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku
|
|||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc=
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
|
||||
github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
|
||||
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:o64h9XF42kVEUuhuer2ehqrlX8rZmvQSU0+Vpj1rF6Q=
|
||||
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:Rp8e0DCtEKwXFOC6JPJQVTz8tuGoGvw6Xfexggh/ed0=
|
||||
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
|
||||
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
|
|
|
|||
49
sdk/api.go
49
sdk/api.go
|
|
@ -408,3 +408,52 @@ func (a *APIConfig) VideoState(videoID string) (string, error) {
|
|||
}
|
||||
return "", errors.Err("invalid API response. Status code: %d", res.StatusCode)
|
||||
}
|
||||
|
||||
type VideoRelease struct {
|
||||
ID uint64 `json:"id"`
|
||||
YoutubeDataID uint64 `json:"youtube_data_id"`
|
||||
VideoID string `json:"video_id"`
|
||||
ReleaseTime string `json:"release_time""`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
|
||||
func (a *APIConfig) GetReleasedDate(videoID string) (*VideoRelease, error) {
|
||||
endpoint := a.ApiURL + "/yt/released"
|
||||
vals := url.Values{
|
||||
"video_id": {videoID},
|
||||
"auth_token": {a.ApiToken},
|
||||
}
|
||||
|
||||
res, err := http.PostForm(endpoint, vals)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode == http.StatusNotFound {
|
||||
return nil, nil
|
||||
}
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(string(body))
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.GetReleasedDate(videoID)
|
||||
}
|
||||
var response struct {
|
||||
Success bool `json:"success"`
|
||||
Error null.String `json:"error"`
|
||||
Data VideoRelease `json:"data"`
|
||||
}
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
if !response.Error.IsNull() {
|
||||
return nil, errors.Err(response.Error.String)
|
||||
}
|
||||
if response.Data.ReleaseTime != "" {
|
||||
return &response.Data, nil
|
||||
}
|
||||
return nil, errors.Err("invalid API response. Status code: %d", res.StatusCode)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -182,7 +182,7 @@ func getVideos(config *sdk.APIConfig, videoIDs []string, stopChan stop.Chan, ipP
|
|||
if state == "published" {
|
||||
continue
|
||||
}
|
||||
video, err := downloader.GetVideoInformation(videoID, stopChan, nil)
|
||||
video, err := downloader.GetVideoInformation(config, videoID, stopChan, nil)
|
||||
if err != nil {
|
||||
//ipPool.ReleaseIP(ip)
|
||||
return nil, errors.Err(err)
|
||||
|
|
|
|||
Loading…
Reference in a new issue