Alex Grintsvayg 2020-07-28 11:05:24 -04:00
parent a05864404d
commit f942bf8025
No known key found for this signature in database
GPG key ID: AEB3F089F86A22B5
3 changed files with 77 additions and 5 deletions

View file

@ -4,12 +4,16 @@ import (
"encoding/json"
"io"
"io/ioutil"
"net/http"
"os/exec"
"strings"
"time"
"github.com/lbryio/ytsync/v5/downloader/ytdl"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/sirupsen/logrus"
)
@ -40,9 +44,76 @@ func GetVideoInformation(videoID string) (*ytdl.YtdlVideo, error) {
if err != nil {
return nil, errors.Err(err)
}
// now get an accurate time
tries := 0
GetTime:
tries++
t, err := getUploadTime(videoID)
if err != nil {
if errors.Is(err, errNotScraped) && tries <= 3 {
triggerScrape(videoID)
time.Sleep(2 * time.Second) // let them scrape it
goto GetTime
}
//return video, errors.Err(err) // just swallow this error and do fallback below
}
if t != "" {
parsed, err := time.Parse("2006-01-02, 15:04:05 (MST)", t) // this will probably be UTC, but Go's timezone parsing is fucked up. it ignores the timezone in the date
if err != nil {
return nil, errors.Err(err)
}
video.UploadDateForReal = parsed
} else {
_ = util.SendToSlack(":warning: Could not get accurate time for %s. Falling back to estimated time.", videoID)
// fall back to UploadDate from youtube-dl
video.UploadDateForReal, err = time.Parse("20060102", video.UploadDate)
if err != nil {
return nil, err
}
}
return video, nil
}
var errNotScraped = errors.Base("not yet scraped by caa.iti.gr")
func triggerScrape(videoID string) error {
res, err := http.Get("https://caa.iti.gr/verify_videoV3?twtimeline=0&url=https://www.youtube.com/watch?v=" + videoID)
if err != nil {
return errors.Err(err)
}
defer res.Body.Close()
return nil
//https://caa.iti.gr/caa/api/v4/videos/reports/h-tuxHS5lSM
}
func getUploadTime(videoID string) (string, error) {
res, err := http.Get("https://caa.iti.gr/get_verificationV3?url=https://www.youtube.com/watch?v=" + videoID)
if err != nil {
return "", errors.Err(err)
}
defer res.Body.Close()
var uploadTime struct {
Time string `json:"video_upload_time"`
Message string `json:"message"`
Status string `json:"status"`
}
err = json.NewDecoder(res.Body).Decode(&uploadTime)
if err != nil {
return "", errors.Err(err)
}
if uploadTime.Status == "ERROR1" {
return "", errNotScraped
}
return uploadTime.Time, nil
}
func run(args []string, withStdErr, withStdOut bool) ([]string, error) {
cmd := exec.Command("youtube-dl", args...)
logrus.Printf("Running command youtube-dl %s", strings.Join(args, " "))

View file

@ -1,7 +1,12 @@
package ytdl
import (
"time"
)
type YtdlVideo struct {
UploadDate string `json:"upload_date"`
UploadDateForReal time.Time // you need to manually set this since the value in the API doesn't include the time
Extractor string `json:"extractor"`
Series interface{} `json:"series"`
Format string `json:"format"`

View file

@ -92,16 +92,12 @@ var youtubeCategories = map[string]string{
func NewYoutubeVideo(directory string, videoData *ytdl.YtdlVideo, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) (*YoutubeVideo, error) {
// youtube-dl returns times in local timezone sometimes. this could break in the future
// maybe we can file a PR to choose the timezone we want from youtube-dl
publishedAt, err := time.ParseInLocation("20060102", videoData.UploadDate, time.Local)
if err != nil {
return nil, errors.Err(err)
}
return &YoutubeVideo{
id: videoData.ID,
title: videoData.Title,
description: videoData.Description,
playlistPosition: playlistPosition,
publishedAt: publishedAt,
publishedAt: videoData.UploadDateForReal,
dir: directory,
youtubeInfo: videoData,
awsConfig: awsConfig,