WIP: trying to get the accurate api to work

downloader/downloader.go

@@ -2,13 +2,16 @@ package downloader
 
 import (
 	"encoding/json"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"os/exec"
 	"strings"
 	"time"
 
+	"github.com/davecgh/go-spew/spew"
 	"github.com/lbryio/ytsync/v5/downloader/ytdl"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
@@ -34,30 +37,38 @@ func GetPlaylistVideoIDs(channelName string, maxVideos int) ([]string, error) {
 }
 
 func GetVideoInformation(videoID string) (*ytdl.YtdlVideo, error) {
-	args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID}
+	//args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID}
-	results, err := run(args, false, true)
+	//results, err := run(args, false, true)
-	if err != nil {
+	//if err != nil {
-		return nil, errors.Err(err)
+	//	return nil, errors.Err(err)
-	}
+	//}
 	var video *ytdl.YtdlVideo
-	err = json.Unmarshal([]byte(results[0]), &video)
+	//err = json.Unmarshal([]byte(results[0]), &video)
-	if err != nil {
+	//if err != nil {
-		return nil, errors.Err(err)
+	//	return nil, errors.Err(err)
-	}
+	//}
+
+	video = &ytdl.YtdlVideo{}
 
 	// now get an accurate time
+	const maxTries = 5
 	tries := 0
 GetTime:
 	tries++
 	t, err := getUploadTime(videoID)
 	if err != nil {
-		if errors.Is(err, errNotScraped) && tries <= 3 {
+		slack(":warning: Upload time error: %v", err)
+		if tries <= maxTries && (errors.Is(err, errNotScraped) || errors.Is(err, errUploadTimeEmpty)) {
 			triggerScrape(videoID)
 			time.Sleep(2 * time.Second) // let them scrape it
 			goto GetTime
+		} else if !errors.Is(err, errNotScraped) && !errors.Is(err, errUploadTimeEmpty) {
+			slack(":warning: Error while trying to get accurate upload time for %s: %v", videoID, err)
+			return nil, errors.Err(err)
 		}
-		//return video, errors.Err(err) // just swallow this error and do fallback below
+		// do fallback below
 	}
+	slack("After all that, upload time for %s is %s", videoID, t)
 
 	if t != "" {
 		parsed, err := time.Parse("2006-01-02, 15:04:05 (MST)", t) // this will probably be UTC, but Go's timezone parsing is fucked up. it ignores the timezone in the date
@@ -66,7 +77,7 @@ GetTime:
 		}
 		video.UploadDateForReal = parsed
 	} else {
-		_ = util.SendToSlack(":warning: Could not get accurate time for %s. Falling back to estimated time.", videoID)
+		slack(":warning: Could not get accurate time for %s. Falling back to estimated time.", videoID)
 		// fall back to UploadDate from youtube-dl
 		video.UploadDateForReal, err = time.Parse("20060102", video.UploadDate)
 		if err != nil {
@@ -78,19 +89,36 @@ GetTime:
 }
 
 var errNotScraped = errors.Base("not yet scraped by caa.iti.gr")
+var errUploadTimeEmpty = errors.Base("upload time is empty")
+
+func slack(format string, a ...interface{}) {
+	fmt.Printf(format+"\n", a...)
+	util.SendToSlack(format, a...)
+}
 
 func triggerScrape(videoID string) error {
-	res, err := http.Get("https://caa.iti.gr/verify_videoV3?twtimeline=0&url=https://www.youtube.com/watch?v=" + videoID)
+	slack("Triggering scrape for %s", videoID)
+	u, err := url.Parse("https://caa.iti.gr/verify_videoV3")
+	q := u.Query()
+	q.Set("twtimeline", "0")
+	q.Set("url", "https://www.youtube.com/watch?v="+videoID)
+	u.RawQuery = q.Encode()
+	slack("GET %s", u.String())
+	res, err := http.Get(u.String())
 	if err != nil {
 		return errors.Err(err)
 	}
 	defer res.Body.Close()
 
+	all, err := ioutil.ReadAll(res.Body)
+	spew.Dump(string(all), err)
+
 	return nil
 	//https://caa.iti.gr/caa/api/v4/videos/reports/h-tuxHS5lSM
 }
 
 func getUploadTime(videoID string) (string, error) {
+	slack("Getting upload time for %s", videoID)
 	res, err := http.Get("https://caa.iti.gr/get_verificationV3?url=https://www.youtube.com/watch?v=" + videoID)
 	if err != nil {
 		return "", errors.Err(err)
@@ -111,6 +139,10 @@ func getUploadTime(videoID string) (string, error) {
 		return "", errNotScraped
 	}
 
+	if uploadTime.Time == "" {
+		return "", errUploadTimeEmpty
+	}
+
 	return uploadTime.Time, nil
 }
 

downloader/ytdl/Video.go

@@ -37,7 +37,39 @@ type YtdlVideo struct {
 	WebpageURLBasename string      `json:"webpage_url_basename"`
 	Acodec             string      `json:"acodec"`
 	DisplayID          string      `json:"display_id"`
-	RequestedFormats   []struct {
+	//RequestedFormats   []RequestedFormat `json:"requested_formats"`
+	//AutomaticCaptions  struct{}          `json:"automatic_captions"`
+	Description        string      `json:"description"`
+	Tags               []string    `json:"tags"`
+	Track              interface{} `json:"track"`
+	RequestedSubtitles interface{} `json:"requested_subtitles"`
+	StartTime          interface{} `json:"start_time"`
+	Uploader           string      `json:"uploader"`
+	ExtractorKey       string      `json:"extractor_key"`
+	FormatID           string      `json:"format_id"`
+	EpisodeNumber      interface{} `json:"episode_number"`
+	UploaderID         string      `json:"uploader_id"`
+	//Subtitles          struct{}          `json:"subtitles"`
+	ReleaseYear interface{} `json:"release_year"`
+	Thumbnails  []Thumbnail `json:"thumbnails"`
+	License     interface{} `json:"license"`
+	Artist      interface{} `json:"artist"`
+	AgeLimit    int         `json:"age_limit"`
+	ReleaseDate interface{} `json:"release_date"`
+	AltTitle    interface{} `json:"alt_title"`
+	Thumbnail   string      `json:"thumbnail"`
+	ChannelID   string      `json:"channel_id"`
+	IsLive      interface{} `json:"is_live"`
+	Width       int         `json:"width"`
+	EndTime     interface{} `json:"end_time"`
+	WebpageURL  string      `json:"webpage_url"`
+	//Formats            []Format          `json:"formats"`
+	ChannelURL string      `json:"channel_url"`
+	Resolution interface{} `json:"resolution"`
+	Vcodec     string      `json:"vcodec"`
+}
+
+type RequestedFormat struct {
 	Asr             interface{} `json:"asr"`
 	Tbr             float64     `json:"tbr"`
 	Container       string      `json:"container"`
@@ -68,35 +100,9 @@ type YtdlVideo struct {
 	} `json:"fragments"`
 	Acodec string `json:"acodec"`
 	Abr    int    `json:"abr,omitempty"`
-	} `json:"requested_formats"`
+}
-	AutomaticCaptions struct {
+
-	} `json:"automatic_captions"`
+type Format struct {
-	Description        string      `json:"description"`
-	Tags               []string    `json:"tags"`
-	Track              interface{} `json:"track"`
-	RequestedSubtitles interface{} `json:"requested_subtitles"`
-	StartTime          interface{} `json:"start_time"`
-	Uploader           string      `json:"uploader"`
-	ExtractorKey       string      `json:"extractor_key"`
-	FormatID           string      `json:"format_id"`
-	EpisodeNumber      interface{} `json:"episode_number"`
-	UploaderID         string      `json:"uploader_id"`
-	Subtitles          struct {
-	} `json:"subtitles"`
-	ReleaseYear interface{} `json:"release_year"`
-	Thumbnails  []Thumbnail `json:"thumbnails"`
-	License     interface{} `json:"license"`
-	Artist      interface{} `json:"artist"`
-	AgeLimit    int         `json:"age_limit"`
-	ReleaseDate interface{} `json:"release_date"`
-	AltTitle    interface{} `json:"alt_title"`
-	Thumbnail   string      `json:"thumbnail"`
-	ChannelID   string      `json:"channel_id"`
-	IsLive      interface{} `json:"is_live"`
-	Width       int         `json:"width"`
-	EndTime     interface{} `json:"end_time"`
-	WebpageURL  string      `json:"webpage_url"`
-	Formats     []struct {
 	Asr             int         `json:"asr"`
 	Tbr             float64     `json:"tbr"`
 	Protocol        string      `json:"protocol"`
@@ -122,10 +128,6 @@ type YtdlVideo struct {
 	} `json:"fragments,omitempty"`
 	Acodec    string      `json:"acodec"`
 	PlayerURL interface{} `json:"player_url,omitempty"`
-	} `json:"formats"`
-	ChannelURL string      `json:"channel_url"`
-	Resolution interface{} `json:"resolution"`
-	Vcodec     string      `json:"vcodec"`
 }
 
 type Thumbnail struct {

main.go

@@ -7,8 +7,10 @@ import (
 	"os"
 	"time"
 
+	"github.com/davecgh/go-spew/spew"
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/extras/util"
+	"github.com/lbryio/ytsync/v5/downloader"
 	"github.com/lbryio/ytsync/v5/manager"
 	"github.com/lbryio/ytsync/v5/sdk"
 	ytUtils "github.com/lbryio/ytsync/v5/util"
@@ -38,6 +40,9 @@ var (
 )
 
 func main() {
+	spew.Dump(downloader.GetVideoInformation("oahaMa3XB0k"))
+	return
+
 	rand.Seed(time.Now().UnixNano())
 	log.SetLevel(log.DebugLevel)
 	http.Handle("/metrics", promhttp.Handler())

manager/ytsync.go

@@ -250,6 +250,7 @@ func (s *Sync) FullCycle() (e error) {
 	defer signal.Stop(interruptChan)
 	go func() {
 		<-interruptChan
+		util.SendToSlack("got interrupt, shutting down")
 		log.Println("Got interrupt signal, shutting down (if publishing, will shut down after current publish)")
 		s.grp.Stop()
 	}()
@@ -856,6 +857,8 @@ func (s *Sync) startWorker(workerNum int) {
 			tryCount++
 			err := s.processVideo(v)
 
+			util.SendToSlack("Tried to process %s. Error: %v", v.ID(), err)
+
 			if err != nil {
 				logMsg := fmt.Sprintf("error processing video %s: %s", v.ID(), err.Error())
 				log.Errorln(logMsg)