.travis.yml

@@ -2,7 +2,7 @@ os: linux
 dist: bionic
 language: go
 go:
-  - 1.17.x
+  - 1.16.3
 
 install: true
 

configs/configs.go

@@ -1,14 +1,10 @@
 package configs
 
 import (
-	"os"
-	"regexp"
-
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
-	log "github.com/sirupsen/logrus"
 	"github.com/tkanos/gonfig"
 )
 
@@ -54,22 +50,3 @@ func (s *S3Configs) GetS3AWSConfig() *aws.Config {
 		S3ForcePathStyle: aws.Bool(true),
 	}
 }
-func (c *Configs) GetHostname() string {
-	var hostname string
-
-	var err error
-	hostname, err = os.Hostname()
-	if err != nil {
-		log.Error("could not detect system hostname")
-		hostname = "ytsync_unknown"
-	}
-	reg, err := regexp.Compile("[^a-zA-Z0-9_]+")
-	if err == nil {
-		hostname = reg.ReplaceAllString(hostname, "_")
-
-	}
-	if len(hostname) > 30 {
-		hostname = hostname[0:30]
-	}
-	return hostname
-}

downloader/downloader.go

@@ -39,6 +39,7 @@ func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan,
 		if v == "" {
 			continue
 		}
+		logrus.Debugf("%d - video id %s", i, v)
 		if i >= maxVideos {
 			break
 		}
@@ -49,7 +50,7 @@ func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan,
 
 const releaseTimeFormat = "2006-01-02, 15:04:05 (MST)"
 
-func GetVideoInformation(videoID string, stopChan stop.Chan, pool *ip_manager.IPPool) (*ytdl.YtdlVideo, error) {
+func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Chan, ip *net.TCPAddr, pool *ip_manager.IPPool) (*ytdl.YtdlVideo, error) {
 	args := []string{
 		"--skip-download",
 		"--write-info-json",
@@ -79,6 +80,50 @@ func GetVideoInformation(videoID string, stopChan stop.Chan, pool *ip_manager.IP
 		return nil, errors.Err(err)
 	}
 
+	// now get an accurate time
+	const maxTries = 5
+	tries := 0
+GetTime:
+	tries++
+	t, err := getUploadTime(config, videoID, ip, video.UploadDate)
+	if err != nil {
+		//slack(":warning: Upload time error: %v", err)
+		if tries <= maxTries && (errors.Is(err, errNotScraped) || errors.Is(err, errUploadTimeEmpty) || errors.Is(err, errStatusParse) || errors.Is(err, errConnectionIssue)) {
+			err := triggerScrape(videoID, ip)
+			if err == nil {
+				time.Sleep(2 * time.Second) // let them scrape it
+				goto GetTime
+			} else {
+				//slack("triggering scrape returned error: %v", err)
+			}
+		} else if !errors.Is(err, errNotScraped) && !errors.Is(err, errUploadTimeEmpty) {
+			//slack(":warning: Error while trying to get accurate upload time for %s: %v", videoID, err)
+			if t == "" {
+				return nil, errors.Err(err)
+			} else {
+				t = "" //TODO: get rid of the other piece below?
+			}
+		}
+		// do fallback below
+	}
+	//slack("After all that, upload time for %s is %s", videoID, t)
+
+	if t != "" {
+		parsed, err := time.Parse("2006-01-02, 15:04:05 (MST)", t) // this will probably be UTC, but Go's timezone parsing is fucked up. it ignores the timezone in the date
+		if err != nil {
+			return nil, errors.Err(err)
+		}
+		//slack(":exclamation: Got an accurate time for %s", videoID)
+		video.UploadDateForReal = parsed
+	} else { //TODO: this is the piece that isn't needed!
+		slack(":warning: Could not get accurate time for %s. Falling back to time from upload ytdl: %s.", videoID, video.UploadDate)
+		// fall back to UploadDate from youtube-dl
+		video.UploadDateForReal, err = time.Parse("20060102", video.UploadDate)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	return video, nil
 }
 
@@ -168,7 +213,45 @@ func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr, uploa
 		}
 	}
 
-	return ytdlUploadDate.Format(releaseTimeFormat), nil
+	if time.Now().AddDate(0, 0, -3).After(ytdlUploadDate) {
+		return ytdlUploadDate.Format(releaseTimeFormat), nil
+	}
+	client := getClient(ip)
+	req, err := http.NewRequest(http.MethodGet, "https://caa.iti.gr/get_verificationV3?url=https://www.youtube.com/watch?v="+videoID, nil)
+	if err != nil {
+		return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
+	}
+	req.Header.Set("User-Agent", ChromeUA)
+
+	res, err := client.Do(req)
+	if err != nil {
+		return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
+	}
+	defer res.Body.Close()
+
+	var uploadTime struct {
+		Time    string `json:"video_upload_time"`
+		Message string `json:"message"`
+		Status  string `json:"status"`
+	}
+	err = json.NewDecoder(res.Body).Decode(&uploadTime)
+	if err != nil {
+		return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
+	}
+
+	if uploadTime.Status == "ERROR1" {
+		return ytdlUploadDate.Format(releaseTimeFormat), errNotScraped
+	}
+
+	if uploadTime.Status == "" && strings.HasPrefix(uploadTime.Message, "CANNOT_RETRIEVE_REPORT_FOR_VIDEO_") {
+		return ytdlUploadDate.Format(releaseTimeFormat), errors.Err("cannot retrieve report for video")
+	}
+
+	if uploadTime.Time == "" {
+		return ytdlUploadDate.Format(releaseTimeFormat), errUploadTimeEmpty
+	}
+
+	return uploadTime.Time, nil
 }
 
 func getClient(ip *net.TCPAddr) *http.Client {
@@ -194,7 +277,7 @@ func getClient(ip *net.TCPAddr) *http.Client {
 
 const (
 	GoogleBotUA             = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
-	ChromeUA                = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36"
+	ChromeUA                = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"
 	maxAttempts             = 3
 	extractionError         = "YouTube said: Unable to extract video data"
 	throttledError          = "HTTP Error 429"
@@ -285,8 +368,7 @@ func runCmd(cmd *exec.Cmd, stopChan stop.Chan) ([]string, error) {
 		return nil, errors.Err("interrupted by user")
 	case err := <-done:
 		if err != nil {
-			//return nil, errors.Prefix("yt-dlp "+strings.Join(cmd.Args, " ")+" ["+string(errorLog)+"]", err)
-			return nil, errors.Prefix(string(errorLog), err)
+			return nil, errors.Prefix("yt-dlp "+strings.Join(cmd.Args, " ")+" ["+string(errorLog)+"]", err)
 		}
 		return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
 	}

downloader/downloader_test.go

@@ -3,11 +3,7 @@ package downloader
 import (
 	"testing"
 
-	"github.com/lbryio/ytsync/v5/ip_manager"
 	"github.com/lbryio/ytsync/v5/sdk"
-
-	"github.com/lbryio/lbry.go/v2/extras/stop"
-
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 )
@@ -23,13 +19,13 @@ func TestGetPlaylistVideoIDs(t *testing.T) {
 }
 
 func TestGetVideoInformation(t *testing.T) {
-	s := stop.New()
-	ip, err := ip_manager.GetIPPool(s)
-	assert.NoError(t, err)
-	video, err := GetVideoInformation("kDGOHNpRjzc", s.Ch(), ip)
-	assert.NoError(t, err)
-	assert.NotNil(t, video)
-	logrus.Info(video.ID)
+	video, err := GetVideoInformation(nil, "zj7pXM9gE5M", nil, nil, nil)
+	if err != nil {
+		logrus.Error(err)
+	}
+	if video != nil {
+		logrus.Info(video.ID)
+	}
 }
 
 func Test_getUploadTime(t *testing.T) {
@@ -37,4 +33,5 @@ func Test_getUploadTime(t *testing.T) {
 	got, err := getUploadTime(&configs, "kDGOHNpRjzc", nil, "20060102")
 	assert.NoError(t, err)
 	t.Log(got)
+
 }

downloader/ytdl/Video.go

@@ -2,136 +2,149 @@ package ytdl
 
 import (
 	"time"
-
-	"github.com/lbryio/ytsync/v5/sdk"
-	"github.com/sirupsen/logrus"
 )
 
 type YtdlVideo struct {
-	ID                string      `json:"id"`
-	Title             string      `json:"title"`
-	Thumbnails        []Thumbnail `json:"thumbnails"`
-	Description       string      `json:"description"`
-	ChannelID         string      `json:"channel_id"`
-	Duration          int         `json:"duration"`
-	Categories        []string    `json:"categories"`
-	Tags              []string    `json:"tags"`
-	IsLive            bool        `json:"is_live"`
-	LiveStatus        string      `json:"live_status"`
-	ReleaseTimestamp  *int64      `json:"release_timestamp"`
-	uploadDateForReal *time.Time
-	Availability      string `json:"availability"`
-	ReleaseDate       string `json:"release_date"`
-	UploadDate        string `json:"upload_date"`
+	UploadDate         string      `json:"upload_date"`
+	UploadDateForReal  time.Time   // you need to manually set this since the value in the API doesn't include the time
+	Extractor          string      `json:"extractor"`
+	Series             interface{} `json:"series"`
+	Format             string      `json:"format"`
+	Vbr                interface{} `json:"vbr"`
+	Chapters           interface{} `json:"chapters"`
+	Height             int         `json:"height"`
+	LikeCount          interface{} `json:"like_count"`
+	Duration           int         `json:"duration"`
+	Fulltitle          string      `json:"fulltitle"`
+	PlaylistIndex      interface{} `json:"playlist_index"`
+	Album              interface{} `json:"album"`
+	ViewCount          int         `json:"view_count"`
+	Playlist           interface{} `json:"playlist"`
+	Title              string      `json:"title"`
+	Filename           string      `json:"_filename"`
+	Creator            interface{} `json:"creator"`
+	Ext                string      `json:"ext"`
+	ID                 string      `json:"id"`
+	DislikeCount       interface{} `json:"dislike_count"`
+	AverageRating      float64     `json:"average_rating"`
+	Abr                float64     `json:"abr"`
+	UploaderURL        string      `json:"uploader_url"`
+	Categories         []string    `json:"categories"`
+	Fps                float64     `json:"fps"`
+	StretchedRatio     interface{} `json:"stretched_ratio"`
+	SeasonNumber       interface{} `json:"season_number"`
+	Annotations        interface{} `json:"annotations"`
+	WebpageURLBasename string      `json:"webpage_url_basename"`
+	Acodec             string      `json:"acodec"`
+	DisplayID          string      `json:"display_id"`
+	//RequestedFormats   []RequestedFormat `json:"requested_formats"`
+	//AutomaticCaptions  struct{}          `json:"automatic_captions"`
+	Description        string      `json:"description"`
+	Tags               []string    `json:"tags"`
+	Track              interface{} `json:"track"`
+	RequestedSubtitles interface{} `json:"requested_subtitles"`
+	StartTime          interface{} `json:"start_time"`
+	Uploader           string      `json:"uploader"`
+	ExtractorKey       string      `json:"extractor_key"`
+	FormatID           string      `json:"format_id"`
+	EpisodeNumber      interface{} `json:"episode_number"`
+	UploaderID         string      `json:"uploader_id"`
+	//Subtitles          struct{}          `json:"subtitles"`
+	ReleaseYear interface{} `json:"release_year"`
+	Thumbnails  []Thumbnail `json:"thumbnails"`
+	License     interface{} `json:"license"`
+	Artist      interface{} `json:"artist"`
+	AgeLimit    int         `json:"age_limit"`
+	ReleaseDate interface{} `json:"release_date"`
+	AltTitle    interface{} `json:"alt_title"`
+	Thumbnail   string      `json:"thumbnail"`
+	ChannelID   string      `json:"channel_id"`
+	IsLive      interface{} `json:"is_live"`
+	Width       int         `json:"width"`
+	EndTime     interface{} `json:"end_time"`
+	WebpageURL  string      `json:"webpage_url"`
+	Formats     []Format    `json:"formats"`
+	ChannelURL  string      `json:"channel_url"`
+	Resolution  interface{} `json:"resolution"`
+	Vcodec      string      `json:"vcodec"`
+}
 
-	//WasLive           bool        `json:"was_live"`
-	//Formats              interface{}   `json:"formats"`
-	//Thumbnail            string        `json:"thumbnail"`
-	//Uploader             string        `json:"uploader"`
-	//UploaderID           string        `json:"uploader_id"`
-	//UploaderURL          string        `json:"uploader_url"`
-	//ChannelURL           string        `json:"channel_url"`
-	//ViewCount            int           `json:"view_count"`
-	//AverageRating        interface{}   `json:"average_rating"`
-	//AgeLimit             int           `json:"age_limit"`
-	//WebpageURL           string        `json:"webpage_url"`
-	//PlayableInEmbed      bool          `json:"playable_in_embed"`
-	//AutomaticCaptions    interface{}   `json:"automatic_captions"`
-	//Subtitles            interface{}   `json:"subtitles"`
-	//Chapters             interface{}   `json:"chapters"`
-	//LikeCount            int           `json:"like_count"`
-	//Channel              string        `json:"channel"`
-	//ChannelFollowerCount int           `json:"channel_follower_count"`
-	//OriginalURL          string        `json:"original_url"`
-	//WebpageURLBasename   string        `json:"webpage_url_basename"`
-	//WebpageURLDomain     string        `json:"webpage_url_domain"`
-	//Extractor            string        `json:"extractor"`
-	//ExtractorKey         string        `json:"extractor_key"`
-	//Playlist             interface{}   `json:"playlist"`
-	//PlaylistIndex        interface{}   `json:"playlist_index"`
-	//DisplayID            string        `json:"display_id"`
-	//Fulltitle            string        `json:"fulltitle"`
-	//DurationString       string        `json:"duration_string"`
-	//RequestedSubtitles   interface{}   `json:"requested_subtitles"`
-	//HasDrm               bool          `json:"__has_drm"`
-	//RequestedFormats     interface{}   `json:"requested_formats"`
-	//Format               string        `json:"format"`
-	//FormatID             string        `json:"format_id"`
-	//Ext                  string        `json:"ext"`
-	//Protocol             string        `json:"protocol"`
-	//Language             interface{}   `json:"language"`
-	//FormatNote           string        `json:"format_note"`
-	//FilesizeApprox       int           `json:"filesize_approx"`
-	//Tbr                  float64       `json:"tbr"`
-	//Width                int           `json:"width"`
-	//Height               int           `json:"height"`
-	//Resolution           string        `json:"resolution"`
-	//Fps                  int           `json:"fps"`
-	//DynamicRange         string        `json:"dynamic_range"`
-	//Vcodec               string        `json:"vcodec"`
-	//Vbr                  float64       `json:"vbr"`
-	//StretchedRatio       interface{}   `json:"stretched_ratio"`
-	//Acodec               string        `json:"acodec"`
-	//Abr                  float64       `json:"abr"`
-	//Asr                  int           `json:"asr"`
-	//Epoch                int           `json:"epoch"`
-	//Filename             string        `json:"filename"`
-	//Urls                 string        `json:"urls"`
-	//Type                 string        `json:"_type"`
+type RequestedFormat struct {
+	Asr             interface{} `json:"asr"`
+	Tbr             float64     `json:"tbr"`
+	Container       string      `json:"container"`
+	Language        interface{} `json:"language"`
+	Format          string      `json:"format"`
+	URL             string      `json:"url"`
+	Vcodec          string      `json:"vcodec"`
+	FormatNote      string      `json:"format_note"`
+	Height          int         `json:"height"`
+	Width           int         `json:"width"`
+	Ext             string      `json:"ext"`
+	FragmentBaseURL string      `json:"fragment_base_url"`
+	Filesize        interface{} `json:"filesize"`
+	Fps             float64     `json:"fps"`
+	ManifestURL     string      `json:"manifest_url"`
+	Protocol        string      `json:"protocol"`
+	FormatID        string      `json:"format_id"`
+	HTTPHeaders     struct {
+		AcceptCharset  string `json:"Accept-Charset"`
+		AcceptLanguage string `json:"Accept-Language"`
+		AcceptEncoding string `json:"Accept-Encoding"`
+		Accept         string `json:"Accept"`
+		UserAgent      string `json:"User-Agent"`
+	} `json:"http_headers"`
+	Fragments []struct {
+		Path     string  `json:"path"`
+		Duration float64 `json:"duration,omitempty"`
+	} `json:"fragments"`
+	Acodec string `json:"acodec"`
+	Abr    int    `json:"abr,omitempty"`
+}
+
+type Format struct {
+	Asr               int         `json:"asr"`
+	Filesize          int         `json:"filesize"`
+	FormatID          string      `json:"format_id"`
+	FormatNote        string      `json:"format_note"`
+	Fps               interface{} `json:"fps"`
+	Height            interface{} `json:"height"`
+	Quality           int         `json:"quality"`
+	Tbr               float64     `json:"tbr"`
+	URL               string      `json:"url"`
+	Width             interface{} `json:"width"`
+	Ext               string      `json:"ext"`
+	Vcodec            string      `json:"vcodec"`
+	Acodec            string      `json:"acodec"`
+	Abr               float64     `json:"abr,omitempty"`
+	DownloaderOptions struct {
+		HTTPChunkSize int `json:"http_chunk_size"`
+	} `json:"downloader_options,omitempty"`
+	Container   string `json:"container,omitempty"`
+	Format      string `json:"format"`
+	Protocol    string `json:"protocol"`
+	HTTPHeaders struct {
+		UserAgent      string `json:"User-Agent"`
+		AcceptCharset  string `json:"Accept-Charset"`
+		Accept         string `json:"Accept"`
+		AcceptEncoding string `json:"Accept-Encoding"`
+		AcceptLanguage string `json:"Accept-Language"`
+	} `json:"http_headers"`
+	Vbr float64 `json:"vbr,omitempty"`
 }
 
 type Thumbnail struct {
 	URL        string `json:"url"`
-	Preference int    `json:"preference"`
+	Width      int    `json:"width"`
+	Resolution string `json:"resolution"`
 	ID         string `json:"id"`
-	Height     int    `json:"height,omitempty"`
-	Width      int    `json:"width,omitempty"`
-	Resolution string `json:"resolution,omitempty"`
+	Height     int    `json:"height"`
 }
 
-func (v *YtdlVideo) GetUploadTime() time.Time {
-	//priority list:
-	// release timestamp from yt
-	// release timestamp from morty
-	// release date from yt
-	// upload date from yt
-	if v.uploadDateForReal != nil {
-		return *v.uploadDateForReal
-	}
-
-	var ytdlReleaseTimestamp time.Time
-	if v.ReleaseTimestamp != nil && *v.ReleaseTimestamp > 0 {
-		ytdlReleaseTimestamp = time.Unix(*v.ReleaseTimestamp, 0).UTC()
-	}
-	//get morty timestamp
-	var mortyReleaseTimestamp time.Time
-	mortyRelease, err := sdk.GetAPIsConfigs().GetReleasedDate(v.ID)
-	if err != nil {
-		logrus.Error(err)
-	} else if mortyRelease != nil {
-		mortyReleaseTimestamp, err = time.ParseInLocation(time.RFC3339, mortyRelease.ReleaseTime, time.UTC)
-		if err != nil {
-			logrus.Error(err)
-		}
-	}
-
-	ytdlReleaseDate, err := time.Parse("20060102", v.ReleaseDate)
-	if err != nil {
-		logrus.Error(err)
-	}
-	ytdlUploadDate, err := time.Parse("20060102", v.UploadDate)
-	if err != nil {
-		logrus.Error(err)
-	}
-	if !ytdlReleaseTimestamp.IsZero() {
-		v.uploadDateForReal = &ytdlReleaseTimestamp
-	} else if !mortyReleaseTimestamp.IsZero() {
-		v.uploadDateForReal = &mortyReleaseTimestamp
-	} else if !ytdlReleaseDate.IsZero() {
-		v.uploadDateForReal = &ytdlReleaseDate
-	} else {
-		v.uploadDateForReal = &ytdlUploadDate
-	}
-
-	return *v.uploadDateForReal
+type HTTPHeaders struct {
+	AcceptCharset  string `json:"Accept-Charset"`
+	AcceptLanguage string `json:"Accept-Language"`
+	AcceptEncoding string `json:"Accept-Encoding"`
+	Accept         string `json:"Accept"`
+	UserAgent      string `json:"User-Agent"`
 }

e2e/docker-compose.yml

@@ -21,7 +21,7 @@ services:
   ## Wallet Server ##
   ###################
   walletserver:
-    image: lbry/wallet-server:v0.101.1
+    image: lbry/wallet-server:latest-release
     restart: always
     environment:
       - DB_DIRECTORY=/database
@@ -81,7 +81,6 @@ services:
       - walletserver
     environment:
       - LBRY_STREAMING_SERVER=0.0.0.0:5280
-      - LBRY_FEE_PER_NAME_CHAR=0
     volumes:
       - "./persist/.lbrynet:/home/lbrynet"
       - ".:/etc/lbry" #Put your daemon_settings.yml here
@@ -110,7 +109,7 @@ services:
   ## Internal APIs ##
   ###################
   internalapis:
-    image: odyseeteam/internal-apis:master
+    image: lbry/internal-apis:master
     restart: "no"
     ports:
       - "15400:8080"
@@ -128,7 +127,7 @@ services:
   ## Chainquery ##
   ################
   chainquery:
-    image: odyseeteam/chainquery:master
+    image: lbry/chainquery:master
     restart: "no"
     ports:
       - 6300:6300

go.mod

@@ -1,5 +1,3 @@
-go 1.17
-
 module github.com/lbryio/ytsync/v5
 
 replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
@@ -8,144 +6,38 @@ replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203
 //replace github.com/lbryio/reflector.go => /home/niko/go/src/github.com/lbryio/reflector.go/
 
 require (
+	github.com/Microsoft/go-winio v0.4.14 // indirect
 	github.com/abadojack/whatlanggo v1.0.1
-	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d
-	github.com/aws/aws-sdk-go v1.44.6
+	github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb
+	github.com/aws/aws-sdk-go v1.25.9
 	github.com/davecgh/go-spew v1.1.1
-	github.com/docker/docker v20.10.17+incompatible
-	github.com/lbryio/lbry.go/v2 v2.7.2-0.20220815204100-2adb8af5b68c
-	github.com/lbryio/reflector.go v1.1.3-0.20220730181028-f5d30b1a6e79
-	github.com/mitchellh/go-ps v1.0.0
-	github.com/prometheus/client_golang v1.12.1
-	github.com/shopspring/decimal v1.3.1
-	github.com/sirupsen/logrus v1.9.0
-	github.com/spf13/cobra v1.4.0
-	github.com/stretchr/testify v1.7.1
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/docker v1.13.1
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
+	github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
+	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
+	github.com/hashicorp/memberlist v0.1.5 // indirect
+	github.com/hashicorp/serf v0.8.5 // indirect
+	github.com/kr/pretty v0.2.1 // indirect
+	github.com/lbryio/lbry.go/v2 v2.7.2-0.20210824154606-3e18b74da08b
+	github.com/lbryio/reflector.go v1.1.3-0.20210412225256-4392c9724262
+	github.com/miekg/dns v1.1.22 // indirect
+	github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
+	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
+	github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 // indirect
+	github.com/prometheus/client_golang v0.9.3
+	github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5
+	github.com/sirupsen/logrus v1.4.2
+	github.com/spf13/cobra v0.0.5
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/testify v1.7.0
 	github.com/tkanos/gonfig v0.0.0-20210106201359-53e13348de2f
-	github.com/vbauerster/mpb/v7 v7.4.1
-	gopkg.in/vansante/go-ffprobe.v2 v2.0.3
+	github.com/vbauerster/mpb/v7 v7.0.2
+	google.golang.org/appengine v1.6.5 // indirect
+	gopkg.in/ini.v1 v1.60.2 // indirect
+	gopkg.in/vansante/go-ffprobe.v2 v2.0.2
 	gotest.tools v2.2.0+incompatible
 )
 
-require (
-	github.com/Microsoft/go-winio v0.5.1 // indirect
-	github.com/VividCortex/ewma v1.2.0 // indirect
-	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
-	github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
-	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/bluele/gcache v0.0.2 // indirect
-	github.com/brk0v/directio v0.0.0-20190225130936-69406e757cf7 // indirect
-	github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 // indirect
-	github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect
-	github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d // indirect
-	github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect
-	github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect
-	github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/cheekybits/genny v1.0.0 // indirect
-	github.com/docker/distribution v2.8.1+incompatible // indirect
-	github.com/docker/go-connections v0.4.0 // indirect
-	github.com/docker/go-units v0.4.0 // indirect
-	github.com/ekyoung/gin-nice-recovery v0.0.0-20160510022553-1654dca486db // indirect
-	github.com/fatih/structs v1.1.0 // indirect
-	github.com/fsnotify/fsnotify v1.4.9 // indirect
-	github.com/ghodss/yaml v1.0.0 // indirect
-	github.com/gin-contrib/sse v0.1.0 // indirect
-	github.com/gin-gonic/gin v1.7.7 // indirect
-	github.com/go-errors/errors v1.1.1 // indirect
-	github.com/go-ini/ini v1.48.0 // indirect
-	github.com/go-playground/locales v0.13.0 // indirect
-	github.com/go-playground/universal-translator v0.17.0 // indirect
-	github.com/go-playground/validator/v10 v10.4.1 // indirect
-	github.com/go-sql-driver/mysql v1.6.0 // indirect
-	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
-	github.com/gofrs/uuid v3.2.0+incompatible // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/btree v1.0.1 // indirect
-	github.com/google/go-cmp v0.5.7 // indirect
-	github.com/gorilla/mux v1.8.0 // indirect
-	github.com/gorilla/rpc v1.2.0 // indirect
-	github.com/gorilla/websocket v1.4.2 // indirect
-	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
-	github.com/hashicorp/go-msgpack v0.5.5 // indirect
-	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
-	github.com/hashicorp/golang-lru v0.5.4 // indirect
-	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/hashicorp/memberlist v0.3.0 // indirect
-	github.com/hashicorp/serf v0.9.7 // indirect
-	github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 // indirect
-	github.com/johntdyer/slackrus v0.0.0-20211215141436-33e4a270affb // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/karrick/godirwalk v1.17.0 // indirect
-	github.com/kr/text v0.2.0 // indirect
-	github.com/lbryio/chainquery v1.9.0 // indirect
-	github.com/lbryio/lbry.go v1.1.2 // indirect
-	github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6 // indirect
-	github.com/leodido/go-urn v1.2.0 // indirect
-	github.com/lucas-clemente/quic-go v0.28.1 // indirect
-	github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 // indirect
-	github.com/magiconair/properties v1.8.1 // indirect
-	github.com/marten-seemann/qpack v0.2.1 // indirect
-	github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
-	github.com/marten-seemann/qtls-go1-17 v0.1.2 // indirect
-	github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect
-	github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1 // indirect
-	github.com/mattn/go-isatty v0.0.12 // indirect
-	github.com/mattn/go-runewidth v0.0.13 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
-	github.com/miekg/dns v1.1.41 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
-	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/morikuni/aec v1.0.0 // indirect
-	github.com/nxadm/tail v1.4.8 // indirect
-	github.com/onsi/ginkgo v1.16.4 // indirect
-	github.com/onsi/gomega v1.17.0 // indirect
-	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
-	github.com/pelletier/go-toml v1.9.3 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.32.1 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
-	github.com/rivo/uniseg v0.2.0 // indirect
-	github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
-	github.com/slack-go/slack v0.10.3 // indirect
-	github.com/spf13/afero v1.4.1 // indirect
-	github.com/spf13/cast v1.4.1 // indirect
-	github.com/spf13/jwalterweatherman v1.0.0 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/spf13/viper v1.7.1 // indirect
-	github.com/subosito/gotenv v1.2.0 // indirect
-	github.com/ugorji/go/codec v1.1.7 // indirect
-	github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d // indirect
-	github.com/volatiletech/null v8.0.0+incompatible // indirect
-	github.com/volatiletech/sqlboiler v3.4.0+incompatible // indirect
-	github.com/ybbus/jsonrpc v2.1.2+incompatible // indirect
-	go.uber.org/atomic v1.9.0 // indirect
-	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
-	golang.org/x/mod v0.4.2 // indirect
-	golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
-	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
-	golang.org/x/tools v0.1.5 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
-	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
-	gopkg.in/ini.v1 v1.60.2 // indirect
-	gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 // indirect
-	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	gotest.tools/v3 v3.2.0 // indirect
-)
+go 1.13

main.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/lbryio/ytsync/v5/configs"
 	"github.com/lbryio/ytsync/v5/manager"
+	"github.com/lbryio/ytsync/v5/sdk"
 	"github.com/lbryio/ytsync/v5/shared"
 	ytUtils "github.com/lbryio/ytsync/v5/util"
 
@@ -32,10 +33,6 @@ var (
 func main() {
 	rand.Seed(time.Now().UnixNano())
 	log.SetLevel(log.DebugLevel)
-	customFormatter := new(log.TextFormatter)
-	customFormatter.TimestampFormat = "2006-01-02 15:04:05"
-	customFormatter.FullTimestamp = true
-	log.SetFormatter(customFormatter)
 	http.Handle("/metrics", promhttp.Handler())
 	go func() {
 		log.Error(http.ListenAndServe(":2112", nil))
@@ -78,11 +75,22 @@ func ytSync(cmd *cobra.Command, args []string) {
 	if err != nil {
 		log.Fatalf("could not parse configuration file: %s", errors.FullTrace(err))
 	}
+	var hostname string
 
 	if configs.Configuration.SlackToken == "" {
 		log.Error("A slack token was not present in the config! Slack messages disabled!")
 	} else {
-		util.InitSlack(configs.Configuration.SlackToken, configs.Configuration.SlackChannel, configs.Configuration.GetHostname())
+		var err error
+		hostname, err = os.Hostname()
+		if err != nil {
+			log.Error("could not detect system hostname")
+			hostname = "ytsync-unknown"
+		}
+		if len(hostname) > 30 {
+			hostname = hostname[0:30]
+		}
+
+		util.InitSlack(configs.Configuration.SlackToken, configs.Configuration.SlackChannel, hostname)
 	}
 
 	if cliFlags.Status != "" && !util.InSlice(cliFlags.Status, shared.SyncStatuses) {
@@ -123,9 +131,17 @@ func ytSync(cmd *cobra.Command, args []string) {
 
 	blobsDir := ytUtils.GetBlobsDir()
 
+	apiConfig := &sdk.APIConfig{
+		ApiURL:   configs.Configuration.InternalApisEndpoint,
+		ApiToken: configs.Configuration.InternalApisAuthToken,
+		HostName: hostname,
+	}
+
 	sm := manager.NewSyncManager(
 		cliFlags,
 		blobsDir,
+		configs.Configuration.LbrycrdString,
+		apiConfig,
 	)
 	err = sm.Start()
 	if err != nil {

manager/manager.go

@@ -8,7 +8,6 @@ import (
 	"time"
 
 	"github.com/lbryio/ytsync/v5/blobs_reflector"
-	"github.com/lbryio/ytsync/v5/configs"
 	"github.com/lbryio/ytsync/v5/ip_manager"
 	"github.com/lbryio/ytsync/v5/namer"
 	"github.com/lbryio/ytsync/v5/sdk"
@@ -30,12 +29,12 @@ type SyncManager struct {
 	channelsToSync []Sync
 }
 
-func NewSyncManager(cliFlags shared.SyncFlags, blobsDir string) *SyncManager {
+func NewSyncManager(cliFlags shared.SyncFlags, blobsDir, lbrycrdDsn string, apiConfig *sdk.APIConfig) *SyncManager {
 	return &SyncManager{
 		CliFlags:   cliFlags,
 		blobsDir:   blobsDir,
-		LbrycrdDsn: configs.Configuration.LbrycrdString,
-		ApiConfig:  sdk.GetAPIsConfigs(),
+		LbrycrdDsn: lbrycrdDsn,
+		ApiConfig:  apiConfig,
 	}
 }
 func (s *SyncManager) enqueueChannel(channel *shared.YoutubeChannel) {
@@ -138,7 +137,7 @@ func (s *SyncManager) Start() error {
 					"WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR",
 					"NotEnoughFunds",
 					"no space left on device",
-					"there was a problem uploading the wallet",
+					"failure uploading wallet",
 					"the channel in the wallet is different than the channel in the database",
 					"this channel does not belong to this wallet!",
 					"You already have a stream claim published under the name",

manager/s3_storage.go

@@ -4,7 +4,6 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/lbryio/ytsync/v5/configs"
 	"github.com/lbryio/ytsync/v5/util"
@@ -216,22 +215,13 @@ func (s *Sync) uploadWallet() error {
 	}
 	defer file.Close()
 
-	start := time.Now()
-
-	for time.Since(start) < 30*time.Minute {
-		_, err = uploader.Upload(&s3manager.UploadInput{
-			Bucket: aws.String(configs.Configuration.WalletS3Config.Bucket),
-			Key:    key,
-			Body:   file,
-		})
-		if err != nil {
-			time.Sleep(30 * time.Second)
-			continue
-		}
-		break
-	}
+	_, err = uploader.Upload(&s3manager.UploadInput{
+		Bucket: aws.String(configs.Configuration.WalletS3Config.Bucket),
+		Key:    key,
+		Body:   file,
+	})
 	if err != nil {
-		return errors.Prefix("there was a problem uploading the wallet to S3", errors.Err(err))
+		return err
 	}
 	log.Println("wallet uploaded to S3")
 

manager/setup.go

@@ -103,9 +103,6 @@ func (s *Sync) walletSetup() error {
 		videosOnYoutube = s.Manager.CliFlags.VideosToSync(s.DbChannelData.TotalSubscribers)
 	}
 	unallocatedVideos := videosOnYoutube - (publishedCount + failedCount)
-	if unallocatedVideos < 0 {
-		unallocatedVideos = 0
-	}
 	channelFee := channelClaimAmount
 	channelAlreadyClaimed := s.DbChannelData.ChannelClaimID != ""
 	if channelAlreadyClaimed {
@@ -113,7 +110,7 @@ func (s *Sync) walletSetup() error {
 	}
 	requiredBalance := float64(unallocatedVideos)*(publishAmount+estimatedMaxTxFee) + channelFee
 	if s.Manager.CliFlags.UpgradeMetadata {
-		requiredBalance += float64(notUpgradedCount) * estimatedMaxTxFee
+		requiredBalance += float64(notUpgradedCount) * 0.001
 	}
 
 	refillAmount := 0.0
@@ -130,12 +127,6 @@ func (s *Sync) walletSetup() error {
 		if err != nil {
 			return errors.Err(err)
 		}
-	} else if balance > requiredBalance {
-		extraLBC := balance - requiredBalance
-		if extraLBC > 5 {
-			sendBackAmount := extraLBC - 1
-			logUtils.SendInfoToSlack("channel %s has %.1f credits which is %.1f more than it requires (%.1f). We should send at least %.1f that back.", s.DbChannelData.ChannelId, balance, extraLBC, requiredBalance, sendBackAmount)
-		}
 	}
 
 	claimAddress, err := s.daemon.AddressList(nil, nil, 1, 20)
@@ -375,12 +366,14 @@ func (s *Sync) ensureChannelOwnership() error {
 
 	channelUsesOldMetadata := false
 	if channelToUse != nil {
-		channelUsesOldMetadata = channelToUse.Value.GetThumbnail() == nil || (len(channelToUse.Value.GetLanguages()) == 0 && s.DbChannelData.Language != "")
+		channelUsesOldMetadata = channelToUse.Value.GetThumbnail() == nil
 		if !channelUsesOldMetadata {
 			return nil
 		}
 	}
 
+	channelBidAmount := channelClaimAmount
+
 	balanceResp, err := s.daemon.AccountBalance(nil)
 	if err != nil {
 		return err
@@ -392,8 +385,8 @@ func (s *Sync) ensureChannelOwnership() error {
 		return errors.Err(err)
 	}
 
-	if balance.LessThan(decimal.NewFromFloat(channelClaimAmount)) {
-		err = s.addCredits(channelClaimAmount + estimatedMaxTxFee*3)
+	if balance.LessThan(decimal.NewFromFloat(channelBidAmount)) {
+		err = s.addCredits(channelBidAmount + 0.3)
 		if err != nil {
 			return err
 		}
@@ -431,16 +424,18 @@ func (s *Sync) ensureChannelOwnership() error {
 	}
 
 	var languages []string = nil
-	if s.DbChannelData.Language != "" {
-		languages = []string{s.DbChannelData.Language}
-	}
-
+	//we don't have this data without the API
+	//if channelInfo.DefaultLanguage != "" {
+	//	if channelInfo.DefaultLanguage == "iw" {
+	//		channelInfo.DefaultLanguage = "he"
+	//	}
+	//	languages = []string{channelInfo.DefaultLanguage}
+	//}
 	var locations []jsonrpc.Location = nil
 	if channelInfo.Topbar.DesktopTopbarRenderer.CountryCode != "" {
 		locations = []jsonrpc.Location{{Country: &channelInfo.Topbar.DesktopTopbarRenderer.CountryCode}}
 	}
 	var c *jsonrpc.TransactionSummary
-	var recoveredChannelClaimID string
 	claimCreateOptions := jsonrpc.ClaimCreateOptions{
 		Title:        &channelInfo.Microformat.MicroformatDataRenderer.Title,
 		Description:  &channelInfo.Metadata.ChannelMetadataRenderer.Description,
@@ -450,20 +445,12 @@ func (s *Sync) ensureChannelOwnership() error {
 		ThumbnailURL: &thumbnailURL,
 	}
 	if channelUsesOldMetadata {
-		da, err := s.getDefaultAccount()
-		if err != nil {
-			return err
-		}
 		if s.DbChannelData.TransferState <= 1 {
 			c, err = s.daemon.ChannelUpdate(s.DbChannelData.ChannelClaimID, jsonrpc.ChannelUpdateOptions{
 				ClearTags:      util.PtrToBool(true),
 				ClearLocations: util.PtrToBool(true),
 				ClearLanguages: util.PtrToBool(true),
 				ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
-					AccountID: &da,
-					FundingAccountIDs: []string{
-						da,
-					},
 					ClaimCreateOptions: claimCreateOptions,
 					CoverURL:           bannerURL,
 				},
@@ -473,50 +460,20 @@ func (s *Sync) ensureChannelOwnership() error {
 			return nil
 		}
 	} else {
-		c, err = s.daemon.ChannelCreate(s.DbChannelData.DesiredChannelName, channelClaimAmount, jsonrpc.ChannelCreateOptions{
+		c, err = s.daemon.ChannelCreate(s.DbChannelData.DesiredChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
 			ClaimCreateOptions: claimCreateOptions,
 			CoverURL:           bannerURL,
 		})
-		if err != nil {
-			claimId, err2 := s.getChannelClaimIDForTimedOutCreation()
-			if err2 != nil {
-				err = errors.Prefix(err2.Error(), err)
-			} else {
-				recoveredChannelClaimID = claimId
-			}
-		}
 	}
+
 	if err != nil {
 		return err
 	}
-	if recoveredChannelClaimID != "" {
-		s.DbChannelData.ChannelClaimID = recoveredChannelClaimID
-	} else {
-		s.DbChannelData.ChannelClaimID = c.Outputs[0].ClaimID
-	}
+
+	s.DbChannelData.ChannelClaimID = c.Outputs[0].ClaimID
 	return s.Manager.ApiConfig.SetChannelClaimID(s.DbChannelData.ChannelId, s.DbChannelData.ChannelClaimID)
 }
 
-//getChannelClaimIDForTimedOutCreation is a raw function that returns the only channel that exists in the wallet
-// this is used because the SDK sucks and can't figure out when to return when creating a claim...
-func (s *Sync) getChannelClaimIDForTimedOutCreation() (string, error) {
-	channels, err := s.daemon.ChannelList(nil, 1, 500, nil)
-	if err != nil {
-		return "", err
-	} else if channels == nil {
-		return "", errors.Err("no channel response")
-	}
-	if len((*channels).Items) != 1 {
-		return "", errors.Err("more than one channel found when trying to recover from SDK failure in creating the channel")
-	}
-	desiredChannel := (*channels).Items[0]
-	if desiredChannel.Name != s.DbChannelData.DesiredChannelName {
-		return "", errors.Err("the channel found in the wallet has a different name than the one we expected")
-	}
-
-	return desiredChannel.ClaimID, nil
-}
-
 func (s *Sync) addCredits(amountToAdd float64) error {
 	start := time.Now()
 	defer func(start time.Time) {

manager/transfer.go

@@ -241,7 +241,7 @@ func transferVideos(s *Sync) error {
 						},
 					},
 				},
-				Bid: util.PtrToString(fmt.Sprintf("%.5f", publishAmount/2.)),
+				Bid: util.PtrToString("0.005"), // Todo - Dont hardcode
 			}
 			videoStatus := shared.VideoStatus{
 				ChannelID:     s.DbChannelData.ChannelId,
@@ -293,7 +293,7 @@ func (s *Sync) streamUpdate(ui *updateInfo) error {
 	timing.TimedComponent("transferStreamUpdate").Add(time.Since(start))
 	if updateError != nil {
 		ui.videoStatus.FailureReason = updateError.Error()
-		ui.videoStatus.Status = shared.VideoStatusTransferFailed
+		ui.videoStatus.Status = shared.VideoStatusTranferFailed
 		ui.videoStatus.IsTransferred = util.PtrToBool(false)
 	} else {
 		ui.videoStatus.IsTransferred = util.PtrToBool(len(result.Outputs) != 0)

manager/ytsync.go

@@ -33,10 +33,11 @@ import (
 
 const (
 	channelClaimAmount    = 0.01
-	estimatedMaxTxFee     = 0.0015
+	estimatedMaxTxFee     = 0.1
 	minimumAccountBalance = 1.0
 	minimumRefillAmount   = 1
-	publishAmount         = 0.002
+	publishAmount         = 0.01
+	maxReasonLength       = 500
 )
 
 // Sync stores the options that control how syncing happens
@@ -285,8 +286,6 @@ func (s *Sync) setChannelTerminationStatus(e *error) {
 			"interrupted during daemon startup",
 			"interrupted by user",
 			"use --skip-space-check to ignore",
-			"failure uploading blockchain DB",
-			"default_wallet already exists",
 		}
 		dbWipeConditions := []string{
 			"Missing inputs",
@@ -336,7 +335,7 @@ func (s *Sync) waitForDaemonStart() error {
 }
 
 func (s *Sync) stopAndUploadWallet(e *error) {
-	log.Println("Stopping daemon")
+	log.Printf("Stopping daemon")
 	shutdownErr := logUtils.StopDaemon()
 	if shutdownErr != nil {
 		logShutdownError(shutdownErr)
@@ -353,7 +352,7 @@ func (s *Sync) stopAndUploadWallet(e *error) {
 				if *e == nil {
 					*e = err
 				} else {
-					*e = errors.Prefix(fmt.Sprintf("%s + original error", errors.FullTrace(err)), *e)
+					*e = errors.Prefix("failure uploading wallet", *e)
 				}
 			}
 			err = s.uploadBlockchainDB()
@@ -496,7 +495,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
 		claimMarkedUnpublished := claimInDatabase && !sv.Published
 		_, isOwnClaim := ownClaimsInfo[videoID]
 		transferred := !isOwnClaim || s.DbChannelData.TransferState == 3
-		transferStatusMismatch := claimInDatabase && sv.Transferred != transferred
+		transferStatusMismatch := sv.Transferred != transferred
 
 		if metadataDiffers {
 			log.Debugf("%s: Mismatch in database for metadata. DB: %d - Blockchain: %d", videoID, sv.MetadataVersion, chainInfo.MetadataVersion)
@@ -557,11 +556,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
 		if sv.Transferred || sv.IsLbryFirst {
 			_, ok := allClaimsInfo[vID]
 			if !ok && sv.Published {
-				searchResponse, err := s.daemon.ClaimSearch(jsonrpc.ClaimSearchArgs{
-					ClaimID:  &sv.ClaimID,
-					Page:     1,
-					PageSize: 20,
-				})
+				searchResponse, err := s.daemon.ClaimSearch(nil, &sv.ClaimID, nil, nil, 1, 20)
 				if err != nil {
 					log.Error(err.Error())
 					continue
@@ -677,8 +672,7 @@ func (s *Sync) checkIntegrity() error {
 
 	if pubsOnWallet > pubsOnDB { //This case should never happen
 		logUtils.SendInfoToSlack("We're claiming to have published %d videos but in reality we published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId)
-		//we never really done anything about those. it happens when a user updates the channel for a publish to another ytsync channel
-		//return errors.Err("not all published videos are in the database")
+		return errors.Err("not all published videos are in the database")
 	}
 	if pubsOnWallet < pubsOnDB {
 		logUtils.SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId)
@@ -867,7 +861,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
 		return err
 	}
 
-	videos, err := ytapi.GetVideosToSync(s.DbChannelData.ChannelId, s.syncedVideos, s.Manager.CliFlags.QuickSync, s.Manager.CliFlags.VideosToSync(s.DbChannelData.TotalSubscribers), ytapi.VideoParams{
+	videos, err := ytapi.GetVideosToSync(s.Manager.ApiConfig, s.DbChannelData.ChannelId, s.syncedVideos, s.Manager.CliFlags.QuickSync, s.Manager.CliFlags.VideosToSync(s.DbChannelData.TotalSubscribers), ytapi.VideoParams{
 		VideoDir: s.videoDirectory,
 		Stopper:  s.grp,
 		IPPool:   ipPool,

metrics/metrics.go

@@ -1,17 +1,31 @@
 package metrics
 
 import (
-	"github.com/lbryio/ytsync/v5/configs"
+	"os"
+	"regexp"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
+	log "github.com/sirupsen/logrus"
 )
 
 var (
 	Durations = promauto.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: "ytsync",
-		Subsystem: configs.Configuration.GetHostname(),
+		Subsystem: getHostname(),
 		Name:      "duration",
 		Help:      "The durations of the individual modules",
 	}, []string{"path"})
 )
+
+func getHostname() string {
+	hostname, err := os.Hostname()
+	if err != nil {
+		hostname = "ytsync_unknown"
+	}
+	reg, err := regexp.Compile("[^a-zA-Z0-9_]+")
+	if err != nil {
+		log.Fatal(err)
+	}
+	return reg.ReplaceAllString(hostname, "_")
+}

sdk/api.go

@@ -13,7 +13,6 @@ import (
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/extras/null"
-	"github.com/lbryio/ytsync/v5/configs"
 	"github.com/lbryio/ytsync/v5/shared"
 
 	"github.com/lbryio/ytsync/v5/util"
@@ -31,19 +30,6 @@ type APIConfig struct {
 	HostName string
 }
 
-var instance *APIConfig
-
-func GetAPIsConfigs() *APIConfig {
-	if instance == nil {
-		instance = &APIConfig{
-			ApiURL:   configs.Configuration.InternalApisEndpoint,
-			ApiToken: configs.Configuration.InternalApisAuthToken,
-			HostName: configs.Configuration.GetHostname(),
-		}
-	}
-	return instance
-}
-
 func (a *APIConfig) FetchChannels(status string, cliFlags *shared.SyncFlags) ([]shared.YoutubeChannel, error) {
 	type apiJobsResponse struct {
 		Success bool                    `json:"success"`
@@ -61,9 +47,12 @@ func (a *APIConfig) FetchChannels(status string, cliFlags *shared.SyncFlags) ([]
 		"channel_id":  {cliFlags.ChannelID},
 	})
 	if err != nil {
-		util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
-		time.Sleep(30 * time.Second)
-		return a.FetchChannels(status, cliFlags)
+		if strings.Contains(err.Error(), "EOF") {
+			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+			time.Sleep(30 * time.Second)
+			return a.FetchChannels(status, cliFlags)
+		}
+		return nil, errors.Err(err)
 	}
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)
@@ -121,9 +110,12 @@ func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
 		"auth_token":       {a.ApiToken},
 	})
 	if err != nil {
-		util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
-		time.Sleep(30 * time.Second)
-		return a.SetChannelCert(certHex, channelID)
+		if strings.Contains(err.Error(), "EOF") {
+			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+			time.Sleep(30 * time.Second)
+			return a.SetChannelCert(certHex, channelID)
+		}
+		return errors.Err(err)
 	}
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)
@@ -166,9 +158,12 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
 	}
 	res, err := http.PostForm(endpoint, params)
 	if err != nil {
-		util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
-		time.Sleep(30 * time.Second)
-		return a.SetChannelStatus(channelID, status, failureReason, transferState)
+		if strings.Contains(err.Error(), "EOF") {
+			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+			time.Sleep(30 * time.Second)
+			return a.SetChannelStatus(channelID, status, failureReason, transferState)
+		}
+		return nil, nil, errors.Err(err)
 	}
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)
@@ -213,9 +208,12 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
 		"channel_claim_id": {channelClaimID},
 	})
 	if err != nil {
-		util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
-		time.Sleep(30 * time.Second)
-		return a.SetChannelClaimID(channelID, channelClaimID)
+		if strings.Contains(err.Error(), "EOF") {
+			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+			time.Sleep(30 * time.Second)
+			return a.SetChannelClaimID(channelID, channelClaimID)
+		}
+		return errors.Err(err)
 	}
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)
@@ -254,9 +252,12 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
 	}
 	res, err := http.PostForm(endpoint, vals)
 	if err != nil {
-		util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
-		time.Sleep(30 * time.Second)
-		return a.DeleteVideos(videos)
+		if strings.Contains(err.Error(), "EOF") {
+			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+			time.Sleep(30 * time.Second)
+			return a.DeleteVideos(videos)
+		}
+		return errors.Err(err)
 	}
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)
@@ -317,9 +318,12 @@ func (a *APIConfig) MarkVideoStatus(status shared.VideoStatus) error {
 	}
 	res, err := http.PostForm(endpoint, vals)
 	if err != nil {
-		util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
-		time.Sleep(30 * time.Second)
-		return a.MarkVideoStatus(status)
+		if strings.Contains(err.Error(), "EOF") {
+			util.SendErrorToSlack("EOF error while trying to call %s for %s. Waiting to retry", endpoint, status.ClaimName)
+			time.Sleep(30 * time.Second)
+			return a.MarkVideoStatus(status)
+		}
+		return errors.Err(err)
 	}
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)
@@ -356,9 +360,12 @@ func (a *APIConfig) VideoState(videoID string) (string, error) {
 
 	res, err := http.PostForm(endpoint, vals)
 	if err != nil {
-		util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
-		time.Sleep(30 * time.Second)
-		return a.VideoState(videoID)
+		if strings.Contains(err.Error(), "EOF") {
+			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+			time.Sleep(30 * time.Second)
+			return a.VideoState(videoID)
+		}
+		return "", errors.Err(err)
 	}
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)
@@ -407,9 +414,12 @@ func (a *APIConfig) GetReleasedDate(videoID string) (*VideoRelease, error) {
 
 	res, err := http.PostForm(endpoint, vals)
 	if err != nil {
-		util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
-		time.Sleep(30 * time.Second)
-		return a.GetReleasedDate(videoID)
+		if strings.Contains(err.Error(), "EOF") {
+			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+			time.Sleep(30 * time.Second)
+			return a.GetReleasedDate(videoID)
+		}
+		return nil, errors.Err(err)
 	}
 	defer res.Body.Close()
 	body, _ := ioutil.ReadAll(res.Body)

shared/shared.go

@@ -26,7 +26,6 @@ type YoutubeChannel struct {
 	SizeLimit          int            `json:"size_limit"`
 	LastUploadedVideo  string         `json:"last_uploaded_video"`
 	WipeDB             bool           `json:"wipe_db"`
-	Language           string         `json:"language"`
 }
 
 type PublishAddress struct {
@@ -78,7 +77,6 @@ var ErrorsNoRetry = []string{
 	"giving up after 0 fragment retries",
 	"Sorry about that",
 	"This video is not available",
-	"Video unavailable",
 	"requested format not available",
 	"interrupted by user",
 	"Sign in to confirm your age",
@@ -91,7 +89,6 @@ var ErrorsNoRetry = []string{
 	"Premiere will begin shortly",
 	"cannot unmarshal number 0.0",
 	"default youtube thumbnail found",
-	"livestream is likely bugged",
 }
 var WalletErrors = []string{
 	"Not enough funds to cover this transaction",
@@ -118,8 +115,6 @@ var NeverRetryFailures = []string{
 	"Playback on other websites has been disabled by the video owner",
 	"uploader has not made this video available in your country",
 	"This video has been removed by the uploader",
-	"Video unavailable",
-	"Video is not available - hardcoded fix",
 }
 
 type SyncFlags struct {
@@ -157,7 +152,7 @@ func (f *SyncFlags) VideosToSync(totalSubscribers uint) int {
 		800:   250,
 		600:   200,
 		200:   80,
-		100:   20,
+		100:   50,
 		1:     10,
 	}
 	videosToSync := 0
@@ -194,25 +189,22 @@ const (
 	StatusSynced         = "synced"         // done
 	StatusWipeDb         = "pendingdbwipe"  // in sync queue. lbryum database will be pruned
 	StatusFailed         = "failed"
-	StatusFinalized      = "finalized"     // no more changes allowed
-	StatusAbandoned      = "abandoned"     // deleted on youtube or banned
-	StatusAgeRestricted  = "agerestricted" // one or more videos are age restricted and should be reprocessed with special keys
+	StatusFinalized      = "finalized" // no more changes allowed
+	StatusAbandoned      = "abandoned" // deleted on youtube or banned
 )
 
-var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned, StatusWipeDb, StatusAgeRestricted}
+var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned, StatusWipeDb}
 
 const LatestMetadataVersion = 2
 
 const (
-	VideoStatusPublished      = "published"
-	VideoStatusFailed         = "failed"
-	VideoStatusUpgradeFailed  = "upgradefailed"
-	VideoStatusUnpublished    = "unpublished"
-	VideoStatusTransferFailed = "transferfailed"
+	VideoStatusPublished     = "published"
+	VideoStatusFailed        = "failed"
+	VideoStatusUpgradeFailed = "upgradefailed"
+	VideoStatusUnpublished   = "unpublished"
+	VideoStatusTranferFailed = "transferfailed"
 )
 
-var VideoSyncStatuses = []string{VideoStatusPublished, VideoStatusFailed, VideoStatusUpgradeFailed, VideoStatusUnpublished, VideoStatusTransferFailed}
-
 const (
 	TransferStateNotTouched = iota
 	TransferStatePending

sources/youtubeVideo.go

@@ -16,12 +16,17 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/abadojack/whatlanggo"
 	"github.com/lbryio/ytsync/v5/downloader"
 	"github.com/lbryio/ytsync/v5/downloader/ytdl"
+	"github.com/lbryio/ytsync/v5/shared"
+	"github.com/vbauerster/mpb/v7"
+	"github.com/vbauerster/mpb/v7/decor"
+	"gopkg.in/vansante/go-ffprobe.v2"
+
 	"github.com/lbryio/ytsync/v5/ip_manager"
 	"github.com/lbryio/ytsync/v5/namer"
 	"github.com/lbryio/ytsync/v5/sdk"
-	"github.com/lbryio/ytsync/v5/shared"
 	"github.com/lbryio/ytsync/v5/tags_manager"
 	"github.com/lbryio/ytsync/v5/thumbs"
 	"github.com/lbryio/ytsync/v5/timing"
@@ -32,12 +37,8 @@ import (
 	"github.com/lbryio/lbry.go/v2/extras/stop"
 	"github.com/lbryio/lbry.go/v2/extras/util"
 
-	"github.com/abadojack/whatlanggo"
 	"github.com/shopspring/decimal"
 	log "github.com/sirupsen/logrus"
-	"github.com/vbauerster/mpb/v7"
-	"github.com/vbauerster/mpb/v7/decor"
-	"gopkg.in/vansante/go-ffprobe.v2"
 )
 
 type YoutubeVideo struct {
@@ -106,7 +107,7 @@ func NewYoutubeVideo(directory string, videoData *ytdl.YtdlVideo, playlistPositi
 		title:            videoData.Title,
 		description:      videoData.Description,
 		playlistPosition: playlistPosition,
-		publishedAt:      videoData.GetUploadTime(),
+		publishedAt:      videoData.UploadDateForReal,
 		dir:              directory,
 		youtubeInfo:      videoData,
 		mocked:           false,
@@ -175,7 +176,7 @@ func (v *YoutubeVideo) getFullPath() string {
 }
 
 func (v *YoutubeVideo) getAbbrevDescription() string {
-	maxLength := 6500
+	maxLength := 2800
 	description := strings.TrimSpace(v.description)
 	additionalDescription := "\nhttps://www.youtube.com/watch?v=" + v.id
 	khanAcademyClaimID := "5fc52291980268b82413ca4c0ace1b8d749f3ffb"
@@ -320,7 +321,7 @@ func (v *YoutubeVideo) download() error {
 	//speedThrottleRetries := 3
 	for i := 0; i < len(qualities); i++ {
 		quality := qualities[i]
-		argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][vcodec!*=av01][height<="+quality+"]+bestaudio[ext!=webm][format_id!=258][format_id!=380][format_id!=251][format_id!=256][format_id!=327][format_id!=328]")
+		argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][vcodec!*=av01][height<="+quality+"]+bestaudio[ext!=webm][format_id!=258][format_id!=251][format_id!=256][format_id!=327]")
 		argsWithFilters = append(argsWithFilters, userAgent...)
 		//if speedThrottleRetries > 0 {
 		//	speedThrottleRetries--
@@ -514,28 +515,21 @@ func (v *YoutubeVideo) trackProgressBar(argsWithFilters []string, ticker *time.T
 			bar.Completed()
 			bar.Abort(true)
 		}()
-		origSize := int64(0)
-		lastUpdate := time.Now()
 		for {
 			select {
 			case <-done.Ch():
 				return
 			case <-ticker.C:
-				var err error
 				size, err := logUtils.DirSize(v.videoDir())
 				if err != nil {
 					log.Errorf("error while getting size of download directory: %s", errors.FullTrace(err))
 					return
 				}
-				if size > origSize {
-					origSize = size
-					bar.SetCurrent(size)
-					if size > int64(videoSize+audioSize) {
-						bar.SetTotal(size+2048, false)
-					}
-					bar.DecoratorEwmaUpdate(time.Since(lastUpdate))
-					lastUpdate = time.Now()
+				bar.SetCurrent(size)
+				if size > int64(videoSize+audioSize) {
+					bar.SetTotal(size+2048, false)
 				}
+				bar.DecoratorEwmaUpdate(400 * time.Millisecond)
 			}
 		}
 	}()
@@ -780,9 +774,6 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
 
 func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
 	var err error
-	if v.youtubeInfo == nil {
-		return nil, errors.Err("Video is not available - hardcoded fix")
-	}
 
 	dur := time.Duration(v.youtubeInfo.Duration) * time.Second
 	minDuration := 7 * time.Second
@@ -790,9 +781,6 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
 	if v.youtubeInfo.IsLive == true {
 		return nil, errors.Err("video is a live stream and hasn't completed yet")
 	}
-	if v.youtubeInfo.Availability != "public" {
-		return nil, errors.Err("video is not public")
-	}
 	if dur > v.maxVideoLength {
 		logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, dur.String(), v.maxVideoLength.String())
 		return nil, errors.Err("video is too long to process")
@@ -801,11 +789,6 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
 		logUtils.SendErrorToSlack("%s is %s long and the minimum is %s", v.id, dur.String(), minDuration.String())
 		return nil, errors.Err("video is too short to process")
 	}
-
-	buggedLivestream := v.youtubeInfo.LiveStatus == "post_live"
-	if buggedLivestream && dur >= 2*time.Hour {
-		return nil, errors.Err("livestream is likely bugged as it was recently published and has a length of %s which is more than 2 hours", dur.String())
-	}
 	for {
 		err = v.download()
 		if err != nil && strings.Contains(err.Error(), "HTTP Error 429") {
@@ -875,11 +858,7 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
 }
 
 func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo) (*SyncSummary, error) {
-	c, err := daemon.ClaimSearch(jsonrpc.ClaimSearchArgs{
-		ClaimID:  &existingVideoData.ClaimID,
-		Page:     1,
-		PageSize: 20,
-	})
+	c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil, 1, 20)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
@@ -942,13 +921,12 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
 				params.DefaultAccount,
 			},
 		},
-		Author:      util.PtrToString(""),
-		License:     util.PtrToString("Copyrighted (contact publisher)"),
-		ChannelID:   &v.lbryChannelID,
-		Height:      util.PtrToUint(720),
-		Width:       util.PtrToUint(1280),
-		Fee:         fee,
-		ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
+		Author:    util.PtrToString(""),
+		License:   util.PtrToString("Copyrighted (contact publisher)"),
+		ChannelID: &v.lbryChannelID,
+		Height:    util.PtrToUint(720),
+		Width:     util.PtrToUint(1280),
+		Fee:       fee,
 	}
 
 	v.walletLock.RLock()

util/log_wrapper.go

@@ -14,9 +14,7 @@ func SendErrorToSlack(format string, a ...interface{}) {
 		message = fmt.Sprintf(format, a...)
 	}
 	log.Errorln(message)
-	log.SetLevel(log.InfoLevel) //I don't want to change the underlying lib so this will do...
 	err := util.SendToSlack(":sos: ```" + message + "```")
-	log.SetLevel(log.DebugLevel)
 	if err != nil {
 		log.Errorln(err)
 	}
@@ -29,9 +27,7 @@ func SendInfoToSlack(format string, a ...interface{}) {
 		message = fmt.Sprintf(format, a...)
 	}
 	log.Infoln(message)
-	log.SetLevel(log.InfoLevel) //I don't want to change the underlying lib so this will do...
 	err := util.SendToSlack(":information_source: " + message)
-	log.SetLevel(log.DebugLevel)
 	if err != nil {
 		log.Errorln(err)
 	}

ytapi/ytapi.go

@@ -55,7 +55,7 @@ type VideoParams struct {
 
 var mostRecentlyFailedChannel string // TODO: fix this hack!
 
-func GetVideosToSync(channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams, lastUploadedVideo string) ([]Video, error) {
+func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams, lastUploadedVideo string) ([]Video, error) {
 	var videos []Video
 	if quickSync && maxVideos > 50 {
 		maxVideos = 50
@@ -94,7 +94,7 @@ func GetVideosToSync(channelID string, syncedVideos map[string]sdk.SyncedVideo,
 		mostRecentlyFailedChannel = channelID
 	}
 
-	vids, err := getVideos(channelID, videoIDs, videoParams.Stopper.Ch(), videoParams.IPPool)
+	vids, err := getVideos(config, channelID, videoIDs, videoParams.Stopper.Ch(), videoParams.IPPool)
 	if err != nil {
 		return nil, err
 	}
@@ -109,8 +109,7 @@ func GetVideosToSync(channelID string, syncedVideos map[string]sdk.SyncedVideo,
 	}
 
 	for k, v := range syncedVideos {
-		newMetadataVersion := int8(2)
-		if !v.Published && v.MetadataVersion >= newMetadataVersion {
+		if !v.Published {
 			continue
 		}
 		if _, ok := playlistMap[k]; !ok {
@@ -204,8 +203,7 @@ func ChannelInfo(channelID string) (*YoutubeStatsResponse, error) {
 	return &decodedResponse, nil
 }
 
-func getVideos(channelID string, videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool) ([]*ytdl.YtdlVideo, error) {
-	config := sdk.GetAPIsConfigs()
+func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool) ([]*ytdl.YtdlVideo, error) {
 	var videos []*ytdl.YtdlVideo
 	for _, videoID := range videoIDs {
 		if len(videoID) < 5 {
@@ -217,6 +215,11 @@ func getVideos(channelID string, videoIDs []string, stopChan stop.Chan, ipPool *
 		default:
 		}
 
+		//ip, err := ipPool.GetIP(videoID)
+		//if err != nil {
+		//	return nil, err
+		//}
+		//video, err := downloader.GetVideoInformation(videoID, &net.TCPAddr{IP: net.ParseIP(ip)})
 		state, err := config.VideoState(videoID)
 		if err != nil {
 			return nil, errors.Err(err)
@@ -224,7 +227,7 @@ func getVideos(channelID string, videoIDs []string, stopChan stop.Chan, ipPool *
 		if state == "published" {
 			continue
 		}
-		video, err := downloader.GetVideoInformation(videoID, stopChan, ipPool)
+		video, err := downloader.GetVideoInformation(config, videoID, stopChan, nil, ipPool)
 		if err != nil {
 			errSDK := config.MarkVideoStatus(shared.VideoStatus{
 				ChannelID:     channelID,