never retry those hardcoded errors

reduce verbosity of debug logs

downloader/downloader.go

@@ -39,7 +39,6 @@ func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan,
 		if v == "" {
 			continue
 		}
-		logrus.Debugf("%d - video id %s", i, v)
 		if i >= maxVideos {
 			break
 		}
@@ -286,7 +285,8 @@ func runCmd(cmd *exec.Cmd, stopChan stop.Chan) ([]string, error) {
 		return nil, errors.Err("interrupted by user")
 	case err := <-done:
 		if err != nil {
-			return nil, errors.Prefix("yt-dlp "+strings.Join(cmd.Args, " ")+" ["+string(errorLog)+"]", err)
+			//return nil, errors.Prefix("yt-dlp "+strings.Join(cmd.Args, " ")+" ["+string(errorLog)+"]", err)
+			return nil, errors.Prefix(string(errorLog), err)
 		}
 		return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
 	}

downloader/downloader_test.go

@@ -3,7 +3,11 @@ package downloader
 import (
 	"testing"
 
+	"github.com/lbryio/ytsync/v5/ip_manager"
 	"github.com/lbryio/ytsync/v5/sdk"
+
+	"github.com/lbryio/lbry.go/v2/extras/stop"
+
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 )
@@ -19,13 +23,13 @@ func TestGetPlaylistVideoIDs(t *testing.T) {
 }
 
 func TestGetVideoInformation(t *testing.T) {
-	video, err := GetVideoInformation("zj7pXM9gE5M", nil, nil)
-	if err != nil {
-		logrus.Error(err)
-	}
-	if video != nil {
-		logrus.Info(video.ID)
-	}
+	s := stop.New()
+	ip, err := ip_manager.GetIPPool(s)
+	assert.NoError(t, err)
+	video, err := GetVideoInformation("kDGOHNpRjzc", s.Ch(), ip)
+	assert.NoError(t, err)
+	assert.NotNil(t, video)
+	logrus.Info(video.ID)
 }
 
 func Test_getUploadTime(t *testing.T) {
@@ -33,5 +37,4 @@ func Test_getUploadTime(t *testing.T) {
 	got, err := getUploadTime(&configs, "kDGOHNpRjzc", nil, "20060102")
 	assert.NoError(t, err)
 	t.Log(got)
-
 }

downloader/ytdl/Video.go

@@ -18,10 +18,11 @@ type YtdlVideo struct {
 	Tags              []string    `json:"tags"`
 	IsLive            bool        `json:"is_live"`
 	LiveStatus        string      `json:"live_status"`
-	ReleaseTimestamp  int64       `json:"release_timestamp"`
+	ReleaseTimestamp  *int64      `json:"release_timestamp"`
 	uploadDateForReal *time.Time
 	Availability      string `json:"availability"`
 	ReleaseDate       string `json:"release_date"`
+	UploadDate        string `json:"upload_date"`
 
 	//WasLive           bool        `json:"was_live"`
 	//Formats              interface{}   `json:"formats"`
@@ -41,7 +42,6 @@ type YtdlVideo struct {
 	//LikeCount            int           `json:"like_count"`
 	//Channel              string        `json:"channel"`
 	//ChannelFollowerCount int           `json:"channel_follower_count"`
-	//UploadDate        string     `json:"upload_date"`
 	//OriginalURL          string        `json:"original_url"`
 	//WebpageURLBasename   string        `json:"webpage_url_basename"`
 	//WebpageURLDomain     string        `json:"webpage_url_domain"`
@@ -94,20 +94,21 @@ func (v *YtdlVideo) GetUploadTime() time.Time {
 	// release timestamp from yt
 	// release timestamp from morty
 	// release date from yt
+	// upload date from yt
 	if v.uploadDateForReal != nil {
 		return *v.uploadDateForReal
 	}
 
 	var ytdlReleaseTimestamp time.Time
-	if v.ReleaseTimestamp > 0 {
-		ytdlReleaseTimestamp = time.Unix(v.ReleaseTimestamp, 0).UTC()
+	if v.ReleaseTimestamp != nil && *v.ReleaseTimestamp > 0 {
+		ytdlReleaseTimestamp = time.Unix(*v.ReleaseTimestamp, 0).UTC()
 	}
 	//get morty timestamp
 	var mortyReleaseTimestamp time.Time
 	mortyRelease, err := sdk.GetAPIsConfigs().GetReleasedDate(v.ID)
 	if err != nil {
 		logrus.Error(err)
-	} else if mortyRelease.ReleaseTime != "" {
+	} else if mortyRelease != nil {
 		mortyReleaseTimestamp, err = time.ParseInLocation(time.RFC3339, mortyRelease.ReleaseTime, time.UTC)
 		if err != nil {
 			logrus.Error(err)
@@ -118,13 +119,19 @@ func (v *YtdlVideo) GetUploadTime() time.Time {
 	if err != nil {
 		logrus.Error(err)
 	}
-
+	ytdlUploadDate, err := time.Parse("20060102", v.UploadDate)
+	if err != nil {
+		logrus.Error(err)
+	}
 	if !ytdlReleaseTimestamp.IsZero() {
 		v.uploadDateForReal = &ytdlReleaseTimestamp
 	} else if !mortyReleaseTimestamp.IsZero() {
 		v.uploadDateForReal = &mortyReleaseTimestamp
-	} else {
+	} else if !ytdlReleaseDate.IsZero() {
 		v.uploadDateForReal = &ytdlReleaseDate
+	} else {
+		v.uploadDateForReal = &ytdlUploadDate
 	}
+
 	return *v.uploadDateForReal
 }

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/aws/aws-sdk-go v1.44.6
 	github.com/davecgh/go-spew v1.1.1
 	github.com/docker/docker v20.10.17+incompatible
-	github.com/lbryio/lbry.go/v2 v2.7.2-0.20220805175105-9130630afe5e
+	github.com/lbryio/lbry.go/v2 v2.7.2-0.20220815204100-2adb8af5b68c
 	github.com/lbryio/reflector.go v1.1.3-0.20220730181028-f5d30b1a6e79
 	github.com/mitchellh/go-ps v1.0.0
 	github.com/prometheus/client_golang v1.12.1

go.sum

@@ -396,8 +396,8 @@ github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f/go.mod h1:JtyI30b
 github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
 github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
 github.com/lbryio/lbry.go/v2 v2.7.2-0.20220321182539-d0aeb0c22b22/go.mod h1:K7IAMJMjDisynAcDkDdZHbZeWth6sTLl66xh+uJqtGg=
-github.com/lbryio/lbry.go/v2 v2.7.2-0.20220805175105-9130630afe5e h1:W53pVf9D2sXRLv0Ytqyc9A9l2NUBrTK8j7Bl7BifSvk=
-github.com/lbryio/lbry.go/v2 v2.7.2-0.20220805175105-9130630afe5e/go.mod h1:YsNY6a6g2Edl14sBUgXx71c6TyoOm8jDvkzcoNHLSXE=
+github.com/lbryio/lbry.go/v2 v2.7.2-0.20220815204100-2adb8af5b68c h1:WWb3/bwunqeTGjTWVbizuLMaFfSQk0Y4pbMlfjnKwqA=
+github.com/lbryio/lbry.go/v2 v2.7.2-0.20220815204100-2adb8af5b68c/go.mod h1:YsNY6a6g2Edl14sBUgXx71c6TyoOm8jDvkzcoNHLSXE=
 github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
 github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
 github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=

manager/setup.go

@@ -450,12 +450,20 @@ func (s *Sync) ensureChannelOwnership() error {
 		ThumbnailURL: &thumbnailURL,
 	}
 	if channelUsesOldMetadata {
+		da, err := s.getDefaultAccount()
+		if err != nil {
+			return err
+		}
 		if s.DbChannelData.TransferState <= 1 {
 			c, err = s.daemon.ChannelUpdate(s.DbChannelData.ChannelClaimID, jsonrpc.ChannelUpdateOptions{
 				ClearTags:      util.PtrToBool(true),
 				ClearLocations: util.PtrToBool(true),
 				ClearLanguages: util.PtrToBool(true),
 				ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
+					AccountID: &da,
+					FundingAccountIDs: []string{
+						da,
+					},
 					ClaimCreateOptions: claimCreateOptions,
 					CoverURL:           bannerURL,
 				},

shared/shared.go

@@ -78,6 +78,7 @@ var ErrorsNoRetry = []string{
 	"giving up after 0 fragment retries",
 	"Sorry about that",
 	"This video is not available",
+	"Video unavailable",
 	"requested format not available",
 	"interrupted by user",
 	"Sign in to confirm your age",
@@ -117,6 +118,8 @@ var NeverRetryFailures = []string{
 	"Playback on other websites has been disabled by the video owner",
 	"uploader has not made this video available in your country",
 	"This video has been removed by the uploader",
+	"Video unavailable",
+	"Video is not available - hardcoded fix",
 }
 
 type SyncFlags struct {

sources/youtubeVideo.go

@@ -780,6 +780,9 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
 
 func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
 	var err error
+	if v.youtubeInfo == nil {
+		return nil, errors.Err("Video is not available - hardcoded fix")
+	}
 
 	dur := time.Duration(v.youtubeInfo.Duration) * time.Second
 	minDuration := 7 * time.Second
@@ -939,12 +942,13 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
 				params.DefaultAccount,
 			},
 		},
-		Author:    util.PtrToString(""),
-		License:   util.PtrToString("Copyrighted (contact publisher)"),
-		ChannelID: &v.lbryChannelID,
-		Height:    util.PtrToUint(720),
-		Width:     util.PtrToUint(1280),
-		Fee:       fee,
+		Author:      util.PtrToString(""),
+		License:     util.PtrToString("Copyrighted (contact publisher)"),
+		ChannelID:   &v.lbryChannelID,
+		Height:      util.PtrToUint(720),
+		Width:       util.PtrToUint(1280),
+		Fee:         fee,
+		ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
 	}
 
 	v.walletLock.RLock()

ytapi/ytapi.go

@@ -109,7 +109,8 @@ func GetVideosToSync(channelID string, syncedVideos map[string]sdk.SyncedVideo,
 	}
 
 	for k, v := range syncedVideos {
-		if !v.Published {
+		newMetadataVersion := int8(2)
+		if !v.Published && v.MetadataVersion >= newMetadataVersion {
 			continue
 		}
 		if _, ok := playlistMap[k]; !ok {