From 843303301ad771c09b45e5a75494593629a4167f Mon Sep 17 00:00:00 2001
From: Alex Grintsvayg <grin@lbry.com>
Date: Mon, 27 Jul 2020 14:48:05 -0400
Subject: [PATCH 01/57] move all youtube api calls into a single place

---
 manager/count.go        |  27 +----
 manager/setup.go        |  39 +++-----
 manager/ytsync.go       | 134 +++----------------------
 sources/youtubeVideo.go |  20 ++--
 thumbs/uploader.go      |   4 +-
 ytapi/ytapi.go          | 213 ++++++++++++++++++++++++++++++++++++++++
 6 files changed, 253 insertions(+), 184 deletions(-)
 create mode 100644 ytapi/ytapi.go

diff --git a/manager/count.go b/manager/count.go
index c3136bc..51c78dd 100644
--- a/manager/count.go
+++ b/manager/count.go
@@ -1,32 +1,9 @@
 package manager
 
 import (
-	"net/http"
-
-	"github.com/lbryio/lbry.go/v2/extras/errors"
-
-	"google.golang.org/api/googleapi/transport"
-	"google.golang.org/api/youtube/v3"
+	"github.com/lbryio/ytsync/v5/ytapi"
 )
 
 func (s *Sync) CountVideos() (uint64, error) {
-	client := &http.Client{
-		Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
-	}
-
-	service, err := youtube.New(client)
-	if err != nil {
-		return 0, errors.Prefix("error creating YouTube service", err)
-	}
-
-	response, err := service.Channels.List("statistics").Id(s.YoutubeChannelID).Do()
-	if err != nil {
-		return 0, errors.Prefix("error getting channels", err)
-	}
-
-	if len(response.Items) < 1 {
-		return 0, errors.Err("youtube channel not found")
-	}
-
-	return response.Items[0].Statistics.VideoCount, nil
+	return ytapi.VideosInChannel(s.APIConfig.YoutubeAPIKey, s.YoutubeChannelID)
 }
diff --git a/manager/setup.go b/manager/setup.go
index aeab832..951fe41 100644
--- a/manager/setup.go
+++ b/manager/setup.go
@@ -3,7 +3,6 @@ package manager
 import (
 	"fmt"
 	"math"
-	"net/http"
 	"strconv"
 	"time"
 
@@ -12,14 +11,13 @@ import (
 	"github.com/lbryio/lbry.go/v2/extras/util"
 	"github.com/lbryio/ytsync/v5/timing"
 	logUtils "github.com/lbryio/ytsync/v5/util"
+	"github.com/lbryio/ytsync/v5/ytapi"
 
 	"github.com/lbryio/ytsync/v5/tags_manager"
 	"github.com/lbryio/ytsync/v5/thumbs"
 
 	"github.com/shopspring/decimal"
 	log "github.com/sirupsen/logrus"
-	"google.golang.org/api/googleapi/transport"
-	"google.golang.org/api/youtube/v3"
 )
 
 func (s *Sync) enableAddressReuse() error {
@@ -266,15 +264,14 @@ func (s *Sync) ensureEnoughUTXOs() error {
 }
 
 func (s *Sync) waitForNewBlock() error {
-	start := time.Now()
-	defer func(start time.Time) {
-		timing.TimedComponent("waitForNewBlock").Add(time.Since(start))
-	}(start)
+	defer func(start time.Time) { timing.TimedComponent("waitForNewBlock").Add(time.Since(start)) }(time.Now())
+
 	log.Printf("regtest: %t, docker: %t", logUtils.IsRegTest(), logUtils.IsUsingDocker())
 	status, err := s.daemon.Status()
 	if err != nil {
 		return err
 	}
+
 	for status.Wallet.Blocks == 0 || status.Wallet.BlocksBehind != 0 {
 		time.Sleep(5 * time.Second)
 		status, err = s.daemon.Status()
@@ -308,10 +305,12 @@ func (s *Sync) GenerateRegtestBlock() error {
 	if err != nil {
 		return errors.Prefix("error getting lbrycrd client: ", err)
 	}
+
 	txs, err := lbrycrd.Generate(1)
 	if err != nil {
 		return errors.Prefix("error generating new block: ", err)
 	}
+
 	for _, tx := range txs {
 		log.Info("Generated tx: ", tx.String())
 	}
@@ -319,10 +318,8 @@ func (s *Sync) GenerateRegtestBlock() error {
 }
 
 func (s *Sync) ensureChannelOwnership() error {
-	start := time.Now()
-	defer func(start time.Time) {
-		timing.TimedComponent("ensureChannelOwnership").Add(time.Since(start))
-	}(start)
+	defer func(start time.Time) { timing.TimedComponent("ensureChannelOwnership").Add(time.Since(start)) }(time.Now())
+
 	if s.LbryChannelName == "" {
 		return errors.Err("no channel name set")
 	}
@@ -386,27 +383,12 @@ func (s *Sync) ensureChannelOwnership() error {
 			return err
 		}
 	}
-	client := &http.Client{
-		Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
-	}
 
-	service, err := youtube.New(client)
+	channelInfo, channelBranding, err := ytapi.ChannelInfo(s.APIConfig.YoutubeAPIKey, s.YoutubeChannelID)
 	if err != nil {
-		return errors.Prefix("error creating YouTube service", err)
+		return err
 	}
 
-	response, err := service.Channels.List("snippet,brandingSettings").Id(s.YoutubeChannelID).Do()
-	if err != nil {
-		return errors.Prefix("error getting channel details", err)
-	}
-
-	if len(response.Items) < 1 {
-		return errors.Err("youtube channel not found")
-	}
-
-	channelInfo := response.Items[0].Snippet
-	channelBranding := response.Items[0].BrandingSettings
-
 	thumbnail := thumbs.GetBestThumbnail(channelInfo.Thumbnails)
 	thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.Url, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
 	if err != nil {
@@ -467,6 +449,7 @@ func (s *Sync) ensureChannelOwnership() error {
 	if err != nil {
 		return err
 	}
+
 	s.lbryChannelID = c.Outputs[0].ClaimID
 	return s.Manager.apiConfig.SetChannelClaimID(s.YoutubeChannelID, s.lbryChannelID)
 }
diff --git a/manager/ytsync.go b/manager/ytsync.go
index a1de795..956e50f 100644
--- a/manager/ytsync.go
+++ b/manager/ytsync.go
@@ -3,11 +3,9 @@ package manager
 import (
 	"fmt"
 	"io/ioutil"
-	"net/http"
 	"os"
 	"os/signal"
 	"runtime/debug"
-	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -21,6 +19,7 @@ import (
 	"github.com/lbryio/ytsync/v5/thumbs"
 	"github.com/lbryio/ytsync/v5/timing"
 	logUtils "github.com/lbryio/ytsync/v5/util"
+	"github.com/lbryio/ytsync/v5/ytapi"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
@@ -34,8 +33,6 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	log "github.com/sirupsen/logrus"
-	"google.golang.org/api/googleapi/transport"
-	"google.golang.org/api/youtube/v3"
 )
 
 const (
@@ -47,22 +44,6 @@ const (
 	maxReasonLength       = 500
 )
 
-type video interface {
-	Size() *int64
-	ID() string
-	IDAndNum() string
-	PlaylistPosition() int
-	PublishedAt() time.Time
-	Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool, *sync.RWMutex) (*sources.SyncSummary, error)
-}
-
-// sorting videos
-type byPublishedAt []video
-
-func (a byPublishedAt) Len() int           { return len(a) }
-func (a byPublishedAt) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
-func (a byPublishedAt) Less(i, j int) bool { return a[i].PublishedAt().Before(a[j].PublishedAt()) }
-
 // Sync stores the options that control how syncing happens
 type Sync struct {
 	APIConfig            *sdk.APIConfig
@@ -87,7 +68,7 @@ type Sync struct {
 	lbryChannelID        string
 	namer                *namer.Namer
 	walletMux            *sync.RWMutex
-	queue                chan video
+	queue                chan ytapi.Video
 	transferState        int
 	clientPublishAddress string
 	publicKey            string
@@ -263,7 +244,7 @@ func (s *Sync) FullCycle() (e error) {
 	s.syncedVideosMux = &sync.RWMutex{}
 	s.walletMux = &sync.RWMutex{}
 	s.grp = stopGroup
-	s.queue = make(chan video)
+	s.queue = make(chan ytapi.Video)
 	interruptChan := make(chan os.Signal, 1)
 	signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
 	defer signal.Stop(interruptChan)
@@ -847,7 +828,7 @@ func (s *Sync) doSync() error {
 }
 
 func (s *Sync) startWorker(workerNum int) {
-	var v video
+	var v ytapi.Video
 	var more bool
 
 	for {
@@ -996,107 +977,23 @@ func (s *Sync) startWorker(workerNum int) {
 	}
 }
 
-var mostRecentlyFailedChannel string
-
 func (s *Sync) enqueueYoutubeVideos() error {
-	start := time.Now()
-	defer func(start time.Time) {
-		timing.TimedComponent("enqueueYoutubeVideos").Add(time.Since(start))
-	}(start)
-	client := &http.Client{
-		Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
-	}
+	defer func(start time.Time) { timing.TimedComponent("enqueueYoutubeVideos").Add(time.Since(start)) }(time.Now())
 
-	service, err := youtube.New(client)
-	if err != nil {
-		return errors.Prefix("error creating YouTube service", err)
-	}
-
-	response, err := service.Channels.List("contentDetails").Id(s.YoutubeChannelID).Do()
-	if err != nil {
-		return errors.Prefix("error getting channels", err)
-	}
-
-	if len(response.Items) < 1 {
-		return errors.Err("youtube channel not found")
-	}
-
-	if response.Items[0].ContentDetails.RelatedPlaylists == nil {
-		return errors.Err("no related playlists")
-	}
-
-	playlistID := response.Items[0].ContentDetails.RelatedPlaylists.Uploads
-	if playlistID == "" {
-		return errors.Err("no channel playlist")
-	}
-
-	var videos []video
 	ipPool, err := ip_manager.GetIPPool(s.grp)
 	if err != nil {
 		return err
 	}
 
-	playlistMap := make(map[string]*youtube.PlaylistItemSnippet, 50)
-	nextPageToken := ""
-	for {
-		req := service.PlaylistItems.List("snippet").
-			PlaylistId(playlistID).
-			MaxResults(50).
-			PageToken(nextPageToken)
-
-		playlistResponse, err := req.Do()
-		if err != nil {
-			return errors.Prefix("error getting playlist items", err)
-		}
-
-		if len(playlistResponse.Items) < 1 {
-			// If there are 50+ videos in a playlist but less than 50 are actually returned by the API, youtube will still redirect
-			// clients to a next page. Such next page will however be empty. This logic prevents ytsync from failing.
-			youtubeIsLying := len(videos) > 0
-			if youtubeIsLying {
-				break
-			}
-			if s.YoutubeChannelID == mostRecentlyFailedChannel {
-				return errors.Err("playlist items not found")
-			}
-			mostRecentlyFailedChannel = s.YoutubeChannelID
-			break //return errors.Err("playlist items not found") //TODO: will this work?
-		}
-		videoIDs := make([]string, 50)
-		for i, item := range playlistResponse.Items {
-			// normally we'd send the video into the channel here, but youtube api doesn't have sorting
-			// so we have to get ALL the videos, then sort them, then send them in
-			playlistMap[item.Snippet.ResourceId.VideoId] = item.Snippet
-			videoIDs[i] = item.Snippet.ResourceId.VideoId
-		}
-		req2 := service.Videos.List("snippet,contentDetails,recordingDetails").Id(strings.Join(videoIDs[:], ","))
-
-		videosListResponse, err := req2.Do()
-		if err != nil {
-			return errors.Prefix("error getting videos info", err)
-		}
-		for _, item := range videosListResponse.Items {
-			videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig(), s.grp, ipPool))
-		}
-
-		log.Infof("Got info for %d videos from youtube API", len(videos))
-
-		nextPageToken = playlistResponse.NextPageToken
-		if nextPageToken == "" || s.Manager.SyncFlags.QuickSync || len(videos) >= s.Manager.videosLimit {
-			break
-		}
+	videos, err := ytapi.Enqueue(s.APIConfig.YoutubeAPIKey, s.YoutubeChannelID, s.syncedVideos, s.Manager.SyncFlags.QuickSync, s.Manager.videosLimit, ytapi.VideoParams{
+		VideoDir: s.videoDirectory,
+		S3Config: s.Manager.GetS3AWSConfig(),
+		Grp:      s.grp,
+		IPPool:   ipPool,
+	})
+	if err != nil {
+		return err
 	}
-	for k, v := range s.syncedVideos {
-		if !v.Published {
-			continue
-		}
-		_, ok := playlistMap[k]
-		if !ok {
-			videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig(), s.grp, ipPool))
-		}
-
-	}
-	sort.Sort(byPublishedAt(videos))
 
 Enqueue:
 	for _, v := range videos {
@@ -1116,7 +1013,7 @@ Enqueue:
 	return nil
 }
 
-func (s *Sync) processVideo(v video) (err error) {
+func (s *Sync) processVideo(v ytapi.Video) (err error) {
 	defer func() {
 		if p := recover(); p != nil {
 			logUtils.SendErrorToSlack("Video processing panic! %s", debug.Stack())
@@ -1284,8 +1181,7 @@ func (s *Sync) getUnsentSupports() (float64, error) {
 
 // waitForDaemonProcess observes the running processes and returns when the process is no longer running or when the timeout is up
 func waitForDaemonProcess(timeout time.Duration) error {
-	then := time.Now()
-	stopTime := then.Add(time.Duration(timeout * time.Second))
+	stopTime := time.Now().Add(timeout * time.Second)
 	for !time.Now().After(stopTime) {
 		wait := 10 * time.Second
 		log.Println("the daemon is still running, waiting for it to exit")
diff --git a/sources/youtubeVideo.go b/sources/youtubeVideo.go
index 26c2f66..2953240 100644
--- a/sources/youtubeVideo.go
+++ b/sources/youtubeVideo.go
@@ -13,24 +13,24 @@ import (
 	"sync"
 	"time"
 
-	"github.com/lbryio/lbry.go/v2/extras/errors"
-	"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
-	"github.com/lbryio/lbry.go/v2/extras/stop"
-	"github.com/lbryio/lbry.go/v2/extras/util"
-	"github.com/lbryio/ytsync/v5/timing"
-	logUtils "github.com/lbryio/ytsync/v5/util"
-
 	"github.com/lbryio/ytsync/v5/ip_manager"
 	"github.com/lbryio/ytsync/v5/namer"
 	"github.com/lbryio/ytsync/v5/sdk"
 	"github.com/lbryio/ytsync/v5/tags_manager"
 	"github.com/lbryio/ytsync/v5/thumbs"
+	"github.com/lbryio/ytsync/v5/timing"
+	logUtils "github.com/lbryio/ytsync/v5/util"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
+	"github.com/lbryio/lbry.go/v2/extras/stop"
+	"github.com/lbryio/lbry.go/v2/extras/util"
 
 	duration "github.com/ChannelMeter/iso8601duration"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/shopspring/decimal"
 	log "github.com/sirupsen/logrus"
-	"google.golang.org/api/youtube/v3"
+	ytlib "google.golang.org/api/youtube/v3"
 )
 
 type YoutubeVideo struct {
@@ -43,7 +43,7 @@ type YoutubeVideo struct {
 	maxVideoLength   float64
 	publishedAt      time.Time
 	dir              string
-	youtubeInfo      *youtube.Video
+	youtubeInfo      *ytlib.Video
 	youtubeChannelID string
 	tags             []string
 	awsConfig        aws.Config
@@ -90,7 +90,7 @@ var youtubeCategories = map[string]string{
 	"44": "trailers",
 }
 
-func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
+func NewYoutubeVideo(directory string, videoData *ytlib.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
 	publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
 	return &YoutubeVideo{
 		id:               videoData.Id,
diff --git a/thumbs/uploader.go b/thumbs/uploader.go
index 6cf21cd..19e9f84 100644
--- a/thumbs/uploader.go
+++ b/thumbs/uploader.go
@@ -11,7 +11,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	log "github.com/sirupsen/logrus"
-	"google.golang.org/api/youtube/v3"
+	ytlib "google.golang.org/api/youtube/v3"
 )
 
 type thumbnailUploader struct {
@@ -98,7 +98,7 @@ func MirrorThumbnail(url string, name string, s3Config aws.Config) (string, erro
 	return tu.mirroredUrl, nil
 }
 
-func GetBestThumbnail(thumbnails *youtube.ThumbnailDetails) *youtube.Thumbnail {
+func GetBestThumbnail(thumbnails *ytlib.ThumbnailDetails) *ytlib.Thumbnail {
 	if thumbnails.Maxres != nil {
 		return thumbnails.Maxres
 	} else if thumbnails.High != nil {
diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go
new file mode 100644
index 0000000..942e344
--- /dev/null
+++ b/ytapi/ytapi.go
@@ -0,0 +1,213 @@
+package ytapi
+
+import (
+	"net/http"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/lbryio/ytsync/v5/ip_manager"
+	"github.com/lbryio/ytsync/v5/sdk"
+	"github.com/lbryio/ytsync/v5/sources"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
+	"github.com/lbryio/lbry.go/v2/extras/stop"
+
+	"github.com/aws/aws-sdk-go/aws"
+	log "github.com/sirupsen/logrus"
+	"google.golang.org/api/googleapi/transport"
+	ytlib "google.golang.org/api/youtube/v3"
+)
+
+type Video interface {
+	Size() *int64
+	ID() string
+	IDAndNum() string
+	PlaylistPosition() int
+	PublishedAt() time.Time
+	Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool, *sync.RWMutex) (*sources.SyncSummary, error)
+}
+
+type byPublishedAt []Video
+
+func (a byPublishedAt) Len() int           { return len(a) }
+func (a byPublishedAt) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a byPublishedAt) Less(i, j int) bool { return a[i].PublishedAt().Before(a[j].PublishedAt()) }
+
+type VideoParams struct {
+	VideoDir string
+	S3Config aws.Config
+	Grp      *stop.Group
+	IPPool   *ip_manager.IPPool
+}
+
+var mostRecentlyFailedChannel string // TODO: fix this hack!
+
+func Enqueue(apiKey, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams) ([]Video, error) {
+	playlistID, err := PlaylistID(apiKey, channelID)
+	if err != nil {
+		return nil, err
+	}
+
+	playlistMap := make(map[string]*ytlib.PlaylistItemSnippet, 50)
+	var playlistItems []*ytlib.PlaylistItem
+	var nextPageToken string
+	var videos []Video
+
+	for {
+		playlistItems, nextPageToken, err = PlaylistItems(apiKey, playlistID, nextPageToken)
+
+		if len(playlistItems) < 1 {
+			// If there are 50+ videos in a playlist but less than 50 are actually returned by the API, youtube will still redirect
+			// clients to a next page. Such next page will however be empty. This logic prevents ytsync from failing.
+			youtubeIsLying := len(videos) > 0
+			if youtubeIsLying {
+				break
+			}
+			if channelID == mostRecentlyFailedChannel {
+				return nil, errors.Err("playlist items not found")
+			}
+			mostRecentlyFailedChannel = channelID
+			break //return errors.Err("playlist items not found") //TODO: will this work?
+		}
+
+		videoIDs := make([]string, len(playlistItems))
+		for i, item := range playlistItems {
+			// normally we'd send the video into the channel here, but youtube api doesn't have sorting
+			// so we have to get ALL the videos, then sort them, then send them in
+			playlistMap[item.Snippet.ResourceId.VideoId] = item.Snippet
+			videoIDs[i] = item.Snippet.ResourceId.VideoId
+		}
+
+		vids, err := Videos(apiKey, videoIDs)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, item := range vids {
+			videos = append(videos, sources.NewYoutubeVideo(videoParams.VideoDir, item, playlistMap[item.Id].Position, videoParams.S3Config, videoParams.Grp, videoParams.IPPool))
+		}
+
+		log.Infof("Got info for %d videos from youtube API", len(videos))
+
+		if nextPageToken == "" || quickSync || len(videos) >= maxVideos {
+			break
+		}
+	}
+
+	for k, v := range syncedVideos {
+		if !v.Published {
+			continue
+		}
+		if _, ok := playlistMap[k]; !ok {
+			videos = append(videos, sources.NewMockedVideo(videoParams.VideoDir, k, channelID, videoParams.S3Config, videoParams.Grp, videoParams.IPPool))
+		}
+
+	}
+
+	sort.Sort(byPublishedAt(videos))
+
+	return videos, nil
+}
+
+func VideosInChannel(apiKey, channelID string) (uint64, error) {
+	client := &http.Client{
+		Transport: &transport.APIKey{Key: apiKey},
+	}
+
+	service, err := ytlib.New(client)
+	if err != nil {
+		return 0, errors.Prefix("error creating YouTube service", err)
+	}
+
+	response, err := service.Channels.List("statistics").Id(channelID).Do()
+	if err != nil {
+		return 0, errors.Prefix("error getting channels", err)
+	}
+
+	if len(response.Items) < 1 {
+		return 0, errors.Err("youtube channel not found")
+	}
+
+	return response.Items[0].Statistics.VideoCount, nil
+}
+
+func ChannelInfo(apiKey, channelID string) (*ytlib.ChannelSnippet, *ytlib.ChannelBrandingSettings, error) {
+	service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
+	if err != nil {
+		return nil, nil, errors.Prefix("error creating YouTube service", err)
+	}
+
+	response, err := service.Channels.List("snippet,brandingSettings").Id(channelID).Do()
+	if err != nil {
+		return nil, nil, errors.Prefix("error getting channel details", err)
+	}
+
+	if len(response.Items) < 1 {
+		return nil, nil, errors.Err("youtube channel not found")
+	}
+
+	return response.Items[0].Snippet, response.Items[0].BrandingSettings, nil
+}
+
+func PlaylistID(apiKey, channelID string) (string, error) {
+	service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
+	if err != nil {
+		return "", errors.Prefix("error creating YouTube service", err)
+	}
+
+	response, err := service.Channels.List("contentDetails").Id(channelID).Do()
+	if err != nil {
+		return "", errors.Prefix("error getting channel details", err)
+	}
+
+	if len(response.Items) < 1 {
+		return "", errors.Err("youtube channel not found")
+	}
+
+	if response.Items[0].ContentDetails.RelatedPlaylists == nil {
+		return "", errors.Err("no related playlists")
+	}
+
+	playlistID := response.Items[0].ContentDetails.RelatedPlaylists.Uploads
+	if playlistID == "" {
+		return "", errors.Err("no channel playlist")
+	}
+
+	return playlistID, nil
+}
+
+func PlaylistItems(apiKey, playlistID, nextPageToken string) ([]*ytlib.PlaylistItem, string, error) {
+	service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
+	if err != nil {
+		return nil, "", errors.Prefix("error creating YouTube service", err)
+	}
+
+	response, err := service.PlaylistItems.List("snippet").
+		PlaylistId(playlistID).
+		MaxResults(50).
+		PageToken(nextPageToken).
+		Do()
+
+	if err != nil {
+		return nil, "", errors.Prefix("error getting playlist items", err)
+	}
+
+	return response.Items, response.NextPageToken, nil
+}
+
+func Videos(apiKey string, videoIDs []string) ([]*ytlib.Video, error) {
+	service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
+	if err != nil {
+		return nil, errors.Prefix("error creating YouTube service", err)
+	}
+
+	response, err := service.Videos.List("snippet,contentDetails,recordingDetails").Id(strings.Join(videoIDs[:], ",")).Do()
+	if err != nil {
+		return nil, errors.Prefix("error getting videos info", err)
+	}
+
+	return response.Items, nil
+}

From edbb22fcf0d06169481e2cdd35e515657204bcec Mon Sep 17 00:00:00 2001
From: Alex Grintsvayg <grin@lbry.com>
Date: Mon, 27 Jul 2020 14:49:37 -0400
Subject: [PATCH 02/57] rename

---
 manager/ytsync.go | 2 +-
 ytapi/ytapi.go    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/manager/ytsync.go b/manager/ytsync.go
index 956e50f..feebc37 100644
--- a/manager/ytsync.go
+++ b/manager/ytsync.go
@@ -985,7 +985,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
 		return err
 	}
 
-	videos, err := ytapi.Enqueue(s.APIConfig.YoutubeAPIKey, s.YoutubeChannelID, s.syncedVideos, s.Manager.SyncFlags.QuickSync, s.Manager.videosLimit, ytapi.VideoParams{
+	videos, err := ytapi.GetVideosToSync(s.APIConfig.YoutubeAPIKey, s.YoutubeChannelID, s.syncedVideos, s.Manager.SyncFlags.QuickSync, s.Manager.videosLimit, ytapi.VideoParams{
 		VideoDir: s.videoDirectory,
 		S3Config: s.Manager.GetS3AWSConfig(),
 		Grp:      s.grp,
diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go
index 942e344..a1c5d82 100644
--- a/ytapi/ytapi.go
+++ b/ytapi/ytapi.go
@@ -45,7 +45,7 @@ type VideoParams struct {
 
 var mostRecentlyFailedChannel string // TODO: fix this hack!
 
-func Enqueue(apiKey, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams) ([]Video, error) {
+func GetVideosToSync(apiKey, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams) ([]Video, error) {
 	playlistID, err := PlaylistID(apiKey, channelID)
 	if err != nil {
 		return nil, err

From bffc0823be2a787df76397bd096cc918238b5cd3 Mon Sep 17 00:00:00 2001
From: Alex Grintsvayg <grin@lbry.com>
Date: Mon, 27 Jul 2020 14:52:21 -0400
Subject: [PATCH 03/57] more

---
 manager/count.go |  9 ---------
 manager/setup.go |  2 +-
 ytapi/ytapi.go   | 14 +++++++-------
 3 files changed, 8 insertions(+), 17 deletions(-)
 delete mode 100644 manager/count.go

diff --git a/manager/count.go b/manager/count.go
deleted file mode 100644
index 51c78dd..0000000
--- a/manager/count.go
+++ /dev/null
@@ -1,9 +0,0 @@
-package manager
-
-import (
-	"github.com/lbryio/ytsync/v5/ytapi"
-)
-
-func (s *Sync) CountVideos() (uint64, error) {
-	return ytapi.VideosInChannel(s.APIConfig.YoutubeAPIKey, s.YoutubeChannelID)
-}
diff --git a/manager/setup.go b/manager/setup.go
index 951fe41..decfa18 100644
--- a/manager/setup.go
+++ b/manager/setup.go
@@ -72,7 +72,7 @@ func (s *Sync) walletSetup() error {
 	}
 	log.Debugf("Starting balance is %.4f", balance)
 
-	n, err := s.CountVideos()
+	n, err := ytapi.CountVideosInChannel(s.APIConfig.YoutubeAPIKey, s.YoutubeChannelID)
 	if err != nil {
 		return err
 	}
diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go
index a1c5d82..bc28360 100644
--- a/ytapi/ytapi.go
+++ b/ytapi/ytapi.go
@@ -46,7 +46,7 @@ type VideoParams struct {
 var mostRecentlyFailedChannel string // TODO: fix this hack!
 
 func GetVideosToSync(apiKey, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams) ([]Video, error) {
-	playlistID, err := PlaylistID(apiKey, channelID)
+	playlistID, err := getPlaylistID(apiKey, channelID)
 	if err != nil {
 		return nil, err
 	}
@@ -57,7 +57,7 @@ func GetVideosToSync(apiKey, channelID string, syncedVideos map[string]sdk.Synce
 	var videos []Video
 
 	for {
-		playlistItems, nextPageToken, err = PlaylistItems(apiKey, playlistID, nextPageToken)
+		playlistItems, nextPageToken, err = getPlaylistItems(apiKey, playlistID, nextPageToken)
 
 		if len(playlistItems) < 1 {
 			// If there are 50+ videos in a playlist but less than 50 are actually returned by the API, youtube will still redirect
@@ -81,7 +81,7 @@ func GetVideosToSync(apiKey, channelID string, syncedVideos map[string]sdk.Synce
 			videoIDs[i] = item.Snippet.ResourceId.VideoId
 		}
 
-		vids, err := Videos(apiKey, videoIDs)
+		vids, err := getVideos(apiKey, videoIDs)
 		if err != nil {
 			return nil, err
 		}
@@ -112,7 +112,7 @@ func GetVideosToSync(apiKey, channelID string, syncedVideos map[string]sdk.Synce
 	return videos, nil
 }
 
-func VideosInChannel(apiKey, channelID string) (uint64, error) {
+func CountVideosInChannel(apiKey, channelID string) (uint64, error) {
 	client := &http.Client{
 		Transport: &transport.APIKey{Key: apiKey},
 	}
@@ -152,7 +152,7 @@ func ChannelInfo(apiKey, channelID string) (*ytlib.ChannelSnippet, *ytlib.Channe
 	return response.Items[0].Snippet, response.Items[0].BrandingSettings, nil
 }
 
-func PlaylistID(apiKey, channelID string) (string, error) {
+func getPlaylistID(apiKey, channelID string) (string, error) {
 	service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
 	if err != nil {
 		return "", errors.Prefix("error creating YouTube service", err)
@@ -179,7 +179,7 @@ func PlaylistID(apiKey, channelID string) (string, error) {
 	return playlistID, nil
 }
 
-func PlaylistItems(apiKey, playlistID, nextPageToken string) ([]*ytlib.PlaylistItem, string, error) {
+func getPlaylistItems(apiKey, playlistID, nextPageToken string) ([]*ytlib.PlaylistItem, string, error) {
 	service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
 	if err != nil {
 		return nil, "", errors.Prefix("error creating YouTube service", err)
@@ -198,7 +198,7 @@ func PlaylistItems(apiKey, playlistID, nextPageToken string) ([]*ytlib.PlaylistI
 	return response.Items, response.NextPageToken, nil
 }
 
-func Videos(apiKey string, videoIDs []string) ([]*ytlib.Video, error) {
+func getVideos(apiKey string, videoIDs []string) ([]*ytlib.Video, error) {
 	service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
 	if err != nil {
 		return nil, errors.Prefix("error creating YouTube service", err)

From f39fc11697a8c52ea371ab4736a674461d48cf1d Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Mon, 27 Jul 2020 15:42:45 -0400
Subject: [PATCH 04/57] Add downloader to create api adjust GetVideosToSync

---
 downloader/downloader.go      |  44 +++++++++++++
 downloader/downloader_test.go |  20 ++++++
 ytapi/ytapi.go                | 119 ++++++++--------------------------
 3 files changed, 91 insertions(+), 92 deletions(-)
 create mode 100644 downloader/downloader.go
 create mode 100644 downloader/downloader_test.go

diff --git a/downloader/downloader.go b/downloader/downloader.go
new file mode 100644
index 0000000..43a7b06
--- /dev/null
+++ b/downloader/downloader.go
@@ -0,0 +1,44 @@
+package downloader
+
+import (
+	"io/ioutil"
+	"os/exec"
+	"strings"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/sirupsen/logrus"
+)
+
+func GetPlaylistVideoIDs(channelName string) ([]string, error) {
+	args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist"}
+	return run(args)
+}
+
+func run(args []string) ([]string, error) {
+	cmd := exec.Command("youtube-dl", args...)
+	logrus.Printf("Running command youtube-dl %s", strings.Join(args, " "))
+
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+
+	if err := cmd.Start(); err != nil {
+		return nil, errors.Err(err)
+	}
+
+	errorLog, _ := ioutil.ReadAll(stderr)
+	outLog, _ := ioutil.ReadAll(stdout)
+	err = cmd.Wait()
+	if len(errorLog) > 0 {
+		return nil, errors.Err(err)
+	}
+	if len(errorLog) > 0 {
+		return nil, errors.Err(string(errorLog))
+	}
+	return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
+}
diff --git a/downloader/downloader_test.go b/downloader/downloader_test.go
new file mode 100644
index 0000000..6a5b5de
--- /dev/null
+++ b/downloader/downloader_test.go
@@ -0,0 +1,20 @@
+package downloader
+
+import (
+	"testing"
+
+	"github.com/sirupsen/logrus"
+)
+
+func TestRun(t *testing.T) {
+	//args := []string{"--skip-download", "https://www.youtube.com/c/Electroboom", "--get-id", "--flat-playlist"}
+	//videoIDs, err := GetPlaylistVideoIDs("Electroboom")
+	videoIDs, err := GetPlaylistVideoIDs("UCJ0-OtVpF0wOKEqT2Z1HEtA")
+	if err != nil {
+		logrus.Error(err)
+	}
+	for _, id := range videoIDs {
+		println(id)
+	}
+	t.Error("just stop")
+}
diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go
index bc28360..b8eeb06 100644
--- a/ytapi/ytapi.go
+++ b/ytapi/ytapi.go
@@ -7,6 +7,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/lbryio/ytsync/v5/downloader"
+
 	"github.com/lbryio/ytsync/v5/ip_manager"
 	"github.com/lbryio/ytsync/v5/sdk"
 	"github.com/lbryio/ytsync/v5/sources"
@@ -46,55 +48,35 @@ type VideoParams struct {
 var mostRecentlyFailedChannel string // TODO: fix this hack!
 
 func GetVideosToSync(apiKey, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams) ([]Video, error) {
-	playlistID, err := getPlaylistID(apiKey, channelID)
+
+	var videos []Video
+	videoIDs, err := downloader.GetPlaylistVideoIDs(channelID)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+
+	log.Infof("Got info for %d videos from youtube downloader", len(videoIDs))
+
+	playlistMap := make(map[string]int64)
+	for i, videoID := range videoIDs {
+		playlistMap[videoID] = int64(i)
+	}
+
+	if len(videoIDs) < 1 {
+		if channelID == mostRecentlyFailedChannel {
+			return nil, errors.Err("playlist items not found")
+		}
+		mostRecentlyFailedChannel = channelID
+	}
+
+	vids, err := getVideos(apiKey, videoIDs)
 	if err != nil {
 		return nil, err
 	}
 
-	playlistMap := make(map[string]*ytlib.PlaylistItemSnippet, 50)
-	var playlistItems []*ytlib.PlaylistItem
-	var nextPageToken string
-	var videos []Video
-
-	for {
-		playlistItems, nextPageToken, err = getPlaylistItems(apiKey, playlistID, nextPageToken)
-
-		if len(playlistItems) < 1 {
-			// If there are 50+ videos in a playlist but less than 50 are actually returned by the API, youtube will still redirect
-			// clients to a next page. Such next page will however be empty. This logic prevents ytsync from failing.
-			youtubeIsLying := len(videos) > 0
-			if youtubeIsLying {
-				break
-			}
-			if channelID == mostRecentlyFailedChannel {
-				return nil, errors.Err("playlist items not found")
-			}
-			mostRecentlyFailedChannel = channelID
-			break //return errors.Err("playlist items not found") //TODO: will this work?
-		}
-
-		videoIDs := make([]string, len(playlistItems))
-		for i, item := range playlistItems {
-			// normally we'd send the video into the channel here, but youtube api doesn't have sorting
-			// so we have to get ALL the videos, then sort them, then send them in
-			playlistMap[item.Snippet.ResourceId.VideoId] = item.Snippet
-			videoIDs[i] = item.Snippet.ResourceId.VideoId
-		}
-
-		vids, err := getVideos(apiKey, videoIDs)
-		if err != nil {
-			return nil, err
-		}
-
-		for _, item := range vids {
-			videos = append(videos, sources.NewYoutubeVideo(videoParams.VideoDir, item, playlistMap[item.Id].Position, videoParams.S3Config, videoParams.Grp, videoParams.IPPool))
-		}
-
-		log.Infof("Got info for %d videos from youtube API", len(videos))
-
-		if nextPageToken == "" || quickSync || len(videos) >= maxVideos {
-			break
-		}
+	for _, item := range vids {
+		positionInList := playlistMap[item.Id]
+		videos = append(videos, sources.NewYoutubeVideo(videoParams.VideoDir, item, positionInList, videoParams.S3Config, videoParams.Grp, videoParams.IPPool))
 	}
 
 	for k, v := range syncedVideos {
@@ -104,7 +86,6 @@ func GetVideosToSync(apiKey, channelID string, syncedVideos map[string]sdk.Synce
 		if _, ok := playlistMap[k]; !ok {
 			videos = append(videos, sources.NewMockedVideo(videoParams.VideoDir, k, channelID, videoParams.S3Config, videoParams.Grp, videoParams.IPPool))
 		}
-
 	}
 
 	sort.Sort(byPublishedAt(videos))
@@ -152,52 +133,6 @@ func ChannelInfo(apiKey, channelID string) (*ytlib.ChannelSnippet, *ytlib.Channe
 	return response.Items[0].Snippet, response.Items[0].BrandingSettings, nil
 }
 
-func getPlaylistID(apiKey, channelID string) (string, error) {
-	service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
-	if err != nil {
-		return "", errors.Prefix("error creating YouTube service", err)
-	}
-
-	response, err := service.Channels.List("contentDetails").Id(channelID).Do()
-	if err != nil {
-		return "", errors.Prefix("error getting channel details", err)
-	}
-
-	if len(response.Items) < 1 {
-		return "", errors.Err("youtube channel not found")
-	}
-
-	if response.Items[0].ContentDetails.RelatedPlaylists == nil {
-		return "", errors.Err("no related playlists")
-	}
-
-	playlistID := response.Items[0].ContentDetails.RelatedPlaylists.Uploads
-	if playlistID == "" {
-		return "", errors.Err("no channel playlist")
-	}
-
-	return playlistID, nil
-}
-
-func getPlaylistItems(apiKey, playlistID, nextPageToken string) ([]*ytlib.PlaylistItem, string, error) {
-	service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
-	if err != nil {
-		return nil, "", errors.Prefix("error creating YouTube service", err)
-	}
-
-	response, err := service.PlaylistItems.List("snippet").
-		PlaylistId(playlistID).
-		MaxResults(50).
-		PageToken(nextPageToken).
-		Do()
-
-	if err != nil {
-		return nil, "", errors.Prefix("error getting playlist items", err)
-	}
-
-	return response.Items, response.NextPageToken, nil
-}
-
 func getVideos(apiKey string, videoIDs []string) ([]*ytlib.Video, error) {
 	service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
 	if err != nil {

From 1369ed0b48e21761f7439140dbd57df355fd9f9c Mon Sep 17 00:00:00 2001
From: Alex Grintsvayg <grin@lbry.com>
Date: Mon, 27 Jul 2020 15:51:03 -0400
Subject: [PATCH 05/57] replace yt channel video count with a scrape of
 socialblade.com

---
 manager/setup.go |  3 +--
 ytapi/ytapi.go   | 47 ++++++++++++++++++++++++++++++++---------------
 2 files changed, 33 insertions(+), 17 deletions(-)

diff --git a/manager/setup.go b/manager/setup.go
index decfa18..aab2251 100644
--- a/manager/setup.go
+++ b/manager/setup.go
@@ -72,11 +72,10 @@ func (s *Sync) walletSetup() error {
 	}
 	log.Debugf("Starting balance is %.4f", balance)
 
-	n, err := ytapi.CountVideosInChannel(s.APIConfig.YoutubeAPIKey, s.YoutubeChannelID)
+	videosOnYoutube, err := ytapi.CountVideosInChannel(s.YoutubeChannelID)
 	if err != nil {
 		return err
 	}
-	videosOnYoutube := int(n)
 
 	log.Debugf("Source channel has %d videos", videosOnYoutube)
 	if videosOnYoutube == 0 {
diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go
index b8eeb06..093fd80 100644
--- a/ytapi/ytapi.go
+++ b/ytapi/ytapi.go
@@ -1,14 +1,16 @@
 package ytapi
 
 import (
+	"bufio"
 	"net/http"
+	"regexp"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/lbryio/ytsync/v5/downloader"
-
 	"github.com/lbryio/ytsync/v5/ip_manager"
 	"github.com/lbryio/ytsync/v5/sdk"
 	"github.com/lbryio/ytsync/v5/sources"
@@ -93,26 +95,41 @@ func GetVideosToSync(apiKey, channelID string, syncedVideos map[string]sdk.Synce
 	return videos, nil
 }
 
-func CountVideosInChannel(apiKey, channelID string) (uint64, error) {
-	client := &http.Client{
-		Transport: &transport.APIKey{Key: apiKey},
-	}
-
-	service, err := ytlib.New(client)
+func CountVideosInChannel(channelID string) (int, error) {
+	res, err := http.Get("https://socialblade.com/youtube/channel/" + channelID)
 	if err != nil {
-		return 0, errors.Prefix("error creating YouTube service", err)
+		return 0, err
+	}
+	defer res.Body.Close()
+
+	var line string
+
+	scanner := bufio.NewScanner(res.Body)
+	for scanner.Scan() {
+		if strings.Contains(scanner.Text(), "youtube-stats-header-uploads") {
+			line = scanner.Text()
+			break
+		}
 	}
 
-	response, err := service.Channels.List("statistics").Id(channelID).Do()
+	if err := scanner.Err(); err != nil {
+		return 0, err
+	}
+	if line == "" {
+		return 0, errors.Err("upload count line not found")
+	}
+
+	matches := regexp.MustCompile(">([0-9]+)<").FindStringSubmatch(line)
+	if len(matches) != 2 {
+		return 0, errors.Err("upload count not found with regex")
+	}
+
+	num, err := strconv.Atoi(matches[1])
 	if err != nil {
-		return 0, errors.Prefix("error getting channels", err)
+		return 0, errors.Err(err)
 	}
 
-	if len(response.Items) < 1 {
-		return 0, errors.Err("youtube channel not found")
-	}
-
-	return response.Items[0].Statistics.VideoCount, nil
+	return num, nil
 }
 
 func ChannelInfo(apiKey, channelID string) (*ytlib.ChannelSnippet, *ytlib.ChannelBrandingSettings, error) {

From e926a2c1f678b1bf4533881db1d6169845bad323 Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Mon, 27 Jul 2020 15:57:19 -0400
Subject: [PATCH 06/57] Fix quickSync && maxVideos

---
 downloader/downloader.go      | 15 +++++++++++++--
 downloader/downloader_test.go |  3 +--
 ytapi/ytapi.go                |  5 ++++-
 3 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/downloader/downloader.go b/downloader/downloader.go
index 43a7b06..f63ba4b 100644
--- a/downloader/downloader.go
+++ b/downloader/downloader.go
@@ -9,9 +9,20 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func GetPlaylistVideoIDs(channelName string) ([]string, error) {
+func GetPlaylistVideoIDs(channelName string, maxVideos int) ([]string, error) {
 	args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist"}
-	return run(args)
+	ids, err := run(args)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	videoIDs := make([]string, maxVideos)
+	for i, v := range ids {
+		if i >= maxVideos {
+			break
+		}
+		videoIDs[i] = v
+	}
+	return videoIDs, nil
 }
 
 func run(args []string) ([]string, error) {
diff --git a/downloader/downloader_test.go b/downloader/downloader_test.go
index 6a5b5de..26e2aaf 100644
--- a/downloader/downloader_test.go
+++ b/downloader/downloader_test.go
@@ -9,12 +9,11 @@ import (
 func TestRun(t *testing.T) {
 	//args := []string{"--skip-download", "https://www.youtube.com/c/Electroboom", "--get-id", "--flat-playlist"}
 	//videoIDs, err := GetPlaylistVideoIDs("Electroboom")
-	videoIDs, err := GetPlaylistVideoIDs("UCJ0-OtVpF0wOKEqT2Z1HEtA")
+	videoIDs, err := GetPlaylistVideoIDs("UCJ0-OtVpF0wOKEqT2Z1HEtA", 50)
 	if err != nil {
 		logrus.Error(err)
 	}
 	for _, id := range videoIDs {
 		println(id)
 	}
-	t.Error("just stop")
 }
diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go
index 093fd80..a846b02 100644
--- a/ytapi/ytapi.go
+++ b/ytapi/ytapi.go
@@ -52,7 +52,10 @@ var mostRecentlyFailedChannel string // TODO: fix this hack!
 func GetVideosToSync(apiKey, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams) ([]Video, error) {
 
 	var videos []Video
-	videoIDs, err := downloader.GetPlaylistVideoIDs(channelID)
+	if quickSync {
+		maxVideos = 50
+	}
+	videoIDs, err := downloader.GetPlaylistVideoIDs(channelID, maxVideos)
 	if err != nil {
 		return nil, errors.Err(err)
 	}

From b59ef282676bf98ca2afb2cfee03f12262cf80d7 Mon Sep 17 00:00:00 2001
From: Alex Grintsvayg <grin@lbry.com>
Date: Mon, 27 Jul 2020 16:02:52 -0400
Subject: [PATCH 07/57] punting on ChannelInfo for now

---
 ytapi/ytapi.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go
index a846b02..9f6cca0 100644
--- a/ytapi/ytapi.go
+++ b/ytapi/ytapi.go
@@ -136,6 +136,7 @@ func CountVideosInChannel(channelID string) (int, error) {
 }
 
 func ChannelInfo(apiKey, channelID string) (*ytlib.ChannelSnippet, *ytlib.ChannelBrandingSettings, error) {
+	panic("not de-youtube-ified yet")
 	service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
 	if err != nil {
 		return nil, nil, errors.Prefix("error creating YouTube service", err)

From 0bb6b6d833ea7bc70b973a52b670458089f7428d Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Mon, 27 Jul 2020 17:14:06 -0400
Subject: [PATCH 08/57] commit my shit

---
 downloader/downloader.go      |  64 ++++++++++++----
 downloader/downloader_test.go |  14 +++-
 downloader/ytdl/Video.go      | 140 ++++++++++++++++++++++++++++++++++
 sources/youtubeVideo.go       |  74 +++++++++---------
 thumbs/uploader.go            |  22 +++---
 ytapi/ytapi.go                |  32 ++++----
 6 files changed, 265 insertions(+), 81 deletions(-)
 create mode 100644 downloader/ytdl/Video.go

diff --git a/downloader/downloader.go b/downloader/downloader.go
index f63ba4b..b49d6e5 100644
--- a/downloader/downloader.go
+++ b/downloader/downloader.go
@@ -1,17 +1,21 @@
 package downloader
 
 import (
+	"encoding/json"
+	"io"
 	"io/ioutil"
 	"os/exec"
 	"strings"
 
+	"github.com/lbryio/ytsync/v5/downloader/ytdl"
+
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/sirupsen/logrus"
 )
 
 func GetPlaylistVideoIDs(channelName string, maxVideos int) ([]string, error) {
 	args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist"}
-	ids, err := run(args)
+	ids, err := run(args, true, true)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
@@ -25,26 +29,56 @@ func GetPlaylistVideoIDs(channelName string, maxVideos int) ([]string, error) {
 	return videoIDs, nil
 }
 
-func run(args []string) ([]string, error) {
+func GetVideoInformation(videoID string) (*ytdl.YtdlVideo, error) {
+	args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID}
+	results, err := run(args, false, true)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	var video *ytdl.YtdlVideo
+	err = json.Unmarshal([]byte(results[0]), &video)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	return video, nil
+}
+
+func run(args []string, withStdErr, withStdOut bool) ([]string, error) {
 	cmd := exec.Command("youtube-dl", args...)
 	logrus.Printf("Running command youtube-dl %s", strings.Join(args, " "))
 
-	stderr, err := cmd.StderrPipe()
-	if err != nil {
-		return nil, errors.Err(err)
-	}
-	stdout, err := cmd.StdoutPipe()
-	if err != nil {
-		return nil, errors.Err(err)
+	var stderr io.ReadCloser
+	var errorLog []byte
+	if withStdErr {
+		var err error
+		stderr, err = cmd.StderrPipe()
+		if err != nil {
+			return nil, errors.Err(err)
+		}
+		errorLog, err = ioutil.ReadAll(stderr)
+		if err != nil {
+			return nil, errors.Err(err)
+		}
 	}
 
-	if err := cmd.Start(); err != nil {
-		return nil, errors.Err(err)
-	}
+	var stdout io.ReadCloser
+	var outLog []byte
+	if withStdOut {
+		var err error
+		stdout, err = cmd.StdoutPipe()
+		if err != nil {
+			return nil, errors.Err(err)
+		}
 
-	errorLog, _ := ioutil.ReadAll(stderr)
-	outLog, _ := ioutil.ReadAll(stdout)
-	err = cmd.Wait()
+		if err := cmd.Start(); err != nil {
+			return nil, errors.Err(err)
+		}
+		outLog, err = ioutil.ReadAll(stdout)
+		if err != nil {
+			return nil, errors.Err(err)
+		}
+	}
+	err := cmd.Wait()
 	if len(errorLog) > 0 {
 		return nil, errors.Err(err)
 	}
diff --git a/downloader/downloader_test.go b/downloader/downloader_test.go
index 26e2aaf..1a80d6d 100644
--- a/downloader/downloader_test.go
+++ b/downloader/downloader_test.go
@@ -6,9 +6,7 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func TestRun(t *testing.T) {
-	//args := []string{"--skip-download", "https://www.youtube.com/c/Electroboom", "--get-id", "--flat-playlist"}
-	//videoIDs, err := GetPlaylistVideoIDs("Electroboom")
+func TestGetPlaylistVideoIDs(t *testing.T) {
 	videoIDs, err := GetPlaylistVideoIDs("UCJ0-OtVpF0wOKEqT2Z1HEtA", 50)
 	if err != nil {
 		logrus.Error(err)
@@ -17,3 +15,13 @@ func TestRun(t *testing.T) {
 		println(id)
 	}
 }
+
+func TestGetVideoInformation(t *testing.T) {
+	video, err := GetVideoInformation("zj7pXM9gE5M")
+	if err != nil {
+		logrus.Error(err)
+	}
+	if video != nil {
+		logrus.Info(video.ID)
+	}
+}
diff --git a/downloader/ytdl/Video.go b/downloader/ytdl/Video.go
new file mode 100644
index 0000000..4dff358
--- /dev/null
+++ b/downloader/ytdl/Video.go
@@ -0,0 +1,140 @@
+package ytdl
+
+type YtdlVideo struct {
+	UploadDate         string      `json:"upload_date"`
+	Extractor          string      `json:"extractor"`
+	Series             interface{} `json:"series"`
+	Format             string      `json:"format"`
+	Vbr                interface{} `json:"vbr"`
+	Chapters           interface{} `json:"chapters"`
+	Height             int         `json:"height"`
+	LikeCount          interface{} `json:"like_count"`
+	Duration           int         `json:"duration"`
+	Fulltitle          string      `json:"fulltitle"`
+	PlaylistIndex      interface{} `json:"playlist_index"`
+	Album              interface{} `json:"album"`
+	ViewCount          int         `json:"view_count"`
+	Playlist           interface{} `json:"playlist"`
+	Title              string      `json:"title"`
+	Filename           string      `json:"_filename"`
+	Creator            interface{} `json:"creator"`
+	Ext                string      `json:"ext"`
+	ID                 string      `json:"id"`
+	DislikeCount       interface{} `json:"dislike_count"`
+	AverageRating      float64     `json:"average_rating"`
+	Abr                int         `json:"abr"`
+	UploaderURL        string      `json:"uploader_url"`
+	Categories         []string    `json:"categories"`
+	Fps                int         `json:"fps"`
+	StretchedRatio     interface{} `json:"stretched_ratio"`
+	SeasonNumber       interface{} `json:"season_number"`
+	Annotations        interface{} `json:"annotations"`
+	WebpageURLBasename string      `json:"webpage_url_basename"`
+	Acodec             string      `json:"acodec"`
+	DisplayID          string      `json:"display_id"`
+	RequestedFormats   []struct {
+		Asr             interface{} `json:"asr"`
+		Tbr             float64     `json:"tbr"`
+		Container       string      `json:"container"`
+		Language        interface{} `json:"language"`
+		Format          string      `json:"format"`
+		URL             string      `json:"url"`
+		Vcodec          string      `json:"vcodec"`
+		FormatNote      string      `json:"format_note"`
+		Height          int         `json:"height"`
+		Width           int         `json:"width"`
+		Ext             string      `json:"ext"`
+		FragmentBaseURL string      `json:"fragment_base_url"`
+		Filesize        interface{} `json:"filesize"`
+		Fps             int         `json:"fps"`
+		ManifestURL     string      `json:"manifest_url"`
+		Protocol        string      `json:"protocol"`
+		FormatID        string      `json:"format_id"`
+		HTTPHeaders     struct {
+			AcceptCharset  string `json:"Accept-Charset"`
+			AcceptLanguage string `json:"Accept-Language"`
+			AcceptEncoding string `json:"Accept-Encoding"`
+			Accept         string `json:"Accept"`
+			UserAgent      string `json:"User-Agent"`
+		} `json:"http_headers"`
+		Fragments []struct {
+			Path     string  `json:"path"`
+			Duration float64 `json:"duration,omitempty"`
+		} `json:"fragments"`
+		Acodec string `json:"acodec"`
+		Abr    int    `json:"abr,omitempty"`
+	} `json:"requested_formats"`
+	AutomaticCaptions struct {
+	} `json:"automatic_captions"`
+	Description        string      `json:"description"`
+	Tags               []string    `json:"tags"`
+	Track              interface{} `json:"track"`
+	RequestedSubtitles interface{} `json:"requested_subtitles"`
+	StartTime          interface{} `json:"start_time"`
+	Uploader           string      `json:"uploader"`
+	ExtractorKey       string      `json:"extractor_key"`
+	FormatID           string      `json:"format_id"`
+	EpisodeNumber      interface{} `json:"episode_number"`
+	UploaderID         string      `json:"uploader_id"`
+	Subtitles          struct {
+	} `json:"subtitles"`
+	ReleaseYear interface{} `json:"release_year"`
+	Thumbnails  []Thumbnail `json:"thumbnails"`
+	License     interface{} `json:"license"`
+	Artist      interface{} `json:"artist"`
+	AgeLimit    int         `json:"age_limit"`
+	ReleaseDate interface{} `json:"release_date"`
+	AltTitle    interface{} `json:"alt_title"`
+	Thumbnail   string      `json:"thumbnail"`
+	ChannelID   string      `json:"channel_id"`
+	IsLive      interface{} `json:"is_live"`
+	Width       int         `json:"width"`
+	EndTime     interface{} `json:"end_time"`
+	WebpageURL  string      `json:"webpage_url"`
+	Formats     []struct {
+		Asr             int         `json:"asr"`
+		Tbr             float64     `json:"tbr"`
+		Protocol        string      `json:"protocol"`
+		Format          string      `json:"format"`
+		FormatNote      string      `json:"format_note"`
+		Height          interface{} `json:"height"`
+		ManifestURL     string      `json:"manifest_url,omitempty"`
+		FormatID        string      `json:"format_id"`
+		Container       string      `json:"container,omitempty"`
+		Language        interface{} `json:"language,omitempty"`
+		HTTPHeaders     HTTPHeaders `json:"http_headers"`
+		URL             string      `json:"url"`
+		Vcodec          string      `json:"vcodec"`
+		Abr             int         `json:"abr,omitempty"`
+		Width           interface{} `json:"width"`
+		Ext             string      `json:"ext"`
+		FragmentBaseURL string      `json:"fragment_base_url,omitempty"`
+		Filesize        interface{} `json:"filesize"`
+		Fps             interface{} `json:"fps"`
+		Fragments       []struct {
+			Path     string  `json:"path"`
+			Duration float64 `json:"duration,omitempty"`
+		} `json:"fragments,omitempty"`
+		Acodec    string      `json:"acodec"`
+		PlayerURL interface{} `json:"player_url,omitempty"`
+	} `json:"formats"`
+	ChannelURL string      `json:"channel_url"`
+	Resolution interface{} `json:"resolution"`
+	Vcodec     string      `json:"vcodec"`
+}
+
+type Thumbnail struct {
+	URL        string `json:"url"`
+	Width      int    `json:"width"`
+	Resolution string `json:"resolution"`
+	ID         string `json:"id"`
+	Height     int    `json:"height"`
+}
+
+type HTTPHeaders struct {
+	AcceptCharset  string `json:"Accept-Charset"`
+	AcceptLanguage string `json:"Accept-Language"`
+	AcceptEncoding string `json:"Accept-Encoding"`
+	Accept         string `json:"Accept"`
+	UserAgent      string `json:"User-Agent"`
+}
diff --git a/sources/youtubeVideo.go b/sources/youtubeVideo.go
index 2953240..9d11e8a 100644
--- a/sources/youtubeVideo.go
+++ b/sources/youtubeVideo.go
@@ -13,6 +13,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/lbryio/ytsync/v5/downloader/ytdl"
+
 	"github.com/lbryio/ytsync/v5/ip_manager"
 	"github.com/lbryio/ytsync/v5/namer"
 	"github.com/lbryio/ytsync/v5/sdk"
@@ -26,11 +28,9 @@ import (
 	"github.com/lbryio/lbry.go/v2/extras/stop"
 	"github.com/lbryio/lbry.go/v2/extras/util"
 
-	duration "github.com/ChannelMeter/iso8601duration"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/shopspring/decimal"
 	log "github.com/sirupsen/logrus"
-	ytlib "google.golang.org/api/youtube/v3"
 )
 
 type YoutubeVideo struct {
@@ -43,7 +43,7 @@ type YoutubeVideo struct {
 	maxVideoLength   float64
 	publishedAt      time.Time
 	dir              string
-	youtubeInfo      *ytlib.Video
+	youtubeInfo      *ytdl.YtdlVideo
 	youtubeChannelID string
 	tags             []string
 	awsConfig        aws.Config
@@ -90,22 +90,25 @@ var youtubeCategories = map[string]string{
 	"44": "trailers",
 }
 
-func NewYoutubeVideo(directory string, videoData *ytlib.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
-	publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
+func NewYoutubeVideo(directory string, videoData *ytdl.YtdlVideo, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) (*YoutubeVideo, error) {
+	publishedAt, err := time.Parse("20060102", videoData.UploadDate)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
 	return &YoutubeVideo{
-		id:               videoData.Id,
-		title:            videoData.Snippet.Title,
-		description:      videoData.Snippet.Description,
+		id:               videoData.ID,
+		title:            videoData.Title,
+		description:      videoData.Description,
 		playlistPosition: playlistPosition,
 		publishedAt:      publishedAt,
 		dir:              directory,
 		youtubeInfo:      videoData,
 		awsConfig:        awsConfig,
 		mocked:           false,
-		youtubeChannelID: videoData.Snippet.ChannelId,
+		youtubeChannelID: videoData.ChannelID,
 		stopGroup:        stopGroup,
 		pool:             pool,
-	}
+	}, nil
 }
 func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
 	return &YoutubeVideo{
@@ -185,7 +188,7 @@ func (v *YoutubeVideo) download() error {
 	defer func(start time.Time) {
 		timing.TimedComponent("download").Add(time.Since(start))
 	}(start)
-	if v.youtubeInfo.Snippet.LiveBroadcastContent != "none" {
+	if v.youtubeInfo.IsLive != nil {
 		return errors.Err("video is a live stream and hasn't completed yet")
 	}
 	videoPath := v.getFullPath()
@@ -370,8 +373,8 @@ func (v *YoutubeVideo) delete(reason string) error {
 }
 
 func (v *YoutubeVideo) triggerThumbnailSave() (err error) {
-	thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails)
-	v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig)
+	thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails)
+	v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig)
 	return err
 }
 
@@ -448,13 +451,10 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
 
 func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
 	var err error
-	videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
-	if err != nil {
-		return nil, errors.Err(err)
-	}
-	if videoDuration.ToDuration() > time.Duration(v.maxVideoLength*60)*time.Minute {
-		log.Infof("%s is %s long and the limit is %s", v.id, videoDuration.ToDuration().String(), (time.Duration(v.maxVideoLength*60) * time.Minute).String())
-		logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, videoDuration.ToDuration().String(), (time.Duration(v.maxVideoLength*60) * time.Minute).String())
+
+	if float64(v.youtubeInfo.Duration) > v.maxVideoLength {
+		log.Infof("%s is %d long and the limit is %s", v.id, v.youtubeInfo.Duration, (time.Duration(v.maxVideoLength*60) * time.Minute).String())
+		logUtils.SendErrorToSlack("%s is %d long and the limit is %s", v.id, v.youtubeInfo.Duration, (time.Duration(v.maxVideoLength*60) * time.Minute).String())
 		return nil, errors.Err("video is too long to process")
 	}
 	for {
@@ -487,27 +487,30 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
 	locations = nil
 	tags = nil
 	if !v.mocked {
-		if v.youtubeInfo.Snippet.DefaultLanguage != "" {
-			if v.youtubeInfo.Snippet.DefaultLanguage == "iw" {
-				v.youtubeInfo.Snippet.DefaultLanguage = "he"
-			}
-			languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
-		}
+		/*
+			if v.youtubeInfo.Snippet.DefaultLanguage != "" {
+				if v.youtubeInfo.Snippet.DefaultLanguage == "iw" {
+					v.youtubeInfo.Snippet.DefaultLanguage = "he"
+				}
+				languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
+			}*/
 
-		if v.youtubeInfo.RecordingDetails != nil && v.youtubeInfo.RecordingDetails.Location != nil {
+		/*if v.youtubeInfo.!= nil && v.youtubeInfo.RecordingDetails.Location != nil {
 			locations = []jsonrpc.Location{{
 				Latitude:  util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Latitude)),
 				Longitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Longitude)),
 			}}
-		}
-		tags = v.youtubeInfo.Snippet.Tags
+		}*/
+		tags = v.youtubeInfo.Tags
 	}
 	tags, err := tags_manager.SanitizeTags(tags, v.youtubeChannelID)
 	if err != nil {
 		log.Errorln(err.Error())
 	}
 	if !v.mocked {
-		tags = append(tags, youtubeCategories[v.youtubeInfo.Snippet.CategoryId])
+		for _, category := range v.youtubeInfo.Categories {
+			tags = append(tags, youtubeCategories[category])
+		}
 	}
 
 	return languages, locations, tags
@@ -532,8 +535,8 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
 		if v.mocked {
 			return nil, errors.Err("could not find thumbnail for mocked video")
 		}
-		thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails)
-		thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig)
+		thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails)
+		thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig)
 	} else {
 		thumbnailURL = thumbs.ThumbnailEndpoint + v.ID()
 	}
@@ -605,14 +608,9 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
 		}, nil
 	}
 
-	videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
-	if err != nil {
-		return nil, errors.Err(err)
-	}
-
 	streamCreateOptions.ClaimCreateOptions.Title = &v.title
 	streamCreateOptions.ClaimCreateOptions.Description = util.PtrToString(v.getAbbrevDescription())
-	streamCreateOptions.Duration = util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds())))
+	streamCreateOptions.Duration = util.PtrToUint64(uint64(v.youtubeInfo.Duration))
 	streamCreateOptions.ReleaseTime = util.PtrToInt64(v.publishedAt.Unix())
 	start := time.Now()
 	pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
diff --git a/thumbs/uploader.go b/thumbs/uploader.go
index 19e9f84..e1ff62e 100644
--- a/thumbs/uploader.go
+++ b/thumbs/uploader.go
@@ -5,13 +5,14 @@ import (
 	"net/http"
 	"os"
 
+	"github.com/lbryio/ytsync/v5/downloader/ytdl"
+
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	log "github.com/sirupsen/logrus"
-	ytlib "google.golang.org/api/youtube/v3"
 )
 
 type thumbnailUploader struct {
@@ -98,15 +99,14 @@ func MirrorThumbnail(url string, name string, s3Config aws.Config) (string, erro
 	return tu.mirroredUrl, nil
 }
 
-func GetBestThumbnail(thumbnails *ytlib.ThumbnailDetails) *ytlib.Thumbnail {
-	if thumbnails.Maxres != nil {
-		return thumbnails.Maxres
-	} else if thumbnails.High != nil {
-		return thumbnails.High
-	} else if thumbnails.Medium != nil {
-		return thumbnails.Medium
-	} else if thumbnails.Standard != nil {
-		return thumbnails.Standard
+func GetBestThumbnail(thumbnails []ytdl.Thumbnail) *ytdl.Thumbnail {
+	var bestWidth *ytdl.Thumbnail
+	for _, thumbnail := range thumbnails {
+		if bestWidth == nil {
+			bestWidth = &thumbnail
+		} else if bestWidth.Width < thumbnail.Width {
+			bestWidth = &thumbnail
+		}
 	}
-	return thumbnails.Default
+	return bestWidth
 }
diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go
index 9f6cca0..a290b9e 100644
--- a/ytapi/ytapi.go
+++ b/ytapi/ytapi.go
@@ -10,6 +10,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/lbryio/ytsync/v5/downloader/ytdl"
+
 	"github.com/lbryio/ytsync/v5/downloader"
 	"github.com/lbryio/ytsync/v5/ip_manager"
 	"github.com/lbryio/ytsync/v5/sdk"
@@ -74,14 +76,18 @@ func GetVideosToSync(apiKey, channelID string, syncedVideos map[string]sdk.Synce
 		mostRecentlyFailedChannel = channelID
 	}
 
-	vids, err := getVideos(apiKey, videoIDs)
+	vids, err := getVideos(videoIDs)
 	if err != nil {
 		return nil, err
 	}
 
 	for _, item := range vids {
-		positionInList := playlistMap[item.Id]
-		videos = append(videos, sources.NewYoutubeVideo(videoParams.VideoDir, item, positionInList, videoParams.S3Config, videoParams.Grp, videoParams.IPPool))
+		positionInList := playlistMap[item.ID]
+		videoToAdd, err := sources.NewYoutubeVideo(videoParams.VideoDir, item, positionInList, videoParams.S3Config, videoParams.Grp, videoParams.IPPool)
+		if err != nil {
+			return nil, errors.Err(err)
+		}
+		videos = append(videos, videoToAdd)
 	}
 
 	for k, v := range syncedVideos {
@@ -154,16 +160,14 @@ func ChannelInfo(apiKey, channelID string) (*ytlib.ChannelSnippet, *ytlib.Channe
 	return response.Items[0].Snippet, response.Items[0].BrandingSettings, nil
 }
 
-func getVideos(apiKey string, videoIDs []string) ([]*ytlib.Video, error) {
-	service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
-	if err != nil {
-		return nil, errors.Prefix("error creating YouTube service", err)
+func getVideos(videoIDs []string) ([]*ytdl.YtdlVideo, error) {
+	var videos []*ytdl.YtdlVideo
+	for _, videoID := range videoIDs {
+		video, err := downloader.GetVideoInformation(videoID)
+		if err != nil {
+			return nil, errors.Err(err)
+		}
+		videos = append(videos, video)
 	}
-
-	response, err := service.Videos.List("snippet,contentDetails,recordingDetails").Id(strings.Join(videoIDs[:], ",")).Do()
-	if err != nil {
-		return nil, errors.Prefix("error getting videos info", err)
-	}
-
-	return response.Items, nil
+	return videos, nil
 }

From e7fdd21bac9ade187d25bc77246f85bd8569f3b5 Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Mon, 27 Jul 2020 17:19:15 -0400
Subject: [PATCH 09/57] commit some more stuff

---
 manager/setup.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/manager/setup.go b/manager/setup.go
index aab2251..f37c682 100644
--- a/manager/setup.go
+++ b/manager/setup.go
@@ -388,8 +388,8 @@ func (s *Sync) ensureChannelOwnership() error {
 		return err
 	}
 
-	thumbnail := thumbs.GetBestThumbnail(channelInfo.Thumbnails)
-	thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.Url, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
+	thumbnail := thumbs.GetBestThumbnail(nil) //Ignore this for now ( ChannelInfo Panics)
+	thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.URL, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
 	if err != nil {
 		return err
 	}

From 8600077caae9afcc3cb69ed8200d8b0370697406 Mon Sep 17 00:00:00 2001
From: Alex Grintsvayg <grin@lbry.com>
Date: Mon, 27 Jul 2020 17:27:50 -0400
Subject: [PATCH 10/57] no panics. thanks Beam-master

---
 ytapi/ytapi.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go
index a290b9e..db354ec 100644
--- a/ytapi/ytapi.go
+++ b/ytapi/ytapi.go
@@ -142,7 +142,7 @@ func CountVideosInChannel(channelID string) (int, error) {
 }
 
 func ChannelInfo(apiKey, channelID string) (*ytlib.ChannelSnippet, *ytlib.ChannelBrandingSettings, error) {
-	panic("not de-youtube-ified yet")
+	return nil, nil, errors.Err("ChannelInfo doesn't work yet because we're focused on existing channels")
 	service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
 	if err != nil {
 		return nil, nil, errors.Prefix("error creating YouTube service", err)

From 8e61cde0a05c40a77c704b165d0d50f729ad6a8b Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Mon, 27 Jul 2020 17:31:28 -0400
Subject: [PATCH 11/57] turn of std error

---
 downloader/downloader.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/downloader/downloader.go b/downloader/downloader.go
index b49d6e5..94f0bea 100644
--- a/downloader/downloader.go
+++ b/downloader/downloader.go
@@ -15,7 +15,7 @@ import (
 
 func GetPlaylistVideoIDs(channelName string, maxVideos int) ([]string, error) {
 	args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist"}
-	ids, err := run(args, true, true)
+	ids, err := run(args, false, true)
 	if err != nil {
 		return nil, errors.Err(err)
 	}

From 70ad891dfa61de4376a98f2251563580c0541813 Mon Sep 17 00:00:00 2001
From: Alex Grintsvayg <grin@lbry.com>
Date: Mon, 27 Jul 2020 19:35:07 -0400
Subject: [PATCH 12/57] use durations

---
 main.go                 |  6 +++---
 manager/manager.go      |  6 +++---
 manager/ytsync.go       |  4 ++--
 sources/youtubeVideo.go | 15 ++++++++-------
 4 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/main.go b/main.go
index e9f9d3c..40be474 100644
--- a/main.go
+++ b/main.go
@@ -34,7 +34,7 @@ var (
 	concurrentJobs int
 	videosLimit    int
 	maxVideoSize   int
-	maxVideoLength float64
+	maxVideoLength int
 )
 
 func main() {
@@ -69,7 +69,7 @@ func main() {
 	cmd.Flags().IntVar(&concurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently")
 	cmd.Flags().IntVar(&videosLimit, "videos-limit", 1000, "how many videos to process per channel")
 	cmd.Flags().IntVar(&maxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)")
-	cmd.Flags().Float64Var(&maxVideoLength, "max-length", 2.0, "Maximum video length to process (in hours)")
+	cmd.Flags().IntVar(&maxVideoLength, "max-length", 2, "Maximum video length to process (in hours)")
 
 	if err := cmd.Execute(); err != nil {
 		fmt.Println(err)
@@ -186,7 +186,7 @@ func ytSync(cmd *cobra.Command, args []string) {
 		syncStatus,
 		syncProperties,
 		apiConfig,
-		maxVideoLength,
+		time.Duration(maxVideoLength)*time.Hour,
 	)
 	err := sm.Start()
 	if err != nil {
diff --git a/manager/manager.go b/manager/manager.go
index 2f831b9..50737af 100644
--- a/manager/manager.go
+++ b/manager/manager.go
@@ -30,7 +30,7 @@ type SyncManager struct {
 	blobsDir         string
 	videosLimit      int
 	maxVideoSize     int
-	maxVideoLength   float64
+	maxVideoLength   time.Duration
 	lbrycrdString    string
 	awsS3ID          string
 	awsS3Secret      string
@@ -43,7 +43,7 @@ type SyncManager struct {
 
 func NewSyncManager(syncFlags sdk.SyncFlags, maxTries int, refill int, limit int, concurrentJobs int, concurrentVideos int, blobsDir string, videosLimit int,
 	maxVideoSize int, lbrycrdString string, awsS3ID string, awsS3Secret string, awsS3Region string, awsS3Bucket string,
-	syncStatus string, syncProperties *sdk.SyncProperties, apiConfig *sdk.APIConfig, maxVideoLength float64) *SyncManager {
+	syncStatus string, syncProperties *sdk.SyncProperties, apiConfig *sdk.APIConfig, maxVideoLength time.Duration) *SyncManager {
 	return &SyncManager{
 		SyncFlags:        syncFlags,
 		maxTries:         maxTries,
@@ -171,7 +171,7 @@ func (s *SyncManager) Start() error {
 					log.Infof("There are %d channels in the \"%s\" queue", len(channels)-i, q)
 					maxVideoLength := s.maxVideoLength
 					if c.TotalSubscribers < 1000 {
-						maxVideoLength = 1.0
+						maxVideoLength = 1 * time.Hour
 					}
 					syncs = append(syncs, Sync{
 						APIConfig:            s.apiConfig,
diff --git a/manager/ytsync.go b/manager/ytsync.go
index feebc37..f741190 100644
--- a/manager/ytsync.go
+++ b/manager/ytsync.go
@@ -73,7 +73,7 @@ type Sync struct {
 	clientPublishAddress string
 	publicKey            string
 	defaultAccountID     string
-	MaxVideoLength       float64
+	MaxVideoLength       time.Duration
 }
 
 func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) {
@@ -224,7 +224,7 @@ func (s *Sync) setStatusSyncing() error {
 
 func (s *Sync) setExceptions() {
 	if s.YoutubeChannelID == "UCwjQfNRW6sGYb__pd7d4nUg" { //@FreeTalkLive
-		s.MaxVideoLength = 9999.0 // skips max length checks
+		s.MaxVideoLength = 9999 * time.Hour // skips max length checks
 		s.Manager.maxVideoSize = 0
 	}
 }
diff --git a/sources/youtubeVideo.go b/sources/youtubeVideo.go
index 9d11e8a..fd85bf1 100644
--- a/sources/youtubeVideo.go
+++ b/sources/youtubeVideo.go
@@ -3,7 +3,6 @@ package sources
 import (
 	"fmt"
 	"io/ioutil"
-	"math"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -40,7 +39,7 @@ type YoutubeVideo struct {
 	playlistPosition int64
 	size             *int64
 	maxVideoSize     int64
-	maxVideoLength   float64
+	maxVideoLength   time.Duration
 	publishedAt      time.Time
 	dir              string
 	youtubeInfo      *ytdl.YtdlVideo
@@ -236,7 +235,7 @@ func (v *YoutubeVideo) download() error {
 	if v.maxVideoLength > 0 {
 		ytdlArgs = append(ytdlArgs,
 			"--match-filter",
-			fmt.Sprintf("duration <= %d", int(math.Round(v.maxVideoLength*3600))),
+			fmt.Sprintf("duration <= %d", int(v.maxVideoLength.Seconds())),
 		)
 	}
 
@@ -432,7 +431,7 @@ type SyncParams struct {
 	ChannelID      string
 	MaxVideoSize   int
 	Namer          *namer.Namer
-	MaxVideoLength float64
+	MaxVideoLength time.Duration
 	Fee            *sdk.Fee
 	DefaultAccount string
 }
@@ -452,9 +451,11 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
 func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
 	var err error
 
-	if float64(v.youtubeInfo.Duration) > v.maxVideoLength {
-		log.Infof("%s is %d long and the limit is %s", v.id, v.youtubeInfo.Duration, (time.Duration(v.maxVideoLength*60) * time.Minute).String())
-		logUtils.SendErrorToSlack("%s is %d long and the limit is %s", v.id, v.youtubeInfo.Duration, (time.Duration(v.maxVideoLength*60) * time.Minute).String())
+	dur := time.Duration(v.youtubeInfo.Duration) * time.Second
+
+	if dur > v.maxVideoLength {
+		log.Infof("%s is %d long and the limit is %s", v.id, dur.String(), v.maxVideoLength.String())
+		logUtils.SendErrorToSlack("%s is %d long and the limit is %s", v.id, dur.String(), v.maxVideoLength.String())
 		return nil, errors.Err("video is too long to process")
 	}
 	for {

From a05864404da99260d872f6bb314806aeddfd9eb8 Mon Sep 17 00:00:00 2001
From: Alex Grintsvayg <grin@lbry.com>
Date: Mon, 27 Jul 2020 20:59:47 -0400
Subject: [PATCH 13/57] use local timezone for youtube-dl, even though its not
 certain

---
 sources/youtubeVideo.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/sources/youtubeVideo.go b/sources/youtubeVideo.go
index fd85bf1..b7cf793 100644
--- a/sources/youtubeVideo.go
+++ b/sources/youtubeVideo.go
@@ -90,7 +90,9 @@ var youtubeCategories = map[string]string{
 }
 
 func NewYoutubeVideo(directory string, videoData *ytdl.YtdlVideo, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) (*YoutubeVideo, error) {
-	publishedAt, err := time.Parse("20060102", videoData.UploadDate)
+	// youtube-dl returns times in local timezone sometimes. this could break in the future
+	// maybe we can file a PR to choose the timezone we want from youtube-dl
+	publishedAt, err := time.ParseInLocation("20060102", videoData.UploadDate, time.Local)
 	if err != nil {
 		return nil, errors.Err(err)
 	}

From f942bf802526c32be8d5716a838c4faa3047d461 Mon Sep 17 00:00:00 2001
From: Alex Grintsvayg <grin@lbry.com>
Date: Tue, 28 Jul 2020 11:05:24 -0400
Subject: [PATCH 14/57] get upload time from
 https://www.invid-project.eu/tools-and-services/invid-verification-plugin/

---
 downloader/downloader.go | 71 ++++++++++++++++++++++++++++++++++++++++
 downloader/ytdl/Video.go |  5 +++
 sources/youtubeVideo.go  |  6 +---
 3 files changed, 77 insertions(+), 5 deletions(-)

diff --git a/downloader/downloader.go b/downloader/downloader.go
index 94f0bea..e1f17de 100644
--- a/downloader/downloader.go
+++ b/downloader/downloader.go
@@ -4,12 +4,16 @@ import (
 	"encoding/json"
 	"io"
 	"io/ioutil"
+	"net/http"
 	"os/exec"
 	"strings"
+	"time"
 
 	"github.com/lbryio/ytsync/v5/downloader/ytdl"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/util"
+
 	"github.com/sirupsen/logrus"
 )
 
@@ -40,9 +44,76 @@ func GetVideoInformation(videoID string) (*ytdl.YtdlVideo, error) {
 	if err != nil {
 		return nil, errors.Err(err)
 	}
+
+	// now get an accurate time
+	tries := 0
+GetTime:
+	tries++
+	t, err := getUploadTime(videoID)
+	if err != nil {
+		if errors.Is(err, errNotScraped) && tries <= 3 {
+			triggerScrape(videoID)
+			time.Sleep(2 * time.Second) // let them scrape it
+			goto GetTime
+		}
+		//return video, errors.Err(err) // just swallow this error and do fallback below
+	}
+
+	if t != "" {
+		parsed, err := time.Parse("2006-01-02, 15:04:05 (MST)", t) // this will probably be UTC, but Go's timezone parsing is fucked up. it ignores the timezone in the date
+		if err != nil {
+			return nil, errors.Err(err)
+		}
+		video.UploadDateForReal = parsed
+	} else {
+		_ = util.SendToSlack(":warning: Could not get accurate time for %s. Falling back to estimated time.", videoID)
+		// fall back to UploadDate from youtube-dl
+		video.UploadDateForReal, err = time.Parse("20060102", video.UploadDate)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	return video, nil
 }
 
+var errNotScraped = errors.Base("not yet scraped by caa.iti.gr")
+
+func triggerScrape(videoID string) error {
+	res, err := http.Get("https://caa.iti.gr/verify_videoV3?twtimeline=0&url=https://www.youtube.com/watch?v=" + videoID)
+	if err != nil {
+		return errors.Err(err)
+	}
+	defer res.Body.Close()
+
+	return nil
+	//https://caa.iti.gr/caa/api/v4/videos/reports/h-tuxHS5lSM
+}
+
+func getUploadTime(videoID string) (string, error) {
+	res, err := http.Get("https://caa.iti.gr/get_verificationV3?url=https://www.youtube.com/watch?v=" + videoID)
+	if err != nil {
+		return "", errors.Err(err)
+	}
+	defer res.Body.Close()
+
+	var uploadTime struct {
+		Time    string `json:"video_upload_time"`
+		Message string `json:"message"`
+		Status  string `json:"status"`
+	}
+	err = json.NewDecoder(res.Body).Decode(&uploadTime)
+	if err != nil {
+		return "", errors.Err(err)
+	}
+
+	if uploadTime.Status == "ERROR1" {
+		return "", errNotScraped
+	}
+
+	return uploadTime.Time, nil
+}
+
 func run(args []string, withStdErr, withStdOut bool) ([]string, error) {
 	cmd := exec.Command("youtube-dl", args...)
 	logrus.Printf("Running command youtube-dl %s", strings.Join(args, " "))
diff --git a/downloader/ytdl/Video.go b/downloader/ytdl/Video.go
index 4dff358..2f845ba 100644
--- a/downloader/ytdl/Video.go
+++ b/downloader/ytdl/Video.go
@@ -1,7 +1,12 @@
 package ytdl
 
+import (
+	"time"
+)
+
 type YtdlVideo struct {
 	UploadDate         string      `json:"upload_date"`
+	UploadDateForReal  time.Time   // you need to manually set this since the value in the API doesn't include the time
 	Extractor          string      `json:"extractor"`
 	Series             interface{} `json:"series"`
 	Format             string      `json:"format"`
diff --git a/sources/youtubeVideo.go b/sources/youtubeVideo.go
index b7cf793..771a99a 100644
--- a/sources/youtubeVideo.go
+++ b/sources/youtubeVideo.go
@@ -92,16 +92,12 @@ var youtubeCategories = map[string]string{
 func NewYoutubeVideo(directory string, videoData *ytdl.YtdlVideo, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) (*YoutubeVideo, error) {
 	// youtube-dl returns times in local timezone sometimes. this could break in the future
 	// maybe we can file a PR to choose the timezone we want from youtube-dl
-	publishedAt, err := time.ParseInLocation("20060102", videoData.UploadDate, time.Local)
-	if err != nil {
-		return nil, errors.Err(err)
-	}
 	return &YoutubeVideo{
 		id:               videoData.ID,
 		title:            videoData.Title,
 		description:      videoData.Description,
 		playlistPosition: playlistPosition,
-		publishedAt:      publishedAt,
+		publishedAt:      videoData.UploadDateForReal,
 		dir:              directory,
 		youtubeInfo:      videoData,
 		awsConfig:        awsConfig,

From a3dd3dc626d7731fa210e7df3a6aa3cc8f506122 Mon Sep 17 00:00:00 2001
From: Alex Grintsvayg <grin@lbry.com>
Date: Tue, 28 Jul 2020 12:47:28 -0400
Subject: [PATCH 15/57] WIP: trying to get the accurate api to work

---
 downloader/downloader.go |  58 ++++++++++++++----
 downloader/ytdl/Video.go | 128 ++++++++++++++++++++-------------------
 main.go                  |   5 ++
 manager/ytsync.go        |   3 +
 4 files changed, 118 insertions(+), 76 deletions(-)

diff --git a/downloader/downloader.go b/downloader/downloader.go
index e1f17de..441497e 100644
--- a/downloader/downloader.go
+++ b/downloader/downloader.go
@@ -2,13 +2,16 @@ package downloader
 
 import (
 	"encoding/json"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"os/exec"
 	"strings"
 	"time"
 
+	"github.com/davecgh/go-spew/spew"
 	"github.com/lbryio/ytsync/v5/downloader/ytdl"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
@@ -34,30 +37,38 @@ func GetPlaylistVideoIDs(channelName string, maxVideos int) ([]string, error) {
 }
 
 func GetVideoInformation(videoID string) (*ytdl.YtdlVideo, error) {
-	args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID}
-	results, err := run(args, false, true)
-	if err != nil {
-		return nil, errors.Err(err)
-	}
+	//args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID}
+	//results, err := run(args, false, true)
+	//if err != nil {
+	//	return nil, errors.Err(err)
+	//}
 	var video *ytdl.YtdlVideo
-	err = json.Unmarshal([]byte(results[0]), &video)
-	if err != nil {
-		return nil, errors.Err(err)
-	}
+	//err = json.Unmarshal([]byte(results[0]), &video)
+	//if err != nil {
+	//	return nil, errors.Err(err)
+	//}
+
+	video = &ytdl.YtdlVideo{}
 
 	// now get an accurate time
+	const maxTries = 5
 	tries := 0
 GetTime:
 	tries++
 	t, err := getUploadTime(videoID)
 	if err != nil {
-		if errors.Is(err, errNotScraped) && tries <= 3 {
+		slack(":warning: Upload time error: %v", err)
+		if tries <= maxTries && (errors.Is(err, errNotScraped) || errors.Is(err, errUploadTimeEmpty)) {
 			triggerScrape(videoID)
 			time.Sleep(2 * time.Second) // let them scrape it
 			goto GetTime
+		} else if !errors.Is(err, errNotScraped) && !errors.Is(err, errUploadTimeEmpty) {
+			slack(":warning: Error while trying to get accurate upload time for %s: %v", videoID, err)
+			return nil, errors.Err(err)
 		}
-		//return video, errors.Err(err) // just swallow this error and do fallback below
+		// do fallback below
 	}
+	slack("After all that, upload time for %s is %s", videoID, t)
 
 	if t != "" {
 		parsed, err := time.Parse("2006-01-02, 15:04:05 (MST)", t) // this will probably be UTC, but Go's timezone parsing is fucked up. it ignores the timezone in the date
@@ -66,7 +77,7 @@ GetTime:
 		}
 		video.UploadDateForReal = parsed
 	} else {
-		_ = util.SendToSlack(":warning: Could not get accurate time for %s. Falling back to estimated time.", videoID)
+		slack(":warning: Could not get accurate time for %s. Falling back to estimated time.", videoID)
 		// fall back to UploadDate from youtube-dl
 		video.UploadDateForReal, err = time.Parse("20060102", video.UploadDate)
 		if err != nil {
@@ -78,19 +89,36 @@ GetTime:
 }
 
 var errNotScraped = errors.Base("not yet scraped by caa.iti.gr")
+var errUploadTimeEmpty = errors.Base("upload time is empty")
+
+func slack(format string, a ...interface{}) {
+	fmt.Printf(format+"\n", a...)
+	util.SendToSlack(format, a...)
+}
 
 func triggerScrape(videoID string) error {
-	res, err := http.Get("https://caa.iti.gr/verify_videoV3?twtimeline=0&url=https://www.youtube.com/watch?v=" + videoID)
+	slack("Triggering scrape for %s", videoID)
+	u, err := url.Parse("https://caa.iti.gr/verify_videoV3")
+	q := u.Query()
+	q.Set("twtimeline", "0")
+	q.Set("url", "https://www.youtube.com/watch?v="+videoID)
+	u.RawQuery = q.Encode()
+	slack("GET %s", u.String())
+	res, err := http.Get(u.String())
 	if err != nil {
 		return errors.Err(err)
 	}
 	defer res.Body.Close()
 
+	all, err := ioutil.ReadAll(res.Body)
+	spew.Dump(string(all), err)
+
 	return nil
 	//https://caa.iti.gr/caa/api/v4/videos/reports/h-tuxHS5lSM
 }
 
 func getUploadTime(videoID string) (string, error) {
+	slack("Getting upload time for %s", videoID)
 	res, err := http.Get("https://caa.iti.gr/get_verificationV3?url=https://www.youtube.com/watch?v=" + videoID)
 	if err != nil {
 		return "", errors.Err(err)
@@ -111,6 +139,10 @@ func getUploadTime(videoID string) (string, error) {
 		return "", errNotScraped
 	}
 
+	if uploadTime.Time == "" {
+		return "", errUploadTimeEmpty
+	}
+
 	return uploadTime.Time, nil
 }
 
diff --git a/downloader/ytdl/Video.go b/downloader/ytdl/Video.go
index 2f845ba..df032e1 100644
--- a/downloader/ytdl/Video.go
+++ b/downloader/ytdl/Video.go
@@ -37,40 +37,8 @@ type YtdlVideo struct {
 	WebpageURLBasename string      `json:"webpage_url_basename"`
 	Acodec             string      `json:"acodec"`
 	DisplayID          string      `json:"display_id"`
-	RequestedFormats   []struct {
-		Asr             interface{} `json:"asr"`
-		Tbr             float64     `json:"tbr"`
-		Container       string      `json:"container"`
-		Language        interface{} `json:"language"`
-		Format          string      `json:"format"`
-		URL             string      `json:"url"`
-		Vcodec          string      `json:"vcodec"`
-		FormatNote      string      `json:"format_note"`
-		Height          int         `json:"height"`
-		Width           int         `json:"width"`
-		Ext             string      `json:"ext"`
-		FragmentBaseURL string      `json:"fragment_base_url"`
-		Filesize        interface{} `json:"filesize"`
-		Fps             int         `json:"fps"`
-		ManifestURL     string      `json:"manifest_url"`
-		Protocol        string      `json:"protocol"`
-		FormatID        string      `json:"format_id"`
-		HTTPHeaders     struct {
-			AcceptCharset  string `json:"Accept-Charset"`
-			AcceptLanguage string `json:"Accept-Language"`
-			AcceptEncoding string `json:"Accept-Encoding"`
-			Accept         string `json:"Accept"`
-			UserAgent      string `json:"User-Agent"`
-		} `json:"http_headers"`
-		Fragments []struct {
-			Path     string  `json:"path"`
-			Duration float64 `json:"duration,omitempty"`
-		} `json:"fragments"`
-		Acodec string `json:"acodec"`
-		Abr    int    `json:"abr,omitempty"`
-	} `json:"requested_formats"`
-	AutomaticCaptions struct {
-	} `json:"automatic_captions"`
+	//RequestedFormats   []RequestedFormat `json:"requested_formats"`
+	//AutomaticCaptions  struct{}          `json:"automatic_captions"`
 	Description        string      `json:"description"`
 	Tags               []string    `json:"tags"`
 	Track              interface{} `json:"track"`
@@ -81,8 +49,7 @@ type YtdlVideo struct {
 	FormatID           string      `json:"format_id"`
 	EpisodeNumber      interface{} `json:"episode_number"`
 	UploaderID         string      `json:"uploader_id"`
-	Subtitles          struct {
-	} `json:"subtitles"`
+	//Subtitles          struct{}          `json:"subtitles"`
 	ReleaseYear interface{} `json:"release_year"`
 	Thumbnails  []Thumbnail `json:"thumbnails"`
 	License     interface{} `json:"license"`
@@ -96,38 +63,73 @@ type YtdlVideo struct {
 	Width       int         `json:"width"`
 	EndTime     interface{} `json:"end_time"`
 	WebpageURL  string      `json:"webpage_url"`
-	Formats     []struct {
-		Asr             int         `json:"asr"`
-		Tbr             float64     `json:"tbr"`
-		Protocol        string      `json:"protocol"`
-		Format          string      `json:"format"`
-		FormatNote      string      `json:"format_note"`
-		Height          interface{} `json:"height"`
-		ManifestURL     string      `json:"manifest_url,omitempty"`
-		FormatID        string      `json:"format_id"`
-		Container       string      `json:"container,omitempty"`
-		Language        interface{} `json:"language,omitempty"`
-		HTTPHeaders     HTTPHeaders `json:"http_headers"`
-		URL             string      `json:"url"`
-		Vcodec          string      `json:"vcodec"`
-		Abr             int         `json:"abr,omitempty"`
-		Width           interface{} `json:"width"`
-		Ext             string      `json:"ext"`
-		FragmentBaseURL string      `json:"fragment_base_url,omitempty"`
-		Filesize        interface{} `json:"filesize"`
-		Fps             interface{} `json:"fps"`
-		Fragments       []struct {
-			Path     string  `json:"path"`
-			Duration float64 `json:"duration,omitempty"`
-		} `json:"fragments,omitempty"`
-		Acodec    string      `json:"acodec"`
-		PlayerURL interface{} `json:"player_url,omitempty"`
-	} `json:"formats"`
+	//Formats            []Format          `json:"formats"`
 	ChannelURL string      `json:"channel_url"`
 	Resolution interface{} `json:"resolution"`
 	Vcodec     string      `json:"vcodec"`
 }
 
+type RequestedFormat struct {
+	Asr             interface{} `json:"asr"`
+	Tbr             float64     `json:"tbr"`
+	Container       string      `json:"container"`
+	Language        interface{} `json:"language"`
+	Format          string      `json:"format"`
+	URL             string      `json:"url"`
+	Vcodec          string      `json:"vcodec"`
+	FormatNote      string      `json:"format_note"`
+	Height          int         `json:"height"`
+	Width           int         `json:"width"`
+	Ext             string      `json:"ext"`
+	FragmentBaseURL string      `json:"fragment_base_url"`
+	Filesize        interface{} `json:"filesize"`
+	Fps             int         `json:"fps"`
+	ManifestURL     string      `json:"manifest_url"`
+	Protocol        string      `json:"protocol"`
+	FormatID        string      `json:"format_id"`
+	HTTPHeaders     struct {
+		AcceptCharset  string `json:"Accept-Charset"`
+		AcceptLanguage string `json:"Accept-Language"`
+		AcceptEncoding string `json:"Accept-Encoding"`
+		Accept         string `json:"Accept"`
+		UserAgent      string `json:"User-Agent"`
+	} `json:"http_headers"`
+	Fragments []struct {
+		Path     string  `json:"path"`
+		Duration float64 `json:"duration,omitempty"`
+	} `json:"fragments"`
+	Acodec string `json:"acodec"`
+	Abr    int    `json:"abr,omitempty"`
+}
+
+type Format struct {
+	Asr             int         `json:"asr"`
+	Tbr             float64     `json:"tbr"`
+	Protocol        string      `json:"protocol"`
+	Format          string      `json:"format"`
+	FormatNote      string      `json:"format_note"`
+	Height          interface{} `json:"height"`
+	ManifestURL     string      `json:"manifest_url,omitempty"`
+	FormatID        string      `json:"format_id"`
+	Container       string      `json:"container,omitempty"`
+	Language        interface{} `json:"language,omitempty"`
+	HTTPHeaders     HTTPHeaders `json:"http_headers"`
+	URL             string      `json:"url"`
+	Vcodec          string      `json:"vcodec"`
+	Abr             int         `json:"abr,omitempty"`
+	Width           interface{} `json:"width"`
+	Ext             string      `json:"ext"`
+	FragmentBaseURL string      `json:"fragment_base_url,omitempty"`
+	Filesize        interface{} `json:"filesize"`
+	Fps             interface{} `json:"fps"`
+	Fragments       []struct {
+		Path     string  `json:"path"`
+		Duration float64 `json:"duration,omitempty"`
+	} `json:"fragments,omitempty"`
+	Acodec    string      `json:"acodec"`
+	PlayerURL interface{} `json:"player_url,omitempty"`
+}
+
 type Thumbnail struct {
 	URL        string `json:"url"`
 	Width      int    `json:"width"`
diff --git a/main.go b/main.go
index 40be474..6ff9112 100644
--- a/main.go
+++ b/main.go
@@ -7,8 +7,10 @@ import (
 	"os"
 	"time"
 
+	"github.com/davecgh/go-spew/spew"
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/extras/util"
+	"github.com/lbryio/ytsync/v5/downloader"
 	"github.com/lbryio/ytsync/v5/manager"
 	"github.com/lbryio/ytsync/v5/sdk"
 	ytUtils "github.com/lbryio/ytsync/v5/util"
@@ -38,6 +40,9 @@ var (
 )
 
 func main() {
+	spew.Dump(downloader.GetVideoInformation("oahaMa3XB0k"))
+	return
+
 	rand.Seed(time.Now().UnixNano())
 	log.SetLevel(log.DebugLevel)
 	http.Handle("/metrics", promhttp.Handler())
diff --git a/manager/ytsync.go b/manager/ytsync.go
index f741190..97eaaa0 100644
--- a/manager/ytsync.go
+++ b/manager/ytsync.go
@@ -250,6 +250,7 @@ func (s *Sync) FullCycle() (e error) {
 	defer signal.Stop(interruptChan)
 	go func() {
 		<-interruptChan
+		util.SendToSlack("got interrupt, shutting down")
 		log.Println("Got interrupt signal, shutting down (if publishing, will shut down after current publish)")
 		s.grp.Stop()
 	}()
@@ -856,6 +857,8 @@ func (s *Sync) startWorker(workerNum int) {
 			tryCount++
 			err := s.processVideo(v)
 
+			util.SendToSlack("Tried to process %s. Error: %v", v.ID(), err)
+
 			if err != nil {
 				logMsg := fmt.Sprintf("error processing video %s: %s", v.ID(), err.Error())
 				log.Errorln(logMsg)

From 0eef62b5fd073238e3c12a931eb1fef4fa3b367a Mon Sep 17 00:00:00 2001
From: Alex Grintsvayg <grin@lbry.com>
Date: Tue, 28 Jul 2020 21:34:08 -0400
Subject: [PATCH 16/57] i made a huge mess. sorry future me, when you're
 digging through this

---
 downloader/downloader.go      | 145 ++++++++++++++++++++++++++--------
 downloader/downloader_test.go |   2 +-
 ip_manager/throttle.go        |  36 ++++-----
 main.go                       |  21 ++++-
 manager/manager.go            |   2 +-
 manager/ytsync.go             |  12 ++-
 ytapi/ytapi.go                |  30 +++++--
 7 files changed, 179 insertions(+), 69 deletions(-)

diff --git a/downloader/downloader.go b/downloader/downloader.go
index 441497e..e233253 100644
--- a/downloader/downloader.go
+++ b/downloader/downloader.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"net/url"
 	"os/exec"
@@ -15,14 +16,15 @@ import (
 	"github.com/lbryio/ytsync/v5/downloader/ytdl"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/stop"
 	"github.com/lbryio/lbry.go/v2/extras/util"
 
 	"github.com/sirupsen/logrus"
 )
 
-func GetPlaylistVideoIDs(channelName string, maxVideos int) ([]string, error) {
+func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan) ([]string, error) {
 	args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist"}
-	ids, err := run(args, false, true)
+	ids, err := run(args, false, true, stopChan)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
@@ -36,48 +38,51 @@ func GetPlaylistVideoIDs(channelName string, maxVideos int) ([]string, error) {
 	return videoIDs, nil
 }
 
-func GetVideoInformation(videoID string) (*ytdl.YtdlVideo, error) {
-	//args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID}
-	//results, err := run(args, false, true)
-	//if err != nil {
-	//	return nil, errors.Err(err)
-	//}
+func GetVideoInformation(videoID string, stopChan stop.Chan, ip *net.TCPAddr) (*ytdl.YtdlVideo, error) {
+	args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID}
+	results, err := run(args, false, true, stopChan)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
 	var video *ytdl.YtdlVideo
-	//err = json.Unmarshal([]byte(results[0]), &video)
-	//if err != nil {
-	//	return nil, errors.Err(err)
-	//}
-
-	video = &ytdl.YtdlVideo{}
+	err = json.Unmarshal([]byte(results[0]), &video)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
 
 	// now get an accurate time
 	const maxTries = 5
 	tries := 0
 GetTime:
 	tries++
-	t, err := getUploadTime(videoID)
+	t, err := getUploadTime(videoID, ip)
 	if err != nil {
-		slack(":warning: Upload time error: %v", err)
+		//slack(":warning: Upload time error: %v", err)
 		if tries <= maxTries && (errors.Is(err, errNotScraped) || errors.Is(err, errUploadTimeEmpty)) {
-			triggerScrape(videoID)
-			time.Sleep(2 * time.Second) // let them scrape it
-			goto GetTime
+			err := triggerScrape(videoID, ip)
+			if err == nil {
+				time.Sleep(2 * time.Second) // let them scrape it
+				goto GetTime
+			} else {
+				//slack("triggering scrape returned error: %v", err)
+			}
 		} else if !errors.Is(err, errNotScraped) && !errors.Is(err, errUploadTimeEmpty) {
-			slack(":warning: Error while trying to get accurate upload time for %s: %v", videoID, err)
+			//slack(":warning: Error while trying to get accurate upload time for %s: %v", videoID, err)
 			return nil, errors.Err(err)
 		}
 		// do fallback below
 	}
-	slack("After all that, upload time for %s is %s", videoID, t)
+	//slack("After all that, upload time for %s is %s", videoID, t)
 
 	if t != "" {
 		parsed, err := time.Parse("2006-01-02, 15:04:05 (MST)", t) // this will probably be UTC, but Go's timezone parsing is fucked up. it ignores the timezone in the date
 		if err != nil {
 			return nil, errors.Err(err)
 		}
+		slack(":exclamation: Got an accurate time for %s", videoID)
 		video.UploadDateForReal = parsed
 	} else {
-		slack(":warning: Could not get accurate time for %s. Falling back to estimated time.", videoID)
+		//slack(":warning: Could not get accurate time for %s. Falling back to time from upload ytdl: %s.", videoID, video.UploadDate)
 		// fall back to UploadDate from youtube-dl
 		video.UploadDateForReal, err = time.Parse("20060102", video.UploadDate)
 		if err != nil {
@@ -96,30 +101,62 @@ func slack(format string, a ...interface{}) {
 	util.SendToSlack(format, a...)
 }
 
-func triggerScrape(videoID string) error {
-	slack("Triggering scrape for %s", videoID)
+func triggerScrape(videoID string, ip *net.TCPAddr) error {
+	//slack("Triggering scrape for %s", videoID)
 	u, err := url.Parse("https://caa.iti.gr/verify_videoV3")
 	q := u.Query()
 	q.Set("twtimeline", "0")
 	q.Set("url", "https://www.youtube.com/watch?v="+videoID)
 	u.RawQuery = q.Encode()
-	slack("GET %s", u.String())
-	res, err := http.Get(u.String())
+	//slack("GET %s", u.String())
+
+	client := getClient(ip)
+	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
+	if err != nil {
+		return errors.Err(err)
+	}
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36")
+
+	res, err := client.Do(req)
 	if err != nil {
 		return errors.Err(err)
 	}
 	defer res.Body.Close()
 
-	all, err := ioutil.ReadAll(res.Body)
-	spew.Dump(string(all), err)
+	var response struct {
+		Message  string `json:"message"`
+		Status   string `json:"status"`
+		VideoURL string `json:"video_url"`
+	}
+	err = json.NewDecoder(res.Body).Decode(&response)
+	if err != nil {
+		return errors.Err(err)
+	}
+
+	switch response.Status {
+	case "removed_video":
+		return errors.Err("video previously removed from service")
+	case "no_video":
+		return errors.Err("they say 'video cannot be found'. wtf?")
+	default:
+		spew.Dump(response)
+	}
 
 	return nil
 	//https://caa.iti.gr/caa/api/v4/videos/reports/h-tuxHS5lSM
 }
 
-func getUploadTime(videoID string) (string, error) {
-	slack("Getting upload time for %s", videoID)
-	res, err := http.Get("https://caa.iti.gr/get_verificationV3?url=https://www.youtube.com/watch?v=" + videoID)
+func getUploadTime(videoID string, ip *net.TCPAddr) (string, error) {
+	//slack("Getting upload time for %s", videoID)
+
+	client := getClient(ip)
+	req, err := http.NewRequest(http.MethodGet, "https://caa.iti.gr/get_verificationV3?url=https://www.youtube.com/watch?v="+videoID, nil)
+	if err != nil {
+		return "", errors.Err(err)
+	}
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36")
+
+	res, err := client.Do(req)
 	if err != nil {
 		return "", errors.Err(err)
 	}
@@ -139,6 +176,10 @@ func getUploadTime(videoID string) (string, error) {
 		return "", errNotScraped
 	}
 
+	if uploadTime.Status == "" && strings.HasPrefix(uploadTime.Message, "CANNOT_RETRIEVE_REPORT_FOR_VIDEO_") {
+		return "", errors.Err("cannot retrieve report for video")
+	}
+
 	if uploadTime.Time == "" {
 		return "", errUploadTimeEmpty
 	}
@@ -146,7 +187,28 @@ func getUploadTime(videoID string) (string, error) {
 	return uploadTime.Time, nil
 }
 
-func run(args []string, withStdErr, withStdOut bool) ([]string, error) {
+func getClient(ip *net.TCPAddr) *http.Client {
+	if ip == nil {
+		return http.DefaultClient
+	}
+
+	return &http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+			DialContext: (&net.Dialer{
+				LocalAddr: ip,
+				Timeout:   30 * time.Second,
+				KeepAlive: 30 * time.Second,
+			}).DialContext,
+			MaxIdleConns:          100,
+			IdleConnTimeout:       90 * time.Second,
+			TLSHandshakeTimeout:   10 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+		},
+	}
+}
+
+func run(args []string, withStdErr, withStdOut bool, stopChan stop.Chan) ([]string, error) {
 	cmd := exec.Command("youtube-dl", args...)
 	logrus.Printf("Running command youtube-dl %s", strings.Join(args, " "))
 
@@ -181,10 +243,23 @@ func run(args []string, withStdErr, withStdOut bool) ([]string, error) {
 			return nil, errors.Err(err)
 		}
 	}
-	err := cmd.Wait()
-	if len(errorLog) > 0 {
-		return nil, errors.Err(err)
+
+	done := make(chan error, 1)
+	go func() {
+		done <- cmd.Wait()
+	}()
+	select {
+	case <-stopChan:
+		if err := cmd.Process.Kill(); err != nil {
+			return nil, errors.Prefix("failed to kill command after stopper cancellation", err)
+		}
+		return nil, errors.Err("canceled by stopper")
+	case err := <-done:
+		if err != nil {
+			return nil, errors.Prefix("youtube-dl "+strings.Join(args, " "), err)
+		}
 	}
+
 	if len(errorLog) > 0 {
 		return nil, errors.Err(string(errorLog))
 	}
diff --git a/downloader/downloader_test.go b/downloader/downloader_test.go
index 1a80d6d..1f065e6 100644
--- a/downloader/downloader_test.go
+++ b/downloader/downloader_test.go
@@ -17,7 +17,7 @@ func TestGetPlaylistVideoIDs(t *testing.T) {
 }
 
 func TestGetVideoInformation(t *testing.T) {
-	video, err := GetVideoInformation("zj7pXM9gE5M")
+	video, err := GetVideoInformation("zj7pXM9gE5M", nil)
 	if err != nil {
 		logrus.Error(err)
 	}
diff --git a/ip_manager/throttle.go b/ip_manager/throttle.go
index 43be077..e37ba69 100644
--- a/ip_manager/throttle.go
+++ b/ip_manager/throttle.go
@@ -63,21 +63,21 @@ func GetIPPool(stopGrp *stop.Group) (*IPPool, error) {
 		lock:    &sync.RWMutex{},
 		stopGrp: stopGrp,
 	}
-	ticker := time.NewTicker(10 * time.Second)
-	go func() {
-		for {
-			select {
-			case <-stopGrp.Ch():
-				return
-			case <-ticker.C:
-				ipPoolInstance.lock.RLock()
-				for _, ip := range ipPoolInstance.ips {
-					log.Debugf("IP: %s\tInUse: %t\tVideoID: %s\tThrottled: %t\tLastUse: %.1f", ip.IP, ip.InUse, ip.UsedForVideo, ip.Throttled, time.Since(ip.LastUse).Seconds())
-				}
-				ipPoolInstance.lock.RUnlock()
-			}
-		}
-	}()
+	//ticker := time.NewTicker(10 * time.Second)
+	//go func() {
+	//	for {
+	//		select {
+	//		case <-stopGrp.Ch():
+	//			return
+	//		case <-ticker.C:
+	//			ipPoolInstance.lock.RLock()
+	//			for _, ip := range ipPoolInstance.ips {
+	//				log.Debugf("IP: %s\tInUse: %t\tVideoID: %s\tThrottled: %t\tLastUse: %.1f", ip.IP, ip.InUse, ip.UsedForVideo, ip.Throttled, time.Since(ip.LastUse).Seconds())
+	//			}
+	//			ipPoolInstance.lock.RUnlock()
+	//		}
+	//	}
+	//}()
 	return ipPoolInstance, nil
 }
 
@@ -108,7 +108,7 @@ func AllInUse(ips []throttledIP) bool {
 func (i *IPPool) ReleaseIP(ip string) {
 	i.lock.Lock()
 	defer i.lock.Unlock()
-	for j, _ := range i.ips {
+	for j := range i.ips {
 		localIP := &i.ips[j]
 		if localIP.IP == ip {
 			localIP.InUse = false
@@ -122,7 +122,7 @@ func (i *IPPool) ReleaseIP(ip string) {
 func (i *IPPool) ReleaseAll() {
 	i.lock.Lock()
 	defer i.lock.Unlock()
-	for j, _ := range i.ips {
+	for j := range i.ips {
 		if i.ips[j].Throttled {
 			continue
 		}
@@ -183,7 +183,7 @@ func (i *IPPool) nextIP(forVideo string) (*throttledIP, error) {
 		}
 
 		var nextIP *throttledIP
-		for j, _ := range i.ips {
+		for j := range i.ips {
 			ip := &i.ips[j]
 			if ip.InUse || ip.Throttled {
 				continue
diff --git a/main.go b/main.go
index 6ff9112..b36233c 100644
--- a/main.go
+++ b/main.go
@@ -7,10 +7,8 @@ import (
 	"os"
 	"time"
 
-	"github.com/davecgh/go-spew/spew"
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/extras/util"
-	"github.com/lbryio/ytsync/v5/downloader"
 	"github.com/lbryio/ytsync/v5/manager"
 	"github.com/lbryio/ytsync/v5/sdk"
 	ytUtils "github.com/lbryio/ytsync/v5/util"
@@ -40,8 +38,23 @@ var (
 )
 
 func main() {
-	spew.Dump(downloader.GetVideoInformation("oahaMa3XB0k"))
-	return
+	//grp := stop.New()
+	//ipPool, err := ip_manager.GetIPPool(grp)
+	//if err != nil {
+	//	panic(err)
+	//}
+	//
+	//videoID := "vtIzMaLkCaM"
+	//
+	//ip, err := ipPool.GetIP(videoID)
+	//if err != nil {
+	//	panic(err)
+	//}
+	//
+	//spew.Dump(ip)
+	//
+	//spew.Dump(downloader.GetVideoInformation(videoID, &net.TCPAddr{IP: net.ParseIP(ip)}))
+	//return
 
 	rand.Seed(time.Now().UnixNano())
 	log.SetLevel(log.DebugLevel)
diff --git a/manager/manager.go b/manager/manager.go
index 50737af..2c84a9a 100644
--- a/manager/manager.go
+++ b/manager/manager.go
@@ -234,7 +234,7 @@ func (s *SyncManager) Start() error {
 				}
 				shouldNotCount = strings.Contains(err.Error(), "this youtube channel is being managed by another server")
 				if !shouldNotCount {
-					logUtils.SendInfoToSlack("A non fatal error was reported by the sync process. %s\nContinuing...", err.Error())
+					logUtils.SendInfoToSlack("A non fatal error was reported by the sync process.\n%s", errors.FullTrace(err))
 				}
 			}
 			err = blobs_reflector.ReflectAndClean()
diff --git a/manager/ytsync.go b/manager/ytsync.go
index 97eaaa0..c2f4450 100644
--- a/manager/ytsync.go
+++ b/manager/ytsync.go
@@ -253,6 +253,8 @@ func (s *Sync) FullCycle() (e error) {
 		util.SendToSlack("got interrupt, shutting down")
 		log.Println("Got interrupt signal, shutting down (if publishing, will shut down after current publish)")
 		s.grp.Stop()
+		time.Sleep(5 * time.Second)
+		debug.PrintStack() // so we can figure out what's not stopping
 	}()
 	err := s.setStatusSyncing()
 	if err != nil {
@@ -854,6 +856,12 @@ func (s *Sync) startWorker(workerNum int) {
 
 		tryCount := 0
 		for {
+			select { // check again inside the loop so this dies faster
+			case <-s.grp.Ch():
+				log.Printf("Stopping worker %d", workerNum)
+				return
+			default:
+			}
 			tryCount++
 			err := s.processVideo(v)
 
@@ -988,10 +996,10 @@ func (s *Sync) enqueueYoutubeVideos() error {
 		return err
 	}
 
-	videos, err := ytapi.GetVideosToSync(s.APIConfig.YoutubeAPIKey, s.YoutubeChannelID, s.syncedVideos, s.Manager.SyncFlags.QuickSync, s.Manager.videosLimit, ytapi.VideoParams{
+	videos, err := ytapi.GetVideosToSync(s.YoutubeChannelID, s.syncedVideos, s.Manager.SyncFlags.QuickSync, s.Manager.videosLimit, ytapi.VideoParams{
 		VideoDir: s.videoDirectory,
 		S3Config: s.Manager.GetS3AWSConfig(),
-		Grp:      s.grp,
+		Stopper:  s.grp,
 		IPPool:   ipPool,
 	})
 	if err != nil {
diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go
index db354ec..c0e1b99 100644
--- a/ytapi/ytapi.go
+++ b/ytapi/ytapi.go
@@ -45,19 +45,19 @@ func (a byPublishedAt) Less(i, j int) bool { return a[i].PublishedAt().Before(a[
 type VideoParams struct {
 	VideoDir string
 	S3Config aws.Config
-	Grp      *stop.Group
+	Stopper  *stop.Group
 	IPPool   *ip_manager.IPPool
 }
 
 var mostRecentlyFailedChannel string // TODO: fix this hack!
 
-func GetVideosToSync(apiKey, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams) ([]Video, error) {
+func GetVideosToSync(channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams) ([]Video, error) {
 
 	var videos []Video
 	if quickSync {
 		maxVideos = 50
 	}
-	videoIDs, err := downloader.GetPlaylistVideoIDs(channelID, maxVideos)
+	videoIDs, err := downloader.GetPlaylistVideoIDs(channelID, maxVideos, videoParams.Stopper.Ch())
 	if err != nil {
 		return nil, errors.Err(err)
 	}
@@ -76,14 +76,14 @@ func GetVideosToSync(apiKey, channelID string, syncedVideos map[string]sdk.Synce
 		mostRecentlyFailedChannel = channelID
 	}
 
-	vids, err := getVideos(videoIDs)
+	vids, err := getVideos(videoIDs, videoParams.Stopper.Ch(), videoParams.IPPool)
 	if err != nil {
 		return nil, err
 	}
 
 	for _, item := range vids {
 		positionInList := playlistMap[item.ID]
-		videoToAdd, err := sources.NewYoutubeVideo(videoParams.VideoDir, item, positionInList, videoParams.S3Config, videoParams.Grp, videoParams.IPPool)
+		videoToAdd, err := sources.NewYoutubeVideo(videoParams.VideoDir, item, positionInList, videoParams.S3Config, videoParams.Stopper, videoParams.IPPool)
 		if err != nil {
 			return nil, errors.Err(err)
 		}
@@ -95,7 +95,7 @@ func GetVideosToSync(apiKey, channelID string, syncedVideos map[string]sdk.Synce
 			continue
 		}
 		if _, ok := playlistMap[k]; !ok {
-			videos = append(videos, sources.NewMockedVideo(videoParams.VideoDir, k, channelID, videoParams.S3Config, videoParams.Grp, videoParams.IPPool))
+			videos = append(videos, sources.NewMockedVideo(videoParams.VideoDir, k, channelID, videoParams.S3Config, videoParams.Stopper, videoParams.IPPool))
 		}
 	}
 
@@ -143,6 +143,7 @@ func CountVideosInChannel(channelID string) (int, error) {
 
 func ChannelInfo(apiKey, channelID string) (*ytlib.ChannelSnippet, *ytlib.ChannelBrandingSettings, error) {
 	return nil, nil, errors.Err("ChannelInfo doesn't work yet because we're focused on existing channels")
+
 	service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
 	if err != nil {
 		return nil, nil, errors.Prefix("error creating YouTube service", err)
@@ -160,14 +161,27 @@ func ChannelInfo(apiKey, channelID string) (*ytlib.ChannelSnippet, *ytlib.Channe
 	return response.Items[0].Snippet, response.Items[0].BrandingSettings, nil
 }
 
-func getVideos(videoIDs []string) ([]*ytdl.YtdlVideo, error) {
+func getVideos(videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool) ([]*ytdl.YtdlVideo, error) {
 	var videos []*ytdl.YtdlVideo
 	for _, videoID := range videoIDs {
-		video, err := downloader.GetVideoInformation(videoID)
+		select {
+		case <-stopChan:
+			return videos, errors.Err("canceled by stopper")
+		default:
+		}
+
+		//ip, err := ipPool.GetIP(videoID)
+		//if err != nil {
+		//	return nil, err
+		//}
+		//video, err := downloader.GetVideoInformation(videoID, &net.TCPAddr{IP: net.ParseIP(ip)})
+		video, err := downloader.GetVideoInformation(videoID, stopChan, nil)
 		if err != nil {
+			//ipPool.ReleaseIP(ip)
 			return nil, errors.Err(err)
 		}
 		videos = append(videos, video)
+		//ipPool.ReleaseIP(ip)
 	}
 	return videos, nil
 }

From 357aebbccee9a4d0da7158319f9800e9df3199c8 Mon Sep 17 00:00:00 2001
From: Alex Grintsvayg <grin@lbry.com>
Date: Tue, 28 Jul 2020 22:40:56 -0400
Subject: [PATCH 17/57] less loud

---
 manager/ytsync.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/manager/ytsync.go b/manager/ytsync.go
index c2f4450..2d69b93 100644
--- a/manager/ytsync.go
+++ b/manager/ytsync.go
@@ -865,9 +865,8 @@ func (s *Sync) startWorker(workerNum int) {
 			tryCount++
 			err := s.processVideo(v)
 
-			util.SendToSlack("Tried to process %s. Error: %v", v.ID(), err)
-
 			if err != nil {
+				util.SendToSlack("Tried to process %s. Error: %v", v.ID(), err)
 				logMsg := fmt.Sprintf("error processing video %s: %s", v.ID(), err.Error())
 				log.Errorln(logMsg)
 				if strings.Contains(strings.ToLower(err.Error()), "interrupted by user") {

From d99e2001782c19009dd1de6fe143cb9b276ed325 Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Wed, 29 Jul 2020 00:12:23 -0400
Subject: [PATCH 18/57] Add check for already published videos using new
 video-state api.

---
 manager/ytsync.go |  2 +-
 sdk/api.go        | 40 ++++++++++++++++++++++++++++++++++++++++
 ytapi/ytapi.go    | 13 ++++++++++---
 3 files changed, 51 insertions(+), 4 deletions(-)

diff --git a/manager/ytsync.go b/manager/ytsync.go
index 2d69b93..ad387e2 100644
--- a/manager/ytsync.go
+++ b/manager/ytsync.go
@@ -995,7 +995,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
 		return err
 	}
 
-	videos, err := ytapi.GetVideosToSync(s.YoutubeChannelID, s.syncedVideos, s.Manager.SyncFlags.QuickSync, s.Manager.videosLimit, ytapi.VideoParams{
+	videos, err := ytapi.GetVideosToSync(s.APIConfig, s.YoutubeChannelID, s.syncedVideos, s.Manager.SyncFlags.QuickSync, s.Manager.videosLimit, ytapi.VideoParams{
 		VideoDir: s.videoDirectory,
 		S3Config: s.Manager.GetS3AWSConfig(),
 		Stopper:  s.grp,
diff --git a/sdk/api.go b/sdk/api.go
index 903229b..bf66b14 100644
--- a/sdk/api.go
+++ b/sdk/api.go
@@ -368,3 +368,43 @@ func (a *APIConfig) MarkVideoStatus(status VideoStatus) error {
 	}
 	return errors.Err("invalid API response. Status code: %d", res.StatusCode)
 }
+
+func (a *APIConfig) VideoState(videoID string) (string, error) {
+	endpoint := a.ApiURL + "/yt/video_state"
+	vals := url.Values{
+		"video_id":   {videoID},
+		"auth_token": {a.ApiToken},
+	}
+
+	res, err := http.PostForm(endpoint, vals)
+	if err != nil {
+		return "", errors.Err(err)
+	}
+	defer res.Body.Close()
+	body, _ := ioutil.ReadAll(res.Body)
+	if res.StatusCode == http.StatusNotFound {
+		return "not_found", nil
+	}
+	if res.StatusCode != http.StatusOK {
+		util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
+		log.Debugln(string(body))
+		time.Sleep(30 * time.Second)
+		return a.VideoState(videoID)
+	}
+	var response struct {
+		Success bool        `json:"success"`
+		Error   null.String `json:"error"`
+		Data    null.String `json:"data"`
+	}
+	err = json.Unmarshal(body, &response)
+	if err != nil {
+		return "", errors.Err(err)
+	}
+	if !response.Error.IsNull() {
+		return "", errors.Err(response.Error.String)
+	}
+	if !response.Data.IsNull() {
+		return response.Data.String, nil
+	}
+	return "", errors.Err("invalid API response. Status code: %d", res.StatusCode)
+}
diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go
index c0e1b99..1d51532 100644
--- a/ytapi/ytapi.go
+++ b/ytapi/ytapi.go
@@ -51,7 +51,7 @@ type VideoParams struct {
 
 var mostRecentlyFailedChannel string // TODO: fix this hack!
 
-func GetVideosToSync(channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams) ([]Video, error) {
+func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams) ([]Video, error) {
 
 	var videos []Video
 	if quickSync {
@@ -76,7 +76,7 @@ func GetVideosToSync(channelID string, syncedVideos map[string]sdk.SyncedVideo,
 		mostRecentlyFailedChannel = channelID
 	}
 
-	vids, err := getVideos(videoIDs, videoParams.Stopper.Ch(), videoParams.IPPool)
+	vids, err := getVideos(config, videoIDs, videoParams.Stopper.Ch(), videoParams.IPPool)
 	if err != nil {
 		return nil, err
 	}
@@ -161,7 +161,7 @@ func ChannelInfo(apiKey, channelID string) (*ytlib.ChannelSnippet, *ytlib.Channe
 	return response.Items[0].Snippet, response.Items[0].BrandingSettings, nil
 }
 
-func getVideos(videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool) ([]*ytdl.YtdlVideo, error) {
+func getVideos(config *sdk.APIConfig, videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool) ([]*ytdl.YtdlVideo, error) {
 	var videos []*ytdl.YtdlVideo
 	for _, videoID := range videoIDs {
 		select {
@@ -175,6 +175,13 @@ func getVideos(videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool)
 		//	return nil, err
 		//}
 		//video, err := downloader.GetVideoInformation(videoID, &net.TCPAddr{IP: net.ParseIP(ip)})
+		state, err := config.VideoState(videoID)
+		if err != nil {
+			return nil, errors.Err(err)
+		}
+		if state == "published" {
+			continue
+		}
 		video, err := downloader.GetVideoInformation(videoID, stopChan, nil)
 		if err != nil {
 			//ipPool.ReleaseIP(ip)

From 64040ea67a50a34f8be53996e3b7a2e272a13d51 Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Thu, 30 Jul 2020 11:13:19 -0400
Subject: [PATCH 19/57] Add calls to internal-apis and add get released date if
 available.

---
 downloader/downloader.go | 21 +++++++++++++----
 go.mod                   |  3 +--
 go.sum                   |  4 ----
 sdk/api.go               | 49 ++++++++++++++++++++++++++++++++++++++++
 ytapi/ytapi.go           |  2 +-
 5 files changed, 68 insertions(+), 11 deletions(-)

diff --git a/downloader/downloader.go b/downloader/downloader.go
index e233253..0f0293f 100644
--- a/downloader/downloader.go
+++ b/downloader/downloader.go
@@ -12,6 +12,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/lbryio/ytsync/v5/sdk"
+
 	"github.com/davecgh/go-spew/spew"
 	"github.com/lbryio/ytsync/v5/downloader/ytdl"
 
@@ -38,7 +40,9 @@ func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan)
 	return videoIDs, nil
 }
 
-func GetVideoInformation(videoID string, stopChan stop.Chan, ip *net.TCPAddr) (*ytdl.YtdlVideo, error) {
+const releaseTimeFormat = "2006-01-02, 15:04:05 (MST)"
+
+func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Chan, ip *net.TCPAddr) (*ytdl.YtdlVideo, error) {
 	args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID}
 	results, err := run(args, false, true, stopChan)
 	if err != nil {
@@ -55,7 +59,7 @@ func GetVideoInformation(videoID string, stopChan stop.Chan, ip *net.TCPAddr) (*
 	tries := 0
 GetTime:
 	tries++
-	t, err := getUploadTime(videoID, ip)
+	t, err := getUploadTime(config, videoID, ip)
 	if err != nil {
 		//slack(":warning: Upload time error: %v", err)
 		if tries <= maxTries && (errors.Is(err, errNotScraped) || errors.Is(err, errUploadTimeEmpty)) {
@@ -146,9 +150,18 @@ func triggerScrape(videoID string, ip *net.TCPAddr) error {
 	//https://caa.iti.gr/caa/api/v4/videos/reports/h-tuxHS5lSM
 }
 
-func getUploadTime(videoID string, ip *net.TCPAddr) (string, error) {
+func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr) (string, error) {
 	//slack("Getting upload time for %s", videoID)
-
+	release, err := config.GetReleasedDate(videoID)
+	if err != nil {
+		if release != nil {
+			const sqlTimeFormat = "2006-01-02 15:04:05"
+			sqlTime, err := time.ParseInLocation(sqlTimeFormat, release.ReleaseTime, time.UTC)
+			if err != nil {
+				return sqlTime.Format(releaseTimeFormat), nil
+			}
+		}
+	}
 	client := getClient(ip)
 	req, err := http.NewRequest(http.MethodGet, "https://caa.iti.gr/get_verificationV3?url=https://www.youtube.com/watch?v="+videoID, nil)
 	if err != nil {
diff --git a/go.mod b/go.mod
index 558dc66..c915950 100644
--- a/go.mod
+++ b/go.mod
@@ -4,11 +4,10 @@ replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203
 
 require (
 	cloud.google.com/go v0.46.3 // indirect
-	github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61
 	github.com/Microsoft/go-winio v0.4.14 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a
 	github.com/aws/aws-sdk-go v1.25.9
-	github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 // indirect
+	github.com/davecgh/go-spew v1.1.1
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/docker v1.13.1
 	github.com/docker/go-connections v0.4.0 // indirect
diff --git a/go.sum b/go.sum
index 5ba9dde..05ca74d 100644
--- a/go.sum
+++ b/go.sum
@@ -11,8 +11,6 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:N5Vqww5QISEHsWHOWDEx4PzdIay3Cg0Jp7zItq2ZAro=
-github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:GnKXcK+7DYNy/8w2Ex//Uql4IgfaU82Cd5rWKb7ah00=
 github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
 github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
@@ -44,8 +42,6 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku
 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc=
 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
 github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
-github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:o64h9XF42kVEUuhuer2ehqrlX8rZmvQSU0+Vpj1rF6Q=
-github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:Rp8e0DCtEKwXFOC6JPJQVTz8tuGoGvw6Xfexggh/ed0=
 github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
 github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
diff --git a/sdk/api.go b/sdk/api.go
index bf66b14..f6cb2a6 100644
--- a/sdk/api.go
+++ b/sdk/api.go
@@ -408,3 +408,52 @@ func (a *APIConfig) VideoState(videoID string) (string, error) {
 	}
 	return "", errors.Err("invalid API response. Status code: %d", res.StatusCode)
 }
+
+type VideoRelease struct {
+	ID            uint64 `json:"id"`
+	YoutubeDataID uint64 `json:"youtube_data_id"`
+	VideoID       string `json:"video_id"`
+	ReleaseTime   string `json:"release_time""`
+	CreatedAt     string `json:"created_at"`
+	UpdatedAt     string `json:"updated_at"`
+}
+
+func (a *APIConfig) GetReleasedDate(videoID string) (*VideoRelease, error) {
+	endpoint := a.ApiURL + "/yt/released"
+	vals := url.Values{
+		"video_id":   {videoID},
+		"auth_token": {a.ApiToken},
+	}
+
+	res, err := http.PostForm(endpoint, vals)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	defer res.Body.Close()
+	body, _ := ioutil.ReadAll(res.Body)
+	if res.StatusCode == http.StatusNotFound {
+		return nil, nil
+	}
+	if res.StatusCode != http.StatusOK {
+		util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
+		log.Debugln(string(body))
+		time.Sleep(30 * time.Second)
+		return a.GetReleasedDate(videoID)
+	}
+	var response struct {
+		Success bool         `json:"success"`
+		Error   null.String  `json:"error"`
+		Data    VideoRelease `json:"data"`
+	}
+	err = json.Unmarshal(body, &response)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	if !response.Error.IsNull() {
+		return nil, errors.Err(response.Error.String)
+	}
+	if response.Data.ReleaseTime != "" {
+		return &response.Data, nil
+	}
+	return nil, errors.Err("invalid API response. Status code: %d", res.StatusCode)
+}
diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go
index 1d51532..db8afd8 100644
--- a/ytapi/ytapi.go
+++ b/ytapi/ytapi.go
@@ -182,7 +182,7 @@ func getVideos(config *sdk.APIConfig, videoIDs []string, stopChan stop.Chan, ipP
 		if state == "published" {
 			continue
 		}
-		video, err := downloader.GetVideoInformation(videoID, stopChan, nil)
+		video, err := downloader.GetVideoInformation(config, videoID, stopChan, nil)
 		if err != nil {
 			//ipPool.ReleaseIP(ip)
 			return nil, errors.Err(err)

From 81e9378b9d12a498c11a1858ba6734b80a8956a3 Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Thu, 30 Jul 2020 11:38:22 -0400
Subject: [PATCH 20/57] Add new fall through errors for release time

---
 downloader/downloader.go | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/downloader/downloader.go b/downloader/downloader.go
index 0f0293f..9a17cb4 100644
--- a/downloader/downloader.go
+++ b/downloader/downloader.go
@@ -62,7 +62,7 @@ GetTime:
 	t, err := getUploadTime(config, videoID, ip)
 	if err != nil {
 		//slack(":warning: Upload time error: %v", err)
-		if tries <= maxTries && (errors.Is(err, errNotScraped) || errors.Is(err, errUploadTimeEmpty)) {
+		if tries <= maxTries && (errors.Is(err, errNotScraped) || errors.Is(err, errUploadTimeEmpty) || errors.Is(err, errStatusParse) || errors.Is(err, errConnectionIssue)) {
 			err := triggerScrape(videoID, ip)
 			if err == nil {
 				time.Sleep(2 * time.Second) // let them scrape it
@@ -99,6 +99,8 @@ GetTime:
 
 var errNotScraped = errors.Base("not yet scraped by caa.iti.gr")
 var errUploadTimeEmpty = errors.Base("upload time is empty")
+var errStatusParse = errors.Base("could not parse status, got number, need string")
+var errConnectionIssue = errors.Base("there was a connection issue with the api")
 
 func slack(format string, a ...interface{}) {
 	fmt.Printf(format+"\n", a...)
@@ -134,6 +136,12 @@ func triggerScrape(videoID string, ip *net.TCPAddr) error {
 	}
 	err = json.NewDecoder(res.Body).Decode(&response)
 	if err != nil {
+		if strings.Contains(err.Error(), "cannot unmarshal number") {
+			return errors.Err(errStatusParse)
+		}
+		if strings.Contains(err.Error(), "no route to host") {
+			return errors.Err(errConnectionIssue)
+		}
 		return errors.Err(err)
 	}
 

From ad2742547174138b1d59aeb75b9ae9a37970d52f Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Thu, 30 Jul 2020 12:03:07 -0400
Subject: [PATCH 21/57] Add ip switching for being blocked

---
 downloader/downloader.go | 33 ++++++++++++++++++++++++++++-----
 1 file changed, 28 insertions(+), 5 deletions(-)

diff --git a/downloader/downloader.go b/downloader/downloader.go
index 9a17cb4..915b648 100644
--- a/downloader/downloader.go
+++ b/downloader/downloader.go
@@ -12,7 +12,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/lbryio/ytsync/v5/ip_manager"
 	"github.com/lbryio/ytsync/v5/sdk"
+	"github.com/lbryio/ytsync/v5/ytapi"
 
 	"github.com/davecgh/go-spew/spew"
 	"github.com/lbryio/ytsync/v5/downloader/ytdl"
@@ -24,9 +26,9 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan) ([]string, error) {
+func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan, videoParams ytapi.VideoParams) ([]string, error) {
 	args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist"}
-	ids, err := run(args, false, true, stopChan)
+	ids, err := run(channelName, args, false, true, stopChan, videoParams)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
@@ -42,9 +44,9 @@ func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan)
 
 const releaseTimeFormat = "2006-01-02, 15:04:05 (MST)"
 
-func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Chan, ip *net.TCPAddr) (*ytdl.YtdlVideo, error) {
+func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Chan, ip *net.TCPAddr, videoParams ytapi.VideoParams) (*ytdl.YtdlVideo, error) {
 	args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID}
-	results, err := run(args, false, true, stopChan)
+	results, err := run(videoID, args, false, true, stopChan, videoParams)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
@@ -229,7 +231,28 @@ func getClient(ip *net.TCPAddr) *http.Client {
 	}
 }
 
-func run(args []string, withStdErr, withStdOut bool, stopChan stop.Chan) ([]string, error) {
+func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.Chan, v ytapi.VideoParams) ([]string, error) {
+	var sourceAddress string
+	var err error
+	for {
+		sourceAddress, err = v.IPPool.GetIP(use)
+		if err != nil {
+			if errors.Is(err, ip_manager.ErrAllThrottled) {
+				select {
+				case <-stopChan:
+					return nil, errors.Err("interrupted by user")
+				default:
+					time.Sleep(ip_manager.IPCooldownPeriod)
+					continue
+				}
+			} else {
+				return nil, err
+			}
+		}
+		break
+	}
+	defer v.IPPool.ReleaseIP(sourceAddress)
+	args = append(args, "--source-address", sourceAddress)
 	cmd := exec.Command("youtube-dl", args...)
 	logrus.Printf("Running command youtube-dl %s", strings.Join(args, " "))
 

From b1e10e7b09ccffafc657f8eae45b829d36542ae4 Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Thu, 30 Jul 2020 12:14:06 -0400
Subject: [PATCH 22/57] Add retry on IP failures

---
 Makefile                 |   2 +-
 downloader/downloader.go | 142 +++++++++++++++++++++------------------
 2 files changed, 79 insertions(+), 65 deletions(-)

diff --git a/Makefile b/Makefile
index e41cb2e..870e2b0 100644
--- a/Makefile
+++ b/Makefile
@@ -13,7 +13,7 @@ LDFLAGS = -ldflags "-X ${IMPORT_PATH}/meta.Version=${VERSION} -X ${IMPORT_PATH}/
 
 
 build:
-	mkdir -p ${BIN_DIR} && CGO_ENABLED=0 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
+	mkdir -p ${BIN_DIR} && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
 
 clean:
 	if [ -f ${BIN_DIR}/${BINARY} ]; then rm ${BIN_DIR}/${BINARY}; fi
diff --git a/downloader/downloader.go b/downloader/downloader.go
index 915b648..ecebb5e 100644
--- a/downloader/downloader.go
+++ b/downloader/downloader.go
@@ -232,80 +232,94 @@ func getClient(ip *net.TCPAddr) *http.Client {
 }
 
 func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.Chan, v ytapi.VideoParams) ([]string, error) {
-	var sourceAddress string
-	var err error
+	var maxtries = 10
+	var attemps int
 	for {
-		sourceAddress, err = v.IPPool.GetIP(use)
-		if err != nil {
-			if errors.Is(err, ip_manager.ErrAllThrottled) {
-				select {
-				case <-stopChan:
-					return nil, errors.Err("interrupted by user")
-				default:
-					time.Sleep(ip_manager.IPCooldownPeriod)
-					continue
+		var sourceAddress string
+		var err error
+		for {
+			sourceAddress, err = v.IPPool.GetIP(use)
+			if err != nil {
+				if errors.Is(err, ip_manager.ErrAllThrottled) {
+					select {
+					case <-stopChan:
+						return nil, errors.Err("interrupted by user")
+					default:
+						time.Sleep(ip_manager.IPCooldownPeriod)
+						continue
+					}
+				} else {
+					return nil, err
 				}
-			} else {
-				return nil, err
+			}
+			break
+		}
+		defer v.IPPool.ReleaseIP(sourceAddress)
+		args = append(args, "--source-address", sourceAddress)
+		cmd := exec.Command("youtube-dl", args...)
+		logrus.Printf("Running command youtube-dl %s", strings.Join(args, " "))
+
+		var stderr io.ReadCloser
+		var errorLog []byte
+		if withStdErr {
+			var err error
+			stderr, err = cmd.StderrPipe()
+			if err != nil {
+				return nil, errors.Err(err)
+			}
+			errorLog, err = ioutil.ReadAll(stderr)
+			if err != nil {
+				return nil, errors.Err(err)
 			}
 		}
-		break
-	}
-	defer v.IPPool.ReleaseIP(sourceAddress)
-	args = append(args, "--source-address", sourceAddress)
-	cmd := exec.Command("youtube-dl", args...)
-	logrus.Printf("Running command youtube-dl %s", strings.Join(args, " "))
 
-	var stderr io.ReadCloser
-	var errorLog []byte
-	if withStdErr {
-		var err error
-		stderr, err = cmd.StderrPipe()
-		if err != nil {
-			return nil, errors.Err(err)
-		}
-		errorLog, err = ioutil.ReadAll(stderr)
-		if err != nil {
-			return nil, errors.Err(err)
-		}
-	}
+		var stdout io.ReadCloser
+		var outLog []byte
+		if withStdOut {
+			var err error
+			stdout, err = cmd.StdoutPipe()
+			if err != nil {
+				return nil, errors.Err(err)
+			}
 
-	var stdout io.ReadCloser
-	var outLog []byte
-	if withStdOut {
-		var err error
-		stdout, err = cmd.StdoutPipe()
-		if err != nil {
-			return nil, errors.Err(err)
+			if err := cmd.Start(); err != nil {
+				return nil, errors.Err(err)
+			}
+			outLog, err = ioutil.ReadAll(stdout)
+			if err != nil {
+				return nil, errors.Err(err)
+			}
 		}
 
-		if err := cmd.Start(); err != nil {
-			return nil, errors.Err(err)
+		done := make(chan error, 1)
+		go func() {
+			attemps++
+			done <- cmd.Wait()
+		}()
+		select {
+		case <-stopChan:
+			if err := cmd.Process.Kill(); err != nil {
+				return nil, errors.Prefix("failed to kill command after stopper cancellation", err)
+			}
+			return nil, errors.Err("canceled by stopper")
+		case err := <-done:
+			if err != nil {
+				if strings.Contains(err.Error(), "exit status 1") {
+					if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
+						v.IPPool.SetThrottled(sourceAddress)
+					}
+					if attemps > maxtries {
+						break
+					}
+					continue
+				}
+				return nil, errors.Prefix("youtube-dl "+strings.Join(args, " "), err)
+			}
+			return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
 		}
-		outLog, err = ioutil.ReadAll(stdout)
-		if err != nil {
-			return nil, errors.Err(err)
-		}
-	}
 
-	done := make(chan error, 1)
-	go func() {
-		done <- cmd.Wait()
-	}()
-	select {
-	case <-stopChan:
-		if err := cmd.Process.Kill(); err != nil {
-			return nil, errors.Prefix("failed to kill command after stopper cancellation", err)
-		}
-		return nil, errors.Err("canceled by stopper")
-	case err := <-done:
-		if err != nil {
-			return nil, errors.Prefix("youtube-dl "+strings.Join(args, " "), err)
+		if len(errorLog) > 0 {
+			return nil, errors.Err(string(errorLog))
 		}
 	}
-
-	if len(errorLog) > 0 {
-		return nil, errors.Err(string(errorLog))
-	}
-	return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
 }

From a1caea4a2892a2726eac8c5070421d273002a450 Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Thu, 30 Jul 2020 12:48:05 -0400
Subject: [PATCH 23/57] added logging

---
 downloader/downloader.go | 25 +++++++++++++------------
 ytapi/ytapi.go           |  7 +++++--
 2 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/downloader/downloader.go b/downloader/downloader.go
index ecebb5e..30e72ee 100644
--- a/downloader/downloader.go
+++ b/downloader/downloader.go
@@ -12,12 +12,10 @@ import (
 	"strings"
 	"time"
 
-	"github.com/lbryio/ytsync/v5/ip_manager"
-	"github.com/lbryio/ytsync/v5/sdk"
-	"github.com/lbryio/ytsync/v5/ytapi"
-
 	"github.com/davecgh/go-spew/spew"
 	"github.com/lbryio/ytsync/v5/downloader/ytdl"
+	"github.com/lbryio/ytsync/v5/ip_manager"
+	"github.com/lbryio/ytsync/v5/sdk"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/extras/stop"
@@ -26,9 +24,9 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan, videoParams ytapi.VideoParams) ([]string, error) {
+func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
 	args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist"}
-	ids, err := run(channelName, args, false, true, stopChan, videoParams)
+	ids, err := run(channelName, args, false, true, stopChan, pool)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
@@ -44,9 +42,9 @@ func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan,
 
 const releaseTimeFormat = "2006-01-02, 15:04:05 (MST)"
 
-func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Chan, ip *net.TCPAddr, videoParams ytapi.VideoParams) (*ytdl.YtdlVideo, error) {
+func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Chan, ip *net.TCPAddr, pool *ip_manager.IPPool) (*ytdl.YtdlVideo, error) {
 	args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID}
-	results, err := run(videoID, args, false, true, stopChan, videoParams)
+	results, err := run(videoID, args, false, true, stopChan, pool)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
@@ -231,14 +229,14 @@ func getClient(ip *net.TCPAddr) *http.Client {
 	}
 }
 
-func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.Chan, v ytapi.VideoParams) ([]string, error) {
+func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
 	var maxtries = 10
 	var attemps int
 	for {
 		var sourceAddress string
 		var err error
 		for {
-			sourceAddress, err = v.IPPool.GetIP(use)
+			sourceAddress, err = pool.GetIP(use)
 			if err != nil {
 				if errors.Is(err, ip_manager.ErrAllThrottled) {
 					select {
@@ -254,7 +252,7 @@ func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.C
 			}
 			break
 		}
-		defer v.IPPool.ReleaseIP(sourceAddress)
+		defer pool.ReleaseIP(sourceAddress)
 		args = append(args, "--source-address", sourceAddress)
 		cmd := exec.Command("youtube-dl", args...)
 		logrus.Printf("Running command youtube-dl %s", strings.Join(args, " "))
@@ -306,13 +304,16 @@ func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.C
 			if err != nil {
 				if strings.Contains(err.Error(), "exit status 1") {
 					if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
-						v.IPPool.SetThrottled(sourceAddress)
+						pool.SetThrottled(sourceAddress)
 					}
 					if attemps > maxtries {
+						logrus.Debug("too many tries returning failure")
 						break
 					}
+					logrus.Debugf("known throttling error...try again (%d)", attemps)
 					continue
 				}
+				logrus.Debug("Unkown error, returning failure")
 				return nil, errors.Prefix("youtube-dl "+strings.Join(args, " "), err)
 			}
 			return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go
index db8afd8..718f515 100644
--- a/ytapi/ytapi.go
+++ b/ytapi/ytapi.go
@@ -57,7 +57,7 @@ func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[s
 	if quickSync {
 		maxVideos = 50
 	}
-	videoIDs, err := downloader.GetPlaylistVideoIDs(channelID, maxVideos, videoParams.Stopper.Ch())
+	videoIDs, err := downloader.GetPlaylistVideoIDs(channelID, maxVideos, videoParams.Stopper.Ch(), videoParams.IPPool)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
@@ -164,6 +164,9 @@ func ChannelInfo(apiKey, channelID string) (*ytlib.ChannelSnippet, *ytlib.Channe
 func getVideos(config *sdk.APIConfig, videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool) ([]*ytdl.YtdlVideo, error) {
 	var videos []*ytdl.YtdlVideo
 	for _, videoID := range videoIDs {
+		if len(videoID) < 5 {
+			continue
+		}
 		select {
 		case <-stopChan:
 			return videos, errors.Err("canceled by stopper")
@@ -182,7 +185,7 @@ func getVideos(config *sdk.APIConfig, videoIDs []string, stopChan stop.Chan, ipP
 		if state == "published" {
 			continue
 		}
-		video, err := downloader.GetVideoInformation(config, videoID, stopChan, nil)
+		video, err := downloader.GetVideoInformation(config, videoID, stopChan, nil, ipPool)
 		if err != nil {
 			//ipPool.ReleaseIP(ip)
 			return nil, errors.Err(err)

From 140353097e05d799e0cccdd35449c689929fa742 Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Thu, 30 Jul 2020 13:05:12 -0400
Subject: [PATCH 24/57] added logging

---
 downloader/downloader.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/downloader/downloader.go b/downloader/downloader.go
index 30e72ee..ed8e6b6 100644
--- a/downloader/downloader.go
+++ b/downloader/downloader.go
@@ -253,8 +253,8 @@ func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.C
 			break
 		}
 		defer pool.ReleaseIP(sourceAddress)
-		args = append(args, "--source-address", sourceAddress)
-		cmd := exec.Command("youtube-dl", args...)
+		argsForCommand := append(args, "--source-address", sourceAddress)
+		cmd := exec.Command("youtube-dl", argsForCommand...)
 		logrus.Printf("Running command youtube-dl %s", strings.Join(args, " "))
 
 		var stderr io.ReadCloser

From 4d56339756841304706b7dcd13fcd0f8949584df Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Thu, 30 Jul 2020 13:05:49 -0400
Subject: [PATCH 25/57] fixed appending args

---
 downloader/downloader.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/downloader/downloader.go b/downloader/downloader.go
index ed8e6b6..c8500f1 100644
--- a/downloader/downloader.go
+++ b/downloader/downloader.go
@@ -255,7 +255,7 @@ func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.C
 		defer pool.ReleaseIP(sourceAddress)
 		argsForCommand := append(args, "--source-address", sourceAddress)
 		cmd := exec.Command("youtube-dl", argsForCommand...)
-		logrus.Printf("Running command youtube-dl %s", strings.Join(args, " "))
+		logrus.Printf("Running command youtube-dl %s", strings.Join(argsForCommand, " "))
 
 		var stderr io.ReadCloser
 		var errorLog []byte
@@ -314,7 +314,7 @@ func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.C
 					continue
 				}
 				logrus.Debug("Unkown error, returning failure")
-				return nil, errors.Prefix("youtube-dl "+strings.Join(args, " "), err)
+				return nil, errors.Prefix("youtube-dl "+strings.Join(argsForCommand, " "), err)
 			}
 			return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
 		}

From c7c220ecd33e7219d0524157acd6408ef9d5305d Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Thu, 30 Jul 2020 13:37:44 -0400
Subject: [PATCH 26/57] Dont always fall through on every error.

---
 downloader/downloader.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/downloader/downloader.go b/downloader/downloader.go
index c8500f1..7d79ae0 100644
--- a/downloader/downloader.go
+++ b/downloader/downloader.go
@@ -305,15 +305,15 @@ func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.C
 				if strings.Contains(err.Error(), "exit status 1") {
 					if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
 						pool.SetThrottled(sourceAddress)
+						logrus.Debugf("known throttling error...try again (%d)", attemps)
+						continue
 					}
 					if attemps > maxtries {
 						logrus.Debug("too many tries returning failure")
 						break
 					}
-					logrus.Debugf("known throttling error...try again (%d)", attemps)
-					continue
 				}
-				logrus.Debug("Unkown error, returning failure")
+				logrus.Debug("Unkown error, returning failure: %s", err.Error())
 				return nil, errors.Prefix("youtube-dl "+strings.Join(argsForCommand, " "), err)
 			}
 			return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil

From cd11e826761a6fbbd2c3a3f8738a1922a3a709de Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Thu, 30 Jul 2020 13:40:41 -0400
Subject: [PATCH 27/57] Dont always fall through on every error. Add user agent
 to calls

---
 downloader/downloader.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/downloader/downloader.go b/downloader/downloader.go
index 7d79ae0..7431118 100644
--- a/downloader/downloader.go
+++ b/downloader/downloader.go
@@ -254,6 +254,7 @@ func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.C
 		}
 		defer pool.ReleaseIP(sourceAddress)
 		argsForCommand := append(args, "--source-address", sourceAddress)
+		argsForCommand = append(args, "--user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
 		cmd := exec.Command("youtube-dl", argsForCommand...)
 		logrus.Printf("Running command youtube-dl %s", strings.Join(argsForCommand, " "))
 

From 4eba7200d80a65a738421e5f15787667c64a2eff Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Thu, 30 Jul 2020 13:53:27 -0400
Subject: [PATCH 28/57] user agent breaks it

---
 downloader/downloader.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/downloader/downloader.go b/downloader/downloader.go
index 7431118..56872bc 100644
--- a/downloader/downloader.go
+++ b/downloader/downloader.go
@@ -254,7 +254,7 @@ func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.C
 		}
 		defer pool.ReleaseIP(sourceAddress)
 		argsForCommand := append(args, "--source-address", sourceAddress)
-		argsForCommand = append(args, "--user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
+		//argsForCommand = append(args, "--user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
 		cmd := exec.Command("youtube-dl", argsForCommand...)
 		logrus.Printf("Running command youtube-dl %s", strings.Join(argsForCommand, " "))
 

From 13543b20e990b162c399793a370ebdde266fd38f Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Thu, 30 Jul 2020 15:37:14 -0400
Subject: [PATCH 29/57] Don't use api unless the video is from the past 5 days,
 otherwise just use the upload date from youtube donwloader change interface
 to int for now known field.

---
 downloader/downloader.go | 13 ++++++++++---
 downloader/ytdl/Video.go |  2 +-
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/downloader/downloader.go b/downloader/downloader.go
index 56872bc..7ad4155 100644
--- a/downloader/downloader.go
+++ b/downloader/downloader.go
@@ -59,7 +59,7 @@ func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Ch
 	tries := 0
 GetTime:
 	tries++
-	t, err := getUploadTime(config, videoID, ip)
+	t, err := getUploadTime(config, videoID, ip, video.UploadDate)
 	if err != nil {
 		//slack(":warning: Upload time error: %v", err)
 		if tries <= maxTries && (errors.Is(err, errNotScraped) || errors.Is(err, errUploadTimeEmpty) || errors.Is(err, errStatusParse) || errors.Is(err, errConnectionIssue)) {
@@ -158,7 +158,7 @@ func triggerScrape(videoID string, ip *net.TCPAddr) error {
 	//https://caa.iti.gr/caa/api/v4/videos/reports/h-tuxHS5lSM
 }
 
-func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr) (string, error) {
+func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr, uploadDate string) (string, error) {
 	//slack("Getting upload time for %s", videoID)
 	release, err := config.GetReleasedDate(videoID)
 	if err != nil {
@@ -170,6 +170,13 @@ func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr) (stri
 			}
 		}
 	}
+	ytdlUploadDate, err := time.Parse("20060102", uploadDate)
+	if err != nil {
+		logrus.Error(err)
+	}
+	if time.Now().Add(-5 * 24 * time.Hour).After(ytdlUploadDate) {
+		return ytdlUploadDate.Format(releaseTimeFormat), nil
+	}
 	client := getClient(ip)
 	req, err := http.NewRequest(http.MethodGet, "https://caa.iti.gr/get_verificationV3?url=https://www.youtube.com/watch?v="+videoID, nil)
 	if err != nil {
@@ -314,7 +321,7 @@ func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.C
 						break
 					}
 				}
-				logrus.Debug("Unkown error, returning failure: %s", err.Error())
+				logrus.Debugf("Unkown error, returning failure: %s", err.Error())
 				return nil, errors.Prefix("youtube-dl "+strings.Join(argsForCommand, " "), err)
 			}
 			return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
diff --git a/downloader/ytdl/Video.go b/downloader/ytdl/Video.go
index df032e1..c1d5c08 100644
--- a/downloader/ytdl/Video.go
+++ b/downloader/ytdl/Video.go
@@ -121,7 +121,7 @@ type Format struct {
 	Ext             string      `json:"ext"`
 	FragmentBaseURL string      `json:"fragment_base_url,omitempty"`
 	Filesize        interface{} `json:"filesize"`
-	Fps             interface{} `json:"fps"`
+	Fps             int         `json:"fps"`
 	Fragments       []struct {
 		Path     string  `json:"path"`
 		Duration float64 `json:"duration,omitempty"`

From fc18151d777934c431906084e09c14e95a08c359 Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Fri, 31 Jul 2020 23:28:19 -0400
Subject: [PATCH 30/57] Increase max db limit Increase wallet startup time
 allotment for big channels Set FPS to float instead of int

---
 downloader/ytdl/Video.go | 2 +-
 manager/ytsync.go        | 2 +-
 util/util.go             | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/downloader/ytdl/Video.go b/downloader/ytdl/Video.go
index c1d5c08..0c8943d 100644
--- a/downloader/ytdl/Video.go
+++ b/downloader/ytdl/Video.go
@@ -121,7 +121,7 @@ type Format struct {
 	Ext             string      `json:"ext"`
 	FragmentBaseURL string      `json:"fragment_base_url,omitempty"`
 	Filesize        interface{} `json:"filesize"`
-	Fps             int         `json:"fps"`
+	Fps             float64     `json:"fps"`
 	Fragments       []struct {
 		Path     string  `json:"path"`
 		Duration float64 `json:"duration,omitempty"`
diff --git a/manager/ytsync.go b/manager/ytsync.go
index ad387e2..e32dae5 100644
--- a/manager/ytsync.go
+++ b/manager/ytsync.go
@@ -424,7 +424,7 @@ func (s *Sync) waitForDaemonStart() error {
 			if err == nil && status.StartupStatus.Wallet && status.IsRunning {
 				return nil
 			}
-			if time.Since(beginTime).Minutes() > 60 {
+			if time.Since(beginTime).Minutes() > 120 {
 				s.grp.Stop()
 				return errors.Err("the daemon is taking too long to start. Something is wrong")
 			}
diff --git a/util/util.go b/util/util.go
index 096ec71..cfe6fb6 100644
--- a/util/util.go
+++ b/util/util.go
@@ -255,7 +255,7 @@ func CleanupLbrynet() error {
 		}
 		return errors.Err(err)
 	}
-	dbSizeLimit := int64(1 * 1024 * 1024 * 1024)
+	dbSizeLimit := int64(2 * 1024 * 1024 * 1024)
 	if db.Size() > dbSizeLimit {
 		files, err := filepath.Glob(lbryumDir + "/blockchain.db*")
 		if err != nil {

From 5be3551abee2cec6db15b096546aba2c87a5cf7d Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Mon, 3 Aug 2020 01:05:03 -0400
Subject: [PATCH 31/57] Mark video failed if we cannot get video info instead
 of failing the sync Turn on error log after fixing lockup Try different user
 agents if calls fail Prevent double processing a channel in a row, just fail
 the sync instead

---
 downloader/downloader.go | 71 +++++++++++++++++++++++++---------------
 manager/manager.go       |  8 +++++
 ytapi/ytapi.go           | 22 +++++++++----
 3 files changed, 69 insertions(+), 32 deletions(-)

diff --git a/downloader/downloader.go b/downloader/downloader.go
index 7ad4155..eba0c1a 100644
--- a/downloader/downloader.go
+++ b/downloader/downloader.go
@@ -26,12 +26,13 @@ import (
 
 func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
 	args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist"}
-	ids, err := run(channelName, args, false, true, stopChan, pool)
+	ids, err := run(channelName, args, true, true, stopChan, pool)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
 	videoIDs := make([]string, maxVideos)
 	for i, v := range ids {
+		logrus.Debugf("%d - video id %s", i, v)
 		if i >= maxVideos {
 			break
 		}
@@ -44,7 +45,7 @@ const releaseTimeFormat = "2006-01-02, 15:04:05 (MST)"
 
 func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Chan, ip *net.TCPAddr, pool *ip_manager.IPPool) (*ytdl.YtdlVideo, error) {
 	args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID}
-	results, err := run(videoID, args, false, true, stopChan, pool)
+	results, err := run(videoID, args, true, true, stopChan, pool)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
@@ -239,29 +240,15 @@ func getClient(ip *net.TCPAddr) *http.Client {
 func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
 	var maxtries = 10
 	var attemps int
+	var useragent []string
 	for {
-		var sourceAddress string
-		var err error
-		for {
-			sourceAddress, err = pool.GetIP(use)
-			if err != nil {
-				if errors.Is(err, ip_manager.ErrAllThrottled) {
-					select {
-					case <-stopChan:
-						return nil, errors.Err("interrupted by user")
-					default:
-						time.Sleep(ip_manager.IPCooldownPeriod)
-						continue
-					}
-				} else {
-					return nil, err
-				}
-			}
-			break
+		sourceAddress, err := getIPFromPool(use, stopChan, pool)
+		if err != nil {
+			return nil, err
 		}
 		defer pool.ReleaseIP(sourceAddress)
 		argsForCommand := append(args, "--source-address", sourceAddress)
-		//argsForCommand = append(args, "--user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
+		argsForCommand = append(argsForCommand, useragent...)
 		cmd := exec.Command("youtube-dl", argsForCommand...)
 		logrus.Printf("Running command youtube-dl %s", strings.Join(argsForCommand, " "))
 
@@ -273,10 +260,6 @@ func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.C
 			if err != nil {
 				return nil, errors.Err(err)
 			}
-			errorLog, err = ioutil.ReadAll(stderr)
-			if err != nil {
-				return nil, errors.Err(err)
-			}
 		}
 
 		var stdout io.ReadCloser
@@ -295,6 +278,12 @@ func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.C
 			if err != nil {
 				return nil, errors.Err(err)
 			}
+			if withStdErr {
+				errorLog, err = ioutil.ReadAll(stderr)
+				if err != nil {
+					return nil, errors.Err(err)
+				}
+			}
 		}
 
 		done := make(chan error, 1)
@@ -316,13 +305,21 @@ func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.C
 						logrus.Debugf("known throttling error...try again (%d)", attemps)
 						continue
 					}
+					if strings.Contains(string(errorLog), "YouTube said: Unable to extract video data") {
+						useragent = []string{"--user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36"}
+						if attemps == 1 {
+							useragent = []string{"--user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}
+						}
+						logrus.Debugf("known extraction issue, maybe user agent specification will work...try again (%d)", attemps)
+						continue
+					}
 					if attemps > maxtries {
 						logrus.Debug("too many tries returning failure")
 						break
 					}
 				}
 				logrus.Debugf("Unkown error, returning failure: %s", err.Error())
-				return nil, errors.Prefix("youtube-dl "+strings.Join(argsForCommand, " "), err)
+				return nil, errors.Prefix("youtube-dl "+strings.Join(argsForCommand, " ")+" ["+string(errorLog)+"] ", err)
 			}
 			return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
 		}
@@ -332,3 +329,25 @@ func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.C
 		}
 	}
 }
+
+func getIPFromPool(use string, stopChan stop.Chan, pool *ip_manager.IPPool) (sourceAddress string, err error) {
+	for {
+		sourceAddress, err = pool.GetIP(use)
+		if err != nil {
+			if errors.Is(err, ip_manager.ErrAllThrottled) {
+				select {
+				case <-stopChan:
+					return "", errors.Err("interrupted by user")
+
+				default:
+					time.Sleep(ip_manager.IPCooldownPeriod)
+					continue
+				}
+			} else {
+				return "", err
+			}
+		}
+		break
+	}
+	return
+}
diff --git a/manager/manager.go b/manager/manager.go
index 2c84a9a..ac24ad7 100644
--- a/manager/manager.go
+++ b/manager/manager.go
@@ -105,6 +105,7 @@ func (s *SyncManager) Start() error {
 		}
 	}
 
+	var lastChannelProcessed string
 	syncCount := 0
 	for {
 		err := s.checkUsedSpace()
@@ -205,6 +206,13 @@ func (s *SyncManager) Start() error {
 			time.Sleep(5 * time.Minute)
 		}
 		for _, sync := range syncs {
+			if lastChannelProcessed == sync.LbryChannelName {
+				util.SendToSlack("We just killed a sync for %s to stop looping!(%s)", sync.LbryChannelName, sync.YoutubeChannelID)
+				stopTheLoops := errors.Err("Found channel %s running twice, set it to failed, and reprocess later", sync.LbryChannelName)
+				sync.setChannelTerminationStatus(&stopTheLoops)
+				continue
+			}
+			lastChannelProcessed = sync.LbryChannelName
 			shouldNotCount := false
 			logUtils.SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
 			err := sync.FullCycle()
diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go
index 718f515..6b98406 100644
--- a/ytapi/ytapi.go
+++ b/ytapi/ytapi.go
@@ -10,6 +10,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/lbryio/ytsync/v5/util"
+
 	"github.com/lbryio/ytsync/v5/downloader/ytdl"
 
 	"github.com/lbryio/ytsync/v5/downloader"
@@ -76,7 +78,7 @@ func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[s
 		mostRecentlyFailedChannel = channelID
 	}
 
-	vids, err := getVideos(config, videoIDs, videoParams.Stopper.Ch(), videoParams.IPPool)
+	vids, err := getVideos(config, channelID, videoIDs, videoParams.Stopper.Ch(), videoParams.IPPool)
 	if err != nil {
 		return nil, err
 	}
@@ -161,7 +163,7 @@ func ChannelInfo(apiKey, channelID string) (*ytlib.ChannelSnippet, *ytlib.Channe
 	return response.Items[0].Snippet, response.Items[0].BrandingSettings, nil
 }
 
-func getVideos(config *sdk.APIConfig, videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool) ([]*ytdl.YtdlVideo, error) {
+func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool) ([]*ytdl.YtdlVideo, error) {
 	var videos []*ytdl.YtdlVideo
 	for _, videoID := range videoIDs {
 		if len(videoID) < 5 {
@@ -187,11 +189,19 @@ func getVideos(config *sdk.APIConfig, videoIDs []string, stopChan stop.Chan, ipP
 		}
 		video, err := downloader.GetVideoInformation(config, videoID, stopChan, nil, ipPool)
 		if err != nil {
-			//ipPool.ReleaseIP(ip)
-			return nil, errors.Err(err)
+			errSDK := config.MarkVideoStatus(sdk.VideoStatus{
+				ChannelID:     channelID,
+				VideoID:       videoID,
+				Status:        "failed",
+				FailureReason: err.Error(),
+			})
+			util.SendErrorToSlack("Skipping video: " + err.Error())
+			if errSDK != nil {
+				return nil, errors.Err(errSDK)
+			}
+		} else {
+			videos = append(videos, video)
 		}
-		videos = append(videos, video)
-		//ipPool.ReleaseIP(ip)
 	}
 	return videos, nil
 }

From 7f1906d58bb463d8daff5479cd43708cc51fbd6d Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Mon, 3 Aug 2020 01:19:41 -0400
Subject: [PATCH 32/57] change FPS to float from int

---
 downloader/ytdl/Video.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/downloader/ytdl/Video.go b/downloader/ytdl/Video.go
index 0c8943d..079d557 100644
--- a/downloader/ytdl/Video.go
+++ b/downloader/ytdl/Video.go
@@ -30,7 +30,7 @@ type YtdlVideo struct {
 	Abr                int         `json:"abr"`
 	UploaderURL        string      `json:"uploader_url"`
 	Categories         []string    `json:"categories"`
-	Fps                int         `json:"fps"`
+	Fps                float64     `json:"fps"`
 	StretchedRatio     interface{} `json:"stretched_ratio"`
 	SeasonNumber       interface{} `json:"season_number"`
 	Annotations        interface{} `json:"annotations"`
@@ -83,7 +83,7 @@ type RequestedFormat struct {
 	Ext             string      `json:"ext"`
 	FragmentBaseURL string      `json:"fragment_base_url"`
 	Filesize        interface{} `json:"filesize"`
-	Fps             int         `json:"fps"`
+	Fps             float64     `json:"fps"`
 	ManifestURL     string      `json:"manifest_url"`
 	Protocol        string      `json:"protocol"`
 	FormatID        string      `json:"format_id"`

From 41fd9f6844c584e7b5756a19e124f0f44867c4f4 Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Mon, 3 Aug 2020 01:40:59 -0400
Subject: [PATCH 33/57] fix string parameter for error message

---
 sources/youtubeVideo.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sources/youtubeVideo.go b/sources/youtubeVideo.go
index 771a99a..e4bbc9f 100644
--- a/sources/youtubeVideo.go
+++ b/sources/youtubeVideo.go
@@ -453,7 +453,7 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
 
 	if dur > v.maxVideoLength {
 		log.Infof("%s is %d long and the limit is %s", v.id, dur.String(), v.maxVideoLength.String())
-		logUtils.SendErrorToSlack("%s is %d long and the limit is %s", v.id, dur.String(), v.maxVideoLength.String())
+		logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, dur.String(), v.maxVideoLength.String())
 		return nil, errors.Err("video is too long to process")
 	}
 	for {

From 751bc84ce59b65dd4b00bb3b411026a2f8b5e432 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Tue, 4 Aug 2020 00:55:26 +0200
Subject: [PATCH 34/57] grab channel info without APIs

fix new channel syncs
fix e2e tests
---
 e2e/data_setup.sh     |   4 +-
 e2e/e2e.sh            |   2 +-
 go.mod                |   6 +-
 go.sum                |  84 +--------------
 manager/s3_storage.go | 240 ++++++++++++++++++++++++++++++++++++++++++
 manager/setup.go      |  31 +++---
 manager/ytsync.go     | 129 +++--------------------
 util/util.go          |  26 ++++-
 ytapi/ytapi.go        | 186 +++++++++++++++++++++++++++++---
 ytapi/ytapi_test.go   |  12 +++
 10 files changed, 481 insertions(+), 239 deletions(-)
 create mode 100644 manager/s3_storage.go
 create mode 100644 ytapi/ytapi_test.go

diff --git a/e2e/data_setup.sh b/e2e/data_setup.sh
index e15c1db..a6dfae4 100755
--- a/e2e/data_setup.sh
+++ b/e2e/data_setup.sh
@@ -19,6 +19,6 @@ mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER"
 ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(2,"youtubertoken")'
 mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
 #Add their youtube channel to be synced
-ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable)
-VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCNQfQvFMPnInwsU_iGYArJQ','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',10,10,1,1)"
+ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable,total_views,reviewed,last_uploaded_video)
+VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCNQfQvFMPnInwsU_iGYArJQ','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',10,10,1,1,10000,1,'7bBV2Z-9wpo')"
 mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL"
diff --git a/e2e/e2e.sh b/e2e/e2e.sh
index e051e27..f6fbc94 100755
--- a/e2e/e2e.sh
+++ b/e2e/e2e.sh
@@ -26,7 +26,7 @@ export REGTEST=true
 # Local settings
 export BLOBS_DIRECTORY="$(pwd)/e2e/blobsfiles"
 export LBRYNET_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbrynet/"
-export LBRYNET_WALLETS_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum"
+export LBRYUM_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum"
 export TMP_DIR="/var/tmp"
 export CHAINNAME="lbrycrd_regtest"
 export UID
diff --git a/go.mod b/go.mod
index c915950..914602c 100644
--- a/go.mod
+++ b/go.mod
@@ -3,7 +3,6 @@ module github.com/lbryio/ytsync/v5
 replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
 
 require (
-	cloud.google.com/go v0.46.3 // indirect
 	github.com/Microsoft/go-winio v0.4.14 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a
 	github.com/aws/aws-sdk-go v1.25.9
@@ -13,7 +12,7 @@ require (
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
 	github.com/go-sql-driver/mysql v1.4.1 // indirect
-	github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc // indirect
+	github.com/google/btree v1.0.0 // indirect
 	github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
 	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
 	github.com/hashicorp/golang-lru v0.5.3 // indirect
@@ -30,9 +29,8 @@ require (
 	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/pflag v1.0.5 // indirect
-	go.opencensus.io v0.22.1 // indirect
+	github.com/stretchr/testify v1.4.0
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
-	google.golang.org/api v0.11.0
 	google.golang.org/appengine v1.6.5 // indirect
 )
 
diff --git a/go.sum b/go.sum
index 05ca74d..367f184 100644
--- a/go.sum
+++ b/go.sum
@@ -1,16 +1,5 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3 h1:AVXDdKsrtX33oR9fbCMu/+c1o8Ofjq6Ku/MInaLVg5Y=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
 github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
@@ -78,12 +67,7 @@ github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZp
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc h1:55rEp52jU6bkyslZ1+C/7NGfpQsEc6pxGLAGDOctqbw=
-github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -93,16 +77,8 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z
 github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f h1:TyqzGm2z1h3AGhjOoRYyeLcW4WlW81MDQkWa+rx/000=
@@ -139,7 +115,6 @@ github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk=
 github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
@@ -166,7 +141,6 @@ github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22/go.mod h1:u0Jo4
 github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07 h1:+kBG/8rjCa6vxJZbUjAiE4MQmBEBYc8nLEb51frnvBY=
 github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
 github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
@@ -254,7 +228,6 @@ github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jO
 github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
 github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@@ -305,10 +278,6 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d h1:tQo6hjclyv3RHUgZOl6iWb2Y44A/sN9bf9LAYfuioEg=
 github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50=
-go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
@@ -318,20 +287,12 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
 golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5hzyggAkLLlCUjqfRxd8Q4=
 golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@@ -339,17 +300,16 @@ golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -357,13 +317,9 @@ golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20191009170851-d66e71096ffb h1:TR699M2v0qoKTOHxeLgp6zPqaQNs74f01a/ob9W0qko=
 golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -374,25 +330,19 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f h1:hjzMYz/7Ea1mNKfOnFOfktR0mlA5jqhvywClCMHM/qw=
 golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w=
 golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -400,54 +350,26 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.11.0 h1:n/qM3q0/rV2F0pox7o0CvNhlPvZAo7pLbef122cbLJ0=
-google.golang.org/api v0.11.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
 google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03 h1:4HYDjxeNXAOTv3o1N2tjo8UUSlhQgAD52FVkwxnWgM8=
 google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s=
 google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
@@ -465,8 +387,4 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
diff --git a/manager/s3_storage.go b/manager/s3_storage.go
new file mode 100644
index 0000000..cccb5a1
--- /dev/null
+++ b/manager/s3_storage.go
@@ -0,0 +1,240 @@
+package manager
+
+import (
+	"os"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+
+	logUtils "github.com/lbryio/ytsync/v5/util"
+)
+
+func (s *Sync) getS3Downloader() (*s3manager.Downloader, error) {
+	creds := credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, "")
+	s3Session, err := session.NewSession(&aws.Config{Region: aws.String(s.AwsS3Region), Credentials: creds})
+	if err != nil {
+		return nil, errors.Prefix("error starting session: ", err)
+	}
+	downloader := s3manager.NewDownloader(s3Session)
+	return downloader, nil
+}
+func (s *Sync) getS3Uploader() (*s3manager.Uploader, error) {
+	creds := credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, "")
+	s3Session, err := session.NewSession(&aws.Config{Region: aws.String(s.AwsS3Region), Credentials: creds})
+	if err != nil {
+		return nil, errors.Prefix("error starting session: ", err)
+	}
+	uploader := s3manager.NewUploader(s3Session)
+	return uploader, nil
+}
+
+func (s *Sync) downloadWallet() error {
+	defaultWalletDir, defaultTempWalletDir, key, err := s.getWalletPaths()
+	if err != nil {
+		return errors.Err(err)
+	}
+	downloader, err := s.getS3Downloader()
+	if err != nil {
+		return err
+	}
+	out, err := os.Create(defaultTempWalletDir)
+	if err != nil {
+		return errors.Prefix("error creating temp wallet: ", err)
+	}
+	defer out.Close()
+
+	bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
+		Bucket: aws.String(s.AwsS3Bucket),
+		Key:    key,
+	})
+	if err != nil {
+		// Casting to the awserr.Error type will allow you to inspect the error
+		// code returned by the service in code. The error code can be used
+		// to switch on context specific functionality. In this case a context
+		// specific error message is printed to the user based on the bucket
+		// and key existing.
+		//
+		// For information on other S3 API error codes see:
+		// http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
+		if aerr, ok := err.(awserr.Error); ok {
+			code := aerr.Code()
+			if code == s3.ErrCodeNoSuchKey {
+				return errors.Err("wallet not on S3")
+			}
+		}
+		return err
+	} else if bytesWritten == 0 {
+		return errors.Err("zero bytes written")
+	}
+
+	err = os.Rename(defaultTempWalletDir, defaultWalletDir)
+	if err != nil {
+		return errors.Prefix("error replacing temp wallet for default wallet: ", err)
+	}
+
+	return nil
+}
+
+func (s *Sync) downloadBlockchainDB() error {
+	defaultBDBDir, defaultTempBDBDir, key, err := s.getBlockchainDBPaths()
+	if err != nil {
+		return errors.Err(err)
+	}
+	_ = os.Remove(defaultBDBDir)
+
+	downloader, err := s.getS3Downloader()
+	if err != nil {
+		return errors.Err(err)
+	}
+	out, err := os.Create(defaultTempBDBDir)
+	if err != nil {
+		return errors.Prefix("error creating temp wallet: ", err)
+	}
+	defer out.Close()
+
+	bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
+		Bucket: aws.String(s.AwsS3Bucket),
+		Key:    key,
+	})
+	if err != nil {
+		// Casting to the awserr.Error type will allow you to inspect the error
+		// code returned by the service in code. The error code can be used
+		// to switch on context specific functionality. In this case a context
+		// specific error message is printed to the user based on the bucket
+		// and key existing.
+		//
+		// For information on other S3 API error codes see:
+		// http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
+		if aerr, ok := err.(awserr.Error); ok {
+			code := aerr.Code()
+			if code == s3.ErrCodeNoSuchKey {
+				return nil // let ytsync sync the database by itself
+			}
+		}
+		return errors.Err(err)
+	} else if bytesWritten == 0 {
+		return errors.Err("zero bytes written")
+	}
+
+	err = os.Rename(defaultTempBDBDir, defaultBDBDir)
+	if err != nil {
+		return errors.Prefix("error replacing temp blockchain.db for default blockchain.db: ", err)
+	}
+
+	return nil
+}
+
+func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string, err error) {
+	defaultWallet = os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
+	tempWallet = os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet"
+	key = aws.String("/wallets/" + s.YoutubeChannelID)
+	if logUtils.IsRegTest() {
+		defaultWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
+		tempWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet"
+		key = aws.String("/regtest/" + s.YoutubeChannelID)
+	}
+
+	lbryumDir := os.Getenv("LBRYUM_DIR")
+	if lbryumDir != "" {
+		defaultWallet = lbryumDir + "/wallets/default_wallet"
+		tempWallet = lbryumDir + "/wallets/tmp_wallet"
+	}
+
+	if _, err := os.Stat(defaultWallet); !os.IsNotExist(err) {
+		return "", "", nil, errors.Err("default_wallet already exists")
+	}
+	return
+}
+
+func (s *Sync) getBlockchainDBPaths() (defaultDB, tempDB string, key *string, err error) {
+	lbryumDir := os.Getenv("LBRYUM_DIR")
+	if lbryumDir == "" {
+		if logUtils.IsRegTest() {
+			lbryumDir = os.Getenv("HOME") + "/.lbryum_regtest"
+		} else {
+			lbryumDir = os.Getenv("HOME") + "/.lbryum"
+		}
+	}
+	defaultDB = lbryumDir + "/lbc_mainnet/blockchain.db"
+	tempDB = lbryumDir + "/lbc_mainnet/tmp_blockchain.db"
+	key = aws.String("/blockchain_dbs/" + s.YoutubeChannelID)
+	if logUtils.IsRegTest() {
+		defaultDB = lbryumDir + "/lbc_regtest/blockchain.db"
+		tempDB = lbryumDir + "/lbc_regtest/tmp_blockchain.db"
+		key = aws.String("/regtest_dbs/" + s.YoutubeChannelID)
+	}
+	return
+}
+
+func (s *Sync) uploadWallet() error {
+	defaultWalletDir := logUtils.GetDefaultWalletPath()
+	key := aws.String("/wallets/" + s.YoutubeChannelID)
+	if logUtils.IsRegTest() {
+		key = aws.String("/regtest/" + s.YoutubeChannelID)
+	}
+
+	if _, err := os.Stat(defaultWalletDir); os.IsNotExist(err) {
+		return errors.Err("default_wallet does not exist")
+	}
+
+	uploader, err := s.getS3Uploader()
+	if err != nil {
+		return err
+	}
+
+	file, err := os.Open(defaultWalletDir)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	_, err = uploader.Upload(&s3manager.UploadInput{
+		Bucket: aws.String(s.AwsS3Bucket),
+		Key:    key,
+		Body:   file,
+	})
+	if err != nil {
+		return err
+	}
+
+	return os.Remove(defaultWalletDir)
+}
+
+func (s *Sync) uploadBlockchainDB() error {
+	defaultBDBDir, _, key, err := s.getBlockchainDBPaths()
+	if err != nil {
+		return errors.Err(err)
+	}
+
+	if _, err := os.Stat(defaultBDBDir); os.IsNotExist(err) {
+		return errors.Err("blockchain.db does not exist")
+	}
+
+	uploader, err := s.getS3Uploader()
+	if err != nil {
+		return err
+	}
+
+	file, err := os.Open(defaultBDBDir)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	_, err = uploader.Upload(&s3manager.UploadInput{
+		Bucket: aws.String(s.AwsS3Bucket),
+		Key:    key,
+		Body:   file,
+	})
+	if err != nil {
+		return err
+	}
+
+	return os.Remove(defaultBDBDir)
+}
diff --git a/manager/setup.go b/manager/setup.go
index f37c682..af2eaad 100644
--- a/manager/setup.go
+++ b/manager/setup.go
@@ -383,20 +383,20 @@ func (s *Sync) ensureChannelOwnership() error {
 		}
 	}
 
-	channelInfo, channelBranding, err := ytapi.ChannelInfo(s.APIConfig.YoutubeAPIKey, s.YoutubeChannelID)
+	channelInfo, err := ytapi.ChannelInfo(s.YoutubeChannelID)
 	if err != nil {
 		return err
 	}
 
-	thumbnail := thumbs.GetBestThumbnail(nil) //Ignore this for now ( ChannelInfo Panics)
-	thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.URL, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
+	thumbnail := channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails)-1].URL
+	thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
 	if err != nil {
 		return err
 	}
 
 	var bannerURL *string
-	if channelBranding.Image != nil && channelBranding.Image.BannerImageUrl != "" {
-		bURL, err := thumbs.MirrorThumbnail(channelBranding.Image.BannerImageUrl, "banner-"+s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
+	if channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails != nil {
+		bURL, err := thumbs.MirrorThumbnail(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails)-1].URL, "banner-"+s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
 		if err != nil {
 			return err
 		}
@@ -404,20 +404,21 @@ func (s *Sync) ensureChannelOwnership() error {
 	}
 
 	var languages []string = nil
-	if channelInfo.DefaultLanguage != "" {
-		if channelInfo.DefaultLanguage == "iw" {
-			channelInfo.DefaultLanguage = "he"
-		}
-		languages = []string{channelInfo.DefaultLanguage}
-	}
+	//we don't have this data without the API
+	//if channelInfo.DefaultLanguage != "" {
+	//	if channelInfo.DefaultLanguage == "iw" {
+	//		channelInfo.DefaultLanguage = "he"
+	//	}
+	//	languages = []string{channelInfo.DefaultLanguage}
+	//}
 	var locations []jsonrpc.Location = nil
-	if channelInfo.Country != "" {
-		locations = []jsonrpc.Location{{Country: util.PtrToString(channelInfo.Country)}}
+	if channelInfo.Topbar.DesktopTopbarRenderer.CountryCode != "" {
+		locations = []jsonrpc.Location{{Country: util.PtrToString(channelInfo.Topbar.DesktopTopbarRenderer.CountryCode)}}
 	}
 	var c *jsonrpc.TransactionSummary
 	claimCreateOptions := jsonrpc.ClaimCreateOptions{
-		Title:        &channelInfo.Title,
-		Description:  &channelInfo.Description,
+		Title:        &channelInfo.Microformat.MicroformatDataRenderer.Title,
+		Description:  &channelInfo.Microformat.MicroformatDataRenderer.Description,
 		Tags:         tags_manager.GetTagsForChannel(s.YoutubeChannelID),
 		Languages:    languages,
 		Locations:    locations,
diff --git a/manager/ytsync.go b/manager/ytsync.go
index e32dae5..4339656 100644
--- a/manager/ytsync.go
+++ b/manager/ytsync.go
@@ -26,12 +26,6 @@ import (
 	"github.com/lbryio/lbry.go/v2/extras/stop"
 	"github.com/lbryio/lbry.go/v2/extras/util"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/awserr"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -100,116 +94,6 @@ func (s *Sync) IsInterrupted() bool {
 	}
 }
 
-func (s *Sync) downloadWallet() error {
-	defaultWalletDir, defaultTempWalletDir, key, err := s.getWalletPaths()
-	if err != nil {
-		return errors.Err(err)
-	}
-
-	creds := credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, "")
-	s3Session, err := session.NewSession(&aws.Config{Region: aws.String(s.AwsS3Region), Credentials: creds})
-	if err != nil {
-		return errors.Prefix("error starting session: ", err)
-	}
-	downloader := s3manager.NewDownloader(s3Session)
-	out, err := os.Create(defaultTempWalletDir)
-	if err != nil {
-		return errors.Prefix("error creating temp wallet: ", err)
-	}
-	defer out.Close()
-
-	bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
-		Bucket: aws.String(s.AwsS3Bucket),
-		Key:    key,
-	})
-	if err != nil {
-		// Casting to the awserr.Error type will allow you to inspect the error
-		// code returned by the service in code. The error code can be used
-		// to switch on context specific functionality. In this case a context
-		// specific error message is printed to the user based on the bucket
-		// and key existing.
-		//
-		// For information on other S3 API error codes see:
-		// http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
-		if aerr, ok := err.(awserr.Error); ok {
-			code := aerr.Code()
-			if code == s3.ErrCodeNoSuchKey {
-				return errors.Err("wallet not on S3")
-			}
-		}
-		return err
-	} else if bytesWritten == 0 {
-		return errors.Err("zero bytes written")
-	}
-
-	err = os.Rename(defaultTempWalletDir, defaultWalletDir)
-	if err != nil {
-		return errors.Prefix("error replacing temp wallet for default wallet: ", err)
-	}
-
-	return nil
-}
-
-func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string, err error) {
-
-	defaultWallet = os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
-	tempWallet = os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet"
-	key = aws.String("/wallets/" + s.YoutubeChannelID)
-	if logUtils.IsRegTest() {
-		defaultWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
-		tempWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet"
-		key = aws.String("/regtest/" + s.YoutubeChannelID)
-	}
-
-	walletPath := os.Getenv("LBRYNET_WALLETS_DIR")
-	if walletPath != "" {
-		defaultWallet = walletPath + "/wallets/default_wallet"
-		tempWallet = walletPath + "/wallets/tmp_wallet"
-	}
-
-	if _, err := os.Stat(defaultWallet); !os.IsNotExist(err) {
-		return "", "", nil, errors.Err("default_wallet already exists")
-	}
-	return
-}
-
-func (s *Sync) uploadWallet() error {
-	defaultWalletDir := logUtils.GetDefaultWalletPath()
-	key := aws.String("/wallets/" + s.YoutubeChannelID)
-	if logUtils.IsRegTest() {
-		key = aws.String("/regtest/" + s.YoutubeChannelID)
-	}
-
-	if _, err := os.Stat(defaultWalletDir); os.IsNotExist(err) {
-		return errors.Err("default_wallet does not exist")
-	}
-
-	creds := credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, "")
-	s3Session, err := session.NewSession(&aws.Config{Region: aws.String(s.AwsS3Region), Credentials: creds})
-	if err != nil {
-		return err
-	}
-
-	uploader := s3manager.NewUploader(s3Session)
-
-	file, err := os.Open(defaultWalletDir)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	_, err = uploader.Upload(&s3manager.UploadInput{
-		Bucket: aws.String(s.AwsS3Bucket),
-		Key:    key,
-		Body:   file,
-	})
-	if err != nil {
-		return err
-	}
-
-	return os.Remove(defaultWalletDir)
-}
-
 func (s *Sync) setStatusSyncing() error {
 	syncedVideos, claimNames, err := s.Manager.apiConfig.SetChannelStatus(s.YoutubeChannelID, StatusSyncing, "", nil)
 	if err != nil {
@@ -270,6 +154,10 @@ func (s *Sync) FullCycle() (e error) {
 	} else {
 		log.Println("Starting new wallet")
 	}
+	err = s.downloadBlockchainDB()
+	if err != nil {
+		return errors.Prefix("failure in downloading blockchain.db", err)
+	}
 
 	defer s.stopAndUploadWallet(&e)
 
@@ -450,7 +338,14 @@ func (s *Sync) stopAndUploadWallet(e *error) {
 			if err != nil {
 				if *e == nil {
 					e = &err
-					return
+				} else {
+					*e = errors.Prefix("failure uploading wallet", *e)
+				}
+			}
+			err = s.uploadBlockchainDB()
+			if err != nil {
+				if *e == nil {
+					e = &err
 				} else {
 					*e = errors.Prefix("failure uploading wallet", *e)
 				}
diff --git a/util/util.go b/util/util.go
index cfe6fb6..d3b1ab1 100644
--- a/util/util.go
+++ b/util/util.go
@@ -52,7 +52,7 @@ func GetLBRYNetDir() string {
 }
 
 func GetLbryumDir() string {
-	lbryumDir := os.Getenv("LBRYNET_WALLETS_DIR")
+	lbryumDir := os.Getenv("LBRYUM_DIR")
 	if lbryumDir == "" {
 		usr, err := user.Current()
 		if err != nil {
@@ -366,9 +366,31 @@ func GetDefaultWalletPath() string {
 		defaultWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
 	}
 
-	walletPath := os.Getenv("LBRYNET_WALLETS_DIR")
+	walletPath := os.Getenv("LBRYUM_DIR")
 	if walletPath != "" {
 		defaultWalletDir = walletPath + "/wallets/default_wallet"
 	}
 	return defaultWalletDir
 }
+func GetBlockchainDBPath() string {
+	lbryumDir := os.Getenv("LBRYUM_DIR")
+	if lbryumDir == "" {
+		if IsRegTest() {
+			lbryumDir = os.Getenv("HOME") + "/.lbryum_regtest"
+		} else {
+			lbryumDir = os.Getenv("HOME") + "/.lbryum"
+		}
+	}
+	defaultDB := lbryumDir + "/lbc_mainnet/blockchain.db"
+	if IsRegTest() {
+		defaultDB = lbryumDir + "/lbc_regtest/blockchain.db"
+	}
+	return defaultDB
+}
+func GetBlockchainDirectoryName() string {
+	ledger := "lbc_mainnet"
+	if IsRegTest() {
+		ledger = "lbc_regtest"
+	}
+	return ledger
+}
diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go
index 6b98406..5b07d37 100644
--- a/ytapi/ytapi.go
+++ b/ytapi/ytapi.go
@@ -2,6 +2,8 @@ package ytapi
 
 import (
 	"bufio"
+	"encoding/json"
+	"io/ioutil"
 	"net/http"
 	"regexp"
 	"sort"
@@ -25,8 +27,6 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	log "github.com/sirupsen/logrus"
-	"google.golang.org/api/googleapi/transport"
-	ytlib "google.golang.org/api/youtube/v3"
 )
 
 type Video interface {
@@ -56,7 +56,7 @@ var mostRecentlyFailedChannel string // TODO: fix this hack!
 func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams) ([]Video, error) {
 
 	var videos []Video
-	if quickSync {
+	if quickSync && maxVideos > 50 {
 		maxVideos = 50
 	}
 	videoIDs, err := downloader.GetPlaylistVideoIDs(channelID, maxVideos, videoParams.Stopper.Ch(), videoParams.IPPool)
@@ -143,24 +143,36 @@ func CountVideosInChannel(channelID string) (int, error) {
 	return num, nil
 }
 
-func ChannelInfo(apiKey, channelID string) (*ytlib.ChannelSnippet, *ytlib.ChannelBrandingSettings, error) {
-	return nil, nil, errors.Err("ChannelInfo doesn't work yet because we're focused on existing channels")
+func ChannelInfo(channelID string) (*YoutubeStatsResponse, error) {
+	//return nil, nil, errors.Err("ChannelInfo doesn't work yet because we're focused on existing channels")
+	url := "https://www.youtube.com/channel/" + channelID + "/about"
 
-	service, err := ytlib.New(&http.Client{Transport: &transport.APIKey{Key: apiKey}})
+	req, _ := http.NewRequest("GET", url, nil)
+
+	req.Header.Add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36")
+	req.Header.Add("Accept", "*/*")
+
+	res, err := http.DefaultClient.Do(req)
 	if err != nil {
-		return nil, nil, errors.Prefix("error creating YouTube service", err)
+		return nil, errors.Err(err)
 	}
-
-	response, err := service.Channels.List("snippet,brandingSettings").Id(channelID).Do()
+	defer res.Body.Close()
+	body, err := ioutil.ReadAll(res.Body)
 	if err != nil {
-		return nil, nil, errors.Prefix("error getting channel details", err)
+		return nil, errors.Err(err)
+	}
+	pageBody := string(body)
+	dataStartIndex := strings.Index(pageBody, "window[\"ytInitialData\"] = ") + 26
+	dataEndIndex := strings.Index(pageBody, "]}}};") + 4
+
+	data := pageBody[dataStartIndex:dataEndIndex]
+	var decodedResponse YoutubeStatsResponse
+	err = json.Unmarshal([]byte(data), &decodedResponse)
+	if err != nil {
+		return nil, errors.Err(err)
 	}
 
-	if len(response.Items) < 1 {
-		return nil, nil, errors.Err("youtube channel not found")
-	}
-
-	return response.Items[0].Snippet, response.Items[0].BrandingSettings, nil
+	return &decodedResponse, nil
 }
 
 func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool) ([]*ytdl.YtdlVideo, error) {
@@ -205,3 +217,147 @@ func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopC
 	}
 	return videos, nil
 }
+
+type YoutubeStatsResponse struct {
+	Contents struct {
+		TwoColumnBrowseResultsRenderer struct {
+			Tabs []struct {
+				TabRenderer struct {
+					Title    string `json:"title"`
+					Selected bool   `json:"selected"`
+					Content  struct {
+						SectionListRenderer struct {
+							Contents []struct {
+								ItemSectionRenderer struct {
+									Contents []struct {
+										ChannelAboutFullMetadataRenderer struct {
+											Description struct {
+												SimpleText string `json:"simpleText"`
+											} `json:"description"`
+											ViewCountText struct {
+												SimpleText string `json:"simpleText"`
+											} `json:"viewCountText"`
+											JoinedDateText struct {
+												Runs []struct {
+													Text string `json:"text"`
+												} `json:"runs"`
+											} `json:"joinedDateText"`
+											CanonicalChannelURL        string `json:"canonicalChannelUrl"`
+											BypassBusinessEmailCaptcha bool   `json:"bypassBusinessEmailCaptcha"`
+											Title                      struct {
+												SimpleText string `json:"simpleText"`
+											} `json:"title"`
+											Avatar struct {
+												Thumbnails []struct {
+													URL    string `json:"url"`
+													Width  int    `json:"width"`
+													Height int    `json:"height"`
+												} `json:"thumbnails"`
+											} `json:"avatar"`
+											ShowDescription  bool `json:"showDescription"`
+											DescriptionLabel struct {
+												Runs []struct {
+													Text string `json:"text"`
+												} `json:"runs"`
+											} `json:"descriptionLabel"`
+											DetailsLabel struct {
+												Runs []struct {
+													Text string `json:"text"`
+												} `json:"runs"`
+											} `json:"detailsLabel"`
+											ChannelID string `json:"channelId"`
+										} `json:"channelAboutFullMetadataRenderer"`
+									} `json:"contents"`
+								} `json:"itemSectionRenderer"`
+							} `json:"contents"`
+						} `json:"sectionListRenderer"`
+					} `json:"content"`
+				} `json:"tabRenderer"`
+			} `json:"tabs"`
+		} `json:"twoColumnBrowseResultsRenderer"`
+	} `json:"contents"`
+	Header struct {
+		C4TabbedHeaderRenderer struct {
+			ChannelID string `json:"channelId"`
+			Title     string `json:"title"`
+			Avatar    struct {
+				Thumbnails []struct {
+					URL    string `json:"url"`
+					Width  int    `json:"width"`
+					Height int    `json:"height"`
+				} `json:"thumbnails"`
+			} `json:"avatar"`
+			Banner struct {
+				Thumbnails []struct {
+					URL    string `json:"url"`
+					Width  int    `json:"width"`
+					Height int    `json:"height"`
+				} `json:"thumbnails"`
+			} `json:"banner"`
+			VisitTracking struct {
+				RemarketingPing string `json:"remarketingPing"`
+			} `json:"visitTracking"`
+			SubscriberCountText struct {
+				SimpleText string `json:"simpleText"`
+			} `json:"subscriberCountText"`
+		} `json:"c4TabbedHeaderRenderer"`
+	} `json:"header"`
+	Metadata struct {
+		ChannelMetadataRenderer struct {
+			Title                string   `json:"title"`
+			Description          string   `json:"description"`
+			RssURL               string   `json:"rssUrl"`
+			ChannelConversionURL string   `json:"channelConversionUrl"`
+			ExternalID           string   `json:"externalId"`
+			Keywords             string   `json:"keywords"`
+			OwnerUrls            []string `json:"ownerUrls"`
+			Avatar               struct {
+				Thumbnails []struct {
+					URL    string `json:"url"`
+					Width  int    `json:"width"`
+					Height int    `json:"height"`
+				} `json:"thumbnails"`
+			} `json:"avatar"`
+			ChannelURL       string `json:"channelUrl"`
+			IsFamilySafe     bool   `json:"isFamilySafe"`
+			VanityChannelURL string `json:"vanityChannelUrl"`
+		} `json:"channelMetadataRenderer"`
+	} `json:"metadata"`
+	Topbar struct {
+		DesktopTopbarRenderer struct {
+			CountryCode string `json:"countryCode"`
+		} `json:"desktopTopbarRenderer"`
+	} `json:"topbar"`
+	Microformat struct {
+		MicroformatDataRenderer struct {
+			URLCanonical string `json:"urlCanonical"`
+			Title        string `json:"title"`
+			Description  string `json:"description"`
+			Thumbnail    struct {
+				Thumbnails []struct {
+					URL    string `json:"url"`
+					Width  int    `json:"width"`
+					Height int    `json:"height"`
+				} `json:"thumbnails"`
+			} `json:"thumbnail"`
+			SiteName           string   `json:"siteName"`
+			AppName            string   `json:"appName"`
+			AndroidPackage     string   `json:"androidPackage"`
+			IosAppStoreID      string   `json:"iosAppStoreId"`
+			IosAppArguments    string   `json:"iosAppArguments"`
+			OgType             string   `json:"ogType"`
+			URLApplinksWeb     string   `json:"urlApplinksWeb"`
+			URLApplinksIos     string   `json:"urlApplinksIos"`
+			URLApplinksAndroid string   `json:"urlApplinksAndroid"`
+			URLTwitterIos      string   `json:"urlTwitterIos"`
+			URLTwitterAndroid  string   `json:"urlTwitterAndroid"`
+			TwitterCardType    string   `json:"twitterCardType"`
+			TwitterSiteHandle  string   `json:"twitterSiteHandle"`
+			SchemaDotOrgType   string   `json:"schemaDotOrgType"`
+			Noindex            bool     `json:"noindex"`
+			Unlisted           bool     `json:"unlisted"`
+			FamilySafe         bool     `json:"familySafe"`
+			Tags               []string `json:"tags"`
+		} `json:"microformatDataRenderer"`
+	} `json:"microformat"`
+}
diff --git a/ytapi/ytapi_test.go b/ytapi/ytapi_test.go
new file mode 100644
index 0000000..93a612c
--- /dev/null
+++ b/ytapi/ytapi_test.go
@@ -0,0 +1,12 @@
+package ytapi
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestChannelInfo(t *testing.T) {
+	_, _, err := ChannelInfo("", "UCNQfQvFMPnInwsU_iGYArJQ")
+	assert.NoError(t, err)
+}

From eab9bcf1ffd1d9a689cb25867286d58249b569a6 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Tue, 4 Aug 2020 01:23:05 +0200
Subject: [PATCH 35/57] don't wait for confirmations unnecessarily

fix sqlite crap
---
 manager/s3_storage.go | 12 +++++++++++-
 manager/ytsync.go     |  8 +++++---
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/manager/s3_storage.go b/manager/s3_storage.go
index cccb5a1..d682b9c 100644
--- a/manager/s3_storage.go
+++ b/manager/s3_storage.go
@@ -2,6 +2,7 @@ package manager
 
 import (
 	"os"
+	"path/filepath"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
@@ -86,7 +87,16 @@ func (s *Sync) downloadBlockchainDB() error {
 	if err != nil {
 		return errors.Err(err)
 	}
-	_ = os.Remove(defaultBDBDir)
+	files, err := filepath.Glob(defaultBDBDir + "*")
+	if err != nil {
+		return errors.Err(err)
+	}
+	for _, f := range files {
+		err = os.Remove(f)
+		if err != nil {
+			return errors.Err(err)
+		}
+	}
 
 	downloader, err := s.getS3Downloader()
 	if err != nil {
diff --git a/manager/ytsync.go b/manager/ytsync.go
index 4339656..0b45e02 100644
--- a/manager/ytsync.go
+++ b/manager/ytsync.go
@@ -200,9 +200,11 @@ func (s *Sync) FullCycle() (e error) {
 
 func (s *Sync) processTransfers() (e error) {
 	log.Println("Processing transfers")
-	err := waitConfirmations(s)
-	if err != nil {
-		return err
+	if s.transferState != 2 {
+		err := waitConfirmations(s)
+		if err != nil {
+			return err
+		}
 	}
 	supportAmount, err := abandonSupports(s)
 	if err != nil {

From 6c6e93cefc27b7ff388b34908e9ecf433ecab630 Mon Sep 17 00:00:00 2001
From: Alex Grintsvayg <grin@lbry.com>
Date: Wed, 5 Aug 2020 15:48:13 -0400
Subject: [PATCH 36/57] go mod tidy

---
 go.mod |  1 +
 go.sum | 11 ++++-------
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/go.mod b/go.mod
index 914602c..22196db 100644
--- a/go.mod
+++ b/go.mod
@@ -30,6 +30,7 @@ require (
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/testify v1.4.0
+	golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
 	google.golang.org/appengine v1.6.5 // indirect
 )
diff --git a/go.sum b/go.sum
index 367f184..f8386b4 100644
--- a/go.sum
+++ b/go.sum
@@ -294,13 +294,6 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
-golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -316,6 +309,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191009170851-d66e71096ffb h1:TR699M2v0qoKTOHxeLgp6zPqaQNs74f01a/ob9W0qko=
 golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -339,6 +334,8 @@ golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f h1:hjzMYz/7Ea1mNKfOnFOfktR0mlA5jqhvywClCMHM/qw=
 golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=

From ec9f46f5526fb07454f20261db199e3978d572bc Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Thu, 6 Aug 2020 00:25:54 +0200
Subject: [PATCH 37/57] fix release date bug

---
 downloader/downloader.go | 15 ++++++++-------
 ytapi/ytapi.go           |  1 -
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/downloader/downloader.go b/downloader/downloader.go
index eba0c1a..0dc12de 100644
--- a/downloader/downloader.go
+++ b/downloader/downloader.go
@@ -163,19 +163,20 @@ func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr, uploa
 	//slack("Getting upload time for %s", videoID)
 	release, err := config.GetReleasedDate(videoID)
 	if err != nil {
-		if release != nil {
-			const sqlTimeFormat = "2006-01-02 15:04:05"
-			sqlTime, err := time.ParseInLocation(sqlTimeFormat, release.ReleaseTime, time.UTC)
-			if err != nil {
-				return sqlTime.Format(releaseTimeFormat), nil
-			}
+		logrus.Error(err)
+	}
+	if release != nil {
+		const sqlTimeFormat = "2006-01-02 15:04:05"
+		sqlTime, err := time.ParseInLocation(sqlTimeFormat, release.ReleaseTime, time.UTC)
+		if err == nil {
+			return sqlTime.Format(releaseTimeFormat), nil
 		}
 	}
 	ytdlUploadDate, err := time.Parse("20060102", uploadDate)
 	if err != nil {
 		logrus.Error(err)
 	}
-	if time.Now().Add(-5 * 24 * time.Hour).After(ytdlUploadDate) {
+	if time.Now().AddDate(0, 0, -5).After(ytdlUploadDate) {
 		return ytdlUploadDate.Format(releaseTimeFormat), nil
 	}
 	client := getClient(ip)
diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go
index 5b07d37..a15d8a6 100644
--- a/ytapi/ytapi.go
+++ b/ytapi/ytapi.go
@@ -144,7 +144,6 @@ func CountVideosInChannel(channelID string) (int, error) {
 }
 
 func ChannelInfo(channelID string) (*YoutubeStatsResponse, error) {
-	//return nil, nil, errors.Err("ChannelInfo doesn't work yet because we're focused on existing channels")
 	url := "https://www.youtube.com/channel/" + channelID + "/about"
 
 	req, _ := http.NewRequest("GET", url, nil)

From 775e4881cbde9d1e362dbc3ce11f674b54b15712 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Thu, 6 Aug 2020 00:30:48 +0200
Subject: [PATCH 38/57] invert logic so that we don't flood slack

---
 downloader/downloader.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/downloader/downloader.go b/downloader/downloader.go
index 0dc12de..3d8eb57 100644
--- a/downloader/downloader.go
+++ b/downloader/downloader.go
@@ -84,10 +84,10 @@ GetTime:
 		if err != nil {
 			return nil, errors.Err(err)
 		}
-		slack(":exclamation: Got an accurate time for %s", videoID)
+		//slack(":exclamation: Got an accurate time for %s", videoID)
 		video.UploadDateForReal = parsed
 	} else {
-		//slack(":warning: Could not get accurate time for %s. Falling back to time from upload ytdl: %s.", videoID, video.UploadDate)
+		slack(":warning: Could not get accurate time for %s. Falling back to time from upload ytdl: %s.", videoID, video.UploadDate)
 		// fall back to UploadDate from youtube-dl
 		video.UploadDateForReal, err = time.Parse("20060102", video.UploadDate)
 		if err != nil {

From f0280b51b448c61cb1402d298ec31a59bb875adc Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Thu, 6 Aug 2020 02:12:05 +0200
Subject: [PATCH 39/57] fix time parsing for real

---
 downloader/downloader.go      | 28 +++++++++++++++++-----------
 downloader/downloader_test.go | 19 +++++++++++++++++--
 sdk/api.go                    |  2 +-
 sources/youtubeVideo.go       |  1 -
 ytapi/ytapi.go                |  2 +-
 5 files changed, 36 insertions(+), 16 deletions(-)

diff --git a/downloader/downloader.go b/downloader/downloader.go
index 3d8eb57..2c92c1e 100644
--- a/downloader/downloader.go
+++ b/downloader/downloader.go
@@ -73,7 +73,11 @@ GetTime:
 			}
 		} else if !errors.Is(err, errNotScraped) && !errors.Is(err, errUploadTimeEmpty) {
 			//slack(":warning: Error while trying to get accurate upload time for %s: %v", videoID, err)
-			return nil, errors.Err(err)
+			if t == "" {
+				return nil, errors.Err(err)
+			} else {
+				t = "" //TODO: get rid of the other piece below?
+			}
 		}
 		// do fallback below
 	}
@@ -86,7 +90,7 @@ GetTime:
 		}
 		//slack(":exclamation: Got an accurate time for %s", videoID)
 		video.UploadDateForReal = parsed
-	} else {
+	} else { //TODO: this is the piece that isn't needed!
 		slack(":warning: Could not get accurate time for %s. Falling back to time from upload ytdl: %s.", videoID, video.UploadDate)
 		// fall back to UploadDate from youtube-dl
 		video.UploadDateForReal, err = time.Parse("20060102", video.UploadDate)
@@ -166,29 +170,31 @@ func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr, uploa
 		logrus.Error(err)
 	}
 	if release != nil {
-		const sqlTimeFormat = "2006-01-02 15:04:05"
-		sqlTime, err := time.ParseInLocation(sqlTimeFormat, release.ReleaseTime, time.UTC)
+		//const sqlTimeFormat = "2006-01-02 15:04:05"
+		sqlTime, err := time.ParseInLocation(time.RFC3339, release.ReleaseTime, time.UTC)
 		if err == nil {
 			return sqlTime.Format(releaseTimeFormat), nil
+		} else {
+			logrus.Error(err)
 		}
 	}
 	ytdlUploadDate, err := time.Parse("20060102", uploadDate)
 	if err != nil {
 		logrus.Error(err)
 	}
-	if time.Now().AddDate(0, 0, -5).After(ytdlUploadDate) {
+	if time.Now().AddDate(0, 0, -2).After(ytdlUploadDate) {
 		return ytdlUploadDate.Format(releaseTimeFormat), nil
 	}
 	client := getClient(ip)
 	req, err := http.NewRequest(http.MethodGet, "https://caa.iti.gr/get_verificationV3?url=https://www.youtube.com/watch?v="+videoID, nil)
 	if err != nil {
-		return "", errors.Err(err)
+		return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
 	}
 	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36")
 
 	res, err := client.Do(req)
 	if err != nil {
-		return "", errors.Err(err)
+		return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
 	}
 	defer res.Body.Close()
 
@@ -199,19 +205,19 @@ func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr, uploa
 	}
 	err = json.NewDecoder(res.Body).Decode(&uploadTime)
 	if err != nil {
-		return "", errors.Err(err)
+		return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
 	}
 
 	if uploadTime.Status == "ERROR1" {
-		return "", errNotScraped
+		return ytdlUploadDate.Format(releaseTimeFormat), errNotScraped
 	}
 
 	if uploadTime.Status == "" && strings.HasPrefix(uploadTime.Message, "CANNOT_RETRIEVE_REPORT_FOR_VIDEO_") {
-		return "", errors.Err("cannot retrieve report for video")
+		return ytdlUploadDate.Format(releaseTimeFormat), errors.Err("cannot retrieve report for video")
 	}
 
 	if uploadTime.Time == "" {
-		return "", errUploadTimeEmpty
+		return ytdlUploadDate.Format(releaseTimeFormat), errUploadTimeEmpty
 	}
 
 	return uploadTime.Time, nil
diff --git a/downloader/downloader_test.go b/downloader/downloader_test.go
index 1f065e6..f4e981a 100644
--- a/downloader/downloader_test.go
+++ b/downloader/downloader_test.go
@@ -3,11 +3,13 @@ package downloader
 import (
 	"testing"
 
+	"github.com/lbryio/ytsync/v5/sdk"
 	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestGetPlaylistVideoIDs(t *testing.T) {
-	videoIDs, err := GetPlaylistVideoIDs("UCJ0-OtVpF0wOKEqT2Z1HEtA", 50)
+	videoIDs, err := GetPlaylistVideoIDs("UCJ0-OtVpF0wOKEqT2Z1HEtA", 50, nil, nil)
 	if err != nil {
 		logrus.Error(err)
 	}
@@ -17,7 +19,7 @@ func TestGetPlaylistVideoIDs(t *testing.T) {
 }
 
 func TestGetVideoInformation(t *testing.T) {
-	video, err := GetVideoInformation("zj7pXM9gE5M", nil)
+	video, err := GetVideoInformation(nil, "zj7pXM9gE5M", nil, nil, nil)
 	if err != nil {
 		logrus.Error(err)
 	}
@@ -25,3 +27,16 @@ func TestGetVideoInformation(t *testing.T) {
 		logrus.Info(video.ID)
 	}
 }
+
+func Test_getUploadTime(t *testing.T) {
+	configs := sdk.APIConfig{
+		YoutubeAPIKey: "",
+		ApiURL:        "https://api.lbry.com",
+		ApiToken:      "Ht4NETrL5oWKyAaZkuSV68BKhtXkiLh5",
+		HostName:      "test",
+	}
+	got, err := getUploadTime(&configs, "kDGOHNpRjzc", nil, "20060102")
+	assert.NoError(t, err)
+	t.Log(got)
+
+}
diff --git a/sdk/api.go b/sdk/api.go
index f6cb2a6..233f463 100644
--- a/sdk/api.go
+++ b/sdk/api.go
@@ -413,7 +413,7 @@ type VideoRelease struct {
 	ID            uint64 `json:"id"`
 	YoutubeDataID uint64 `json:"youtube_data_id"`
 	VideoID       string `json:"video_id"`
-	ReleaseTime   string `json:"release_time""`
+	ReleaseTime   string `json:"release_time"`
 	CreatedAt     string `json:"created_at"`
 	UpdatedAt     string `json:"updated_at"`
 }
diff --git a/sources/youtubeVideo.go b/sources/youtubeVideo.go
index e4bbc9f..639b394 100644
--- a/sources/youtubeVideo.go
+++ b/sources/youtubeVideo.go
@@ -452,7 +452,6 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
 	dur := time.Duration(v.youtubeInfo.Duration) * time.Second
 
 	if dur > v.maxVideoLength {
-		log.Infof("%s is %d long and the limit is %s", v.id, dur.String(), v.maxVideoLength.String())
 		logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, dur.String(), v.maxVideoLength.String())
 		return nil, errors.Err("video is too long to process")
 	}
diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go
index a15d8a6..ee43f88 100644
--- a/ytapi/ytapi.go
+++ b/ytapi/ytapi.go
@@ -206,7 +206,7 @@ func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopC
 				Status:        "failed",
 				FailureReason: err.Error(),
 			})
-			util.SendErrorToSlack("Skipping video: " + err.Error())
+			util.SendErrorToSlack("Skipping video: " + errors.FullTrace(err))
 			if errSDK != nil {
 				return nil, errors.Err(errSDK)
 			}

From 5d230a6b54f0d011bd2b7040657f92bc64061932 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Thu, 6 Aug 2020 20:32:49 +0200
Subject: [PATCH 40/57] potentially fix looping

add limits by db
---
 manager/manager.go |  6 ++++++
 manager/ytsync.go  |  3 ++-
 sdk/api.go         |  3 +++
 ytapi/ytapi.go     | 10 +++++++++-
 4 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/manager/manager.go b/manager/manager.go
index ac24ad7..9af86cb 100644
--- a/manager/manager.go
+++ b/manager/manager.go
@@ -127,6 +127,8 @@ func (s *SyncManager) Start() error {
 			}
 			lbryChannelName := channels[0].DesiredChannelName
 			syncs = make([]Sync, 1)
+			s.maxVideoLength = time.Duration(channels[0].LengthLimit) * time.Minute
+			s.maxVideoSize = channels[0].SizeLimit
 			syncs[0] = Sync{
 				APIConfig:            s.apiConfig,
 				YoutubeChannelID:     s.syncProperties.YoutubeChannelID,
@@ -147,6 +149,7 @@ func (s *SyncManager) Start() error {
 				clientPublishAddress: channels[0].PublishAddress,
 				publicKey:            channels[0].PublicKey,
 				transferState:        channels[0].TransferState,
+				LastUploadedVideo:    channels[0].LastUploadedVideo,
 			}
 			shouldInterruptLoop = true
 		} else {
@@ -174,6 +177,8 @@ func (s *SyncManager) Start() error {
 					if c.TotalSubscribers < 1000 {
 						maxVideoLength = 1 * time.Hour
 					}
+					maxVideoLength = time.Duration(c.LengthLimit) * time.Minute
+					s.maxVideoSize = c.SizeLimit
 					syncs = append(syncs, Sync{
 						APIConfig:            s.apiConfig,
 						YoutubeChannelID:     c.ChannelId,
@@ -194,6 +199,7 @@ func (s *SyncManager) Start() error {
 						clientPublishAddress: c.PublishAddress,
 						publicKey:            c.PublicKey,
 						transferState:        c.TransferState,
+						LastUploadedVideo:    c.LastUploadedVideo,
 					})
 					if q != StatusFailed {
 						continue queues
diff --git a/manager/ytsync.go b/manager/ytsync.go
index 0b45e02..b1611d3 100644
--- a/manager/ytsync.go
+++ b/manager/ytsync.go
@@ -68,6 +68,7 @@ type Sync struct {
 	publicKey            string
 	defaultAccountID     string
 	MaxVideoLength       time.Duration
+	LastUploadedVideo    string
 }
 
 func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) {
@@ -897,7 +898,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
 		S3Config: s.Manager.GetS3AWSConfig(),
 		Stopper:  s.grp,
 		IPPool:   ipPool,
-	})
+	}, s.LastUploadedVideo)
 	if err != nil {
 		return err
 	}
diff --git a/sdk/api.go b/sdk/api.go
index 233f463..c542cc8 100644
--- a/sdk/api.go
+++ b/sdk/api.go
@@ -63,6 +63,9 @@ type YoutubeChannel struct {
 	TransferState      int    `json:"transfer_state"`
 	PublishAddress     string `json:"publish_address"`
 	PublicKey          string `json:"public_key"`
+	LengthLimit        int    `json:"length_limit"`
+	SizeLimit          int    `json:"size_limit"`
+	LastUploadedVideo  string `json:"last_uploaded_video"`
 }
 
 func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeChannel, error) {
diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go
index ee43f88..2d19e58 100644
--- a/ytapi/ytapi.go
+++ b/ytapi/ytapi.go
@@ -53,7 +53,7 @@ type VideoParams struct {
 
 var mostRecentlyFailedChannel string // TODO: fix this hack!
 
-func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams) ([]Video, error) {
+func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams, lastUploadedVideo string) ([]Video, error) {
 
 	var videos []Video
 	if quickSync && maxVideos > 50 {
@@ -70,6 +70,14 @@ func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[s
 	for i, videoID := range videoIDs {
 		playlistMap[videoID] = int64(i)
 	}
+	//this will ensure that we at least try to sync the video that was marked as last uploaded video in the database.
+	if lastUploadedVideo != "" {
+		_, ok := playlistMap[lastUploadedVideo]
+		if !ok {
+			playlistMap[lastUploadedVideo] = 0
+			videoIDs = append(videoIDs, lastUploadedVideo)
+		}
+	}
 
 	if len(videoIDs) < 1 {
 		if channelID == mostRecentlyFailedChannel {

From b7037900f8217dc64900fc7b507a74dbcc109d9b Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Thu, 6 Aug 2020 23:53:03 +0200
Subject: [PATCH 41/57] fix thumbnail download

---
 thumbs/uploader.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/thumbs/uploader.go b/thumbs/uploader.go
index e1ff62e..b61a05e 100644
--- a/thumbs/uploader.go
+++ b/thumbs/uploader.go
@@ -4,6 +4,7 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"strings"
 
 	"github.com/lbryio/ytsync/v5/downloader/ytdl"
 
@@ -32,7 +33,9 @@ func (u *thumbnailUploader) downloadThumbnail() error {
 		return errors.Err(err)
 	}
 	defer img.Close()
-
+	if strings.HasPrefix(u.originalUrl, "//") {
+		u.originalUrl = "https:" + u.originalUrl
+	}
 	resp, err := http.Get(u.originalUrl)
 	if err != nil {
 		return errors.Err(err)

From 2a33f443177e0be3343e89b2e0d78d3efae273e0 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Fri, 7 Aug 2020 15:24:13 +0200
Subject: [PATCH 42/57] fix e2e tests

---
 e2e/data_setup.sh     | 4 ++--
 manager/s3_storage.go | 3 +++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/e2e/data_setup.sh b/e2e/data_setup.sh
index a6dfae4..9b1ee17 100755
--- a/e2e/data_setup.sh
+++ b/e2e/data_setup.sh
@@ -19,6 +19,6 @@ mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER"
 ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(2,"youtubertoken")'
 mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
 #Add their youtube channel to be synced
-ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable,total_views,reviewed,last_uploaded_video)
-VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCNQfQvFMPnInwsU_iGYArJQ','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',10,10,1,1,10000,1,'7bBV2Z-9wpo')"
+ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable,total_views,reviewed,last_uploaded_video,length_limit,size_limit)
+VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@test"$(date +%s)"','UCNQfQvFMPnInwsU_iGYArJQ','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',10,10,1,1,10000,1,'7bBV2Z-9wpo',60,2048)"
 mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL"
diff --git a/manager/s3_storage.go b/manager/s3_storage.go
index d682b9c..6c8e551 100644
--- a/manager/s3_storage.go
+++ b/manager/s3_storage.go
@@ -83,6 +83,9 @@ func (s *Sync) downloadWallet() error {
 }
 
 func (s *Sync) downloadBlockchainDB() error {
+	if logUtils.IsRegTest() {
+		return nil // tests fail if we re-use the same blockchain DB
+	}
 	defaultBDBDir, defaultTempBDBDir, key, err := s.getBlockchainDBPaths()
 	if err != nil {
 		return errors.Err(err)

From 7c02c5b92d3e10ccb01062f0a5cbabf35424d215 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Sat, 8 Aug 2020 01:12:55 +0200
Subject: [PATCH 43/57] refactor structures

fix various bugs
---
 main.go                 | 112 +++++++--------------
 manager/manager.go      | 212 ++++++++++------------------------------
 manager/s3_storage.go   |  27 +++--
 manager/setup.go        |  67 +++++++------
 manager/transfer.go     |  32 +++---
 manager/ytsync.go       | 168 +++++++++++++------------------
 sdk/api.go              |  70 ++-----------
 shared/shared.go        | 112 +++++++++++++++++++++
 sources/youtubeVideo.go |   3 +-
 ytapi/ytapi.go          |   3 +-
 10 files changed, 346 insertions(+), 460 deletions(-)
 create mode 100644 shared/shared.go

diff --git a/main.go b/main.go
index b36233c..13080eb 100644
--- a/main.go
+++ b/main.go
@@ -11,6 +11,7 @@ import (
 	"github.com/lbryio/lbry.go/v2/extras/util"
 	"github.com/lbryio/ytsync/v5/manager"
 	"github.com/lbryio/ytsync/v5/sdk"
+	"github.com/lbryio/ytsync/v5/shared"
 	ytUtils "github.com/lbryio/ytsync/v5/util"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 
@@ -23,39 +24,11 @@ var Version string
 const defaultMaxTries = 3
 
 var (
-	flags          sdk.SyncFlags
-	maxTries       int
-	refill         int
-	limit          int
-	syncStatus     string
-	channelID      string
-	syncFrom       int64
-	syncUntil      int64
-	concurrentJobs int
-	videosLimit    int
-	maxVideoSize   int
+	cliFlags       shared.SyncFlags
 	maxVideoLength int
 )
 
 func main() {
-	//grp := stop.New()
-	//ipPool, err := ip_manager.GetIPPool(grp)
-	//if err != nil {
-	//	panic(err)
-	//}
-	//
-	//videoID := "vtIzMaLkCaM"
-	//
-	//ip, err := ipPool.GetIP(videoID)
-	//if err != nil {
-	//	panic(err)
-	//}
-	//
-	//spew.Dump(ip)
-	//
-	//spew.Dump(downloader.GetVideoInformation(videoID, &net.TCPAddr{IP: net.ParseIP(ip)}))
-	//return
-
 	rand.Seed(time.Now().UnixNano())
 	log.SetLevel(log.DebugLevel)
 	http.Handle("/metrics", promhttp.Handler())
@@ -69,24 +42,24 @@ func main() {
 		Args:  cobra.RangeArgs(0, 0),
 	}
 
-	cmd.Flags().BoolVar(&flags.StopOnError, "stop-on-error", false, "If a publish fails, stop all publishing and exit")
-	cmd.Flags().IntVar(&maxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
-	cmd.Flags().BoolVar(&flags.TakeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
-	cmd.Flags().IntVar(&limit, "limit", 0, "limit the amount of channels to sync")
-	cmd.Flags().BoolVar(&flags.SkipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
-	cmd.Flags().BoolVar(&flags.SyncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
-	cmd.Flags().BoolVar(&flags.SingleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
-	cmd.Flags().BoolVar(&flags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
-	cmd.Flags().BoolVar(&flags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
-	cmd.Flags().BoolVar(&flags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
-	cmd.Flags().BoolVar(&flags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube")
-	cmd.Flags().StringVar(&syncStatus, "status", "", "Specify which queue to pull from. Overrides --update")
-	cmd.Flags().StringVar(&channelID, "channelID", "", "If specified, only this channel will be synced.")
-	cmd.Flags().Int64Var(&syncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
-	cmd.Flags().Int64Var(&syncUntil, "before", time.Now().AddDate(1, 0, 0).Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)")
-	cmd.Flags().IntVar(&concurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently")
-	cmd.Flags().IntVar(&videosLimit, "videos-limit", 1000, "how many videos to process per channel")
-	cmd.Flags().IntVar(&maxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)")
+	cmd.Flags().BoolVar(&cliFlags.StopOnError, "stop-on-error", false, "If a publish fails, stop all publishing and exit")
+	cmd.Flags().IntVar(&cliFlags.MaxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
+	cmd.Flags().BoolVar(&cliFlags.TakeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
+	cmd.Flags().IntVar(&cliFlags.Limit, "limit", 0, "limit the amount of channels to sync")
+	cmd.Flags().BoolVar(&cliFlags.SkipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
+	cmd.Flags().BoolVar(&cliFlags.SyncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
+	cmd.Flags().BoolVar(&cliFlags.SingleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
+	cmd.Flags().BoolVar(&cliFlags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
+	cmd.Flags().BoolVar(&cliFlags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
+	cmd.Flags().BoolVar(&cliFlags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
+	cmd.Flags().BoolVar(&cliFlags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube")
+	cmd.Flags().StringVar(&cliFlags.SyncStatus, "status", "", "Specify which queue to pull from. Overrides --update")
+	cmd.Flags().StringVar(&cliFlags.ChannelID, "channelID", "", "If specified, only this channel will be synced.")
+	cmd.Flags().Int64Var(&cliFlags.SyncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
+	cmd.Flags().Int64Var(&cliFlags.SyncUntil, "before", time.Now().AddDate(1, 0, 0).Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)")
+	cmd.Flags().IntVar(&cliFlags.ConcurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently")
+	cmd.Flags().IntVar(&cliFlags.VideosLimit, "videos-limit", 1000, "how many videos to process per channel")
+	cmd.Flags().IntVar(&cliFlags.MaxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)")
 	cmd.Flags().IntVar(&maxVideoLength, "max-length", 2, "Maximum video length to process (in hours)")
 
 	if err := cmd.Execute(); err != nil {
@@ -114,29 +87,30 @@ func ytSync(cmd *cobra.Command, args []string) {
 		util.InitSlack(os.Getenv("SLACK_TOKEN"), os.Getenv("SLACK_CHANNEL"), hostname)
 	}
 
-	if syncStatus != "" && !util.InSlice(syncStatus, manager.SyncStatuses) {
-		log.Errorf("status must be one of the following: %v\n", manager.SyncStatuses)
+	if cliFlags.SyncStatus != "" && !util.InSlice(cliFlags.SyncStatus, shared.SyncStatuses) {
+		log.Errorf("status must be one of the following: %v\n", shared.SyncStatuses)
 		return
 	}
 
-	if flags.StopOnError && maxTries != defaultMaxTries {
+	if cliFlags.StopOnError && cliFlags.MaxTries != defaultMaxTries {
 		log.Errorln("--stop-on-error and --max-tries are mutually exclusive")
 		return
 	}
-	if maxTries < 1 {
+	if cliFlags.MaxTries < 1 {
 		log.Errorln("setting --max-tries less than 1 doesn't make sense")
 		return
 	}
 
-	if limit < 0 {
+	if cliFlags.Limit < 0 {
 		log.Errorln("setting --limit less than 0 (unlimited) doesn't make sense")
 		return
 	}
+	cliFlags.MaxVideoLength = time.Duration(maxVideoLength) * time.Hour
 
 	apiURL := os.Getenv("LBRY_WEB_API")
 	apiToken := os.Getenv("LBRY_API_TOKEN")
 	youtubeAPIKey := os.Getenv("YOUTUBE_API_KEY")
-	lbrycrdString := os.Getenv("LBRYCRD_STRING")
+	lbrycrdDsn := os.Getenv("LBRYCRD_STRING")
 	awsS3ID := os.Getenv("AWS_S3_ID")
 	awsS3Secret := os.Getenv("AWS_S3_SECRET")
 	awsS3Region := os.Getenv("AWS_S3_REGION")
@@ -169,42 +143,30 @@ func ytSync(cmd *cobra.Command, args []string) {
 		log.Errorln("AWS S3 Bucket was not defined. Please set the environment variable AWS_S3_BUCKET")
 		return
 	}
-	if lbrycrdString == "" {
+	if lbrycrdDsn == "" {
 		log.Infoln("Using default (local) lbrycrd instance. Set LBRYCRD_STRING if you want to use something else")
 	}
 
 	blobsDir := ytUtils.GetBlobsDir()
 
-	syncProperties := &sdk.SyncProperties{
-		SyncFrom:         syncFrom,
-		SyncUntil:        syncUntil,
-		YoutubeChannelID: channelID,
-	}
 	apiConfig := &sdk.APIConfig{
 		YoutubeAPIKey: youtubeAPIKey,
 		ApiURL:        apiURL,
 		ApiToken:      apiToken,
 		HostName:      hostname,
 	}
+	awsConfig := &shared.AwsConfigs{
+		AwsS3ID:     awsS3ID,
+		AwsS3Secret: awsS3Secret,
+		AwsS3Region: awsS3Region,
+		AwsS3Bucket: awsS3Bucket,
+	}
 	sm := manager.NewSyncManager(
-		flags,
-		maxTries,
-		refill,
-		limit,
-		concurrentJobs,
-		concurrentJobs,
+		cliFlags,
 		blobsDir,
-		videosLimit,
-		maxVideoSize,
-		lbrycrdString,
-		awsS3ID,
-		awsS3Secret,
-		awsS3Region,
-		awsS3Bucket,
-		syncStatus,
-		syncProperties,
+		lbrycrdDsn,
+		awsConfig,
 		apiConfig,
-		time.Duration(maxVideoLength)*time.Hour,
 	)
 	err := sm.Start()
 	if err != nil {
diff --git a/manager/manager.go b/manager/manager.go
index 9af86cb..8a09b77 100644
--- a/manager/manager.go
+++ b/manager/manager.go
@@ -10,94 +10,43 @@ import (
 	"github.com/lbryio/ytsync/v5/ip_manager"
 	"github.com/lbryio/ytsync/v5/namer"
 	"github.com/lbryio/ytsync/v5/sdk"
+	"github.com/lbryio/ytsync/v5/shared"
 	logUtils "github.com/lbryio/ytsync/v5/util"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/extras/util"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
 	log "github.com/sirupsen/logrus"
 )
 
 type SyncManager struct {
-	SyncFlags        sdk.SyncFlags
-	maxTries         int
-	refill           int
-	limit            int
-	concurrentJobs   int
-	concurrentVideos int
-	blobsDir         string
-	videosLimit      int
-	maxVideoSize     int
-	maxVideoLength   time.Duration
-	lbrycrdString    string
-	awsS3ID          string
-	awsS3Secret      string
-	awsS3Region      string
-	syncStatus       string
-	awsS3Bucket      string
-	syncProperties   *sdk.SyncProperties
-	apiConfig        *sdk.APIConfig
+	CliFlags   shared.SyncFlags
+	ApiConfig  *sdk.APIConfig
+	LbrycrdDsn string
+	AwsConfigs *shared.AwsConfigs
+
+	blobsDir       string
+	channelsToSync []Sync
 }
 
-func NewSyncManager(syncFlags sdk.SyncFlags, maxTries int, refill int, limit int, concurrentJobs int, concurrentVideos int, blobsDir string, videosLimit int,
-	maxVideoSize int, lbrycrdString string, awsS3ID string, awsS3Secret string, awsS3Region string, awsS3Bucket string,
-	syncStatus string, syncProperties *sdk.SyncProperties, apiConfig *sdk.APIConfig, maxVideoLength time.Duration) *SyncManager {
+func NewSyncManager(cliFlags shared.SyncFlags, blobsDir, lbrycrdDsn string, awsConfigs *shared.AwsConfigs, apiConfig *sdk.APIConfig) *SyncManager {
 	return &SyncManager{
-		SyncFlags:        syncFlags,
-		maxTries:         maxTries,
-		refill:           refill,
-		limit:            limit,
-		concurrentJobs:   concurrentJobs,
-		concurrentVideos: concurrentVideos,
-		blobsDir:         blobsDir,
-		videosLimit:      videosLimit,
-		maxVideoSize:     maxVideoSize,
-		maxVideoLength:   maxVideoLength,
-		lbrycrdString:    lbrycrdString,
-		awsS3ID:          awsS3ID,
-		awsS3Secret:      awsS3Secret,
-		awsS3Region:      awsS3Region,
-		awsS3Bucket:      awsS3Bucket,
-		syncStatus:       syncStatus,
-		syncProperties:   syncProperties,
-		apiConfig:        apiConfig,
+		CliFlags:   cliFlags,
+		blobsDir:   blobsDir,
+		LbrycrdDsn: lbrycrdDsn,
+		AwsConfigs: awsConfigs,
+		ApiConfig:  apiConfig,
 	}
 }
-
-const (
-	StatusPending        = "pending"        // waiting for permission to sync
-	StatusPendingEmail   = "pendingemail"   // permission granted but missing email
-	StatusQueued         = "queued"         // in sync queue. will be synced soon
-	StatusPendingUpgrade = "pendingupgrade" // in sync queue. will be synced soon
-	StatusSyncing        = "syncing"        // syncing now
-	StatusSynced         = "synced"         // done
-	StatusFailed         = "failed"
-	StatusFinalized      = "finalized" // no more changes allowed
-	StatusAbandoned      = "abandoned" // deleted on youtube or banned
-)
-const LatestMetadataVersion = 2
-
-var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned}
-
-const (
-	VideoStatusPublished     = "published"
-	VideoStatusFailed        = "failed"
-	VideoStatusUpgradeFailed = "upgradefailed"
-	VideoStatusUnpublished   = "unpublished"
-	VideoStatusTranferFailed = "transferfailed"
-)
-
-const (
-	TransferStateNotTouched = iota
-	TransferStatePending
-	TransferStateComplete
-	TransferStateManual
-)
+func (s *SyncManager) enqueueChannel(channel *shared.YoutubeChannel) {
+	s.channelsToSync = append(s.channelsToSync, Sync{
+		DbChannelData: channel,
+		Manager:       s,
+		namer:         namer.NewNamer(),
+	})
+}
 
 func (s *SyncManager) Start() error {
-
 	if logUtils.ShouldCleanOnStartup() {
 		err := logUtils.CleanForStartup()
 		if err != nil {
@@ -108,119 +57,63 @@ func (s *SyncManager) Start() error {
 	var lastChannelProcessed string
 	syncCount := 0
 	for {
+		s.channelsToSync = make([]Sync, 0, 10) // reset sync queue
 		err := s.checkUsedSpace()
 		if err != nil {
 			return errors.Err(err)
 		}
-
-		var syncs []Sync
 		shouldInterruptLoop := false
 
-		isSingleChannelSync := s.syncProperties.YoutubeChannelID != ""
-		if isSingleChannelSync {
-			channels, err := s.apiConfig.FetchChannels("", s.syncProperties)
+		if s.CliFlags.IsSingleChannelSync() {
+			channels, err := s.ApiConfig.FetchChannels("", &s.CliFlags)
 			if err != nil {
 				return errors.Err(err)
 			}
 			if len(channels) != 1 {
 				return errors.Err("Expected 1 channel, %d returned", len(channels))
 			}
-			lbryChannelName := channels[0].DesiredChannelName
-			syncs = make([]Sync, 1)
-			s.maxVideoLength = time.Duration(channels[0].LengthLimit) * time.Minute
-			s.maxVideoSize = channels[0].SizeLimit
-			syncs[0] = Sync{
-				APIConfig:            s.apiConfig,
-				YoutubeChannelID:     s.syncProperties.YoutubeChannelID,
-				LbryChannelName:      lbryChannelName,
-				lbryChannelID:        channels[0].ChannelClaimID,
-				MaxTries:             s.maxTries,
-				ConcurrentVideos:     s.concurrentVideos,
-				Refill:               s.refill,
-				Manager:              s,
-				MaxVideoLength:       s.maxVideoLength,
-				LbrycrdString:        s.lbrycrdString,
-				AwsS3ID:              s.awsS3ID,
-				AwsS3Secret:          s.awsS3Secret,
-				AwsS3Region:          s.awsS3Region,
-				AwsS3Bucket:          s.awsS3Bucket,
-				namer:                namer.NewNamer(),
-				Fee:                  channels[0].Fee,
-				clientPublishAddress: channels[0].PublishAddress,
-				publicKey:            channels[0].PublicKey,
-				transferState:        channels[0].TransferState,
-				LastUploadedVideo:    channels[0].LastUploadedVideo,
-			}
+			s.enqueueChannel(&channels[0])
 			shouldInterruptLoop = true
 		} else {
 			var queuesToSync []string
-			if s.syncStatus != "" {
-				queuesToSync = append(queuesToSync, s.syncStatus)
-			} else if s.SyncFlags.SyncUpdate {
-				queuesToSync = append(queuesToSync, StatusSyncing, StatusSynced)
+			if s.CliFlags.SyncStatus != "" {
+				queuesToSync = append(queuesToSync, s.CliFlags.SyncStatus)
+			} else if s.CliFlags.SyncUpdate {
+				queuesToSync = append(queuesToSync, shared.StatusSyncing, shared.StatusSynced)
 			} else {
-				queuesToSync = append(queuesToSync, StatusSyncing, StatusQueued)
+				queuesToSync = append(queuesToSync, shared.StatusSyncing, shared.StatusQueued)
 			}
 		queues:
 			for _, q := range queuesToSync {
-				//temporary override for sync-until to give tom the time to review the channels
-				if q == StatusQueued {
-					s.syncProperties.SyncUntil = time.Now().Add(-8 * time.Hour).Unix()
-				}
-				channels, err := s.apiConfig.FetchChannels(q, s.syncProperties)
+				channels, err := s.ApiConfig.FetchChannels(q, &s.CliFlags)
 				if err != nil {
 					return err
 				}
-				for i, c := range channels {
-					log.Infof("There are %d channels in the \"%s\" queue", len(channels)-i, q)
-					maxVideoLength := s.maxVideoLength
-					if c.TotalSubscribers < 1000 {
-						maxVideoLength = 1 * time.Hour
-					}
-					maxVideoLength = time.Duration(c.LengthLimit) * time.Minute
-					s.maxVideoSize = c.SizeLimit
-					syncs = append(syncs, Sync{
-						APIConfig:            s.apiConfig,
-						YoutubeChannelID:     c.ChannelId,
-						LbryChannelName:      c.DesiredChannelName,
-						lbryChannelID:        c.ChannelClaimID,
-						MaxTries:             s.maxTries,
-						ConcurrentVideos:     s.concurrentVideos,
-						MaxVideoLength:       maxVideoLength,
-						Refill:               s.refill,
-						Manager:              s,
-						LbrycrdString:        s.lbrycrdString,
-						AwsS3ID:              s.awsS3ID,
-						AwsS3Secret:          s.awsS3Secret,
-						AwsS3Region:          s.awsS3Region,
-						AwsS3Bucket:          s.awsS3Bucket,
-						namer:                namer.NewNamer(),
-						Fee:                  c.Fee,
-						clientPublishAddress: c.PublishAddress,
-						publicKey:            c.PublicKey,
-						transferState:        c.TransferState,
-						LastUploadedVideo:    c.LastUploadedVideo,
-					})
-					if q != StatusFailed {
-						continue queues
+				log.Infof("Currently processing the \"%s\" queue with %d channels", q, len(channels))
+				for _, c := range channels {
+					s.enqueueChannel(&c)
+					queueAll := q == shared.StatusFailed || q == shared.StatusSyncing
+					if !queueAll {
+						break queues
 					}
 				}
+				log.Infof("Drained the \"%s\" queue", q)
 			}
 		}
-		if len(syncs) == 0 {
+		if len(s.channelsToSync) == 0 {
 			log.Infoln("No channels to sync. Pausing 5 minutes!")
 			time.Sleep(5 * time.Minute)
 		}
-		for _, sync := range syncs {
-			if lastChannelProcessed == sync.LbryChannelName {
-				util.SendToSlack("We just killed a sync for %s to stop looping!(%s)", sync.LbryChannelName, sync.YoutubeChannelID)
-				stopTheLoops := errors.Err("Found channel %s running twice, set it to failed, and reprocess later", sync.LbryChannelName)
+		for _, sync := range s.channelsToSync {
+			if lastChannelProcessed == sync.DbChannelData.ChannelId {
+				util.SendToSlack("We just killed a sync for %s to stop looping! (%s)", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId)
+				stopTheLoops := errors.Err("Found channel %s running twice, set it to failed, and reprocess later", sync.DbChannelData.DesiredChannelName)
 				sync.setChannelTerminationStatus(&stopTheLoops)
 				continue
 			}
-			lastChannelProcessed = sync.LbryChannelName
+			lastChannelProcessed = sync.DbChannelData.ChannelId
 			shouldNotCount := false
-			logUtils.SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
+			logUtils.SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId, syncCount+1)
 			err := sync.FullCycle()
 			//TODO: THIS IS A TEMPORARY WORK AROUND FOR THE STUPID IP LOCKUP BUG
 			ipPool, _ := ip_manager.GetIPPool(sync.grp)
@@ -255,33 +148,28 @@ func (s *SyncManager) Start() error {
 			if err != nil {
 				return errors.Prefix("@Nikooo777 something went wrong while reflecting blobs", err)
 			}
-			logUtils.SendInfoToSlack("Syncing %s (%s) reached an end. total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
+			logUtils.SendInfoToSlack("%s (%s) reached an end. Total processed channels since startup: %d", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId, syncCount+1)
 			if !shouldNotCount {
 				syncCount++
 			}
-			if sync.IsInterrupted() || (s.limit != 0 && syncCount >= s.limit) {
+			if sync.IsInterrupted() || (s.CliFlags.Limit != 0 && syncCount >= s.CliFlags.Limit) {
 				shouldInterruptLoop = true
 				break
 			}
 		}
-		if shouldInterruptLoop || s.SyncFlags.SingleRun {
+		if shouldInterruptLoop || s.CliFlags.SingleRun {
 			break
 		}
 	}
 	return nil
 }
-func (s *SyncManager) GetS3AWSConfig() aws.Config {
-	return aws.Config{
-		Credentials: credentials.NewStaticCredentials(s.awsS3ID, s.awsS3Secret, ""),
-		Region:      &s.awsS3Region,
-	}
-}
+
 func (s *SyncManager) checkUsedSpace() error {
 	usedPctile, err := GetUsedSpace(logUtils.GetBlobsDir())
 	if err != nil {
 		return errors.Err(err)
 	}
-	if usedPctile >= 0.90 && !s.SyncFlags.SkipSpaceCheck {
+	if usedPctile >= 0.90 && !s.CliFlags.SkipSpaceCheck {
 		return errors.Err(fmt.Sprintf("more than 90%% of the space has been used. use --skip-space-check to ignore. Used: %.1f%%", usedPctile*100))
 	}
 	log.Infof("disk usage: %.1f%%", usedPctile*100)
diff --git a/manager/s3_storage.go b/manager/s3_storage.go
index 6c8e551..8b78c89 100644
--- a/manager/s3_storage.go
+++ b/manager/s3_storage.go
@@ -6,7 +6,6 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
-	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
@@ -17,8 +16,7 @@ import (
 )
 
 func (s *Sync) getS3Downloader() (*s3manager.Downloader, error) {
-	creds := credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, "")
-	s3Session, err := session.NewSession(&aws.Config{Region: aws.String(s.AwsS3Region), Credentials: creds})
+	s3Session, err := session.NewSession(s.Manager.AwsConfigs.GetS3AWSConfig())
 	if err != nil {
 		return nil, errors.Prefix("error starting session: ", err)
 	}
@@ -26,8 +24,7 @@ func (s *Sync) getS3Downloader() (*s3manager.Downloader, error) {
 	return downloader, nil
 }
 func (s *Sync) getS3Uploader() (*s3manager.Uploader, error) {
-	creds := credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, "")
-	s3Session, err := session.NewSession(&aws.Config{Region: aws.String(s.AwsS3Region), Credentials: creds})
+	s3Session, err := session.NewSession(s.Manager.AwsConfigs.GetS3AWSConfig())
 	if err != nil {
 		return nil, errors.Prefix("error starting session: ", err)
 	}
@@ -51,7 +48,7 @@ func (s *Sync) downloadWallet() error {
 	defer out.Close()
 
 	bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
-		Bucket: aws.String(s.AwsS3Bucket),
+		Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
 		Key:    key,
 	})
 	if err != nil {
@@ -112,7 +109,7 @@ func (s *Sync) downloadBlockchainDB() error {
 	defer out.Close()
 
 	bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
-		Bucket: aws.String(s.AwsS3Bucket),
+		Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
 		Key:    key,
 	})
 	if err != nil {
@@ -146,11 +143,11 @@ func (s *Sync) downloadBlockchainDB() error {
 func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string, err error) {
 	defaultWallet = os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
 	tempWallet = os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet"
-	key = aws.String("/wallets/" + s.YoutubeChannelID)
+	key = aws.String("/wallets/" + s.DbChannelData.ChannelId)
 	if logUtils.IsRegTest() {
 		defaultWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
 		tempWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet"
-		key = aws.String("/regtest/" + s.YoutubeChannelID)
+		key = aws.String("/regtest/" + s.DbChannelData.ChannelId)
 	}
 
 	lbryumDir := os.Getenv("LBRYUM_DIR")
@@ -176,20 +173,20 @@ func (s *Sync) getBlockchainDBPaths() (defaultDB, tempDB string, key *string, er
 	}
 	defaultDB = lbryumDir + "/lbc_mainnet/blockchain.db"
 	tempDB = lbryumDir + "/lbc_mainnet/tmp_blockchain.db"
-	key = aws.String("/blockchain_dbs/" + s.YoutubeChannelID)
+	key = aws.String("/blockchain_dbs/" + s.DbChannelData.ChannelId)
 	if logUtils.IsRegTest() {
 		defaultDB = lbryumDir + "/lbc_regtest/blockchain.db"
 		tempDB = lbryumDir + "/lbc_regtest/tmp_blockchain.db"
-		key = aws.String("/regtest_dbs/" + s.YoutubeChannelID)
+		key = aws.String("/regtest_dbs/" + s.DbChannelData.ChannelId)
 	}
 	return
 }
 
 func (s *Sync) uploadWallet() error {
 	defaultWalletDir := logUtils.GetDefaultWalletPath()
-	key := aws.String("/wallets/" + s.YoutubeChannelID)
+	key := aws.String("/wallets/" + s.DbChannelData.ChannelId)
 	if logUtils.IsRegTest() {
-		key = aws.String("/regtest/" + s.YoutubeChannelID)
+		key = aws.String("/regtest/" + s.DbChannelData.ChannelId)
 	}
 
 	if _, err := os.Stat(defaultWalletDir); os.IsNotExist(err) {
@@ -208,7 +205,7 @@ func (s *Sync) uploadWallet() error {
 	defer file.Close()
 
 	_, err = uploader.Upload(&s3manager.UploadInput{
-		Bucket: aws.String(s.AwsS3Bucket),
+		Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
 		Key:    key,
 		Body:   file,
 	})
@@ -241,7 +238,7 @@ func (s *Sync) uploadBlockchainDB() error {
 	defer file.Close()
 
 	_, err = uploader.Upload(&s3manager.UploadInput{
-		Bucket: aws.String(s.AwsS3Bucket),
+		Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
 		Key:    key,
 		Body:   file,
 	})
diff --git a/manager/setup.go b/manager/setup.go
index af2eaad..ec1e771 100644
--- a/manager/setup.go
+++ b/manager/setup.go
@@ -9,6 +9,7 @@ import (
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
 	"github.com/lbryio/lbry.go/v2/extras/util"
+	"github.com/lbryio/ytsync/v5/shared"
 	"github.com/lbryio/ytsync/v5/timing"
 	logUtils "github.com/lbryio/ytsync/v5/util"
 	"github.com/lbryio/ytsync/v5/ytapi"
@@ -72,7 +73,7 @@ func (s *Sync) walletSetup() error {
 	}
 	log.Debugf("Starting balance is %.4f", balance)
 
-	videosOnYoutube, err := ytapi.CountVideosInChannel(s.YoutubeChannelID)
+	videosOnYoutube, err := ytapi.CountVideosInChannel(s.DbChannelData.ChannelId)
 	if err != nil {
 		return err
 	}
@@ -100,17 +101,17 @@ func (s *Sync) walletSetup() error {
 
 	log.Debugf("We already allocated credits for %d published videos and %d failed videos", publishedCount, failedCount)
 
-	if videosOnYoutube > s.Manager.videosLimit {
-		videosOnYoutube = s.Manager.videosLimit
+	if videosOnYoutube > s.Manager.CliFlags.VideosLimit {
+		videosOnYoutube = s.Manager.CliFlags.VideosLimit
 	}
 	unallocatedVideos := videosOnYoutube - (publishedCount + failedCount)
 	channelFee := channelClaimAmount
-	channelAlreadyClaimed := s.lbryChannelID != ""
+	channelAlreadyClaimed := s.DbChannelData.ChannelClaimID != ""
 	if channelAlreadyClaimed {
 		channelFee = 0.0
 	}
 	requiredBalance := float64(unallocatedVideos)*(publishAmount+estimatedMaxTxFee) + channelFee
-	if s.Manager.SyncFlags.UpgradeMetadata {
+	if s.Manager.CliFlags.UpgradeMetadata {
 		requiredBalance += float64(notUpgradedCount) * 0.001
 	}
 
@@ -119,8 +120,8 @@ func (s *Sync) walletSetup() error {
 		refillAmount = math.Max(math.Max(requiredBalance-balance, minimumAccountBalance-balance), minimumRefillAmount)
 	}
 
-	if s.Refill > 0 {
-		refillAmount += float64(s.Refill)
+	if s.Manager.CliFlags.Refill > 0 {
+		refillAmount += float64(s.Manager.CliFlags.Refill)
 	}
 
 	if refillAmount > 0 {
@@ -136,12 +137,11 @@ func (s *Sync) walletSetup() error {
 	} else if claimAddress == nil {
 		return errors.Err("could not get an address")
 	}
-	s.claimAddress = string(claimAddress.Items[0].Address)
-	if s.claimAddress == "" {
-		return errors.Err("found blank claim address")
+	if s.DbChannelData.PublishAddress == "" || !s.shouldTransfer() {
+		s.DbChannelData.PublishAddress = string(claimAddress.Items[0].Address)
 	}
-	if s.shouldTransfer() {
-		s.claimAddress = s.clientPublishAddress
+	if s.DbChannelData.PublishAddress == "" {
+		return errors.Err("found blank claim address")
 	}
 
 	err = s.ensureEnoughUTXOs()
@@ -300,7 +300,7 @@ func (s *Sync) waitForNewBlock() error {
 }
 
 func (s *Sync) GenerateRegtestBlock() error {
-	lbrycrd, err := logUtils.GetLbrycrdClient(s.LbrycrdString)
+	lbrycrd, err := logUtils.GetLbrycrdClient(s.Manager.LbrycrdDsn)
 	if err != nil {
 		return errors.Prefix("error getting lbrycrd client: ", err)
 	}
@@ -319,7 +319,7 @@ func (s *Sync) GenerateRegtestBlock() error {
 func (s *Sync) ensureChannelOwnership() error {
 	defer func(start time.Time) { timing.TimedComponent("ensureChannelOwnership").Add(time.Since(start)) }(time.Now())
 
-	if s.LbryChannelName == "" {
+	if s.DbChannelData.DesiredChannelName == "" {
 		return errors.Err("no channel name set")
 	}
 
@@ -332,27 +332,27 @@ func (s *Sync) ensureChannelOwnership() error {
 
 	var channelToUse *jsonrpc.Transaction
 	if len((*channels).Items) > 0 {
-		if s.lbryChannelID == "" {
+		if s.DbChannelData.ChannelClaimID == "" {
 			return errors.Err("this channel does not have a recorded claimID in the database. To prevent failures, updates are not supported until an entry is manually added in the database")
 		}
 		for _, c := range (*channels).Items {
 			log.Debugf("checking listed channel %s (%s)", c.ClaimID, c.Name)
-			if c.ClaimID != s.lbryChannelID {
+			if c.ClaimID != s.DbChannelData.ChannelClaimID {
 				continue
 			}
-			if c.Name != s.LbryChannelName {
+			if c.Name != s.DbChannelData.DesiredChannelName {
 				return errors.Err("the channel in the wallet is different than the channel in the database")
 			}
 			channelToUse = &c
 			break
 		}
 		if channelToUse == nil {
-			return errors.Err("this wallet has channels but not a single one is ours! Expected claim_id: %s (%s)", s.lbryChannelID, s.LbryChannelName)
+			return errors.Err("this wallet has channels but not a single one is ours! Expected claim_id: %s (%s)", s.DbChannelData.ChannelClaimID, s.DbChannelData.DesiredChannelName)
 		}
-	} else if s.transferState == TransferStateComplete {
+	} else if s.DbChannelData.TransferState == shared.TransferStateComplete {
 		return errors.Err("the channel was transferred but appears to have been abandoned!")
-	} else if s.lbryChannelID != "" {
-		return errors.Err("the database has a channel recorded (%s) but nothing was found in our control", s.lbryChannelID)
+	} else if s.DbChannelData.ChannelClaimID != "" {
+		return errors.Err("the database has a channel recorded (%s) but nothing was found in our control", s.DbChannelData.ChannelClaimID)
 	}
 
 	channelUsesOldMetadata := false
@@ -383,20 +383,23 @@ func (s *Sync) ensureChannelOwnership() error {
 		}
 	}
 
-	channelInfo, err := ytapi.ChannelInfo(s.YoutubeChannelID)
+	channelInfo, err := ytapi.ChannelInfo(s.DbChannelData.ChannelId)
 	if err != nil {
 		return err
 	}
 
 	thumbnail := channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails)-1].URL
-	thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
+	thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail, s.DbChannelData.ChannelId, *s.Manager.AwsConfigs.GetS3AWSConfig())
 	if err != nil {
 		return err
 	}
 
 	var bannerURL *string
 	if channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails != nil {
-		bURL, err := thumbs.MirrorThumbnail(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails)-1].URL, "banner-"+s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
+		bURL, err := thumbs.MirrorThumbnail(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails)-1].URL,
+			"banner-"+s.DbChannelData.ChannelId,
+			*s.Manager.AwsConfigs.GetS3AWSConfig(),
+		)
 		if err != nil {
 			return err
 		}
@@ -419,14 +422,14 @@ func (s *Sync) ensureChannelOwnership() error {
 	claimCreateOptions := jsonrpc.ClaimCreateOptions{
 		Title:        &channelInfo.Microformat.MicroformatDataRenderer.Title,
 		Description:  &channelInfo.Microformat.MicroformatDataRenderer.Description,
-		Tags:         tags_manager.GetTagsForChannel(s.YoutubeChannelID),
+		Tags:         tags_manager.GetTagsForChannel(s.DbChannelData.ChannelId),
 		Languages:    languages,
 		Locations:    locations,
 		ThumbnailURL: &thumbnailURL,
 	}
 	if channelUsesOldMetadata {
-		if s.transferState <= 1 {
-			c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
+		if s.DbChannelData.TransferState <= 1 {
+			c, err = s.daemon.ChannelUpdate(s.DbChannelData.ChannelClaimID, jsonrpc.ChannelUpdateOptions{
 				ClearTags:      util.PtrToBool(true),
 				ClearLocations: util.PtrToBool(true),
 				ClearLanguages: util.PtrToBool(true),
@@ -436,11 +439,11 @@ func (s *Sync) ensureChannelOwnership() error {
 				},
 			})
 		} else {
-			logUtils.SendInfoToSlack("%s (%s) has a channel with old metadata but isn't in our control anymore. Ignoring", s.LbryChannelName, s.lbryChannelID)
+			logUtils.SendInfoToSlack("%s (%s) has a channel with old metadata but isn't in our control anymore. Ignoring", s.DbChannelData.DesiredChannelName, s.DbChannelData.ChannelClaimID)
 			return nil
 		}
 	} else {
-		c, err = s.daemon.ChannelCreate(s.LbryChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
+		c, err = s.daemon.ChannelCreate(s.DbChannelData.DesiredChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
 			ClaimCreateOptions: claimCreateOptions,
 			CoverURL:           bannerURL,
 		})
@@ -450,8 +453,8 @@ func (s *Sync) ensureChannelOwnership() error {
 		return err
 	}
 
-	s.lbryChannelID = c.Outputs[0].ClaimID
-	return s.Manager.apiConfig.SetChannelClaimID(s.YoutubeChannelID, s.lbryChannelID)
+	s.DbChannelData.ChannelClaimID = c.Outputs[0].ClaimID
+	return s.Manager.ApiConfig.SetChannelClaimID(s.DbChannelData.ChannelId, s.DbChannelData.ChannelClaimID)
 }
 
 func (s *Sync) addCredits(amountToAdd float64) error {
@@ -460,7 +463,7 @@ func (s *Sync) addCredits(amountToAdd float64) error {
 		timing.TimedComponent("addCredits").Add(time.Since(start))
 	}(start)
 	log.Printf("Adding %f credits", amountToAdd)
-	lbrycrdd, err := logUtils.GetLbrycrdClient(s.LbrycrdString)
+	lbrycrdd, err := logUtils.GetLbrycrdClient(s.Manager.LbrycrdDsn)
 	if err != nil {
 		return err
 	}
diff --git a/manager/transfer.go b/manager/transfer.go
index 1fdd8fb..d04272d 100644
--- a/manager/transfer.go
+++ b/manager/transfer.go
@@ -10,7 +10,7 @@ import (
 	"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
 	"github.com/lbryio/lbry.go/v2/extras/stop"
 	"github.com/lbryio/lbry.go/v2/extras/util"
-	"github.com/lbryio/ytsync/v5/sdk"
+	"github.com/lbryio/ytsync/v5/shared"
 	"github.com/lbryio/ytsync/v5/timing"
 
 	log "github.com/sirupsen/logrus"
@@ -97,7 +97,7 @@ func abandonSupports(s *Sync) (float64, error) {
 	//TODO: remove this once the SDK team fixes their RPC bugs....
 	s.daemon.SetRPCTimeout(60 * time.Second)
 	defer s.daemon.SetRPCTimeout(5 * time.Minute)
-	for i := 0; i < s.ConcurrentVideos; i++ {
+	for i := 0; i < s.Manager.CliFlags.ConcurrentJobs; i++ {
 		consumerWG.Add(1)
 		go func() {
 			defer consumerWG.Done()
@@ -189,7 +189,7 @@ func abandonSupports(s *Sync) (float64, error) {
 type updateInfo struct {
 	ClaimID             string
 	streamUpdateOptions *jsonrpc.StreamUpdateOptions
-	videoStatus         *sdk.VideoStatus
+	videoStatus         *shared.VideoStatus
 }
 
 func transferVideos(s *Sync) error {
@@ -199,7 +199,7 @@ func transferVideos(s *Sync) error {
 	}(start)
 	cleanTransfer := true
 
-	streamChan := make(chan updateInfo, s.ConcurrentVideos)
+	streamChan := make(chan updateInfo, s.Manager.CliFlags.ConcurrentJobs)
 	account, err := s.getDefaultAccount()
 	if err != nil {
 		return err
@@ -213,13 +213,13 @@ func transferVideos(s *Sync) error {
 	go func() {
 		defer producerWG.Done()
 		for _, video := range s.syncedVideos {
-			if !video.Published || video.Transferred || video.MetadataVersion != LatestMetadataVersion {
+			if !video.Published || video.Transferred || video.MetadataVersion != shared.LatestMetadataVersion {
 				continue
 			}
 
 			var stream *jsonrpc.Claim = nil
 			for _, c := range streams.Items {
-				if c.ClaimID != video.ClaimID || (c.SigningChannel != nil && c.SigningChannel.ClaimID != s.lbryChannelID) {
+				if c.ClaimID != video.ClaimID || (c.SigningChannel != nil && c.SigningChannel.ClaimID != s.DbChannelData.ChannelClaimID) {
 					continue
 				}
 				stream = &c
@@ -232,7 +232,7 @@ func transferVideos(s *Sync) error {
 			streamUpdateOptions := jsonrpc.StreamUpdateOptions{
 				StreamCreateOptions: &jsonrpc.StreamCreateOptions{
 					ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
-						ClaimAddress: &s.clientPublishAddress,
+						ClaimAddress: &s.DbChannelData.PublishAddress,
 						FundingAccountIDs: []string{
 							account,
 						},
@@ -240,12 +240,12 @@ func transferVideos(s *Sync) error {
 				},
 				Bid: util.PtrToString("0.005"), // Todo - Dont hardcode
 			}
-			videoStatus := sdk.VideoStatus{
-				ChannelID:     s.YoutubeChannelID,
+			videoStatus := shared.VideoStatus{
+				ChannelID:     s.DbChannelData.ChannelId,
 				VideoID:       video.VideoID,
 				ClaimID:       video.ClaimID,
 				ClaimName:     video.ClaimName,
-				Status:        VideoStatusPublished,
+				Status:        shared.VideoStatusPublished,
 				IsTransferred: util.PtrToBool(true),
 			}
 			streamChan <- updateInfo{
@@ -257,7 +257,7 @@ func transferVideos(s *Sync) error {
 	}()
 
 	consumerWG := &stop.Group{}
-	for i := 0; i < s.ConcurrentVideos; i++ {
+	for i := 0; i < s.Manager.CliFlags.ConcurrentJobs; i++ {
 		consumerWG.Add(1)
 		go func(worker int) {
 			defer consumerWG.Done()
@@ -290,13 +290,13 @@ func (s *Sync) streamUpdate(ui *updateInfo) error {
 	timing.TimedComponent("transferStreamUpdate").Add(time.Since(start))
 	if updateError != nil {
 		ui.videoStatus.FailureReason = updateError.Error()
-		ui.videoStatus.Status = VideoStatusTranferFailed
+		ui.videoStatus.Status = shared.VideoStatusTranferFailed
 		ui.videoStatus.IsTransferred = util.PtrToBool(false)
 	} else {
 		ui.videoStatus.IsTransferred = util.PtrToBool(len(result.Outputs) != 0)
 	}
 	log.Infof("TRANSFERRED %t", *ui.videoStatus.IsTransferred)
-	statusErr := s.APIConfig.MarkVideoStatus(*ui.videoStatus)
+	statusErr := s.Manager.ApiConfig.MarkVideoStatus(*ui.videoStatus)
 	if statusErr != nil {
 		return errors.Prefix(statusErr.Error(), updateError)
 	}
@@ -318,7 +318,7 @@ func transferChannel(s *Sync) error {
 	}
 	var channelClaim *jsonrpc.Transaction = nil
 	for _, c := range channelClaims.Items {
-		if c.ClaimID != s.lbryChannelID {
+		if c.ClaimID != s.DbChannelData.ChannelClaimID {
 			continue
 		}
 		channelClaim = &c
@@ -332,11 +332,11 @@ func transferChannel(s *Sync) error {
 		Bid: util.PtrToString(fmt.Sprintf("%.6f", channelClaimAmount-0.005)),
 		ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
 			ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
-				ClaimAddress: &s.clientPublishAddress,
+				ClaimAddress: &s.DbChannelData.PublishAddress,
 			},
 		},
 	}
-	result, err := s.daemon.ChannelUpdate(s.lbryChannelID, updateOptions)
+	result, err := s.daemon.ChannelUpdate(s.DbChannelData.ChannelClaimID, updateOptions)
 	if err != nil {
 		return errors.Err(err)
 	}
diff --git a/manager/ytsync.go b/manager/ytsync.go
index b1611d3..dad8020 100644
--- a/manager/ytsync.go
+++ b/manager/ytsync.go
@@ -15,6 +15,7 @@ import (
 	"github.com/lbryio/ytsync/v5/ip_manager"
 	"github.com/lbryio/ytsync/v5/namer"
 	"github.com/lbryio/ytsync/v5/sdk"
+	"github.com/lbryio/ytsync/v5/shared"
 	"github.com/lbryio/ytsync/v5/sources"
 	"github.com/lbryio/ytsync/v5/thumbs"
 	"github.com/lbryio/ytsync/v5/timing"
@@ -40,35 +41,18 @@ const (
 
 // Sync stores the options that control how syncing happens
 type Sync struct {
-	APIConfig            *sdk.APIConfig
-	YoutubeChannelID     string
-	LbryChannelName      string
-	MaxTries             int
-	ConcurrentVideos     int
-	Refill               int
-	Manager              *SyncManager
-	LbrycrdString        string
-	AwsS3ID              string
-	AwsS3Secret          string
-	AwsS3Region          string
-	AwsS3Bucket          string
-	Fee                  *sdk.Fee
-	daemon               *jsonrpc.Client
-	claimAddress         string
-	videoDirectory       string
-	syncedVideosMux      *sync.RWMutex
-	syncedVideos         map[string]sdk.SyncedVideo
-	grp                  *stop.Group
-	lbryChannelID        string
-	namer                *namer.Namer
-	walletMux            *sync.RWMutex
-	queue                chan ytapi.Video
-	transferState        int
-	clientPublishAddress string
-	publicKey            string
-	defaultAccountID     string
-	MaxVideoLength       time.Duration
-	LastUploadedVideo    string
+	DbChannelData *shared.YoutubeChannel
+	Manager       *SyncManager
+
+	daemon           *jsonrpc.Client
+	videoDirectory   string
+	syncedVideosMux  *sync.RWMutex
+	syncedVideos     map[string]sdk.SyncedVideo
+	grp              *stop.Group
+	namer            *namer.Namer
+	walletMux        *sync.RWMutex
+	queue            chan ytapi.Video
+	defaultAccountID string
 }
 
 func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) {
@@ -96,7 +80,7 @@ func (s *Sync) IsInterrupted() bool {
 }
 
 func (s *Sync) setStatusSyncing() error {
-	syncedVideos, claimNames, err := s.Manager.apiConfig.SetChannelStatus(s.YoutubeChannelID, StatusSyncing, "", nil)
+	syncedVideos, claimNames, err := s.Manager.ApiConfig.SetChannelStatus(s.DbChannelData.ChannelId, shared.StatusSyncing, "", nil)
 	if err != nil {
 		return err
 	}
@@ -107,24 +91,12 @@ func (s *Sync) setStatusSyncing() error {
 	return nil
 }
 
-func (s *Sync) setExceptions() {
-	if s.YoutubeChannelID == "UCwjQfNRW6sGYb__pd7d4nUg" { //@FreeTalkLive
-		s.MaxVideoLength = 9999 * time.Hour // skips max length checks
-		s.Manager.maxVideoSize = 0
-	}
-}
-
 var stopGroup = stop.New()
 
 func (s *Sync) FullCycle() (e error) {
 	if os.Getenv("HOME") == "" {
 		return errors.Err("no $HOME env var found")
 	}
-	if s.YoutubeChannelID == "" {
-		return errors.Err("channel ID not provided")
-	}
-
-	s.setExceptions()
 	defer timing.ClearTimings()
 	s.syncedVideosMux = &sync.RWMutex{}
 	s.walletMux = &sync.RWMutex{}
@@ -201,7 +173,7 @@ func (s *Sync) FullCycle() (e error) {
 
 func (s *Sync) processTransfers() (e error) {
 	log.Println("Processing transfers")
-	if s.transferState != 2 {
+	if s.DbChannelData.TransferState != 2 {
 		err := waitConfirmations(s)
 		if err != nil {
 			return err
@@ -212,7 +184,7 @@ func (s *Sync) processTransfers() (e error) {
 		return errors.Prefix(fmt.Sprintf("%.6f LBCs were abandoned before failing", supportAmount), err)
 	}
 	if supportAmount > 0 {
-		logUtils.SendInfoToSlack("(%s) %.6f LBCs were abandoned and should be used as support", s.YoutubeChannelID, supportAmount)
+		logUtils.SendInfoToSlack("(%s) %.6f LBCs were abandoned and should be used as support", s.DbChannelData.ChannelId, supportAmount)
 	}
 	err = transferVideos(s)
 	if err != nil {
@@ -233,14 +205,14 @@ func (s *Sync) processTransfers() (e error) {
 			return err
 		}
 		isTip := true
-		summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount}, nil)
+		summary, err := s.daemon.SupportCreate(s.DbChannelData.ChannelClaimID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount}, nil)
 		if err != nil {
 			if strings.Contains(err.Error(), "tx-size") { //TODO: this is a silly workaround and should be written in an recursive function
-				summary, err = s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount/2.0), &isTip, nil, []string{defaultAccount}, nil)
+				summary, err = s.daemon.SupportCreate(s.DbChannelData.ChannelClaimID, fmt.Sprintf("%.6f", supportAmount/2.0), &isTip, nil, []string{defaultAccount}, nil)
 				if err != nil {
 					return errors.Prefix(fmt.Sprintf("something went wrong while tipping the channel for %.6f LBCs", supportAmount), err)
 				}
-				summary, err = s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount/2.0), &isTip, nil, []string{defaultAccount}, nil)
+				summary, err = s.daemon.SupportCreate(s.DbChannelData.ChannelClaimID, fmt.Sprintf("%.6f", supportAmount/2.0), &isTip, nil, []string{defaultAccount}, nil)
 				if err != nil {
 					return errors.Err(err)
 				}
@@ -267,7 +239,7 @@ func deleteSyncFolder(videoDirectory string) {
 }
 
 func (s *Sync) shouldTransfer() bool {
-	return s.transferState >= 1 && s.clientPublishAddress != "" && !s.Manager.SyncFlags.DisableTransfers
+	return s.DbChannelData.TransferState >= 1 && s.DbChannelData.PublishAddress != "" && !s.Manager.CliFlags.DisableTransfers
 }
 
 func (s *Sync) setChannelTerminationStatus(e *error) {
@@ -275,7 +247,7 @@ func (s *Sync) setChannelTerminationStatus(e *error) {
 
 	if s.shouldTransfer() {
 		if *e == nil {
-			transferState = util.PtrToInt(TransferStateComplete)
+			transferState = util.PtrToInt(shared.TransferStateComplete)
 		}
 	}
 	if *e != nil {
@@ -288,13 +260,13 @@ func (s *Sync) setChannelTerminationStatus(e *error) {
 			return
 		}
 		failureReason := (*e).Error()
-		_, _, err := s.Manager.apiConfig.SetChannelStatus(s.YoutubeChannelID, StatusFailed, failureReason, transferState)
+		_, _, err := s.Manager.ApiConfig.SetChannelStatus(s.DbChannelData.ChannelId, shared.StatusFailed, failureReason, transferState)
 		if err != nil {
-			msg := fmt.Sprintf("Failed setting failed state for channel %s", s.LbryChannelName)
+			msg := fmt.Sprintf("Failed setting failed state for channel %s", s.DbChannelData.DesiredChannelName)
 			*e = errors.Prefix(msg+err.Error(), *e)
 		}
 	} else if !s.IsInterrupted() {
-		_, _, err := s.Manager.apiConfig.SetChannelStatus(s.YoutubeChannelID, StatusSynced, "", transferState)
+		_, _, err := s.Manager.ApiConfig.SetChannelStatus(s.DbChannelData.ChannelId, shared.StatusSynced, "", transferState)
 		if err != nil {
 			*e = err
 		}
@@ -397,7 +369,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
 	abandonedClaims := false
 	videoIDs := make(map[string]jsonrpc.Claim)
 	for _, c := range claims {
-		if !isYtsyncClaim(c, s.lbryChannelID) {
+		if !isYtsyncClaim(c, s.DbChannelData.ChannelClaimID) {
 			continue
 		}
 		tn := c.Value.GetThumbnail().GetUrl()
@@ -415,7 +387,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
 			claimToAbandon = cl
 			videoIDs[videoID] = c
 		}
-		if claimToAbandon.Address != s.clientPublishAddress && !s.syncedVideos[videoID].Transferred {
+		if claimToAbandon.Address != s.DbChannelData.PublishAddress && !s.syncedVideos[videoID].Transferred {
 			log.Debugf("abandoning %+v", claimToAbandon)
 			_, err := s.daemon.StreamAbandon(claimToAbandon.Txid, claimToAbandon.Nout, nil, false)
 			if err != nil {
@@ -444,7 +416,7 @@ type ytsyncClaim struct {
 func (s *Sync) mapFromClaims(claims []jsonrpc.Claim) map[string]ytsyncClaim {
 	videoIDMap := make(map[string]ytsyncClaim, len(claims))
 	for _, c := range claims {
-		if !isYtsyncClaim(c, s.lbryChannelID) {
+		if !isYtsyncClaim(c, s.DbChannelData.ChannelClaimID) {
 			continue
 		}
 		tn := c.Value.GetThumbnail().GetUrl()
@@ -513,10 +485,10 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
 			}
 			fixed++
 			log.Debugf("updating %s in the database", videoID)
-			err = s.Manager.apiConfig.MarkVideoStatus(sdk.VideoStatus{
-				ChannelID:       s.YoutubeChannelID,
+			err = s.Manager.ApiConfig.MarkVideoStatus(shared.VideoStatus{
+				ChannelID:       s.DbChannelData.ChannelId,
 				VideoID:         videoID,
-				Status:          VideoStatusPublished,
+				Status:          shared.VideoStatusPublished,
 				ClaimID:         chainInfo.ClaimID,
 				ClaimName:       chainInfo.ClaimName,
 				Size:            util.PtrToInt64(int64(claimSize)),
@@ -562,13 +534,13 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
 		}
 		_, ok := ownClaimsInfo[vID]
 		if !ok && sv.Published {
-			log.Debugf("%s: claims to be published but wasn't found in the list of claims and will be removed if --remove-db-unpublished was specified (%t)", vID, s.Manager.SyncFlags.RemoveDBUnpublished)
+			log.Debugf("%s: claims to be published but wasn't found in the list of claims and will be removed if --remove-db-unpublished was specified (%t)", vID, s.Manager.CliFlags.RemoveDBUnpublished)
 			idsToRemove = append(idsToRemove, vID)
 		}
 	}
-	if s.Manager.SyncFlags.RemoveDBUnpublished && len(idsToRemove) > 0 {
+	if s.Manager.CliFlags.RemoveDBUnpublished && len(idsToRemove) > 0 {
 		log.Infof("removing: %s", strings.Join(idsToRemove, ","))
-		err := s.Manager.apiConfig.DeleteVideos(idsToRemove)
+		err := s.Manager.ApiConfig.DeleteVideos(idsToRemove)
 		if err != nil {
 			return count, fixed, len(idsToRemove), err
 		}
@@ -599,7 +571,7 @@ func (s *Sync) getClaims(defaultOnly bool) ([]jsonrpc.Claim, error) {
 	}
 	items := make([]jsonrpc.Claim, 0, len(claims.Items))
 	for _, c := range claims.Items {
-		if c.SigningChannel != nil && c.SigningChannel.ClaimID == s.lbryChannelID {
+		if c.SigningChannel != nil && c.SigningChannel.ClaimID == s.DbChannelData.ChannelClaimID {
 			items = append(items, c)
 		}
 	}
@@ -656,11 +628,11 @@ func (s *Sync) checkIntegrity() error {
 	}
 
 	if pubsOnWallet > pubsOnDB { //This case should never happen
-		logUtils.SendInfoToSlack("We're claiming to have published %d videos but in reality we published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID)
+		logUtils.SendInfoToSlack("We're claiming to have published %d videos but in reality we published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId)
 		return errors.Err("not all published videos are in the database")
 	}
 	if pubsOnWallet < pubsOnDB {
-		logUtils.SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID)
+		logUtils.SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId)
 	}
 
 	_, err = s.getUnsentSupports() //TODO: use the returned value when it works
@@ -693,24 +665,24 @@ func (s *Sync) doSync() error {
 		return err
 	}
 
-	if s.transferState < TransferStateComplete {
-		cert, err := s.daemon.ChannelExport(s.lbryChannelID, nil, nil)
+	if s.DbChannelData.TransferState < shared.TransferStateComplete {
+		cert, err := s.daemon.ChannelExport(s.DbChannelData.ChannelClaimID, nil, nil)
 		if err != nil {
 			return errors.Prefix("error getting channel cert", err)
 		}
 		if cert != nil {
-			err = s.APIConfig.SetChannelCert(string(*cert), s.lbryChannelID)
+			err = s.Manager.ApiConfig.SetChannelCert(string(*cert), s.DbChannelData.ChannelClaimID)
 			if err != nil {
 				return errors.Prefix("error setting channel cert", err)
 			}
 		}
 	}
 
-	if s.Manager.SyncFlags.StopOnError {
+	if s.Manager.CliFlags.StopOnError {
 		log.Println("Will stop publishing if an error is detected")
 	}
 
-	for i := 0; i < s.ConcurrentVideos; i++ {
+	for i := 0; i < s.Manager.CliFlags.ConcurrentJobs; i++ {
 		s.grp.Add(1)
 		go func(i int) {
 			defer s.grp.Done()
@@ -718,7 +690,7 @@ func (s *Sync) doSync() error {
 		}(i)
 	}
 
-	if s.LbryChannelName == "@UCBerkeley" {
+	if s.DbChannelData.DesiredChannelName == "@UCBerkeley" {
 		err = errors.Err("UCB is not supported in this version of YTSYNC")
 	} else {
 		err = s.enqueueYoutubeVideos()
@@ -780,9 +752,9 @@ func (s *Sync) startWorker(workerNum int) {
 					"Couldn't find private key for id",
 					"You already have a stream claim published under the name",
 				}
-				if util.SubstringInSlice(err.Error(), fatalErrors) || s.Manager.SyncFlags.StopOnError {
+				if util.SubstringInSlice(err.Error(), fatalErrors) || s.Manager.CliFlags.StopOnError {
 					s.grp.Stop()
-				} else if s.MaxTries > 1 {
+				} else if s.Manager.CliFlags.MaxTries > 1 {
 					errorsNoRetry := []string{
 						"non 200 status code received",
 						"This video contains content from",
@@ -812,7 +784,7 @@ func (s *Sync) startWorker(workerNum int) {
 					}
 					if util.SubstringInSlice(err.Error(), errorsNoRetry) {
 						log.Println("This error should not be retried at all")
-					} else if tryCount < s.MaxTries {
+					} else if tryCount < s.Manager.CliFlags.MaxTries {
 						if util.SubstringInSlice(err.Error(), []string{
 							"txn-mempool-conflict",
 							"too-long-mempool-chain",
@@ -861,14 +833,14 @@ func (s *Sync) startWorker(workerNum int) {
 						existingClaimSize = existingClaim.Size
 					}
 				}
-				videoStatus := VideoStatusFailed
+				videoStatus := shared.VideoStatusFailed
 				if strings.Contains(err.Error(), "upgrade failed") {
-					videoStatus = VideoStatusUpgradeFailed
+					videoStatus = shared.VideoStatusUpgradeFailed
 				} else {
 					s.AppendSyncedVideo(v.ID(), false, err.Error(), existingClaimName, existingClaimID, 0, existingClaimSize)
 				}
-				err = s.Manager.apiConfig.MarkVideoStatus(sdk.VideoStatus{
-					ChannelID:     s.YoutubeChannelID,
+				err = s.Manager.ApiConfig.MarkVideoStatus(shared.VideoStatus{
+					ChannelID:     s.DbChannelData.ChannelId,
 					VideoID:       v.ID(),
 					Status:        videoStatus,
 					ClaimID:       existingClaimID,
@@ -893,12 +865,12 @@ func (s *Sync) enqueueYoutubeVideos() error {
 		return err
 	}
 
-	videos, err := ytapi.GetVideosToSync(s.APIConfig, s.YoutubeChannelID, s.syncedVideos, s.Manager.SyncFlags.QuickSync, s.Manager.videosLimit, ytapi.VideoParams{
+	videos, err := ytapi.GetVideosToSync(s.Manager.ApiConfig, s.DbChannelData.ChannelId, s.syncedVideos, s.Manager.CliFlags.QuickSync, s.Manager.CliFlags.VideosLimit, ytapi.VideoParams{
 		VideoDir: s.videoDirectory,
-		S3Config: s.Manager.GetS3AWSConfig(),
+		S3Config: *s.Manager.AwsConfigs.GetS3AWSConfig(),
 		Stopper:  s.grp,
 		IPPool:   ipPool,
-	}, s.LastUploadedVideo)
+	}, s.DbChannelData.LastUploadedVideo)
 	if err != nil {
 		return err
 	}
@@ -944,7 +916,7 @@ func (s *Sync) processVideo(v ytapi.Video) (err error) {
 	s.syncedVideosMux.RUnlock()
 	newMetadataVersion := int8(2)
 	alreadyPublished := ok && sv.Published
-	videoRequiresUpgrade := ok && s.Manager.SyncFlags.UpgradeMetadata && sv.MetadataVersion < newMetadataVersion
+	videoRequiresUpgrade := ok && s.Manager.CliFlags.UpgradeMetadata && sv.MetadataVersion < newMetadataVersion
 
 	neverRetryFailures := []string{
 		"Error extracting sts from embedded url response",
@@ -972,7 +944,7 @@ func (s *Sync) processVideo(v ytapi.Video) (err error) {
 		return nil
 	}
 
-	if !videoRequiresUpgrade && v.PlaylistPosition() >= s.Manager.videosLimit {
+	if !videoRequiresUpgrade && v.PlaylistPosition() >= s.Manager.CliFlags.VideosLimit {
 		log.Println(v.ID() + " is old: skipping")
 		return nil
 	}
@@ -985,13 +957,13 @@ func (s *Sync) processVideo(v ytapi.Video) (err error) {
 		return err
 	}
 	sp := sources.SyncParams{
-		ClaimAddress:   s.claimAddress,
+		ClaimAddress:   s.DbChannelData.PublishAddress,
 		Amount:         publishAmount,
-		ChannelID:      s.lbryChannelID,
-		MaxVideoSize:   s.Manager.maxVideoSize,
+		ChannelID:      s.DbChannelData.ChannelClaimID,
+		MaxVideoSize:   s.DbChannelData.SizeLimit,
 		Namer:          s.namer,
-		MaxVideoLength: s.MaxVideoLength,
-		Fee:            s.Fee,
+		MaxVideoLength: time.Duration(s.DbChannelData.LengthLimit) * time.Minute,
+		Fee:            s.DbChannelData.Fee,
 		DefaultAccount: da,
 	}
 
@@ -1001,14 +973,14 @@ func (s *Sync) processVideo(v ytapi.Video) (err error) {
 	}
 
 	s.AppendSyncedVideo(v.ID(), true, "", summary.ClaimName, summary.ClaimID, newMetadataVersion, *v.Size())
-	err = s.Manager.apiConfig.MarkVideoStatus(sdk.VideoStatus{
-		ChannelID:       s.YoutubeChannelID,
+	err = s.Manager.ApiConfig.MarkVideoStatus(shared.VideoStatus{
+		ChannelID:       s.DbChannelData.ChannelId,
 		VideoID:         v.ID(),
-		Status:          VideoStatusPublished,
+		Status:          shared.VideoStatusPublished,
 		ClaimID:         summary.ClaimID,
 		ClaimName:       summary.ClaimName,
 		Size:            v.Size(),
-		MetaDataVersion: LatestMetadataVersion,
+		MetaDataVersion: shared.LatestMetadataVersion,
 		IsTransferred:   util.PtrToBool(s.shouldTransfer()),
 	})
 	if err != nil {
@@ -1019,7 +991,7 @@ func (s *Sync) processVideo(v ytapi.Video) (err error) {
 }
 
 func (s *Sync) importPublicKey() error {
-	if s.publicKey != "" {
+	if s.DbChannelData.PublicKey != "" {
 		accountsResponse, err := s.daemon.AccountList(1, 50)
 		if err != nil {
 			return errors.Err(err)
@@ -1030,13 +1002,13 @@ func (s *Sync) importPublicKey() error {
 		}
 		for _, a := range accountsResponse.Items {
 			if *a.Ledger == ledger {
-				if a.PublicKey == s.publicKey {
+				if a.PublicKey == s.DbChannelData.PublicKey {
 					return nil
 				}
 			}
 		}
-		log.Infof("Could not find public key %s in the wallet. Importing it...", s.publicKey)
-		_, err = s.daemon.AccountAdd(s.LbryChannelName, nil, nil, &s.publicKey, util.PtrToBool(true), nil)
+		log.Infof("Could not find public key %s in the wallet. Importing it...", s.DbChannelData.PublicKey)
+		_, err = s.daemon.AccountAdd(s.DbChannelData.DesiredChannelName, nil, nil, &s.DbChannelData.PublicKey, util.PtrToBool(true), nil)
 		return errors.Err(err)
 	}
 	return nil
@@ -1048,7 +1020,7 @@ func (s *Sync) getUnsentSupports() (float64, error) {
 	if err != nil {
 		return 0, errors.Err(err)
 	}
-	if s.transferState == 2 {
+	if s.DbChannelData.TransferState == 2 {
 		balance, err := s.daemon.AccountBalance(&defaultAccount)
 		if err != nil {
 			return 0, err
@@ -1079,8 +1051,8 @@ func (s *Sync) getUnsentSupports() (float64, error) {
 				}
 			}
 		}
-		if balanceAmount > 10 && sentSupports < 1 {
-			logUtils.SendErrorToSlack("(%s) this channel has quite some LBCs in it (%.2f) and %.2f LBC in sent tips, it's likely that the tips weren't actually sent or the wallet has unnecessary extra credits in it", s.YoutubeChannelID, balanceAmount, sentSupports)
+		if balanceAmount > 10 && sentSupports < 1 && s.DbChannelData.TransferState > 1 {
+			logUtils.SendErrorToSlack("(%s) this channel has quite some LBCs in it (%.2f) and %.2f LBC in sent tips, it's likely that the tips weren't actually sent or the wallet has unnecessary extra credits in it", s.DbChannelData.ChannelId, balanceAmount, sentSupports)
 			return balanceAmount - 10, nil
 		}
 	}
diff --git a/sdk/api.go b/sdk/api.go
index c542cc8..4d15ae6 100644
--- a/sdk/api.go
+++ b/sdk/api.go
@@ -13,6 +13,7 @@ import (
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/extras/null"
+	"github.com/lbryio/ytsync/v5/shared"
 
 	"github.com/lbryio/ytsync/v5/util"
 
@@ -30,59 +31,21 @@ type APIConfig struct {
 	HostName      string
 }
 
-type SyncProperties struct {
-	SyncFrom         int64
-	SyncUntil        int64
-	YoutubeChannelID string
-}
-
-type SyncFlags struct {
-	StopOnError             bool
-	TakeOverExistingChannel bool
-	SkipSpaceCheck          bool
-	SyncUpdate              bool
-	SingleRun               bool
-	RemoveDBUnpublished     bool
-	UpgradeMetadata         bool
-	DisableTransfers        bool
-	QuickSync               bool
-}
-
-type Fee struct {
-	Amount   string `json:"amount"`
-	Address  string `json:"address"`
-	Currency string `json:"currency"`
-}
-type YoutubeChannel struct {
-	ChannelId          string `json:"channel_id"`
-	TotalVideos        uint   `json:"total_videos"`
-	TotalSubscribers   uint   `json:"total_subscribers"`
-	DesiredChannelName string `json:"desired_channel_name"`
-	Fee                *Fee   `json:"fee"`
-	ChannelClaimID     string `json:"channel_claim_id"`
-	TransferState      int    `json:"transfer_state"`
-	PublishAddress     string `json:"publish_address"`
-	PublicKey          string `json:"public_key"`
-	LengthLimit        int    `json:"length_limit"`
-	SizeLimit          int    `json:"size_limit"`
-	LastUploadedVideo  string `json:"last_uploaded_video"`
-}
-
-func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeChannel, error) {
+func (a *APIConfig) FetchChannels(status string, cliFlags *shared.SyncFlags) ([]shared.YoutubeChannel, error) {
 	type apiJobsResponse struct {
-		Success bool             `json:"success"`
-		Error   null.String      `json:"error"`
-		Data    []YoutubeChannel `json:"data"`
+		Success bool                    `json:"success"`
+		Error   null.String             `json:"error"`
+		Data    []shared.YoutubeChannel `json:"data"`
 	}
 	endpoint := a.ApiURL + "/yt/jobs"
 	res, err := http.PostForm(endpoint, url.Values{
 		"auth_token":  {a.ApiToken},
 		"sync_status": {status},
 		"min_videos":  {strconv.Itoa(1)},
-		"after":       {strconv.Itoa(int(cp.SyncFrom))},
-		"before":      {strconv.Itoa(int(cp.SyncUntil))},
+		"after":       {strconv.Itoa(int(cliFlags.SyncFrom))},
+		"before":      {strconv.Itoa(int(cliFlags.SyncUntil))},
 		"sync_server": {a.HostName},
-		"channel_id":  {cp.YoutubeChannelID},
+		"channel_id":  {cliFlags.ChannelID},
 	})
 	if err != nil {
 		return nil, errors.Err(err)
@@ -93,7 +56,7 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC
 		util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
 		log.Debugln(string(body))
 		time.Sleep(30 * time.Second)
-		return a.FetchChannels(status, cp)
+		return a.FetchChannels(status, cliFlags)
 	}
 	var response apiJobsResponse
 	err = json.Unmarshal(body, &response)
@@ -129,7 +92,6 @@ func sanitizeFailureReason(s *string) {
 }
 
 func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
-
 	type apiSetChannelCertResponse struct {
 		Success bool        `json:"success"`
 		Error   null.String `json:"error"`
@@ -300,19 +262,7 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
 	return errors.Err("invalid API response. Status code: %d", res.StatusCode)
 }
 
-type VideoStatus struct {
-	ChannelID       string
-	VideoID         string
-	Status          string
-	ClaimID         string
-	ClaimName       string
-	FailureReason   string
-	Size            *int64
-	MetaDataVersion uint
-	IsTransferred   *bool
-}
-
-func (a *APIConfig) MarkVideoStatus(status VideoStatus) error {
+func (a *APIConfig) MarkVideoStatus(status shared.VideoStatus) error {
 	endpoint := a.ApiURL + "/yt/video_status"
 
 	sanitizeFailureReason(&status.FailureReason)
diff --git a/shared/shared.go b/shared/shared.go
new file mode 100644
index 0000000..95f425b
--- /dev/null
+++ b/shared/shared.go
@@ -0,0 +1,112 @@
+package shared
+
+import (
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+)
+
+type Fee struct {
+	Amount   string `json:"amount"`
+	Address  string `json:"address"`
+	Currency string `json:"currency"`
+}
+type YoutubeChannel struct {
+	ChannelId          string `json:"channel_id"`
+	TotalVideos        uint   `json:"total_videos"`
+	TotalSubscribers   uint   `json:"total_subscribers"`
+	DesiredChannelName string `json:"desired_channel_name"`
+	Fee                *Fee   `json:"fee"`
+	ChannelClaimID     string `json:"channel_claim_id"`
+	TransferState      int    `json:"transfer_state"`
+	PublishAddress     string `json:"publish_address"`
+	PublicKey          string `json:"public_key"`
+	LengthLimit        int    `json:"length_limit"`
+	SizeLimit          int    `json:"size_limit"`
+	LastUploadedVideo  string `json:"last_uploaded_video"`
+}
+
+type SyncFlags struct {
+	StopOnError             bool
+	TakeOverExistingChannel bool
+	SkipSpaceCheck          bool
+	SyncUpdate              bool
+	SingleRun               bool
+	RemoveDBUnpublished     bool
+	UpgradeMetadata         bool
+	DisableTransfers        bool
+	QuickSync               bool
+	MaxTries                int
+	Refill                  int
+	Limit                   int
+	SyncStatus              string
+	ChannelID               string
+	SyncFrom                int64
+	SyncUntil               int64
+	ConcurrentJobs          int
+	VideosLimit             int
+	MaxVideoSize            int
+	MaxVideoLength          time.Duration
+}
+
+func (f *SyncFlags) IsSingleChannelSync() bool {
+	return f.ChannelID != ""
+}
+
+type VideoStatus struct {
+	ChannelID       string
+	VideoID         string
+	Status          string
+	ClaimID         string
+	ClaimName       string
+	FailureReason   string
+	Size            *int64
+	MetaDataVersion uint
+	IsTransferred   *bool
+}
+
+const (
+	StatusPending        = "pending"        // waiting for permission to sync
+	StatusPendingEmail   = "pendingemail"   // permission granted but missing email
+	StatusQueued         = "queued"         // in sync queue. will be synced soon
+	StatusPendingUpgrade = "pendingupgrade" // in sync queue. will be synced soon
+	StatusSyncing        = "syncing"        // syncing now
+	StatusSynced         = "synced"         // done
+	StatusFailed         = "failed"
+	StatusFinalized      = "finalized" // no more changes allowed
+	StatusAbandoned      = "abandoned" // deleted on youtube or banned
+)
+
+var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned}
+
+const LatestMetadataVersion = 2
+
+const (
+	VideoStatusPublished     = "published"
+	VideoStatusFailed        = "failed"
+	VideoStatusUpgradeFailed = "upgradefailed"
+	VideoStatusUnpublished   = "unpublished"
+	VideoStatusTranferFailed = "transferfailed"
+)
+
+const (
+	TransferStateNotTouched = iota
+	TransferStatePending
+	TransferStateComplete
+	TransferStateManual
+)
+
+type AwsConfigs struct {
+	AwsS3ID     string
+	AwsS3Secret string
+	AwsS3Region string
+	AwsS3Bucket string
+}
+
+func (a *AwsConfigs) GetS3AWSConfig() *aws.Config {
+	return &aws.Config{
+		Credentials: credentials.NewStaticCredentials(a.AwsS3ID, a.AwsS3Secret, ""),
+		Region:      &a.AwsS3Region,
+	}
+}
diff --git a/sources/youtubeVideo.go b/sources/youtubeVideo.go
index 639b394..0b0a3bb 100644
--- a/sources/youtubeVideo.go
+++ b/sources/youtubeVideo.go
@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/lbryio/ytsync/v5/downloader/ytdl"
+	"github.com/lbryio/ytsync/v5/shared"
 
 	"github.com/lbryio/ytsync/v5/ip_manager"
 	"github.com/lbryio/ytsync/v5/namer"
@@ -430,7 +431,7 @@ type SyncParams struct {
 	MaxVideoSize   int
 	Namer          *namer.Namer
 	MaxVideoLength time.Duration
-	Fee            *sdk.Fee
+	Fee            *shared.Fee
 	DefaultAccount string
 }
 
diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go
index 2d19e58..ddf13bc 100644
--- a/ytapi/ytapi.go
+++ b/ytapi/ytapi.go
@@ -12,6 +12,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/lbryio/ytsync/v5/shared"
 	"github.com/lbryio/ytsync/v5/util"
 
 	"github.com/lbryio/ytsync/v5/downloader/ytdl"
@@ -208,7 +209,7 @@ func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopC
 		}
 		video, err := downloader.GetVideoInformation(config, videoID, stopChan, nil, ipPool)
 		if err != nil {
-			errSDK := config.MarkVideoStatus(sdk.VideoStatus{
+			errSDK := config.MarkVideoStatus(shared.VideoStatus{
 				ChannelID:     channelID,
 				VideoID:       videoID,
 				Status:        "failed",

From 7e83b17b69f991b37faeabcfb7c138ab26b2c435 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Sat, 8 Aug 2020 02:31:26 +0200
Subject: [PATCH 44/57] add fallback for socialblade API failures

---
 manager/setup.go | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/manager/setup.go b/manager/setup.go
index ec1e771..17de7fe 100644
--- a/manager/setup.go
+++ b/manager/setup.go
@@ -75,7 +75,8 @@ func (s *Sync) walletSetup() error {
 
 	videosOnYoutube, err := ytapi.CountVideosInChannel(s.DbChannelData.ChannelId)
 	if err != nil {
-		return err
+		logUtils.SendErrorToSlack(errors.FullTrace(errors.Prefix("failed to get video count through socialblade. Falling back to API count", err)))
+		videosOnYoutube = int(s.DbChannelData.TotalVideos)
 	}
 
 	log.Debugf("Source channel has %d videos", videosOnYoutube)
@@ -416,7 +417,7 @@ func (s *Sync) ensureChannelOwnership() error {
 	//}
 	var locations []jsonrpc.Location = nil
 	if channelInfo.Topbar.DesktopTopbarRenderer.CountryCode != "" {
-		locations = []jsonrpc.Location{{Country: util.PtrToString(channelInfo.Topbar.DesktopTopbarRenderer.CountryCode)}}
+		locations = []jsonrpc.Location{{Country: &channelInfo.Topbar.DesktopTopbarRenderer.CountryCode}}
 	}
 	var c *jsonrpc.TransactionSummary
 	claimCreateOptions := jsonrpc.ClaimCreateOptions{

From fb0e567cafb7a90df1fc25e4bd8cc5e922a402b5 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Sat, 8 Aug 2020 03:29:57 +0200
Subject: [PATCH 45/57] be more specific on video failures

---
 ytapi/ytapi.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go
index ddf13bc..f2c61b9 100644
--- a/ytapi/ytapi.go
+++ b/ytapi/ytapi.go
@@ -3,6 +3,7 @@ package ytapi
 import (
 	"bufio"
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"regexp"
@@ -215,7 +216,7 @@ func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopC
 				Status:        "failed",
 				FailureReason: err.Error(),
 			})
-			util.SendErrorToSlack("Skipping video: " + errors.FullTrace(err))
+			util.SendErrorToSlack(fmt.Sprintf("Skipping video (%s): %s", videoID, errors.FullTrace(err)))
 			if errSDK != nil {
 				return nil, errors.Err(errSDK)
 			}

From 67da4142d549c24df24bb5f4c5dbd75d88eb4af8 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Mon, 10 Aug 2020 18:33:05 +0200
Subject: [PATCH 46/57] fix old transferred channels

---
 manager/ytsync.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/manager/ytsync.go b/manager/ytsync.go
index dad8020..45f0da1 100644
--- a/manager/ytsync.go
+++ b/manager/ytsync.go
@@ -239,7 +239,7 @@ func deleteSyncFolder(videoDirectory string) {
 }
 
 func (s *Sync) shouldTransfer() bool {
-	return s.DbChannelData.TransferState >= 1 && s.DbChannelData.PublishAddress != "" && !s.Manager.CliFlags.DisableTransfers
+	return s.DbChannelData.TransferState >= 1 && s.DbChannelData.PublishAddress != "" && !s.Manager.CliFlags.DisableTransfers && s.DbChannelData.TransferState != 3
 }
 
 func (s *Sync) setChannelTerminationStatus(e *error) {
@@ -456,8 +456,8 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
 		claimNameDiffers := claimInDatabase && sv.ClaimName != chainInfo.ClaimName
 		claimMarkedUnpublished := claimInDatabase && !sv.Published
 		_, isOwnClaim := ownClaimsInfo[videoID]
-		tranferred := !isOwnClaim
-		transferStatusMismatch := sv.Transferred != tranferred
+		transferred := !isOwnClaim || s.DbChannelData.TransferState == 3
+		transferStatusMismatch := sv.Transferred != transferred
 
 		if metadataDiffers {
 			log.Debugf("%s: Mismatch in database for metadata. DB: %d - Blockchain: %d", videoID, sv.MetadataVersion, chainInfo.MetadataVersion)
@@ -475,7 +475,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
 			log.Debugf("%s: Published but is not in database (%s - %s)", videoID, chainInfo.ClaimName, chainInfo.ClaimID)
 		}
 		if transferStatusMismatch {
-			log.Debugf("%s: is marked as transferred %t on it's actually %t", videoID, sv.Transferred, tranferred)
+			log.Debugf("%s: is marked as transferred %t on it's actually %t", videoID, sv.Transferred, transferred)
 		}
 
 		if !claimInDatabase || metadataDiffers || claimIDDiffers || claimNameDiffers || claimMarkedUnpublished || transferStatusMismatch {
@@ -493,7 +493,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
 				ClaimName:       chainInfo.ClaimName,
 				Size:            util.PtrToInt64(int64(claimSize)),
 				MetaDataVersion: chainInfo.MetadataVersion,
-				IsTransferred:   &tranferred,
+				IsTransferred:   &transferred,
 			})
 			if err != nil {
 				return count, fixed, 0, err

From eb8900c66ae2e9bd065074c9f45e1998563dd2bd Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Wed, 12 Aug 2020 03:37:18 +0200
Subject: [PATCH 47/57] don't waste loops

fix a deadlock?
---
 downloader/downloader.go | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/downloader/downloader.go b/downloader/downloader.go
index 2c92c1e..60a66b4 100644
--- a/downloader/downloader.go
+++ b/downloader/downloader.go
@@ -245,8 +245,8 @@ func getClient(ip *net.TCPAddr) *http.Client {
 }
 
 func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
-	var maxtries = 10
-	var attemps int
+	var maxTries = 10
+	var attempts int
 	var useragent []string
 	for {
 		sourceAddress, err := getIPFromPool(use, stopChan, pool)
@@ -295,7 +295,7 @@ func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.C
 
 		done := make(chan error, 1)
 		go func() {
-			attemps++
+			attempts++
 			done <- cmd.Wait()
 		}()
 		select {
@@ -309,23 +309,26 @@ func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.C
 				if strings.Contains(err.Error(), "exit status 1") {
 					if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
 						pool.SetThrottled(sourceAddress)
-						logrus.Debugf("known throttling error...try again (%d)", attemps)
-						continue
+						logrus.Debugf("known throttling error...try again (%d)", attempts)
 					}
 					if strings.Contains(string(errorLog), "YouTube said: Unable to extract video data") {
 						useragent = []string{"--user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36"}
-						if attemps == 1 {
+						if attempts == 1 {
 							useragent = []string{"--user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}
 						}
-						logrus.Debugf("known extraction issue, maybe user agent specification will work...try again (%d)", attemps)
-						continue
+						if attempts > 3 {
+							logrus.Debugf("It's pointless to keep trying here... skipping (%d)", attempts)
+							break
+						}
+						logrus.Debugf("known extraction issue, maybe user agent specification will work...try again (%d)", attempts)
 					}
-					if attemps > maxtries {
+					if attempts > maxTries {
 						logrus.Debug("too many tries returning failure")
 						break
 					}
+					continue
 				}
-				logrus.Debugf("Unkown error, returning failure: %s", err.Error())
+				logrus.Debugf("Unknown error, returning failure: %s", err.Error())
 				return nil, errors.Prefix("youtube-dl "+strings.Join(argsForCommand, " ")+" ["+string(errorLog)+"] ", err)
 			}
 			return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil

From 31ce612e2e349b7ee572b97721e1c03aaa8d690e Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Wed, 12 Aug 2020 03:38:52 +0200
Subject: [PATCH 48/57] temporarily disable ipv6

---
 ip_manager/throttle.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ip_manager/throttle.go b/ip_manager/throttle.go
index e37ba69..e500eaf 100644
--- a/ip_manager/throttle.go
+++ b/ip_manager/throttle.go
@@ -45,7 +45,7 @@ func GetIPPool(stopGrp *stop.Group) (*IPPool, error) {
 	var pool []throttledIP
 	for _, address := range addrs {
 		if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsGlobalUnicast() {
-			if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
+			if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) && false {
 				pool = append(pool, throttledIP{
 					IP:      ipnet.IP.String(),
 					LastUse: time.Now().Add(-5 * time.Minute),

From ddca850c17c7d2b7c115bcc9b5917e8d17c9d269 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Wed, 12 Aug 2020 04:17:21 +0200
Subject: [PATCH 49/57] nevermind... re-enable ipv6 (i should make it a flag)

---
 ip_manager/throttle.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ip_manager/throttle.go b/ip_manager/throttle.go
index e500eaf..e37ba69 100644
--- a/ip_manager/throttle.go
+++ b/ip_manager/throttle.go
@@ -45,7 +45,7 @@ func GetIPPool(stopGrp *stop.Group) (*IPPool, error) {
 	var pool []throttledIP
 	for _, address := range addrs {
 		if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsGlobalUnicast() {
-			if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) && false {
+			if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
 				pool = append(pool, throttledIP{
 					IP:      ipnet.IP.String(),
 					LastUse: time.Now().Add(-5 * time.Minute),

From a56166ee51cc2069d46da4d00196208c3b7ea174 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Wed, 12 Aug 2020 19:44:57 +0200
Subject: [PATCH 50/57] refactor youtube-dl execution process

---
 downloader/downloader.go | 162 +++++++++++++++++++--------------------
 1 file changed, 79 insertions(+), 83 deletions(-)

diff --git a/downloader/downloader.go b/downloader/downloader.go
index 60a66b4..4479402 100644
--- a/downloader/downloader.go
+++ b/downloader/downloader.go
@@ -3,7 +3,6 @@ package downloader
 import (
 	"encoding/json"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"net"
 	"net/http"
@@ -25,8 +24,8 @@ import (
 )
 
 func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
-	args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist"}
-	ids, err := run(channelName, args, true, true, stopChan, pool)
+	args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist", "--cookies", "cookies.txt"}
+	ids, err := run(channelName, args, stopChan, pool)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
@@ -44,8 +43,8 @@ func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan,
 const releaseTimeFormat = "2006-01-02, 15:04:05 (MST)"
 
 func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Chan, ip *net.TCPAddr, pool *ip_manager.IPPool) (*ytdl.YtdlVideo, error) {
-	args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID}
-	results, err := run(videoID, args, true, true, stopChan, pool)
+	args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID, "--cookies", "cookies.txt"}
+	results, err := run(videoID, args, stopChan, pool)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
@@ -244,99 +243,96 @@ func getClient(ip *net.TCPAddr) *http.Client {
 	}
 }
 
-func run(use string, args []string, withStdErr, withStdOut bool, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
-	var maxTries = 10
-	var attempts int
+const (
+	googleBotUA             = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
+	chromeUA                = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36"
+	maxAttempts             = 3
+	extractionError         = "YouTube said: Unable to extract video data"
+	throttledError          = "HTTP Error 429"
+	AlternateThrottledError = "returned non-zero exit status 8"
+	youtubeDlError          = "exit status 1"
+)
+
+func run(use string, args []string, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
 	var useragent []string
-	for {
+	var lastError error
+	for attempts := 0; attempts < maxAttempts; attempts++ {
 		sourceAddress, err := getIPFromPool(use, stopChan, pool)
 		if err != nil {
 			return nil, err
 		}
-		defer pool.ReleaseIP(sourceAddress)
 		argsForCommand := append(args, "--source-address", sourceAddress)
 		argsForCommand = append(argsForCommand, useragent...)
 		cmd := exec.Command("youtube-dl", argsForCommand...)
-		logrus.Printf("Running command youtube-dl %s", strings.Join(argsForCommand, " "))
 
-		var stderr io.ReadCloser
-		var errorLog []byte
-		if withStdErr {
-			var err error
-			stderr, err = cmd.StderrPipe()
-			if err != nil {
-				return nil, errors.Err(err)
+		res, err := runCmd(cmd, stopChan)
+		pool.ReleaseIP(sourceAddress)
+		if err == nil {
+			return res, nil
+		}
+		lastError = err
+		if strings.Contains(err.Error(), youtubeDlError) {
+			if strings.Contains(err.Error(), extractionError) {
+				logrus.Warnf("known extraction error: %s", errors.FullTrace(err))
+				useragent = nextUA(useragent)
+			}
+			if strings.Contains(err.Error(), throttledError) || strings.Contains(err.Error(), AlternateThrottledError) {
+				pool.SetThrottled(sourceAddress)
+				//we don't want throttle errors to count toward the max retries
+				attempts--
 			}
 		}
+	}
+	return nil, lastError
+}
 
-		var stdout io.ReadCloser
-		var outLog []byte
-		if withStdOut {
-			var err error
-			stdout, err = cmd.StdoutPipe()
-			if err != nil {
-				return nil, errors.Err(err)
-			}
+func nextUA(current []string) []string {
+	if len(current) == 0 {
+		return []string{"--user-agent", googleBotUA}
+	}
+	return []string{"--user-agent", chromeUA}
+}
 
-			if err := cmd.Start(); err != nil {
-				return nil, errors.Err(err)
-			}
-			outLog, err = ioutil.ReadAll(stdout)
-			if err != nil {
-				return nil, errors.Err(err)
-			}
-			if withStdErr {
-				errorLog, err = ioutil.ReadAll(stderr)
-				if err != nil {
-					return nil, errors.Err(err)
-				}
-			}
+func runCmd(cmd *exec.Cmd, stopChan stop.Chan) ([]string, error) {
+	logrus.Infof("running youtube-dl cmd: %s", strings.Join(cmd.Args, " "))
+	var err error
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	err = cmd.Start()
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	outLog, err := ioutil.ReadAll(stdout)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	errorLog, err := ioutil.ReadAll(stderr)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	done := make(chan error, 1)
+	go func() {
+		done <- cmd.Wait()
+	}()
+
+	select {
+	case <-stopChan:
+		err := cmd.Process.Kill()
+		if err != nil {
+			return nil, errors.Prefix("failed to kill command after stopper cancellation", err)
 		}
-
-		done := make(chan error, 1)
-		go func() {
-			attempts++
-			done <- cmd.Wait()
-		}()
-		select {
-		case <-stopChan:
-			if err := cmd.Process.Kill(); err != nil {
-				return nil, errors.Prefix("failed to kill command after stopper cancellation", err)
-			}
-			return nil, errors.Err("canceled by stopper")
-		case err := <-done:
-			if err != nil {
-				if strings.Contains(err.Error(), "exit status 1") {
-					if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
-						pool.SetThrottled(sourceAddress)
-						logrus.Debugf("known throttling error...try again (%d)", attempts)
-					}
-					if strings.Contains(string(errorLog), "YouTube said: Unable to extract video data") {
-						useragent = []string{"--user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36"}
-						if attempts == 1 {
-							useragent = []string{"--user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}
-						}
-						if attempts > 3 {
-							logrus.Debugf("It's pointless to keep trying here... skipping (%d)", attempts)
-							break
-						}
-						logrus.Debugf("known extraction issue, maybe user agent specification will work...try again (%d)", attempts)
-					}
-					if attempts > maxTries {
-						logrus.Debug("too many tries returning failure")
-						break
-					}
-					continue
-				}
-				logrus.Debugf("Unknown error, returning failure: %s", err.Error())
-				return nil, errors.Prefix("youtube-dl "+strings.Join(argsForCommand, " ")+" ["+string(errorLog)+"] ", err)
-			}
-			return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
-		}
-
-		if len(errorLog) > 0 {
-			return nil, errors.Err(string(errorLog))
+		return nil, errors.Err("canceled by stopper")
+	case err := <-done:
+		if err != nil {
+			return nil, errors.Prefix("youtube-dl "+strings.Join(cmd.Args, " ")+" ["+string(errorLog)+"]", err)
 		}
+		return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
 	}
 }
 

From 0b002c82284a4af858db2597620f41e5f676a1f8 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Tue, 18 Aug 2020 00:03:38 +0200
Subject: [PATCH 51/57] lots of bug fixes and improvements

---
 downloader/downloader.go | 31 ++++++++++++++++++-------------
 manager/ytsync.go        | 13 +------------
 sdk/api.go               | 40 ++++++++++++++++++++++++++++++++++++++++
 shared/shared.go         | 13 +++++++++++++
 sources/youtubeVideo.go  | 23 +++++++++++++++++++----
 ytapi/ytapi.go           | 17 ++++++++++++-----
 6 files changed, 103 insertions(+), 34 deletions(-)

diff --git a/downloader/downloader.go b/downloader/downloader.go
index 4479402..404bab6 100644
--- a/downloader/downloader.go
+++ b/downloader/downloader.go
@@ -125,7 +125,7 @@ func triggerScrape(videoID string, ip *net.TCPAddr) error {
 	if err != nil {
 		return errors.Err(err)
 	}
-	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36")
+	req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36")
 
 	res, err := client.Do(req)
 	if err != nil {
@@ -168,20 +168,25 @@ func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr, uploa
 	if err != nil {
 		logrus.Error(err)
 	}
-	if release != nil {
-		//const sqlTimeFormat = "2006-01-02 15:04:05"
-		sqlTime, err := time.ParseInLocation(time.RFC3339, release.ReleaseTime, time.UTC)
-		if err == nil {
-			return sqlTime.Format(releaseTimeFormat), nil
-		} else {
-			logrus.Error(err)
-		}
-	}
 	ytdlUploadDate, err := time.Parse("20060102", uploadDate)
 	if err != nil {
 		logrus.Error(err)
 	}
-	if time.Now().AddDate(0, 0, -2).After(ytdlUploadDate) {
+	if release != nil {
+		//const sqlTimeFormat = "2006-01-02 15:04:05"
+		sqlTime, err := time.ParseInLocation(time.RFC3339, release.ReleaseTime, time.UTC)
+		if err == nil {
+			if sqlTime.Day() != ytdlUploadDate.Day() {
+				logrus.Infof("upload day from APIs differs from the ytdl one by more than 1 day.")
+			} else {
+				return sqlTime.Format(releaseTimeFormat), nil
+			}
+		} else {
+			logrus.Error(err)
+		}
+	}
+
+	if time.Now().AddDate(0, 0, -3).After(ytdlUploadDate) {
 		return ytdlUploadDate.Format(releaseTimeFormat), nil
 	}
 	client := getClient(ip)
@@ -189,7 +194,7 @@ func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr, uploa
 	if err != nil {
 		return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
 	}
-	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36")
+	req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36")
 
 	res, err := client.Do(req)
 	if err != nil {
@@ -245,7 +250,7 @@ func getClient(ip *net.TCPAddr) *http.Client {
 
 const (
 	googleBotUA             = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
-	chromeUA                = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36"
+	chromeUA                = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"
 	maxAttempts             = 3
 	extractionError         = "YouTube said: Unable to extract video data"
 	throttledError          = "HTTP Error 429"
diff --git a/manager/ytsync.go b/manager/ytsync.go
index 45f0da1..e1ab683 100644
--- a/manager/ytsync.go
+++ b/manager/ytsync.go
@@ -918,18 +918,7 @@ func (s *Sync) processVideo(v ytapi.Video) (err error) {
 	alreadyPublished := ok && sv.Published
 	videoRequiresUpgrade := ok && s.Manager.CliFlags.UpgradeMetadata && sv.MetadataVersion < newMetadataVersion
 
-	neverRetryFailures := []string{
-		"Error extracting sts from embedded url response",
-		"Unable to extract signature tokens",
-		"the video is too big to sync, skipping for now",
-		"video is too long to process",
-		"This video contains content from",
-		"no compatible format available for this video",
-		"Watch this video on YouTube.",
-		"have blocked it on copyright grounds",
-		"giving up after 0 fragment retries",
-		"Sign in to confirm your age",
-	}
+	neverRetryFailures := shared.NeverRetryFailures
 	if ok && !sv.Published && util.SubstringInSlice(sv.FailureReason, neverRetryFailures) {
 		log.Println(v.ID() + " can't ever be published")
 		return nil
diff --git a/sdk/api.go b/sdk/api.go
index 4d15ae6..8dbb5ca 100644
--- a/sdk/api.go
+++ b/sdk/api.go
@@ -48,6 +48,11 @@ func (a *APIConfig) FetchChannels(status string, cliFlags *shared.SyncFlags) ([]
 		"channel_id":  {cliFlags.ChannelID},
 	})
 	if err != nil {
+		if strings.Contains(err.Error(), "EOF") {
+			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+			time.Sleep(30 * time.Second)
+			return a.FetchChannels(status, cliFlags)
+		}
 		return nil, errors.Err(err)
 	}
 	defer res.Body.Close()
@@ -106,6 +111,11 @@ func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
 		"auth_token":       {a.ApiToken},
 	})
 	if err != nil {
+		if strings.Contains(err.Error(), "EOF") {
+			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+			time.Sleep(30 * time.Second)
+			return a.SetChannelCert(certHex, channelID)
+		}
 		return errors.Err(err)
 	}
 	defer res.Body.Close()
@@ -149,6 +159,11 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
 	}
 	res, err := http.PostForm(endpoint, params)
 	if err != nil {
+		if strings.Contains(err.Error(), "EOF") {
+			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+			time.Sleep(30 * time.Second)
+			return a.SetChannelStatus(channelID, status, failureReason, transferState)
+		}
 		return nil, nil, errors.Err(err)
 	}
 	defer res.Body.Close()
@@ -194,6 +209,11 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
 		"channel_claim_id": {channelClaimID},
 	})
 	if err != nil {
+		if strings.Contains(err.Error(), "EOF") {
+			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+			time.Sleep(30 * time.Second)
+			return a.SetChannelClaimID(channelID, channelClaimID)
+		}
 		return errors.Err(err)
 	}
 	defer res.Body.Close()
@@ -233,6 +253,11 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
 	}
 	res, err := http.PostForm(endpoint, vals)
 	if err != nil {
+		if strings.Contains(err.Error(), "EOF") {
+			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+			time.Sleep(30 * time.Second)
+			return a.DeleteVideos(videos)
+		}
 		return errors.Err(err)
 	}
 	defer res.Body.Close()
@@ -294,6 +319,11 @@ func (a *APIConfig) MarkVideoStatus(status shared.VideoStatus) error {
 	}
 	res, err := http.PostForm(endpoint, vals)
 	if err != nil {
+		if strings.Contains(err.Error(), "EOF") {
+			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+			time.Sleep(30 * time.Second)
+			return a.MarkVideoStatus(status)
+		}
 		return errors.Err(err)
 	}
 	defer res.Body.Close()
@@ -331,6 +361,11 @@ func (a *APIConfig) VideoState(videoID string) (string, error) {
 
 	res, err := http.PostForm(endpoint, vals)
 	if err != nil {
+		if strings.Contains(err.Error(), "EOF") {
+			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+			time.Sleep(30 * time.Second)
+			return a.VideoState(videoID)
+		}
 		return "", errors.Err(err)
 	}
 	defer res.Body.Close()
@@ -380,6 +415,11 @@ func (a *APIConfig) GetReleasedDate(videoID string) (*VideoRelease, error) {
 
 	res, err := http.PostForm(endpoint, vals)
 	if err != nil {
+		if strings.Contains(err.Error(), "EOF") {
+			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+			time.Sleep(30 * time.Second)
+			return a.GetReleasedDate(videoID)
+		}
 		return nil, errors.Err(err)
 	}
 	defer res.Body.Close()
diff --git a/shared/shared.go b/shared/shared.go
index 95f425b..28b8077 100644
--- a/shared/shared.go
+++ b/shared/shared.go
@@ -27,6 +27,19 @@ type YoutubeChannel struct {
 	LastUploadedVideo  string `json:"last_uploaded_video"`
 }
 
+var NeverRetryFailures = []string{
+	"Error extracting sts from embedded url response",
+	"Unable to extract signature tokens",
+	"the video is too big to sync, skipping for now",
+	"video is too long to process",
+	"This video contains content from",
+	"no compatible format available for this video",
+	"Watch this video on YouTube.",
+	"have blocked it on copyright grounds",
+	"giving up after 0 fragment retries",
+	"Sign in to confirm your age",
+}
+
 type SyncFlags struct {
 	StopOnError             bool
 	TakeOverExistingChannel bool
diff --git a/sources/youtubeVideo.go b/sources/youtubeVideo.go
index 0b0a3bb..6118745 100644
--- a/sources/youtubeVideo.go
+++ b/sources/youtubeVideo.go
@@ -209,6 +209,15 @@ func (v *YoutubeVideo) download() error {
 		"480",
 		"320",
 	}
+	dur := time.Duration(v.youtubeInfo.Duration) * time.Second
+	if dur.Hours() > 2 { //for videos longer than 2 hours only sync up to 720p
+		qualities = []string{
+			"720",
+			"480",
+			"320",
+		}
+	}
+
 	ytdlArgs := []string{
 		"--no-progress",
 		"-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"),
@@ -219,12 +228,11 @@ func (v *YoutubeVideo) download() error {
 		"-movflags faststart",
 		"--abort-on-unavailable-fragment",
 		"--fragment-retries",
-		"0",
-		"--user-agent",
-		"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36",
+		"1",
 		"--cookies",
 		"cookies.txt",
 	}
+	userAgent := []string{"--user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"}
 	if v.maxVideoSize > 0 {
 		ytdlArgs = append(ytdlArgs,
 			"--max-filesize",
@@ -264,8 +272,10 @@ func (v *YoutubeVideo) download() error {
 		"https://www.youtube.com/watch?v="+v.ID(),
 	)
 
-	for i, quality := range qualities {
+	for i := 0; i < len(qualities); i++ {
+		quality := qualities[i]
 		argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+quality+"]+bestaudio[ext!=webm]")
+		argsWithFilters = append(argsWithFilters, userAgent...)
 		cmd := exec.Command("youtube-dl", argsWithFilters...)
 		log.Printf("Running command youtube-dl %s", strings.Join(argsWithFilters, " "))
 
@@ -294,6 +304,11 @@ func (v *YoutubeVideo) download() error {
 						return errors.Err(string(errorLog))
 					}
 					continue //this bypasses the yt throttling IP redistribution... TODO: don't
+				} else if strings.Contains(string(errorLog), "YouTube said: Unable to extract video data") && !strings.Contains(userAgent[1], "Googlebot") {
+					i-- //do not lower quality when trying a different user agent
+					userAgent = []string{"--user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}
+					log.Infof("trying different user agent for video %s", v.ID())
+					continue
 				}
 				return errors.Err(string(errorLog))
 			}
diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go
index f2c61b9..9cb2acf 100644
--- a/ytapi/ytapi.go
+++ b/ytapi/ytapi.go
@@ -14,7 +14,7 @@ import (
 	"time"
 
 	"github.com/lbryio/ytsync/v5/shared"
-	"github.com/lbryio/ytsync/v5/util"
+	logUtils "github.com/lbryio/ytsync/v5/util"
 
 	"github.com/lbryio/ytsync/v5/downloader/ytdl"
 
@@ -26,6 +26,7 @@ import (
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
 	"github.com/lbryio/lbry.go/v2/extras/stop"
+	"github.com/lbryio/lbry.go/v2/extras/util"
 
 	"github.com/aws/aws-sdk-go/aws"
 	log "github.com/sirupsen/logrus"
@@ -56,16 +57,22 @@ type VideoParams struct {
 var mostRecentlyFailedChannel string // TODO: fix this hack!
 
 func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams, lastUploadedVideo string) ([]Video, error) {
-
 	var videos []Video
 	if quickSync && maxVideos > 50 {
 		maxVideos = 50
 	}
-	videoIDs, err := downloader.GetPlaylistVideoIDs(channelID, maxVideos, videoParams.Stopper.Ch(), videoParams.IPPool)
+	allVideos, err := downloader.GetPlaylistVideoIDs(channelID, maxVideos, videoParams.Stopper.Ch(), videoParams.IPPool)
 	if err != nil {
 		return nil, errors.Err(err)
 	}
-
+	videoIDs := make([]string, 0, len(allVideos))
+	for _, video := range allVideos {
+		sv, ok := syncedVideos[video]
+		if ok && util.SubstringInSlice(sv.FailureReason, shared.NeverRetryFailures) {
+			continue
+		}
+		videoIDs = append(videoIDs, video)
+	}
 	log.Infof("Got info for %d videos from youtube downloader", len(videoIDs))
 
 	playlistMap := make(map[string]int64)
@@ -216,7 +223,7 @@ func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopC
 				Status:        "failed",
 				FailureReason: err.Error(),
 			})
-			util.SendErrorToSlack(fmt.Sprintf("Skipping video (%s): %s", videoID, errors.FullTrace(err)))
+			logUtils.SendErrorToSlack(fmt.Sprintf("Skipping video (%s): %s", videoID, errors.FullTrace(err)))
 			if errSDK != nil {
 				return nil, errors.Err(errSDK)
 			}

From 24cf937e14a4ac37fca3766287a79295185b4b24 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Tue, 18 Aug 2020 01:09:53 +0200
Subject: [PATCH 52/57] retry on support list failures

---
 manager/transfer.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/manager/transfer.go b/manager/transfer.go
index d04272d..fde2530 100644
--- a/manager/transfer.go
+++ b/manager/transfer.go
@@ -71,7 +71,10 @@ func abandonSupports(s *Sync) (float64, error) {
 	for page := uint64(1); page <= totalPages; page++ {
 		supports, err := s.daemon.SupportList(&defaultAccount, page, 50)
 		if err != nil {
-			return 0, errors.Prefix("cannot list claims", err)
+			supports, err = s.daemon.SupportList(&defaultAccount, page, 50)
+			if err != nil {
+				return 0, errors.Prefix("cannot list supports", err)
+			}
 		}
 		allSupports = append(allSupports, (*supports).Items...)
 		totalPages = (*supports).TotalPages

From ecda80b02dee251820d3b6f21429c8292d0bd94c Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Fri, 21 Aug 2020 00:15:14 +0200
Subject: [PATCH 53/57] bypass socialblade block

stop using socialblade for now
---
 manager/setup.go |  6 +-----
 ytapi/ytapi.go   | 14 +++++++++++---
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/manager/setup.go b/manager/setup.go
index 17de7fe..e61768e 100644
--- a/manager/setup.go
+++ b/manager/setup.go
@@ -73,11 +73,7 @@ func (s *Sync) walletSetup() error {
 	}
 	log.Debugf("Starting balance is %.4f", balance)
 
-	videosOnYoutube, err := ytapi.CountVideosInChannel(s.DbChannelData.ChannelId)
-	if err != nil {
-		logUtils.SendErrorToSlack(errors.FullTrace(errors.Prefix("failed to get video count through socialblade. Falling back to API count", err)))
-		videosOnYoutube = int(s.DbChannelData.TotalVideos)
-	}
+	videosOnYoutube := int(s.DbChannelData.TotalVideos)
 
 	log.Debugf("Source channel has %d videos", videosOnYoutube)
 	if videosOnYoutube == 0 {
diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go
index 9cb2acf..575e82e 100644
--- a/ytapi/ytapi.go
+++ b/ytapi/ytapi.go
@@ -123,15 +123,23 @@ func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[s
 	return videos, nil
 }
 
+// CountVideosInChannel is unused for now... keeping it here just in case
 func CountVideosInChannel(channelID string) (int, error) {
-	res, err := http.Get("https://socialblade.com/youtube/channel/" + channelID)
+	url := "https://socialblade.com/youtube/channel/" + channelID
+
+	req, _ := http.NewRequest("GET", url, nil)
+
+	req.Header.Add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36")
+	req.Header.Add("Accept", "*/*")
+	req.Header.Add("Host", "socialblade.com")
+
+	res, err := http.DefaultClient.Do(req)
 	if err != nil {
-		return 0, err
+		return 0, errors.Err(err)
 	}
 	defer res.Body.Close()
 
 	var line string
-
 	scanner := bufio.NewScanner(res.Body)
 	for scanner.Scan() {
 		if strings.Contains(scanner.Text(), "youtube-stats-header-uploads") {

From ad5a30da9e7aeaafed2025f5f6f3807d371a2fdd Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Tue, 25 Aug 2020 18:44:44 +0200
Subject: [PATCH 54/57] work around transfer failures

---
 manager/ytsync.go | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/manager/ytsync.go b/manager/ytsync.go
index e1ab683..b866be3 100644
--- a/manager/ytsync.go
+++ b/manager/ytsync.go
@@ -207,17 +207,21 @@ func (s *Sync) processTransfers() (e error) {
 		isTip := true
 		summary, err := s.daemon.SupportCreate(s.DbChannelData.ChannelClaimID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount}, nil)
 		if err != nil {
-			if strings.Contains(err.Error(), "tx-size") { //TODO: this is a silly workaround and should be written in an recursive function
-				summary, err = s.daemon.SupportCreate(s.DbChannelData.ChannelClaimID, fmt.Sprintf("%.6f", supportAmount/2.0), &isTip, nil, []string{defaultAccount}, nil)
+			if strings.Contains(err.Error(), "tx-size") { //TODO: this is a silly workaround...
+				_, spendErr := s.daemon.TxoSpend(util.PtrToString("other"), nil, nil, nil, nil, &s.defaultAccountID)
+				if spendErr != nil {
+					return errors.Prefix(fmt.Sprintf("something went wrong while tipping the channel for %.6f LBCs", supportAmount), err)
+				}
+				err = s.waitForNewBlock()
+				if err != nil {
+					return errors.Prefix(fmt.Sprintf("something went wrong while tipping the channel for %.6f LBCs (waiting for new block)", supportAmount), err)
+				}
+				summary, err = s.daemon.SupportCreate(s.DbChannelData.ChannelClaimID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount}, nil)
 				if err != nil {
 					return errors.Prefix(fmt.Sprintf("something went wrong while tipping the channel for %.6f LBCs", supportAmount), err)
 				}
-				summary, err = s.daemon.SupportCreate(s.DbChannelData.ChannelClaimID, fmt.Sprintf("%.6f", supportAmount/2.0), &isTip, nil, []string{defaultAccount}, nil)
-				if err != nil {
-					return errors.Err(err)
-				}
 			} else {
-				return errors.Err(err)
+				return errors.Prefix(fmt.Sprintf("something went wrong while tipping the channel for %.6f LBCs", supportAmount), err)
 			}
 		}
 		if len(summary.Outputs) < 1 {

From c53cf4c1b31fd070fa8387bbb87478dddc9416d4 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Wed, 26 Aug 2020 19:05:16 +0200
Subject: [PATCH 55/57] prevent channels from failing when manually killed

---
 ytapi/ytapi.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go
index 575e82e..32b99f9 100644
--- a/ytapi/ytapi.go
+++ b/ytapi/ytapi.go
@@ -207,7 +207,7 @@ func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopC
 		}
 		select {
 		case <-stopChan:
-			return videos, errors.Err("canceled by stopper")
+			return videos, errors.Err("interrupted by user")
 		default:
 		}
 

From d4ca71a89d3d0a6570f862c3b5b52a5e7ce1ca6e Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Mon, 31 Aug 2020 21:27:39 +0200
Subject: [PATCH 56/57] fix channel description

---
 manager/setup.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/manager/setup.go b/manager/setup.go
index e61768e..e2a809e 100644
--- a/manager/setup.go
+++ b/manager/setup.go
@@ -418,7 +418,7 @@ func (s *Sync) ensureChannelOwnership() error {
 	var c *jsonrpc.TransactionSummary
 	claimCreateOptions := jsonrpc.ClaimCreateOptions{
 		Title:        &channelInfo.Microformat.MicroformatDataRenderer.Title,
-		Description:  &channelInfo.Microformat.MicroformatDataRenderer.Description,
+		Description:  &channelInfo.Metadata.ChannelMetadataRenderer.Description,
 		Tags:         tags_manager.GetTagsForChannel(s.DbChannelData.ChannelId),
 		Languages:    languages,
 		Locations:    locations,

From ad6fa4d725354615b8c087f6a5fb511f4e860d3e Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Tue, 1 Sep 2020 20:27:39 +0200
Subject: [PATCH 57/57] update dependencies

---
 blobs_reflector/reflect.go |   4 +-
 go.mod                     |  14 +-
 go.sum                     | 260 ++++++++++++++++++++++++++++++++-----
 3 files changed, 237 insertions(+), 41 deletions(-)

diff --git a/blobs_reflector/reflect.go b/blobs_reflector/reflect.go
index 7758c24..81e3cd2 100644
--- a/blobs_reflector/reflect.go
+++ b/blobs_reflector/reflect.go
@@ -72,12 +72,12 @@ func reflectBlobs() error {
 			return errors.Err(err)
 		}
 	}
-	st := store.NewDBBackedS3Store(
+	st := store.NewDBBackedStore(
 		store.NewS3BlobStore(config.AwsID, config.AwsSecret, config.BucketRegion, config.BucketName),
 		dbHandle)
 
 	uploadWorkers := 10
-	uploader := reflector.NewUploader(dbHandle, st, uploadWorkers, false)
+	uploader := reflector.NewUploader(dbHandle, st, uploadWorkers, false, false)
 	usr, err := user.Current()
 	if err != nil {
 		return errors.Err(err)
diff --git a/go.mod b/go.mod
index 22196db..da33ca3 100644
--- a/go.mod
+++ b/go.mod
@@ -4,22 +4,22 @@ replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203
 
 require (
 	github.com/Microsoft/go-winio v0.4.14 // indirect
-	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a
+	github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb
 	github.com/aws/aws-sdk-go v1.25.9
 	github.com/davecgh/go-spew v1.1.1
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/docker v1.13.1
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
-	github.com/go-sql-driver/mysql v1.4.1 // indirect
-	github.com/google/btree v1.0.0 // indirect
+	github.com/golang/protobuf v1.4.2 // indirect
 	github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
 	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
-	github.com/hashicorp/golang-lru v0.5.3 // indirect
 	github.com/hashicorp/memberlist v0.1.5 // indirect
 	github.com/hashicorp/serf v0.8.5 // indirect
-	github.com/lbryio/lbry.go/v2 v2.6.1-0.20200710180140-fcade7475323
-	github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6
+	github.com/kr/pretty v0.2.1 // indirect
+	github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128
+	github.com/lbryio/reflector.go v1.1.3-0.20200901181534-e8b98bc862d5
+	github.com/lbryio/types v0.0.0-20200605192618-366870b2862d // indirect
 	github.com/miekg/dns v1.1.22 // indirect
 	github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
 	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
@@ -27,12 +27,14 @@ require (
 	github.com/prometheus/client_golang v0.9.2
 	github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5
 	github.com/sirupsen/logrus v1.4.2
+	github.com/smartystreets/goconvey v1.6.4 // indirect
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/testify v1.4.0
 	golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
 	google.golang.org/appengine v1.6.5 // indirect
+	gopkg.in/ini.v1 v1.60.2 // indirect
 )
 
 go 1.13
diff --git a/go.sum b/go.sum
index f8386b4..409d083 100644
--- a/go.sum
+++ b/go.sum
@@ -1,9 +1,20 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo=
+dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU=
+dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU=
+dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
+dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
+git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
 github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
+github.com/StackExchange/wmi v0.0.0-20170410192909-ea383cf3ba6e/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
@@ -11,14 +22,16 @@ github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7
 github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb h1:kvlW1qyM1aU3xeyeIVTU2jx5fSvjKpsU3aXvuaCMg3Q=
+github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.25.9 h1:WtVzerf5wSgPwlTTwl+ktCq/0GCS5MI9ZlLIcjsTr+Q=
 github.com/aws/aws-sdk-go v1.25.9/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo=
 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
 github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
@@ -31,17 +44,22 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku
 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc=
 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
 github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
+github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
+github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
+github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
 github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
 github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
@@ -50,50 +68,79 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
+github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
 github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-ini/ini v1.48.0 h1:TvO60hO/2xgaaTWp2P0wUe4CFxwdMzfbkv3+343Xzqw=
 github.com/go-ini/ini v1.48.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
+github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
 github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
 github.com/go-ozzo/ozzo-validation v3.6.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
-github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
+github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.4.0 h1:Rd1kQnQu0Hq3qvJppYSG0HtP+f5LPPUiDswTLiEegLg=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/gops v0.3.7/go.mod h1:bj0cwMmX1X4XIJFTjR99R5sCxNssNJ8HebFNvoQlmgY=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
+github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f h1:TyqzGm2z1h3AGhjOoRYyeLcW4WlW81MDQkWa+rx/000=
 github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc=
+github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
 github.com/gorilla/rpc v1.2.0 h1:WvvdC2lNeT1SP32zrIce5l0ECBfbAlmrmSBsuc57wfk=
 github.com/gorilla/rpc v1.2.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
+github.com/gorilla/schema v1.0.2/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU=
 github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
 github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
@@ -117,6 +164,8 @@ github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk=
 github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
@@ -133,54 +182,78 @@ github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7V
 github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
 github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmoiron/sqlx v0.0.0-20170430194603-d9bd385d68c0/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU=
 github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 h1:jKUP9TQ0c7X3w6+IPyMit07RE42MtTWNd77sN2cHngQ=
 github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22/go.mod h1:u0Jo4f2dNlTJeeOywkM6bLwxq6gC3pZ9rEFHn3AhTdk=
+github.com/johntdyer/slackrus v0.0.0-20170926115001-3992f319fd0a/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
 github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07 h1:+kBG/8rjCa6vxJZbUjAiE4MQmBEBYc8nLEb51frnvBY=
 github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
 github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jteeuwen/go-bindata v3.0.7+incompatible/go.mod h1:JVvhzYOiGBnFSYRyV00iY8q7/0PThjIYav1p9h5dmKs=
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
+github.com/keybase/go-ps v0.0.0-20161005175911-668c8856d999/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
+github.com/lbryio/chainquery v1.9.0 h1:NfBZ3eKYwD3PqXU/vt+2tF3ox3WUWoW4J5YdEQ0rxw0=
+github.com/lbryio/chainquery v1.9.0/go.mod h1:7G8l7jNtANS1I7fQOvtzbiHsv6qKVmN4codXHc3C4kk=
 github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
-github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303/go.mod h1:qR+Ui0hYhemIU4fXqM3d1P9eiaRFlof777VJgV7KJ8w=
+github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
 github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
 github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
-github.com/lbryio/lbry.go/v2 v2.6.1-0.20200710180140-fcade7475323 h1:Fvngg+hybcA1O2783YBw07S2MGpemSO+vWYSk97QHdM=
-github.com/lbryio/lbry.go/v2 v2.6.1-0.20200710180140-fcade7475323/go.mod h1:LgFKEpZzJE72DVgSXXfg+2IGOhrC9Lzj6eLma18iNz8=
+github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128 h1:VL209c+AGKixMFpxT+TsOAPzBPuoUzyjXf47iNe7OzY=
+github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128/go.mod h1:RqOv4V5eWY/JGmduCPcQVdN19SEYnNY3SuF+arTKIU4=
 github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
 github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
 github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
-github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46 h1:LemfR+rMxhf7nnOrzy2HqS7Me7SZ5gEwOcNFzKC8ySQ=
 github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
 github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
-github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6 h1:8k12lI18EEgMHkgGrgtRvJGl2Bq1vDSEbE4s+QaQ5Qc=
-github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6/go.mod h1:Q1Cnuv5iLsB2rS4Vr0o0EJr4Gs/EgDXFH2V4WtoM5o8=
+github.com/lbryio/reflector.go v1.1.3-0.20200901181534-e8b98bc862d5 h1:Y+zuIZQ1QwpBKLsdfThALHMwQSR4i0H67Znen6YdT54=
+github.com/lbryio/reflector.go v1.1.3-0.20200901181534-e8b98bc862d5/go.mod h1:QHKtgkmOCJJKZ578AFmN3jlBUhJFykbVEGgOuEiQ+aY=
 github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
-github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8 h1:jSNW/rK6DQsz7Zh+iv1zR384PeQdHt0gS4hKY17tkuM=
 github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
+github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
+github.com/lbryio/types v0.0.0-20200605192618-366870b2862d h1:Pg/4L7ExGXsseTMy1d/LK9TY55c8CP2gtmafmYzed0Q=
+github.com/lbryio/types v0.0.0-20200605192618-366870b2862d/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
+github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lucas-clemente/quic-go v0.17.2 h1:4iQInIuNQkPNZmsy9rCnwuOzpH0qGnDo4jn0QfI/qE4=
+github.com/lucas-clemente/quic-go v0.17.2/go.mod h1:I0+fcNTdb9eS1ZcjQZbDVPGchJ86chcIxPALn9lEJqE=
+github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
 github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
 github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcXMWfkoumVwhcCbf3jHF9QKv/m37BkM0=
 github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4=
+github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/marten-seemann/qpack v0.1.0 h1:/0M7lkda/6mus9B8u34Asqm8ZhHAAt9Ho0vniNuVSVg=
+github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI=
+github.com/marten-seemann/qtls v0.9.1 h1:O0YKQxNVPaiFgMng0suWEOY2Sb4LT2sRn9Qimq3Z1IQ=
+github.com/marten-seemann/qtls v0.9.1/go.mod h1:T1MmAdDPyISzxlK6kjRr0pcZFBVd1OZbBb/j3cvzHhk=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.22 h1:Jm64b3bO9kP43ddLjL2EY3Io6bmy1qGb9Xxz6TqS6rc=
 github.com/miekg/dns v1.1.22/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
@@ -195,21 +268,29 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F
 github.com/mitchellh/mapstructure v0.0.0-20180511142126-bb74f1db0675/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
+github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
 github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
 github.com/nlopes/slack v0.6.0 h1:jt0jxVQGhssx1Ib7naAOZEZcGdtIhTzkP0nopK0AsRA=
 github.com/nlopes/slack v0.6.0/go.mod h1:JzQ9m3PMAqcpeCam7UaHSuBuupz7CmpjehYMayT6YOk=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94=
 github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw=
+github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34=
+github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
 github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=
@@ -217,17 +298,22 @@ github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rK
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/profile v1.3.0/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
+github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
 github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
 github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
 github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/rubenv/sql-migrate v0.0.0-20170330050058-38004e7a77f2/go.mod h1:WS0rl9eEliYI8DPnr3TOwz4439pay+qNgzJoVya/DmY=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@@ -238,11 +324,36 @@ github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77 h1:Msb6XRY62jQOueNNl
 github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4=
 github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/sfreiberg/gotwilio v0.0.0-20180612161623-8fb7259ba8bf/go.mod h1:60PiR0SAnAcYSiwrXB6BaxeqHdXMf172toCosHfV+Yk=
+github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
 github.com/shopspring/decimal v0.0.0-20180607144847-19e3cb6c2930/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
+github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
 github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5 h1:Gojs/hac/DoYEM7WEICT45+hNWczIeuL5D21e5/HPAw=
 github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
+github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
+github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM=
+github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0=
+github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
+github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
+github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
+github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI=
+github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU=
+github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag=
+github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg=
+github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw=
+github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y=
+github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
+github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q=
+github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ=
+github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I=
+github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0=
+github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ=
+github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk=
+github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4=
+github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -250,19 +361,29 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
 github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
 github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
-github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
 github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
+github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
+github.com/spf13/afero v1.1.1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
 github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.0-20180722215644-7c4570c3ebeb/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
+github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -270,108 +391,174 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
-github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o=
-github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
+github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
+github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d/go.mod h1:jspfvgf53t5NLUT4o9L1IX0kIBNKamGq1tWc/MgWK9Q=
+github.com/volatiletech/null v8.0.0+incompatible/go.mod h1:0wD98JzdqB+rLyZ70fN05VDbXbafIb0KU0MdVhCzmOQ=
+github.com/volatiletech/sqlboiler v3.4.0+incompatible/go.mod h1:jLfDkkHWPbS2cWRLkyC20vQWaIQsASEY7gM7zSo11Yw=
+github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d h1:tQo6hjclyv3RHUgZOl6iWb2Y44A/sN9bf9LAYfuioEg=
 github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE=
+github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
+go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM=
+go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
+golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
 golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
-golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5hzyggAkLLlCUjqfRxd8Q4=
 golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5 h1:Q7tZBpemrlsc2I7IyODzhtallWRSm4Q0d09pL6XbQtU=
+golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190228165749-92fc7df08ae7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191009170851-d66e71096ffb h1:TR699M2v0qoKTOHxeLgp6zPqaQNs74f01a/ob9W0qko=
 golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20171017063910-8dbc5d05d6ed/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f h1:hjzMYz/7Ea1mNKfOnFOfktR0mlA5jqhvywClCMHM/qw=
 golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w=
 golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 h1:Toz2IK7k8rbltAXwNAxKcn9OzqyNfMUhUNjz3sL0NMk=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
+google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
+google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03 h1:4HYDjxeNXAOTv3o1N2tjo8UUSlhQgAD52FVkwxnWgM8=
+google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
+google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
+google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s=
 google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/gorp.v1 v1.7.1/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/ini.v1 v1.48.0 h1:URjZc+8ugRY5mL5uUeQH/a63JcHwdX9xZaWvmNWD7z8=
 gopkg.in/ini.v1 v1.48.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.60.2 h1:7i8mqModL63zqi8nQn8Q3+0zvSCZy1AxhBgthKfi4WU=
+gopkg.in/ini.v1 v1.60.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 h1:FpCr9V8wuOei4BAen+93HtVJ+XSi+KPbaPKm0Vj5R64=
 gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79/go.mod h1:gWkaRU7CoXpezCBWfWjm3999QqS+1pYPXGbqQCTMzo8=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
@@ -382,6 +569,13 @@ gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
+grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+rsc.io/goversion v1.0.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
+sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=