Merge branch 'fuck-yt'

Makefile

@@ -13,7 +13,7 @@ LDFLAGS = -ldflags "-X ${IMPORT_PATH}/meta.Version=${VERSION} -X ${IMPORT_PATH}/
 
 
 build:
-	mkdir -p ${BIN_DIR} && CGO_ENABLED=0 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
+	mkdir -p ${BIN_DIR} && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
 
 clean:
 	if [ -f ${BIN_DIR}/${BINARY} ]; then rm ${BIN_DIR}/${BINARY}; fi

blobs_reflector/reflect.go

@@ -72,12 +72,12 @@ func reflectBlobs() error {
 			return errors.Err(err)
 		}
 	}
-	st := store.NewDBBackedS3Store(
+	st := store.NewDBBackedStore(
 		store.NewS3BlobStore(config.AwsID, config.AwsSecret, config.BucketRegion, config.BucketName),
 		dbHandle)
 
 	uploadWorkers := 10
-	uploader := reflector.NewUploader(dbHandle, st, uploadWorkers, false)
+	uploader := reflector.NewUploader(dbHandle, st, uploadWorkers, false, false)
 	usr, err := user.Current()
 	if err != nil {
 		return errors.Err(err)

downloader/downloader.go

@@ -0,0 +1,364 @@
+package downloader
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/lbryio/ytsync/v5/downloader/ytdl"
+	"github.com/lbryio/ytsync/v5/ip_manager"
+	"github.com/lbryio/ytsync/v5/sdk"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/stop"
+	"github.com/lbryio/lbry.go/v2/extras/util"
+
+	"github.com/sirupsen/logrus"
+)
+
+func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
+	args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist", "--cookies", "cookies.txt"}
+	ids, err := run(channelName, args, stopChan, pool)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	videoIDs := make([]string, maxVideos)
+	for i, v := range ids {
+		logrus.Debugf("%d - video id %s", i, v)
+		if i >= maxVideos {
+			break
+		}
+		videoIDs[i] = v
+	}
+	return videoIDs, nil
+}
+
+const releaseTimeFormat = "2006-01-02, 15:04:05 (MST)"
+
+func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Chan, ip *net.TCPAddr, pool *ip_manager.IPPool) (*ytdl.YtdlVideo, error) {
+	args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID, "--cookies", "cookies.txt"}
+	results, err := run(videoID, args, stopChan, pool)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	var video *ytdl.YtdlVideo
+	err = json.Unmarshal([]byte(results[0]), &video)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+
+	// now get an accurate time
+	const maxTries = 5
+	tries := 0
+GetTime:
+	tries++
+	t, err := getUploadTime(config, videoID, ip, video.UploadDate)
+	if err != nil {
+		//slack(":warning: Upload time error: %v", err)
+		if tries <= maxTries && (errors.Is(err, errNotScraped) || errors.Is(err, errUploadTimeEmpty) || errors.Is(err, errStatusParse) || errors.Is(err, errConnectionIssue)) {
+			err := triggerScrape(videoID, ip)
+			if err == nil {
+				time.Sleep(2 * time.Second) // let them scrape it
+				goto GetTime
+			} else {
+				//slack("triggering scrape returned error: %v", err)
+			}
+		} else if !errors.Is(err, errNotScraped) && !errors.Is(err, errUploadTimeEmpty) {
+			//slack(":warning: Error while trying to get accurate upload time for %s: %v", videoID, err)
+			if t == "" {
+				return nil, errors.Err(err)
+			} else {
+				t = "" //TODO: get rid of the other piece below?
+			}
+		}
+		// do fallback below
+	}
+	//slack("After all that, upload time for %s is %s", videoID, t)
+
+	if t != "" {
+		parsed, err := time.Parse("2006-01-02, 15:04:05 (MST)", t) // this will probably be UTC, but Go's timezone parsing is fucked up. it ignores the timezone in the date
+		if err != nil {
+			return nil, errors.Err(err)
+		}
+		//slack(":exclamation: Got an accurate time for %s", videoID)
+		video.UploadDateForReal = parsed
+	} else { //TODO: this is the piece that isn't needed!
+		slack(":warning: Could not get accurate time for %s. Falling back to time from upload ytdl: %s.", videoID, video.UploadDate)
+		// fall back to UploadDate from youtube-dl
+		video.UploadDateForReal, err = time.Parse("20060102", video.UploadDate)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return video, nil
+}
+
+var errNotScraped = errors.Base("not yet scraped by caa.iti.gr")
+var errUploadTimeEmpty = errors.Base("upload time is empty")
+var errStatusParse = errors.Base("could not parse status, got number, need string")
+var errConnectionIssue = errors.Base("there was a connection issue with the api")
+
+func slack(format string, a ...interface{}) {
+	fmt.Printf(format+"\n", a...)
+	util.SendToSlack(format, a...)
+}
+
+func triggerScrape(videoID string, ip *net.TCPAddr) error {
+	//slack("Triggering scrape for %s", videoID)
+	u, err := url.Parse("https://caa.iti.gr/verify_videoV3")
+	q := u.Query()
+	q.Set("twtimeline", "0")
+	q.Set("url", "https://www.youtube.com/watch?v="+videoID)
+	u.RawQuery = q.Encode()
+	//slack("GET %s", u.String())
+
+	client := getClient(ip)
+	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
+	if err != nil {
+		return errors.Err(err)
+	}
+	req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36")
+
+	res, err := client.Do(req)
+	if err != nil {
+		return errors.Err(err)
+	}
+	defer res.Body.Close()
+
+	var response struct {
+		Message  string `json:"message"`
+		Status   string `json:"status"`
+		VideoURL string `json:"video_url"`
+	}
+	err = json.NewDecoder(res.Body).Decode(&response)
+	if err != nil {
+		if strings.Contains(err.Error(), "cannot unmarshal number") {
+			return errors.Err(errStatusParse)
+		}
+		if strings.Contains(err.Error(), "no route to host") {
+			return errors.Err(errConnectionIssue)
+		}
+		return errors.Err(err)
+	}
+
+	switch response.Status {
+	case "removed_video":
+		return errors.Err("video previously removed from service")
+	case "no_video":
+		return errors.Err("they say 'video cannot be found'. wtf?")
+	default:
+		spew.Dump(response)
+	}
+
+	return nil
+	//https://caa.iti.gr/caa/api/v4/videos/reports/h-tuxHS5lSM
+}
+
+func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr, uploadDate string) (string, error) {
+	//slack("Getting upload time for %s", videoID)
+	release, err := config.GetReleasedDate(videoID)
+	if err != nil {
+		logrus.Error(err)
+	}
+	ytdlUploadDate, err := time.Parse("20060102", uploadDate)
+	if err != nil {
+		logrus.Error(err)
+	}
+	if release != nil {
+		//const sqlTimeFormat = "2006-01-02 15:04:05"
+		sqlTime, err := time.ParseInLocation(time.RFC3339, release.ReleaseTime, time.UTC)
+		if err == nil {
+			if sqlTime.Day() != ytdlUploadDate.Day() {
+				logrus.Infof("upload day from APIs differs from the ytdl one by more than 1 day.")
+			} else {
+				return sqlTime.Format(releaseTimeFormat), nil
+			}
+		} else {
+			logrus.Error(err)
+		}
+	}
+
+	if time.Now().AddDate(0, 0, -3).After(ytdlUploadDate) {
+		return ytdlUploadDate.Format(releaseTimeFormat), nil
+	}
+	client := getClient(ip)
+	req, err := http.NewRequest(http.MethodGet, "https://caa.iti.gr/get_verificationV3?url=https://www.youtube.com/watch?v="+videoID, nil)
+	if err != nil {
+		return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
+	}
+	req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36")
+
+	res, err := client.Do(req)
+	if err != nil {
+		return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
+	}
+	defer res.Body.Close()
+
+	var uploadTime struct {
+		Time    string `json:"video_upload_time"`
+		Message string `json:"message"`
+		Status  string `json:"status"`
+	}
+	err = json.NewDecoder(res.Body).Decode(&uploadTime)
+	if err != nil {
+		return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
+	}
+
+	if uploadTime.Status == "ERROR1" {
+		return ytdlUploadDate.Format(releaseTimeFormat), errNotScraped
+	}
+
+	if uploadTime.Status == "" && strings.HasPrefix(uploadTime.Message, "CANNOT_RETRIEVE_REPORT_FOR_VIDEO_") {
+		return ytdlUploadDate.Format(releaseTimeFormat), errors.Err("cannot retrieve report for video")
+	}
+
+	if uploadTime.Time == "" {
+		return ytdlUploadDate.Format(releaseTimeFormat), errUploadTimeEmpty
+	}
+
+	return uploadTime.Time, nil
+}
+
+func getClient(ip *net.TCPAddr) *http.Client {
+	if ip == nil {
+		return http.DefaultClient
+	}
+
+	return &http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+			DialContext: (&net.Dialer{
+				LocalAddr: ip,
+				Timeout:   30 * time.Second,
+				KeepAlive: 30 * time.Second,
+			}).DialContext,
+			MaxIdleConns:          100,
+			IdleConnTimeout:       90 * time.Second,
+			TLSHandshakeTimeout:   10 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+		},
+	}
+}
+
+const (
+	googleBotUA             = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
+	chromeUA                = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"
+	maxAttempts             = 3
+	extractionError         = "YouTube said: Unable to extract video data"
+	throttledError          = "HTTP Error 429"
+	AlternateThrottledError = "returned non-zero exit status 8"
+	youtubeDlError          = "exit status 1"
+)
+
+func run(use string, args []string, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
+	var useragent []string
+	var lastError error
+	for attempts := 0; attempts < maxAttempts; attempts++ {
+		sourceAddress, err := getIPFromPool(use, stopChan, pool)
+		if err != nil {
+			return nil, err
+		}
+		argsForCommand := append(args, "--source-address", sourceAddress)
+		argsForCommand = append(argsForCommand, useragent...)
+		cmd := exec.Command("youtube-dl", argsForCommand...)
+
+		res, err := runCmd(cmd, stopChan)
+		pool.ReleaseIP(sourceAddress)
+		if err == nil {
+			return res, nil
+		}
+		lastError = err
+		if strings.Contains(err.Error(), youtubeDlError) {
+			if strings.Contains(err.Error(), extractionError) {
+				logrus.Warnf("known extraction error: %s", errors.FullTrace(err))
+				useragent = nextUA(useragent)
+			}
+			if strings.Contains(err.Error(), throttledError) || strings.Contains(err.Error(), AlternateThrottledError) {
+				pool.SetThrottled(sourceAddress)
+				//we don't want throttle errors to count toward the max retries
+				attempts--
+			}
+		}
+	}
+	return nil, lastError
+}
+
+func nextUA(current []string) []string {
+	if len(current) == 0 {
+		return []string{"--user-agent", googleBotUA}
+	}
+	return []string{"--user-agent", chromeUA}
+}
+
+func runCmd(cmd *exec.Cmd, stopChan stop.Chan) ([]string, error) {
+	logrus.Infof("running youtube-dl cmd: %s", strings.Join(cmd.Args, " "))
+	var err error
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	err = cmd.Start()
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	outLog, err := ioutil.ReadAll(stdout)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	errorLog, err := ioutil.ReadAll(stderr)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	done := make(chan error, 1)
+	go func() {
+		done <- cmd.Wait()
+	}()
+
+	select {
+	case <-stopChan:
+		err := cmd.Process.Kill()
+		if err != nil {
+			return nil, errors.Prefix("failed to kill command after stopper cancellation", err)
+		}
+		return nil, errors.Err("canceled by stopper")
+	case err := <-done:
+		if err != nil {
+			return nil, errors.Prefix("youtube-dl "+strings.Join(cmd.Args, " ")+" ["+string(errorLog)+"]", err)
+		}
+		return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
+	}
+}
+
+func getIPFromPool(use string, stopChan stop.Chan, pool *ip_manager.IPPool) (sourceAddress string, err error) {
+	for {
+		sourceAddress, err = pool.GetIP(use)
+		if err != nil {
+			if errors.Is(err, ip_manager.ErrAllThrottled) {
+				select {
+				case <-stopChan:
+					return "", errors.Err("interrupted by user")
+
+				default:
+					time.Sleep(ip_manager.IPCooldownPeriod)
+					continue
+				}
+			} else {
+				return "", err
+			}
+		}
+		break
+	}
+	return
+}

downloader/downloader_test.go

@@ -0,0 +1,42 @@
+package downloader
+
+import (
+	"testing"
+
+	"github.com/lbryio/ytsync/v5/sdk"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetPlaylistVideoIDs(t *testing.T) {
+	videoIDs, err := GetPlaylistVideoIDs("UCJ0-OtVpF0wOKEqT2Z1HEtA", 50, nil, nil)
+	if err != nil {
+		logrus.Error(err)
+	}
+	for _, id := range videoIDs {
+		println(id)
+	}
+}
+
+func TestGetVideoInformation(t *testing.T) {
+	video, err := GetVideoInformation(nil, "zj7pXM9gE5M", nil, nil, nil)
+	if err != nil {
+		logrus.Error(err)
+	}
+	if video != nil {
+		logrus.Info(video.ID)
+	}
+}
+
+func Test_getUploadTime(t *testing.T) {
+	configs := sdk.APIConfig{
+		YoutubeAPIKey: "",
+		ApiURL:        "https://api.lbry.com",
+		ApiToken:      "Ht4NETrL5oWKyAaZkuSV68BKhtXkiLh5",
+		HostName:      "test",
+	}
+	got, err := getUploadTime(&configs, "kDGOHNpRjzc", nil, "20060102")
+	assert.NoError(t, err)
+	t.Log(got)
+
+}

downloader/ytdl/Video.go

@@ -0,0 +1,147 @@
+package ytdl
+
+import (
+	"time"
+)
+
+type YtdlVideo struct {
+	UploadDate         string      `json:"upload_date"`
+	UploadDateForReal  time.Time   // you need to manually set this since the value in the API doesn't include the time
+	Extractor          string      `json:"extractor"`
+	Series             interface{} `json:"series"`
+	Format             string      `json:"format"`
+	Vbr                interface{} `json:"vbr"`
+	Chapters           interface{} `json:"chapters"`
+	Height             int         `json:"height"`
+	LikeCount          interface{} `json:"like_count"`
+	Duration           int         `json:"duration"`
+	Fulltitle          string      `json:"fulltitle"`
+	PlaylistIndex      interface{} `json:"playlist_index"`
+	Album              interface{} `json:"album"`
+	ViewCount          int         `json:"view_count"`
+	Playlist           interface{} `json:"playlist"`
+	Title              string      `json:"title"`
+	Filename           string      `json:"_filename"`
+	Creator            interface{} `json:"creator"`
+	Ext                string      `json:"ext"`
+	ID                 string      `json:"id"`
+	DislikeCount       interface{} `json:"dislike_count"`
+	AverageRating      float64     `json:"average_rating"`
+	Abr                int         `json:"abr"`
+	UploaderURL        string      `json:"uploader_url"`
+	Categories         []string    `json:"categories"`
+	Fps                float64     `json:"fps"`
+	StretchedRatio     interface{} `json:"stretched_ratio"`
+	SeasonNumber       interface{} `json:"season_number"`
+	Annotations        interface{} `json:"annotations"`
+	WebpageURLBasename string      `json:"webpage_url_basename"`
+	Acodec             string      `json:"acodec"`
+	DisplayID          string      `json:"display_id"`
+	//RequestedFormats   []RequestedFormat `json:"requested_formats"`
+	//AutomaticCaptions  struct{}          `json:"automatic_captions"`
+	Description        string      `json:"description"`
+	Tags               []string    `json:"tags"`
+	Track              interface{} `json:"track"`
+	RequestedSubtitles interface{} `json:"requested_subtitles"`
+	StartTime          interface{} `json:"start_time"`
+	Uploader           string      `json:"uploader"`
+	ExtractorKey       string      `json:"extractor_key"`
+	FormatID           string      `json:"format_id"`
+	EpisodeNumber      interface{} `json:"episode_number"`
+	UploaderID         string      `json:"uploader_id"`
+	//Subtitles          struct{}          `json:"subtitles"`
+	ReleaseYear interface{} `json:"release_year"`
+	Thumbnails  []Thumbnail `json:"thumbnails"`
+	License     interface{} `json:"license"`
+	Artist      interface{} `json:"artist"`
+	AgeLimit    int         `json:"age_limit"`
+	ReleaseDate interface{} `json:"release_date"`
+	AltTitle    interface{} `json:"alt_title"`
+	Thumbnail   string      `json:"thumbnail"`
+	ChannelID   string      `json:"channel_id"`
+	IsLive      interface{} `json:"is_live"`
+	Width       int         `json:"width"`
+	EndTime     interface{} `json:"end_time"`
+	WebpageURL  string      `json:"webpage_url"`
+	//Formats            []Format          `json:"formats"`
+	ChannelURL string      `json:"channel_url"`
+	Resolution interface{} `json:"resolution"`
+	Vcodec     string      `json:"vcodec"`
+}
+
+type RequestedFormat struct {
+	Asr             interface{} `json:"asr"`
+	Tbr             float64     `json:"tbr"`
+	Container       string      `json:"container"`
+	Language        interface{} `json:"language"`
+	Format          string      `json:"format"`
+	URL             string      `json:"url"`
+	Vcodec          string      `json:"vcodec"`
+	FormatNote      string      `json:"format_note"`
+	Height          int         `json:"height"`
+	Width           int         `json:"width"`
+	Ext             string      `json:"ext"`
+	FragmentBaseURL string      `json:"fragment_base_url"`
+	Filesize        interface{} `json:"filesize"`
+	Fps             float64     `json:"fps"`
+	ManifestURL     string      `json:"manifest_url"`
+	Protocol        string      `json:"protocol"`
+	FormatID        string      `json:"format_id"`
+	HTTPHeaders     struct {
+		AcceptCharset  string `json:"Accept-Charset"`
+		AcceptLanguage string `json:"Accept-Language"`
+		AcceptEncoding string `json:"Accept-Encoding"`
+		Accept         string `json:"Accept"`
+		UserAgent      string `json:"User-Agent"`
+	} `json:"http_headers"`
+	Fragments []struct {
+		Path     string  `json:"path"`
+		Duration float64 `json:"duration,omitempty"`
+	} `json:"fragments"`
+	Acodec string `json:"acodec"`
+	Abr    int    `json:"abr,omitempty"`
+}
+
+type Format struct {
+	Asr             int         `json:"asr"`
+	Tbr             float64     `json:"tbr"`
+	Protocol        string      `json:"protocol"`
+	Format          string      `json:"format"`
+	FormatNote      string      `json:"format_note"`
+	Height          interface{} `json:"height"`
+	ManifestURL     string      `json:"manifest_url,omitempty"`
+	FormatID        string      `json:"format_id"`
+	Container       string      `json:"container,omitempty"`
+	Language        interface{} `json:"language,omitempty"`
+	HTTPHeaders     HTTPHeaders `json:"http_headers"`
+	URL             string      `json:"url"`
+	Vcodec          string      `json:"vcodec"`
+	Abr             int         `json:"abr,omitempty"`
+	Width           interface{} `json:"width"`
+	Ext             string      `json:"ext"`
+	FragmentBaseURL string      `json:"fragment_base_url,omitempty"`
+	Filesize        interface{} `json:"filesize"`
+	Fps             float64     `json:"fps"`
+	Fragments       []struct {
+		Path     string  `json:"path"`
+		Duration float64 `json:"duration,omitempty"`
+	} `json:"fragments,omitempty"`
+	Acodec    string      `json:"acodec"`
+	PlayerURL interface{} `json:"player_url,omitempty"`
+}
+
+type Thumbnail struct {
+	URL        string `json:"url"`
+	Width      int    `json:"width"`
+	Resolution string `json:"resolution"`
+	ID         string `json:"id"`
+	Height     int    `json:"height"`
+}
+
+type HTTPHeaders struct {
+	AcceptCharset  string `json:"Accept-Charset"`
+	AcceptLanguage string `json:"Accept-Language"`
+	AcceptEncoding string `json:"Accept-Encoding"`
+	Accept         string `json:"Accept"`
+	UserAgent      string `json:"User-Agent"`
+}

e2e/data_setup.sh

@@ -19,6 +19,6 @@ mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER"
 ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(2,"youtubertoken")'
 mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
 #Add their youtube channel to be synced
-ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable)
+ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable,total_views,reviewed,last_uploaded_video,length_limit,size_limit)
-VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCNQfQvFMPnInwsU_iGYArJQ','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',10,10,1,1)"
+VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@test"$(date +%s)"','UCNQfQvFMPnInwsU_iGYArJQ','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',10,10,1,1,10000,1,'7bBV2Z-9wpo',60,2048)"
 mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL"

e2e/e2e.sh

@@ -26,7 +26,7 @@ export REGTEST=true
 # Local settings
 export BLOBS_DIRECTORY="$(pwd)/e2e/blobsfiles"
 export LBRYNET_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbrynet/"
-export LBRYNET_WALLETS_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum"
+export LBRYUM_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum"
 export TMP_DIR="/var/tmp"
 export CHAINNAME="lbrycrd_regtest"
 export UID

go.mod

@@ -3,25 +3,23 @@ module github.com/lbryio/ytsync/v5
 replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
 
 require (
-	cloud.google.com/go v0.46.3 // indirect
-	github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61
 	github.com/Microsoft/go-winio v0.4.14 // indirect
-	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a
+	github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb
 	github.com/aws/aws-sdk-go v1.25.9
-	github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 // indirect
+	github.com/davecgh/go-spew v1.1.1
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/docker v1.13.1
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
-	github.com/go-sql-driver/mysql v1.4.1 // indirect
+	github.com/golang/protobuf v1.4.2 // indirect
-	github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc // indirect
 	github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
 	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
-	github.com/hashicorp/golang-lru v0.5.3 // indirect
 	github.com/hashicorp/memberlist v0.1.5 // indirect
 	github.com/hashicorp/serf v0.8.5 // indirect
-	github.com/lbryio/lbry.go/v2 v2.6.1-0.20200710180140-fcade7475323
+	github.com/kr/pretty v0.2.1 // indirect
-	github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6
+	github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128
+	github.com/lbryio/reflector.go v1.1.3-0.20200901181534-e8b98bc862d5
+	github.com/lbryio/types v0.0.0-20200605192618-366870b2862d // indirect
 	github.com/miekg/dns v1.1.22 // indirect
 	github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
 	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
@@ -29,12 +27,14 @@ require (
 	github.com/prometheus/client_golang v0.9.2
 	github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5
 	github.com/sirupsen/logrus v1.4.2
+	github.com/smartystreets/goconvey v1.6.4 // indirect
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/pflag v1.0.5 // indirect
-	go.opencensus.io v0.22.1 // indirect
+	github.com/stretchr/testify v1.4.0
+	golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
-	google.golang.org/api v0.11.0
 	google.golang.org/appengine v1.6.5 // indirect
+	gopkg.in/ini.v1 v1.60.2 // indirect
 )
 
 go 1.13

go.sum

@@ -1,22 +1,20 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
-cloud.google.com/go v0.46.3 h1:AVXDdKsrtX33oR9fbCMu/+c1o8Ofjq6Ku/MInaLVg5Y=
+dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:N5Vqww5QISEHsWHOWDEx4PzdIay3Cg0Jp7zItq2ZAro=
-github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:GnKXcK+7DYNy/8w2Ex//Uql4IgfaU82Cd5rWKb7ah00=
 github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
 github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
+github.com/StackExchange/wmi v0.0.0-20170410192909-ea383cf3ba6e/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
@@ -24,14 +22,16 @@ github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7
 github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb h1:kvlW1qyM1aU3xeyeIVTU2jx5fSvjKpsU3aXvuaCMg3Q=
+github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.25.9 h1:WtVzerf5wSgPwlTTwl+ktCq/0GCS5MI9ZlLIcjsTr+Q=
 github.com/aws/aws-sdk-go v1.25.9/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo=
 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
 github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
@@ -44,19 +44,22 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku
 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc=
 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
 github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
-github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:o64h9XF42kVEUuhuer2ehqrlX8rZmvQSU0+Vpj1rF6Q=
+github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
-github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:Rp8e0DCtEKwXFOC6JPJQVTz8tuGoGvw6Xfexggh/ed0=
+github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
+github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
 github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
 github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
@@ -65,63 +68,79 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
+github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
 github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-ini/ini v1.48.0 h1:TvO60hO/2xgaaTWp2P0wUe4CFxwdMzfbkv3+343Xzqw=
 github.com/go-ini/ini v1.48.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
+github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
 github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
 github.com/go-ozzo/ozzo-validation v3.6.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
-github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
+github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
-github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc h1:55rEp52jU6bkyslZ1+C/7NGfpQsEc6pxGLAGDOctqbw=
-github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0 h1:Rd1kQnQu0Hq3qvJppYSG0HtP+f5LPPUiDswTLiEegLg=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/gops v0.3.7/go.mod h1:bj0cwMmX1X4XIJFTjR99R5sCxNssNJ8HebFNvoQlmgY=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f h1:TyqzGm2z1h3AGhjOoRYyeLcW4WlW81MDQkWa+rx/000=
 github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc=
+github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
 github.com/gorilla/rpc v1.2.0 h1:WvvdC2lNeT1SP32zrIce5l0ECBfbAlmrmSBsuc57wfk=
 github.com/gorilla/rpc v1.2.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
+github.com/gorilla/schema v1.0.2/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU=
 github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
 github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
@@ -143,9 +162,10 @@ github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk=
 github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
@@ -162,55 +182,78 @@ github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7V
 github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
 github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmoiron/sqlx v0.0.0-20170430194603-d9bd385d68c0/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU=
 github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 h1:jKUP9TQ0c7X3w6+IPyMit07RE42MtTWNd77sN2cHngQ=
 github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22/go.mod h1:u0Jo4f2dNlTJeeOywkM6bLwxq6gC3pZ9rEFHn3AhTdk=
+github.com/johntdyer/slackrus v0.0.0-20170926115001-3992f319fd0a/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
 github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07 h1:+kBG/8rjCa6vxJZbUjAiE4MQmBEBYc8nLEb51frnvBY=
 github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
 github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jteeuwen/go-bindata v3.0.7+incompatible/go.mod h1:JVvhzYOiGBnFSYRyV00iY8q7/0PThjIYav1p9h5dmKs=
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
+github.com/keybase/go-ps v0.0.0-20161005175911-668c8856d999/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
+github.com/lbryio/chainquery v1.9.0 h1:NfBZ3eKYwD3PqXU/vt+2tF3ox3WUWoW4J5YdEQ0rxw0=
+github.com/lbryio/chainquery v1.9.0/go.mod h1:7G8l7jNtANS1I7fQOvtzbiHsv6qKVmN4codXHc3C4kk=
 github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
-github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303/go.mod h1:qR+Ui0hYhemIU4fXqM3d1P9eiaRFlof777VJgV7KJ8w=
+github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
 github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
 github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
-github.com/lbryio/lbry.go/v2 v2.6.1-0.20200710180140-fcade7475323 h1:Fvngg+hybcA1O2783YBw07S2MGpemSO+vWYSk97QHdM=
+github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128 h1:VL209c+AGKixMFpxT+TsOAPzBPuoUzyjXf47iNe7OzY=
-github.com/lbryio/lbry.go/v2 v2.6.1-0.20200710180140-fcade7475323/go.mod h1:LgFKEpZzJE72DVgSXXfg+2IGOhrC9Lzj6eLma18iNz8=
+github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128/go.mod h1:RqOv4V5eWY/JGmduCPcQVdN19SEYnNY3SuF+arTKIU4=
 github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
 github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
 github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
-github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46 h1:LemfR+rMxhf7nnOrzy2HqS7Me7SZ5gEwOcNFzKC8ySQ=
 github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
 github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
-github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6 h1:8k12lI18EEgMHkgGrgtRvJGl2Bq1vDSEbE4s+QaQ5Qc=
+github.com/lbryio/reflector.go v1.1.3-0.20200901181534-e8b98bc862d5 h1:Y+zuIZQ1QwpBKLsdfThALHMwQSR4i0H67Znen6YdT54=
-github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6/go.mod h1:Q1Cnuv5iLsB2rS4Vr0o0EJr4Gs/EgDXFH2V4WtoM5o8=
+github.com/lbryio/reflector.go v1.1.3-0.20200901181534-e8b98bc862d5/go.mod h1:QHKtgkmOCJJKZ578AFmN3jlBUhJFykbVEGgOuEiQ+aY=
 github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
-github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8 h1:jSNW/rK6DQsz7Zh+iv1zR384PeQdHt0gS4hKY17tkuM=
 github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
+github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
+github.com/lbryio/types v0.0.0-20200605192618-366870b2862d h1:Pg/4L7ExGXsseTMy1d/LK9TY55c8CP2gtmafmYzed0Q=
+github.com/lbryio/types v0.0.0-20200605192618-366870b2862d/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
+github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lucas-clemente/quic-go v0.17.2 h1:4iQInIuNQkPNZmsy9rCnwuOzpH0qGnDo4jn0QfI/qE4=
+github.com/lucas-clemente/quic-go v0.17.2/go.mod h1:I0+fcNTdb9eS1ZcjQZbDVPGchJ86chcIxPALn9lEJqE=
+github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
 github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
 github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcXMWfkoumVwhcCbf3jHF9QKv/m37BkM0=
 github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4=
+github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/marten-seemann/qpack v0.1.0 h1:/0M7lkda/6mus9B8u34Asqm8ZhHAAt9Ho0vniNuVSVg=
+github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI=
+github.com/marten-seemann/qtls v0.9.1 h1:O0YKQxNVPaiFgMng0suWEOY2Sb4LT2sRn9Qimq3Z1IQ=
+github.com/marten-seemann/qtls v0.9.1/go.mod h1:T1MmAdDPyISzxlK6kjRr0pcZFBVd1OZbBb/j3cvzHhk=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.22 h1:Jm64b3bO9kP43ddLjL2EY3Io6bmy1qGb9Xxz6TqS6rc=
 github.com/miekg/dns v1.1.22/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
@@ -225,21 +268,29 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F
 github.com/mitchellh/mapstructure v0.0.0-20180511142126-bb74f1db0675/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
+github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
 github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
 github.com/nlopes/slack v0.6.0 h1:jt0jxVQGhssx1Ib7naAOZEZcGdtIhTzkP0nopK0AsRA=
 github.com/nlopes/slack v0.6.0/go.mod h1:JzQ9m3PMAqcpeCam7UaHSuBuupz7CmpjehYMayT6YOk=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94=
 github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw=
+github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34=
+github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
 github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=
@@ -247,18 +298,22 @@ github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rK
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/profile v1.3.0/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
+github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
 github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
 github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
 github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rubenv/sql-migrate v0.0.0-20170330050058-38004e7a77f2/go.mod h1:WS0rl9eEliYI8DPnr3TOwz4439pay+qNgzJoVya/DmY=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@@ -269,11 +324,36 @@ github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77 h1:Msb6XRY62jQOueNNl
 github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4=
 github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/sfreiberg/gotwilio v0.0.0-20180612161623-8fb7259ba8bf/go.mod h1:60PiR0SAnAcYSiwrXB6BaxeqHdXMf172toCosHfV+Yk=
+github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
 github.com/shopspring/decimal v0.0.0-20180607144847-19e3cb6c2930/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
+github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
 github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5 h1:Gojs/hac/DoYEM7WEICT45+hNWczIeuL5D21e5/HPAw=
 github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
+github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
+github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM=
+github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0=
+github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
+github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
+github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
+github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI=
+github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU=
+github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag=
+github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg=
+github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw=
+github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y=
+github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
+github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q=
+github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ=
+github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I=
+github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0=
+github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ=
+github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk=
+github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4=
+github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -281,19 +361,29 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
 github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
 github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
-github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
 github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
+github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
+github.com/spf13/afero v1.1.1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
 github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.0-20180722215644-7c4570c3ebeb/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
+github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -301,69 +391,77 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
-github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o=
-github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
+github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
+github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d/go.mod h1:jspfvgf53t5NLUT4o9L1IX0kIBNKamGq1tWc/MgWK9Q=
+github.com/volatiletech/null v8.0.0+incompatible/go.mod h1:0wD98JzdqB+rLyZ70fN05VDbXbafIb0KU0MdVhCzmOQ=
+github.com/volatiletech/sqlboiler v3.4.0+incompatible/go.mod h1:jLfDkkHWPbS2cWRLkyC20vQWaIQsASEY7gM7zSo11Yw=
+github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d h1:tQo6hjclyv3RHUgZOl6iWb2Y44A/sN9bf9LAYfuioEg=
 github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
-go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50=
-go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM=
+go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
+golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
 golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
-golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5hzyggAkLLlCUjqfRxd8Q4=
 golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5 h1:Q7tZBpemrlsc2I7IyODzhtallWRSm4Q0d09pL6XbQtU=
+golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190228165749-92fc7df08ae7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191009170851-d66e71096ffb h1:TR699M2v0qoKTOHxeLgp6zPqaQNs74f01a/ob9W0qko=
 golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
+golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -371,92 +469,96 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20171017063910-8dbc5d05d6ed/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f h1:hjzMYz/7Ea1mNKfOnFOfktR0mlA5jqhvywClCMHM/qw=
 golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w=
 golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 h1:Toz2IK7k8rbltAXwNAxKcn9OzqyNfMUhUNjz3sL0NMk=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
-google.golang.org/api v0.11.0 h1:n/qM3q0/rV2F0pox7o0CvNhlPvZAo7pLbef122cbLJ0=
+google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
-google.golang.org/api v0.11.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03 h1:4HYDjxeNXAOTv3o1N2tjo8UUSlhQgAD52FVkwxnWgM8=
 google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
+google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s=
 google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/gorp.v1 v1.7.1/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/ini.v1 v1.48.0 h1:URjZc+8ugRY5mL5uUeQH/a63JcHwdX9xZaWvmNWD7z8=
 gopkg.in/ini.v1 v1.48.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.60.2 h1:7i8mqModL63zqi8nQn8Q3+0zvSCZy1AxhBgthKfi4WU=
+gopkg.in/ini.v1 v1.60.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 h1:FpCr9V8wuOei4BAen+93HtVJ+XSi+KPbaPKm0Vj5R64=
 gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79/go.mod h1:gWkaRU7CoXpezCBWfWjm3999QqS+1pYPXGbqQCTMzo8=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
@@ -467,10 +569,13 @@ gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
+grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+rsc.io/goversion v1.0.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=
-rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
+sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=

ip_manager/throttle.go

@@ -63,21 +63,21 @@ func GetIPPool(stopGrp *stop.Group) (*IPPool, error) {
 		lock:    &sync.RWMutex{},
 		stopGrp: stopGrp,
 	}
-	ticker := time.NewTicker(10 * time.Second)
+	//ticker := time.NewTicker(10 * time.Second)
-	go func() {
+	//go func() {
-		for {
+	//	for {
-			select {
+	//		select {
-			case <-stopGrp.Ch():
+	//		case <-stopGrp.Ch():
-				return
+	//			return
-			case <-ticker.C:
+	//		case <-ticker.C:
-				ipPoolInstance.lock.RLock()
+	//			ipPoolInstance.lock.RLock()
-				for _, ip := range ipPoolInstance.ips {
+	//			for _, ip := range ipPoolInstance.ips {
-					log.Debugf("IP: %s\tInUse: %t\tVideoID: %s\tThrottled: %t\tLastUse: %.1f", ip.IP, ip.InUse, ip.UsedForVideo, ip.Throttled, time.Since(ip.LastUse).Seconds())
+	//				log.Debugf("IP: %s\tInUse: %t\tVideoID: %s\tThrottled: %t\tLastUse: %.1f", ip.IP, ip.InUse, ip.UsedForVideo, ip.Throttled, time.Since(ip.LastUse).Seconds())
-				}
+	//			}
-				ipPoolInstance.lock.RUnlock()
+	//			ipPoolInstance.lock.RUnlock()
-			}
+	//		}
-		}
+	//	}
-	}()
+	//}()
 	return ipPoolInstance, nil
 }
 
@@ -108,7 +108,7 @@ func AllInUse(ips []throttledIP) bool {
 func (i *IPPool) ReleaseIP(ip string) {
 	i.lock.Lock()
 	defer i.lock.Unlock()
-	for j, _ := range i.ips {
+	for j := range i.ips {
 		localIP := &i.ips[j]
 		if localIP.IP == ip {
 			localIP.InUse = false
@@ -122,7 +122,7 @@ func (i *IPPool) ReleaseIP(ip string) {
 func (i *IPPool) ReleaseAll() {
 	i.lock.Lock()
 	defer i.lock.Unlock()
-	for j, _ := range i.ips {
+	for j := range i.ips {
 		if i.ips[j].Throttled {
 			continue
 		}
@@ -183,7 +183,7 @@ func (i *IPPool) nextIP(forVideo string) (*throttledIP, error) {
 		}
 
 		var nextIP *throttledIP
-		for j, _ := range i.ips {
+		for j := range i.ips {
 			ip := &i.ips[j]
 			if ip.InUse || ip.Throttled {
 				continue

main.go

@@ -11,6 +11,7 @@ import (
 	"github.com/lbryio/lbry.go/v2/extras/util"
 	"github.com/lbryio/ytsync/v5/manager"
 	"github.com/lbryio/ytsync/v5/sdk"
+	"github.com/lbryio/ytsync/v5/shared"
 	ytUtils "github.com/lbryio/ytsync/v5/util"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 
@@ -23,18 +24,8 @@ var Version string
 const defaultMaxTries = 3
 
 var (
-	flags          sdk.SyncFlags
+	cliFlags       shared.SyncFlags
-	maxTries       int
+	maxVideoLength int
-	refill         int
-	limit          int
-	syncStatus     string
-	channelID      string
-	syncFrom       int64
-	syncUntil      int64
-	concurrentJobs int
-	videosLimit    int
-	maxVideoSize   int
-	maxVideoLength float64
 )
 
 func main() {
@@ -51,25 +42,25 @@ func main() {
 		Args:  cobra.RangeArgs(0, 0),
 	}
 
-	cmd.Flags().BoolVar(&flags.StopOnError, "stop-on-error", false, "If a publish fails, stop all publishing and exit")
+	cmd.Flags().BoolVar(&cliFlags.StopOnError, "stop-on-error", false, "If a publish fails, stop all publishing and exit")
-	cmd.Flags().IntVar(&maxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
+	cmd.Flags().IntVar(&cliFlags.MaxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
-	cmd.Flags().BoolVar(&flags.TakeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
+	cmd.Flags().BoolVar(&cliFlags.TakeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
-	cmd.Flags().IntVar(&limit, "limit", 0, "limit the amount of channels to sync")
+	cmd.Flags().IntVar(&cliFlags.Limit, "limit", 0, "limit the amount of channels to sync")
-	cmd.Flags().BoolVar(&flags.SkipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
+	cmd.Flags().BoolVar(&cliFlags.SkipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
-	cmd.Flags().BoolVar(&flags.SyncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
+	cmd.Flags().BoolVar(&cliFlags.SyncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
-	cmd.Flags().BoolVar(&flags.SingleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
+	cmd.Flags().BoolVar(&cliFlags.SingleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
-	cmd.Flags().BoolVar(&flags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
+	cmd.Flags().BoolVar(&cliFlags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
-	cmd.Flags().BoolVar(&flags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
+	cmd.Flags().BoolVar(&cliFlags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
-	cmd.Flags().BoolVar(&flags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
+	cmd.Flags().BoolVar(&cliFlags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
-	cmd.Flags().BoolVar(&flags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube")
+	cmd.Flags().BoolVar(&cliFlags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube")
-	cmd.Flags().StringVar(&syncStatus, "status", "", "Specify which queue to pull from. Overrides --update")
+	cmd.Flags().StringVar(&cliFlags.SyncStatus, "status", "", "Specify which queue to pull from. Overrides --update")
-	cmd.Flags().StringVar(&channelID, "channelID", "", "If specified, only this channel will be synced.")
+	cmd.Flags().StringVar(&cliFlags.ChannelID, "channelID", "", "If specified, only this channel will be synced.")
-	cmd.Flags().Int64Var(&syncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
+	cmd.Flags().Int64Var(&cliFlags.SyncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
-	cmd.Flags().Int64Var(&syncUntil, "before", time.Now().AddDate(1, 0, 0).Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)")
+	cmd.Flags().Int64Var(&cliFlags.SyncUntil, "before", time.Now().AddDate(1, 0, 0).Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)")
-	cmd.Flags().IntVar(&concurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently")
+	cmd.Flags().IntVar(&cliFlags.ConcurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently")
-	cmd.Flags().IntVar(&videosLimit, "videos-limit", 1000, "how many videos to process per channel")
+	cmd.Flags().IntVar(&cliFlags.VideosLimit, "videos-limit", 1000, "how many videos to process per channel")
-	cmd.Flags().IntVar(&maxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)")
+	cmd.Flags().IntVar(&cliFlags.MaxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)")
-	cmd.Flags().Float64Var(&maxVideoLength, "max-length", 2.0, "Maximum video length to process (in hours)")
+	cmd.Flags().IntVar(&maxVideoLength, "max-length", 2, "Maximum video length to process (in hours)")
 
 	if err := cmd.Execute(); err != nil {
 		fmt.Println(err)
@@ -96,29 +87,30 @@ func ytSync(cmd *cobra.Command, args []string) {
 		util.InitSlack(os.Getenv("SLACK_TOKEN"), os.Getenv("SLACK_CHANNEL"), hostname)
 	}
 
-	if syncStatus != "" && !util.InSlice(syncStatus, manager.SyncStatuses) {
+	if cliFlags.SyncStatus != "" && !util.InSlice(cliFlags.SyncStatus, shared.SyncStatuses) {
-		log.Errorf("status must be one of the following: %v\n", manager.SyncStatuses)
+		log.Errorf("status must be one of the following: %v\n", shared.SyncStatuses)
 		return
 	}
 
-	if flags.StopOnError && maxTries != defaultMaxTries {
+	if cliFlags.StopOnError && cliFlags.MaxTries != defaultMaxTries {
 		log.Errorln("--stop-on-error and --max-tries are mutually exclusive")
 		return
 	}
-	if maxTries < 1 {
+	if cliFlags.MaxTries < 1 {
 		log.Errorln("setting --max-tries less than 1 doesn't make sense")
 		return
 	}
 
-	if limit < 0 {
+	if cliFlags.Limit < 0 {
 		log.Errorln("setting --limit less than 0 (unlimited) doesn't make sense")
 		return
 	}
+	cliFlags.MaxVideoLength = time.Duration(maxVideoLength) * time.Hour
 
 	apiURL := os.Getenv("LBRY_WEB_API")
 	apiToken := os.Getenv("LBRY_API_TOKEN")
 	youtubeAPIKey := os.Getenv("YOUTUBE_API_KEY")
-	lbrycrdString := os.Getenv("LBRYCRD_STRING")
+	lbrycrdDsn := os.Getenv("LBRYCRD_STRING")
 	awsS3ID := os.Getenv("AWS_S3_ID")
 	awsS3Secret := os.Getenv("AWS_S3_SECRET")
 	awsS3Region := os.Getenv("AWS_S3_REGION")
@@ -151,42 +143,30 @@ func ytSync(cmd *cobra.Command, args []string) {
 		log.Errorln("AWS S3 Bucket was not defined. Please set the environment variable AWS_S3_BUCKET")
 		return
 	}
-	if lbrycrdString == "" {
+	if lbrycrdDsn == "" {
 		log.Infoln("Using default (local) lbrycrd instance. Set LBRYCRD_STRING if you want to use something else")
 	}
 
 	blobsDir := ytUtils.GetBlobsDir()
 
-	syncProperties := &sdk.SyncProperties{
-		SyncFrom:         syncFrom,
-		SyncUntil:        syncUntil,
-		YoutubeChannelID: channelID,
-	}
 	apiConfig := &sdk.APIConfig{
 		YoutubeAPIKey: youtubeAPIKey,
 		ApiURL:        apiURL,
 		ApiToken:      apiToken,
 		HostName:      hostname,
 	}
+	awsConfig := &shared.AwsConfigs{
+		AwsS3ID:     awsS3ID,
+		AwsS3Secret: awsS3Secret,
+		AwsS3Region: awsS3Region,
+		AwsS3Bucket: awsS3Bucket,
+	}
 	sm := manager.NewSyncManager(
-		flags,
+		cliFlags,
-		maxTries,
-		refill,
-		limit,
-		concurrentJobs,
-		concurrentJobs,
 		blobsDir,
-		videosLimit,
+		lbrycrdDsn,
-		maxVideoSize,
+		awsConfig,
-		lbrycrdString,
-		awsS3ID,
-		awsS3Secret,
-		awsS3Region,
-		awsS3Bucket,
-		syncStatus,
-		syncProperties,
 		apiConfig,
-		maxVideoLength,
 	)
 	err := sm.Start()
 	if err != nil {

manager/count.go

@@ -1,32 +0,0 @@
-package manager
-
-import (
-	"net/http"
-
-	"github.com/lbryio/lbry.go/v2/extras/errors"
-
-	"google.golang.org/api/googleapi/transport"
-	"google.golang.org/api/youtube/v3"
-)
-
-func (s *Sync) CountVideos() (uint64, error) {
-	client := &http.Client{
-		Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
-	}
-
-	service, err := youtube.New(client)
-	if err != nil {
-		return 0, errors.Prefix("error creating YouTube service", err)
-	}
-
-	response, err := service.Channels.List("statistics").Id(s.YoutubeChannelID).Do()
-	if err != nil {
-		return 0, errors.Prefix("error getting channels", err)
-	}
-
-	if len(response.Items) < 1 {
-		return 0, errors.Err("youtube channel not found")
-	}
-
-	return response.Items[0].Statistics.VideoCount, nil
-}

manager/manager.go

@@ -10,94 +10,43 @@ import (
 	"github.com/lbryio/ytsync/v5/ip_manager"
 	"github.com/lbryio/ytsync/v5/namer"
 	"github.com/lbryio/ytsync/v5/sdk"
+	"github.com/lbryio/ytsync/v5/shared"
 	logUtils "github.com/lbryio/ytsync/v5/util"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/extras/util"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
 	log "github.com/sirupsen/logrus"
 )
 
 type SyncManager struct {
-	SyncFlags        sdk.SyncFlags
+	CliFlags   shared.SyncFlags
-	maxTries         int
+	ApiConfig  *sdk.APIConfig
-	refill           int
+	LbrycrdDsn string
-	limit            int
+	AwsConfigs *shared.AwsConfigs
-	concurrentJobs   int
+
-	concurrentVideos int
+	blobsDir       string
-	blobsDir         string
+	channelsToSync []Sync
-	videosLimit      int
-	maxVideoSize     int
-	maxVideoLength   float64
-	lbrycrdString    string
-	awsS3ID          string
-	awsS3Secret      string
-	awsS3Region      string
-	syncStatus       string
-	awsS3Bucket      string
-	syncProperties   *sdk.SyncProperties
-	apiConfig        *sdk.APIConfig
 }
 
-func NewSyncManager(syncFlags sdk.SyncFlags, maxTries int, refill int, limit int, concurrentJobs int, concurrentVideos int, blobsDir string, videosLimit int,
+func NewSyncManager(cliFlags shared.SyncFlags, blobsDir, lbrycrdDsn string, awsConfigs *shared.AwsConfigs, apiConfig *sdk.APIConfig) *SyncManager {
-	maxVideoSize int, lbrycrdString string, awsS3ID string, awsS3Secret string, awsS3Region string, awsS3Bucket string,
-	syncStatus string, syncProperties *sdk.SyncProperties, apiConfig *sdk.APIConfig, maxVideoLength float64) *SyncManager {
 	return &SyncManager{
-		SyncFlags:        syncFlags,
+		CliFlags:   cliFlags,
-		maxTries:         maxTries,
+		blobsDir:   blobsDir,
-		refill:           refill,
+		LbrycrdDsn: lbrycrdDsn,
-		limit:            limit,
+		AwsConfigs: awsConfigs,
-		concurrentJobs:   concurrentJobs,
+		ApiConfig:  apiConfig,
-		concurrentVideos: concurrentVideos,
-		blobsDir:         blobsDir,
-		videosLimit:      videosLimit,
-		maxVideoSize:     maxVideoSize,
-		maxVideoLength:   maxVideoLength,
-		lbrycrdString:    lbrycrdString,
-		awsS3ID:          awsS3ID,
-		awsS3Secret:      awsS3Secret,
-		awsS3Region:      awsS3Region,
-		awsS3Bucket:      awsS3Bucket,
-		syncStatus:       syncStatus,
-		syncProperties:   syncProperties,
-		apiConfig:        apiConfig,
 	}
 }
-
+func (s *SyncManager) enqueueChannel(channel *shared.YoutubeChannel) {
-const (
+	s.channelsToSync = append(s.channelsToSync, Sync{
-	StatusPending        = "pending"        // waiting for permission to sync
+		DbChannelData: channel,
-	StatusPendingEmail   = "pendingemail"   // permission granted but missing email
+		Manager:       s,
-	StatusQueued         = "queued"         // in sync queue. will be synced soon
+		namer:         namer.NewNamer(),
-	StatusPendingUpgrade = "pendingupgrade" // in sync queue. will be synced soon
+	})
-	StatusSyncing        = "syncing"        // syncing now
+}
-	StatusSynced         = "synced"         // done
-	StatusFailed         = "failed"
-	StatusFinalized      = "finalized" // no more changes allowed
-	StatusAbandoned      = "abandoned" // deleted on youtube or banned
-)
-const LatestMetadataVersion = 2
-
-var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned}
-
-const (
-	VideoStatusPublished     = "published"
-	VideoStatusFailed        = "failed"
-	VideoStatusUpgradeFailed = "upgradefailed"
-	VideoStatusUnpublished   = "unpublished"
-	VideoStatusTranferFailed = "transferfailed"
-)
-
-const (
-	TransferStateNotTouched = iota
-	TransferStatePending
-	TransferStateComplete
-	TransferStateManual
-)
 
 func (s *SyncManager) Start() error {
-
 	if logUtils.ShouldCleanOnStartup() {
 		err := logUtils.CleanForStartup()
 		if err != nil {
@@ -105,108 +54,66 @@ func (s *SyncManager) Start() error {
 		}
 	}
 
+	var lastChannelProcessed string
 	syncCount := 0
 	for {
+		s.channelsToSync = make([]Sync, 0, 10) // reset sync queue
 		err := s.checkUsedSpace()
 		if err != nil {
 			return errors.Err(err)
 		}
-
-		var syncs []Sync
 		shouldInterruptLoop := false
 
-		isSingleChannelSync := s.syncProperties.YoutubeChannelID != ""
+		if s.CliFlags.IsSingleChannelSync() {
-		if isSingleChannelSync {
+			channels, err := s.ApiConfig.FetchChannels("", &s.CliFlags)
-			channels, err := s.apiConfig.FetchChannels("", s.syncProperties)
 			if err != nil {
 				return errors.Err(err)
 			}
 			if len(channels) != 1 {
 				return errors.Err("Expected 1 channel, %d returned", len(channels))
 			}
-			lbryChannelName := channels[0].DesiredChannelName
+			s.enqueueChannel(&channels[0])
-			syncs = make([]Sync, 1)
-			syncs[0] = Sync{
-				APIConfig:            s.apiConfig,
-				YoutubeChannelID:     s.syncProperties.YoutubeChannelID,
-				LbryChannelName:      lbryChannelName,
-				lbryChannelID:        channels[0].ChannelClaimID,
-				MaxTries:             s.maxTries,
-				ConcurrentVideos:     s.concurrentVideos,
-				Refill:               s.refill,
-				Manager:              s,
-				MaxVideoLength:       s.maxVideoLength,
-				LbrycrdString:        s.lbrycrdString,
-				AwsS3ID:              s.awsS3ID,
-				AwsS3Secret:          s.awsS3Secret,
-				AwsS3Region:          s.awsS3Region,
-				AwsS3Bucket:          s.awsS3Bucket,
-				namer:                namer.NewNamer(),
-				Fee:                  channels[0].Fee,
-				clientPublishAddress: channels[0].PublishAddress,
-				publicKey:            channels[0].PublicKey,
-				transferState:        channels[0].TransferState,
-			}
 			shouldInterruptLoop = true
 		} else {
 			var queuesToSync []string
-			if s.syncStatus != "" {
+			if s.CliFlags.SyncStatus != "" {
-				queuesToSync = append(queuesToSync, s.syncStatus)
+				queuesToSync = append(queuesToSync, s.CliFlags.SyncStatus)
-			} else if s.SyncFlags.SyncUpdate {
+			} else if s.CliFlags.SyncUpdate {
-				queuesToSync = append(queuesToSync, StatusSyncing, StatusSynced)
+				queuesToSync = append(queuesToSync, shared.StatusSyncing, shared.StatusSynced)
 			} else {
-				queuesToSync = append(queuesToSync, StatusSyncing, StatusQueued)
+				queuesToSync = append(queuesToSync, shared.StatusSyncing, shared.StatusQueued)
 			}
 		queues:
 			for _, q := range queuesToSync {
-				//temporary override for sync-until to give tom the time to review the channels
+				channels, err := s.ApiConfig.FetchChannels(q, &s.CliFlags)
-				if q == StatusQueued {
-					s.syncProperties.SyncUntil = time.Now().Add(-8 * time.Hour).Unix()
-				}
-				channels, err := s.apiConfig.FetchChannels(q, s.syncProperties)
 				if err != nil {
 					return err
 				}
-				for i, c := range channels {
+				log.Infof("Currently processing the \"%s\" queue with %d channels", q, len(channels))
-					log.Infof("There are %d channels in the \"%s\" queue", len(channels)-i, q)
+				for _, c := range channels {
-					maxVideoLength := s.maxVideoLength
+					s.enqueueChannel(&c)
-					if c.TotalSubscribers < 1000 {
+					queueAll := q == shared.StatusFailed || q == shared.StatusSyncing
-						maxVideoLength = 1.0
+					if !queueAll {
-					}
+						break queues
-					syncs = append(syncs, Sync{
-						APIConfig:            s.apiConfig,
-						YoutubeChannelID:     c.ChannelId,
-						LbryChannelName:      c.DesiredChannelName,
-						lbryChannelID:        c.ChannelClaimID,
-						MaxTries:             s.maxTries,
-						ConcurrentVideos:     s.concurrentVideos,
-						MaxVideoLength:       maxVideoLength,
-						Refill:               s.refill,
-						Manager:              s,
-						LbrycrdString:        s.lbrycrdString,
-						AwsS3ID:              s.awsS3ID,
-						AwsS3Secret:          s.awsS3Secret,
-						AwsS3Region:          s.awsS3Region,
-						AwsS3Bucket:          s.awsS3Bucket,
-						namer:                namer.NewNamer(),
-						Fee:                  c.Fee,
-						clientPublishAddress: c.PublishAddress,
-						publicKey:            c.PublicKey,
-						transferState:        c.TransferState,
-					})
-					if q != StatusFailed {
-						continue queues
 					}
 				}
+				log.Infof("Drained the \"%s\" queue", q)
 			}
 		}
-		if len(syncs) == 0 {
+		if len(s.channelsToSync) == 0 {
 			log.Infoln("No channels to sync. Pausing 5 minutes!")
 			time.Sleep(5 * time.Minute)
 		}
-		for _, sync := range syncs {
+		for _, sync := range s.channelsToSync {
+			if lastChannelProcessed == sync.DbChannelData.ChannelId {
+				util.SendToSlack("We just killed a sync for %s to stop looping! (%s)", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId)
+				stopTheLoops := errors.Err("Found channel %s running twice, set it to failed, and reprocess later", sync.DbChannelData.DesiredChannelName)
+				sync.setChannelTerminationStatus(&stopTheLoops)
+				continue
+			}
+			lastChannelProcessed = sync.DbChannelData.ChannelId
 			shouldNotCount := false
-			logUtils.SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
+			logUtils.SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId, syncCount+1)
 			err := sync.FullCycle()
 			//TODO: THIS IS A TEMPORARY WORK AROUND FOR THE STUPID IP LOCKUP BUG
 			ipPool, _ := ip_manager.GetIPPool(sync.grp)
@@ -234,40 +141,35 @@ func (s *SyncManager) Start() error {
 				}
 				shouldNotCount = strings.Contains(err.Error(), "this youtube channel is being managed by another server")
 				if !shouldNotCount {
-					logUtils.SendInfoToSlack("A non fatal error was reported by the sync process. %s\nContinuing...", err.Error())
+					logUtils.SendInfoToSlack("A non fatal error was reported by the sync process.\n%s", errors.FullTrace(err))
 				}
 			}
 			err = blobs_reflector.ReflectAndClean()
 			if err != nil {
 				return errors.Prefix("@Nikooo777 something went wrong while reflecting blobs", err)
 			}
-			logUtils.SendInfoToSlack("Syncing %s (%s) reached an end. total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
+			logUtils.SendInfoToSlack("%s (%s) reached an end. Total processed channels since startup: %d", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId, syncCount+1)
 			if !shouldNotCount {
 				syncCount++
 			}
-			if sync.IsInterrupted() || (s.limit != 0 && syncCount >= s.limit) {
+			if sync.IsInterrupted() || (s.CliFlags.Limit != 0 && syncCount >= s.CliFlags.Limit) {
 				shouldInterruptLoop = true
 				break
 			}
 		}
-		if shouldInterruptLoop || s.SyncFlags.SingleRun {
+		if shouldInterruptLoop || s.CliFlags.SingleRun {
 			break
 		}
 	}
 	return nil
 }
-func (s *SyncManager) GetS3AWSConfig() aws.Config {
+
-	return aws.Config{
-		Credentials: credentials.NewStaticCredentials(s.awsS3ID, s.awsS3Secret, ""),
-		Region:      &s.awsS3Region,
-	}
-}
 func (s *SyncManager) checkUsedSpace() error {
 	usedPctile, err := GetUsedSpace(logUtils.GetBlobsDir())
 	if err != nil {
 		return errors.Err(err)
 	}
-	if usedPctile >= 0.90 && !s.SyncFlags.SkipSpaceCheck {
+	if usedPctile >= 0.90 && !s.CliFlags.SkipSpaceCheck {
 		return errors.Err(fmt.Sprintf("more than 90%% of the space has been used. use --skip-space-check to ignore. Used: %.1f%%", usedPctile*100))
 	}
 	log.Infof("disk usage: %.1f%%", usedPctile*100)

manager/s3_storage.go

@@ -0,0 +1,250 @@
+package manager
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+
+	logUtils "github.com/lbryio/ytsync/v5/util"
+)
+
+func (s *Sync) getS3Downloader() (*s3manager.Downloader, error) {
+	s3Session, err := session.NewSession(s.Manager.AwsConfigs.GetS3AWSConfig())
+	if err != nil {
+		return nil, errors.Prefix("error starting session: ", err)
+	}
+	downloader := s3manager.NewDownloader(s3Session)
+	return downloader, nil
+}
+func (s *Sync) getS3Uploader() (*s3manager.Uploader, error) {
+	s3Session, err := session.NewSession(s.Manager.AwsConfigs.GetS3AWSConfig())
+	if err != nil {
+		return nil, errors.Prefix("error starting session: ", err)
+	}
+	uploader := s3manager.NewUploader(s3Session)
+	return uploader, nil
+}
+
+func (s *Sync) downloadWallet() error {
+	defaultWalletDir, defaultTempWalletDir, key, err := s.getWalletPaths()
+	if err != nil {
+		return errors.Err(err)
+	}
+	downloader, err := s.getS3Downloader()
+	if err != nil {
+		return err
+	}
+	out, err := os.Create(defaultTempWalletDir)
+	if err != nil {
+		return errors.Prefix("error creating temp wallet: ", err)
+	}
+	defer out.Close()
+
+	bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
+		Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
+		Key:    key,
+	})
+	if err != nil {
+		// Casting to the awserr.Error type will allow you to inspect the error
+		// code returned by the service in code. The error code can be used
+		// to switch on context specific functionality. In this case a context
+		// specific error message is printed to the user based on the bucket
+		// and key existing.
+		//
+		// For information on other S3 API error codes see:
+		// http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
+		if aerr, ok := err.(awserr.Error); ok {
+			code := aerr.Code()
+			if code == s3.ErrCodeNoSuchKey {
+				return errors.Err("wallet not on S3")
+			}
+		}
+		return err
+	} else if bytesWritten == 0 {
+		return errors.Err("zero bytes written")
+	}
+
+	err = os.Rename(defaultTempWalletDir, defaultWalletDir)
+	if err != nil {
+		return errors.Prefix("error replacing temp wallet for default wallet: ", err)
+	}
+
+	return nil
+}
+
+func (s *Sync) downloadBlockchainDB() error {
+	if logUtils.IsRegTest() {
+		return nil // tests fail if we re-use the same blockchain DB
+	}
+	defaultBDBDir, defaultTempBDBDir, key, err := s.getBlockchainDBPaths()
+	if err != nil {
+		return errors.Err(err)
+	}
+	files, err := filepath.Glob(defaultBDBDir + "*")
+	if err != nil {
+		return errors.Err(err)
+	}
+	for _, f := range files {
+		err = os.Remove(f)
+		if err != nil {
+			return errors.Err(err)
+		}
+	}
+
+	downloader, err := s.getS3Downloader()
+	if err != nil {
+		return errors.Err(err)
+	}
+	out, err := os.Create(defaultTempBDBDir)
+	if err != nil {
+		return errors.Prefix("error creating temp wallet: ", err)
+	}
+	defer out.Close()
+
+	bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
+		Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
+		Key:    key,
+	})
+	if err != nil {
+		// Casting to the awserr.Error type will allow you to inspect the error
+		// code returned by the service in code. The error code can be used
+		// to switch on context specific functionality. In this case a context
+		// specific error message is printed to the user based on the bucket
+		// and key existing.
+		//
+		// For information on other S3 API error codes see:
+		// http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
+		if aerr, ok := err.(awserr.Error); ok {
+			code := aerr.Code()
+			if code == s3.ErrCodeNoSuchKey {
+				return nil // let ytsync sync the database by itself
+			}
+		}
+		return errors.Err(err)
+	} else if bytesWritten == 0 {
+		return errors.Err("zero bytes written")
+	}
+
+	err = os.Rename(defaultTempBDBDir, defaultBDBDir)
+	if err != nil {
+		return errors.Prefix("error replacing temp blockchain.db for default blockchain.db: ", err)
+	}
+
+	return nil
+}
+
+func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string, err error) {
+	defaultWallet = os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
+	tempWallet = os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet"
+	key = aws.String("/wallets/" + s.DbChannelData.ChannelId)
+	if logUtils.IsRegTest() {
+		defaultWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
+		tempWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet"
+		key = aws.String("/regtest/" + s.DbChannelData.ChannelId)
+	}
+
+	lbryumDir := os.Getenv("LBRYUM_DIR")
+	if lbryumDir != "" {
+		defaultWallet = lbryumDir + "/wallets/default_wallet"
+		tempWallet = lbryumDir + "/wallets/tmp_wallet"
+	}
+
+	if _, err := os.Stat(defaultWallet); !os.IsNotExist(err) {
+		return "", "", nil, errors.Err("default_wallet already exists")
+	}
+	return
+}
+
+func (s *Sync) getBlockchainDBPaths() (defaultDB, tempDB string, key *string, err error) {
+	lbryumDir := os.Getenv("LBRYUM_DIR")
+	if lbryumDir == "" {
+		if logUtils.IsRegTest() {
+			lbryumDir = os.Getenv("HOME") + "/.lbryum_regtest"
+		} else {
+			lbryumDir = os.Getenv("HOME") + "/.lbryum"
+		}
+	}
+	defaultDB = lbryumDir + "/lbc_mainnet/blockchain.db"
+	tempDB = lbryumDir + "/lbc_mainnet/tmp_blockchain.db"
+	key = aws.String("/blockchain_dbs/" + s.DbChannelData.ChannelId)
+	if logUtils.IsRegTest() {
+		defaultDB = lbryumDir + "/lbc_regtest/blockchain.db"
+		tempDB = lbryumDir + "/lbc_regtest/tmp_blockchain.db"
+		key = aws.String("/regtest_dbs/" + s.DbChannelData.ChannelId)
+	}
+	return
+}
+
+func (s *Sync) uploadWallet() error {
+	defaultWalletDir := logUtils.GetDefaultWalletPath()
+	key := aws.String("/wallets/" + s.DbChannelData.ChannelId)
+	if logUtils.IsRegTest() {
+		key = aws.String("/regtest/" + s.DbChannelData.ChannelId)
+	}
+
+	if _, err := os.Stat(defaultWalletDir); os.IsNotExist(err) {
+		return errors.Err("default_wallet does not exist")
+	}
+
+	uploader, err := s.getS3Uploader()
+	if err != nil {
+		return err
+	}
+
+	file, err := os.Open(defaultWalletDir)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	_, err = uploader.Upload(&s3manager.UploadInput{
+		Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
+		Key:    key,
+		Body:   file,
+	})
+	if err != nil {
+		return err
+	}
+
+	return os.Remove(defaultWalletDir)
+}
+
+func (s *Sync) uploadBlockchainDB() error {
+	defaultBDBDir, _, key, err := s.getBlockchainDBPaths()
+	if err != nil {
+		return errors.Err(err)
+	}
+
+	if _, err := os.Stat(defaultBDBDir); os.IsNotExist(err) {
+		return errors.Err("blockchain.db does not exist")
+	}
+
+	uploader, err := s.getS3Uploader()
+	if err != nil {
+		return err
+	}
+
+	file, err := os.Open(defaultBDBDir)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	_, err = uploader.Upload(&s3manager.UploadInput{
+		Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
+		Key:    key,
+		Body:   file,
+	})
+	if err != nil {
+		return err
+	}
+
+	return os.Remove(defaultBDBDir)
+}

manager/setup.go

@@ -3,23 +3,22 @@ package manager
 import (
 	"fmt"
 	"math"
-	"net/http"
 	"strconv"
 	"time"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
 	"github.com/lbryio/lbry.go/v2/extras/util"
+	"github.com/lbryio/ytsync/v5/shared"
 	"github.com/lbryio/ytsync/v5/timing"
 	logUtils "github.com/lbryio/ytsync/v5/util"
+	"github.com/lbryio/ytsync/v5/ytapi"
 
 	"github.com/lbryio/ytsync/v5/tags_manager"
 	"github.com/lbryio/ytsync/v5/thumbs"
 
 	"github.com/shopspring/decimal"
 	log "github.com/sirupsen/logrus"
-	"google.golang.org/api/googleapi/transport"
-	"google.golang.org/api/youtube/v3"
 )
 
 func (s *Sync) enableAddressReuse() error {
@@ -74,11 +73,7 @@ func (s *Sync) walletSetup() error {
 	}
 	log.Debugf("Starting balance is %.4f", balance)
 
-	n, err := s.CountVideos()
+	videosOnYoutube := int(s.DbChannelData.TotalVideos)
-	if err != nil {
-		return err
-	}
-	videosOnYoutube := int(n)
 
 	log.Debugf("Source channel has %d videos", videosOnYoutube)
 	if videosOnYoutube == 0 {
@@ -103,17 +98,17 @@ func (s *Sync) walletSetup() error {
 
 	log.Debugf("We already allocated credits for %d published videos and %d failed videos", publishedCount, failedCount)
 
-	if videosOnYoutube > s.Manager.videosLimit {
+	if videosOnYoutube > s.Manager.CliFlags.VideosLimit {
-		videosOnYoutube = s.Manager.videosLimit
+		videosOnYoutube = s.Manager.CliFlags.VideosLimit
 	}
 	unallocatedVideos := videosOnYoutube - (publishedCount + failedCount)
 	channelFee := channelClaimAmount
-	channelAlreadyClaimed := s.lbryChannelID != ""
+	channelAlreadyClaimed := s.DbChannelData.ChannelClaimID != ""
 	if channelAlreadyClaimed {
 		channelFee = 0.0
 	}
 	requiredBalance := float64(unallocatedVideos)*(publishAmount+estimatedMaxTxFee) + channelFee
-	if s.Manager.SyncFlags.UpgradeMetadata {
+	if s.Manager.CliFlags.UpgradeMetadata {
 		requiredBalance += float64(notUpgradedCount) * 0.001
 	}
 
@@ -122,8 +117,8 @@ func (s *Sync) walletSetup() error {
 		refillAmount = math.Max(math.Max(requiredBalance-balance, minimumAccountBalance-balance), minimumRefillAmount)
 	}
 
-	if s.Refill > 0 {
+	if s.Manager.CliFlags.Refill > 0 {
-		refillAmount += float64(s.Refill)
+		refillAmount += float64(s.Manager.CliFlags.Refill)
 	}
 
 	if refillAmount > 0 {
@@ -139,12 +134,11 @@ func (s *Sync) walletSetup() error {
 	} else if claimAddress == nil {
 		return errors.Err("could not get an address")
 	}
-	s.claimAddress = string(claimAddress.Items[0].Address)
+	if s.DbChannelData.PublishAddress == "" || !s.shouldTransfer() {
-	if s.claimAddress == "" {
+		s.DbChannelData.PublishAddress = string(claimAddress.Items[0].Address)
-		return errors.Err("found blank claim address")
 	}
-	if s.shouldTransfer() {
+	if s.DbChannelData.PublishAddress == "" {
-		s.claimAddress = s.clientPublishAddress
+		return errors.Err("found blank claim address")
 	}
 
 	err = s.ensureEnoughUTXOs()
@@ -266,15 +260,14 @@ func (s *Sync) ensureEnoughUTXOs() error {
 }
 
 func (s *Sync) waitForNewBlock() error {
-	start := time.Now()
+	defer func(start time.Time) { timing.TimedComponent("waitForNewBlock").Add(time.Since(start)) }(time.Now())
-	defer func(start time.Time) {
+
-		timing.TimedComponent("waitForNewBlock").Add(time.Since(start))
-	}(start)
 	log.Printf("regtest: %t, docker: %t", logUtils.IsRegTest(), logUtils.IsUsingDocker())
 	status, err := s.daemon.Status()
 	if err != nil {
 		return err
 	}
+
 	for status.Wallet.Blocks == 0 || status.Wallet.BlocksBehind != 0 {
 		time.Sleep(5 * time.Second)
 		status, err = s.daemon.Status()
@@ -304,14 +297,16 @@ func (s *Sync) waitForNewBlock() error {
 }
 
 func (s *Sync) GenerateRegtestBlock() error {
-	lbrycrd, err := logUtils.GetLbrycrdClient(s.LbrycrdString)
+	lbrycrd, err := logUtils.GetLbrycrdClient(s.Manager.LbrycrdDsn)
 	if err != nil {
 		return errors.Prefix("error getting lbrycrd client: ", err)
 	}
+
 	txs, err := lbrycrd.Generate(1)
 	if err != nil {
 		return errors.Prefix("error generating new block: ", err)
 	}
+
 	for _, tx := range txs {
 		log.Info("Generated tx: ", tx.String())
 	}
@@ -319,11 +314,9 @@ func (s *Sync) GenerateRegtestBlock() error {
 }
 
 func (s *Sync) ensureChannelOwnership() error {
-	start := time.Now()
+	defer func(start time.Time) { timing.TimedComponent("ensureChannelOwnership").Add(time.Since(start)) }(time.Now())
-	defer func(start time.Time) {
+
-		timing.TimedComponent("ensureChannelOwnership").Add(time.Since(start))
+	if s.DbChannelData.DesiredChannelName == "" {
-	}(start)
-	if s.LbryChannelName == "" {
 		return errors.Err("no channel name set")
 	}
 
@@ -336,27 +329,27 @@ func (s *Sync) ensureChannelOwnership() error {
 
 	var channelToUse *jsonrpc.Transaction
 	if len((*channels).Items) > 0 {
-		if s.lbryChannelID == "" {
+		if s.DbChannelData.ChannelClaimID == "" {
 			return errors.Err("this channel does not have a recorded claimID in the database. To prevent failures, updates are not supported until an entry is manually added in the database")
 		}
 		for _, c := range (*channels).Items {
 			log.Debugf("checking listed channel %s (%s)", c.ClaimID, c.Name)
-			if c.ClaimID != s.lbryChannelID {
+			if c.ClaimID != s.DbChannelData.ChannelClaimID {
 				continue
 			}
-			if c.Name != s.LbryChannelName {
+			if c.Name != s.DbChannelData.DesiredChannelName {
 				return errors.Err("the channel in the wallet is different than the channel in the database")
 			}
 			channelToUse = &c
 			break
 		}
 		if channelToUse == nil {
-			return errors.Err("this wallet has channels but not a single one is ours! Expected claim_id: %s (%s)", s.lbryChannelID, s.LbryChannelName)
+			return errors.Err("this wallet has channels but not a single one is ours! Expected claim_id: %s (%s)", s.DbChannelData.ChannelClaimID, s.DbChannelData.DesiredChannelName)
 		}
-	} else if s.transferState == TransferStateComplete {
+	} else if s.DbChannelData.TransferState == shared.TransferStateComplete {
 		return errors.Err("the channel was transferred but appears to have been abandoned!")
-	} else if s.lbryChannelID != "" {
+	} else if s.DbChannelData.ChannelClaimID != "" {
-		return errors.Err("the database has a channel recorded (%s) but nothing was found in our control", s.lbryChannelID)
+		return errors.Err("the database has a channel recorded (%s) but nothing was found in our control", s.DbChannelData.ChannelClaimID)
 	}
 
 	channelUsesOldMetadata := false
@@ -386,36 +379,24 @@ func (s *Sync) ensureChannelOwnership() error {
 			return err
 		}
 	}
-	client := &http.Client{
-		Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
-	}
 
-	service, err := youtube.New(client)
+	channelInfo, err := ytapi.ChannelInfo(s.DbChannelData.ChannelId)
 	if err != nil {
-		return errors.Prefix("error creating YouTube service", err)
+		return err
 	}
 
-	response, err := service.Channels.List("snippet,brandingSettings").Id(s.YoutubeChannelID).Do()
+	thumbnail := channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails)-1].URL
-	if err != nil {
+	thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail, s.DbChannelData.ChannelId, *s.Manager.AwsConfigs.GetS3AWSConfig())
-		return errors.Prefix("error getting channel details", err)
-	}
-
-	if len(response.Items) < 1 {
-		return errors.Err("youtube channel not found")
-	}
-
-	channelInfo := response.Items[0].Snippet
-	channelBranding := response.Items[0].BrandingSettings
-
-	thumbnail := thumbs.GetBestThumbnail(channelInfo.Thumbnails)
-	thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.Url, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
 	if err != nil {
 		return err
 	}
 
 	var bannerURL *string
-	if channelBranding.Image != nil && channelBranding.Image.BannerImageUrl != "" {
+	if channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails != nil {
-		bURL, err := thumbs.MirrorThumbnail(channelBranding.Image.BannerImageUrl, "banner-"+s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
+		bURL, err := thumbs.MirrorThumbnail(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails)-1].URL,
+			"banner-"+s.DbChannelData.ChannelId,
+			*s.Manager.AwsConfigs.GetS3AWSConfig(),
+		)
 		if err != nil {
 			return err
 		}
@@ -423,28 +404,29 @@ func (s *Sync) ensureChannelOwnership() error {
 	}
 
 	var languages []string = nil
-	if channelInfo.DefaultLanguage != "" {
+	//we don't have this data without the API
-		if channelInfo.DefaultLanguage == "iw" {
+	//if channelInfo.DefaultLanguage != "" {
-			channelInfo.DefaultLanguage = "he"
+	//	if channelInfo.DefaultLanguage == "iw" {
-		}
+	//		channelInfo.DefaultLanguage = "he"
-		languages = []string{channelInfo.DefaultLanguage}
+	//	}
-	}
+	//	languages = []string{channelInfo.DefaultLanguage}
+	//}
 	var locations []jsonrpc.Location = nil
-	if channelInfo.Country != "" {
+	if channelInfo.Topbar.DesktopTopbarRenderer.CountryCode != "" {
-		locations = []jsonrpc.Location{{Country: util.PtrToString(channelInfo.Country)}}
+		locations = []jsonrpc.Location{{Country: &channelInfo.Topbar.DesktopTopbarRenderer.CountryCode}}
 	}
 	var c *jsonrpc.TransactionSummary
 	claimCreateOptions := jsonrpc.ClaimCreateOptions{
-		Title:        &channelInfo.Title,
+		Title:        &channelInfo.Microformat.MicroformatDataRenderer.Title,
-		Description:  &channelInfo.Description,
+		Description:  &channelInfo.Metadata.ChannelMetadataRenderer.Description,
-		Tags:         tags_manager.GetTagsForChannel(s.YoutubeChannelID),
+		Tags:         tags_manager.GetTagsForChannel(s.DbChannelData.ChannelId),
 		Languages:    languages,
 		Locations:    locations,
 		ThumbnailURL: &thumbnailURL,
 	}
 	if channelUsesOldMetadata {
-		if s.transferState <= 1 {
+		if s.DbChannelData.TransferState <= 1 {
-			c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
+			c, err = s.daemon.ChannelUpdate(s.DbChannelData.ChannelClaimID, jsonrpc.ChannelUpdateOptions{
 				ClearTags:      util.PtrToBool(true),
 				ClearLocations: util.PtrToBool(true),
 				ClearLanguages: util.PtrToBool(true),
@@ -454,11 +436,11 @@ func (s *Sync) ensureChannelOwnership() error {
 				},
 			})
 		} else {
-			logUtils.SendInfoToSlack("%s (%s) has a channel with old metadata but isn't in our control anymore. Ignoring", s.LbryChannelName, s.lbryChannelID)
+			logUtils.SendInfoToSlack("%s (%s) has a channel with old metadata but isn't in our control anymore. Ignoring", s.DbChannelData.DesiredChannelName, s.DbChannelData.ChannelClaimID)
 			return nil
 		}
 	} else {
-		c, err = s.daemon.ChannelCreate(s.LbryChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
+		c, err = s.daemon.ChannelCreate(s.DbChannelData.DesiredChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
 			ClaimCreateOptions: claimCreateOptions,
 			CoverURL:           bannerURL,
 		})
@@ -467,8 +449,9 @@ func (s *Sync) ensureChannelOwnership() error {
 	if err != nil {
 		return err
 	}
-	s.lbryChannelID = c.Outputs[0].ClaimID
+
-	return s.Manager.apiConfig.SetChannelClaimID(s.YoutubeChannelID, s.lbryChannelID)
+	s.DbChannelData.ChannelClaimID = c.Outputs[0].ClaimID
+	return s.Manager.ApiConfig.SetChannelClaimID(s.DbChannelData.ChannelId, s.DbChannelData.ChannelClaimID)
 }
 
 func (s *Sync) addCredits(amountToAdd float64) error {
@@ -477,7 +460,7 @@ func (s *Sync) addCredits(amountToAdd float64) error {
 		timing.TimedComponent("addCredits").Add(time.Since(start))
 	}(start)
 	log.Printf("Adding %f credits", amountToAdd)
-	lbrycrdd, err := logUtils.GetLbrycrdClient(s.LbrycrdString)
+	lbrycrdd, err := logUtils.GetLbrycrdClient(s.Manager.LbrycrdDsn)
 	if err != nil {
 		return err
 	}

manager/transfer.go

@@ -10,7 +10,7 @@ import (
 	"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
 	"github.com/lbryio/lbry.go/v2/extras/stop"
 	"github.com/lbryio/lbry.go/v2/extras/util"
-	"github.com/lbryio/ytsync/v5/sdk"
+	"github.com/lbryio/ytsync/v5/shared"
 	"github.com/lbryio/ytsync/v5/timing"
 
 	log "github.com/sirupsen/logrus"
@@ -71,7 +71,10 @@ func abandonSupports(s *Sync) (float64, error) {
 	for page := uint64(1); page <= totalPages; page++ {
 		supports, err := s.daemon.SupportList(&defaultAccount, page, 50)
 		if err != nil {
-			return 0, errors.Prefix("cannot list claims", err)
+			supports, err = s.daemon.SupportList(&defaultAccount, page, 50)
+			if err != nil {
+				return 0, errors.Prefix("cannot list supports", err)
+			}
 		}
 		allSupports = append(allSupports, (*supports).Items...)
 		totalPages = (*supports).TotalPages
@@ -97,7 +100,7 @@ func abandonSupports(s *Sync) (float64, error) {
 	//TODO: remove this once the SDK team fixes their RPC bugs....
 	s.daemon.SetRPCTimeout(60 * time.Second)
 	defer s.daemon.SetRPCTimeout(5 * time.Minute)
-	for i := 0; i < s.ConcurrentVideos; i++ {
+	for i := 0; i < s.Manager.CliFlags.ConcurrentJobs; i++ {
 		consumerWG.Add(1)
 		go func() {
 			defer consumerWG.Done()
@@ -189,7 +192,7 @@ func abandonSupports(s *Sync) (float64, error) {
 type updateInfo struct {
 	ClaimID             string
 	streamUpdateOptions *jsonrpc.StreamUpdateOptions
-	videoStatus         *sdk.VideoStatus
+	videoStatus         *shared.VideoStatus
 }
 
 func transferVideos(s *Sync) error {
@@ -199,7 +202,7 @@ func transferVideos(s *Sync) error {
 	}(start)
 	cleanTransfer := true
 
-	streamChan := make(chan updateInfo, s.ConcurrentVideos)
+	streamChan := make(chan updateInfo, s.Manager.CliFlags.ConcurrentJobs)
 	account, err := s.getDefaultAccount()
 	if err != nil {
 		return err
@@ -213,13 +216,13 @@ func transferVideos(s *Sync) error {
 	go func() {
 		defer producerWG.Done()
 		for _, video := range s.syncedVideos {
-			if !video.Published || video.Transferred || video.MetadataVersion != LatestMetadataVersion {
+			if !video.Published || video.Transferred || video.MetadataVersion != shared.LatestMetadataVersion {
 				continue
 			}
 
 			var stream *jsonrpc.Claim = nil
 			for _, c := range streams.Items {
-				if c.ClaimID != video.ClaimID || (c.SigningChannel != nil && c.SigningChannel.ClaimID != s.lbryChannelID) {
+				if c.ClaimID != video.ClaimID || (c.SigningChannel != nil && c.SigningChannel.ClaimID != s.DbChannelData.ChannelClaimID) {
 					continue
 				}
 				stream = &c
@@ -232,7 +235,7 @@ func transferVideos(s *Sync) error {
 			streamUpdateOptions := jsonrpc.StreamUpdateOptions{
 				StreamCreateOptions: &jsonrpc.StreamCreateOptions{
 					ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
-						ClaimAddress: &s.clientPublishAddress,
+						ClaimAddress: &s.DbChannelData.PublishAddress,
 						FundingAccountIDs: []string{
 							account,
 						},
@@ -240,12 +243,12 @@ func transferVideos(s *Sync) error {
 				},
 				Bid: util.PtrToString("0.005"), // Todo - Dont hardcode
 			}
-			videoStatus := sdk.VideoStatus{
+			videoStatus := shared.VideoStatus{
-				ChannelID:     s.YoutubeChannelID,
+				ChannelID:     s.DbChannelData.ChannelId,
 				VideoID:       video.VideoID,
 				ClaimID:       video.ClaimID,
 				ClaimName:     video.ClaimName,
-				Status:        VideoStatusPublished,
+				Status:        shared.VideoStatusPublished,
 				IsTransferred: util.PtrToBool(true),
 			}
 			streamChan <- updateInfo{
@@ -257,7 +260,7 @@ func transferVideos(s *Sync) error {
 	}()
 
 	consumerWG := &stop.Group{}
-	for i := 0; i < s.ConcurrentVideos; i++ {
+	for i := 0; i < s.Manager.CliFlags.ConcurrentJobs; i++ {
 		consumerWG.Add(1)
 		go func(worker int) {
 			defer consumerWG.Done()
@@ -290,13 +293,13 @@ func (s *Sync) streamUpdate(ui *updateInfo) error {
 	timing.TimedComponent("transferStreamUpdate").Add(time.Since(start))
 	if updateError != nil {
 		ui.videoStatus.FailureReason = updateError.Error()
-		ui.videoStatus.Status = VideoStatusTranferFailed
+		ui.videoStatus.Status = shared.VideoStatusTranferFailed
 		ui.videoStatus.IsTransferred = util.PtrToBool(false)
 	} else {
 		ui.videoStatus.IsTransferred = util.PtrToBool(len(result.Outputs) != 0)
 	}
 	log.Infof("TRANSFERRED %t", *ui.videoStatus.IsTransferred)
-	statusErr := s.APIConfig.MarkVideoStatus(*ui.videoStatus)
+	statusErr := s.Manager.ApiConfig.MarkVideoStatus(*ui.videoStatus)
 	if statusErr != nil {
 		return errors.Prefix(statusErr.Error(), updateError)
 	}
@@ -318,7 +321,7 @@ func transferChannel(s *Sync) error {
 	}
 	var channelClaim *jsonrpc.Transaction = nil
 	for _, c := range channelClaims.Items {
-		if c.ClaimID != s.lbryChannelID {
+		if c.ClaimID != s.DbChannelData.ChannelClaimID {
 			continue
 		}
 		channelClaim = &c
@@ -332,11 +335,11 @@ func transferChannel(s *Sync) error {
 		Bid: util.PtrToString(fmt.Sprintf("%.6f", channelClaimAmount-0.005)),
 		ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
 			ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
-				ClaimAddress: &s.clientPublishAddress,
+				ClaimAddress: &s.DbChannelData.PublishAddress,
 			},
 		},
 	}
-	result, err := s.daemon.ChannelUpdate(s.lbryChannelID, updateOptions)
+	result, err := s.daemon.ChannelUpdate(s.DbChannelData.ChannelClaimID, updateOptions)
 	if err != nil {
 		return errors.Err(err)
 	}

manager/ytsync.go

@@ -3,11 +3,9 @@ package manager
 import (
 	"fmt"
 	"io/ioutil"
-	"net/http"
 	"os"
 	"os/signal"
 	"runtime/debug"
-	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -17,25 +15,19 @@ import (
 	"github.com/lbryio/ytsync/v5/ip_manager"
 	"github.com/lbryio/ytsync/v5/namer"
 	"github.com/lbryio/ytsync/v5/sdk"
+	"github.com/lbryio/ytsync/v5/shared"
 	"github.com/lbryio/ytsync/v5/sources"
 	"github.com/lbryio/ytsync/v5/thumbs"
 	"github.com/lbryio/ytsync/v5/timing"
 	logUtils "github.com/lbryio/ytsync/v5/util"
+	"github.com/lbryio/ytsync/v5/ytapi"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
 	"github.com/lbryio/lbry.go/v2/extras/stop"
 	"github.com/lbryio/lbry.go/v2/extras/util"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/awserr"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	log "github.com/sirupsen/logrus"
-	"google.golang.org/api/googleapi/transport"
-	"google.golang.org/api/youtube/v3"
 )
 
 const (
@@ -47,52 +39,20 @@ const (
 	maxReasonLength       = 500
 )
 
-type video interface {
-	Size() *int64
-	ID() string
-	IDAndNum() string
-	PlaylistPosition() int
-	PublishedAt() time.Time
-	Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool, *sync.RWMutex) (*sources.SyncSummary, error)
-}
-
-// sorting videos
-type byPublishedAt []video
-
-func (a byPublishedAt) Len() int           { return len(a) }
-func (a byPublishedAt) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
-func (a byPublishedAt) Less(i, j int) bool { return a[i].PublishedAt().Before(a[j].PublishedAt()) }
-
 // Sync stores the options that control how syncing happens
 type Sync struct {
-	APIConfig            *sdk.APIConfig
+	DbChannelData *shared.YoutubeChannel
-	YoutubeChannelID     string
+	Manager       *SyncManager
-	LbryChannelName      string
+
-	MaxTries             int
+	daemon           *jsonrpc.Client
-	ConcurrentVideos     int
+	videoDirectory   string
-	Refill               int
+	syncedVideosMux  *sync.RWMutex
-	Manager              *SyncManager
+	syncedVideos     map[string]sdk.SyncedVideo
-	LbrycrdString        string
+	grp              *stop.Group
-	AwsS3ID              string
+	namer            *namer.Namer
-	AwsS3Secret          string
+	walletMux        *sync.RWMutex
-	AwsS3Region          string
+	queue            chan ytapi.Video
-	AwsS3Bucket          string
+	defaultAccountID string
-	Fee                  *sdk.Fee
-	daemon               *jsonrpc.Client
-	claimAddress         string
-	videoDirectory       string
-	syncedVideosMux      *sync.RWMutex
-	syncedVideos         map[string]sdk.SyncedVideo
-	grp                  *stop.Group
-	lbryChannelID        string
-	namer                *namer.Namer
-	walletMux            *sync.RWMutex
-	queue                chan video
-	transferState        int
-	clientPublishAddress string
-	publicKey            string
-	defaultAccountID     string
-	MaxVideoLength       float64
 }
 
 func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) {
@@ -119,118 +79,8 @@ func (s *Sync) IsInterrupted() bool {
 	}
 }
 
-func (s *Sync) downloadWallet() error {
-	defaultWalletDir, defaultTempWalletDir, key, err := s.getWalletPaths()
-	if err != nil {
-		return errors.Err(err)
-	}
-
-	creds := credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, "")
-	s3Session, err := session.NewSession(&aws.Config{Region: aws.String(s.AwsS3Region), Credentials: creds})
-	if err != nil {
-		return errors.Prefix("error starting session: ", err)
-	}
-	downloader := s3manager.NewDownloader(s3Session)
-	out, err := os.Create(defaultTempWalletDir)
-	if err != nil {
-		return errors.Prefix("error creating temp wallet: ", err)
-	}
-	defer out.Close()
-
-	bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
-		Bucket: aws.String(s.AwsS3Bucket),
-		Key:    key,
-	})
-	if err != nil {
-		// Casting to the awserr.Error type will allow you to inspect the error
-		// code returned by the service in code. The error code can be used
-		// to switch on context specific functionality. In this case a context
-		// specific error message is printed to the user based on the bucket
-		// and key existing.
-		//
-		// For information on other S3 API error codes see:
-		// http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
-		if aerr, ok := err.(awserr.Error); ok {
-			code := aerr.Code()
-			if code == s3.ErrCodeNoSuchKey {
-				return errors.Err("wallet not on S3")
-			}
-		}
-		return err
-	} else if bytesWritten == 0 {
-		return errors.Err("zero bytes written")
-	}
-
-	err = os.Rename(defaultTempWalletDir, defaultWalletDir)
-	if err != nil {
-		return errors.Prefix("error replacing temp wallet for default wallet: ", err)
-	}
-
-	return nil
-}
-
-func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string, err error) {
-
-	defaultWallet = os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
-	tempWallet = os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet"
-	key = aws.String("/wallets/" + s.YoutubeChannelID)
-	if logUtils.IsRegTest() {
-		defaultWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
-		tempWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet"
-		key = aws.String("/regtest/" + s.YoutubeChannelID)
-	}
-
-	walletPath := os.Getenv("LBRYNET_WALLETS_DIR")
-	if walletPath != "" {
-		defaultWallet = walletPath + "/wallets/default_wallet"
-		tempWallet = walletPath + "/wallets/tmp_wallet"
-	}
-
-	if _, err := os.Stat(defaultWallet); !os.IsNotExist(err) {
-		return "", "", nil, errors.Err("default_wallet already exists")
-	}
-	return
-}
-
-func (s *Sync) uploadWallet() error {
-	defaultWalletDir := logUtils.GetDefaultWalletPath()
-	key := aws.String("/wallets/" + s.YoutubeChannelID)
-	if logUtils.IsRegTest() {
-		key = aws.String("/regtest/" + s.YoutubeChannelID)
-	}
-
-	if _, err := os.Stat(defaultWalletDir); os.IsNotExist(err) {
-		return errors.Err("default_wallet does not exist")
-	}
-
-	creds := credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, "")
-	s3Session, err := session.NewSession(&aws.Config{Region: aws.String(s.AwsS3Region), Credentials: creds})
-	if err != nil {
-		return err
-	}
-
-	uploader := s3manager.NewUploader(s3Session)
-
-	file, err := os.Open(defaultWalletDir)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	_, err = uploader.Upload(&s3manager.UploadInput{
-		Bucket: aws.String(s.AwsS3Bucket),
-		Key:    key,
-		Body:   file,
-	})
-	if err != nil {
-		return err
-	}
-
-	return os.Remove(defaultWalletDir)
-}
-
 func (s *Sync) setStatusSyncing() error {
-	syncedVideos, claimNames, err := s.Manager.apiConfig.SetChannelStatus(s.YoutubeChannelID, StatusSyncing, "", nil)
+	syncedVideos, claimNames, err := s.Manager.ApiConfig.SetChannelStatus(s.DbChannelData.ChannelId, shared.StatusSyncing, "", nil)
 	if err != nil {
 		return err
 	}
@@ -241,36 +91,27 @@ func (s *Sync) setStatusSyncing() error {
 	return nil
 }
 
-func (s *Sync) setExceptions() {
-	if s.YoutubeChannelID == "UCwjQfNRW6sGYb__pd7d4nUg" { //@FreeTalkLive
-		s.MaxVideoLength = 9999.0 // skips max length checks
-		s.Manager.maxVideoSize = 0
-	}
-}
-
 var stopGroup = stop.New()
 
 func (s *Sync) FullCycle() (e error) {
 	if os.Getenv("HOME") == "" {
 		return errors.Err("no $HOME env var found")
 	}
-	if s.YoutubeChannelID == "" {
-		return errors.Err("channel ID not provided")
-	}
-
-	s.setExceptions()
 	defer timing.ClearTimings()
 	s.syncedVideosMux = &sync.RWMutex{}
 	s.walletMux = &sync.RWMutex{}
 	s.grp = stopGroup
-	s.queue = make(chan video)
+	s.queue = make(chan ytapi.Video)
 	interruptChan := make(chan os.Signal, 1)
 	signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
 	defer signal.Stop(interruptChan)
 	go func() {
 		<-interruptChan
+		util.SendToSlack("got interrupt, shutting down")
 		log.Println("Got interrupt signal, shutting down (if publishing, will shut down after current publish)")
 		s.grp.Stop()
+		time.Sleep(5 * time.Second)
+		debug.PrintStack() // so we can figure out what's not stopping
 	}()
 	err := s.setStatusSyncing()
 	if err != nil {
@@ -286,6 +127,10 @@ func (s *Sync) FullCycle() (e error) {
 	} else {
 		log.Println("Starting new wallet")
 	}
+	err = s.downloadBlockchainDB()
+	if err != nil {
+		return errors.Prefix("failure in downloading blockchain.db", err)
+	}
 
 	defer s.stopAndUploadWallet(&e)
 
@@ -328,16 +173,18 @@ func (s *Sync) FullCycle() (e error) {
 
 func (s *Sync) processTransfers() (e error) {
 	log.Println("Processing transfers")
-	err := waitConfirmations(s)
+	if s.DbChannelData.TransferState != 2 {
-	if err != nil {
+		err := waitConfirmations(s)
-		return err
+		if err != nil {
+			return err
+		}
 	}
 	supportAmount, err := abandonSupports(s)
 	if err != nil {
 		return errors.Prefix(fmt.Sprintf("%.6f LBCs were abandoned before failing", supportAmount), err)
 	}
 	if supportAmount > 0 {
-		logUtils.SendInfoToSlack("(%s) %.6f LBCs were abandoned and should be used as support", s.YoutubeChannelID, supportAmount)
+		logUtils.SendInfoToSlack("(%s) %.6f LBCs were abandoned and should be used as support", s.DbChannelData.ChannelId, supportAmount)
 	}
 	err = transferVideos(s)
 	if err != nil {
@@ -358,19 +205,23 @@ func (s *Sync) processTransfers() (e error) {
 			return err
 		}
 		isTip := true
-		summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount}, nil)
+		summary, err := s.daemon.SupportCreate(s.DbChannelData.ChannelClaimID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount}, nil)
 		if err != nil {
-			if strings.Contains(err.Error(), "tx-size") { //TODO: this is a silly workaround and should be written in an recursive function
+			if strings.Contains(err.Error(), "tx-size") { //TODO: this is a silly workaround...
-				summary, err = s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount/2.0), &isTip, nil, []string{defaultAccount}, nil)
+				_, spendErr := s.daemon.TxoSpend(util.PtrToString("other"), nil, nil, nil, nil, &s.defaultAccountID)
+				if spendErr != nil {
+					return errors.Prefix(fmt.Sprintf("something went wrong while tipping the channel for %.6f LBCs", supportAmount), err)
+				}
+				err = s.waitForNewBlock()
+				if err != nil {
+					return errors.Prefix(fmt.Sprintf("something went wrong while tipping the channel for %.6f LBCs (waiting for new block)", supportAmount), err)
+				}
+				summary, err = s.daemon.SupportCreate(s.DbChannelData.ChannelClaimID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount}, nil)
 				if err != nil {
 					return errors.Prefix(fmt.Sprintf("something went wrong while tipping the channel for %.6f LBCs", supportAmount), err)
 				}
-				summary, err = s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount/2.0), &isTip, nil, []string{defaultAccount}, nil)
-				if err != nil {
-					return errors.Err(err)
-				}
 			} else {
-				return errors.Err(err)
+				return errors.Prefix(fmt.Sprintf("something went wrong while tipping the channel for %.6f LBCs", supportAmount), err)
 			}
 		}
 		if len(summary.Outputs) < 1 {
@@ -392,7 +243,7 @@ func deleteSyncFolder(videoDirectory string) {
 }
 
 func (s *Sync) shouldTransfer() bool {
-	return s.transferState >= 1 && s.clientPublishAddress != "" && !s.Manager.SyncFlags.DisableTransfers
+	return s.DbChannelData.TransferState >= 1 && s.DbChannelData.PublishAddress != "" && !s.Manager.CliFlags.DisableTransfers && s.DbChannelData.TransferState != 3
 }
 
 func (s *Sync) setChannelTerminationStatus(e *error) {
@@ -400,7 +251,7 @@ func (s *Sync) setChannelTerminationStatus(e *error) {
 
 	if s.shouldTransfer() {
 		if *e == nil {
-			transferState = util.PtrToInt(TransferStateComplete)
+			transferState = util.PtrToInt(shared.TransferStateComplete)
 		}
 	}
 	if *e != nil {
@@ -413,13 +264,13 @@ func (s *Sync) setChannelTerminationStatus(e *error) {
 			return
 		}
 		failureReason := (*e).Error()
-		_, _, err := s.Manager.apiConfig.SetChannelStatus(s.YoutubeChannelID, StatusFailed, failureReason, transferState)
+		_, _, err := s.Manager.ApiConfig.SetChannelStatus(s.DbChannelData.ChannelId, shared.StatusFailed, failureReason, transferState)
 		if err != nil {
-			msg := fmt.Sprintf("Failed setting failed state for channel %s", s.LbryChannelName)
+			msg := fmt.Sprintf("Failed setting failed state for channel %s", s.DbChannelData.DesiredChannelName)
 			*e = errors.Prefix(msg+err.Error(), *e)
 		}
 	} else if !s.IsInterrupted() {
-		_, _, err := s.Manager.apiConfig.SetChannelStatus(s.YoutubeChannelID, StatusSynced, "", transferState)
+		_, _, err := s.Manager.ApiConfig.SetChannelStatus(s.DbChannelData.ChannelId, shared.StatusSynced, "", transferState)
 		if err != nil {
 			*e = err
 		}
@@ -440,7 +291,7 @@ func (s *Sync) waitForDaemonStart() error {
 			if err == nil && status.StartupStatus.Wallet && status.IsRunning {
 				return nil
 			}
-			if time.Since(beginTime).Minutes() > 60 {
+			if time.Since(beginTime).Minutes() > 120 {
 				s.grp.Stop()
 				return errors.Err("the daemon is taking too long to start. Something is wrong")
 			}
@@ -466,7 +317,14 @@ func (s *Sync) stopAndUploadWallet(e *error) {
 			if err != nil {
 				if *e == nil {
 					e = &err
-					return
+				} else {
+					*e = errors.Prefix("failure uploading wallet", *e)
+				}
+			}
+			err = s.uploadBlockchainDB()
+			if err != nil {
+				if *e == nil {
+					e = &err
 				} else {
 					*e = errors.Prefix("failure uploading wallet", *e)
 				}
@@ -515,7 +373,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
 	abandonedClaims := false
 	videoIDs := make(map[string]jsonrpc.Claim)
 	for _, c := range claims {
-		if !isYtsyncClaim(c, s.lbryChannelID) {
+		if !isYtsyncClaim(c, s.DbChannelData.ChannelClaimID) {
 			continue
 		}
 		tn := c.Value.GetThumbnail().GetUrl()
@@ -533,7 +391,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
 			claimToAbandon = cl
 			videoIDs[videoID] = c
 		}
-		if claimToAbandon.Address != s.clientPublishAddress && !s.syncedVideos[videoID].Transferred {
+		if claimToAbandon.Address != s.DbChannelData.PublishAddress && !s.syncedVideos[videoID].Transferred {
 			log.Debugf("abandoning %+v", claimToAbandon)
 			_, err := s.daemon.StreamAbandon(claimToAbandon.Txid, claimToAbandon.Nout, nil, false)
 			if err != nil {
@@ -562,7 +420,7 @@ type ytsyncClaim struct {
 func (s *Sync) mapFromClaims(claims []jsonrpc.Claim) map[string]ytsyncClaim {
 	videoIDMap := make(map[string]ytsyncClaim, len(claims))
 	for _, c := range claims {
-		if !isYtsyncClaim(c, s.lbryChannelID) {
+		if !isYtsyncClaim(c, s.DbChannelData.ChannelClaimID) {
 			continue
 		}
 		tn := c.Value.GetThumbnail().GetUrl()
@@ -602,8 +460,8 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
 		claimNameDiffers := claimInDatabase && sv.ClaimName != chainInfo.ClaimName
 		claimMarkedUnpublished := claimInDatabase && !sv.Published
 		_, isOwnClaim := ownClaimsInfo[videoID]
-		tranferred := !isOwnClaim
+		transferred := !isOwnClaim || s.DbChannelData.TransferState == 3
-		transferStatusMismatch := sv.Transferred != tranferred
+		transferStatusMismatch := sv.Transferred != transferred
 
 		if metadataDiffers {
 			log.Debugf("%s: Mismatch in database for metadata. DB: %d - Blockchain: %d", videoID, sv.MetadataVersion, chainInfo.MetadataVersion)
@@ -621,7 +479,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
 			log.Debugf("%s: Published but is not in database (%s - %s)", videoID, chainInfo.ClaimName, chainInfo.ClaimID)
 		}
 		if transferStatusMismatch {
-			log.Debugf("%s: is marked as transferred %t on it's actually %t", videoID, sv.Transferred, tranferred)
+			log.Debugf("%s: is marked as transferred %t on it's actually %t", videoID, sv.Transferred, transferred)
 		}
 
 		if !claimInDatabase || metadataDiffers || claimIDDiffers || claimNameDiffers || claimMarkedUnpublished || transferStatusMismatch {
@@ -631,15 +489,15 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
 			}
 			fixed++
 			log.Debugf("updating %s in the database", videoID)
-			err = s.Manager.apiConfig.MarkVideoStatus(sdk.VideoStatus{
+			err = s.Manager.ApiConfig.MarkVideoStatus(shared.VideoStatus{
-				ChannelID:       s.YoutubeChannelID,
+				ChannelID:       s.DbChannelData.ChannelId,
 				VideoID:         videoID,
-				Status:          VideoStatusPublished,
+				Status:          shared.VideoStatusPublished,
 				ClaimID:         chainInfo.ClaimID,
 				ClaimName:       chainInfo.ClaimName,
 				Size:            util.PtrToInt64(int64(claimSize)),
 				MetaDataVersion: chainInfo.MetadataVersion,
-				IsTransferred:   &tranferred,
+				IsTransferred:   &transferred,
 			})
 			if err != nil {
 				return count, fixed, 0, err
@@ -680,13 +538,13 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
 		}
 		_, ok := ownClaimsInfo[vID]
 		if !ok && sv.Published {
-			log.Debugf("%s: claims to be published but wasn't found in the list of claims and will be removed if --remove-db-unpublished was specified (%t)", vID, s.Manager.SyncFlags.RemoveDBUnpublished)
+			log.Debugf("%s: claims to be published but wasn't found in the list of claims and will be removed if --remove-db-unpublished was specified (%t)", vID, s.Manager.CliFlags.RemoveDBUnpublished)
 			idsToRemove = append(idsToRemove, vID)
 		}
 	}
-	if s.Manager.SyncFlags.RemoveDBUnpublished && len(idsToRemove) > 0 {
+	if s.Manager.CliFlags.RemoveDBUnpublished && len(idsToRemove) > 0 {
 		log.Infof("removing: %s", strings.Join(idsToRemove, ","))
-		err := s.Manager.apiConfig.DeleteVideos(idsToRemove)
+		err := s.Manager.ApiConfig.DeleteVideos(idsToRemove)
 		if err != nil {
 			return count, fixed, len(idsToRemove), err
 		}
@@ -717,7 +575,7 @@ func (s *Sync) getClaims(defaultOnly bool) ([]jsonrpc.Claim, error) {
 	}
 	items := make([]jsonrpc.Claim, 0, len(claims.Items))
 	for _, c := range claims.Items {
-		if c.SigningChannel != nil && c.SigningChannel.ClaimID == s.lbryChannelID {
+		if c.SigningChannel != nil && c.SigningChannel.ClaimID == s.DbChannelData.ChannelClaimID {
 			items = append(items, c)
 		}
 	}
@@ -774,11 +632,11 @@ func (s *Sync) checkIntegrity() error {
 	}
 
 	if pubsOnWallet > pubsOnDB { //This case should never happen
-		logUtils.SendInfoToSlack("We're claiming to have published %d videos but in reality we published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID)
+		logUtils.SendInfoToSlack("We're claiming to have published %d videos but in reality we published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId)
 		return errors.Err("not all published videos are in the database")
 	}
 	if pubsOnWallet < pubsOnDB {
-		logUtils.SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID)
+		logUtils.SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId)
 	}
 
 	_, err = s.getUnsentSupports() //TODO: use the returned value when it works
@@ -811,24 +669,24 @@ func (s *Sync) doSync() error {
 		return err
 	}
 
-	if s.transferState < TransferStateComplete {
+	if s.DbChannelData.TransferState < shared.TransferStateComplete {
-		cert, err := s.daemon.ChannelExport(s.lbryChannelID, nil, nil)
+		cert, err := s.daemon.ChannelExport(s.DbChannelData.ChannelClaimID, nil, nil)
 		if err != nil {
 			return errors.Prefix("error getting channel cert", err)
 		}
 		if cert != nil {
-			err = s.APIConfig.SetChannelCert(string(*cert), s.lbryChannelID)
+			err = s.Manager.ApiConfig.SetChannelCert(string(*cert), s.DbChannelData.ChannelClaimID)
 			if err != nil {
 				return errors.Prefix("error setting channel cert", err)
 			}
 		}
 	}
 
-	if s.Manager.SyncFlags.StopOnError {
+	if s.Manager.CliFlags.StopOnError {
 		log.Println("Will stop publishing if an error is detected")
 	}
 
-	for i := 0; i < s.ConcurrentVideos; i++ {
+	for i := 0; i < s.Manager.CliFlags.ConcurrentJobs; i++ {
 		s.grp.Add(1)
 		go func(i int) {
 			defer s.grp.Done()
@@ -836,7 +694,7 @@ func (s *Sync) doSync() error {
 		}(i)
 	}
 
-	if s.LbryChannelName == "@UCBerkeley" {
+	if s.DbChannelData.DesiredChannelName == "@UCBerkeley" {
 		err = errors.Err("UCB is not supported in this version of YTSYNC")
 	} else {
 		err = s.enqueueYoutubeVideos()
@@ -847,7 +705,7 @@ func (s *Sync) doSync() error {
 }
 
 func (s *Sync) startWorker(workerNum int) {
-	var v video
+	var v ytapi.Video
 	var more bool
 
 	for {
@@ -872,10 +730,17 @@ func (s *Sync) startWorker(workerNum int) {
 
 		tryCount := 0
 		for {
+			select { // check again inside the loop so this dies faster
+			case <-s.grp.Ch():
+				log.Printf("Stopping worker %d", workerNum)
+				return
+			default:
+			}
 			tryCount++
 			err := s.processVideo(v)
 
 			if err != nil {
+				util.SendToSlack("Tried to process %s. Error: %v", v.ID(), err)
 				logMsg := fmt.Sprintf("error processing video %s: %s", v.ID(), err.Error())
 				log.Errorln(logMsg)
 				if strings.Contains(strings.ToLower(err.Error()), "interrupted by user") {
@@ -891,9 +756,9 @@ func (s *Sync) startWorker(workerNum int) {
 					"Couldn't find private key for id",
 					"You already have a stream claim published under the name",
 				}
-				if util.SubstringInSlice(err.Error(), fatalErrors) || s.Manager.SyncFlags.StopOnError {
+				if util.SubstringInSlice(err.Error(), fatalErrors) || s.Manager.CliFlags.StopOnError {
 					s.grp.Stop()
-				} else if s.MaxTries > 1 {
+				} else if s.Manager.CliFlags.MaxTries > 1 {
 					errorsNoRetry := []string{
 						"non 200 status code received",
 						"This video contains content from",
@@ -923,7 +788,7 @@ func (s *Sync) startWorker(workerNum int) {
 					}
 					if util.SubstringInSlice(err.Error(), errorsNoRetry) {
 						log.Println("This error should not be retried at all")
-					} else if tryCount < s.MaxTries {
+					} else if tryCount < s.Manager.CliFlags.MaxTries {
 						if util.SubstringInSlice(err.Error(), []string{
 							"txn-mempool-conflict",
 							"too-long-mempool-chain",
@@ -972,14 +837,14 @@ func (s *Sync) startWorker(workerNum int) {
 						existingClaimSize = existingClaim.Size
 					}
 				}
-				videoStatus := VideoStatusFailed
+				videoStatus := shared.VideoStatusFailed
 				if strings.Contains(err.Error(), "upgrade failed") {
-					videoStatus = VideoStatusUpgradeFailed
+					videoStatus = shared.VideoStatusUpgradeFailed
 				} else {
 					s.AppendSyncedVideo(v.ID(), false, err.Error(), existingClaimName, existingClaimID, 0, existingClaimSize)
 				}
-				err = s.Manager.apiConfig.MarkVideoStatus(sdk.VideoStatus{
+				err = s.Manager.ApiConfig.MarkVideoStatus(shared.VideoStatus{
-					ChannelID:     s.YoutubeChannelID,
+					ChannelID:     s.DbChannelData.ChannelId,
 					VideoID:       v.ID(),
 					Status:        videoStatus,
 					ClaimID:       existingClaimID,
@@ -996,107 +861,23 @@ func (s *Sync) startWorker(workerNum int) {
 	}
 }
 
-var mostRecentlyFailedChannel string
-
 func (s *Sync) enqueueYoutubeVideos() error {
-	start := time.Now()
+	defer func(start time.Time) { timing.TimedComponent("enqueueYoutubeVideos").Add(time.Since(start)) }(time.Now())
-	defer func(start time.Time) {
-		timing.TimedComponent("enqueueYoutubeVideos").Add(time.Since(start))
-	}(start)
-	client := &http.Client{
-		Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
-	}
 
-	service, err := youtube.New(client)
-	if err != nil {
-		return errors.Prefix("error creating YouTube service", err)
-	}
-
-	response, err := service.Channels.List("contentDetails").Id(s.YoutubeChannelID).Do()
-	if err != nil {
-		return errors.Prefix("error getting channels", err)
-	}
-
-	if len(response.Items) < 1 {
-		return errors.Err("youtube channel not found")
-	}
-
-	if response.Items[0].ContentDetails.RelatedPlaylists == nil {
-		return errors.Err("no related playlists")
-	}
-
-	playlistID := response.Items[0].ContentDetails.RelatedPlaylists.Uploads
-	if playlistID == "" {
-		return errors.Err("no channel playlist")
-	}
-
-	var videos []video
 	ipPool, err := ip_manager.GetIPPool(s.grp)
 	if err != nil {
 		return err
 	}
 
-	playlistMap := make(map[string]*youtube.PlaylistItemSnippet, 50)
+	videos, err := ytapi.GetVideosToSync(s.Manager.ApiConfig, s.DbChannelData.ChannelId, s.syncedVideos, s.Manager.CliFlags.QuickSync, s.Manager.CliFlags.VideosLimit, ytapi.VideoParams{
-	nextPageToken := ""
+		VideoDir: s.videoDirectory,
-	for {
+		S3Config: *s.Manager.AwsConfigs.GetS3AWSConfig(),
-		req := service.PlaylistItems.List("snippet").
+		Stopper:  s.grp,
-			PlaylistId(playlistID).
+		IPPool:   ipPool,
-			MaxResults(50).
+	}, s.DbChannelData.LastUploadedVideo)
-			PageToken(nextPageToken)
+	if err != nil {
-
+		return err
-		playlistResponse, err := req.Do()
-		if err != nil {
-			return errors.Prefix("error getting playlist items", err)
-		}
-
-		if len(playlistResponse.Items) < 1 {
-			// If there are 50+ videos in a playlist but less than 50 are actually returned by the API, youtube will still redirect
-			// clients to a next page. Such next page will however be empty. This logic prevents ytsync from failing.
-			youtubeIsLying := len(videos) > 0
-			if youtubeIsLying {
-				break
-			}
-			if s.YoutubeChannelID == mostRecentlyFailedChannel {
-				return errors.Err("playlist items not found")
-			}
-			mostRecentlyFailedChannel = s.YoutubeChannelID
-			break //return errors.Err("playlist items not found") //TODO: will this work?
-		}
-		videoIDs := make([]string, 50)
-		for i, item := range playlistResponse.Items {
-			// normally we'd send the video into the channel here, but youtube api doesn't have sorting
-			// so we have to get ALL the videos, then sort them, then send them in
-			playlistMap[item.Snippet.ResourceId.VideoId] = item.Snippet
-			videoIDs[i] = item.Snippet.ResourceId.VideoId
-		}
-		req2 := service.Videos.List("snippet,contentDetails,recordingDetails").Id(strings.Join(videoIDs[:], ","))
-
-		videosListResponse, err := req2.Do()
-		if err != nil {
-			return errors.Prefix("error getting videos info", err)
-		}
-		for _, item := range videosListResponse.Items {
-			videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig(), s.grp, ipPool))
-		}
-
-		log.Infof("Got info for %d videos from youtube API", len(videos))
-
-		nextPageToken = playlistResponse.NextPageToken
-		if nextPageToken == "" || s.Manager.SyncFlags.QuickSync || len(videos) >= s.Manager.videosLimit {
-			break
-		}
 	}
-	for k, v := range s.syncedVideos {
-		if !v.Published {
-			continue
-		}
-		_, ok := playlistMap[k]
-		if !ok {
-			videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig(), s.grp, ipPool))
-		}
-
-	}
-	sort.Sort(byPublishedAt(videos))
 
 Enqueue:
 	for _, v := range videos {
@@ -1116,7 +897,7 @@ Enqueue:
 	return nil
 }
 
-func (s *Sync) processVideo(v video) (err error) {
+func (s *Sync) processVideo(v ytapi.Video) (err error) {
 	defer func() {
 		if p := recover(); p != nil {
 			logUtils.SendErrorToSlack("Video processing panic! %s", debug.Stack())
@@ -1139,20 +920,9 @@ func (s *Sync) processVideo(v video) (err error) {
 	s.syncedVideosMux.RUnlock()
 	newMetadataVersion := int8(2)
 	alreadyPublished := ok && sv.Published
-	videoRequiresUpgrade := ok && s.Manager.SyncFlags.UpgradeMetadata && sv.MetadataVersion < newMetadataVersion
+	videoRequiresUpgrade := ok && s.Manager.CliFlags.UpgradeMetadata && sv.MetadataVersion < newMetadataVersion
 
-	neverRetryFailures := []string{
+	neverRetryFailures := shared.NeverRetryFailures
-		"Error extracting sts from embedded url response",
-		"Unable to extract signature tokens",
-		"the video is too big to sync, skipping for now",
-		"video is too long to process",
-		"This video contains content from",
-		"no compatible format available for this video",
-		"Watch this video on YouTube.",
-		"have blocked it on copyright grounds",
-		"giving up after 0 fragment retries",
-		"Sign in to confirm your age",
-	}
 	if ok && !sv.Published && util.SubstringInSlice(sv.FailureReason, neverRetryFailures) {
 		log.Println(v.ID() + " can't ever be published")
 		return nil
@@ -1167,7 +937,7 @@ func (s *Sync) processVideo(v video) (err error) {
 		return nil
 	}
 
-	if !videoRequiresUpgrade && v.PlaylistPosition() >= s.Manager.videosLimit {
+	if !videoRequiresUpgrade && v.PlaylistPosition() >= s.Manager.CliFlags.VideosLimit {
 		log.Println(v.ID() + " is old: skipping")
 		return nil
 	}
@@ -1180,13 +950,13 @@ func (s *Sync) processVideo(v video) (err error) {
 		return err
 	}
 	sp := sources.SyncParams{
-		ClaimAddress:   s.claimAddress,
+		ClaimAddress:   s.DbChannelData.PublishAddress,
 		Amount:         publishAmount,
-		ChannelID:      s.lbryChannelID,
+		ChannelID:      s.DbChannelData.ChannelClaimID,
-		MaxVideoSize:   s.Manager.maxVideoSize,
+		MaxVideoSize:   s.DbChannelData.SizeLimit,
 		Namer:          s.namer,
-		MaxVideoLength: s.MaxVideoLength,
+		MaxVideoLength: time.Duration(s.DbChannelData.LengthLimit) * time.Minute,
-		Fee:            s.Fee,
+		Fee:            s.DbChannelData.Fee,
 		DefaultAccount: da,
 	}
 
@@ -1196,14 +966,14 @@ func (s *Sync) processVideo(v video) (err error) {
 	}
 
 	s.AppendSyncedVideo(v.ID(), true, "", summary.ClaimName, summary.ClaimID, newMetadataVersion, *v.Size())
-	err = s.Manager.apiConfig.MarkVideoStatus(sdk.VideoStatus{
+	err = s.Manager.ApiConfig.MarkVideoStatus(shared.VideoStatus{
-		ChannelID:       s.YoutubeChannelID,
+		ChannelID:       s.DbChannelData.ChannelId,
 		VideoID:         v.ID(),
-		Status:          VideoStatusPublished,
+		Status:          shared.VideoStatusPublished,
 		ClaimID:         summary.ClaimID,
 		ClaimName:       summary.ClaimName,
 		Size:            v.Size(),
-		MetaDataVersion: LatestMetadataVersion,
+		MetaDataVersion: shared.LatestMetadataVersion,
 		IsTransferred:   util.PtrToBool(s.shouldTransfer()),
 	})
 	if err != nil {
@@ -1214,7 +984,7 @@ func (s *Sync) processVideo(v video) (err error) {
 }
 
 func (s *Sync) importPublicKey() error {
-	if s.publicKey != "" {
+	if s.DbChannelData.PublicKey != "" {
 		accountsResponse, err := s.daemon.AccountList(1, 50)
 		if err != nil {
 			return errors.Err(err)
@@ -1225,13 +995,13 @@ func (s *Sync) importPublicKey() error {
 		}
 		for _, a := range accountsResponse.Items {
 			if *a.Ledger == ledger {
-				if a.PublicKey == s.publicKey {
+				if a.PublicKey == s.DbChannelData.PublicKey {
 					return nil
 				}
 			}
 		}
-		log.Infof("Could not find public key %s in the wallet. Importing it...", s.publicKey)
+		log.Infof("Could not find public key %s in the wallet. Importing it...", s.DbChannelData.PublicKey)
-		_, err = s.daemon.AccountAdd(s.LbryChannelName, nil, nil, &s.publicKey, util.PtrToBool(true), nil)
+		_, err = s.daemon.AccountAdd(s.DbChannelData.DesiredChannelName, nil, nil, &s.DbChannelData.PublicKey, util.PtrToBool(true), nil)
 		return errors.Err(err)
 	}
 	return nil
@@ -1243,7 +1013,7 @@ func (s *Sync) getUnsentSupports() (float64, error) {
 	if err != nil {
 		return 0, errors.Err(err)
 	}
-	if s.transferState == 2 {
+	if s.DbChannelData.TransferState == 2 {
 		balance, err := s.daemon.AccountBalance(&defaultAccount)
 		if err != nil {
 			return 0, err
@@ -1274,8 +1044,8 @@ func (s *Sync) getUnsentSupports() (float64, error) {
 				}
 			}
 		}
-		if balanceAmount > 10 && sentSupports < 1 {
+		if balanceAmount > 10 && sentSupports < 1 && s.DbChannelData.TransferState > 1 {
-			logUtils.SendErrorToSlack("(%s) this channel has quite some LBCs in it (%.2f) and %.2f LBC in sent tips, it's likely that the tips weren't actually sent or the wallet has unnecessary extra credits in it", s.YoutubeChannelID, balanceAmount, sentSupports)
+			logUtils.SendErrorToSlack("(%s) this channel has quite some LBCs in it (%.2f) and %.2f LBC in sent tips, it's likely that the tips weren't actually sent or the wallet has unnecessary extra credits in it", s.DbChannelData.ChannelId, balanceAmount, sentSupports)
 			return balanceAmount - 10, nil
 		}
 	}
@@ -1284,8 +1054,7 @@ func (s *Sync) getUnsentSupports() (float64, error) {
 
 // waitForDaemonProcess observes the running processes and returns when the process is no longer running or when the timeout is up
 func waitForDaemonProcess(timeout time.Duration) error {
-	then := time.Now()
+	stopTime := time.Now().Add(timeout * time.Second)
-	stopTime := then.Add(time.Duration(timeout * time.Second))
 	for !time.Now().After(stopTime) {
 		wait := 10 * time.Second
 		log.Println("the daemon is still running, waiting for it to exit")

sdk/api.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/extras/null"
+	"github.com/lbryio/ytsync/v5/shared"
 
 	"github.com/lbryio/ytsync/v5/util"
 
@@ -30,58 +31,28 @@ type APIConfig struct {
 	HostName      string
 }
 
-type SyncProperties struct {
+func (a *APIConfig) FetchChannels(status string, cliFlags *shared.SyncFlags) ([]shared.YoutubeChannel, error) {
-	SyncFrom         int64
-	SyncUntil        int64
-	YoutubeChannelID string
-}
-
-type SyncFlags struct {
-	StopOnError             bool
-	TakeOverExistingChannel bool
-	SkipSpaceCheck          bool
-	SyncUpdate              bool
-	SingleRun               bool
-	RemoveDBUnpublished     bool
-	UpgradeMetadata         bool
-	DisableTransfers        bool
-	QuickSync               bool
-}
-
-type Fee struct {
-	Amount   string `json:"amount"`
-	Address  string `json:"address"`
-	Currency string `json:"currency"`
-}
-type YoutubeChannel struct {
-	ChannelId          string `json:"channel_id"`
-	TotalVideos        uint   `json:"total_videos"`
-	TotalSubscribers   uint   `json:"total_subscribers"`
-	DesiredChannelName string `json:"desired_channel_name"`
-	Fee                *Fee   `json:"fee"`
-	ChannelClaimID     string `json:"channel_claim_id"`
-	TransferState      int    `json:"transfer_state"`
-	PublishAddress     string `json:"publish_address"`
-	PublicKey          string `json:"public_key"`
-}
-
-func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeChannel, error) {
 	type apiJobsResponse struct {
-		Success bool             `json:"success"`
+		Success bool                    `json:"success"`
-		Error   null.String      `json:"error"`
+		Error   null.String             `json:"error"`
-		Data    []YoutubeChannel `json:"data"`
+		Data    []shared.YoutubeChannel `json:"data"`
 	}
 	endpoint := a.ApiURL + "/yt/jobs"
 	res, err := http.PostForm(endpoint, url.Values{
 		"auth_token":  {a.ApiToken},
 		"sync_status": {status},
 		"min_videos":  {strconv.Itoa(1)},
-		"after":       {strconv.Itoa(int(cp.SyncFrom))},
+		"after":       {strconv.Itoa(int(cliFlags.SyncFrom))},
-		"before":      {strconv.Itoa(int(cp.SyncUntil))},
+		"before":      {strconv.Itoa(int(cliFlags.SyncUntil))},
 		"sync_server": {a.HostName},
-		"channel_id":  {cp.YoutubeChannelID},
+		"channel_id":  {cliFlags.ChannelID},
 	})
 	if err != nil {
+		if strings.Contains(err.Error(), "EOF") {
+			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+			time.Sleep(30 * time.Second)
+			return a.FetchChannels(status, cliFlags)
+		}
 		return nil, errors.Err(err)
 	}
 	defer res.Body.Close()
@@ -90,7 +61,7 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC
 		util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
 		log.Debugln(string(body))
 		time.Sleep(30 * time.Second)
-		return a.FetchChannels(status, cp)
+		return a.FetchChannels(status, cliFlags)
 	}
 	var response apiJobsResponse
 	err = json.Unmarshal(body, &response)
@@ -126,7 +97,6 @@ func sanitizeFailureReason(s *string) {
 }
 
 func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
-
 	type apiSetChannelCertResponse struct {
 		Success bool        `json:"success"`
 		Error   null.String `json:"error"`
@@ -141,6 +111,11 @@ func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
 		"auth_token":       {a.ApiToken},
 	})
 	if err != nil {
+		if strings.Contains(err.Error(), "EOF") {
+			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+			time.Sleep(30 * time.Second)
+			return a.SetChannelCert(certHex, channelID)
+		}
 		return errors.Err(err)
 	}
 	defer res.Body.Close()
@@ -184,6 +159,11 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
 	}
 	res, err := http.PostForm(endpoint, params)
 	if err != nil {
+		if strings.Contains(err.Error(), "EOF") {
+			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+			time.Sleep(30 * time.Second)
+			return a.SetChannelStatus(channelID, status, failureReason, transferState)
+		}
 		return nil, nil, errors.Err(err)
 	}
 	defer res.Body.Close()
@@ -229,6 +209,11 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
 		"channel_claim_id": {channelClaimID},
 	})
 	if err != nil {
+		if strings.Contains(err.Error(), "EOF") {
+			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+			time.Sleep(30 * time.Second)
+			return a.SetChannelClaimID(channelID, channelClaimID)
+		}
 		return errors.Err(err)
 	}
 	defer res.Body.Close()
@@ -268,6 +253,11 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
 	}
 	res, err := http.PostForm(endpoint, vals)
 	if err != nil {
+		if strings.Contains(err.Error(), "EOF") {
+			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+			time.Sleep(30 * time.Second)
+			return a.DeleteVideos(videos)
+		}
 		return errors.Err(err)
 	}
 	defer res.Body.Close()
@@ -297,19 +287,7 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
 	return errors.Err("invalid API response. Status code: %d", res.StatusCode)
 }
 
-type VideoStatus struct {
+func (a *APIConfig) MarkVideoStatus(status shared.VideoStatus) error {
-	ChannelID       string
-	VideoID         string
-	Status          string
-	ClaimID         string
-	ClaimName       string
-	FailureReason   string
-	Size            *int64
-	MetaDataVersion uint
-	IsTransferred   *bool
-}
-
-func (a *APIConfig) MarkVideoStatus(status VideoStatus) error {
 	endpoint := a.ApiURL + "/yt/video_status"
 
 	sanitizeFailureReason(&status.FailureReason)
@@ -341,6 +319,11 @@ func (a *APIConfig) MarkVideoStatus(status VideoStatus) error {
 	}
 	res, err := http.PostForm(endpoint, vals)
 	if err != nil {
+		if strings.Contains(err.Error(), "EOF") {
+			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+			time.Sleep(30 * time.Second)
+			return a.MarkVideoStatus(status)
+		}
 		return errors.Err(err)
 	}
 	defer res.Body.Close()
@@ -368,3 +351,102 @@ func (a *APIConfig) MarkVideoStatus(status VideoStatus) error {
 	}
 	return errors.Err("invalid API response. Status code: %d", res.StatusCode)
 }
+
+func (a *APIConfig) VideoState(videoID string) (string, error) {
+	endpoint := a.ApiURL + "/yt/video_state"
+	vals := url.Values{
+		"video_id":   {videoID},
+		"auth_token": {a.ApiToken},
+	}
+
+	res, err := http.PostForm(endpoint, vals)
+	if err != nil {
+		if strings.Contains(err.Error(), "EOF") {
+			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+			time.Sleep(30 * time.Second)
+			return a.VideoState(videoID)
+		}
+		return "", errors.Err(err)
+	}
+	defer res.Body.Close()
+	body, _ := ioutil.ReadAll(res.Body)
+	if res.StatusCode == http.StatusNotFound {
+		return "not_found", nil
+	}
+	if res.StatusCode != http.StatusOK {
+		util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
+		log.Debugln(string(body))
+		time.Sleep(30 * time.Second)
+		return a.VideoState(videoID)
+	}
+	var response struct {
+		Success bool        `json:"success"`
+		Error   null.String `json:"error"`
+		Data    null.String `json:"data"`
+	}
+	err = json.Unmarshal(body, &response)
+	if err != nil {
+		return "", errors.Err(err)
+	}
+	if !response.Error.IsNull() {
+		return "", errors.Err(response.Error.String)
+	}
+	if !response.Data.IsNull() {
+		return response.Data.String, nil
+	}
+	return "", errors.Err("invalid API response. Status code: %d", res.StatusCode)
+}
+
+type VideoRelease struct {
+	ID            uint64 `json:"id"`
+	YoutubeDataID uint64 `json:"youtube_data_id"`
+	VideoID       string `json:"video_id"`
+	ReleaseTime   string `json:"release_time"`
+	CreatedAt     string `json:"created_at"`
+	UpdatedAt     string `json:"updated_at"`
+}
+
+func (a *APIConfig) GetReleasedDate(videoID string) (*VideoRelease, error) {
+	endpoint := a.ApiURL + "/yt/released"
+	vals := url.Values{
+		"video_id":   {videoID},
+		"auth_token": {a.ApiToken},
+	}
+
+	res, err := http.PostForm(endpoint, vals)
+	if err != nil {
+		if strings.Contains(err.Error(), "EOF") {
+			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+			time.Sleep(30 * time.Second)
+			return a.GetReleasedDate(videoID)
+		}
+		return nil, errors.Err(err)
+	}
+	defer res.Body.Close()
+	body, _ := ioutil.ReadAll(res.Body)
+	if res.StatusCode == http.StatusNotFound {
+		return nil, nil
+	}
+	if res.StatusCode != http.StatusOK {
+		util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
+		log.Debugln(string(body))
+		time.Sleep(30 * time.Second)
+		return a.GetReleasedDate(videoID)
+	}
+	var response struct {
+		Success bool         `json:"success"`
+		Error   null.String  `json:"error"`
+		Data    VideoRelease `json:"data"`
+	}
+	err = json.Unmarshal(body, &response)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	if !response.Error.IsNull() {
+		return nil, errors.Err(response.Error.String)
+	}
+	if response.Data.ReleaseTime != "" {
+		return &response.Data, nil
+	}
+	return nil, errors.Err("invalid API response. Status code: %d", res.StatusCode)
+}

shared/shared.go

@@ -0,0 +1,125 @@
+package shared
+
+import (
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+)
+
+type Fee struct {
+	Amount   string `json:"amount"`
+	Address  string `json:"address"`
+	Currency string `json:"currency"`
+}
+type YoutubeChannel struct {
+	ChannelId          string `json:"channel_id"`
+	TotalVideos        uint   `json:"total_videos"`
+	TotalSubscribers   uint   `json:"total_subscribers"`
+	DesiredChannelName string `json:"desired_channel_name"`
+	Fee                *Fee   `json:"fee"`
+	ChannelClaimID     string `json:"channel_claim_id"`
+	TransferState      int    `json:"transfer_state"`
+	PublishAddress     string `json:"publish_address"`
+	PublicKey          string `json:"public_key"`
+	LengthLimit        int    `json:"length_limit"`
+	SizeLimit          int    `json:"size_limit"`
+	LastUploadedVideo  string `json:"last_uploaded_video"`
+}
+
+var NeverRetryFailures = []string{
+	"Error extracting sts from embedded url response",
+	"Unable to extract signature tokens",
+	"the video is too big to sync, skipping for now",
+	"video is too long to process",
+	"This video contains content from",
+	"no compatible format available for this video",
+	"Watch this video on YouTube.",
+	"have blocked it on copyright grounds",
+	"giving up after 0 fragment retries",
+	"Sign in to confirm your age",
+}
+
+type SyncFlags struct {
+	StopOnError             bool
+	TakeOverExistingChannel bool
+	SkipSpaceCheck          bool
+	SyncUpdate              bool
+	SingleRun               bool
+	RemoveDBUnpublished     bool
+	UpgradeMetadata         bool
+	DisableTransfers        bool
+	QuickSync               bool
+	MaxTries                int
+	Refill                  int
+	Limit                   int
+	SyncStatus              string
+	ChannelID               string
+	SyncFrom                int64
+	SyncUntil               int64
+	ConcurrentJobs          int
+	VideosLimit             int
+	MaxVideoSize            int
+	MaxVideoLength          time.Duration
+}
+
+func (f *SyncFlags) IsSingleChannelSync() bool {
+	return f.ChannelID != ""
+}
+
+type VideoStatus struct {
+	ChannelID       string
+	VideoID         string
+	Status          string
+	ClaimID         string
+	ClaimName       string
+	FailureReason   string
+	Size            *int64
+	MetaDataVersion uint
+	IsTransferred   *bool
+}
+
+const (
+	StatusPending        = "pending"        // waiting for permission to sync
+	StatusPendingEmail   = "pendingemail"   // permission granted but missing email
+	StatusQueued         = "queued"         // in sync queue. will be synced soon
+	StatusPendingUpgrade = "pendingupgrade" // in sync queue. will be synced soon
+	StatusSyncing        = "syncing"        // syncing now
+	StatusSynced         = "synced"         // done
+	StatusFailed         = "failed"
+	StatusFinalized      = "finalized" // no more changes allowed
+	StatusAbandoned      = "abandoned" // deleted on youtube or banned
+)
+
+var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned}
+
+const LatestMetadataVersion = 2
+
+const (
+	VideoStatusPublished     = "published"
+	VideoStatusFailed        = "failed"
+	VideoStatusUpgradeFailed = "upgradefailed"
+	VideoStatusUnpublished   = "unpublished"
+	VideoStatusTranferFailed = "transferfailed"
+)
+
+const (
+	TransferStateNotTouched = iota
+	TransferStatePending
+	TransferStateComplete
+	TransferStateManual
+)
+
+type AwsConfigs struct {
+	AwsS3ID     string
+	AwsS3Secret string
+	AwsS3Region string
+	AwsS3Bucket string
+}
+
+func (a *AwsConfigs) GetS3AWSConfig() *aws.Config {
+	return &aws.Config{
+		Credentials: credentials.NewStaticCredentials(a.AwsS3ID, a.AwsS3Secret, ""),
+		Region:      &a.AwsS3Region,
+	}
+}

sources/youtubeVideo.go

@@ -3,7 +3,6 @@ package sources
 import (
 	"fmt"
 	"io/ioutil"
-	"math"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -13,24 +12,25 @@ import (
 	"sync"
 	"time"
 
-	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/ytsync/v5/downloader/ytdl"
-	"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
+	"github.com/lbryio/ytsync/v5/shared"
-	"github.com/lbryio/lbry.go/v2/extras/stop"
-	"github.com/lbryio/lbry.go/v2/extras/util"
-	"github.com/lbryio/ytsync/v5/timing"
-	logUtils "github.com/lbryio/ytsync/v5/util"
 
 	"github.com/lbryio/ytsync/v5/ip_manager"
 	"github.com/lbryio/ytsync/v5/namer"
 	"github.com/lbryio/ytsync/v5/sdk"
 	"github.com/lbryio/ytsync/v5/tags_manager"
 	"github.com/lbryio/ytsync/v5/thumbs"
+	"github.com/lbryio/ytsync/v5/timing"
+	logUtils "github.com/lbryio/ytsync/v5/util"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
+	"github.com/lbryio/lbry.go/v2/extras/stop"
+	"github.com/lbryio/lbry.go/v2/extras/util"
 
-	duration "github.com/ChannelMeter/iso8601duration"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/shopspring/decimal"
 	log "github.com/sirupsen/logrus"
-	"google.golang.org/api/youtube/v3"
 )
 
 type YoutubeVideo struct {
@@ -40,10 +40,10 @@ type YoutubeVideo struct {
 	playlistPosition int64
 	size             *int64
 	maxVideoSize     int64
-	maxVideoLength   float64
+	maxVideoLength   time.Duration
 	publishedAt      time.Time
 	dir              string
-	youtubeInfo      *youtube.Video
+	youtubeInfo      *ytdl.YtdlVideo
 	youtubeChannelID string
 	tags             []string
 	awsConfig        aws.Config
@@ -90,22 +90,23 @@ var youtubeCategories = map[string]string{
 	"44": "trailers",
 }
 
-func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
+func NewYoutubeVideo(directory string, videoData *ytdl.YtdlVideo, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) (*YoutubeVideo, error) {
-	publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
+	// youtube-dl returns times in local timezone sometimes. this could break in the future
+	// maybe we can file a PR to choose the timezone we want from youtube-dl
 	return &YoutubeVideo{
-		id:               videoData.Id,
+		id:               videoData.ID,
-		title:            videoData.Snippet.Title,
+		title:            videoData.Title,
-		description:      videoData.Snippet.Description,
+		description:      videoData.Description,
 		playlistPosition: playlistPosition,
-		publishedAt:      publishedAt,
+		publishedAt:      videoData.UploadDateForReal,
 		dir:              directory,
 		youtubeInfo:      videoData,
 		awsConfig:        awsConfig,
 		mocked:           false,
-		youtubeChannelID: videoData.Snippet.ChannelId,
+		youtubeChannelID: videoData.ChannelID,
 		stopGroup:        stopGroup,
 		pool:             pool,
-	}
+	}, nil
 }
 func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
 	return &YoutubeVideo{
@@ -185,7 +186,7 @@ func (v *YoutubeVideo) download() error {
 	defer func(start time.Time) {
 		timing.TimedComponent("download").Add(time.Since(start))
 	}(start)
-	if v.youtubeInfo.Snippet.LiveBroadcastContent != "none" {
+	if v.youtubeInfo.IsLive != nil {
 		return errors.Err("video is a live stream and hasn't completed yet")
 	}
 	videoPath := v.getFullPath()
@@ -208,6 +209,15 @@ func (v *YoutubeVideo) download() error {
 		"480",
 		"320",
 	}
+	dur := time.Duration(v.youtubeInfo.Duration) * time.Second
+	if dur.Hours() > 2 { //for videos longer than 2 hours only sync up to 720p
+		qualities = []string{
+			"720",
+			"480",
+			"320",
+		}
+	}
+
 	ytdlArgs := []string{
 		"--no-progress",
 		"-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"),
@@ -218,12 +228,11 @@ func (v *YoutubeVideo) download() error {
 		"-movflags faststart",
 		"--abort-on-unavailable-fragment",
 		"--fragment-retries",
-		"0",
+		"1",
-		"--user-agent",
-		"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36",
 		"--cookies",
 		"cookies.txt",
 	}
+	userAgent := []string{"--user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"}
 	if v.maxVideoSize > 0 {
 		ytdlArgs = append(ytdlArgs,
 			"--max-filesize",
@@ -233,7 +242,7 @@ func (v *YoutubeVideo) download() error {
 	if v.maxVideoLength > 0 {
 		ytdlArgs = append(ytdlArgs,
 			"--match-filter",
-			fmt.Sprintf("duration <= %d", int(math.Round(v.maxVideoLength*3600))),
+			fmt.Sprintf("duration <= %d", int(v.maxVideoLength.Seconds())),
 		)
 	}
 
@@ -263,8 +272,10 @@ func (v *YoutubeVideo) download() error {
 		"https://www.youtube.com/watch?v="+v.ID(),
 	)
 
-	for i, quality := range qualities {
+	for i := 0; i < len(qualities); i++ {
+		quality := qualities[i]
 		argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+quality+"]+bestaudio[ext!=webm]")
+		argsWithFilters = append(argsWithFilters, userAgent...)
 		cmd := exec.Command("youtube-dl", argsWithFilters...)
 		log.Printf("Running command youtube-dl %s", strings.Join(argsWithFilters, " "))
 
@@ -293,6 +304,11 @@ func (v *YoutubeVideo) download() error {
 						return errors.Err(string(errorLog))
 					}
 					continue //this bypasses the yt throttling IP redistribution... TODO: don't
+				} else if strings.Contains(string(errorLog), "YouTube said: Unable to extract video data") && !strings.Contains(userAgent[1], "Googlebot") {
+					i-- //do not lower quality when trying a different user agent
+					userAgent = []string{"--user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}
+					log.Infof("trying different user agent for video %s", v.ID())
+					continue
 				}
 				return errors.Err(string(errorLog))
 			}
@@ -370,8 +386,8 @@ func (v *YoutubeVideo) delete(reason string) error {
 }
 
 func (v *YoutubeVideo) triggerThumbnailSave() (err error) {
-	thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails)
+	thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails)
-	v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig)
+	v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig)
 	return err
 }
 
@@ -429,8 +445,8 @@ type SyncParams struct {
 	ChannelID      string
 	MaxVideoSize   int
 	Namer          *namer.Namer
-	MaxVideoLength float64
+	MaxVideoLength time.Duration
-	Fee            *sdk.Fee
+	Fee            *shared.Fee
 	DefaultAccount string
 }
 
@@ -448,13 +464,11 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
 
 func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
 	var err error
-	videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
+
-	if err != nil {
+	dur := time.Duration(v.youtubeInfo.Duration) * time.Second
-		return nil, errors.Err(err)
+
-	}
+	if dur > v.maxVideoLength {
-	if videoDuration.ToDuration() > time.Duration(v.maxVideoLength*60)*time.Minute {
+		logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, dur.String(), v.maxVideoLength.String())
-		log.Infof("%s is %s long and the limit is %s", v.id, videoDuration.ToDuration().String(), (time.Duration(v.maxVideoLength*60) * time.Minute).String())
-		logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, videoDuration.ToDuration().String(), (time.Duration(v.maxVideoLength*60) * time.Minute).String())
 		return nil, errors.Err("video is too long to process")
 	}
 	for {
@@ -487,27 +501,30 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
 	locations = nil
 	tags = nil
 	if !v.mocked {
-		if v.youtubeInfo.Snippet.DefaultLanguage != "" {
+		/*
-			if v.youtubeInfo.Snippet.DefaultLanguage == "iw" {
+			if v.youtubeInfo.Snippet.DefaultLanguage != "" {
-				v.youtubeInfo.Snippet.DefaultLanguage = "he"
+				if v.youtubeInfo.Snippet.DefaultLanguage == "iw" {
-			}
+					v.youtubeInfo.Snippet.DefaultLanguage = "he"
-			languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
+				}
-		}
+				languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
+			}*/
 
-		if v.youtubeInfo.RecordingDetails != nil && v.youtubeInfo.RecordingDetails.Location != nil {
+		/*if v.youtubeInfo.!= nil && v.youtubeInfo.RecordingDetails.Location != nil {
 			locations = []jsonrpc.Location{{
 				Latitude:  util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Latitude)),
 				Longitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Longitude)),
 			}}
-		}
+		}*/
-		tags = v.youtubeInfo.Snippet.Tags
+		tags = v.youtubeInfo.Tags
 	}
 	tags, err := tags_manager.SanitizeTags(tags, v.youtubeChannelID)
 	if err != nil {
 		log.Errorln(err.Error())
 	}
 	if !v.mocked {
-		tags = append(tags, youtubeCategories[v.youtubeInfo.Snippet.CategoryId])
+		for _, category := range v.youtubeInfo.Categories {
+			tags = append(tags, youtubeCategories[category])
+		}
 	}
 
 	return languages, locations, tags
@@ -532,8 +549,8 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
 		if v.mocked {
 			return nil, errors.Err("could not find thumbnail for mocked video")
 		}
-		thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails)
+		thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails)
-		thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig)
+		thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig)
 	} else {
 		thumbnailURL = thumbs.ThumbnailEndpoint + v.ID()
 	}
@@ -605,14 +622,9 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
 		}, nil
 	}
 
-	videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
-	if err != nil {
-		return nil, errors.Err(err)
-	}
-
 	streamCreateOptions.ClaimCreateOptions.Title = &v.title
 	streamCreateOptions.ClaimCreateOptions.Description = util.PtrToString(v.getAbbrevDescription())
-	streamCreateOptions.Duration = util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds())))
+	streamCreateOptions.Duration = util.PtrToUint64(uint64(v.youtubeInfo.Duration))
 	streamCreateOptions.ReleaseTime = util.PtrToInt64(v.publishedAt.Unix())
 	start := time.Now()
 	pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{

thumbs/uploader.go

@@ -4,6 +4,9 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"strings"
+
+	"github.com/lbryio/ytsync/v5/downloader/ytdl"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 
@@ -11,7 +14,6 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	log "github.com/sirupsen/logrus"
-	"google.golang.org/api/youtube/v3"
 )
 
 type thumbnailUploader struct {
@@ -31,7 +33,9 @@ func (u *thumbnailUploader) downloadThumbnail() error {
 		return errors.Err(err)
 	}
 	defer img.Close()
-
+	if strings.HasPrefix(u.originalUrl, "//") {
+		u.originalUrl = "https:" + u.originalUrl
+	}
 	resp, err := http.Get(u.originalUrl)
 	if err != nil {
 		return errors.Err(err)
@@ -98,15 +102,14 @@ func MirrorThumbnail(url string, name string, s3Config aws.Config) (string, erro
 	return tu.mirroredUrl, nil
 }
 
-func GetBestThumbnail(thumbnails *youtube.ThumbnailDetails) *youtube.Thumbnail {
+func GetBestThumbnail(thumbnails []ytdl.Thumbnail) *ytdl.Thumbnail {
-	if thumbnails.Maxres != nil {
+	var bestWidth *ytdl.Thumbnail
-		return thumbnails.Maxres
+	for _, thumbnail := range thumbnails {
-	} else if thumbnails.High != nil {
+		if bestWidth == nil {
-		return thumbnails.High
+			bestWidth = &thumbnail
-	} else if thumbnails.Medium != nil {
+		} else if bestWidth.Width < thumbnail.Width {
-		return thumbnails.Medium
+			bestWidth = &thumbnail
-	} else if thumbnails.Standard != nil {
+		}
-		return thumbnails.Standard
 	}
-	return thumbnails.Default
+	return bestWidth
 }

util/util.go

@@ -52,7 +52,7 @@ func GetLBRYNetDir() string {
 }
 
 func GetLbryumDir() string {
-	lbryumDir := os.Getenv("LBRYNET_WALLETS_DIR")
+	lbryumDir := os.Getenv("LBRYUM_DIR")
 	if lbryumDir == "" {
 		usr, err := user.Current()
 		if err != nil {
@@ -255,7 +255,7 @@ func CleanupLbrynet() error {
 		}
 		return errors.Err(err)
 	}
-	dbSizeLimit := int64(1 * 1024 * 1024 * 1024)
+	dbSizeLimit := int64(2 * 1024 * 1024 * 1024)
 	if db.Size() > dbSizeLimit {
 		files, err := filepath.Glob(lbryumDir + "/blockchain.db*")
 		if err != nil {
@@ -366,9 +366,31 @@ func GetDefaultWalletPath() string {
 		defaultWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
 	}
 
-	walletPath := os.Getenv("LBRYNET_WALLETS_DIR")
+	walletPath := os.Getenv("LBRYUM_DIR")
 	if walletPath != "" {
 		defaultWalletDir = walletPath + "/wallets/default_wallet"
 	}
 	return defaultWalletDir
 }
+func GetBlockchainDBPath() string {
+	lbryumDir := os.Getenv("LBRYUM_DIR")
+	if lbryumDir == "" {
+		if IsRegTest() {
+			lbryumDir = os.Getenv("HOME") + "/.lbryum_regtest"
+		} else {
+			lbryumDir = os.Getenv("HOME") + "/.lbryum"
+		}
+	}
+	defaultDB := lbryumDir + "/lbc_mainnet/blockchain.db"
+	if IsRegTest() {
+		defaultDB = lbryumDir + "/lbc_regtest/blockchain.db"
+	}
+	return defaultDB
+}
+func GetBlockchainDirectoryName() string {
+	ledger := "lbc_mainnet"
+	if IsRegTest() {
+		ledger = "lbc_regtest"
+	}
+	return ledger
+}

ytapi/ytapi.go

@@ -0,0 +1,387 @@
+package ytapi
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/lbryio/ytsync/v5/shared"
+	logUtils "github.com/lbryio/ytsync/v5/util"
+
+	"github.com/lbryio/ytsync/v5/downloader/ytdl"
+
+	"github.com/lbryio/ytsync/v5/downloader"
+	"github.com/lbryio/ytsync/v5/ip_manager"
+	"github.com/lbryio/ytsync/v5/sdk"
+	"github.com/lbryio/ytsync/v5/sources"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
+	"github.com/lbryio/lbry.go/v2/extras/stop"
+	"github.com/lbryio/lbry.go/v2/extras/util"
+
+	"github.com/aws/aws-sdk-go/aws"
+	log "github.com/sirupsen/logrus"
+)
+
+type Video interface {
+	Size() *int64
+	ID() string
+	IDAndNum() string
+	PlaylistPosition() int
+	PublishedAt() time.Time
+	Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool, *sync.RWMutex) (*sources.SyncSummary, error)
+}
+
+type byPublishedAt []Video
+
+func (a byPublishedAt) Len() int           { return len(a) }
+func (a byPublishedAt) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a byPublishedAt) Less(i, j int) bool { return a[i].PublishedAt().Before(a[j].PublishedAt()) }
+
+type VideoParams struct {
+	VideoDir string
+	S3Config aws.Config
+	Stopper  *stop.Group
+	IPPool   *ip_manager.IPPool
+}
+
+var mostRecentlyFailedChannel string // TODO: fix this hack!
+
+func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams, lastUploadedVideo string) ([]Video, error) {
+	var videos []Video
+	if quickSync && maxVideos > 50 {
+		maxVideos = 50
+	}
+	allVideos, err := downloader.GetPlaylistVideoIDs(channelID, maxVideos, videoParams.Stopper.Ch(), videoParams.IPPool)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	videoIDs := make([]string, 0, len(allVideos))
+	for _, video := range allVideos {
+		sv, ok := syncedVideos[video]
+		if ok && util.SubstringInSlice(sv.FailureReason, shared.NeverRetryFailures) {
+			continue
+		}
+		videoIDs = append(videoIDs, video)
+	}
+	log.Infof("Got info for %d videos from youtube downloader", len(videoIDs))
+
+	playlistMap := make(map[string]int64)
+	for i, videoID := range videoIDs {
+		playlistMap[videoID] = int64(i)
+	}
+	//this will ensure that we at least try to sync the video that was marked as last uploaded video in the database.
+	if lastUploadedVideo != "" {
+		_, ok := playlistMap[lastUploadedVideo]
+		if !ok {
+			playlistMap[lastUploadedVideo] = 0
+			videoIDs = append(videoIDs, lastUploadedVideo)
+		}
+	}
+
+	if len(videoIDs) < 1 {
+		if channelID == mostRecentlyFailedChannel {
+			return nil, errors.Err("playlist items not found")
+		}
+		mostRecentlyFailedChannel = channelID
+	}
+
+	vids, err := getVideos(config, channelID, videoIDs, videoParams.Stopper.Ch(), videoParams.IPPool)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, item := range vids {
+		positionInList := playlistMap[item.ID]
+		videoToAdd, err := sources.NewYoutubeVideo(videoParams.VideoDir, item, positionInList, videoParams.S3Config, videoParams.Stopper, videoParams.IPPool)
+		if err != nil {
+			return nil, errors.Err(err)
+		}
+		videos = append(videos, videoToAdd)
+	}
+
+	for k, v := range syncedVideos {
+		if !v.Published {
+			continue
+		}
+		if _, ok := playlistMap[k]; !ok {
+			videos = append(videos, sources.NewMockedVideo(videoParams.VideoDir, k, channelID, videoParams.S3Config, videoParams.Stopper, videoParams.IPPool))
+		}
+	}
+
+	sort.Sort(byPublishedAt(videos))
+
+	return videos, nil
+}
+
+// CountVideosInChannel is unused for now... keeping it here just in case
+func CountVideosInChannel(channelID string) (int, error) {
+	url := "https://socialblade.com/youtube/channel/" + channelID
+
+	req, _ := http.NewRequest("GET", url, nil)
+
+	req.Header.Add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36")
+	req.Header.Add("Accept", "*/*")
+	req.Header.Add("Host", "socialblade.com")
+
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return 0, errors.Err(err)
+	}
+	defer res.Body.Close()
+
+	var line string
+	scanner := bufio.NewScanner(res.Body)
+	for scanner.Scan() {
+		if strings.Contains(scanner.Text(), "youtube-stats-header-uploads") {
+			line = scanner.Text()
+			break
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return 0, err
+	}
+	if line == "" {
+		return 0, errors.Err("upload count line not found")
+	}
+
+	matches := regexp.MustCompile(">([0-9]+)<").FindStringSubmatch(line)
+	if len(matches) != 2 {
+		return 0, errors.Err("upload count not found with regex")
+	}
+
+	num, err := strconv.Atoi(matches[1])
+	if err != nil {
+		return 0, errors.Err(err)
+	}
+
+	return num, nil
+}
+
+func ChannelInfo(channelID string) (*YoutubeStatsResponse, error) {
+	url := "https://www.youtube.com/channel/" + channelID + "/about"
+
+	req, _ := http.NewRequest("GET", url, nil)
+
+	req.Header.Add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36")
+	req.Header.Add("Accept", "*/*")
+
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	defer res.Body.Close()
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	pageBody := string(body)
+	dataStartIndex := strings.Index(pageBody, "window[\"ytInitialData\"] = ") + 26
+	dataEndIndex := strings.Index(pageBody, "]}}};") + 4
+
+	data := pageBody[dataStartIndex:dataEndIndex]
+	var decodedResponse YoutubeStatsResponse
+	err = json.Unmarshal([]byte(data), &decodedResponse)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+
+	return &decodedResponse, nil
+}
+
+func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool) ([]*ytdl.YtdlVideo, error) {
+	var videos []*ytdl.YtdlVideo
+	for _, videoID := range videoIDs {
+		if len(videoID) < 5 {
+			continue
+		}
+		select {
+		case <-stopChan:
+			return videos, errors.Err("interrupted by user")
+		default:
+		}
+
+		//ip, err := ipPool.GetIP(videoID)
+		//if err != nil {
+		//	return nil, err
+		//}
+		//video, err := downloader.GetVideoInformation(videoID, &net.TCPAddr{IP: net.ParseIP(ip)})
+		state, err := config.VideoState(videoID)
+		if err != nil {
+			return nil, errors.Err(err)
+		}
+		if state == "published" {
+			continue
+		}
+		video, err := downloader.GetVideoInformation(config, videoID, stopChan, nil, ipPool)
+		if err != nil {
+			errSDK := config.MarkVideoStatus(shared.VideoStatus{
+				ChannelID:     channelID,
+				VideoID:       videoID,
+				Status:        "failed",
+				FailureReason: err.Error(),
+			})
+			logUtils.SendErrorToSlack(fmt.Sprintf("Skipping video (%s): %s", videoID, errors.FullTrace(err)))
+			if errSDK != nil {
+				return nil, errors.Err(errSDK)
+			}
+		} else {
+			videos = append(videos, video)
+		}
+	}
+	return videos, nil
+}
+
+type YoutubeStatsResponse struct {
+	Contents struct {
+		TwoColumnBrowseResultsRenderer struct {
+			Tabs []struct {
+				TabRenderer struct {
+					Title    string `json:"title"`
+					Selected bool   `json:"selected"`
+					Content  struct {
+						SectionListRenderer struct {
+							Contents []struct {
+								ItemSectionRenderer struct {
+									Contents []struct {
+										ChannelAboutFullMetadataRenderer struct {
+											Description struct {
+												SimpleText string `json:"simpleText"`
+											} `json:"description"`
+											ViewCountText struct {
+												SimpleText string `json:"simpleText"`
+											} `json:"viewCountText"`
+											JoinedDateText struct {
+												Runs []struct {
+													Text string `json:"text"`
+												} `json:"runs"`
+											} `json:"joinedDateText"`
+											CanonicalChannelURL        string `json:"canonicalChannelUrl"`
+											BypassBusinessEmailCaptcha bool   `json:"bypassBusinessEmailCaptcha"`
+											Title                      struct {
+												SimpleText string `json:"simpleText"`
+											} `json:"title"`
+											Avatar struct {
+												Thumbnails []struct {
+													URL    string `json:"url"`
+													Width  int    `json:"width"`
+													Height int    `json:"height"`
+												} `json:"thumbnails"`
+											} `json:"avatar"`
+											ShowDescription  bool `json:"showDescription"`
+											DescriptionLabel struct {
+												Runs []struct {
+													Text string `json:"text"`
+												} `json:"runs"`
+											} `json:"descriptionLabel"`
+											DetailsLabel struct {
+												Runs []struct {
+													Text string `json:"text"`
+												} `json:"runs"`
+											} `json:"detailsLabel"`
+											ChannelID string `json:"channelId"`
+										} `json:"channelAboutFullMetadataRenderer"`
+									} `json:"contents"`
+								} `json:"itemSectionRenderer"`
+							} `json:"contents"`
+						} `json:"sectionListRenderer"`
+					} `json:"content"`
+				} `json:"tabRenderer"`
+			} `json:"tabs"`
+		} `json:"twoColumnBrowseResultsRenderer"`
+	} `json:"contents"`
+	Header struct {
+		C4TabbedHeaderRenderer struct {
+			ChannelID string `json:"channelId"`
+			Title     string `json:"title"`
+			Avatar    struct {
+				Thumbnails []struct {
+					URL    string `json:"url"`
+					Width  int    `json:"width"`
+					Height int    `json:"height"`
+				} `json:"thumbnails"`
+			} `json:"avatar"`
+			Banner struct {
+				Thumbnails []struct {
+					URL    string `json:"url"`
+					Width  int    `json:"width"`
+					Height int    `json:"height"`
+				} `json:"thumbnails"`
+			} `json:"banner"`
+			VisitTracking struct {
+				RemarketingPing string `json:"remarketingPing"`
+			} `json:"visitTracking"`
+			SubscriberCountText struct {
+				SimpleText string `json:"simpleText"`
+			} `json:"subscriberCountText"`
+		} `json:"c4TabbedHeaderRenderer"`
+	} `json:"header"`
+	Metadata struct {
+		ChannelMetadataRenderer struct {
+			Title                string   `json:"title"`
+			Description          string   `json:"description"`
+			RssURL               string   `json:"rssUrl"`
+			ChannelConversionURL string   `json:"channelConversionUrl"`
+			ExternalID           string   `json:"externalId"`
+			Keywords             string   `json:"keywords"`
+			OwnerUrls            []string `json:"ownerUrls"`
+			Avatar               struct {
+				Thumbnails []struct {
+					URL    string `json:"url"`
+					Width  int    `json:"width"`
+					Height int    `json:"height"`
+				} `json:"thumbnails"`
+			} `json:"avatar"`
+			ChannelURL       string `json:"channelUrl"`
+			IsFamilySafe     bool   `json:"isFamilySafe"`
+			VanityChannelURL string `json:"vanityChannelUrl"`
+		} `json:"channelMetadataRenderer"`
+	} `json:"metadata"`
+	Topbar struct {
+		DesktopTopbarRenderer struct {
+			CountryCode string `json:"countryCode"`
+		} `json:"desktopTopbarRenderer"`
+	} `json:"topbar"`
+	Microformat struct {
+		MicroformatDataRenderer struct {
+			URLCanonical string `json:"urlCanonical"`
+			Title        string `json:"title"`
+			Description  string `json:"description"`
+			Thumbnail    struct {
+				Thumbnails []struct {
+					URL    string `json:"url"`
+					Width  int    `json:"width"`
+					Height int    `json:"height"`
+				} `json:"thumbnails"`
+			} `json:"thumbnail"`
+			SiteName           string   `json:"siteName"`
+			AppName            string   `json:"appName"`
+			AndroidPackage     string   `json:"androidPackage"`
+			IosAppStoreID      string   `json:"iosAppStoreId"`
+			IosAppArguments    string   `json:"iosAppArguments"`
+			OgType             string   `json:"ogType"`
+			URLApplinksWeb     string   `json:"urlApplinksWeb"`
+			URLApplinksIos     string   `json:"urlApplinksIos"`
+			URLApplinksAndroid string   `json:"urlApplinksAndroid"`
+			URLTwitterIos      string   `json:"urlTwitterIos"`
+			URLTwitterAndroid  string   `json:"urlTwitterAndroid"`
+			TwitterCardType    string   `json:"twitterCardType"`
+			TwitterSiteHandle  string   `json:"twitterSiteHandle"`
+			SchemaDotOrgType   string   `json:"schemaDotOrgType"`
+			Noindex            bool     `json:"noindex"`
+			Unlisted           bool     `json:"unlisted"`
+			FamilySafe         bool     `json:"familySafe"`
+			Tags               []string `json:"tags"`
+		} `json:"microformatDataRenderer"`
+	} `json:"microformat"`
+}

ytapi/ytapi_test.go

@@ -0,0 +1,12 @@
+package ytapi
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestChannelInfo(t *testing.T) {
+	_, _, err := ChannelInfo("", "UCNQfQvFMPnInwsU_iGYArJQ")
+	assert.NoError(t, err)
+}