diff --git a/Makefile b/Makefile
index e41cb2e..870e2b0 100644
--- a/Makefile
+++ b/Makefile
@@ -13,7 +13,7 @@ LDFLAGS = -ldflags "-X ${IMPORT_PATH}/meta.Version=${VERSION} -X ${IMPORT_PATH}/
build:
- mkdir -p ${BIN_DIR} && CGO_ENABLED=0 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
+ mkdir -p ${BIN_DIR} && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
clean:
if [ -f ${BIN_DIR}/${BINARY} ]; then rm ${BIN_DIR}/${BINARY}; fi
diff --git a/blobs_reflector/reflect.go b/blobs_reflector/reflect.go
index 7758c24..81e3cd2 100644
--- a/blobs_reflector/reflect.go
+++ b/blobs_reflector/reflect.go
@@ -72,12 +72,12 @@ func reflectBlobs() error {
return errors.Err(err)
}
}
- st := store.NewDBBackedS3Store(
+ st := store.NewDBBackedStore(
store.NewS3BlobStore(config.AwsID, config.AwsSecret, config.BucketRegion, config.BucketName),
dbHandle)
uploadWorkers := 10
- uploader := reflector.NewUploader(dbHandle, st, uploadWorkers, false)
+ uploader := reflector.NewUploader(dbHandle, st, uploadWorkers, false, false)
usr, err := user.Current()
if err != nil {
return errors.Err(err)
diff --git a/downloader/downloader.go b/downloader/downloader.go
new file mode 100644
index 0000000..404bab6
--- /dev/null
+++ b/downloader/downloader.go
@@ -0,0 +1,364 @@
+package downloader
+
+import (
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "net"
+ "net/http"
+ "net/url"
+ "os/exec"
+ "strings"
+ "time"
+
+ "github.com/davecgh/go-spew/spew"
+ "github.com/lbryio/ytsync/v5/downloader/ytdl"
+ "github.com/lbryio/ytsync/v5/ip_manager"
+ "github.com/lbryio/ytsync/v5/sdk"
+
+ "github.com/lbryio/lbry.go/v2/extras/errors"
+ "github.com/lbryio/lbry.go/v2/extras/stop"
+ "github.com/lbryio/lbry.go/v2/extras/util"
+
+ "github.com/sirupsen/logrus"
+)
+
+func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
+ args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist", "--cookies", "cookies.txt"}
+ ids, err := run(channelName, args, stopChan, pool)
+ if err != nil {
+ return nil, errors.Err(err)
+ }
+ videoIDs := make([]string, maxVideos)
+ for i, v := range ids {
+ logrus.Debugf("%d - video id %s", i, v)
+ if i >= maxVideos {
+ break
+ }
+ videoIDs[i] = v
+ }
+ return videoIDs, nil
+}
+
+const releaseTimeFormat = "2006-01-02, 15:04:05 (MST)"
+
+func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Chan, ip *net.TCPAddr, pool *ip_manager.IPPool) (*ytdl.YtdlVideo, error) {
+ args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID, "--cookies", "cookies.txt"}
+ results, err := run(videoID, args, stopChan, pool)
+ if err != nil {
+ return nil, errors.Err(err)
+ }
+ var video *ytdl.YtdlVideo
+ err = json.Unmarshal([]byte(results[0]), &video)
+ if err != nil {
+ return nil, errors.Err(err)
+ }
+
+ // now get an accurate time
+ const maxTries = 5
+ tries := 0
+GetTime:
+ tries++
+ t, err := getUploadTime(config, videoID, ip, video.UploadDate)
+ if err != nil {
+ //slack(":warning: Upload time error: %v", err)
+ if tries <= maxTries && (errors.Is(err, errNotScraped) || errors.Is(err, errUploadTimeEmpty) || errors.Is(err, errStatusParse) || errors.Is(err, errConnectionIssue)) {
+ err := triggerScrape(videoID, ip)
+ if err == nil {
+ time.Sleep(2 * time.Second) // let them scrape it
+ goto GetTime
+ } else {
+ //slack("triggering scrape returned error: %v", err)
+ }
+ } else if !errors.Is(err, errNotScraped) && !errors.Is(err, errUploadTimeEmpty) {
+ //slack(":warning: Error while trying to get accurate upload time for %s: %v", videoID, err)
+ if t == "" {
+ return nil, errors.Err(err)
+ } else {
+ t = "" //TODO: get rid of the other piece below?
+ }
+ }
+ // do fallback below
+ }
+ //slack("After all that, upload time for %s is %s", videoID, t)
+
+ if t != "" {
+ parsed, err := time.Parse("2006-01-02, 15:04:05 (MST)", t) // this will probably be UTC, but Go's timezone parsing is fucked up. it ignores the timezone in the date
+ if err != nil {
+ return nil, errors.Err(err)
+ }
+ //slack(":exclamation: Got an accurate time for %s", videoID)
+ video.UploadDateForReal = parsed
+ } else { //TODO: this is the piece that isn't needed!
+ slack(":warning: Could not get accurate time for %s. Falling back to time from upload ytdl: %s.", videoID, video.UploadDate)
+ // fall back to UploadDate from youtube-dl
+ video.UploadDateForReal, err = time.Parse("20060102", video.UploadDate)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ return video, nil
+}
+
+var errNotScraped = errors.Base("not yet scraped by caa.iti.gr")
+var errUploadTimeEmpty = errors.Base("upload time is empty")
+var errStatusParse = errors.Base("could not parse status, got number, need string")
+var errConnectionIssue = errors.Base("there was a connection issue with the api")
+
+func slack(format string, a ...interface{}) {
+ fmt.Printf(format+"\n", a...)
+ util.SendToSlack(format, a...)
+}
+
+func triggerScrape(videoID string, ip *net.TCPAddr) error {
+ //slack("Triggering scrape for %s", videoID)
+ u, err := url.Parse("https://caa.iti.gr/verify_videoV3")
+ q := u.Query()
+ q.Set("twtimeline", "0")
+ q.Set("url", "https://www.youtube.com/watch?v="+videoID)
+ u.RawQuery = q.Encode()
+ //slack("GET %s", u.String())
+
+ client := getClient(ip)
+ req, err := http.NewRequest(http.MethodGet, u.String(), nil)
+ if err != nil {
+ return errors.Err(err)
+ }
+ req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36")
+
+ res, err := client.Do(req)
+ if err != nil {
+ return errors.Err(err)
+ }
+ defer res.Body.Close()
+
+ var response struct {
+ Message string `json:"message"`
+ Status string `json:"status"`
+ VideoURL string `json:"video_url"`
+ }
+ err = json.NewDecoder(res.Body).Decode(&response)
+ if err != nil {
+ if strings.Contains(err.Error(), "cannot unmarshal number") {
+ return errors.Err(errStatusParse)
+ }
+ if strings.Contains(err.Error(), "no route to host") {
+ return errors.Err(errConnectionIssue)
+ }
+ return errors.Err(err)
+ }
+
+ switch response.Status {
+ case "removed_video":
+ return errors.Err("video previously removed from service")
+ case "no_video":
+ return errors.Err("they say 'video cannot be found'. wtf?")
+ default:
+ spew.Dump(response)
+ }
+
+ return nil
+ //https://caa.iti.gr/caa/api/v4/videos/reports/h-tuxHS5lSM
+}
+
+func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr, uploadDate string) (string, error) {
+ //slack("Getting upload time for %s", videoID)
+ release, err := config.GetReleasedDate(videoID)
+ if err != nil {
+ logrus.Error(err)
+ }
+ ytdlUploadDate, err := time.Parse("20060102", uploadDate)
+ if err != nil {
+ logrus.Error(err)
+ }
+ if release != nil {
+ //const sqlTimeFormat = "2006-01-02 15:04:05"
+ sqlTime, err := time.ParseInLocation(time.RFC3339, release.ReleaseTime, time.UTC)
+ if err == nil {
+ if sqlTime.Day() != ytdlUploadDate.Day() {
+ logrus.Infof("upload day from APIs differs from the ytdl one by more than 1 day.")
+ } else {
+ return sqlTime.Format(releaseTimeFormat), nil
+ }
+ } else {
+ logrus.Error(err)
+ }
+ }
+
+ if time.Now().AddDate(0, 0, -3).After(ytdlUploadDate) {
+ return ytdlUploadDate.Format(releaseTimeFormat), nil
+ }
+ client := getClient(ip)
+ req, err := http.NewRequest(http.MethodGet, "https://caa.iti.gr/get_verificationV3?url=https://www.youtube.com/watch?v="+videoID, nil)
+ if err != nil {
+ return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
+ }
+ req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36")
+
+ res, err := client.Do(req)
+ if err != nil {
+ return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
+ }
+ defer res.Body.Close()
+
+ var uploadTime struct {
+ Time string `json:"video_upload_time"`
+ Message string `json:"message"`
+ Status string `json:"status"`
+ }
+ err = json.NewDecoder(res.Body).Decode(&uploadTime)
+ if err != nil {
+ return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
+ }
+
+ if uploadTime.Status == "ERROR1" {
+ return ytdlUploadDate.Format(releaseTimeFormat), errNotScraped
+ }
+
+ if uploadTime.Status == "" && strings.HasPrefix(uploadTime.Message, "CANNOT_RETRIEVE_REPORT_FOR_VIDEO_") {
+ return ytdlUploadDate.Format(releaseTimeFormat), errors.Err("cannot retrieve report for video")
+ }
+
+ if uploadTime.Time == "" {
+ return ytdlUploadDate.Format(releaseTimeFormat), errUploadTimeEmpty
+ }
+
+ return uploadTime.Time, nil
+}
+
+func getClient(ip *net.TCPAddr) *http.Client {
+ if ip == nil {
+ return http.DefaultClient
+ }
+
+ return &http.Client{
+ Transport: &http.Transport{
+ Proxy: http.ProxyFromEnvironment,
+ DialContext: (&net.Dialer{
+ LocalAddr: ip,
+ Timeout: 30 * time.Second,
+ KeepAlive: 30 * time.Second,
+ }).DialContext,
+ MaxIdleConns: 100,
+ IdleConnTimeout: 90 * time.Second,
+ TLSHandshakeTimeout: 10 * time.Second,
+ ExpectContinueTimeout: 1 * time.Second,
+ },
+ }
+}
+
+const (
+ googleBotUA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
+ chromeUA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"
+ maxAttempts = 3
+ extractionError = "YouTube said: Unable to extract video data"
+ throttledError = "HTTP Error 429"
+ AlternateThrottledError = "returned non-zero exit status 8"
+ youtubeDlError = "exit status 1"
+)
+
+func run(use string, args []string, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
+ var useragent []string
+ var lastError error
+ for attempts := 0; attempts < maxAttempts; attempts++ {
+ sourceAddress, err := getIPFromPool(use, stopChan, pool)
+ if err != nil {
+ return nil, err
+ }
+ argsForCommand := append(args, "--source-address", sourceAddress)
+ argsForCommand = append(argsForCommand, useragent...)
+ cmd := exec.Command("youtube-dl", argsForCommand...)
+
+ res, err := runCmd(cmd, stopChan)
+ pool.ReleaseIP(sourceAddress)
+ if err == nil {
+ return res, nil
+ }
+ lastError = err
+ if strings.Contains(err.Error(), youtubeDlError) {
+ if strings.Contains(err.Error(), extractionError) {
+ logrus.Warnf("known extraction error: %s", errors.FullTrace(err))
+ useragent = nextUA(useragent)
+ }
+ if strings.Contains(err.Error(), throttledError) || strings.Contains(err.Error(), AlternateThrottledError) {
+ pool.SetThrottled(sourceAddress)
+ //we don't want throttle errors to count toward the max retries
+ attempts--
+ }
+ }
+ }
+ return nil, lastError
+}
+
+func nextUA(current []string) []string {
+ if len(current) == 0 {
+ return []string{"--user-agent", googleBotUA}
+ }
+ return []string{"--user-agent", chromeUA}
+}
+
+func runCmd(cmd *exec.Cmd, stopChan stop.Chan) ([]string, error) {
+ logrus.Infof("running youtube-dl cmd: %s", strings.Join(cmd.Args, " "))
+ var err error
+ stderr, err := cmd.StderrPipe()
+ if err != nil {
+ return nil, errors.Err(err)
+ }
+ stdout, err := cmd.StdoutPipe()
+ if err != nil {
+ return nil, errors.Err(err)
+ }
+ err = cmd.Start()
+ if err != nil {
+ return nil, errors.Err(err)
+ }
+ outLog, err := ioutil.ReadAll(stdout)
+ if err != nil {
+ return nil, errors.Err(err)
+ }
+ errorLog, err := ioutil.ReadAll(stderr)
+ if err != nil {
+ return nil, errors.Err(err)
+ }
+ done := make(chan error, 1)
+ go func() {
+ done <- cmd.Wait()
+ }()
+
+ select {
+ case <-stopChan:
+ err := cmd.Process.Kill()
+ if err != nil {
+ return nil, errors.Prefix("failed to kill command after stopper cancellation", err)
+ }
+ return nil, errors.Err("canceled by stopper")
+ case err := <-done:
+ if err != nil {
+ return nil, errors.Prefix("youtube-dl "+strings.Join(cmd.Args, " ")+" ["+string(errorLog)+"]", err)
+ }
+ return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
+ }
+}
+
+func getIPFromPool(use string, stopChan stop.Chan, pool *ip_manager.IPPool) (sourceAddress string, err error) {
+ for {
+ sourceAddress, err = pool.GetIP(use)
+ if err != nil {
+ if errors.Is(err, ip_manager.ErrAllThrottled) {
+ select {
+ case <-stopChan:
+ return "", errors.Err("interrupted by user")
+
+ default:
+ time.Sleep(ip_manager.IPCooldownPeriod)
+ continue
+ }
+ } else {
+ return "", err
+ }
+ }
+ break
+ }
+ return
+}
diff --git a/downloader/downloader_test.go b/downloader/downloader_test.go
new file mode 100644
index 0000000..f4e981a
--- /dev/null
+++ b/downloader/downloader_test.go
@@ -0,0 +1,42 @@
+package downloader
+
+import (
+ "testing"
+
+ "github.com/lbryio/ytsync/v5/sdk"
+ "github.com/sirupsen/logrus"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestGetPlaylistVideoIDs(t *testing.T) {
+ videoIDs, err := GetPlaylistVideoIDs("UCJ0-OtVpF0wOKEqT2Z1HEtA", 50, nil, nil)
+ if err != nil {
+ logrus.Error(err)
+ }
+ for _, id := range videoIDs {
+ println(id)
+ }
+}
+
+func TestGetVideoInformation(t *testing.T) {
+ video, err := GetVideoInformation(nil, "zj7pXM9gE5M", nil, nil, nil)
+ if err != nil {
+ logrus.Error(err)
+ }
+ if video != nil {
+ logrus.Info(video.ID)
+ }
+}
+
+func Test_getUploadTime(t *testing.T) {
+ configs := sdk.APIConfig{
+ YoutubeAPIKey: "",
+ ApiURL: "https://api.lbry.com",
+ ApiToken: "Ht4NETrL5oWKyAaZkuSV68BKhtXkiLh5",
+ HostName: "test",
+ }
+ got, err := getUploadTime(&configs, "kDGOHNpRjzc", nil, "20060102")
+ assert.NoError(t, err)
+ t.Log(got)
+
+}
diff --git a/downloader/ytdl/Video.go b/downloader/ytdl/Video.go
new file mode 100644
index 0000000..079d557
--- /dev/null
+++ b/downloader/ytdl/Video.go
@@ -0,0 +1,147 @@
+package ytdl
+
+import (
+ "time"
+)
+
+type YtdlVideo struct {
+ UploadDate string `json:"upload_date"`
+ UploadDateForReal time.Time // you need to manually set this since the value in the API doesn't include the time
+ Extractor string `json:"extractor"`
+ Series interface{} `json:"series"`
+ Format string `json:"format"`
+ Vbr interface{} `json:"vbr"`
+ Chapters interface{} `json:"chapters"`
+ Height int `json:"height"`
+ LikeCount interface{} `json:"like_count"`
+ Duration int `json:"duration"`
+ Fulltitle string `json:"fulltitle"`
+ PlaylistIndex interface{} `json:"playlist_index"`
+ Album interface{} `json:"album"`
+ ViewCount int `json:"view_count"`
+ Playlist interface{} `json:"playlist"`
+ Title string `json:"title"`
+ Filename string `json:"_filename"`
+ Creator interface{} `json:"creator"`
+ Ext string `json:"ext"`
+ ID string `json:"id"`
+ DislikeCount interface{} `json:"dislike_count"`
+ AverageRating float64 `json:"average_rating"`
+ Abr int `json:"abr"`
+ UploaderURL string `json:"uploader_url"`
+ Categories []string `json:"categories"`
+ Fps float64 `json:"fps"`
+ StretchedRatio interface{} `json:"stretched_ratio"`
+ SeasonNumber interface{} `json:"season_number"`
+ Annotations interface{} `json:"annotations"`
+ WebpageURLBasename string `json:"webpage_url_basename"`
+ Acodec string `json:"acodec"`
+ DisplayID string `json:"display_id"`
+ //RequestedFormats []RequestedFormat `json:"requested_formats"`
+ //AutomaticCaptions struct{} `json:"automatic_captions"`
+ Description string `json:"description"`
+ Tags []string `json:"tags"`
+ Track interface{} `json:"track"`
+ RequestedSubtitles interface{} `json:"requested_subtitles"`
+ StartTime interface{} `json:"start_time"`
+ Uploader string `json:"uploader"`
+ ExtractorKey string `json:"extractor_key"`
+ FormatID string `json:"format_id"`
+ EpisodeNumber interface{} `json:"episode_number"`
+ UploaderID string `json:"uploader_id"`
+ //Subtitles struct{} `json:"subtitles"`
+ ReleaseYear interface{} `json:"release_year"`
+ Thumbnails []Thumbnail `json:"thumbnails"`
+ License interface{} `json:"license"`
+ Artist interface{} `json:"artist"`
+ AgeLimit int `json:"age_limit"`
+ ReleaseDate interface{} `json:"release_date"`
+ AltTitle interface{} `json:"alt_title"`
+ Thumbnail string `json:"thumbnail"`
+ ChannelID string `json:"channel_id"`
+ IsLive interface{} `json:"is_live"`
+ Width int `json:"width"`
+ EndTime interface{} `json:"end_time"`
+ WebpageURL string `json:"webpage_url"`
+ //Formats []Format `json:"formats"`
+ ChannelURL string `json:"channel_url"`
+ Resolution interface{} `json:"resolution"`
+ Vcodec string `json:"vcodec"`
+}
+
+type RequestedFormat struct {
+ Asr interface{} `json:"asr"`
+ Tbr float64 `json:"tbr"`
+ Container string `json:"container"`
+ Language interface{} `json:"language"`
+ Format string `json:"format"`
+ URL string `json:"url"`
+ Vcodec string `json:"vcodec"`
+ FormatNote string `json:"format_note"`
+ Height int `json:"height"`
+ Width int `json:"width"`
+ Ext string `json:"ext"`
+ FragmentBaseURL string `json:"fragment_base_url"`
+ Filesize interface{} `json:"filesize"`
+ Fps float64 `json:"fps"`
+ ManifestURL string `json:"manifest_url"`
+ Protocol string `json:"protocol"`
+ FormatID string `json:"format_id"`
+ HTTPHeaders struct {
+ AcceptCharset string `json:"Accept-Charset"`
+ AcceptLanguage string `json:"Accept-Language"`
+ AcceptEncoding string `json:"Accept-Encoding"`
+ Accept string `json:"Accept"`
+ UserAgent string `json:"User-Agent"`
+ } `json:"http_headers"`
+ Fragments []struct {
+ Path string `json:"path"`
+ Duration float64 `json:"duration,omitempty"`
+ } `json:"fragments"`
+ Acodec string `json:"acodec"`
+ Abr int `json:"abr,omitempty"`
+}
+
+type Format struct {
+ Asr int `json:"asr"`
+ Tbr float64 `json:"tbr"`
+ Protocol string `json:"protocol"`
+ Format string `json:"format"`
+ FormatNote string `json:"format_note"`
+ Height interface{} `json:"height"`
+ ManifestURL string `json:"manifest_url,omitempty"`
+ FormatID string `json:"format_id"`
+ Container string `json:"container,omitempty"`
+ Language interface{} `json:"language,omitempty"`
+ HTTPHeaders HTTPHeaders `json:"http_headers"`
+ URL string `json:"url"`
+ Vcodec string `json:"vcodec"`
+ Abr int `json:"abr,omitempty"`
+ Width interface{} `json:"width"`
+ Ext string `json:"ext"`
+ FragmentBaseURL string `json:"fragment_base_url,omitempty"`
+ Filesize interface{} `json:"filesize"`
+ Fps float64 `json:"fps"`
+ Fragments []struct {
+ Path string `json:"path"`
+ Duration float64 `json:"duration,omitempty"`
+ } `json:"fragments,omitempty"`
+ Acodec string `json:"acodec"`
+ PlayerURL interface{} `json:"player_url,omitempty"`
+}
+
+type Thumbnail struct {
+ URL string `json:"url"`
+ Width int `json:"width"`
+ Resolution string `json:"resolution"`
+ ID string `json:"id"`
+ Height int `json:"height"`
+}
+
+type HTTPHeaders struct {
+ AcceptCharset string `json:"Accept-Charset"`
+ AcceptLanguage string `json:"Accept-Language"`
+ AcceptEncoding string `json:"Accept-Encoding"`
+ Accept string `json:"Accept"`
+ UserAgent string `json:"User-Agent"`
+}
diff --git a/e2e/data_setup.sh b/e2e/data_setup.sh
index e15c1db..9b1ee17 100755
--- a/e2e/data_setup.sh
+++ b/e2e/data_setup.sh
@@ -19,6 +19,6 @@ mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER"
ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(2,"youtubertoken")'
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
#Add their youtube channel to be synced
-ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable)
-VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCNQfQvFMPnInwsU_iGYArJQ','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',10,10,1,1)"
+ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable,total_views,reviewed,last_uploaded_video,length_limit,size_limit)
+VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@test"$(date +%s)"','UCNQfQvFMPnInwsU_iGYArJQ','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',10,10,1,1,10000,1,'7bBV2Z-9wpo',60,2048)"
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL"
diff --git a/e2e/e2e.sh b/e2e/e2e.sh
index e051e27..f6fbc94 100755
--- a/e2e/e2e.sh
+++ b/e2e/e2e.sh
@@ -26,7 +26,7 @@ export REGTEST=true
# Local settings
export BLOBS_DIRECTORY="$(pwd)/e2e/blobsfiles"
export LBRYNET_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbrynet/"
-export LBRYNET_WALLETS_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum"
+export LBRYUM_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum"
export TMP_DIR="/var/tmp"
export CHAINNAME="lbrycrd_regtest"
export UID
diff --git a/go.mod b/go.mod
index 558dc66..da33ca3 100644
--- a/go.mod
+++ b/go.mod
@@ -3,25 +3,23 @@ module github.com/lbryio/ytsync/v5
replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
require (
- cloud.google.com/go v0.46.3 // indirect
- github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61
github.com/Microsoft/go-winio v0.4.14 // indirect
- github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a
+ github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb
github.com/aws/aws-sdk-go v1.25.9
- github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 // indirect
+ github.com/davecgh/go-spew v1.1.1
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/docker/docker v1.13.1
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
- github.com/go-sql-driver/mysql v1.4.1 // indirect
- github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc // indirect
+ github.com/golang/protobuf v1.4.2 // indirect
github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
- github.com/hashicorp/golang-lru v0.5.3 // indirect
github.com/hashicorp/memberlist v0.1.5 // indirect
github.com/hashicorp/serf v0.8.5 // indirect
- github.com/lbryio/lbry.go/v2 v2.6.1-0.20200710180140-fcade7475323
- github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6
+ github.com/kr/pretty v0.2.1 // indirect
+ github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128
+ github.com/lbryio/reflector.go v1.1.3-0.20200901181534-e8b98bc862d5
+ github.com/lbryio/types v0.0.0-20200605192618-366870b2862d // indirect
github.com/miekg/dns v1.1.22 // indirect
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
@@ -29,12 +27,14 @@ require (
github.com/prometheus/client_golang v0.9.2
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5
github.com/sirupsen/logrus v1.4.2
+ github.com/smartystreets/goconvey v1.6.4 // indirect
github.com/spf13/cobra v0.0.5
github.com/spf13/pflag v1.0.5 // indirect
- go.opencensus.io v0.22.1 // indirect
+ github.com/stretchr/testify v1.4.0
+ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
- google.golang.org/api v0.11.0
google.golang.org/appengine v1.6.5 // indirect
+ gopkg.in/ini.v1 v1.60.2 // indirect
)
go 1.13
diff --git a/go.sum b/go.sum
index 5ba9dde..409d083 100644
--- a/go.sum
+++ b/go.sum
@@ -1,22 +1,20 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3 h1:AVXDdKsrtX33oR9fbCMu/+c1o8Ofjq6Ku/MInaLVg5Y=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo=
+dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU=
+dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU=
+dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
+dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
+git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:N5Vqww5QISEHsWHOWDEx4PzdIay3Cg0Jp7zItq2ZAro=
-github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:GnKXcK+7DYNy/8w2Ex//Uql4IgfaU82Cd5rWKb7ah00=
github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
+github.com/StackExchange/wmi v0.0.0-20170410192909-ea383cf3ba6e/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
@@ -24,14 +22,16 @@ github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb h1:kvlW1qyM1aU3xeyeIVTU2jx5fSvjKpsU3aXvuaCMg3Q=
+github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.25.9 h1:WtVzerf5wSgPwlTTwl+ktCq/0GCS5MI9ZlLIcjsTr+Q=
github.com/aws/aws-sdk-go v1.25.9/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo=
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
@@ -44,19 +44,22 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc=
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
-github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:o64h9XF42kVEUuhuer2ehqrlX8rZmvQSU0+Vpj1rF6Q=
-github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:Rp8e0DCtEKwXFOC6JPJQVTz8tuGoGvw6Xfexggh/ed0=
+github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
+github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
+github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
@@ -65,63 +68,79 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
+github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-ini/ini v1.48.0 h1:TvO60hO/2xgaaTWp2P0wUe4CFxwdMzfbkv3+343Xzqw=
github.com/go-ini/ini v1.48.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
+github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
github.com/go-ozzo/ozzo-validation v3.6.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
-github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
+github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc h1:55rEp52jU6bkyslZ1+C/7NGfpQsEc6pxGLAGDOctqbw=
-github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0 h1:Rd1kQnQu0Hq3qvJppYSG0HtP+f5LPPUiDswTLiEegLg=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/gops v0.3.7/go.mod h1:bj0cwMmX1X4XIJFTjR99R5sCxNssNJ8HebFNvoQlmgY=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
+github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f h1:TyqzGm2z1h3AGhjOoRYyeLcW4WlW81MDQkWa+rx/000=
github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc=
+github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
github.com/gorilla/rpc v1.2.0 h1:WvvdC2lNeT1SP32zrIce5l0ECBfbAlmrmSBsuc57wfk=
github.com/gorilla/rpc v1.2.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
+github.com/gorilla/schema v1.0.2/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU=
github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
@@ -143,9 +162,10 @@ github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk=
github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
@@ -162,55 +182,78 @@ github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7V
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmoiron/sqlx v0.0.0-20170430194603-d9bd385d68c0/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU=
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 h1:jKUP9TQ0c7X3w6+IPyMit07RE42MtTWNd77sN2cHngQ=
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22/go.mod h1:u0Jo4f2dNlTJeeOywkM6bLwxq6gC3pZ9rEFHn3AhTdk=
+github.com/johntdyer/slackrus v0.0.0-20170926115001-3992f319fd0a/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07 h1:+kBG/8rjCa6vxJZbUjAiE4MQmBEBYc8nLEb51frnvBY=
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jteeuwen/go-bindata v3.0.7+incompatible/go.mod h1:JVvhzYOiGBnFSYRyV00iY8q7/0PThjIYav1p9h5dmKs=
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
+github.com/keybase/go-ps v0.0.0-20161005175911-668c8856d999/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
+github.com/lbryio/chainquery v1.9.0 h1:NfBZ3eKYwD3PqXU/vt+2tF3ox3WUWoW4J5YdEQ0rxw0=
+github.com/lbryio/chainquery v1.9.0/go.mod h1:7G8l7jNtANS1I7fQOvtzbiHsv6qKVmN4codXHc3C4kk=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
-github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303/go.mod h1:qR+Ui0hYhemIU4fXqM3d1P9eiaRFlof777VJgV7KJ8w=
+github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
-github.com/lbryio/lbry.go/v2 v2.6.1-0.20200710180140-fcade7475323 h1:Fvngg+hybcA1O2783YBw07S2MGpemSO+vWYSk97QHdM=
-github.com/lbryio/lbry.go/v2 v2.6.1-0.20200710180140-fcade7475323/go.mod h1:LgFKEpZzJE72DVgSXXfg+2IGOhrC9Lzj6eLma18iNz8=
+github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128 h1:VL209c+AGKixMFpxT+TsOAPzBPuoUzyjXf47iNe7OzY=
+github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128/go.mod h1:RqOv4V5eWY/JGmduCPcQVdN19SEYnNY3SuF+arTKIU4=
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
-github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46 h1:LemfR+rMxhf7nnOrzy2HqS7Me7SZ5gEwOcNFzKC8ySQ=
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
-github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6 h1:8k12lI18EEgMHkgGrgtRvJGl2Bq1vDSEbE4s+QaQ5Qc=
-github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6/go.mod h1:Q1Cnuv5iLsB2rS4Vr0o0EJr4Gs/EgDXFH2V4WtoM5o8=
+github.com/lbryio/reflector.go v1.1.3-0.20200901181534-e8b98bc862d5 h1:Y+zuIZQ1QwpBKLsdfThALHMwQSR4i0H67Znen6YdT54=
+github.com/lbryio/reflector.go v1.1.3-0.20200901181534-e8b98bc862d5/go.mod h1:QHKtgkmOCJJKZ578AFmN3jlBUhJFykbVEGgOuEiQ+aY=
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
-github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8 h1:jSNW/rK6DQsz7Zh+iv1zR384PeQdHt0gS4hKY17tkuM=
github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
+github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
+github.com/lbryio/types v0.0.0-20200605192618-366870b2862d h1:Pg/4L7ExGXsseTMy1d/LK9TY55c8CP2gtmafmYzed0Q=
+github.com/lbryio/types v0.0.0-20200605192618-366870b2862d/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
+github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lucas-clemente/quic-go v0.17.2 h1:4iQInIuNQkPNZmsy9rCnwuOzpH0qGnDo4jn0QfI/qE4=
+github.com/lucas-clemente/quic-go v0.17.2/go.mod h1:I0+fcNTdb9eS1ZcjQZbDVPGchJ86chcIxPALn9lEJqE=
+github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcXMWfkoumVwhcCbf3jHF9QKv/m37BkM0=
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4=
+github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/marten-seemann/qpack v0.1.0 h1:/0M7lkda/6mus9B8u34Asqm8ZhHAAt9Ho0vniNuVSVg=
+github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI=
+github.com/marten-seemann/qtls v0.9.1 h1:O0YKQxNVPaiFgMng0suWEOY2Sb4LT2sRn9Qimq3Z1IQ=
+github.com/marten-seemann/qtls v0.9.1/go.mod h1:T1MmAdDPyISzxlK6kjRr0pcZFBVd1OZbBb/j3cvzHhk=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.22 h1:Jm64b3bO9kP43ddLjL2EY3Io6bmy1qGb9Xxz6TqS6rc=
github.com/miekg/dns v1.1.22/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
@@ -225,21 +268,29 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F
github.com/mitchellh/mapstructure v0.0.0-20180511142126-bb74f1db0675/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
+github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
github.com/nlopes/slack v0.6.0 h1:jt0jxVQGhssx1Ib7naAOZEZcGdtIhTzkP0nopK0AsRA=
github.com/nlopes/slack v0.6.0/go.mod h1:JzQ9m3PMAqcpeCam7UaHSuBuupz7CmpjehYMayT6YOk=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94=
github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw=
+github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34=
+github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=
@@ -247,18 +298,22 @@ github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rK
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/profile v1.3.0/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
+github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rubenv/sql-migrate v0.0.0-20170330050058-38004e7a77f2/go.mod h1:WS0rl9eEliYI8DPnr3TOwz4439pay+qNgzJoVya/DmY=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@@ -269,11 +324,36 @@ github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77 h1:Msb6XRY62jQOueNNl
github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/sfreiberg/gotwilio v0.0.0-20180612161623-8fb7259ba8bf/go.mod h1:60PiR0SAnAcYSiwrXB6BaxeqHdXMf172toCosHfV+Yk=
+github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
github.com/shopspring/decimal v0.0.0-20180607144847-19e3cb6c2930/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
+github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5 h1:Gojs/hac/DoYEM7WEICT45+hNWczIeuL5D21e5/HPAw=
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
+github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
+github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM=
+github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0=
+github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
+github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
+github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
+github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI=
+github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU=
+github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag=
+github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg=
+github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw=
+github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y=
+github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
+github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q=
+github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ=
+github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I=
+github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0=
+github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ=
+github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk=
+github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4=
+github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -281,19 +361,29 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
-github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
+github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
+github.com/spf13/afero v1.1.1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.0-20180722215644-7c4570c3ebeb/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
+github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -301,69 +391,77 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
-github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o=
-github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
+github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
+github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d/go.mod h1:jspfvgf53t5NLUT4o9L1IX0kIBNKamGq1tWc/MgWK9Q=
+github.com/volatiletech/null v8.0.0+incompatible/go.mod h1:0wD98JzdqB+rLyZ70fN05VDbXbafIb0KU0MdVhCzmOQ=
+github.com/volatiletech/sqlboiler v3.4.0+incompatible/go.mod h1:jLfDkkHWPbS2cWRLkyC20vQWaIQsASEY7gM7zSo11Yw=
+github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d h1:tQo6hjclyv3RHUgZOl6iWb2Y44A/sN9bf9LAYfuioEg=
github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50=
-go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=
+github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
+go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM=
+go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
+golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
-golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5hzyggAkLLlCUjqfRxd8Q4=
golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5 h1:Q7tZBpemrlsc2I7IyODzhtallWRSm4Q0d09pL6XbQtU=
+golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190228165749-92fc7df08ae7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191009170851-d66e71096ffb h1:TR699M2v0qoKTOHxeLgp6zPqaQNs74f01a/ob9W0qko=
golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -371,92 +469,96 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20171017063910-8dbc5d05d6ed/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f h1:hjzMYz/7Ea1mNKfOnFOfktR0mlA5jqhvywClCMHM/qw=
golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w=
golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 h1:Toz2IK7k8rbltAXwNAxKcn9OzqyNfMUhUNjz3sL0NMk=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.11.0 h1:n/qM3q0/rV2F0pox7o0CvNhlPvZAo7pLbef122cbLJ0=
-google.golang.org/api v0.11.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
+google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
+google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03 h1:4HYDjxeNXAOTv3o1N2tjo8UUSlhQgAD52FVkwxnWgM8=
+google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
+google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
+google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s=
google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/gorp.v1 v1.7.1/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/ini.v1 v1.48.0 h1:URjZc+8ugRY5mL5uUeQH/a63JcHwdX9xZaWvmNWD7z8=
gopkg.in/ini.v1 v1.48.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.60.2 h1:7i8mqModL63zqi8nQn8Q3+0zvSCZy1AxhBgthKfi4WU=
+gopkg.in/ini.v1 v1.60.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 h1:FpCr9V8wuOei4BAen+93HtVJ+XSi+KPbaPKm0Vj5R64=
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79/go.mod h1:gWkaRU7CoXpezCBWfWjm3999QqS+1pYPXGbqQCTMzo8=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
@@ -467,10 +569,13 @@ gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
+grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/goversion v1.0.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
+sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
diff --git a/ip_manager/throttle.go b/ip_manager/throttle.go
index 43be077..e37ba69 100644
--- a/ip_manager/throttle.go
+++ b/ip_manager/throttle.go
@@ -63,21 +63,21 @@ func GetIPPool(stopGrp *stop.Group) (*IPPool, error) {
lock: &sync.RWMutex{},
stopGrp: stopGrp,
}
- ticker := time.NewTicker(10 * time.Second)
- go func() {
- for {
- select {
- case <-stopGrp.Ch():
- return
- case <-ticker.C:
- ipPoolInstance.lock.RLock()
- for _, ip := range ipPoolInstance.ips {
- log.Debugf("IP: %s\tInUse: %t\tVideoID: %s\tThrottled: %t\tLastUse: %.1f", ip.IP, ip.InUse, ip.UsedForVideo, ip.Throttled, time.Since(ip.LastUse).Seconds())
- }
- ipPoolInstance.lock.RUnlock()
- }
- }
- }()
+ //ticker := time.NewTicker(10 * time.Second)
+ //go func() {
+ // for {
+ // select {
+ // case <-stopGrp.Ch():
+ // return
+ // case <-ticker.C:
+ // ipPoolInstance.lock.RLock()
+ // for _, ip := range ipPoolInstance.ips {
+ // log.Debugf("IP: %s\tInUse: %t\tVideoID: %s\tThrottled: %t\tLastUse: %.1f", ip.IP, ip.InUse, ip.UsedForVideo, ip.Throttled, time.Since(ip.LastUse).Seconds())
+ // }
+ // ipPoolInstance.lock.RUnlock()
+ // }
+ // }
+ //}()
return ipPoolInstance, nil
}
@@ -108,7 +108,7 @@ func AllInUse(ips []throttledIP) bool {
func (i *IPPool) ReleaseIP(ip string) {
i.lock.Lock()
defer i.lock.Unlock()
- for j, _ := range i.ips {
+ for j := range i.ips {
localIP := &i.ips[j]
if localIP.IP == ip {
localIP.InUse = false
@@ -122,7 +122,7 @@ func (i *IPPool) ReleaseIP(ip string) {
func (i *IPPool) ReleaseAll() {
i.lock.Lock()
defer i.lock.Unlock()
- for j, _ := range i.ips {
+ for j := range i.ips {
if i.ips[j].Throttled {
continue
}
@@ -183,7 +183,7 @@ func (i *IPPool) nextIP(forVideo string) (*throttledIP, error) {
}
var nextIP *throttledIP
- for j, _ := range i.ips {
+ for j := range i.ips {
ip := &i.ips[j]
if ip.InUse || ip.Throttled {
continue
diff --git a/main.go b/main.go
index e9f9d3c..13080eb 100644
--- a/main.go
+++ b/main.go
@@ -11,6 +11,7 @@ import (
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/lbryio/ytsync/v5/manager"
"github.com/lbryio/ytsync/v5/sdk"
+ "github.com/lbryio/ytsync/v5/shared"
ytUtils "github.com/lbryio/ytsync/v5/util"
"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -23,18 +24,8 @@ var Version string
const defaultMaxTries = 3
var (
- flags sdk.SyncFlags
- maxTries int
- refill int
- limit int
- syncStatus string
- channelID string
- syncFrom int64
- syncUntil int64
- concurrentJobs int
- videosLimit int
- maxVideoSize int
- maxVideoLength float64
+ cliFlags shared.SyncFlags
+ maxVideoLength int
)
func main() {
@@ -51,25 +42,25 @@ func main() {
Args: cobra.RangeArgs(0, 0),
}
- cmd.Flags().BoolVar(&flags.StopOnError, "stop-on-error", false, "If a publish fails, stop all publishing and exit")
- cmd.Flags().IntVar(&maxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
- cmd.Flags().BoolVar(&flags.TakeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
- cmd.Flags().IntVar(&limit, "limit", 0, "limit the amount of channels to sync")
- cmd.Flags().BoolVar(&flags.SkipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
- cmd.Flags().BoolVar(&flags.SyncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
- cmd.Flags().BoolVar(&flags.SingleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
- cmd.Flags().BoolVar(&flags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
- cmd.Flags().BoolVar(&flags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
- cmd.Flags().BoolVar(&flags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
- cmd.Flags().BoolVar(&flags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube")
- cmd.Flags().StringVar(&syncStatus, "status", "", "Specify which queue to pull from. Overrides --update")
- cmd.Flags().StringVar(&channelID, "channelID", "", "If specified, only this channel will be synced.")
- cmd.Flags().Int64Var(&syncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
- cmd.Flags().Int64Var(&syncUntil, "before", time.Now().AddDate(1, 0, 0).Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)")
- cmd.Flags().IntVar(&concurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently")
- cmd.Flags().IntVar(&videosLimit, "videos-limit", 1000, "how many videos to process per channel")
- cmd.Flags().IntVar(&maxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)")
- cmd.Flags().Float64Var(&maxVideoLength, "max-length", 2.0, "Maximum video length to process (in hours)")
+ cmd.Flags().BoolVar(&cliFlags.StopOnError, "stop-on-error", false, "If a publish fails, stop all publishing and exit")
+ cmd.Flags().IntVar(&cliFlags.MaxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
+ cmd.Flags().BoolVar(&cliFlags.TakeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
+ cmd.Flags().IntVar(&cliFlags.Limit, "limit", 0, "limit the amount of channels to sync")
+ cmd.Flags().BoolVar(&cliFlags.SkipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
+ cmd.Flags().BoolVar(&cliFlags.SyncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
+ cmd.Flags().BoolVar(&cliFlags.SingleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
+ cmd.Flags().BoolVar(&cliFlags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
+ cmd.Flags().BoolVar(&cliFlags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
+ cmd.Flags().BoolVar(&cliFlags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
+ cmd.Flags().BoolVar(&cliFlags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube")
+ cmd.Flags().StringVar(&cliFlags.SyncStatus, "status", "", "Specify which queue to pull from. Overrides --update")
+ cmd.Flags().StringVar(&cliFlags.ChannelID, "channelID", "", "If specified, only this channel will be synced.")
+ cmd.Flags().Int64Var(&cliFlags.SyncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
+ cmd.Flags().Int64Var(&cliFlags.SyncUntil, "before", time.Now().AddDate(1, 0, 0).Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)")
+ cmd.Flags().IntVar(&cliFlags.ConcurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently")
+ cmd.Flags().IntVar(&cliFlags.VideosLimit, "videos-limit", 1000, "how many videos to process per channel")
+ cmd.Flags().IntVar(&cliFlags.MaxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)")
+ cmd.Flags().IntVar(&maxVideoLength, "max-length", 2, "Maximum video length to process (in hours)")
if err := cmd.Execute(); err != nil {
fmt.Println(err)
@@ -96,29 +87,30 @@ func ytSync(cmd *cobra.Command, args []string) {
util.InitSlack(os.Getenv("SLACK_TOKEN"), os.Getenv("SLACK_CHANNEL"), hostname)
}
- if syncStatus != "" && !util.InSlice(syncStatus, manager.SyncStatuses) {
- log.Errorf("status must be one of the following: %v\n", manager.SyncStatuses)
+ if cliFlags.SyncStatus != "" && !util.InSlice(cliFlags.SyncStatus, shared.SyncStatuses) {
+ log.Errorf("status must be one of the following: %v\n", shared.SyncStatuses)
return
}
- if flags.StopOnError && maxTries != defaultMaxTries {
+ if cliFlags.StopOnError && cliFlags.MaxTries != defaultMaxTries {
log.Errorln("--stop-on-error and --max-tries are mutually exclusive")
return
}
- if maxTries < 1 {
+ if cliFlags.MaxTries < 1 {
log.Errorln("setting --max-tries less than 1 doesn't make sense")
return
}
- if limit < 0 {
+ if cliFlags.Limit < 0 {
log.Errorln("setting --limit less than 0 (unlimited) doesn't make sense")
return
}
+ cliFlags.MaxVideoLength = time.Duration(maxVideoLength) * time.Hour
apiURL := os.Getenv("LBRY_WEB_API")
apiToken := os.Getenv("LBRY_API_TOKEN")
youtubeAPIKey := os.Getenv("YOUTUBE_API_KEY")
- lbrycrdString := os.Getenv("LBRYCRD_STRING")
+ lbrycrdDsn := os.Getenv("LBRYCRD_STRING")
awsS3ID := os.Getenv("AWS_S3_ID")
awsS3Secret := os.Getenv("AWS_S3_SECRET")
awsS3Region := os.Getenv("AWS_S3_REGION")
@@ -151,42 +143,30 @@ func ytSync(cmd *cobra.Command, args []string) {
log.Errorln("AWS S3 Bucket was not defined. Please set the environment variable AWS_S3_BUCKET")
return
}
- if lbrycrdString == "" {
+ if lbrycrdDsn == "" {
log.Infoln("Using default (local) lbrycrd instance. Set LBRYCRD_STRING if you want to use something else")
}
blobsDir := ytUtils.GetBlobsDir()
- syncProperties := &sdk.SyncProperties{
- SyncFrom: syncFrom,
- SyncUntil: syncUntil,
- YoutubeChannelID: channelID,
- }
apiConfig := &sdk.APIConfig{
YoutubeAPIKey: youtubeAPIKey,
ApiURL: apiURL,
ApiToken: apiToken,
HostName: hostname,
}
+ awsConfig := &shared.AwsConfigs{
+ AwsS3ID: awsS3ID,
+ AwsS3Secret: awsS3Secret,
+ AwsS3Region: awsS3Region,
+ AwsS3Bucket: awsS3Bucket,
+ }
sm := manager.NewSyncManager(
- flags,
- maxTries,
- refill,
- limit,
- concurrentJobs,
- concurrentJobs,
+ cliFlags,
blobsDir,
- videosLimit,
- maxVideoSize,
- lbrycrdString,
- awsS3ID,
- awsS3Secret,
- awsS3Region,
- awsS3Bucket,
- syncStatus,
- syncProperties,
+ lbrycrdDsn,
+ awsConfig,
apiConfig,
- maxVideoLength,
)
err := sm.Start()
if err != nil {
diff --git a/manager/count.go b/manager/count.go
deleted file mode 100644
index c3136bc..0000000
--- a/manager/count.go
+++ /dev/null
@@ -1,32 +0,0 @@
-package manager
-
-import (
- "net/http"
-
- "github.com/lbryio/lbry.go/v2/extras/errors"
-
- "google.golang.org/api/googleapi/transport"
- "google.golang.org/api/youtube/v3"
-)
-
-func (s *Sync) CountVideos() (uint64, error) {
- client := &http.Client{
- Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
- }
-
- service, err := youtube.New(client)
- if err != nil {
- return 0, errors.Prefix("error creating YouTube service", err)
- }
-
- response, err := service.Channels.List("statistics").Id(s.YoutubeChannelID).Do()
- if err != nil {
- return 0, errors.Prefix("error getting channels", err)
- }
-
- if len(response.Items) < 1 {
- return 0, errors.Err("youtube channel not found")
- }
-
- return response.Items[0].Statistics.VideoCount, nil
-}
diff --git a/manager/manager.go b/manager/manager.go
index 2f831b9..8a09b77 100644
--- a/manager/manager.go
+++ b/manager/manager.go
@@ -10,94 +10,43 @@ import (
"github.com/lbryio/ytsync/v5/ip_manager"
"github.com/lbryio/ytsync/v5/namer"
"github.com/lbryio/ytsync/v5/sdk"
+ "github.com/lbryio/ytsync/v5/shared"
logUtils "github.com/lbryio/ytsync/v5/util"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/util"
- "github.com/aws/aws-sdk-go/aws"
- "github.com/aws/aws-sdk-go/aws/credentials"
log "github.com/sirupsen/logrus"
)
type SyncManager struct {
- SyncFlags sdk.SyncFlags
- maxTries int
- refill int
- limit int
- concurrentJobs int
- concurrentVideos int
- blobsDir string
- videosLimit int
- maxVideoSize int
- maxVideoLength float64
- lbrycrdString string
- awsS3ID string
- awsS3Secret string
- awsS3Region string
- syncStatus string
- awsS3Bucket string
- syncProperties *sdk.SyncProperties
- apiConfig *sdk.APIConfig
+ CliFlags shared.SyncFlags
+ ApiConfig *sdk.APIConfig
+ LbrycrdDsn string
+ AwsConfigs *shared.AwsConfigs
+
+ blobsDir string
+ channelsToSync []Sync
}
-func NewSyncManager(syncFlags sdk.SyncFlags, maxTries int, refill int, limit int, concurrentJobs int, concurrentVideos int, blobsDir string, videosLimit int,
- maxVideoSize int, lbrycrdString string, awsS3ID string, awsS3Secret string, awsS3Region string, awsS3Bucket string,
- syncStatus string, syncProperties *sdk.SyncProperties, apiConfig *sdk.APIConfig, maxVideoLength float64) *SyncManager {
+func NewSyncManager(cliFlags shared.SyncFlags, blobsDir, lbrycrdDsn string, awsConfigs *shared.AwsConfigs, apiConfig *sdk.APIConfig) *SyncManager {
return &SyncManager{
- SyncFlags: syncFlags,
- maxTries: maxTries,
- refill: refill,
- limit: limit,
- concurrentJobs: concurrentJobs,
- concurrentVideos: concurrentVideos,
- blobsDir: blobsDir,
- videosLimit: videosLimit,
- maxVideoSize: maxVideoSize,
- maxVideoLength: maxVideoLength,
- lbrycrdString: lbrycrdString,
- awsS3ID: awsS3ID,
- awsS3Secret: awsS3Secret,
- awsS3Region: awsS3Region,
- awsS3Bucket: awsS3Bucket,
- syncStatus: syncStatus,
- syncProperties: syncProperties,
- apiConfig: apiConfig,
+ CliFlags: cliFlags,
+ blobsDir: blobsDir,
+ LbrycrdDsn: lbrycrdDsn,
+ AwsConfigs: awsConfigs,
+ ApiConfig: apiConfig,
}
}
-
-const (
- StatusPending = "pending" // waiting for permission to sync
- StatusPendingEmail = "pendingemail" // permission granted but missing email
- StatusQueued = "queued" // in sync queue. will be synced soon
- StatusPendingUpgrade = "pendingupgrade" // in sync queue. will be synced soon
- StatusSyncing = "syncing" // syncing now
- StatusSynced = "synced" // done
- StatusFailed = "failed"
- StatusFinalized = "finalized" // no more changes allowed
- StatusAbandoned = "abandoned" // deleted on youtube or banned
-)
-const LatestMetadataVersion = 2
-
-var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned}
-
-const (
- VideoStatusPublished = "published"
- VideoStatusFailed = "failed"
- VideoStatusUpgradeFailed = "upgradefailed"
- VideoStatusUnpublished = "unpublished"
- VideoStatusTranferFailed = "transferfailed"
-)
-
-const (
- TransferStateNotTouched = iota
- TransferStatePending
- TransferStateComplete
- TransferStateManual
-)
+func (s *SyncManager) enqueueChannel(channel *shared.YoutubeChannel) {
+ s.channelsToSync = append(s.channelsToSync, Sync{
+ DbChannelData: channel,
+ Manager: s,
+ namer: namer.NewNamer(),
+ })
+}
func (s *SyncManager) Start() error {
-
if logUtils.ShouldCleanOnStartup() {
err := logUtils.CleanForStartup()
if err != nil {
@@ -105,108 +54,66 @@ func (s *SyncManager) Start() error {
}
}
+ var lastChannelProcessed string
syncCount := 0
for {
+ s.channelsToSync = make([]Sync, 0, 10) // reset sync queue
err := s.checkUsedSpace()
if err != nil {
return errors.Err(err)
}
-
- var syncs []Sync
shouldInterruptLoop := false
- isSingleChannelSync := s.syncProperties.YoutubeChannelID != ""
- if isSingleChannelSync {
- channels, err := s.apiConfig.FetchChannels("", s.syncProperties)
+ if s.CliFlags.IsSingleChannelSync() {
+ channels, err := s.ApiConfig.FetchChannels("", &s.CliFlags)
if err != nil {
return errors.Err(err)
}
if len(channels) != 1 {
return errors.Err("Expected 1 channel, %d returned", len(channels))
}
- lbryChannelName := channels[0].DesiredChannelName
- syncs = make([]Sync, 1)
- syncs[0] = Sync{
- APIConfig: s.apiConfig,
- YoutubeChannelID: s.syncProperties.YoutubeChannelID,
- LbryChannelName: lbryChannelName,
- lbryChannelID: channels[0].ChannelClaimID,
- MaxTries: s.maxTries,
- ConcurrentVideos: s.concurrentVideos,
- Refill: s.refill,
- Manager: s,
- MaxVideoLength: s.maxVideoLength,
- LbrycrdString: s.lbrycrdString,
- AwsS3ID: s.awsS3ID,
- AwsS3Secret: s.awsS3Secret,
- AwsS3Region: s.awsS3Region,
- AwsS3Bucket: s.awsS3Bucket,
- namer: namer.NewNamer(),
- Fee: channels[0].Fee,
- clientPublishAddress: channels[0].PublishAddress,
- publicKey: channels[0].PublicKey,
- transferState: channels[0].TransferState,
- }
+ s.enqueueChannel(&channels[0])
shouldInterruptLoop = true
} else {
var queuesToSync []string
- if s.syncStatus != "" {
- queuesToSync = append(queuesToSync, s.syncStatus)
- } else if s.SyncFlags.SyncUpdate {
- queuesToSync = append(queuesToSync, StatusSyncing, StatusSynced)
+ if s.CliFlags.SyncStatus != "" {
+ queuesToSync = append(queuesToSync, s.CliFlags.SyncStatus)
+ } else if s.CliFlags.SyncUpdate {
+ queuesToSync = append(queuesToSync, shared.StatusSyncing, shared.StatusSynced)
} else {
- queuesToSync = append(queuesToSync, StatusSyncing, StatusQueued)
+ queuesToSync = append(queuesToSync, shared.StatusSyncing, shared.StatusQueued)
}
queues:
for _, q := range queuesToSync {
- //temporary override for sync-until to give tom the time to review the channels
- if q == StatusQueued {
- s.syncProperties.SyncUntil = time.Now().Add(-8 * time.Hour).Unix()
- }
- channels, err := s.apiConfig.FetchChannels(q, s.syncProperties)
+ channels, err := s.ApiConfig.FetchChannels(q, &s.CliFlags)
if err != nil {
return err
}
- for i, c := range channels {
- log.Infof("There are %d channels in the \"%s\" queue", len(channels)-i, q)
- maxVideoLength := s.maxVideoLength
- if c.TotalSubscribers < 1000 {
- maxVideoLength = 1.0
- }
- syncs = append(syncs, Sync{
- APIConfig: s.apiConfig,
- YoutubeChannelID: c.ChannelId,
- LbryChannelName: c.DesiredChannelName,
- lbryChannelID: c.ChannelClaimID,
- MaxTries: s.maxTries,
- ConcurrentVideos: s.concurrentVideos,
- MaxVideoLength: maxVideoLength,
- Refill: s.refill,
- Manager: s,
- LbrycrdString: s.lbrycrdString,
- AwsS3ID: s.awsS3ID,
- AwsS3Secret: s.awsS3Secret,
- AwsS3Region: s.awsS3Region,
- AwsS3Bucket: s.awsS3Bucket,
- namer: namer.NewNamer(),
- Fee: c.Fee,
- clientPublishAddress: c.PublishAddress,
- publicKey: c.PublicKey,
- transferState: c.TransferState,
- })
- if q != StatusFailed {
- continue queues
+ log.Infof("Currently processing the \"%s\" queue with %d channels", q, len(channels))
+ for _, c := range channels {
+ s.enqueueChannel(&c)
+ queueAll := q == shared.StatusFailed || q == shared.StatusSyncing
+ if !queueAll {
+ break queues
}
}
+ log.Infof("Drained the \"%s\" queue", q)
}
}
- if len(syncs) == 0 {
+ if len(s.channelsToSync) == 0 {
log.Infoln("No channels to sync. Pausing 5 minutes!")
time.Sleep(5 * time.Minute)
}
- for _, sync := range syncs {
+ for _, sync := range s.channelsToSync {
+ if lastChannelProcessed == sync.DbChannelData.ChannelId {
+ util.SendToSlack("We just killed a sync for %s to stop looping! (%s)", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId)
+ stopTheLoops := errors.Err("Found channel %s running twice, set it to failed, and reprocess later", sync.DbChannelData.DesiredChannelName)
+ sync.setChannelTerminationStatus(&stopTheLoops)
+ continue
+ }
+ lastChannelProcessed = sync.DbChannelData.ChannelId
shouldNotCount := false
- logUtils.SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
+ logUtils.SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId, syncCount+1)
err := sync.FullCycle()
//TODO: THIS IS A TEMPORARY WORK AROUND FOR THE STUPID IP LOCKUP BUG
ipPool, _ := ip_manager.GetIPPool(sync.grp)
@@ -234,40 +141,35 @@ func (s *SyncManager) Start() error {
}
shouldNotCount = strings.Contains(err.Error(), "this youtube channel is being managed by another server")
if !shouldNotCount {
- logUtils.SendInfoToSlack("A non fatal error was reported by the sync process. %s\nContinuing...", err.Error())
+ logUtils.SendInfoToSlack("A non fatal error was reported by the sync process.\n%s", errors.FullTrace(err))
}
}
err = blobs_reflector.ReflectAndClean()
if err != nil {
return errors.Prefix("@Nikooo777 something went wrong while reflecting blobs", err)
}
- logUtils.SendInfoToSlack("Syncing %s (%s) reached an end. total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
+ logUtils.SendInfoToSlack("%s (%s) reached an end. Total processed channels since startup: %d", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId, syncCount+1)
if !shouldNotCount {
syncCount++
}
- if sync.IsInterrupted() || (s.limit != 0 && syncCount >= s.limit) {
+ if sync.IsInterrupted() || (s.CliFlags.Limit != 0 && syncCount >= s.CliFlags.Limit) {
shouldInterruptLoop = true
break
}
}
- if shouldInterruptLoop || s.SyncFlags.SingleRun {
+ if shouldInterruptLoop || s.CliFlags.SingleRun {
break
}
}
return nil
}
-func (s *SyncManager) GetS3AWSConfig() aws.Config {
- return aws.Config{
- Credentials: credentials.NewStaticCredentials(s.awsS3ID, s.awsS3Secret, ""),
- Region: &s.awsS3Region,
- }
-}
+
func (s *SyncManager) checkUsedSpace() error {
usedPctile, err := GetUsedSpace(logUtils.GetBlobsDir())
if err != nil {
return errors.Err(err)
}
- if usedPctile >= 0.90 && !s.SyncFlags.SkipSpaceCheck {
+ if usedPctile >= 0.90 && !s.CliFlags.SkipSpaceCheck {
return errors.Err(fmt.Sprintf("more than 90%% of the space has been used. use --skip-space-check to ignore. Used: %.1f%%", usedPctile*100))
}
log.Infof("disk usage: %.1f%%", usedPctile*100)
diff --git a/manager/s3_storage.go b/manager/s3_storage.go
new file mode 100644
index 0000000..8b78c89
--- /dev/null
+++ b/manager/s3_storage.go
@@ -0,0 +1,250 @@
+package manager
+
+import (
+ "os"
+ "path/filepath"
+
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/aws/awserr"
+ "github.com/aws/aws-sdk-go/aws/session"
+ "github.com/aws/aws-sdk-go/service/s3"
+ "github.com/aws/aws-sdk-go/service/s3/s3manager"
+
+ "github.com/lbryio/lbry.go/v2/extras/errors"
+
+ logUtils "github.com/lbryio/ytsync/v5/util"
+)
+
+func (s *Sync) getS3Downloader() (*s3manager.Downloader, error) {
+ s3Session, err := session.NewSession(s.Manager.AwsConfigs.GetS3AWSConfig())
+ if err != nil {
+ return nil, errors.Prefix("error starting session: ", err)
+ }
+ downloader := s3manager.NewDownloader(s3Session)
+ return downloader, nil
+}
+func (s *Sync) getS3Uploader() (*s3manager.Uploader, error) {
+ s3Session, err := session.NewSession(s.Manager.AwsConfigs.GetS3AWSConfig())
+ if err != nil {
+ return nil, errors.Prefix("error starting session: ", err)
+ }
+ uploader := s3manager.NewUploader(s3Session)
+ return uploader, nil
+}
+
+func (s *Sync) downloadWallet() error {
+ defaultWalletDir, defaultTempWalletDir, key, err := s.getWalletPaths()
+ if err != nil {
+ return errors.Err(err)
+ }
+ downloader, err := s.getS3Downloader()
+ if err != nil {
+ return err
+ }
+ out, err := os.Create(defaultTempWalletDir)
+ if err != nil {
+ return errors.Prefix("error creating temp wallet: ", err)
+ }
+ defer out.Close()
+
+ bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
+ Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
+ Key: key,
+ })
+ if err != nil {
+ // Casting to the awserr.Error type will allow you to inspect the error
+ // code returned by the service in code. The error code can be used
+ // to switch on context specific functionality. In this case a context
+ // specific error message is printed to the user based on the bucket
+ // and key existing.
+ //
+ // For information on other S3 API error codes see:
+ // http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
+ if aerr, ok := err.(awserr.Error); ok {
+ code := aerr.Code()
+ if code == s3.ErrCodeNoSuchKey {
+ return errors.Err("wallet not on S3")
+ }
+ }
+ return err
+ } else if bytesWritten == 0 {
+ return errors.Err("zero bytes written")
+ }
+
+ err = os.Rename(defaultTempWalletDir, defaultWalletDir)
+ if err != nil {
+ return errors.Prefix("error replacing temp wallet for default wallet: ", err)
+ }
+
+ return nil
+}
+
+func (s *Sync) downloadBlockchainDB() error {
+ if logUtils.IsRegTest() {
+ return nil // tests fail if we re-use the same blockchain DB
+ }
+ defaultBDBDir, defaultTempBDBDir, key, err := s.getBlockchainDBPaths()
+ if err != nil {
+ return errors.Err(err)
+ }
+ files, err := filepath.Glob(defaultBDBDir + "*")
+ if err != nil {
+ return errors.Err(err)
+ }
+ for _, f := range files {
+ err = os.Remove(f)
+ if err != nil {
+ return errors.Err(err)
+ }
+ }
+
+ downloader, err := s.getS3Downloader()
+ if err != nil {
+ return errors.Err(err)
+ }
+ out, err := os.Create(defaultTempBDBDir)
+ if err != nil {
+ return errors.Prefix("error creating temp wallet: ", err)
+ }
+ defer out.Close()
+
+ bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
+ Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
+ Key: key,
+ })
+ if err != nil {
+ // Casting to the awserr.Error type will allow you to inspect the error
+ // code returned by the service in code. The error code can be used
+ // to switch on context specific functionality. In this case a context
+ // specific error message is printed to the user based on the bucket
+ // and key existing.
+ //
+ // For information on other S3 API error codes see:
+ // http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
+ if aerr, ok := err.(awserr.Error); ok {
+ code := aerr.Code()
+ if code == s3.ErrCodeNoSuchKey {
+ return nil // let ytsync sync the database by itself
+ }
+ }
+ return errors.Err(err)
+ } else if bytesWritten == 0 {
+ return errors.Err("zero bytes written")
+ }
+
+ err = os.Rename(defaultTempBDBDir, defaultBDBDir)
+ if err != nil {
+ return errors.Prefix("error replacing temp blockchain.db for default blockchain.db: ", err)
+ }
+
+ return nil
+}
+
+func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string, err error) {
+ defaultWallet = os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
+ tempWallet = os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet"
+ key = aws.String("/wallets/" + s.DbChannelData.ChannelId)
+ if logUtils.IsRegTest() {
+ defaultWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
+ tempWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet"
+ key = aws.String("/regtest/" + s.DbChannelData.ChannelId)
+ }
+
+ lbryumDir := os.Getenv("LBRYUM_DIR")
+ if lbryumDir != "" {
+ defaultWallet = lbryumDir + "/wallets/default_wallet"
+ tempWallet = lbryumDir + "/wallets/tmp_wallet"
+ }
+
+ if _, err := os.Stat(defaultWallet); !os.IsNotExist(err) {
+ return "", "", nil, errors.Err("default_wallet already exists")
+ }
+ return
+}
+
+func (s *Sync) getBlockchainDBPaths() (defaultDB, tempDB string, key *string, err error) {
+ lbryumDir := os.Getenv("LBRYUM_DIR")
+ if lbryumDir == "" {
+ if logUtils.IsRegTest() {
+ lbryumDir = os.Getenv("HOME") + "/.lbryum_regtest"
+ } else {
+ lbryumDir = os.Getenv("HOME") + "/.lbryum"
+ }
+ }
+ defaultDB = lbryumDir + "/lbc_mainnet/blockchain.db"
+ tempDB = lbryumDir + "/lbc_mainnet/tmp_blockchain.db"
+ key = aws.String("/blockchain_dbs/" + s.DbChannelData.ChannelId)
+ if logUtils.IsRegTest() {
+ defaultDB = lbryumDir + "/lbc_regtest/blockchain.db"
+ tempDB = lbryumDir + "/lbc_regtest/tmp_blockchain.db"
+ key = aws.String("/regtest_dbs/" + s.DbChannelData.ChannelId)
+ }
+ return
+}
+
+func (s *Sync) uploadWallet() error {
+ defaultWalletDir := logUtils.GetDefaultWalletPath()
+ key := aws.String("/wallets/" + s.DbChannelData.ChannelId)
+ if logUtils.IsRegTest() {
+ key = aws.String("/regtest/" + s.DbChannelData.ChannelId)
+ }
+
+ if _, err := os.Stat(defaultWalletDir); os.IsNotExist(err) {
+ return errors.Err("default_wallet does not exist")
+ }
+
+ uploader, err := s.getS3Uploader()
+ if err != nil {
+ return err
+ }
+
+ file, err := os.Open(defaultWalletDir)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+
+ _, err = uploader.Upload(&s3manager.UploadInput{
+ Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
+ Key: key,
+ Body: file,
+ })
+ if err != nil {
+ return err
+ }
+
+ return os.Remove(defaultWalletDir)
+}
+
+func (s *Sync) uploadBlockchainDB() error {
+ defaultBDBDir, _, key, err := s.getBlockchainDBPaths()
+ if err != nil {
+ return errors.Err(err)
+ }
+
+ if _, err := os.Stat(defaultBDBDir); os.IsNotExist(err) {
+ return errors.Err("blockchain.db does not exist")
+ }
+
+ uploader, err := s.getS3Uploader()
+ if err != nil {
+ return err
+ }
+
+ file, err := os.Open(defaultBDBDir)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+
+ _, err = uploader.Upload(&s3manager.UploadInput{
+ Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
+ Key: key,
+ Body: file,
+ })
+ if err != nil {
+ return err
+ }
+
+ return os.Remove(defaultBDBDir)
+}
diff --git a/manager/setup.go b/manager/setup.go
index aeab832..e2a809e 100644
--- a/manager/setup.go
+++ b/manager/setup.go
@@ -3,23 +3,22 @@ package manager
import (
"fmt"
"math"
- "net/http"
"strconv"
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
"github.com/lbryio/lbry.go/v2/extras/util"
+ "github.com/lbryio/ytsync/v5/shared"
"github.com/lbryio/ytsync/v5/timing"
logUtils "github.com/lbryio/ytsync/v5/util"
+ "github.com/lbryio/ytsync/v5/ytapi"
"github.com/lbryio/ytsync/v5/tags_manager"
"github.com/lbryio/ytsync/v5/thumbs"
"github.com/shopspring/decimal"
log "github.com/sirupsen/logrus"
- "google.golang.org/api/googleapi/transport"
- "google.golang.org/api/youtube/v3"
)
func (s *Sync) enableAddressReuse() error {
@@ -74,11 +73,7 @@ func (s *Sync) walletSetup() error {
}
log.Debugf("Starting balance is %.4f", balance)
- n, err := s.CountVideos()
- if err != nil {
- return err
- }
- videosOnYoutube := int(n)
+ videosOnYoutube := int(s.DbChannelData.TotalVideos)
log.Debugf("Source channel has %d videos", videosOnYoutube)
if videosOnYoutube == 0 {
@@ -103,17 +98,17 @@ func (s *Sync) walletSetup() error {
log.Debugf("We already allocated credits for %d published videos and %d failed videos", publishedCount, failedCount)
- if videosOnYoutube > s.Manager.videosLimit {
- videosOnYoutube = s.Manager.videosLimit
+ if videosOnYoutube > s.Manager.CliFlags.VideosLimit {
+ videosOnYoutube = s.Manager.CliFlags.VideosLimit
}
unallocatedVideos := videosOnYoutube - (publishedCount + failedCount)
channelFee := channelClaimAmount
- channelAlreadyClaimed := s.lbryChannelID != ""
+ channelAlreadyClaimed := s.DbChannelData.ChannelClaimID != ""
if channelAlreadyClaimed {
channelFee = 0.0
}
requiredBalance := float64(unallocatedVideos)*(publishAmount+estimatedMaxTxFee) + channelFee
- if s.Manager.SyncFlags.UpgradeMetadata {
+ if s.Manager.CliFlags.UpgradeMetadata {
requiredBalance += float64(notUpgradedCount) * 0.001
}
@@ -122,8 +117,8 @@ func (s *Sync) walletSetup() error {
refillAmount = math.Max(math.Max(requiredBalance-balance, minimumAccountBalance-balance), minimumRefillAmount)
}
- if s.Refill > 0 {
- refillAmount += float64(s.Refill)
+ if s.Manager.CliFlags.Refill > 0 {
+ refillAmount += float64(s.Manager.CliFlags.Refill)
}
if refillAmount > 0 {
@@ -139,12 +134,11 @@ func (s *Sync) walletSetup() error {
} else if claimAddress == nil {
return errors.Err("could not get an address")
}
- s.claimAddress = string(claimAddress.Items[0].Address)
- if s.claimAddress == "" {
- return errors.Err("found blank claim address")
+ if s.DbChannelData.PublishAddress == "" || !s.shouldTransfer() {
+ s.DbChannelData.PublishAddress = string(claimAddress.Items[0].Address)
}
- if s.shouldTransfer() {
- s.claimAddress = s.clientPublishAddress
+ if s.DbChannelData.PublishAddress == "" {
+ return errors.Err("found blank claim address")
}
err = s.ensureEnoughUTXOs()
@@ -266,15 +260,14 @@ func (s *Sync) ensureEnoughUTXOs() error {
}
func (s *Sync) waitForNewBlock() error {
- start := time.Now()
- defer func(start time.Time) {
- timing.TimedComponent("waitForNewBlock").Add(time.Since(start))
- }(start)
+ defer func(start time.Time) { timing.TimedComponent("waitForNewBlock").Add(time.Since(start)) }(time.Now())
+
log.Printf("regtest: %t, docker: %t", logUtils.IsRegTest(), logUtils.IsUsingDocker())
status, err := s.daemon.Status()
if err != nil {
return err
}
+
for status.Wallet.Blocks == 0 || status.Wallet.BlocksBehind != 0 {
time.Sleep(5 * time.Second)
status, err = s.daemon.Status()
@@ -304,14 +297,16 @@ func (s *Sync) waitForNewBlock() error {
}
func (s *Sync) GenerateRegtestBlock() error {
- lbrycrd, err := logUtils.GetLbrycrdClient(s.LbrycrdString)
+ lbrycrd, err := logUtils.GetLbrycrdClient(s.Manager.LbrycrdDsn)
if err != nil {
return errors.Prefix("error getting lbrycrd client: ", err)
}
+
txs, err := lbrycrd.Generate(1)
if err != nil {
return errors.Prefix("error generating new block: ", err)
}
+
for _, tx := range txs {
log.Info("Generated tx: ", tx.String())
}
@@ -319,11 +314,9 @@ func (s *Sync) GenerateRegtestBlock() error {
}
func (s *Sync) ensureChannelOwnership() error {
- start := time.Now()
- defer func(start time.Time) {
- timing.TimedComponent("ensureChannelOwnership").Add(time.Since(start))
- }(start)
- if s.LbryChannelName == "" {
+ defer func(start time.Time) { timing.TimedComponent("ensureChannelOwnership").Add(time.Since(start)) }(time.Now())
+
+ if s.DbChannelData.DesiredChannelName == "" {
return errors.Err("no channel name set")
}
@@ -336,27 +329,27 @@ func (s *Sync) ensureChannelOwnership() error {
var channelToUse *jsonrpc.Transaction
if len((*channels).Items) > 0 {
- if s.lbryChannelID == "" {
+ if s.DbChannelData.ChannelClaimID == "" {
return errors.Err("this channel does not have a recorded claimID in the database. To prevent failures, updates are not supported until an entry is manually added in the database")
}
for _, c := range (*channels).Items {
log.Debugf("checking listed channel %s (%s)", c.ClaimID, c.Name)
- if c.ClaimID != s.lbryChannelID {
+ if c.ClaimID != s.DbChannelData.ChannelClaimID {
continue
}
- if c.Name != s.LbryChannelName {
+ if c.Name != s.DbChannelData.DesiredChannelName {
return errors.Err("the channel in the wallet is different than the channel in the database")
}
channelToUse = &c
break
}
if channelToUse == nil {
- return errors.Err("this wallet has channels but not a single one is ours! Expected claim_id: %s (%s)", s.lbryChannelID, s.LbryChannelName)
+ return errors.Err("this wallet has channels but not a single one is ours! Expected claim_id: %s (%s)", s.DbChannelData.ChannelClaimID, s.DbChannelData.DesiredChannelName)
}
- } else if s.transferState == TransferStateComplete {
+ } else if s.DbChannelData.TransferState == shared.TransferStateComplete {
return errors.Err("the channel was transferred but appears to have been abandoned!")
- } else if s.lbryChannelID != "" {
- return errors.Err("the database has a channel recorded (%s) but nothing was found in our control", s.lbryChannelID)
+ } else if s.DbChannelData.ChannelClaimID != "" {
+ return errors.Err("the database has a channel recorded (%s) but nothing was found in our control", s.DbChannelData.ChannelClaimID)
}
channelUsesOldMetadata := false
@@ -386,36 +379,24 @@ func (s *Sync) ensureChannelOwnership() error {
return err
}
}
- client := &http.Client{
- Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
- }
- service, err := youtube.New(client)
+ channelInfo, err := ytapi.ChannelInfo(s.DbChannelData.ChannelId)
if err != nil {
- return errors.Prefix("error creating YouTube service", err)
+ return err
}
- response, err := service.Channels.List("snippet,brandingSettings").Id(s.YoutubeChannelID).Do()
- if err != nil {
- return errors.Prefix("error getting channel details", err)
- }
-
- if len(response.Items) < 1 {
- return errors.Err("youtube channel not found")
- }
-
- channelInfo := response.Items[0].Snippet
- channelBranding := response.Items[0].BrandingSettings
-
- thumbnail := thumbs.GetBestThumbnail(channelInfo.Thumbnails)
- thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.Url, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
+ thumbnail := channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails)-1].URL
+ thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail, s.DbChannelData.ChannelId, *s.Manager.AwsConfigs.GetS3AWSConfig())
if err != nil {
return err
}
var bannerURL *string
- if channelBranding.Image != nil && channelBranding.Image.BannerImageUrl != "" {
- bURL, err := thumbs.MirrorThumbnail(channelBranding.Image.BannerImageUrl, "banner-"+s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
+ if channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails != nil {
+ bURL, err := thumbs.MirrorThumbnail(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails)-1].URL,
+ "banner-"+s.DbChannelData.ChannelId,
+ *s.Manager.AwsConfigs.GetS3AWSConfig(),
+ )
if err != nil {
return err
}
@@ -423,28 +404,29 @@ func (s *Sync) ensureChannelOwnership() error {
}
var languages []string = nil
- if channelInfo.DefaultLanguage != "" {
- if channelInfo.DefaultLanguage == "iw" {
- channelInfo.DefaultLanguage = "he"
- }
- languages = []string{channelInfo.DefaultLanguage}
- }
+ //we don't have this data without the API
+ //if channelInfo.DefaultLanguage != "" {
+ // if channelInfo.DefaultLanguage == "iw" {
+ // channelInfo.DefaultLanguage = "he"
+ // }
+ // languages = []string{channelInfo.DefaultLanguage}
+ //}
var locations []jsonrpc.Location = nil
- if channelInfo.Country != "" {
- locations = []jsonrpc.Location{{Country: util.PtrToString(channelInfo.Country)}}
+ if channelInfo.Topbar.DesktopTopbarRenderer.CountryCode != "" {
+ locations = []jsonrpc.Location{{Country: &channelInfo.Topbar.DesktopTopbarRenderer.CountryCode}}
}
var c *jsonrpc.TransactionSummary
claimCreateOptions := jsonrpc.ClaimCreateOptions{
- Title: &channelInfo.Title,
- Description: &channelInfo.Description,
- Tags: tags_manager.GetTagsForChannel(s.YoutubeChannelID),
+ Title: &channelInfo.Microformat.MicroformatDataRenderer.Title,
+ Description: &channelInfo.Metadata.ChannelMetadataRenderer.Description,
+ Tags: tags_manager.GetTagsForChannel(s.DbChannelData.ChannelId),
Languages: languages,
Locations: locations,
ThumbnailURL: &thumbnailURL,
}
if channelUsesOldMetadata {
- if s.transferState <= 1 {
- c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
+ if s.DbChannelData.TransferState <= 1 {
+ c, err = s.daemon.ChannelUpdate(s.DbChannelData.ChannelClaimID, jsonrpc.ChannelUpdateOptions{
ClearTags: util.PtrToBool(true),
ClearLocations: util.PtrToBool(true),
ClearLanguages: util.PtrToBool(true),
@@ -454,11 +436,11 @@ func (s *Sync) ensureChannelOwnership() error {
},
})
} else {
- logUtils.SendInfoToSlack("%s (%s) has a channel with old metadata but isn't in our control anymore. Ignoring", s.LbryChannelName, s.lbryChannelID)
+ logUtils.SendInfoToSlack("%s (%s) has a channel with old metadata but isn't in our control anymore. Ignoring", s.DbChannelData.DesiredChannelName, s.DbChannelData.ChannelClaimID)
return nil
}
} else {
- c, err = s.daemon.ChannelCreate(s.LbryChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
+ c, err = s.daemon.ChannelCreate(s.DbChannelData.DesiredChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
ClaimCreateOptions: claimCreateOptions,
CoverURL: bannerURL,
})
@@ -467,8 +449,9 @@ func (s *Sync) ensureChannelOwnership() error {
if err != nil {
return err
}
- s.lbryChannelID = c.Outputs[0].ClaimID
- return s.Manager.apiConfig.SetChannelClaimID(s.YoutubeChannelID, s.lbryChannelID)
+
+ s.DbChannelData.ChannelClaimID = c.Outputs[0].ClaimID
+ return s.Manager.ApiConfig.SetChannelClaimID(s.DbChannelData.ChannelId, s.DbChannelData.ChannelClaimID)
}
func (s *Sync) addCredits(amountToAdd float64) error {
@@ -477,7 +460,7 @@ func (s *Sync) addCredits(amountToAdd float64) error {
timing.TimedComponent("addCredits").Add(time.Since(start))
}(start)
log.Printf("Adding %f credits", amountToAdd)
- lbrycrdd, err := logUtils.GetLbrycrdClient(s.LbrycrdString)
+ lbrycrdd, err := logUtils.GetLbrycrdClient(s.Manager.LbrycrdDsn)
if err != nil {
return err
}
diff --git a/manager/transfer.go b/manager/transfer.go
index 1fdd8fb..fde2530 100644
--- a/manager/transfer.go
+++ b/manager/transfer.go
@@ -10,7 +10,7 @@ import (
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/util"
- "github.com/lbryio/ytsync/v5/sdk"
+ "github.com/lbryio/ytsync/v5/shared"
"github.com/lbryio/ytsync/v5/timing"
log "github.com/sirupsen/logrus"
@@ -71,7 +71,10 @@ func abandonSupports(s *Sync) (float64, error) {
for page := uint64(1); page <= totalPages; page++ {
supports, err := s.daemon.SupportList(&defaultAccount, page, 50)
if err != nil {
- return 0, errors.Prefix("cannot list claims", err)
+ supports, err = s.daemon.SupportList(&defaultAccount, page, 50)
+ if err != nil {
+ return 0, errors.Prefix("cannot list supports", err)
+ }
}
allSupports = append(allSupports, (*supports).Items...)
totalPages = (*supports).TotalPages
@@ -97,7 +100,7 @@ func abandonSupports(s *Sync) (float64, error) {
//TODO: remove this once the SDK team fixes their RPC bugs....
s.daemon.SetRPCTimeout(60 * time.Second)
defer s.daemon.SetRPCTimeout(5 * time.Minute)
- for i := 0; i < s.ConcurrentVideos; i++ {
+ for i := 0; i < s.Manager.CliFlags.ConcurrentJobs; i++ {
consumerWG.Add(1)
go func() {
defer consumerWG.Done()
@@ -189,7 +192,7 @@ func abandonSupports(s *Sync) (float64, error) {
type updateInfo struct {
ClaimID string
streamUpdateOptions *jsonrpc.StreamUpdateOptions
- videoStatus *sdk.VideoStatus
+ videoStatus *shared.VideoStatus
}
func transferVideos(s *Sync) error {
@@ -199,7 +202,7 @@ func transferVideos(s *Sync) error {
}(start)
cleanTransfer := true
- streamChan := make(chan updateInfo, s.ConcurrentVideos)
+ streamChan := make(chan updateInfo, s.Manager.CliFlags.ConcurrentJobs)
account, err := s.getDefaultAccount()
if err != nil {
return err
@@ -213,13 +216,13 @@ func transferVideos(s *Sync) error {
go func() {
defer producerWG.Done()
for _, video := range s.syncedVideos {
- if !video.Published || video.Transferred || video.MetadataVersion != LatestMetadataVersion {
+ if !video.Published || video.Transferred || video.MetadataVersion != shared.LatestMetadataVersion {
continue
}
var stream *jsonrpc.Claim = nil
for _, c := range streams.Items {
- if c.ClaimID != video.ClaimID || (c.SigningChannel != nil && c.SigningChannel.ClaimID != s.lbryChannelID) {
+ if c.ClaimID != video.ClaimID || (c.SigningChannel != nil && c.SigningChannel.ClaimID != s.DbChannelData.ChannelClaimID) {
continue
}
stream = &c
@@ -232,7 +235,7 @@ func transferVideos(s *Sync) error {
streamUpdateOptions := jsonrpc.StreamUpdateOptions{
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
- ClaimAddress: &s.clientPublishAddress,
+ ClaimAddress: &s.DbChannelData.PublishAddress,
FundingAccountIDs: []string{
account,
},
@@ -240,12 +243,12 @@ func transferVideos(s *Sync) error {
},
Bid: util.PtrToString("0.005"), // Todo - Dont hardcode
}
- videoStatus := sdk.VideoStatus{
- ChannelID: s.YoutubeChannelID,
+ videoStatus := shared.VideoStatus{
+ ChannelID: s.DbChannelData.ChannelId,
VideoID: video.VideoID,
ClaimID: video.ClaimID,
ClaimName: video.ClaimName,
- Status: VideoStatusPublished,
+ Status: shared.VideoStatusPublished,
IsTransferred: util.PtrToBool(true),
}
streamChan <- updateInfo{
@@ -257,7 +260,7 @@ func transferVideos(s *Sync) error {
}()
consumerWG := &stop.Group{}
- for i := 0; i < s.ConcurrentVideos; i++ {
+ for i := 0; i < s.Manager.CliFlags.ConcurrentJobs; i++ {
consumerWG.Add(1)
go func(worker int) {
defer consumerWG.Done()
@@ -290,13 +293,13 @@ func (s *Sync) streamUpdate(ui *updateInfo) error {
timing.TimedComponent("transferStreamUpdate").Add(time.Since(start))
if updateError != nil {
ui.videoStatus.FailureReason = updateError.Error()
- ui.videoStatus.Status = VideoStatusTranferFailed
+ ui.videoStatus.Status = shared.VideoStatusTranferFailed
ui.videoStatus.IsTransferred = util.PtrToBool(false)
} else {
ui.videoStatus.IsTransferred = util.PtrToBool(len(result.Outputs) != 0)
}
log.Infof("TRANSFERRED %t", *ui.videoStatus.IsTransferred)
- statusErr := s.APIConfig.MarkVideoStatus(*ui.videoStatus)
+ statusErr := s.Manager.ApiConfig.MarkVideoStatus(*ui.videoStatus)
if statusErr != nil {
return errors.Prefix(statusErr.Error(), updateError)
}
@@ -318,7 +321,7 @@ func transferChannel(s *Sync) error {
}
var channelClaim *jsonrpc.Transaction = nil
for _, c := range channelClaims.Items {
- if c.ClaimID != s.lbryChannelID {
+ if c.ClaimID != s.DbChannelData.ChannelClaimID {
continue
}
channelClaim = &c
@@ -332,11 +335,11 @@ func transferChannel(s *Sync) error {
Bid: util.PtrToString(fmt.Sprintf("%.6f", channelClaimAmount-0.005)),
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
- ClaimAddress: &s.clientPublishAddress,
+ ClaimAddress: &s.DbChannelData.PublishAddress,
},
},
}
- result, err := s.daemon.ChannelUpdate(s.lbryChannelID, updateOptions)
+ result, err := s.daemon.ChannelUpdate(s.DbChannelData.ChannelClaimID, updateOptions)
if err != nil {
return errors.Err(err)
}
diff --git a/manager/ytsync.go b/manager/ytsync.go
index a1de795..b866be3 100644
--- a/manager/ytsync.go
+++ b/manager/ytsync.go
@@ -3,11 +3,9 @@ package manager
import (
"fmt"
"io/ioutil"
- "net/http"
"os"
"os/signal"
"runtime/debug"
- "sort"
"strconv"
"strings"
"sync"
@@ -17,25 +15,19 @@ import (
"github.com/lbryio/ytsync/v5/ip_manager"
"github.com/lbryio/ytsync/v5/namer"
"github.com/lbryio/ytsync/v5/sdk"
+ "github.com/lbryio/ytsync/v5/shared"
"github.com/lbryio/ytsync/v5/sources"
"github.com/lbryio/ytsync/v5/thumbs"
"github.com/lbryio/ytsync/v5/timing"
logUtils "github.com/lbryio/ytsync/v5/util"
+ "github.com/lbryio/ytsync/v5/ytapi"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/util"
- "github.com/aws/aws-sdk-go/aws"
- "github.com/aws/aws-sdk-go/aws/awserr"
- "github.com/aws/aws-sdk-go/aws/credentials"
- "github.com/aws/aws-sdk-go/aws/session"
- "github.com/aws/aws-sdk-go/service/s3"
- "github.com/aws/aws-sdk-go/service/s3/s3manager"
log "github.com/sirupsen/logrus"
- "google.golang.org/api/googleapi/transport"
- "google.golang.org/api/youtube/v3"
)
const (
@@ -47,52 +39,20 @@ const (
maxReasonLength = 500
)
-type video interface {
- Size() *int64
- ID() string
- IDAndNum() string
- PlaylistPosition() int
- PublishedAt() time.Time
- Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool, *sync.RWMutex) (*sources.SyncSummary, error)
-}
-
-// sorting videos
-type byPublishedAt []video
-
-func (a byPublishedAt) Len() int { return len(a) }
-func (a byPublishedAt) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
-func (a byPublishedAt) Less(i, j int) bool { return a[i].PublishedAt().Before(a[j].PublishedAt()) }
-
// Sync stores the options that control how syncing happens
type Sync struct {
- APIConfig *sdk.APIConfig
- YoutubeChannelID string
- LbryChannelName string
- MaxTries int
- ConcurrentVideos int
- Refill int
- Manager *SyncManager
- LbrycrdString string
- AwsS3ID string
- AwsS3Secret string
- AwsS3Region string
- AwsS3Bucket string
- Fee *sdk.Fee
- daemon *jsonrpc.Client
- claimAddress string
- videoDirectory string
- syncedVideosMux *sync.RWMutex
- syncedVideos map[string]sdk.SyncedVideo
- grp *stop.Group
- lbryChannelID string
- namer *namer.Namer
- walletMux *sync.RWMutex
- queue chan video
- transferState int
- clientPublishAddress string
- publicKey string
- defaultAccountID string
- MaxVideoLength float64
+ DbChannelData *shared.YoutubeChannel
+ Manager *SyncManager
+
+ daemon *jsonrpc.Client
+ videoDirectory string
+ syncedVideosMux *sync.RWMutex
+ syncedVideos map[string]sdk.SyncedVideo
+ grp *stop.Group
+ namer *namer.Namer
+ walletMux *sync.RWMutex
+ queue chan ytapi.Video
+ defaultAccountID string
}
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) {
@@ -119,118 +79,8 @@ func (s *Sync) IsInterrupted() bool {
}
}
-func (s *Sync) downloadWallet() error {
- defaultWalletDir, defaultTempWalletDir, key, err := s.getWalletPaths()
- if err != nil {
- return errors.Err(err)
- }
-
- creds := credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, "")
- s3Session, err := session.NewSession(&aws.Config{Region: aws.String(s.AwsS3Region), Credentials: creds})
- if err != nil {
- return errors.Prefix("error starting session: ", err)
- }
- downloader := s3manager.NewDownloader(s3Session)
- out, err := os.Create(defaultTempWalletDir)
- if err != nil {
- return errors.Prefix("error creating temp wallet: ", err)
- }
- defer out.Close()
-
- bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
- Bucket: aws.String(s.AwsS3Bucket),
- Key: key,
- })
- if err != nil {
- // Casting to the awserr.Error type will allow you to inspect the error
- // code returned by the service in code. The error code can be used
- // to switch on context specific functionality. In this case a context
- // specific error message is printed to the user based on the bucket
- // and key existing.
- //
- // For information on other S3 API error codes see:
- // http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
- if aerr, ok := err.(awserr.Error); ok {
- code := aerr.Code()
- if code == s3.ErrCodeNoSuchKey {
- return errors.Err("wallet not on S3")
- }
- }
- return err
- } else if bytesWritten == 0 {
- return errors.Err("zero bytes written")
- }
-
- err = os.Rename(defaultTempWalletDir, defaultWalletDir)
- if err != nil {
- return errors.Prefix("error replacing temp wallet for default wallet: ", err)
- }
-
- return nil
-}
-
-func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string, err error) {
-
- defaultWallet = os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
- tempWallet = os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet"
- key = aws.String("/wallets/" + s.YoutubeChannelID)
- if logUtils.IsRegTest() {
- defaultWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
- tempWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet"
- key = aws.String("/regtest/" + s.YoutubeChannelID)
- }
-
- walletPath := os.Getenv("LBRYNET_WALLETS_DIR")
- if walletPath != "" {
- defaultWallet = walletPath + "/wallets/default_wallet"
- tempWallet = walletPath + "/wallets/tmp_wallet"
- }
-
- if _, err := os.Stat(defaultWallet); !os.IsNotExist(err) {
- return "", "", nil, errors.Err("default_wallet already exists")
- }
- return
-}
-
-func (s *Sync) uploadWallet() error {
- defaultWalletDir := logUtils.GetDefaultWalletPath()
- key := aws.String("/wallets/" + s.YoutubeChannelID)
- if logUtils.IsRegTest() {
- key = aws.String("/regtest/" + s.YoutubeChannelID)
- }
-
- if _, err := os.Stat(defaultWalletDir); os.IsNotExist(err) {
- return errors.Err("default_wallet does not exist")
- }
-
- creds := credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, "")
- s3Session, err := session.NewSession(&aws.Config{Region: aws.String(s.AwsS3Region), Credentials: creds})
- if err != nil {
- return err
- }
-
- uploader := s3manager.NewUploader(s3Session)
-
- file, err := os.Open(defaultWalletDir)
- if err != nil {
- return err
- }
- defer file.Close()
-
- _, err = uploader.Upload(&s3manager.UploadInput{
- Bucket: aws.String(s.AwsS3Bucket),
- Key: key,
- Body: file,
- })
- if err != nil {
- return err
- }
-
- return os.Remove(defaultWalletDir)
-}
-
func (s *Sync) setStatusSyncing() error {
- syncedVideos, claimNames, err := s.Manager.apiConfig.SetChannelStatus(s.YoutubeChannelID, StatusSyncing, "", nil)
+ syncedVideos, claimNames, err := s.Manager.ApiConfig.SetChannelStatus(s.DbChannelData.ChannelId, shared.StatusSyncing, "", nil)
if err != nil {
return err
}
@@ -241,36 +91,27 @@ func (s *Sync) setStatusSyncing() error {
return nil
}
-func (s *Sync) setExceptions() {
- if s.YoutubeChannelID == "UCwjQfNRW6sGYb__pd7d4nUg" { //@FreeTalkLive
- s.MaxVideoLength = 9999.0 // skips max length checks
- s.Manager.maxVideoSize = 0
- }
-}
-
var stopGroup = stop.New()
func (s *Sync) FullCycle() (e error) {
if os.Getenv("HOME") == "" {
return errors.Err("no $HOME env var found")
}
- if s.YoutubeChannelID == "" {
- return errors.Err("channel ID not provided")
- }
-
- s.setExceptions()
defer timing.ClearTimings()
s.syncedVideosMux = &sync.RWMutex{}
s.walletMux = &sync.RWMutex{}
s.grp = stopGroup
- s.queue = make(chan video)
+ s.queue = make(chan ytapi.Video)
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
defer signal.Stop(interruptChan)
go func() {
<-interruptChan
+ util.SendToSlack("got interrupt, shutting down")
log.Println("Got interrupt signal, shutting down (if publishing, will shut down after current publish)")
s.grp.Stop()
+ time.Sleep(5 * time.Second)
+ debug.PrintStack() // so we can figure out what's not stopping
}()
err := s.setStatusSyncing()
if err != nil {
@@ -286,6 +127,10 @@ func (s *Sync) FullCycle() (e error) {
} else {
log.Println("Starting new wallet")
}
+ err = s.downloadBlockchainDB()
+ if err != nil {
+ return errors.Prefix("failure in downloading blockchain.db", err)
+ }
defer s.stopAndUploadWallet(&e)
@@ -328,16 +173,18 @@ func (s *Sync) FullCycle() (e error) {
func (s *Sync) processTransfers() (e error) {
log.Println("Processing transfers")
- err := waitConfirmations(s)
- if err != nil {
- return err
+ if s.DbChannelData.TransferState != 2 {
+ err := waitConfirmations(s)
+ if err != nil {
+ return err
+ }
}
supportAmount, err := abandonSupports(s)
if err != nil {
return errors.Prefix(fmt.Sprintf("%.6f LBCs were abandoned before failing", supportAmount), err)
}
if supportAmount > 0 {
- logUtils.SendInfoToSlack("(%s) %.6f LBCs were abandoned and should be used as support", s.YoutubeChannelID, supportAmount)
+ logUtils.SendInfoToSlack("(%s) %.6f LBCs were abandoned and should be used as support", s.DbChannelData.ChannelId, supportAmount)
}
err = transferVideos(s)
if err != nil {
@@ -358,19 +205,23 @@ func (s *Sync) processTransfers() (e error) {
return err
}
isTip := true
- summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount}, nil)
+ summary, err := s.daemon.SupportCreate(s.DbChannelData.ChannelClaimID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount}, nil)
if err != nil {
- if strings.Contains(err.Error(), "tx-size") { //TODO: this is a silly workaround and should be written in an recursive function
- summary, err = s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount/2.0), &isTip, nil, []string{defaultAccount}, nil)
+ if strings.Contains(err.Error(), "tx-size") { //TODO: this is a silly workaround...
+ _, spendErr := s.daemon.TxoSpend(util.PtrToString("other"), nil, nil, nil, nil, &s.defaultAccountID)
+ if spendErr != nil {
+ return errors.Prefix(fmt.Sprintf("something went wrong while tipping the channel for %.6f LBCs", supportAmount), err)
+ }
+ err = s.waitForNewBlock()
+ if err != nil {
+ return errors.Prefix(fmt.Sprintf("something went wrong while tipping the channel for %.6f LBCs (waiting for new block)", supportAmount), err)
+ }
+ summary, err = s.daemon.SupportCreate(s.DbChannelData.ChannelClaimID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount}, nil)
if err != nil {
return errors.Prefix(fmt.Sprintf("something went wrong while tipping the channel for %.6f LBCs", supportAmount), err)
}
- summary, err = s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount/2.0), &isTip, nil, []string{defaultAccount}, nil)
- if err != nil {
- return errors.Err(err)
- }
} else {
- return errors.Err(err)
+ return errors.Prefix(fmt.Sprintf("something went wrong while tipping the channel for %.6f LBCs", supportAmount), err)
}
}
if len(summary.Outputs) < 1 {
@@ -392,7 +243,7 @@ func deleteSyncFolder(videoDirectory string) {
}
func (s *Sync) shouldTransfer() bool {
- return s.transferState >= 1 && s.clientPublishAddress != "" && !s.Manager.SyncFlags.DisableTransfers
+ return s.DbChannelData.TransferState >= 1 && s.DbChannelData.PublishAddress != "" && !s.Manager.CliFlags.DisableTransfers && s.DbChannelData.TransferState != 3
}
func (s *Sync) setChannelTerminationStatus(e *error) {
@@ -400,7 +251,7 @@ func (s *Sync) setChannelTerminationStatus(e *error) {
if s.shouldTransfer() {
if *e == nil {
- transferState = util.PtrToInt(TransferStateComplete)
+ transferState = util.PtrToInt(shared.TransferStateComplete)
}
}
if *e != nil {
@@ -413,13 +264,13 @@ func (s *Sync) setChannelTerminationStatus(e *error) {
return
}
failureReason := (*e).Error()
- _, _, err := s.Manager.apiConfig.SetChannelStatus(s.YoutubeChannelID, StatusFailed, failureReason, transferState)
+ _, _, err := s.Manager.ApiConfig.SetChannelStatus(s.DbChannelData.ChannelId, shared.StatusFailed, failureReason, transferState)
if err != nil {
- msg := fmt.Sprintf("Failed setting failed state for channel %s", s.LbryChannelName)
+ msg := fmt.Sprintf("Failed setting failed state for channel %s", s.DbChannelData.DesiredChannelName)
*e = errors.Prefix(msg+err.Error(), *e)
}
} else if !s.IsInterrupted() {
- _, _, err := s.Manager.apiConfig.SetChannelStatus(s.YoutubeChannelID, StatusSynced, "", transferState)
+ _, _, err := s.Manager.ApiConfig.SetChannelStatus(s.DbChannelData.ChannelId, shared.StatusSynced, "", transferState)
if err != nil {
*e = err
}
@@ -440,7 +291,7 @@ func (s *Sync) waitForDaemonStart() error {
if err == nil && status.StartupStatus.Wallet && status.IsRunning {
return nil
}
- if time.Since(beginTime).Minutes() > 60 {
+ if time.Since(beginTime).Minutes() > 120 {
s.grp.Stop()
return errors.Err("the daemon is taking too long to start. Something is wrong")
}
@@ -466,7 +317,14 @@ func (s *Sync) stopAndUploadWallet(e *error) {
if err != nil {
if *e == nil {
e = &err
- return
+ } else {
+ *e = errors.Prefix("failure uploading wallet", *e)
+ }
+ }
+ err = s.uploadBlockchainDB()
+ if err != nil {
+ if *e == nil {
+ e = &err
} else {
*e = errors.Prefix("failure uploading wallet", *e)
}
@@ -515,7 +373,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
abandonedClaims := false
videoIDs := make(map[string]jsonrpc.Claim)
for _, c := range claims {
- if !isYtsyncClaim(c, s.lbryChannelID) {
+ if !isYtsyncClaim(c, s.DbChannelData.ChannelClaimID) {
continue
}
tn := c.Value.GetThumbnail().GetUrl()
@@ -533,7 +391,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
claimToAbandon = cl
videoIDs[videoID] = c
}
- if claimToAbandon.Address != s.clientPublishAddress && !s.syncedVideos[videoID].Transferred {
+ if claimToAbandon.Address != s.DbChannelData.PublishAddress && !s.syncedVideos[videoID].Transferred {
log.Debugf("abandoning %+v", claimToAbandon)
_, err := s.daemon.StreamAbandon(claimToAbandon.Txid, claimToAbandon.Nout, nil, false)
if err != nil {
@@ -562,7 +420,7 @@ type ytsyncClaim struct {
func (s *Sync) mapFromClaims(claims []jsonrpc.Claim) map[string]ytsyncClaim {
videoIDMap := make(map[string]ytsyncClaim, len(claims))
for _, c := range claims {
- if !isYtsyncClaim(c, s.lbryChannelID) {
+ if !isYtsyncClaim(c, s.DbChannelData.ChannelClaimID) {
continue
}
tn := c.Value.GetThumbnail().GetUrl()
@@ -602,8 +460,8 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
claimNameDiffers := claimInDatabase && sv.ClaimName != chainInfo.ClaimName
claimMarkedUnpublished := claimInDatabase && !sv.Published
_, isOwnClaim := ownClaimsInfo[videoID]
- tranferred := !isOwnClaim
- transferStatusMismatch := sv.Transferred != tranferred
+ transferred := !isOwnClaim || s.DbChannelData.TransferState == 3
+ transferStatusMismatch := sv.Transferred != transferred
if metadataDiffers {
log.Debugf("%s: Mismatch in database for metadata. DB: %d - Blockchain: %d", videoID, sv.MetadataVersion, chainInfo.MetadataVersion)
@@ -621,7 +479,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
log.Debugf("%s: Published but is not in database (%s - %s)", videoID, chainInfo.ClaimName, chainInfo.ClaimID)
}
if transferStatusMismatch {
- log.Debugf("%s: is marked as transferred %t on it's actually %t", videoID, sv.Transferred, tranferred)
+ log.Debugf("%s: is marked as transferred %t on it's actually %t", videoID, sv.Transferred, transferred)
}
if !claimInDatabase || metadataDiffers || claimIDDiffers || claimNameDiffers || claimMarkedUnpublished || transferStatusMismatch {
@@ -631,15 +489,15 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
}
fixed++
log.Debugf("updating %s in the database", videoID)
- err = s.Manager.apiConfig.MarkVideoStatus(sdk.VideoStatus{
- ChannelID: s.YoutubeChannelID,
+ err = s.Manager.ApiConfig.MarkVideoStatus(shared.VideoStatus{
+ ChannelID: s.DbChannelData.ChannelId,
VideoID: videoID,
- Status: VideoStatusPublished,
+ Status: shared.VideoStatusPublished,
ClaimID: chainInfo.ClaimID,
ClaimName: chainInfo.ClaimName,
Size: util.PtrToInt64(int64(claimSize)),
MetaDataVersion: chainInfo.MetadataVersion,
- IsTransferred: &tranferred,
+ IsTransferred: &transferred,
})
if err != nil {
return count, fixed, 0, err
@@ -680,13 +538,13 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
}
_, ok := ownClaimsInfo[vID]
if !ok && sv.Published {
- log.Debugf("%s: claims to be published but wasn't found in the list of claims and will be removed if --remove-db-unpublished was specified (%t)", vID, s.Manager.SyncFlags.RemoveDBUnpublished)
+ log.Debugf("%s: claims to be published but wasn't found in the list of claims and will be removed if --remove-db-unpublished was specified (%t)", vID, s.Manager.CliFlags.RemoveDBUnpublished)
idsToRemove = append(idsToRemove, vID)
}
}
- if s.Manager.SyncFlags.RemoveDBUnpublished && len(idsToRemove) > 0 {
+ if s.Manager.CliFlags.RemoveDBUnpublished && len(idsToRemove) > 0 {
log.Infof("removing: %s", strings.Join(idsToRemove, ","))
- err := s.Manager.apiConfig.DeleteVideos(idsToRemove)
+ err := s.Manager.ApiConfig.DeleteVideos(idsToRemove)
if err != nil {
return count, fixed, len(idsToRemove), err
}
@@ -717,7 +575,7 @@ func (s *Sync) getClaims(defaultOnly bool) ([]jsonrpc.Claim, error) {
}
items := make([]jsonrpc.Claim, 0, len(claims.Items))
for _, c := range claims.Items {
- if c.SigningChannel != nil && c.SigningChannel.ClaimID == s.lbryChannelID {
+ if c.SigningChannel != nil && c.SigningChannel.ClaimID == s.DbChannelData.ChannelClaimID {
items = append(items, c)
}
}
@@ -774,11 +632,11 @@ func (s *Sync) checkIntegrity() error {
}
if pubsOnWallet > pubsOnDB { //This case should never happen
- logUtils.SendInfoToSlack("We're claiming to have published %d videos but in reality we published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID)
+ logUtils.SendInfoToSlack("We're claiming to have published %d videos but in reality we published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId)
return errors.Err("not all published videos are in the database")
}
if pubsOnWallet < pubsOnDB {
- logUtils.SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID)
+ logUtils.SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId)
}
_, err = s.getUnsentSupports() //TODO: use the returned value when it works
@@ -811,24 +669,24 @@ func (s *Sync) doSync() error {
return err
}
- if s.transferState < TransferStateComplete {
- cert, err := s.daemon.ChannelExport(s.lbryChannelID, nil, nil)
+ if s.DbChannelData.TransferState < shared.TransferStateComplete {
+ cert, err := s.daemon.ChannelExport(s.DbChannelData.ChannelClaimID, nil, nil)
if err != nil {
return errors.Prefix("error getting channel cert", err)
}
if cert != nil {
- err = s.APIConfig.SetChannelCert(string(*cert), s.lbryChannelID)
+ err = s.Manager.ApiConfig.SetChannelCert(string(*cert), s.DbChannelData.ChannelClaimID)
if err != nil {
return errors.Prefix("error setting channel cert", err)
}
}
}
- if s.Manager.SyncFlags.StopOnError {
+ if s.Manager.CliFlags.StopOnError {
log.Println("Will stop publishing if an error is detected")
}
- for i := 0; i < s.ConcurrentVideos; i++ {
+ for i := 0; i < s.Manager.CliFlags.ConcurrentJobs; i++ {
s.grp.Add(1)
go func(i int) {
defer s.grp.Done()
@@ -836,7 +694,7 @@ func (s *Sync) doSync() error {
}(i)
}
- if s.LbryChannelName == "@UCBerkeley" {
+ if s.DbChannelData.DesiredChannelName == "@UCBerkeley" {
err = errors.Err("UCB is not supported in this version of YTSYNC")
} else {
err = s.enqueueYoutubeVideos()
@@ -847,7 +705,7 @@ func (s *Sync) doSync() error {
}
func (s *Sync) startWorker(workerNum int) {
- var v video
+ var v ytapi.Video
var more bool
for {
@@ -872,10 +730,17 @@ func (s *Sync) startWorker(workerNum int) {
tryCount := 0
for {
+ select { // check again inside the loop so this dies faster
+ case <-s.grp.Ch():
+ log.Printf("Stopping worker %d", workerNum)
+ return
+ default:
+ }
tryCount++
err := s.processVideo(v)
if err != nil {
+ util.SendToSlack("Tried to process %s. Error: %v", v.ID(), err)
logMsg := fmt.Sprintf("error processing video %s: %s", v.ID(), err.Error())
log.Errorln(logMsg)
if strings.Contains(strings.ToLower(err.Error()), "interrupted by user") {
@@ -891,9 +756,9 @@ func (s *Sync) startWorker(workerNum int) {
"Couldn't find private key for id",
"You already have a stream claim published under the name",
}
- if util.SubstringInSlice(err.Error(), fatalErrors) || s.Manager.SyncFlags.StopOnError {
+ if util.SubstringInSlice(err.Error(), fatalErrors) || s.Manager.CliFlags.StopOnError {
s.grp.Stop()
- } else if s.MaxTries > 1 {
+ } else if s.Manager.CliFlags.MaxTries > 1 {
errorsNoRetry := []string{
"non 200 status code received",
"This video contains content from",
@@ -923,7 +788,7 @@ func (s *Sync) startWorker(workerNum int) {
}
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
log.Println("This error should not be retried at all")
- } else if tryCount < s.MaxTries {
+ } else if tryCount < s.Manager.CliFlags.MaxTries {
if util.SubstringInSlice(err.Error(), []string{
"txn-mempool-conflict",
"too-long-mempool-chain",
@@ -972,14 +837,14 @@ func (s *Sync) startWorker(workerNum int) {
existingClaimSize = existingClaim.Size
}
}
- videoStatus := VideoStatusFailed
+ videoStatus := shared.VideoStatusFailed
if strings.Contains(err.Error(), "upgrade failed") {
- videoStatus = VideoStatusUpgradeFailed
+ videoStatus = shared.VideoStatusUpgradeFailed
} else {
s.AppendSyncedVideo(v.ID(), false, err.Error(), existingClaimName, existingClaimID, 0, existingClaimSize)
}
- err = s.Manager.apiConfig.MarkVideoStatus(sdk.VideoStatus{
- ChannelID: s.YoutubeChannelID,
+ err = s.Manager.ApiConfig.MarkVideoStatus(shared.VideoStatus{
+ ChannelID: s.DbChannelData.ChannelId,
VideoID: v.ID(),
Status: videoStatus,
ClaimID: existingClaimID,
@@ -996,107 +861,23 @@ func (s *Sync) startWorker(workerNum int) {
}
}
-var mostRecentlyFailedChannel string
-
func (s *Sync) enqueueYoutubeVideos() error {
- start := time.Now()
- defer func(start time.Time) {
- timing.TimedComponent("enqueueYoutubeVideos").Add(time.Since(start))
- }(start)
- client := &http.Client{
- Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
- }
+ defer func(start time.Time) { timing.TimedComponent("enqueueYoutubeVideos").Add(time.Since(start)) }(time.Now())
- service, err := youtube.New(client)
- if err != nil {
- return errors.Prefix("error creating YouTube service", err)
- }
-
- response, err := service.Channels.List("contentDetails").Id(s.YoutubeChannelID).Do()
- if err != nil {
- return errors.Prefix("error getting channels", err)
- }
-
- if len(response.Items) < 1 {
- return errors.Err("youtube channel not found")
- }
-
- if response.Items[0].ContentDetails.RelatedPlaylists == nil {
- return errors.Err("no related playlists")
- }
-
- playlistID := response.Items[0].ContentDetails.RelatedPlaylists.Uploads
- if playlistID == "" {
- return errors.Err("no channel playlist")
- }
-
- var videos []video
ipPool, err := ip_manager.GetIPPool(s.grp)
if err != nil {
return err
}
- playlistMap := make(map[string]*youtube.PlaylistItemSnippet, 50)
- nextPageToken := ""
- for {
- req := service.PlaylistItems.List("snippet").
- PlaylistId(playlistID).
- MaxResults(50).
- PageToken(nextPageToken)
-
- playlistResponse, err := req.Do()
- if err != nil {
- return errors.Prefix("error getting playlist items", err)
- }
-
- if len(playlistResponse.Items) < 1 {
- // If there are 50+ videos in a playlist but less than 50 are actually returned by the API, youtube will still redirect
- // clients to a next page. Such next page will however be empty. This logic prevents ytsync from failing.
- youtubeIsLying := len(videos) > 0
- if youtubeIsLying {
- break
- }
- if s.YoutubeChannelID == mostRecentlyFailedChannel {
- return errors.Err("playlist items not found")
- }
- mostRecentlyFailedChannel = s.YoutubeChannelID
- break //return errors.Err("playlist items not found") //TODO: will this work?
- }
- videoIDs := make([]string, 50)
- for i, item := range playlistResponse.Items {
- // normally we'd send the video into the channel here, but youtube api doesn't have sorting
- // so we have to get ALL the videos, then sort them, then send them in
- playlistMap[item.Snippet.ResourceId.VideoId] = item.Snippet
- videoIDs[i] = item.Snippet.ResourceId.VideoId
- }
- req2 := service.Videos.List("snippet,contentDetails,recordingDetails").Id(strings.Join(videoIDs[:], ","))
-
- videosListResponse, err := req2.Do()
- if err != nil {
- return errors.Prefix("error getting videos info", err)
- }
- for _, item := range videosListResponse.Items {
- videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig(), s.grp, ipPool))
- }
-
- log.Infof("Got info for %d videos from youtube API", len(videos))
-
- nextPageToken = playlistResponse.NextPageToken
- if nextPageToken == "" || s.Manager.SyncFlags.QuickSync || len(videos) >= s.Manager.videosLimit {
- break
- }
+ videos, err := ytapi.GetVideosToSync(s.Manager.ApiConfig, s.DbChannelData.ChannelId, s.syncedVideos, s.Manager.CliFlags.QuickSync, s.Manager.CliFlags.VideosLimit, ytapi.VideoParams{
+ VideoDir: s.videoDirectory,
+ S3Config: *s.Manager.AwsConfigs.GetS3AWSConfig(),
+ Stopper: s.grp,
+ IPPool: ipPool,
+ }, s.DbChannelData.LastUploadedVideo)
+ if err != nil {
+ return err
}
- for k, v := range s.syncedVideos {
- if !v.Published {
- continue
- }
- _, ok := playlistMap[k]
- if !ok {
- videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig(), s.grp, ipPool))
- }
-
- }
- sort.Sort(byPublishedAt(videos))
Enqueue:
for _, v := range videos {
@@ -1116,7 +897,7 @@ Enqueue:
return nil
}
-func (s *Sync) processVideo(v video) (err error) {
+func (s *Sync) processVideo(v ytapi.Video) (err error) {
defer func() {
if p := recover(); p != nil {
logUtils.SendErrorToSlack("Video processing panic! %s", debug.Stack())
@@ -1139,20 +920,9 @@ func (s *Sync) processVideo(v video) (err error) {
s.syncedVideosMux.RUnlock()
newMetadataVersion := int8(2)
alreadyPublished := ok && sv.Published
- videoRequiresUpgrade := ok && s.Manager.SyncFlags.UpgradeMetadata && sv.MetadataVersion < newMetadataVersion
+ videoRequiresUpgrade := ok && s.Manager.CliFlags.UpgradeMetadata && sv.MetadataVersion < newMetadataVersion
- neverRetryFailures := []string{
- "Error extracting sts from embedded url response",
- "Unable to extract signature tokens",
- "the video is too big to sync, skipping for now",
- "video is too long to process",
- "This video contains content from",
- "no compatible format available for this video",
- "Watch this video on YouTube.",
- "have blocked it on copyright grounds",
- "giving up after 0 fragment retries",
- "Sign in to confirm your age",
- }
+ neverRetryFailures := shared.NeverRetryFailures
if ok && !sv.Published && util.SubstringInSlice(sv.FailureReason, neverRetryFailures) {
log.Println(v.ID() + " can't ever be published")
return nil
@@ -1167,7 +937,7 @@ func (s *Sync) processVideo(v video) (err error) {
return nil
}
- if !videoRequiresUpgrade && v.PlaylistPosition() >= s.Manager.videosLimit {
+ if !videoRequiresUpgrade && v.PlaylistPosition() >= s.Manager.CliFlags.VideosLimit {
log.Println(v.ID() + " is old: skipping")
return nil
}
@@ -1180,13 +950,13 @@ func (s *Sync) processVideo(v video) (err error) {
return err
}
sp := sources.SyncParams{
- ClaimAddress: s.claimAddress,
+ ClaimAddress: s.DbChannelData.PublishAddress,
Amount: publishAmount,
- ChannelID: s.lbryChannelID,
- MaxVideoSize: s.Manager.maxVideoSize,
+ ChannelID: s.DbChannelData.ChannelClaimID,
+ MaxVideoSize: s.DbChannelData.SizeLimit,
Namer: s.namer,
- MaxVideoLength: s.MaxVideoLength,
- Fee: s.Fee,
+ MaxVideoLength: time.Duration(s.DbChannelData.LengthLimit) * time.Minute,
+ Fee: s.DbChannelData.Fee,
DefaultAccount: da,
}
@@ -1196,14 +966,14 @@ func (s *Sync) processVideo(v video) (err error) {
}
s.AppendSyncedVideo(v.ID(), true, "", summary.ClaimName, summary.ClaimID, newMetadataVersion, *v.Size())
- err = s.Manager.apiConfig.MarkVideoStatus(sdk.VideoStatus{
- ChannelID: s.YoutubeChannelID,
+ err = s.Manager.ApiConfig.MarkVideoStatus(shared.VideoStatus{
+ ChannelID: s.DbChannelData.ChannelId,
VideoID: v.ID(),
- Status: VideoStatusPublished,
+ Status: shared.VideoStatusPublished,
ClaimID: summary.ClaimID,
ClaimName: summary.ClaimName,
Size: v.Size(),
- MetaDataVersion: LatestMetadataVersion,
+ MetaDataVersion: shared.LatestMetadataVersion,
IsTransferred: util.PtrToBool(s.shouldTransfer()),
})
if err != nil {
@@ -1214,7 +984,7 @@ func (s *Sync) processVideo(v video) (err error) {
}
func (s *Sync) importPublicKey() error {
- if s.publicKey != "" {
+ if s.DbChannelData.PublicKey != "" {
accountsResponse, err := s.daemon.AccountList(1, 50)
if err != nil {
return errors.Err(err)
@@ -1225,13 +995,13 @@ func (s *Sync) importPublicKey() error {
}
for _, a := range accountsResponse.Items {
if *a.Ledger == ledger {
- if a.PublicKey == s.publicKey {
+ if a.PublicKey == s.DbChannelData.PublicKey {
return nil
}
}
}
- log.Infof("Could not find public key %s in the wallet. Importing it...", s.publicKey)
- _, err = s.daemon.AccountAdd(s.LbryChannelName, nil, nil, &s.publicKey, util.PtrToBool(true), nil)
+ log.Infof("Could not find public key %s in the wallet. Importing it...", s.DbChannelData.PublicKey)
+ _, err = s.daemon.AccountAdd(s.DbChannelData.DesiredChannelName, nil, nil, &s.DbChannelData.PublicKey, util.PtrToBool(true), nil)
return errors.Err(err)
}
return nil
@@ -1243,7 +1013,7 @@ func (s *Sync) getUnsentSupports() (float64, error) {
if err != nil {
return 0, errors.Err(err)
}
- if s.transferState == 2 {
+ if s.DbChannelData.TransferState == 2 {
balance, err := s.daemon.AccountBalance(&defaultAccount)
if err != nil {
return 0, err
@@ -1274,8 +1044,8 @@ func (s *Sync) getUnsentSupports() (float64, error) {
}
}
}
- if balanceAmount > 10 && sentSupports < 1 {
- logUtils.SendErrorToSlack("(%s) this channel has quite some LBCs in it (%.2f) and %.2f LBC in sent tips, it's likely that the tips weren't actually sent or the wallet has unnecessary extra credits in it", s.YoutubeChannelID, balanceAmount, sentSupports)
+ if balanceAmount > 10 && sentSupports < 1 && s.DbChannelData.TransferState > 1 {
+ logUtils.SendErrorToSlack("(%s) this channel has quite some LBCs in it (%.2f) and %.2f LBC in sent tips, it's likely that the tips weren't actually sent or the wallet has unnecessary extra credits in it", s.DbChannelData.ChannelId, balanceAmount, sentSupports)
return balanceAmount - 10, nil
}
}
@@ -1284,8 +1054,7 @@ func (s *Sync) getUnsentSupports() (float64, error) {
// waitForDaemonProcess observes the running processes and returns when the process is no longer running or when the timeout is up
func waitForDaemonProcess(timeout time.Duration) error {
- then := time.Now()
- stopTime := then.Add(time.Duration(timeout * time.Second))
+ stopTime := time.Now().Add(timeout * time.Second)
for !time.Now().After(stopTime) {
wait := 10 * time.Second
log.Println("the daemon is still running, waiting for it to exit")
diff --git a/sdk/api.go b/sdk/api.go
index 903229b..8dbb5ca 100644
--- a/sdk/api.go
+++ b/sdk/api.go
@@ -13,6 +13,7 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/null"
+ "github.com/lbryio/ytsync/v5/shared"
"github.com/lbryio/ytsync/v5/util"
@@ -30,58 +31,28 @@ type APIConfig struct {
HostName string
}
-type SyncProperties struct {
- SyncFrom int64
- SyncUntil int64
- YoutubeChannelID string
-}
-
-type SyncFlags struct {
- StopOnError bool
- TakeOverExistingChannel bool
- SkipSpaceCheck bool
- SyncUpdate bool
- SingleRun bool
- RemoveDBUnpublished bool
- UpgradeMetadata bool
- DisableTransfers bool
- QuickSync bool
-}
-
-type Fee struct {
- Amount string `json:"amount"`
- Address string `json:"address"`
- Currency string `json:"currency"`
-}
-type YoutubeChannel struct {
- ChannelId string `json:"channel_id"`
- TotalVideos uint `json:"total_videos"`
- TotalSubscribers uint `json:"total_subscribers"`
- DesiredChannelName string `json:"desired_channel_name"`
- Fee *Fee `json:"fee"`
- ChannelClaimID string `json:"channel_claim_id"`
- TransferState int `json:"transfer_state"`
- PublishAddress string `json:"publish_address"`
- PublicKey string `json:"public_key"`
-}
-
-func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeChannel, error) {
+func (a *APIConfig) FetchChannels(status string, cliFlags *shared.SyncFlags) ([]shared.YoutubeChannel, error) {
type apiJobsResponse struct {
- Success bool `json:"success"`
- Error null.String `json:"error"`
- Data []YoutubeChannel `json:"data"`
+ Success bool `json:"success"`
+ Error null.String `json:"error"`
+ Data []shared.YoutubeChannel `json:"data"`
}
endpoint := a.ApiURL + "/yt/jobs"
res, err := http.PostForm(endpoint, url.Values{
"auth_token": {a.ApiToken},
"sync_status": {status},
"min_videos": {strconv.Itoa(1)},
- "after": {strconv.Itoa(int(cp.SyncFrom))},
- "before": {strconv.Itoa(int(cp.SyncUntil))},
+ "after": {strconv.Itoa(int(cliFlags.SyncFrom))},
+ "before": {strconv.Itoa(int(cliFlags.SyncUntil))},
"sync_server": {a.HostName},
- "channel_id": {cp.YoutubeChannelID},
+ "channel_id": {cliFlags.ChannelID},
})
if err != nil {
+ if strings.Contains(err.Error(), "EOF") {
+ util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+ time.Sleep(30 * time.Second)
+ return a.FetchChannels(status, cliFlags)
+ }
return nil, errors.Err(err)
}
defer res.Body.Close()
@@ -90,7 +61,7 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(string(body))
time.Sleep(30 * time.Second)
- return a.FetchChannels(status, cp)
+ return a.FetchChannels(status, cliFlags)
}
var response apiJobsResponse
err = json.Unmarshal(body, &response)
@@ -126,7 +97,6 @@ func sanitizeFailureReason(s *string) {
}
func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
-
type apiSetChannelCertResponse struct {
Success bool `json:"success"`
Error null.String `json:"error"`
@@ -141,6 +111,11 @@ func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
"auth_token": {a.ApiToken},
})
if err != nil {
+ if strings.Contains(err.Error(), "EOF") {
+ util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+ time.Sleep(30 * time.Second)
+ return a.SetChannelCert(certHex, channelID)
+ }
return errors.Err(err)
}
defer res.Body.Close()
@@ -184,6 +159,11 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
}
res, err := http.PostForm(endpoint, params)
if err != nil {
+ if strings.Contains(err.Error(), "EOF") {
+ util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+ time.Sleep(30 * time.Second)
+ return a.SetChannelStatus(channelID, status, failureReason, transferState)
+ }
return nil, nil, errors.Err(err)
}
defer res.Body.Close()
@@ -229,6 +209,11 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
"channel_claim_id": {channelClaimID},
})
if err != nil {
+ if strings.Contains(err.Error(), "EOF") {
+ util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+ time.Sleep(30 * time.Second)
+ return a.SetChannelClaimID(channelID, channelClaimID)
+ }
return errors.Err(err)
}
defer res.Body.Close()
@@ -268,6 +253,11 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
}
res, err := http.PostForm(endpoint, vals)
if err != nil {
+ if strings.Contains(err.Error(), "EOF") {
+ util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+ time.Sleep(30 * time.Second)
+ return a.DeleteVideos(videos)
+ }
return errors.Err(err)
}
defer res.Body.Close()
@@ -297,19 +287,7 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
return errors.Err("invalid API response. Status code: %d", res.StatusCode)
}
-type VideoStatus struct {
- ChannelID string
- VideoID string
- Status string
- ClaimID string
- ClaimName string
- FailureReason string
- Size *int64
- MetaDataVersion uint
- IsTransferred *bool
-}
-
-func (a *APIConfig) MarkVideoStatus(status VideoStatus) error {
+func (a *APIConfig) MarkVideoStatus(status shared.VideoStatus) error {
endpoint := a.ApiURL + "/yt/video_status"
sanitizeFailureReason(&status.FailureReason)
@@ -341,6 +319,11 @@ func (a *APIConfig) MarkVideoStatus(status VideoStatus) error {
}
res, err := http.PostForm(endpoint, vals)
if err != nil {
+ if strings.Contains(err.Error(), "EOF") {
+ util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+ time.Sleep(30 * time.Second)
+ return a.MarkVideoStatus(status)
+ }
return errors.Err(err)
}
defer res.Body.Close()
@@ -368,3 +351,102 @@ func (a *APIConfig) MarkVideoStatus(status VideoStatus) error {
}
return errors.Err("invalid API response. Status code: %d", res.StatusCode)
}
+
+func (a *APIConfig) VideoState(videoID string) (string, error) {
+ endpoint := a.ApiURL + "/yt/video_state"
+ vals := url.Values{
+ "video_id": {videoID},
+ "auth_token": {a.ApiToken},
+ }
+
+ res, err := http.PostForm(endpoint, vals)
+ if err != nil {
+ if strings.Contains(err.Error(), "EOF") {
+ util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+ time.Sleep(30 * time.Second)
+ return a.VideoState(videoID)
+ }
+ return "", errors.Err(err)
+ }
+ defer res.Body.Close()
+ body, _ := ioutil.ReadAll(res.Body)
+ if res.StatusCode == http.StatusNotFound {
+ return "not_found", nil
+ }
+ if res.StatusCode != http.StatusOK {
+ util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
+ log.Debugln(string(body))
+ time.Sleep(30 * time.Second)
+ return a.VideoState(videoID)
+ }
+ var response struct {
+ Success bool `json:"success"`
+ Error null.String `json:"error"`
+ Data null.String `json:"data"`
+ }
+ err = json.Unmarshal(body, &response)
+ if err != nil {
+ return "", errors.Err(err)
+ }
+ if !response.Error.IsNull() {
+ return "", errors.Err(response.Error.String)
+ }
+ if !response.Data.IsNull() {
+ return response.Data.String, nil
+ }
+ return "", errors.Err("invalid API response. Status code: %d", res.StatusCode)
+}
+
+type VideoRelease struct {
+ ID uint64 `json:"id"`
+ YoutubeDataID uint64 `json:"youtube_data_id"`
+ VideoID string `json:"video_id"`
+ ReleaseTime string `json:"release_time"`
+ CreatedAt string `json:"created_at"`
+ UpdatedAt string `json:"updated_at"`
+}
+
+func (a *APIConfig) GetReleasedDate(videoID string) (*VideoRelease, error) {
+ endpoint := a.ApiURL + "/yt/released"
+ vals := url.Values{
+ "video_id": {videoID},
+ "auth_token": {a.ApiToken},
+ }
+
+ res, err := http.PostForm(endpoint, vals)
+ if err != nil {
+ if strings.Contains(err.Error(), "EOF") {
+ util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+ time.Sleep(30 * time.Second)
+ return a.GetReleasedDate(videoID)
+ }
+ return nil, errors.Err(err)
+ }
+ defer res.Body.Close()
+ body, _ := ioutil.ReadAll(res.Body)
+ if res.StatusCode == http.StatusNotFound {
+ return nil, nil
+ }
+ if res.StatusCode != http.StatusOK {
+ util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
+ log.Debugln(string(body))
+ time.Sleep(30 * time.Second)
+ return a.GetReleasedDate(videoID)
+ }
+ var response struct {
+ Success bool `json:"success"`
+ Error null.String `json:"error"`
+ Data VideoRelease `json:"data"`
+ }
+ err = json.Unmarshal(body, &response)
+ if err != nil {
+ return nil, errors.Err(err)
+ }
+ if !response.Error.IsNull() {
+ return nil, errors.Err(response.Error.String)
+ }
+ if response.Data.ReleaseTime != "" {
+ return &response.Data, nil
+ }
+ return nil, errors.Err("invalid API response. Status code: %d", res.StatusCode)
+}
diff --git a/shared/shared.go b/shared/shared.go
new file mode 100644
index 0000000..28b8077
--- /dev/null
+++ b/shared/shared.go
@@ -0,0 +1,125 @@
+package shared
+
+import (
+ "time"
+
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/aws/credentials"
+)
+
+type Fee struct {
+ Amount string `json:"amount"`
+ Address string `json:"address"`
+ Currency string `json:"currency"`
+}
+type YoutubeChannel struct {
+ ChannelId string `json:"channel_id"`
+ TotalVideos uint `json:"total_videos"`
+ TotalSubscribers uint `json:"total_subscribers"`
+ DesiredChannelName string `json:"desired_channel_name"`
+ Fee *Fee `json:"fee"`
+ ChannelClaimID string `json:"channel_claim_id"`
+ TransferState int `json:"transfer_state"`
+ PublishAddress string `json:"publish_address"`
+ PublicKey string `json:"public_key"`
+ LengthLimit int `json:"length_limit"`
+ SizeLimit int `json:"size_limit"`
+ LastUploadedVideo string `json:"last_uploaded_video"`
+}
+
+var NeverRetryFailures = []string{
+ "Error extracting sts from embedded url response",
+ "Unable to extract signature tokens",
+ "the video is too big to sync, skipping for now",
+ "video is too long to process",
+ "This video contains content from",
+ "no compatible format available for this video",
+ "Watch this video on YouTube.",
+ "have blocked it on copyright grounds",
+ "giving up after 0 fragment retries",
+ "Sign in to confirm your age",
+}
+
+type SyncFlags struct {
+ StopOnError bool
+ TakeOverExistingChannel bool
+ SkipSpaceCheck bool
+ SyncUpdate bool
+ SingleRun bool
+ RemoveDBUnpublished bool
+ UpgradeMetadata bool
+ DisableTransfers bool
+ QuickSync bool
+ MaxTries int
+ Refill int
+ Limit int
+ SyncStatus string
+ ChannelID string
+ SyncFrom int64
+ SyncUntil int64
+ ConcurrentJobs int
+ VideosLimit int
+ MaxVideoSize int
+ MaxVideoLength time.Duration
+}
+
+func (f *SyncFlags) IsSingleChannelSync() bool {
+ return f.ChannelID != ""
+}
+
+type VideoStatus struct {
+ ChannelID string
+ VideoID string
+ Status string
+ ClaimID string
+ ClaimName string
+ FailureReason string
+ Size *int64
+ MetaDataVersion uint
+ IsTransferred *bool
+}
+
+const (
+ StatusPending = "pending" // waiting for permission to sync
+ StatusPendingEmail = "pendingemail" // permission granted but missing email
+ StatusQueued = "queued" // in sync queue. will be synced soon
+ StatusPendingUpgrade = "pendingupgrade" // in sync queue. will be synced soon
+ StatusSyncing = "syncing" // syncing now
+ StatusSynced = "synced" // done
+ StatusFailed = "failed"
+ StatusFinalized = "finalized" // no more changes allowed
+ StatusAbandoned = "abandoned" // deleted on youtube or banned
+)
+
+var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned}
+
+const LatestMetadataVersion = 2
+
+const (
+ VideoStatusPublished = "published"
+ VideoStatusFailed = "failed"
+ VideoStatusUpgradeFailed = "upgradefailed"
+ VideoStatusUnpublished = "unpublished"
+ VideoStatusTranferFailed = "transferfailed"
+)
+
+const (
+ TransferStateNotTouched = iota
+ TransferStatePending
+ TransferStateComplete
+ TransferStateManual
+)
+
+type AwsConfigs struct {
+ AwsS3ID string
+ AwsS3Secret string
+ AwsS3Region string
+ AwsS3Bucket string
+}
+
+func (a *AwsConfigs) GetS3AWSConfig() *aws.Config {
+ return &aws.Config{
+ Credentials: credentials.NewStaticCredentials(a.AwsS3ID, a.AwsS3Secret, ""),
+ Region: &a.AwsS3Region,
+ }
+}
diff --git a/sources/youtubeVideo.go b/sources/youtubeVideo.go
index 26c2f66..6118745 100644
--- a/sources/youtubeVideo.go
+++ b/sources/youtubeVideo.go
@@ -3,7 +3,6 @@ package sources
import (
"fmt"
"io/ioutil"
- "math"
"os"
"os/exec"
"path/filepath"
@@ -13,24 +12,25 @@ import (
"sync"
"time"
- "github.com/lbryio/lbry.go/v2/extras/errors"
- "github.com/lbryio/lbry.go/v2/extras/jsonrpc"
- "github.com/lbryio/lbry.go/v2/extras/stop"
- "github.com/lbryio/lbry.go/v2/extras/util"
- "github.com/lbryio/ytsync/v5/timing"
- logUtils "github.com/lbryio/ytsync/v5/util"
+ "github.com/lbryio/ytsync/v5/downloader/ytdl"
+ "github.com/lbryio/ytsync/v5/shared"
"github.com/lbryio/ytsync/v5/ip_manager"
"github.com/lbryio/ytsync/v5/namer"
"github.com/lbryio/ytsync/v5/sdk"
"github.com/lbryio/ytsync/v5/tags_manager"
"github.com/lbryio/ytsync/v5/thumbs"
+ "github.com/lbryio/ytsync/v5/timing"
+ logUtils "github.com/lbryio/ytsync/v5/util"
+
+ "github.com/lbryio/lbry.go/v2/extras/errors"
+ "github.com/lbryio/lbry.go/v2/extras/jsonrpc"
+ "github.com/lbryio/lbry.go/v2/extras/stop"
+ "github.com/lbryio/lbry.go/v2/extras/util"
- duration "github.com/ChannelMeter/iso8601duration"
"github.com/aws/aws-sdk-go/aws"
"github.com/shopspring/decimal"
log "github.com/sirupsen/logrus"
- "google.golang.org/api/youtube/v3"
)
type YoutubeVideo struct {
@@ -40,10 +40,10 @@ type YoutubeVideo struct {
playlistPosition int64
size *int64
maxVideoSize int64
- maxVideoLength float64
+ maxVideoLength time.Duration
publishedAt time.Time
dir string
- youtubeInfo *youtube.Video
+ youtubeInfo *ytdl.YtdlVideo
youtubeChannelID string
tags []string
awsConfig aws.Config
@@ -90,22 +90,23 @@ var youtubeCategories = map[string]string{
"44": "trailers",
}
-func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
- publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
+func NewYoutubeVideo(directory string, videoData *ytdl.YtdlVideo, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) (*YoutubeVideo, error) {
+ // youtube-dl returns times in local timezone sometimes. this could break in the future
+ // maybe we can file a PR to choose the timezone we want from youtube-dl
return &YoutubeVideo{
- id: videoData.Id,
- title: videoData.Snippet.Title,
- description: videoData.Snippet.Description,
+ id: videoData.ID,
+ title: videoData.Title,
+ description: videoData.Description,
playlistPosition: playlistPosition,
- publishedAt: publishedAt,
+ publishedAt: videoData.UploadDateForReal,
dir: directory,
youtubeInfo: videoData,
awsConfig: awsConfig,
mocked: false,
- youtubeChannelID: videoData.Snippet.ChannelId,
+ youtubeChannelID: videoData.ChannelID,
stopGroup: stopGroup,
pool: pool,
- }
+ }, nil
}
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
return &YoutubeVideo{
@@ -185,7 +186,7 @@ func (v *YoutubeVideo) download() error {
defer func(start time.Time) {
timing.TimedComponent("download").Add(time.Since(start))
}(start)
- if v.youtubeInfo.Snippet.LiveBroadcastContent != "none" {
+ if v.youtubeInfo.IsLive != nil {
return errors.Err("video is a live stream and hasn't completed yet")
}
videoPath := v.getFullPath()
@@ -208,6 +209,15 @@ func (v *YoutubeVideo) download() error {
"480",
"320",
}
+ dur := time.Duration(v.youtubeInfo.Duration) * time.Second
+ if dur.Hours() > 2 { //for videos longer than 2 hours only sync up to 720p
+ qualities = []string{
+ "720",
+ "480",
+ "320",
+ }
+ }
+
ytdlArgs := []string{
"--no-progress",
"-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"),
@@ -218,12 +228,11 @@ func (v *YoutubeVideo) download() error {
"-movflags faststart",
"--abort-on-unavailable-fragment",
"--fragment-retries",
- "0",
- "--user-agent",
- "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36",
+ "1",
"--cookies",
"cookies.txt",
}
+ userAgent := []string{"--user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"}
if v.maxVideoSize > 0 {
ytdlArgs = append(ytdlArgs,
"--max-filesize",
@@ -233,7 +242,7 @@ func (v *YoutubeVideo) download() error {
if v.maxVideoLength > 0 {
ytdlArgs = append(ytdlArgs,
"--match-filter",
- fmt.Sprintf("duration <= %d", int(math.Round(v.maxVideoLength*3600))),
+ fmt.Sprintf("duration <= %d", int(v.maxVideoLength.Seconds())),
)
}
@@ -263,8 +272,10 @@ func (v *YoutubeVideo) download() error {
"https://www.youtube.com/watch?v="+v.ID(),
)
- for i, quality := range qualities {
+ for i := 0; i < len(qualities); i++ {
+ quality := qualities[i]
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+quality+"]+bestaudio[ext!=webm]")
+ argsWithFilters = append(argsWithFilters, userAgent...)
cmd := exec.Command("youtube-dl", argsWithFilters...)
log.Printf("Running command youtube-dl %s", strings.Join(argsWithFilters, " "))
@@ -293,6 +304,11 @@ func (v *YoutubeVideo) download() error {
return errors.Err(string(errorLog))
}
continue //this bypasses the yt throttling IP redistribution... TODO: don't
+ } else if strings.Contains(string(errorLog), "YouTube said: Unable to extract video data") && !strings.Contains(userAgent[1], "Googlebot") {
+ i-- //do not lower quality when trying a different user agent
+ userAgent = []string{"--user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}
+ log.Infof("trying different user agent for video %s", v.ID())
+ continue
}
return errors.Err(string(errorLog))
}
@@ -370,8 +386,8 @@ func (v *YoutubeVideo) delete(reason string) error {
}
func (v *YoutubeVideo) triggerThumbnailSave() (err error) {
- thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails)
- v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig)
+ thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails)
+ v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig)
return err
}
@@ -429,8 +445,8 @@ type SyncParams struct {
ChannelID string
MaxVideoSize int
Namer *namer.Namer
- MaxVideoLength float64
- Fee *sdk.Fee
+ MaxVideoLength time.Duration
+ Fee *shared.Fee
DefaultAccount string
}
@@ -448,13 +464,11 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
var err error
- videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
- if err != nil {
- return nil, errors.Err(err)
- }
- if videoDuration.ToDuration() > time.Duration(v.maxVideoLength*60)*time.Minute {
- log.Infof("%s is %s long and the limit is %s", v.id, videoDuration.ToDuration().String(), (time.Duration(v.maxVideoLength*60) * time.Minute).String())
- logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, videoDuration.ToDuration().String(), (time.Duration(v.maxVideoLength*60) * time.Minute).String())
+
+ dur := time.Duration(v.youtubeInfo.Duration) * time.Second
+
+ if dur > v.maxVideoLength {
+ logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, dur.String(), v.maxVideoLength.String())
return nil, errors.Err("video is too long to process")
}
for {
@@ -487,27 +501,30 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
locations = nil
tags = nil
if !v.mocked {
- if v.youtubeInfo.Snippet.DefaultLanguage != "" {
- if v.youtubeInfo.Snippet.DefaultLanguage == "iw" {
- v.youtubeInfo.Snippet.DefaultLanguage = "he"
- }
- languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
- }
+ /*
+ if v.youtubeInfo.Snippet.DefaultLanguage != "" {
+ if v.youtubeInfo.Snippet.DefaultLanguage == "iw" {
+ v.youtubeInfo.Snippet.DefaultLanguage = "he"
+ }
+ languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
+ }*/
- if v.youtubeInfo.RecordingDetails != nil && v.youtubeInfo.RecordingDetails.Location != nil {
+ /*if v.youtubeInfo.!= nil && v.youtubeInfo.RecordingDetails.Location != nil {
locations = []jsonrpc.Location{{
Latitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Latitude)),
Longitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Longitude)),
}}
- }
- tags = v.youtubeInfo.Snippet.Tags
+ }*/
+ tags = v.youtubeInfo.Tags
}
tags, err := tags_manager.SanitizeTags(tags, v.youtubeChannelID)
if err != nil {
log.Errorln(err.Error())
}
if !v.mocked {
- tags = append(tags, youtubeCategories[v.youtubeInfo.Snippet.CategoryId])
+ for _, category := range v.youtubeInfo.Categories {
+ tags = append(tags, youtubeCategories[category])
+ }
}
return languages, locations, tags
@@ -532,8 +549,8 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
if v.mocked {
return nil, errors.Err("could not find thumbnail for mocked video")
}
- thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails)
- thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig)
+ thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails)
+ thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig)
} else {
thumbnailURL = thumbs.ThumbnailEndpoint + v.ID()
}
@@ -605,14 +622,9 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
}, nil
}
- videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
- if err != nil {
- return nil, errors.Err(err)
- }
-
streamCreateOptions.ClaimCreateOptions.Title = &v.title
streamCreateOptions.ClaimCreateOptions.Description = util.PtrToString(v.getAbbrevDescription())
- streamCreateOptions.Duration = util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds())))
+ streamCreateOptions.Duration = util.PtrToUint64(uint64(v.youtubeInfo.Duration))
streamCreateOptions.ReleaseTime = util.PtrToInt64(v.publishedAt.Unix())
start := time.Now()
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
diff --git a/thumbs/uploader.go b/thumbs/uploader.go
index 6cf21cd..b61a05e 100644
--- a/thumbs/uploader.go
+++ b/thumbs/uploader.go
@@ -4,6 +4,9 @@ import (
"io"
"net/http"
"os"
+ "strings"
+
+ "github.com/lbryio/ytsync/v5/downloader/ytdl"
"github.com/lbryio/lbry.go/v2/extras/errors"
@@ -11,7 +14,6 @@ import (
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
log "github.com/sirupsen/logrus"
- "google.golang.org/api/youtube/v3"
)
type thumbnailUploader struct {
@@ -31,7 +33,9 @@ func (u *thumbnailUploader) downloadThumbnail() error {
return errors.Err(err)
}
defer img.Close()
-
+ if strings.HasPrefix(u.originalUrl, "//") {
+ u.originalUrl = "https:" + u.originalUrl
+ }
resp, err := http.Get(u.originalUrl)
if err != nil {
return errors.Err(err)
@@ -98,15 +102,14 @@ func MirrorThumbnail(url string, name string, s3Config aws.Config) (string, erro
return tu.mirroredUrl, nil
}
-func GetBestThumbnail(thumbnails *youtube.ThumbnailDetails) *youtube.Thumbnail {
- if thumbnails.Maxres != nil {
- return thumbnails.Maxres
- } else if thumbnails.High != nil {
- return thumbnails.High
- } else if thumbnails.Medium != nil {
- return thumbnails.Medium
- } else if thumbnails.Standard != nil {
- return thumbnails.Standard
+func GetBestThumbnail(thumbnails []ytdl.Thumbnail) *ytdl.Thumbnail {
+ var bestWidth *ytdl.Thumbnail
+ for _, thumbnail := range thumbnails {
+ if bestWidth == nil {
+ bestWidth = &thumbnail
+ } else if bestWidth.Width < thumbnail.Width {
+ bestWidth = &thumbnail
+ }
}
- return thumbnails.Default
+ return bestWidth
}
diff --git a/util/util.go b/util/util.go
index 096ec71..d3b1ab1 100644
--- a/util/util.go
+++ b/util/util.go
@@ -52,7 +52,7 @@ func GetLBRYNetDir() string {
}
func GetLbryumDir() string {
- lbryumDir := os.Getenv("LBRYNET_WALLETS_DIR")
+ lbryumDir := os.Getenv("LBRYUM_DIR")
if lbryumDir == "" {
usr, err := user.Current()
if err != nil {
@@ -255,7 +255,7 @@ func CleanupLbrynet() error {
}
return errors.Err(err)
}
- dbSizeLimit := int64(1 * 1024 * 1024 * 1024)
+ dbSizeLimit := int64(2 * 1024 * 1024 * 1024)
if db.Size() > dbSizeLimit {
files, err := filepath.Glob(lbryumDir + "/blockchain.db*")
if err != nil {
@@ -366,9 +366,31 @@ func GetDefaultWalletPath() string {
defaultWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
}
- walletPath := os.Getenv("LBRYNET_WALLETS_DIR")
+ walletPath := os.Getenv("LBRYUM_DIR")
if walletPath != "" {
defaultWalletDir = walletPath + "/wallets/default_wallet"
}
return defaultWalletDir
}
+func GetBlockchainDBPath() string {
+ lbryumDir := os.Getenv("LBRYUM_DIR")
+ if lbryumDir == "" {
+ if IsRegTest() {
+ lbryumDir = os.Getenv("HOME") + "/.lbryum_regtest"
+ } else {
+ lbryumDir = os.Getenv("HOME") + "/.lbryum"
+ }
+ }
+ defaultDB := lbryumDir + "/lbc_mainnet/blockchain.db"
+ if IsRegTest() {
+ defaultDB = lbryumDir + "/lbc_regtest/blockchain.db"
+ }
+ return defaultDB
+}
+func GetBlockchainDirectoryName() string {
+ ledger := "lbc_mainnet"
+ if IsRegTest() {
+ ledger = "lbc_regtest"
+ }
+ return ledger
+}
diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go
new file mode 100644
index 0000000..32b99f9
--- /dev/null
+++ b/ytapi/ytapi.go
@@ -0,0 +1,387 @@
+package ytapi
+
+import (
+ "bufio"
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "regexp"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/lbryio/ytsync/v5/shared"
+ logUtils "github.com/lbryio/ytsync/v5/util"
+
+ "github.com/lbryio/ytsync/v5/downloader/ytdl"
+
+ "github.com/lbryio/ytsync/v5/downloader"
+ "github.com/lbryio/ytsync/v5/ip_manager"
+ "github.com/lbryio/ytsync/v5/sdk"
+ "github.com/lbryio/ytsync/v5/sources"
+
+ "github.com/lbryio/lbry.go/v2/extras/errors"
+ "github.com/lbryio/lbry.go/v2/extras/jsonrpc"
+ "github.com/lbryio/lbry.go/v2/extras/stop"
+ "github.com/lbryio/lbry.go/v2/extras/util"
+
+ "github.com/aws/aws-sdk-go/aws"
+ log "github.com/sirupsen/logrus"
+)
+
+type Video interface {
+ Size() *int64
+ ID() string
+ IDAndNum() string
+ PlaylistPosition() int
+ PublishedAt() time.Time
+ Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool, *sync.RWMutex) (*sources.SyncSummary, error)
+}
+
+type byPublishedAt []Video
+
+func (a byPublishedAt) Len() int { return len(a) }
+func (a byPublishedAt) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a byPublishedAt) Less(i, j int) bool { return a[i].PublishedAt().Before(a[j].PublishedAt()) }
+
+type VideoParams struct {
+ VideoDir string
+ S3Config aws.Config
+ Stopper *stop.Group
+ IPPool *ip_manager.IPPool
+}
+
+var mostRecentlyFailedChannel string // TODO: fix this hack!
+
+func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams, lastUploadedVideo string) ([]Video, error) {
+ var videos []Video
+ if quickSync && maxVideos > 50 {
+ maxVideos = 50
+ }
+ allVideos, err := downloader.GetPlaylistVideoIDs(channelID, maxVideos, videoParams.Stopper.Ch(), videoParams.IPPool)
+ if err != nil {
+ return nil, errors.Err(err)
+ }
+ videoIDs := make([]string, 0, len(allVideos))
+ for _, video := range allVideos {
+ sv, ok := syncedVideos[video]
+ if ok && util.SubstringInSlice(sv.FailureReason, shared.NeverRetryFailures) {
+ continue
+ }
+ videoIDs = append(videoIDs, video)
+ }
+ log.Infof("Got info for %d videos from youtube downloader", len(videoIDs))
+
+ playlistMap := make(map[string]int64)
+ for i, videoID := range videoIDs {
+ playlistMap[videoID] = int64(i)
+ }
+ //this will ensure that we at least try to sync the video that was marked as last uploaded video in the database.
+ if lastUploadedVideo != "" {
+ _, ok := playlistMap[lastUploadedVideo]
+ if !ok {
+ playlistMap[lastUploadedVideo] = 0
+ videoIDs = append(videoIDs, lastUploadedVideo)
+ }
+ }
+
+ if len(videoIDs) < 1 {
+ if channelID == mostRecentlyFailedChannel {
+ return nil, errors.Err("playlist items not found")
+ }
+ mostRecentlyFailedChannel = channelID
+ }
+
+ vids, err := getVideos(config, channelID, videoIDs, videoParams.Stopper.Ch(), videoParams.IPPool)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, item := range vids {
+ positionInList := playlistMap[item.ID]
+ videoToAdd, err := sources.NewYoutubeVideo(videoParams.VideoDir, item, positionInList, videoParams.S3Config, videoParams.Stopper, videoParams.IPPool)
+ if err != nil {
+ return nil, errors.Err(err)
+ }
+ videos = append(videos, videoToAdd)
+ }
+
+ for k, v := range syncedVideos {
+ if !v.Published {
+ continue
+ }
+ if _, ok := playlistMap[k]; !ok {
+ videos = append(videos, sources.NewMockedVideo(videoParams.VideoDir, k, channelID, videoParams.S3Config, videoParams.Stopper, videoParams.IPPool))
+ }
+ }
+
+ sort.Sort(byPublishedAt(videos))
+
+ return videos, nil
+}
+
+// CountVideosInChannel is unused for now... keeping it here just in case
+func CountVideosInChannel(channelID string) (int, error) {
+ url := "https://socialblade.com/youtube/channel/" + channelID
+
+ req, _ := http.NewRequest("GET", url, nil)
+
+ req.Header.Add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36")
+ req.Header.Add("Accept", "*/*")
+ req.Header.Add("Host", "socialblade.com")
+
+ res, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return 0, errors.Err(err)
+ }
+ defer res.Body.Close()
+
+ var line string
+ scanner := bufio.NewScanner(res.Body)
+ for scanner.Scan() {
+ if strings.Contains(scanner.Text(), "youtube-stats-header-uploads") {
+ line = scanner.Text()
+ break
+ }
+ }
+
+ if err := scanner.Err(); err != nil {
+ return 0, err
+ }
+ if line == "" {
+ return 0, errors.Err("upload count line not found")
+ }
+
+ matches := regexp.MustCompile(">([0-9]+)<").FindStringSubmatch(line)
+ if len(matches) != 2 {
+ return 0, errors.Err("upload count not found with regex")
+ }
+
+ num, err := strconv.Atoi(matches[1])
+ if err != nil {
+ return 0, errors.Err(err)
+ }
+
+ return num, nil
+}
+
+func ChannelInfo(channelID string) (*YoutubeStatsResponse, error) {
+ url := "https://www.youtube.com/channel/" + channelID + "/about"
+
+ req, _ := http.NewRequest("GET", url, nil)
+
+ req.Header.Add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36")
+ req.Header.Add("Accept", "*/*")
+
+ res, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return nil, errors.Err(err)
+ }
+ defer res.Body.Close()
+ body, err := ioutil.ReadAll(res.Body)
+ if err != nil {
+ return nil, errors.Err(err)
+ }
+ pageBody := string(body)
+ dataStartIndex := strings.Index(pageBody, "window[\"ytInitialData\"] = ") + 26
+ dataEndIndex := strings.Index(pageBody, "]}}};") + 4
+
+ data := pageBody[dataStartIndex:dataEndIndex]
+ var decodedResponse YoutubeStatsResponse
+ err = json.Unmarshal([]byte(data), &decodedResponse)
+ if err != nil {
+ return nil, errors.Err(err)
+ }
+
+ return &decodedResponse, nil
+}
+
+func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool) ([]*ytdl.YtdlVideo, error) {
+ var videos []*ytdl.YtdlVideo
+ for _, videoID := range videoIDs {
+ if len(videoID) < 5 {
+ continue
+ }
+ select {
+ case <-stopChan:
+ return videos, errors.Err("interrupted by user")
+ default:
+ }
+
+ //ip, err := ipPool.GetIP(videoID)
+ //if err != nil {
+ // return nil, err
+ //}
+ //video, err := downloader.GetVideoInformation(videoID, &net.TCPAddr{IP: net.ParseIP(ip)})
+ state, err := config.VideoState(videoID)
+ if err != nil {
+ return nil, errors.Err(err)
+ }
+ if state == "published" {
+ continue
+ }
+ video, err := downloader.GetVideoInformation(config, videoID, stopChan, nil, ipPool)
+ if err != nil {
+ errSDK := config.MarkVideoStatus(shared.VideoStatus{
+ ChannelID: channelID,
+ VideoID: videoID,
+ Status: "failed",
+ FailureReason: err.Error(),
+ })
+ logUtils.SendErrorToSlack(fmt.Sprintf("Skipping video (%s): %s", videoID, errors.FullTrace(err)))
+ if errSDK != nil {
+ return nil, errors.Err(errSDK)
+ }
+ } else {
+ videos = append(videos, video)
+ }
+ }
+ return videos, nil
+}
+
+type YoutubeStatsResponse struct {
+ Contents struct {
+ TwoColumnBrowseResultsRenderer struct {
+ Tabs []struct {
+ TabRenderer struct {
+ Title string `json:"title"`
+ Selected bool `json:"selected"`
+ Content struct {
+ SectionListRenderer struct {
+ Contents []struct {
+ ItemSectionRenderer struct {
+ Contents []struct {
+ ChannelAboutFullMetadataRenderer struct {
+ Description struct {
+ SimpleText string `json:"simpleText"`
+ } `json:"description"`
+ ViewCountText struct {
+ SimpleText string `json:"simpleText"`
+ } `json:"viewCountText"`
+ JoinedDateText struct {
+ Runs []struct {
+ Text string `json:"text"`
+ } `json:"runs"`
+ } `json:"joinedDateText"`
+ CanonicalChannelURL string `json:"canonicalChannelUrl"`
+ BypassBusinessEmailCaptcha bool `json:"bypassBusinessEmailCaptcha"`
+ Title struct {
+ SimpleText string `json:"simpleText"`
+ } `json:"title"`
+ Avatar struct {
+ Thumbnails []struct {
+ URL string `json:"url"`
+ Width int `json:"width"`
+ Height int `json:"height"`
+ } `json:"thumbnails"`
+ } `json:"avatar"`
+ ShowDescription bool `json:"showDescription"`
+ DescriptionLabel struct {
+ Runs []struct {
+ Text string `json:"text"`
+ } `json:"runs"`
+ } `json:"descriptionLabel"`
+ DetailsLabel struct {
+ Runs []struct {
+ Text string `json:"text"`
+ } `json:"runs"`
+ } `json:"detailsLabel"`
+ ChannelID string `json:"channelId"`
+ } `json:"channelAboutFullMetadataRenderer"`
+ } `json:"contents"`
+ } `json:"itemSectionRenderer"`
+ } `json:"contents"`
+ } `json:"sectionListRenderer"`
+ } `json:"content"`
+ } `json:"tabRenderer"`
+ } `json:"tabs"`
+ } `json:"twoColumnBrowseResultsRenderer"`
+ } `json:"contents"`
+ Header struct {
+ C4TabbedHeaderRenderer struct {
+ ChannelID string `json:"channelId"`
+ Title string `json:"title"`
+ Avatar struct {
+ Thumbnails []struct {
+ URL string `json:"url"`
+ Width int `json:"width"`
+ Height int `json:"height"`
+ } `json:"thumbnails"`
+ } `json:"avatar"`
+ Banner struct {
+ Thumbnails []struct {
+ URL string `json:"url"`
+ Width int `json:"width"`
+ Height int `json:"height"`
+ } `json:"thumbnails"`
+ } `json:"banner"`
+ VisitTracking struct {
+ RemarketingPing string `json:"remarketingPing"`
+ } `json:"visitTracking"`
+ SubscriberCountText struct {
+ SimpleText string `json:"simpleText"`
+ } `json:"subscriberCountText"`
+ } `json:"c4TabbedHeaderRenderer"`
+ } `json:"header"`
+ Metadata struct {
+ ChannelMetadataRenderer struct {
+ Title string `json:"title"`
+ Description string `json:"description"`
+ RssURL string `json:"rssUrl"`
+ ChannelConversionURL string `json:"channelConversionUrl"`
+ ExternalID string `json:"externalId"`
+ Keywords string `json:"keywords"`
+ OwnerUrls []string `json:"ownerUrls"`
+ Avatar struct {
+ Thumbnails []struct {
+ URL string `json:"url"`
+ Width int `json:"width"`
+ Height int `json:"height"`
+ } `json:"thumbnails"`
+ } `json:"avatar"`
+ ChannelURL string `json:"channelUrl"`
+ IsFamilySafe bool `json:"isFamilySafe"`
+ VanityChannelURL string `json:"vanityChannelUrl"`
+ } `json:"channelMetadataRenderer"`
+ } `json:"metadata"`
+ Topbar struct {
+ DesktopTopbarRenderer struct {
+ CountryCode string `json:"countryCode"`
+ } `json:"desktopTopbarRenderer"`
+ } `json:"topbar"`
+ Microformat struct {
+ MicroformatDataRenderer struct {
+ URLCanonical string `json:"urlCanonical"`
+ Title string `json:"title"`
+ Description string `json:"description"`
+ Thumbnail struct {
+ Thumbnails []struct {
+ URL string `json:"url"`
+ Width int `json:"width"`
+ Height int `json:"height"`
+ } `json:"thumbnails"`
+ } `json:"thumbnail"`
+ SiteName string `json:"siteName"`
+ AppName string `json:"appName"`
+ AndroidPackage string `json:"androidPackage"`
+ IosAppStoreID string `json:"iosAppStoreId"`
+ IosAppArguments string `json:"iosAppArguments"`
+ OgType string `json:"ogType"`
+ URLApplinksWeb string `json:"urlApplinksWeb"`
+ URLApplinksIos string `json:"urlApplinksIos"`
+ URLApplinksAndroid string `json:"urlApplinksAndroid"`
+ URLTwitterIos string `json:"urlTwitterIos"`
+ URLTwitterAndroid string `json:"urlTwitterAndroid"`
+ TwitterCardType string `json:"twitterCardType"`
+ TwitterSiteHandle string `json:"twitterSiteHandle"`
+ SchemaDotOrgType string `json:"schemaDotOrgType"`
+ Noindex bool `json:"noindex"`
+ Unlisted bool `json:"unlisted"`
+ FamilySafe bool `json:"familySafe"`
+ Tags []string `json:"tags"`
+ } `json:"microformatDataRenderer"`
+ } `json:"microformat"`
+}
diff --git a/ytapi/ytapi_test.go b/ytapi/ytapi_test.go
new file mode 100644
index 0000000..93a612c
--- /dev/null
+++ b/ytapi/ytapi_test.go
@@ -0,0 +1,12 @@
+package ytapi
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestChannelInfo(t *testing.T) {
+ _, _, err := ChannelInfo("", "UCNQfQvFMPnInwsU_iGYArJQ")
+ assert.NoError(t, err)
+}