diff --git a/Makefile b/Makefile index e41cb2e..870e2b0 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ LDFLAGS = -ldflags "-X ${IMPORT_PATH}/meta.Version=${VERSION} -X ${IMPORT_PATH}/ build: - mkdir -p ${BIN_DIR} && CGO_ENABLED=0 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go + mkdir -p ${BIN_DIR} && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go clean: if [ -f ${BIN_DIR}/${BINARY} ]; then rm ${BIN_DIR}/${BINARY}; fi diff --git a/blobs_reflector/reflect.go b/blobs_reflector/reflect.go index 7758c24..81e3cd2 100644 --- a/blobs_reflector/reflect.go +++ b/blobs_reflector/reflect.go @@ -72,12 +72,12 @@ func reflectBlobs() error { return errors.Err(err) } } - st := store.NewDBBackedS3Store( + st := store.NewDBBackedStore( store.NewS3BlobStore(config.AwsID, config.AwsSecret, config.BucketRegion, config.BucketName), dbHandle) uploadWorkers := 10 - uploader := reflector.NewUploader(dbHandle, st, uploadWorkers, false) + uploader := reflector.NewUploader(dbHandle, st, uploadWorkers, false, false) usr, err := user.Current() if err != nil { return errors.Err(err) diff --git a/downloader/downloader.go b/downloader/downloader.go new file mode 100644 index 0000000..404bab6 --- /dev/null +++ b/downloader/downloader.go @@ -0,0 +1,364 @@ +package downloader + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net" + "net/http" + "net/url" + "os/exec" + "strings" + "time" + + "github.com/davecgh/go-spew/spew" + "github.com/lbryio/ytsync/v5/downloader/ytdl" + "github.com/lbryio/ytsync/v5/ip_manager" + "github.com/lbryio/ytsync/v5/sdk" + + "github.com/lbryio/lbry.go/v2/extras/errors" + "github.com/lbryio/lbry.go/v2/extras/stop" + "github.com/lbryio/lbry.go/v2/extras/util" + + "github.com/sirupsen/logrus" +) + +func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) { + args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist", "--cookies", "cookies.txt"} + ids, err := run(channelName, args, stopChan, pool) + if err != nil { + return nil, errors.Err(err) + } + videoIDs := make([]string, maxVideos) + for i, v := range ids { + logrus.Debugf("%d - video id %s", i, v) + if i >= maxVideos { + break + } + videoIDs[i] = v + } + return videoIDs, nil +} + +const releaseTimeFormat = "2006-01-02, 15:04:05 (MST)" + +func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Chan, ip *net.TCPAddr, pool *ip_manager.IPPool) (*ytdl.YtdlVideo, error) { + args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID, "--cookies", "cookies.txt"} + results, err := run(videoID, args, stopChan, pool) + if err != nil { + return nil, errors.Err(err) + } + var video *ytdl.YtdlVideo + err = json.Unmarshal([]byte(results[0]), &video) + if err != nil { + return nil, errors.Err(err) + } + + // now get an accurate time + const maxTries = 5 + tries := 0 +GetTime: + tries++ + t, err := getUploadTime(config, videoID, ip, video.UploadDate) + if err != nil { + //slack(":warning: Upload time error: %v", err) + if tries <= maxTries && (errors.Is(err, errNotScraped) || errors.Is(err, errUploadTimeEmpty) || errors.Is(err, errStatusParse) || errors.Is(err, errConnectionIssue)) { + err := triggerScrape(videoID, ip) + if err == nil { + time.Sleep(2 * time.Second) // let them scrape it + goto GetTime + } else { + //slack("triggering scrape returned error: %v", err) + } + } else if !errors.Is(err, errNotScraped) && !errors.Is(err, errUploadTimeEmpty) { + //slack(":warning: Error while trying to get accurate upload time for %s: %v", videoID, err) + if t == "" { + return nil, errors.Err(err) + } else { + t = "" //TODO: get rid of the other piece below? + } + } + // do fallback below + } + //slack("After all that, upload time for %s is %s", videoID, t) + + if t != "" { + parsed, err := time.Parse("2006-01-02, 15:04:05 (MST)", t) // this will probably be UTC, but Go's timezone parsing is fucked up. it ignores the timezone in the date + if err != nil { + return nil, errors.Err(err) + } + //slack(":exclamation: Got an accurate time for %s", videoID) + video.UploadDateForReal = parsed + } else { //TODO: this is the piece that isn't needed! + slack(":warning: Could not get accurate time for %s. Falling back to time from upload ytdl: %s.", videoID, video.UploadDate) + // fall back to UploadDate from youtube-dl + video.UploadDateForReal, err = time.Parse("20060102", video.UploadDate) + if err != nil { + return nil, err + } + } + + return video, nil +} + +var errNotScraped = errors.Base("not yet scraped by caa.iti.gr") +var errUploadTimeEmpty = errors.Base("upload time is empty") +var errStatusParse = errors.Base("could not parse status, got number, need string") +var errConnectionIssue = errors.Base("there was a connection issue with the api") + +func slack(format string, a ...interface{}) { + fmt.Printf(format+"\n", a...) + util.SendToSlack(format, a...) +} + +func triggerScrape(videoID string, ip *net.TCPAddr) error { + //slack("Triggering scrape for %s", videoID) + u, err := url.Parse("https://caa.iti.gr/verify_videoV3") + q := u.Query() + q.Set("twtimeline", "0") + q.Set("url", "https://www.youtube.com/watch?v="+videoID) + u.RawQuery = q.Encode() + //slack("GET %s", u.String()) + + client := getClient(ip) + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return errors.Err(err) + } + req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36") + + res, err := client.Do(req) + if err != nil { + return errors.Err(err) + } + defer res.Body.Close() + + var response struct { + Message string `json:"message"` + Status string `json:"status"` + VideoURL string `json:"video_url"` + } + err = json.NewDecoder(res.Body).Decode(&response) + if err != nil { + if strings.Contains(err.Error(), "cannot unmarshal number") { + return errors.Err(errStatusParse) + } + if strings.Contains(err.Error(), "no route to host") { + return errors.Err(errConnectionIssue) + } + return errors.Err(err) + } + + switch response.Status { + case "removed_video": + return errors.Err("video previously removed from service") + case "no_video": + return errors.Err("they say 'video cannot be found'. wtf?") + default: + spew.Dump(response) + } + + return nil + //https://caa.iti.gr/caa/api/v4/videos/reports/h-tuxHS5lSM +} + +func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr, uploadDate string) (string, error) { + //slack("Getting upload time for %s", videoID) + release, err := config.GetReleasedDate(videoID) + if err != nil { + logrus.Error(err) + } + ytdlUploadDate, err := time.Parse("20060102", uploadDate) + if err != nil { + logrus.Error(err) + } + if release != nil { + //const sqlTimeFormat = "2006-01-02 15:04:05" + sqlTime, err := time.ParseInLocation(time.RFC3339, release.ReleaseTime, time.UTC) + if err == nil { + if sqlTime.Day() != ytdlUploadDate.Day() { + logrus.Infof("upload day from APIs differs from the ytdl one by more than 1 day.") + } else { + return sqlTime.Format(releaseTimeFormat), nil + } + } else { + logrus.Error(err) + } + } + + if time.Now().AddDate(0, 0, -3).After(ytdlUploadDate) { + return ytdlUploadDate.Format(releaseTimeFormat), nil + } + client := getClient(ip) + req, err := http.NewRequest(http.MethodGet, "https://caa.iti.gr/get_verificationV3?url=https://www.youtube.com/watch?v="+videoID, nil) + if err != nil { + return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err) + } + req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36") + + res, err := client.Do(req) + if err != nil { + return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err) + } + defer res.Body.Close() + + var uploadTime struct { + Time string `json:"video_upload_time"` + Message string `json:"message"` + Status string `json:"status"` + } + err = json.NewDecoder(res.Body).Decode(&uploadTime) + if err != nil { + return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err) + } + + if uploadTime.Status == "ERROR1" { + return ytdlUploadDate.Format(releaseTimeFormat), errNotScraped + } + + if uploadTime.Status == "" && strings.HasPrefix(uploadTime.Message, "CANNOT_RETRIEVE_REPORT_FOR_VIDEO_") { + return ytdlUploadDate.Format(releaseTimeFormat), errors.Err("cannot retrieve report for video") + } + + if uploadTime.Time == "" { + return ytdlUploadDate.Format(releaseTimeFormat), errUploadTimeEmpty + } + + return uploadTime.Time, nil +} + +func getClient(ip *net.TCPAddr) *http.Client { + if ip == nil { + return http.DefaultClient + } + + return &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + LocalAddr: ip, + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + }, + } +} + +const ( + googleBotUA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" + chromeUA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36" + maxAttempts = 3 + extractionError = "YouTube said: Unable to extract video data" + throttledError = "HTTP Error 429" + AlternateThrottledError = "returned non-zero exit status 8" + youtubeDlError = "exit status 1" +) + +func run(use string, args []string, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) { + var useragent []string + var lastError error + for attempts := 0; attempts < maxAttempts; attempts++ { + sourceAddress, err := getIPFromPool(use, stopChan, pool) + if err != nil { + return nil, err + } + argsForCommand := append(args, "--source-address", sourceAddress) + argsForCommand = append(argsForCommand, useragent...) + cmd := exec.Command("youtube-dl", argsForCommand...) + + res, err := runCmd(cmd, stopChan) + pool.ReleaseIP(sourceAddress) + if err == nil { + return res, nil + } + lastError = err + if strings.Contains(err.Error(), youtubeDlError) { + if strings.Contains(err.Error(), extractionError) { + logrus.Warnf("known extraction error: %s", errors.FullTrace(err)) + useragent = nextUA(useragent) + } + if strings.Contains(err.Error(), throttledError) || strings.Contains(err.Error(), AlternateThrottledError) { + pool.SetThrottled(sourceAddress) + //we don't want throttle errors to count toward the max retries + attempts-- + } + } + } + return nil, lastError +} + +func nextUA(current []string) []string { + if len(current) == 0 { + return []string{"--user-agent", googleBotUA} + } + return []string{"--user-agent", chromeUA} +} + +func runCmd(cmd *exec.Cmd, stopChan stop.Chan) ([]string, error) { + logrus.Infof("running youtube-dl cmd: %s", strings.Join(cmd.Args, " ")) + var err error + stderr, err := cmd.StderrPipe() + if err != nil { + return nil, errors.Err(err) + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, errors.Err(err) + } + err = cmd.Start() + if err != nil { + return nil, errors.Err(err) + } + outLog, err := ioutil.ReadAll(stdout) + if err != nil { + return nil, errors.Err(err) + } + errorLog, err := ioutil.ReadAll(stderr) + if err != nil { + return nil, errors.Err(err) + } + done := make(chan error, 1) + go func() { + done <- cmd.Wait() + }() + + select { + case <-stopChan: + err := cmd.Process.Kill() + if err != nil { + return nil, errors.Prefix("failed to kill command after stopper cancellation", err) + } + return nil, errors.Err("canceled by stopper") + case err := <-done: + if err != nil { + return nil, errors.Prefix("youtube-dl "+strings.Join(cmd.Args, " ")+" ["+string(errorLog)+"]", err) + } + return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil + } +} + +func getIPFromPool(use string, stopChan stop.Chan, pool *ip_manager.IPPool) (sourceAddress string, err error) { + for { + sourceAddress, err = pool.GetIP(use) + if err != nil { + if errors.Is(err, ip_manager.ErrAllThrottled) { + select { + case <-stopChan: + return "", errors.Err("interrupted by user") + + default: + time.Sleep(ip_manager.IPCooldownPeriod) + continue + } + } else { + return "", err + } + } + break + } + return +} diff --git a/downloader/downloader_test.go b/downloader/downloader_test.go new file mode 100644 index 0000000..f4e981a --- /dev/null +++ b/downloader/downloader_test.go @@ -0,0 +1,42 @@ +package downloader + +import ( + "testing" + + "github.com/lbryio/ytsync/v5/sdk" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" +) + +func TestGetPlaylistVideoIDs(t *testing.T) { + videoIDs, err := GetPlaylistVideoIDs("UCJ0-OtVpF0wOKEqT2Z1HEtA", 50, nil, nil) + if err != nil { + logrus.Error(err) + } + for _, id := range videoIDs { + println(id) + } +} + +func TestGetVideoInformation(t *testing.T) { + video, err := GetVideoInformation(nil, "zj7pXM9gE5M", nil, nil, nil) + if err != nil { + logrus.Error(err) + } + if video != nil { + logrus.Info(video.ID) + } +} + +func Test_getUploadTime(t *testing.T) { + configs := sdk.APIConfig{ + YoutubeAPIKey: "", + ApiURL: "https://api.lbry.com", + ApiToken: "Ht4NETrL5oWKyAaZkuSV68BKhtXkiLh5", + HostName: "test", + } + got, err := getUploadTime(&configs, "kDGOHNpRjzc", nil, "20060102") + assert.NoError(t, err) + t.Log(got) + +} diff --git a/downloader/ytdl/Video.go b/downloader/ytdl/Video.go new file mode 100644 index 0000000..079d557 --- /dev/null +++ b/downloader/ytdl/Video.go @@ -0,0 +1,147 @@ +package ytdl + +import ( + "time" +) + +type YtdlVideo struct { + UploadDate string `json:"upload_date"` + UploadDateForReal time.Time // you need to manually set this since the value in the API doesn't include the time + Extractor string `json:"extractor"` + Series interface{} `json:"series"` + Format string `json:"format"` + Vbr interface{} `json:"vbr"` + Chapters interface{} `json:"chapters"` + Height int `json:"height"` + LikeCount interface{} `json:"like_count"` + Duration int `json:"duration"` + Fulltitle string `json:"fulltitle"` + PlaylistIndex interface{} `json:"playlist_index"` + Album interface{} `json:"album"` + ViewCount int `json:"view_count"` + Playlist interface{} `json:"playlist"` + Title string `json:"title"` + Filename string `json:"_filename"` + Creator interface{} `json:"creator"` + Ext string `json:"ext"` + ID string `json:"id"` + DislikeCount interface{} `json:"dislike_count"` + AverageRating float64 `json:"average_rating"` + Abr int `json:"abr"` + UploaderURL string `json:"uploader_url"` + Categories []string `json:"categories"` + Fps float64 `json:"fps"` + StretchedRatio interface{} `json:"stretched_ratio"` + SeasonNumber interface{} `json:"season_number"` + Annotations interface{} `json:"annotations"` + WebpageURLBasename string `json:"webpage_url_basename"` + Acodec string `json:"acodec"` + DisplayID string `json:"display_id"` + //RequestedFormats []RequestedFormat `json:"requested_formats"` + //AutomaticCaptions struct{} `json:"automatic_captions"` + Description string `json:"description"` + Tags []string `json:"tags"` + Track interface{} `json:"track"` + RequestedSubtitles interface{} `json:"requested_subtitles"` + StartTime interface{} `json:"start_time"` + Uploader string `json:"uploader"` + ExtractorKey string `json:"extractor_key"` + FormatID string `json:"format_id"` + EpisodeNumber interface{} `json:"episode_number"` + UploaderID string `json:"uploader_id"` + //Subtitles struct{} `json:"subtitles"` + ReleaseYear interface{} `json:"release_year"` + Thumbnails []Thumbnail `json:"thumbnails"` + License interface{} `json:"license"` + Artist interface{} `json:"artist"` + AgeLimit int `json:"age_limit"` + ReleaseDate interface{} `json:"release_date"` + AltTitle interface{} `json:"alt_title"` + Thumbnail string `json:"thumbnail"` + ChannelID string `json:"channel_id"` + IsLive interface{} `json:"is_live"` + Width int `json:"width"` + EndTime interface{} `json:"end_time"` + WebpageURL string `json:"webpage_url"` + //Formats []Format `json:"formats"` + ChannelURL string `json:"channel_url"` + Resolution interface{} `json:"resolution"` + Vcodec string `json:"vcodec"` +} + +type RequestedFormat struct { + Asr interface{} `json:"asr"` + Tbr float64 `json:"tbr"` + Container string `json:"container"` + Language interface{} `json:"language"` + Format string `json:"format"` + URL string `json:"url"` + Vcodec string `json:"vcodec"` + FormatNote string `json:"format_note"` + Height int `json:"height"` + Width int `json:"width"` + Ext string `json:"ext"` + FragmentBaseURL string `json:"fragment_base_url"` + Filesize interface{} `json:"filesize"` + Fps float64 `json:"fps"` + ManifestURL string `json:"manifest_url"` + Protocol string `json:"protocol"` + FormatID string `json:"format_id"` + HTTPHeaders struct { + AcceptCharset string `json:"Accept-Charset"` + AcceptLanguage string `json:"Accept-Language"` + AcceptEncoding string `json:"Accept-Encoding"` + Accept string `json:"Accept"` + UserAgent string `json:"User-Agent"` + } `json:"http_headers"` + Fragments []struct { + Path string `json:"path"` + Duration float64 `json:"duration,omitempty"` + } `json:"fragments"` + Acodec string `json:"acodec"` + Abr int `json:"abr,omitempty"` +} + +type Format struct { + Asr int `json:"asr"` + Tbr float64 `json:"tbr"` + Protocol string `json:"protocol"` + Format string `json:"format"` + FormatNote string `json:"format_note"` + Height interface{} `json:"height"` + ManifestURL string `json:"manifest_url,omitempty"` + FormatID string `json:"format_id"` + Container string `json:"container,omitempty"` + Language interface{} `json:"language,omitempty"` + HTTPHeaders HTTPHeaders `json:"http_headers"` + URL string `json:"url"` + Vcodec string `json:"vcodec"` + Abr int `json:"abr,omitempty"` + Width interface{} `json:"width"` + Ext string `json:"ext"` + FragmentBaseURL string `json:"fragment_base_url,omitempty"` + Filesize interface{} `json:"filesize"` + Fps float64 `json:"fps"` + Fragments []struct { + Path string `json:"path"` + Duration float64 `json:"duration,omitempty"` + } `json:"fragments,omitempty"` + Acodec string `json:"acodec"` + PlayerURL interface{} `json:"player_url,omitempty"` +} + +type Thumbnail struct { + URL string `json:"url"` + Width int `json:"width"` + Resolution string `json:"resolution"` + ID string `json:"id"` + Height int `json:"height"` +} + +type HTTPHeaders struct { + AcceptCharset string `json:"Accept-Charset"` + AcceptLanguage string `json:"Accept-Language"` + AcceptEncoding string `json:"Accept-Encoding"` + Accept string `json:"Accept"` + UserAgent string `json:"User-Agent"` +} diff --git a/e2e/data_setup.sh b/e2e/data_setup.sh index e15c1db..9b1ee17 100755 --- a/e2e/data_setup.sh +++ b/e2e/data_setup.sh @@ -19,6 +19,6 @@ mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER" ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(2,"youtubertoken")' mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN" #Add their youtube channel to be synced -ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable) -VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCNQfQvFMPnInwsU_iGYArJQ','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',10,10,1,1)" +ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable,total_views,reviewed,last_uploaded_video,length_limit,size_limit) +VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@test"$(date +%s)"','UCNQfQvFMPnInwsU_iGYArJQ','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',10,10,1,1,10000,1,'7bBV2Z-9wpo',60,2048)" mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL" diff --git a/e2e/e2e.sh b/e2e/e2e.sh index e051e27..f6fbc94 100755 --- a/e2e/e2e.sh +++ b/e2e/e2e.sh @@ -26,7 +26,7 @@ export REGTEST=true # Local settings export BLOBS_DIRECTORY="$(pwd)/e2e/blobsfiles" export LBRYNET_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbrynet/" -export LBRYNET_WALLETS_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum" +export LBRYUM_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum" export TMP_DIR="/var/tmp" export CHAINNAME="lbrycrd_regtest" export UID diff --git a/go.mod b/go.mod index 558dc66..da33ca3 100644 --- a/go.mod +++ b/go.mod @@ -3,25 +3,23 @@ module github.com/lbryio/ytsync/v5 replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 require ( - cloud.google.com/go v0.46.3 // indirect - github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 github.com/Microsoft/go-winio v0.4.14 // indirect - github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a + github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb github.com/aws/aws-sdk-go v1.25.9 - github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 // indirect + github.com/davecgh/go-spew v1.1.1 github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v1.13.1 github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect - github.com/go-sql-driver/mysql v1.4.1 // indirect - github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc // indirect + github.com/golang/protobuf v1.4.2 // indirect github.com/hashicorp/go-immutable-radix v1.1.0 // indirect github.com/hashicorp/go-sockaddr v1.0.2 // indirect - github.com/hashicorp/golang-lru v0.5.3 // indirect github.com/hashicorp/memberlist v0.1.5 // indirect github.com/hashicorp/serf v0.8.5 // indirect - github.com/lbryio/lbry.go/v2 v2.6.1-0.20200710180140-fcade7475323 - github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6 + github.com/kr/pretty v0.2.1 // indirect + github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128 + github.com/lbryio/reflector.go v1.1.3-0.20200901181534-e8b98bc862d5 + github.com/lbryio/types v0.0.0-20200605192618-366870b2862d // indirect github.com/miekg/dns v1.1.22 // indirect github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b github.com/opencontainers/go-digest v1.0.0-rc1 // indirect @@ -29,12 +27,14 @@ require ( github.com/prometheus/client_golang v0.9.2 github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5 github.com/sirupsen/logrus v1.4.2 + github.com/smartystreets/goconvey v1.6.4 // indirect github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 // indirect - go.opencensus.io v0.22.1 // indirect + github.com/stretchr/testify v1.4.0 + golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect - google.golang.org/api v0.11.0 google.golang.org/appengine v1.6.5 // indirect + gopkg.in/ini.v1 v1.60.2 // indirect ) go 1.13 diff --git a/go.sum b/go.sum index 5ba9dde..409d083 100644 --- a/go.sum +++ b/go.sum @@ -1,22 +1,20 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3 h1:AVXDdKsrtX33oR9fbCMu/+c1o8Ofjq6Ku/MInaLVg5Y= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= +dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= +dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= +dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= +dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= +git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:N5Vqww5QISEHsWHOWDEx4PzdIay3Cg0Jp7zItq2ZAro= -github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:GnKXcK+7DYNy/8w2Ex//Uql4IgfaU82Cd5rWKb7ah00= github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/StackExchange/wmi v0.0.0-20170410192909-ea383cf3ba6e/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -24,14 +22,16 @@ github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7 github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb h1:kvlW1qyM1aU3xeyeIVTU2jx5fSvjKpsU3aXvuaCMg3Q= +github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.25.9 h1:WtVzerf5wSgPwlTTwl+ktCq/0GCS5MI9ZlLIcjsTr+Q= github.com/aws/aws-sdk-go v1.25.9/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= @@ -44,19 +44,22 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:o64h9XF42kVEUuhuer2ehqrlX8rZmvQSU0+Vpj1rF6Q= -github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:Rp8e0DCtEKwXFOC6JPJQVTz8tuGoGvw6Xfexggh/ed0= +github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= +github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo= @@ -65,63 +68,79 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= +github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-ini/ini v1.48.0 h1:TvO60hO/2xgaaTWp2P0wUe4CFxwdMzfbkv3+343Xzqw= github.com/go-ini/ini v1.48.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= github.com/go-ozzo/ozzo-validation v3.6.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= -github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc h1:55rEp52jU6bkyslZ1+C/7NGfpQsEc6pxGLAGDOctqbw= -github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0 h1:Rd1kQnQu0Hq3qvJppYSG0HtP+f5LPPUiDswTLiEegLg= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/gops v0.3.7/go.mod h1:bj0cwMmX1X4XIJFTjR99R5sCxNssNJ8HebFNvoQlmgY= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f h1:TyqzGm2z1h3AGhjOoRYyeLcW4WlW81MDQkWa+rx/000= github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= +github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ= github.com/gorilla/rpc v1.2.0 h1:WvvdC2lNeT1SP32zrIce5l0ECBfbAlmrmSBsuc57wfk= github.com/gorilla/rpc v1.2.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ= +github.com/gorilla/schema v1.0.2/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= @@ -143,9 +162,10 @@ github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= @@ -162,55 +182,78 @@ github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7V github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmoiron/sqlx v0.0.0-20170430194603-d9bd385d68c0/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU= github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 h1:jKUP9TQ0c7X3w6+IPyMit07RE42MtTWNd77sN2cHngQ= github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22/go.mod h1:u0Jo4f2dNlTJeeOywkM6bLwxq6gC3pZ9rEFHn3AhTdk= +github.com/johntdyer/slackrus v0.0.0-20170926115001-3992f319fd0a/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo= github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07 h1:+kBG/8rjCa6vxJZbUjAiE4MQmBEBYc8nLEb51frnvBY= github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jteeuwen/go-bindata v3.0.7+incompatible/go.mod h1:JVvhzYOiGBnFSYRyV00iY8q7/0PThjIYav1p9h5dmKs= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/keybase/go-ps v0.0.0-20161005175911-668c8856d999/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M= +github.com/lbryio/chainquery v1.9.0 h1:NfBZ3eKYwD3PqXU/vt+2tF3ox3WUWoW4J5YdEQ0rxw0= +github.com/lbryio/chainquery v1.9.0/go.mod h1:7G8l7jNtANS1I7fQOvtzbiHsv6qKVmN4codXHc3C4kk= github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8= -github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303/go.mod h1:qR+Ui0hYhemIU4fXqM3d1P9eiaRFlof777VJgV7KJ8w= +github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c= github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU= -github.com/lbryio/lbry.go/v2 v2.6.1-0.20200710180140-fcade7475323 h1:Fvngg+hybcA1O2783YBw07S2MGpemSO+vWYSk97QHdM= -github.com/lbryio/lbry.go/v2 v2.6.1-0.20200710180140-fcade7475323/go.mod h1:LgFKEpZzJE72DVgSXXfg+2IGOhrC9Lzj6eLma18iNz8= +github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128 h1:VL209c+AGKixMFpxT+TsOAPzBPuoUzyjXf47iNe7OzY= +github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128/go.mod h1:RqOv4V5eWY/JGmduCPcQVdN19SEYnNY3SuF+arTKIU4= github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY= github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo= -github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46 h1:LemfR+rMxhf7nnOrzy2HqS7Me7SZ5gEwOcNFzKC8ySQ= github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo= github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4= -github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6 h1:8k12lI18EEgMHkgGrgtRvJGl2Bq1vDSEbE4s+QaQ5Qc= -github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6/go.mod h1:Q1Cnuv5iLsB2rS4Vr0o0EJr4Gs/EgDXFH2V4WtoM5o8= +github.com/lbryio/reflector.go v1.1.3-0.20200901181534-e8b98bc862d5 h1:Y+zuIZQ1QwpBKLsdfThALHMwQSR4i0H67Znen6YdT54= +github.com/lbryio/reflector.go v1.1.3-0.20200901181534-e8b98bc862d5/go.mod h1:QHKtgkmOCJJKZ578AFmN3jlBUhJFykbVEGgOuEiQ+aY= github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= -github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8 h1:jSNW/rK6DQsz7Zh+iv1zR384PeQdHt0gS4hKY17tkuM= github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= +github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= +github.com/lbryio/types v0.0.0-20200605192618-366870b2862d h1:Pg/4L7ExGXsseTMy1d/LK9TY55c8CP2gtmafmYzed0Q= +github.com/lbryio/types v0.0.0-20200605192618-366870b2862d/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE= +github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lucas-clemente/quic-go v0.17.2 h1:4iQInIuNQkPNZmsy9rCnwuOzpH0qGnDo4jn0QfI/qE4= +github.com/lucas-clemente/quic-go v0.17.2/go.mod h1:I0+fcNTdb9eS1ZcjQZbDVPGchJ86chcIxPALn9lEJqE= +github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0= github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg= github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcXMWfkoumVwhcCbf3jHF9QKv/m37BkM0= github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4= +github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/marten-seemann/qpack v0.1.0 h1:/0M7lkda/6mus9B8u34Asqm8ZhHAAt9Ho0vniNuVSVg= +github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI= +github.com/marten-seemann/qtls v0.9.1 h1:O0YKQxNVPaiFgMng0suWEOY2Sb4LT2sRn9Qimq3Z1IQ= +github.com/marten-seemann/qtls v0.9.1/go.mod h1:T1MmAdDPyISzxlK6kjRr0pcZFBVd1OZbBb/j3cvzHhk= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.22 h1:Jm64b3bO9kP43ddLjL2EY3Io6bmy1qGb9Xxz6TqS6rc= github.com/miekg/dns v1.1.22/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= @@ -225,21 +268,29 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F github.com/mitchellh/mapstructure v0.0.0-20180511142126-bb74f1db0675/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= +github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM= github.com/nlopes/slack v0.6.0 h1:jt0jxVQGhssx1Ib7naAOZEZcGdtIhTzkP0nopK0AsRA= github.com/nlopes/slack v0.6.0/go.mod h1:JzQ9m3PMAqcpeCam7UaHSuBuupz7CmpjehYMayT6YOk= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94= github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34= +github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= @@ -247,18 +298,22 @@ github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rK github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.3.0/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rubenv/sql-migrate v0.0.0-20170330050058-38004e7a77f2/go.mod h1:WS0rl9eEliYI8DPnr3TOwz4439pay+qNgzJoVya/DmY= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -269,11 +324,36 @@ github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77 h1:Msb6XRY62jQOueNNl github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sfreiberg/gotwilio v0.0.0-20180612161623-8fb7259ba8bf/go.mod h1:60PiR0SAnAcYSiwrXB6BaxeqHdXMf172toCosHfV+Yk= +github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shopspring/decimal v0.0.0-20180607144847-19e3cb6c2930/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5 h1:Gojs/hac/DoYEM7WEICT45+hNWczIeuL5D21e5/HPAw= github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= +github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= +github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= +github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= +github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI= +github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU= +github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag= +github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg= +github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw= +github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y= +github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= +github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ= +github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I= +github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0= +github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ= +github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk= +github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= +github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -281,19 +361,29 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= -github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= +github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= +github.com/spf13/afero v1.1.1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.0-20180722215644-7c4570c3ebeb/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= +github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -301,69 +391,77 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= -github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o= -github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= +github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= +github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d/go.mod h1:jspfvgf53t5NLUT4o9L1IX0kIBNKamGq1tWc/MgWK9Q= +github.com/volatiletech/null v8.0.0+incompatible/go.mod h1:0wD98JzdqB+rLyZ70fN05VDbXbafIb0KU0MdVhCzmOQ= +github.com/volatiletech/sqlboiler v3.4.0+incompatible/go.mod h1:jLfDkkHWPbS2cWRLkyC20vQWaIQsASEY7gM7zSo11Yw= +github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d h1:tQo6hjclyv3RHUgZOl6iWb2Y44A/sN9bf9LAYfuioEg= github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50= -go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= +github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= +go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM= +go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= +golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= -golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5hzyggAkLLlCUjqfRxd8Q4= golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5 h1:Q7tZBpemrlsc2I7IyODzhtallWRSm4Q0d09pL6XbQtU= +golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190228165749-92fc7df08ae7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191009170851-d66e71096ffb h1:TR699M2v0qoKTOHxeLgp6zPqaQNs74f01a/ob9W0qko= golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -371,92 +469,96 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20171017063910-8dbc5d05d6ed/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f h1:hjzMYz/7Ea1mNKfOnFOfktR0mlA5jqhvywClCMHM/qw= golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w= golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 h1:Toz2IK7k8rbltAXwNAxKcn9OzqyNfMUhUNjz3sL0NMk= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.11.0 h1:n/qM3q0/rV2F0pox7o0CvNhlPvZAo7pLbef122cbLJ0= -google.golang.org/api v0.11.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03 h1:4HYDjxeNXAOTv3o1N2tjo8UUSlhQgAD52FVkwxnWgM8= +google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= +google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gorp.v1 v1.7.1/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.48.0 h1:URjZc+8ugRY5mL5uUeQH/a63JcHwdX9xZaWvmNWD7z8= gopkg.in/ini.v1 v1.48.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.60.2 h1:7i8mqModL63zqi8nQn8Q3+0zvSCZy1AxhBgthKfi4WU= +gopkg.in/ini.v1 v1.60.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 h1:FpCr9V8wuOei4BAen+93HtVJ+XSi+KPbaPKm0Vj5R64= gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79/go.mod h1:gWkaRU7CoXpezCBWfWjm3999QqS+1pYPXGbqQCTMzo8= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= @@ -467,10 +569,13 @@ gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/goversion v1.0.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= +sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/ip_manager/throttle.go b/ip_manager/throttle.go index 43be077..e37ba69 100644 --- a/ip_manager/throttle.go +++ b/ip_manager/throttle.go @@ -63,21 +63,21 @@ func GetIPPool(stopGrp *stop.Group) (*IPPool, error) { lock: &sync.RWMutex{}, stopGrp: stopGrp, } - ticker := time.NewTicker(10 * time.Second) - go func() { - for { - select { - case <-stopGrp.Ch(): - return - case <-ticker.C: - ipPoolInstance.lock.RLock() - for _, ip := range ipPoolInstance.ips { - log.Debugf("IP: %s\tInUse: %t\tVideoID: %s\tThrottled: %t\tLastUse: %.1f", ip.IP, ip.InUse, ip.UsedForVideo, ip.Throttled, time.Since(ip.LastUse).Seconds()) - } - ipPoolInstance.lock.RUnlock() - } - } - }() + //ticker := time.NewTicker(10 * time.Second) + //go func() { + // for { + // select { + // case <-stopGrp.Ch(): + // return + // case <-ticker.C: + // ipPoolInstance.lock.RLock() + // for _, ip := range ipPoolInstance.ips { + // log.Debugf("IP: %s\tInUse: %t\tVideoID: %s\tThrottled: %t\tLastUse: %.1f", ip.IP, ip.InUse, ip.UsedForVideo, ip.Throttled, time.Since(ip.LastUse).Seconds()) + // } + // ipPoolInstance.lock.RUnlock() + // } + // } + //}() return ipPoolInstance, nil } @@ -108,7 +108,7 @@ func AllInUse(ips []throttledIP) bool { func (i *IPPool) ReleaseIP(ip string) { i.lock.Lock() defer i.lock.Unlock() - for j, _ := range i.ips { + for j := range i.ips { localIP := &i.ips[j] if localIP.IP == ip { localIP.InUse = false @@ -122,7 +122,7 @@ func (i *IPPool) ReleaseIP(ip string) { func (i *IPPool) ReleaseAll() { i.lock.Lock() defer i.lock.Unlock() - for j, _ := range i.ips { + for j := range i.ips { if i.ips[j].Throttled { continue } @@ -183,7 +183,7 @@ func (i *IPPool) nextIP(forVideo string) (*throttledIP, error) { } var nextIP *throttledIP - for j, _ := range i.ips { + for j := range i.ips { ip := &i.ips[j] if ip.InUse || ip.Throttled { continue diff --git a/main.go b/main.go index e9f9d3c..13080eb 100644 --- a/main.go +++ b/main.go @@ -11,6 +11,7 @@ import ( "github.com/lbryio/lbry.go/v2/extras/util" "github.com/lbryio/ytsync/v5/manager" "github.com/lbryio/ytsync/v5/sdk" + "github.com/lbryio/ytsync/v5/shared" ytUtils "github.com/lbryio/ytsync/v5/util" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -23,18 +24,8 @@ var Version string const defaultMaxTries = 3 var ( - flags sdk.SyncFlags - maxTries int - refill int - limit int - syncStatus string - channelID string - syncFrom int64 - syncUntil int64 - concurrentJobs int - videosLimit int - maxVideoSize int - maxVideoLength float64 + cliFlags shared.SyncFlags + maxVideoLength int ) func main() { @@ -51,25 +42,25 @@ func main() { Args: cobra.RangeArgs(0, 0), } - cmd.Flags().BoolVar(&flags.StopOnError, "stop-on-error", false, "If a publish fails, stop all publishing and exit") - cmd.Flags().IntVar(&maxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails") - cmd.Flags().BoolVar(&flags.TakeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel") - cmd.Flags().IntVar(&limit, "limit", 0, "limit the amount of channels to sync") - cmd.Flags().BoolVar(&flags.SkipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup") - cmd.Flags().BoolVar(&flags.SyncUpdate, "update", false, "Update previously synced channels instead of syncing new ones") - cmd.Flags().BoolVar(&flags.SingleRun, "run-once", false, "Whether the process should be stopped after one cycle or not") - cmd.Flags().BoolVar(&flags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published") - cmd.Flags().BoolVar(&flags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version") - cmd.Flags().BoolVar(&flags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports") - cmd.Flags().BoolVar(&flags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube") - cmd.Flags().StringVar(&syncStatus, "status", "", "Specify which queue to pull from. Overrides --update") - cmd.Flags().StringVar(&channelID, "channelID", "", "If specified, only this channel will be synced.") - cmd.Flags().Int64Var(&syncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)") - cmd.Flags().Int64Var(&syncUntil, "before", time.Now().AddDate(1, 0, 0).Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)") - cmd.Flags().IntVar(&concurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently") - cmd.Flags().IntVar(&videosLimit, "videos-limit", 1000, "how many videos to process per channel") - cmd.Flags().IntVar(&maxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)") - cmd.Flags().Float64Var(&maxVideoLength, "max-length", 2.0, "Maximum video length to process (in hours)") + cmd.Flags().BoolVar(&cliFlags.StopOnError, "stop-on-error", false, "If a publish fails, stop all publishing and exit") + cmd.Flags().IntVar(&cliFlags.MaxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails") + cmd.Flags().BoolVar(&cliFlags.TakeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel") + cmd.Flags().IntVar(&cliFlags.Limit, "limit", 0, "limit the amount of channels to sync") + cmd.Flags().BoolVar(&cliFlags.SkipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup") + cmd.Flags().BoolVar(&cliFlags.SyncUpdate, "update", false, "Update previously synced channels instead of syncing new ones") + cmd.Flags().BoolVar(&cliFlags.SingleRun, "run-once", false, "Whether the process should be stopped after one cycle or not") + cmd.Flags().BoolVar(&cliFlags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published") + cmd.Flags().BoolVar(&cliFlags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version") + cmd.Flags().BoolVar(&cliFlags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports") + cmd.Flags().BoolVar(&cliFlags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube") + cmd.Flags().StringVar(&cliFlags.SyncStatus, "status", "", "Specify which queue to pull from. Overrides --update") + cmd.Flags().StringVar(&cliFlags.ChannelID, "channelID", "", "If specified, only this channel will be synced.") + cmd.Flags().Int64Var(&cliFlags.SyncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)") + cmd.Flags().Int64Var(&cliFlags.SyncUntil, "before", time.Now().AddDate(1, 0, 0).Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)") + cmd.Flags().IntVar(&cliFlags.ConcurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently") + cmd.Flags().IntVar(&cliFlags.VideosLimit, "videos-limit", 1000, "how many videos to process per channel") + cmd.Flags().IntVar(&cliFlags.MaxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)") + cmd.Flags().IntVar(&maxVideoLength, "max-length", 2, "Maximum video length to process (in hours)") if err := cmd.Execute(); err != nil { fmt.Println(err) @@ -96,29 +87,30 @@ func ytSync(cmd *cobra.Command, args []string) { util.InitSlack(os.Getenv("SLACK_TOKEN"), os.Getenv("SLACK_CHANNEL"), hostname) } - if syncStatus != "" && !util.InSlice(syncStatus, manager.SyncStatuses) { - log.Errorf("status must be one of the following: %v\n", manager.SyncStatuses) + if cliFlags.SyncStatus != "" && !util.InSlice(cliFlags.SyncStatus, shared.SyncStatuses) { + log.Errorf("status must be one of the following: %v\n", shared.SyncStatuses) return } - if flags.StopOnError && maxTries != defaultMaxTries { + if cliFlags.StopOnError && cliFlags.MaxTries != defaultMaxTries { log.Errorln("--stop-on-error and --max-tries are mutually exclusive") return } - if maxTries < 1 { + if cliFlags.MaxTries < 1 { log.Errorln("setting --max-tries less than 1 doesn't make sense") return } - if limit < 0 { + if cliFlags.Limit < 0 { log.Errorln("setting --limit less than 0 (unlimited) doesn't make sense") return } + cliFlags.MaxVideoLength = time.Duration(maxVideoLength) * time.Hour apiURL := os.Getenv("LBRY_WEB_API") apiToken := os.Getenv("LBRY_API_TOKEN") youtubeAPIKey := os.Getenv("YOUTUBE_API_KEY") - lbrycrdString := os.Getenv("LBRYCRD_STRING") + lbrycrdDsn := os.Getenv("LBRYCRD_STRING") awsS3ID := os.Getenv("AWS_S3_ID") awsS3Secret := os.Getenv("AWS_S3_SECRET") awsS3Region := os.Getenv("AWS_S3_REGION") @@ -151,42 +143,30 @@ func ytSync(cmd *cobra.Command, args []string) { log.Errorln("AWS S3 Bucket was not defined. Please set the environment variable AWS_S3_BUCKET") return } - if lbrycrdString == "" { + if lbrycrdDsn == "" { log.Infoln("Using default (local) lbrycrd instance. Set LBRYCRD_STRING if you want to use something else") } blobsDir := ytUtils.GetBlobsDir() - syncProperties := &sdk.SyncProperties{ - SyncFrom: syncFrom, - SyncUntil: syncUntil, - YoutubeChannelID: channelID, - } apiConfig := &sdk.APIConfig{ YoutubeAPIKey: youtubeAPIKey, ApiURL: apiURL, ApiToken: apiToken, HostName: hostname, } + awsConfig := &shared.AwsConfigs{ + AwsS3ID: awsS3ID, + AwsS3Secret: awsS3Secret, + AwsS3Region: awsS3Region, + AwsS3Bucket: awsS3Bucket, + } sm := manager.NewSyncManager( - flags, - maxTries, - refill, - limit, - concurrentJobs, - concurrentJobs, + cliFlags, blobsDir, - videosLimit, - maxVideoSize, - lbrycrdString, - awsS3ID, - awsS3Secret, - awsS3Region, - awsS3Bucket, - syncStatus, - syncProperties, + lbrycrdDsn, + awsConfig, apiConfig, - maxVideoLength, ) err := sm.Start() if err != nil { diff --git a/manager/count.go b/manager/count.go deleted file mode 100644 index c3136bc..0000000 --- a/manager/count.go +++ /dev/null @@ -1,32 +0,0 @@ -package manager - -import ( - "net/http" - - "github.com/lbryio/lbry.go/v2/extras/errors" - - "google.golang.org/api/googleapi/transport" - "google.golang.org/api/youtube/v3" -) - -func (s *Sync) CountVideos() (uint64, error) { - client := &http.Client{ - Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey}, - } - - service, err := youtube.New(client) - if err != nil { - return 0, errors.Prefix("error creating YouTube service", err) - } - - response, err := service.Channels.List("statistics").Id(s.YoutubeChannelID).Do() - if err != nil { - return 0, errors.Prefix("error getting channels", err) - } - - if len(response.Items) < 1 { - return 0, errors.Err("youtube channel not found") - } - - return response.Items[0].Statistics.VideoCount, nil -} diff --git a/manager/manager.go b/manager/manager.go index 2f831b9..8a09b77 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -10,94 +10,43 @@ import ( "github.com/lbryio/ytsync/v5/ip_manager" "github.com/lbryio/ytsync/v5/namer" "github.com/lbryio/ytsync/v5/sdk" + "github.com/lbryio/ytsync/v5/shared" logUtils "github.com/lbryio/ytsync/v5/util" "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/extras/util" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" log "github.com/sirupsen/logrus" ) type SyncManager struct { - SyncFlags sdk.SyncFlags - maxTries int - refill int - limit int - concurrentJobs int - concurrentVideos int - blobsDir string - videosLimit int - maxVideoSize int - maxVideoLength float64 - lbrycrdString string - awsS3ID string - awsS3Secret string - awsS3Region string - syncStatus string - awsS3Bucket string - syncProperties *sdk.SyncProperties - apiConfig *sdk.APIConfig + CliFlags shared.SyncFlags + ApiConfig *sdk.APIConfig + LbrycrdDsn string + AwsConfigs *shared.AwsConfigs + + blobsDir string + channelsToSync []Sync } -func NewSyncManager(syncFlags sdk.SyncFlags, maxTries int, refill int, limit int, concurrentJobs int, concurrentVideos int, blobsDir string, videosLimit int, - maxVideoSize int, lbrycrdString string, awsS3ID string, awsS3Secret string, awsS3Region string, awsS3Bucket string, - syncStatus string, syncProperties *sdk.SyncProperties, apiConfig *sdk.APIConfig, maxVideoLength float64) *SyncManager { +func NewSyncManager(cliFlags shared.SyncFlags, blobsDir, lbrycrdDsn string, awsConfigs *shared.AwsConfigs, apiConfig *sdk.APIConfig) *SyncManager { return &SyncManager{ - SyncFlags: syncFlags, - maxTries: maxTries, - refill: refill, - limit: limit, - concurrentJobs: concurrentJobs, - concurrentVideos: concurrentVideos, - blobsDir: blobsDir, - videosLimit: videosLimit, - maxVideoSize: maxVideoSize, - maxVideoLength: maxVideoLength, - lbrycrdString: lbrycrdString, - awsS3ID: awsS3ID, - awsS3Secret: awsS3Secret, - awsS3Region: awsS3Region, - awsS3Bucket: awsS3Bucket, - syncStatus: syncStatus, - syncProperties: syncProperties, - apiConfig: apiConfig, + CliFlags: cliFlags, + blobsDir: blobsDir, + LbrycrdDsn: lbrycrdDsn, + AwsConfigs: awsConfigs, + ApiConfig: apiConfig, } } - -const ( - StatusPending = "pending" // waiting for permission to sync - StatusPendingEmail = "pendingemail" // permission granted but missing email - StatusQueued = "queued" // in sync queue. will be synced soon - StatusPendingUpgrade = "pendingupgrade" // in sync queue. will be synced soon - StatusSyncing = "syncing" // syncing now - StatusSynced = "synced" // done - StatusFailed = "failed" - StatusFinalized = "finalized" // no more changes allowed - StatusAbandoned = "abandoned" // deleted on youtube or banned -) -const LatestMetadataVersion = 2 - -var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned} - -const ( - VideoStatusPublished = "published" - VideoStatusFailed = "failed" - VideoStatusUpgradeFailed = "upgradefailed" - VideoStatusUnpublished = "unpublished" - VideoStatusTranferFailed = "transferfailed" -) - -const ( - TransferStateNotTouched = iota - TransferStatePending - TransferStateComplete - TransferStateManual -) +func (s *SyncManager) enqueueChannel(channel *shared.YoutubeChannel) { + s.channelsToSync = append(s.channelsToSync, Sync{ + DbChannelData: channel, + Manager: s, + namer: namer.NewNamer(), + }) +} func (s *SyncManager) Start() error { - if logUtils.ShouldCleanOnStartup() { err := logUtils.CleanForStartup() if err != nil { @@ -105,108 +54,66 @@ func (s *SyncManager) Start() error { } } + var lastChannelProcessed string syncCount := 0 for { + s.channelsToSync = make([]Sync, 0, 10) // reset sync queue err := s.checkUsedSpace() if err != nil { return errors.Err(err) } - - var syncs []Sync shouldInterruptLoop := false - isSingleChannelSync := s.syncProperties.YoutubeChannelID != "" - if isSingleChannelSync { - channels, err := s.apiConfig.FetchChannels("", s.syncProperties) + if s.CliFlags.IsSingleChannelSync() { + channels, err := s.ApiConfig.FetchChannels("", &s.CliFlags) if err != nil { return errors.Err(err) } if len(channels) != 1 { return errors.Err("Expected 1 channel, %d returned", len(channels)) } - lbryChannelName := channels[0].DesiredChannelName - syncs = make([]Sync, 1) - syncs[0] = Sync{ - APIConfig: s.apiConfig, - YoutubeChannelID: s.syncProperties.YoutubeChannelID, - LbryChannelName: lbryChannelName, - lbryChannelID: channels[0].ChannelClaimID, - MaxTries: s.maxTries, - ConcurrentVideos: s.concurrentVideos, - Refill: s.refill, - Manager: s, - MaxVideoLength: s.maxVideoLength, - LbrycrdString: s.lbrycrdString, - AwsS3ID: s.awsS3ID, - AwsS3Secret: s.awsS3Secret, - AwsS3Region: s.awsS3Region, - AwsS3Bucket: s.awsS3Bucket, - namer: namer.NewNamer(), - Fee: channels[0].Fee, - clientPublishAddress: channels[0].PublishAddress, - publicKey: channels[0].PublicKey, - transferState: channels[0].TransferState, - } + s.enqueueChannel(&channels[0]) shouldInterruptLoop = true } else { var queuesToSync []string - if s.syncStatus != "" { - queuesToSync = append(queuesToSync, s.syncStatus) - } else if s.SyncFlags.SyncUpdate { - queuesToSync = append(queuesToSync, StatusSyncing, StatusSynced) + if s.CliFlags.SyncStatus != "" { + queuesToSync = append(queuesToSync, s.CliFlags.SyncStatus) + } else if s.CliFlags.SyncUpdate { + queuesToSync = append(queuesToSync, shared.StatusSyncing, shared.StatusSynced) } else { - queuesToSync = append(queuesToSync, StatusSyncing, StatusQueued) + queuesToSync = append(queuesToSync, shared.StatusSyncing, shared.StatusQueued) } queues: for _, q := range queuesToSync { - //temporary override for sync-until to give tom the time to review the channels - if q == StatusQueued { - s.syncProperties.SyncUntil = time.Now().Add(-8 * time.Hour).Unix() - } - channels, err := s.apiConfig.FetchChannels(q, s.syncProperties) + channels, err := s.ApiConfig.FetchChannels(q, &s.CliFlags) if err != nil { return err } - for i, c := range channels { - log.Infof("There are %d channels in the \"%s\" queue", len(channels)-i, q) - maxVideoLength := s.maxVideoLength - if c.TotalSubscribers < 1000 { - maxVideoLength = 1.0 - } - syncs = append(syncs, Sync{ - APIConfig: s.apiConfig, - YoutubeChannelID: c.ChannelId, - LbryChannelName: c.DesiredChannelName, - lbryChannelID: c.ChannelClaimID, - MaxTries: s.maxTries, - ConcurrentVideos: s.concurrentVideos, - MaxVideoLength: maxVideoLength, - Refill: s.refill, - Manager: s, - LbrycrdString: s.lbrycrdString, - AwsS3ID: s.awsS3ID, - AwsS3Secret: s.awsS3Secret, - AwsS3Region: s.awsS3Region, - AwsS3Bucket: s.awsS3Bucket, - namer: namer.NewNamer(), - Fee: c.Fee, - clientPublishAddress: c.PublishAddress, - publicKey: c.PublicKey, - transferState: c.TransferState, - }) - if q != StatusFailed { - continue queues + log.Infof("Currently processing the \"%s\" queue with %d channels", q, len(channels)) + for _, c := range channels { + s.enqueueChannel(&c) + queueAll := q == shared.StatusFailed || q == shared.StatusSyncing + if !queueAll { + break queues } } + log.Infof("Drained the \"%s\" queue", q) } } - if len(syncs) == 0 { + if len(s.channelsToSync) == 0 { log.Infoln("No channels to sync. Pausing 5 minutes!") time.Sleep(5 * time.Minute) } - for _, sync := range syncs { + for _, sync := range s.channelsToSync { + if lastChannelProcessed == sync.DbChannelData.ChannelId { + util.SendToSlack("We just killed a sync for %s to stop looping! (%s)", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId) + stopTheLoops := errors.Err("Found channel %s running twice, set it to failed, and reprocess later", sync.DbChannelData.DesiredChannelName) + sync.setChannelTerminationStatus(&stopTheLoops) + continue + } + lastChannelProcessed = sync.DbChannelData.ChannelId shouldNotCount := false - logUtils.SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1) + logUtils.SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId, syncCount+1) err := sync.FullCycle() //TODO: THIS IS A TEMPORARY WORK AROUND FOR THE STUPID IP LOCKUP BUG ipPool, _ := ip_manager.GetIPPool(sync.grp) @@ -234,40 +141,35 @@ func (s *SyncManager) Start() error { } shouldNotCount = strings.Contains(err.Error(), "this youtube channel is being managed by another server") if !shouldNotCount { - logUtils.SendInfoToSlack("A non fatal error was reported by the sync process. %s\nContinuing...", err.Error()) + logUtils.SendInfoToSlack("A non fatal error was reported by the sync process.\n%s", errors.FullTrace(err)) } } err = blobs_reflector.ReflectAndClean() if err != nil { return errors.Prefix("@Nikooo777 something went wrong while reflecting blobs", err) } - logUtils.SendInfoToSlack("Syncing %s (%s) reached an end. total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1) + logUtils.SendInfoToSlack("%s (%s) reached an end. Total processed channels since startup: %d", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId, syncCount+1) if !shouldNotCount { syncCount++ } - if sync.IsInterrupted() || (s.limit != 0 && syncCount >= s.limit) { + if sync.IsInterrupted() || (s.CliFlags.Limit != 0 && syncCount >= s.CliFlags.Limit) { shouldInterruptLoop = true break } } - if shouldInterruptLoop || s.SyncFlags.SingleRun { + if shouldInterruptLoop || s.CliFlags.SingleRun { break } } return nil } -func (s *SyncManager) GetS3AWSConfig() aws.Config { - return aws.Config{ - Credentials: credentials.NewStaticCredentials(s.awsS3ID, s.awsS3Secret, ""), - Region: &s.awsS3Region, - } -} + func (s *SyncManager) checkUsedSpace() error { usedPctile, err := GetUsedSpace(logUtils.GetBlobsDir()) if err != nil { return errors.Err(err) } - if usedPctile >= 0.90 && !s.SyncFlags.SkipSpaceCheck { + if usedPctile >= 0.90 && !s.CliFlags.SkipSpaceCheck { return errors.Err(fmt.Sprintf("more than 90%% of the space has been used. use --skip-space-check to ignore. Used: %.1f%%", usedPctile*100)) } log.Infof("disk usage: %.1f%%", usedPctile*100) diff --git a/manager/s3_storage.go b/manager/s3_storage.go new file mode 100644 index 0000000..8b78c89 --- /dev/null +++ b/manager/s3_storage.go @@ -0,0 +1,250 @@ +package manager + +import ( + "os" + "path/filepath" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + + "github.com/lbryio/lbry.go/v2/extras/errors" + + logUtils "github.com/lbryio/ytsync/v5/util" +) + +func (s *Sync) getS3Downloader() (*s3manager.Downloader, error) { + s3Session, err := session.NewSession(s.Manager.AwsConfigs.GetS3AWSConfig()) + if err != nil { + return nil, errors.Prefix("error starting session: ", err) + } + downloader := s3manager.NewDownloader(s3Session) + return downloader, nil +} +func (s *Sync) getS3Uploader() (*s3manager.Uploader, error) { + s3Session, err := session.NewSession(s.Manager.AwsConfigs.GetS3AWSConfig()) + if err != nil { + return nil, errors.Prefix("error starting session: ", err) + } + uploader := s3manager.NewUploader(s3Session) + return uploader, nil +} + +func (s *Sync) downloadWallet() error { + defaultWalletDir, defaultTempWalletDir, key, err := s.getWalletPaths() + if err != nil { + return errors.Err(err) + } + downloader, err := s.getS3Downloader() + if err != nil { + return err + } + out, err := os.Create(defaultTempWalletDir) + if err != nil { + return errors.Prefix("error creating temp wallet: ", err) + } + defer out.Close() + + bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{ + Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket), + Key: key, + }) + if err != nil { + // Casting to the awserr.Error type will allow you to inspect the error + // code returned by the service in code. The error code can be used + // to switch on context specific functionality. In this case a context + // specific error message is printed to the user based on the bucket + // and key existing. + // + // For information on other S3 API error codes see: + // http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html + if aerr, ok := err.(awserr.Error); ok { + code := aerr.Code() + if code == s3.ErrCodeNoSuchKey { + return errors.Err("wallet not on S3") + } + } + return err + } else if bytesWritten == 0 { + return errors.Err("zero bytes written") + } + + err = os.Rename(defaultTempWalletDir, defaultWalletDir) + if err != nil { + return errors.Prefix("error replacing temp wallet for default wallet: ", err) + } + + return nil +} + +func (s *Sync) downloadBlockchainDB() error { + if logUtils.IsRegTest() { + return nil // tests fail if we re-use the same blockchain DB + } + defaultBDBDir, defaultTempBDBDir, key, err := s.getBlockchainDBPaths() + if err != nil { + return errors.Err(err) + } + files, err := filepath.Glob(defaultBDBDir + "*") + if err != nil { + return errors.Err(err) + } + for _, f := range files { + err = os.Remove(f) + if err != nil { + return errors.Err(err) + } + } + + downloader, err := s.getS3Downloader() + if err != nil { + return errors.Err(err) + } + out, err := os.Create(defaultTempBDBDir) + if err != nil { + return errors.Prefix("error creating temp wallet: ", err) + } + defer out.Close() + + bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{ + Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket), + Key: key, + }) + if err != nil { + // Casting to the awserr.Error type will allow you to inspect the error + // code returned by the service in code. The error code can be used + // to switch on context specific functionality. In this case a context + // specific error message is printed to the user based on the bucket + // and key existing. + // + // For information on other S3 API error codes see: + // http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html + if aerr, ok := err.(awserr.Error); ok { + code := aerr.Code() + if code == s3.ErrCodeNoSuchKey { + return nil // let ytsync sync the database by itself + } + } + return errors.Err(err) + } else if bytesWritten == 0 { + return errors.Err("zero bytes written") + } + + err = os.Rename(defaultTempBDBDir, defaultBDBDir) + if err != nil { + return errors.Prefix("error replacing temp blockchain.db for default blockchain.db: ", err) + } + + return nil +} + +func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string, err error) { + defaultWallet = os.Getenv("HOME") + "/.lbryum/wallets/default_wallet" + tempWallet = os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet" + key = aws.String("/wallets/" + s.DbChannelData.ChannelId) + if logUtils.IsRegTest() { + defaultWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet" + tempWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet" + key = aws.String("/regtest/" + s.DbChannelData.ChannelId) + } + + lbryumDir := os.Getenv("LBRYUM_DIR") + if lbryumDir != "" { + defaultWallet = lbryumDir + "/wallets/default_wallet" + tempWallet = lbryumDir + "/wallets/tmp_wallet" + } + + if _, err := os.Stat(defaultWallet); !os.IsNotExist(err) { + return "", "", nil, errors.Err("default_wallet already exists") + } + return +} + +func (s *Sync) getBlockchainDBPaths() (defaultDB, tempDB string, key *string, err error) { + lbryumDir := os.Getenv("LBRYUM_DIR") + if lbryumDir == "" { + if logUtils.IsRegTest() { + lbryumDir = os.Getenv("HOME") + "/.lbryum_regtest" + } else { + lbryumDir = os.Getenv("HOME") + "/.lbryum" + } + } + defaultDB = lbryumDir + "/lbc_mainnet/blockchain.db" + tempDB = lbryumDir + "/lbc_mainnet/tmp_blockchain.db" + key = aws.String("/blockchain_dbs/" + s.DbChannelData.ChannelId) + if logUtils.IsRegTest() { + defaultDB = lbryumDir + "/lbc_regtest/blockchain.db" + tempDB = lbryumDir + "/lbc_regtest/tmp_blockchain.db" + key = aws.String("/regtest_dbs/" + s.DbChannelData.ChannelId) + } + return +} + +func (s *Sync) uploadWallet() error { + defaultWalletDir := logUtils.GetDefaultWalletPath() + key := aws.String("/wallets/" + s.DbChannelData.ChannelId) + if logUtils.IsRegTest() { + key = aws.String("/regtest/" + s.DbChannelData.ChannelId) + } + + if _, err := os.Stat(defaultWalletDir); os.IsNotExist(err) { + return errors.Err("default_wallet does not exist") + } + + uploader, err := s.getS3Uploader() + if err != nil { + return err + } + + file, err := os.Open(defaultWalletDir) + if err != nil { + return err + } + defer file.Close() + + _, err = uploader.Upload(&s3manager.UploadInput{ + Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket), + Key: key, + Body: file, + }) + if err != nil { + return err + } + + return os.Remove(defaultWalletDir) +} + +func (s *Sync) uploadBlockchainDB() error { + defaultBDBDir, _, key, err := s.getBlockchainDBPaths() + if err != nil { + return errors.Err(err) + } + + if _, err := os.Stat(defaultBDBDir); os.IsNotExist(err) { + return errors.Err("blockchain.db does not exist") + } + + uploader, err := s.getS3Uploader() + if err != nil { + return err + } + + file, err := os.Open(defaultBDBDir) + if err != nil { + return err + } + defer file.Close() + + _, err = uploader.Upload(&s3manager.UploadInput{ + Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket), + Key: key, + Body: file, + }) + if err != nil { + return err + } + + return os.Remove(defaultBDBDir) +} diff --git a/manager/setup.go b/manager/setup.go index aeab832..e2a809e 100644 --- a/manager/setup.go +++ b/manager/setup.go @@ -3,23 +3,22 @@ package manager import ( "fmt" "math" - "net/http" "strconv" "time" "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/extras/jsonrpc" "github.com/lbryio/lbry.go/v2/extras/util" + "github.com/lbryio/ytsync/v5/shared" "github.com/lbryio/ytsync/v5/timing" logUtils "github.com/lbryio/ytsync/v5/util" + "github.com/lbryio/ytsync/v5/ytapi" "github.com/lbryio/ytsync/v5/tags_manager" "github.com/lbryio/ytsync/v5/thumbs" "github.com/shopspring/decimal" log "github.com/sirupsen/logrus" - "google.golang.org/api/googleapi/transport" - "google.golang.org/api/youtube/v3" ) func (s *Sync) enableAddressReuse() error { @@ -74,11 +73,7 @@ func (s *Sync) walletSetup() error { } log.Debugf("Starting balance is %.4f", balance) - n, err := s.CountVideos() - if err != nil { - return err - } - videosOnYoutube := int(n) + videosOnYoutube := int(s.DbChannelData.TotalVideos) log.Debugf("Source channel has %d videos", videosOnYoutube) if videosOnYoutube == 0 { @@ -103,17 +98,17 @@ func (s *Sync) walletSetup() error { log.Debugf("We already allocated credits for %d published videos and %d failed videos", publishedCount, failedCount) - if videosOnYoutube > s.Manager.videosLimit { - videosOnYoutube = s.Manager.videosLimit + if videosOnYoutube > s.Manager.CliFlags.VideosLimit { + videosOnYoutube = s.Manager.CliFlags.VideosLimit } unallocatedVideos := videosOnYoutube - (publishedCount + failedCount) channelFee := channelClaimAmount - channelAlreadyClaimed := s.lbryChannelID != "" + channelAlreadyClaimed := s.DbChannelData.ChannelClaimID != "" if channelAlreadyClaimed { channelFee = 0.0 } requiredBalance := float64(unallocatedVideos)*(publishAmount+estimatedMaxTxFee) + channelFee - if s.Manager.SyncFlags.UpgradeMetadata { + if s.Manager.CliFlags.UpgradeMetadata { requiredBalance += float64(notUpgradedCount) * 0.001 } @@ -122,8 +117,8 @@ func (s *Sync) walletSetup() error { refillAmount = math.Max(math.Max(requiredBalance-balance, minimumAccountBalance-balance), minimumRefillAmount) } - if s.Refill > 0 { - refillAmount += float64(s.Refill) + if s.Manager.CliFlags.Refill > 0 { + refillAmount += float64(s.Manager.CliFlags.Refill) } if refillAmount > 0 { @@ -139,12 +134,11 @@ func (s *Sync) walletSetup() error { } else if claimAddress == nil { return errors.Err("could not get an address") } - s.claimAddress = string(claimAddress.Items[0].Address) - if s.claimAddress == "" { - return errors.Err("found blank claim address") + if s.DbChannelData.PublishAddress == "" || !s.shouldTransfer() { + s.DbChannelData.PublishAddress = string(claimAddress.Items[0].Address) } - if s.shouldTransfer() { - s.claimAddress = s.clientPublishAddress + if s.DbChannelData.PublishAddress == "" { + return errors.Err("found blank claim address") } err = s.ensureEnoughUTXOs() @@ -266,15 +260,14 @@ func (s *Sync) ensureEnoughUTXOs() error { } func (s *Sync) waitForNewBlock() error { - start := time.Now() - defer func(start time.Time) { - timing.TimedComponent("waitForNewBlock").Add(time.Since(start)) - }(start) + defer func(start time.Time) { timing.TimedComponent("waitForNewBlock").Add(time.Since(start)) }(time.Now()) + log.Printf("regtest: %t, docker: %t", logUtils.IsRegTest(), logUtils.IsUsingDocker()) status, err := s.daemon.Status() if err != nil { return err } + for status.Wallet.Blocks == 0 || status.Wallet.BlocksBehind != 0 { time.Sleep(5 * time.Second) status, err = s.daemon.Status() @@ -304,14 +297,16 @@ func (s *Sync) waitForNewBlock() error { } func (s *Sync) GenerateRegtestBlock() error { - lbrycrd, err := logUtils.GetLbrycrdClient(s.LbrycrdString) + lbrycrd, err := logUtils.GetLbrycrdClient(s.Manager.LbrycrdDsn) if err != nil { return errors.Prefix("error getting lbrycrd client: ", err) } + txs, err := lbrycrd.Generate(1) if err != nil { return errors.Prefix("error generating new block: ", err) } + for _, tx := range txs { log.Info("Generated tx: ", tx.String()) } @@ -319,11 +314,9 @@ func (s *Sync) GenerateRegtestBlock() error { } func (s *Sync) ensureChannelOwnership() error { - start := time.Now() - defer func(start time.Time) { - timing.TimedComponent("ensureChannelOwnership").Add(time.Since(start)) - }(start) - if s.LbryChannelName == "" { + defer func(start time.Time) { timing.TimedComponent("ensureChannelOwnership").Add(time.Since(start)) }(time.Now()) + + if s.DbChannelData.DesiredChannelName == "" { return errors.Err("no channel name set") } @@ -336,27 +329,27 @@ func (s *Sync) ensureChannelOwnership() error { var channelToUse *jsonrpc.Transaction if len((*channels).Items) > 0 { - if s.lbryChannelID == "" { + if s.DbChannelData.ChannelClaimID == "" { return errors.Err("this channel does not have a recorded claimID in the database. To prevent failures, updates are not supported until an entry is manually added in the database") } for _, c := range (*channels).Items { log.Debugf("checking listed channel %s (%s)", c.ClaimID, c.Name) - if c.ClaimID != s.lbryChannelID { + if c.ClaimID != s.DbChannelData.ChannelClaimID { continue } - if c.Name != s.LbryChannelName { + if c.Name != s.DbChannelData.DesiredChannelName { return errors.Err("the channel in the wallet is different than the channel in the database") } channelToUse = &c break } if channelToUse == nil { - return errors.Err("this wallet has channels but not a single one is ours! Expected claim_id: %s (%s)", s.lbryChannelID, s.LbryChannelName) + return errors.Err("this wallet has channels but not a single one is ours! Expected claim_id: %s (%s)", s.DbChannelData.ChannelClaimID, s.DbChannelData.DesiredChannelName) } - } else if s.transferState == TransferStateComplete { + } else if s.DbChannelData.TransferState == shared.TransferStateComplete { return errors.Err("the channel was transferred but appears to have been abandoned!") - } else if s.lbryChannelID != "" { - return errors.Err("the database has a channel recorded (%s) but nothing was found in our control", s.lbryChannelID) + } else if s.DbChannelData.ChannelClaimID != "" { + return errors.Err("the database has a channel recorded (%s) but nothing was found in our control", s.DbChannelData.ChannelClaimID) } channelUsesOldMetadata := false @@ -386,36 +379,24 @@ func (s *Sync) ensureChannelOwnership() error { return err } } - client := &http.Client{ - Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey}, - } - service, err := youtube.New(client) + channelInfo, err := ytapi.ChannelInfo(s.DbChannelData.ChannelId) if err != nil { - return errors.Prefix("error creating YouTube service", err) + return err } - response, err := service.Channels.List("snippet,brandingSettings").Id(s.YoutubeChannelID).Do() - if err != nil { - return errors.Prefix("error getting channel details", err) - } - - if len(response.Items) < 1 { - return errors.Err("youtube channel not found") - } - - channelInfo := response.Items[0].Snippet - channelBranding := response.Items[0].BrandingSettings - - thumbnail := thumbs.GetBestThumbnail(channelInfo.Thumbnails) - thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.Url, s.YoutubeChannelID, s.Manager.GetS3AWSConfig()) + thumbnail := channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails)-1].URL + thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail, s.DbChannelData.ChannelId, *s.Manager.AwsConfigs.GetS3AWSConfig()) if err != nil { return err } var bannerURL *string - if channelBranding.Image != nil && channelBranding.Image.BannerImageUrl != "" { - bURL, err := thumbs.MirrorThumbnail(channelBranding.Image.BannerImageUrl, "banner-"+s.YoutubeChannelID, s.Manager.GetS3AWSConfig()) + if channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails != nil { + bURL, err := thumbs.MirrorThumbnail(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails)-1].URL, + "banner-"+s.DbChannelData.ChannelId, + *s.Manager.AwsConfigs.GetS3AWSConfig(), + ) if err != nil { return err } @@ -423,28 +404,29 @@ func (s *Sync) ensureChannelOwnership() error { } var languages []string = nil - if channelInfo.DefaultLanguage != "" { - if channelInfo.DefaultLanguage == "iw" { - channelInfo.DefaultLanguage = "he" - } - languages = []string{channelInfo.DefaultLanguage} - } + //we don't have this data without the API + //if channelInfo.DefaultLanguage != "" { + // if channelInfo.DefaultLanguage == "iw" { + // channelInfo.DefaultLanguage = "he" + // } + // languages = []string{channelInfo.DefaultLanguage} + //} var locations []jsonrpc.Location = nil - if channelInfo.Country != "" { - locations = []jsonrpc.Location{{Country: util.PtrToString(channelInfo.Country)}} + if channelInfo.Topbar.DesktopTopbarRenderer.CountryCode != "" { + locations = []jsonrpc.Location{{Country: &channelInfo.Topbar.DesktopTopbarRenderer.CountryCode}} } var c *jsonrpc.TransactionSummary claimCreateOptions := jsonrpc.ClaimCreateOptions{ - Title: &channelInfo.Title, - Description: &channelInfo.Description, - Tags: tags_manager.GetTagsForChannel(s.YoutubeChannelID), + Title: &channelInfo.Microformat.MicroformatDataRenderer.Title, + Description: &channelInfo.Metadata.ChannelMetadataRenderer.Description, + Tags: tags_manager.GetTagsForChannel(s.DbChannelData.ChannelId), Languages: languages, Locations: locations, ThumbnailURL: &thumbnailURL, } if channelUsesOldMetadata { - if s.transferState <= 1 { - c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{ + if s.DbChannelData.TransferState <= 1 { + c, err = s.daemon.ChannelUpdate(s.DbChannelData.ChannelClaimID, jsonrpc.ChannelUpdateOptions{ ClearTags: util.PtrToBool(true), ClearLocations: util.PtrToBool(true), ClearLanguages: util.PtrToBool(true), @@ -454,11 +436,11 @@ func (s *Sync) ensureChannelOwnership() error { }, }) } else { - logUtils.SendInfoToSlack("%s (%s) has a channel with old metadata but isn't in our control anymore. Ignoring", s.LbryChannelName, s.lbryChannelID) + logUtils.SendInfoToSlack("%s (%s) has a channel with old metadata but isn't in our control anymore. Ignoring", s.DbChannelData.DesiredChannelName, s.DbChannelData.ChannelClaimID) return nil } } else { - c, err = s.daemon.ChannelCreate(s.LbryChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{ + c, err = s.daemon.ChannelCreate(s.DbChannelData.DesiredChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{ ClaimCreateOptions: claimCreateOptions, CoverURL: bannerURL, }) @@ -467,8 +449,9 @@ func (s *Sync) ensureChannelOwnership() error { if err != nil { return err } - s.lbryChannelID = c.Outputs[0].ClaimID - return s.Manager.apiConfig.SetChannelClaimID(s.YoutubeChannelID, s.lbryChannelID) + + s.DbChannelData.ChannelClaimID = c.Outputs[0].ClaimID + return s.Manager.ApiConfig.SetChannelClaimID(s.DbChannelData.ChannelId, s.DbChannelData.ChannelClaimID) } func (s *Sync) addCredits(amountToAdd float64) error { @@ -477,7 +460,7 @@ func (s *Sync) addCredits(amountToAdd float64) error { timing.TimedComponent("addCredits").Add(time.Since(start)) }(start) log.Printf("Adding %f credits", amountToAdd) - lbrycrdd, err := logUtils.GetLbrycrdClient(s.LbrycrdString) + lbrycrdd, err := logUtils.GetLbrycrdClient(s.Manager.LbrycrdDsn) if err != nil { return err } diff --git a/manager/transfer.go b/manager/transfer.go index 1fdd8fb..fde2530 100644 --- a/manager/transfer.go +++ b/manager/transfer.go @@ -10,7 +10,7 @@ import ( "github.com/lbryio/lbry.go/v2/extras/jsonrpc" "github.com/lbryio/lbry.go/v2/extras/stop" "github.com/lbryio/lbry.go/v2/extras/util" - "github.com/lbryio/ytsync/v5/sdk" + "github.com/lbryio/ytsync/v5/shared" "github.com/lbryio/ytsync/v5/timing" log "github.com/sirupsen/logrus" @@ -71,7 +71,10 @@ func abandonSupports(s *Sync) (float64, error) { for page := uint64(1); page <= totalPages; page++ { supports, err := s.daemon.SupportList(&defaultAccount, page, 50) if err != nil { - return 0, errors.Prefix("cannot list claims", err) + supports, err = s.daemon.SupportList(&defaultAccount, page, 50) + if err != nil { + return 0, errors.Prefix("cannot list supports", err) + } } allSupports = append(allSupports, (*supports).Items...) totalPages = (*supports).TotalPages @@ -97,7 +100,7 @@ func abandonSupports(s *Sync) (float64, error) { //TODO: remove this once the SDK team fixes their RPC bugs.... s.daemon.SetRPCTimeout(60 * time.Second) defer s.daemon.SetRPCTimeout(5 * time.Minute) - for i := 0; i < s.ConcurrentVideos; i++ { + for i := 0; i < s.Manager.CliFlags.ConcurrentJobs; i++ { consumerWG.Add(1) go func() { defer consumerWG.Done() @@ -189,7 +192,7 @@ func abandonSupports(s *Sync) (float64, error) { type updateInfo struct { ClaimID string streamUpdateOptions *jsonrpc.StreamUpdateOptions - videoStatus *sdk.VideoStatus + videoStatus *shared.VideoStatus } func transferVideos(s *Sync) error { @@ -199,7 +202,7 @@ func transferVideos(s *Sync) error { }(start) cleanTransfer := true - streamChan := make(chan updateInfo, s.ConcurrentVideos) + streamChan := make(chan updateInfo, s.Manager.CliFlags.ConcurrentJobs) account, err := s.getDefaultAccount() if err != nil { return err @@ -213,13 +216,13 @@ func transferVideos(s *Sync) error { go func() { defer producerWG.Done() for _, video := range s.syncedVideos { - if !video.Published || video.Transferred || video.MetadataVersion != LatestMetadataVersion { + if !video.Published || video.Transferred || video.MetadataVersion != shared.LatestMetadataVersion { continue } var stream *jsonrpc.Claim = nil for _, c := range streams.Items { - if c.ClaimID != video.ClaimID || (c.SigningChannel != nil && c.SigningChannel.ClaimID != s.lbryChannelID) { + if c.ClaimID != video.ClaimID || (c.SigningChannel != nil && c.SigningChannel.ClaimID != s.DbChannelData.ChannelClaimID) { continue } stream = &c @@ -232,7 +235,7 @@ func transferVideos(s *Sync) error { streamUpdateOptions := jsonrpc.StreamUpdateOptions{ StreamCreateOptions: &jsonrpc.StreamCreateOptions{ ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ - ClaimAddress: &s.clientPublishAddress, + ClaimAddress: &s.DbChannelData.PublishAddress, FundingAccountIDs: []string{ account, }, @@ -240,12 +243,12 @@ func transferVideos(s *Sync) error { }, Bid: util.PtrToString("0.005"), // Todo - Dont hardcode } - videoStatus := sdk.VideoStatus{ - ChannelID: s.YoutubeChannelID, + videoStatus := shared.VideoStatus{ + ChannelID: s.DbChannelData.ChannelId, VideoID: video.VideoID, ClaimID: video.ClaimID, ClaimName: video.ClaimName, - Status: VideoStatusPublished, + Status: shared.VideoStatusPublished, IsTransferred: util.PtrToBool(true), } streamChan <- updateInfo{ @@ -257,7 +260,7 @@ func transferVideos(s *Sync) error { }() consumerWG := &stop.Group{} - for i := 0; i < s.ConcurrentVideos; i++ { + for i := 0; i < s.Manager.CliFlags.ConcurrentJobs; i++ { consumerWG.Add(1) go func(worker int) { defer consumerWG.Done() @@ -290,13 +293,13 @@ func (s *Sync) streamUpdate(ui *updateInfo) error { timing.TimedComponent("transferStreamUpdate").Add(time.Since(start)) if updateError != nil { ui.videoStatus.FailureReason = updateError.Error() - ui.videoStatus.Status = VideoStatusTranferFailed + ui.videoStatus.Status = shared.VideoStatusTranferFailed ui.videoStatus.IsTransferred = util.PtrToBool(false) } else { ui.videoStatus.IsTransferred = util.PtrToBool(len(result.Outputs) != 0) } log.Infof("TRANSFERRED %t", *ui.videoStatus.IsTransferred) - statusErr := s.APIConfig.MarkVideoStatus(*ui.videoStatus) + statusErr := s.Manager.ApiConfig.MarkVideoStatus(*ui.videoStatus) if statusErr != nil { return errors.Prefix(statusErr.Error(), updateError) } @@ -318,7 +321,7 @@ func transferChannel(s *Sync) error { } var channelClaim *jsonrpc.Transaction = nil for _, c := range channelClaims.Items { - if c.ClaimID != s.lbryChannelID { + if c.ClaimID != s.DbChannelData.ChannelClaimID { continue } channelClaim = &c @@ -332,11 +335,11 @@ func transferChannel(s *Sync) error { Bid: util.PtrToString(fmt.Sprintf("%.6f", channelClaimAmount-0.005)), ChannelCreateOptions: jsonrpc.ChannelCreateOptions{ ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ - ClaimAddress: &s.clientPublishAddress, + ClaimAddress: &s.DbChannelData.PublishAddress, }, }, } - result, err := s.daemon.ChannelUpdate(s.lbryChannelID, updateOptions) + result, err := s.daemon.ChannelUpdate(s.DbChannelData.ChannelClaimID, updateOptions) if err != nil { return errors.Err(err) } diff --git a/manager/ytsync.go b/manager/ytsync.go index a1de795..b866be3 100644 --- a/manager/ytsync.go +++ b/manager/ytsync.go @@ -3,11 +3,9 @@ package manager import ( "fmt" "io/ioutil" - "net/http" "os" "os/signal" "runtime/debug" - "sort" "strconv" "strings" "sync" @@ -17,25 +15,19 @@ import ( "github.com/lbryio/ytsync/v5/ip_manager" "github.com/lbryio/ytsync/v5/namer" "github.com/lbryio/ytsync/v5/sdk" + "github.com/lbryio/ytsync/v5/shared" "github.com/lbryio/ytsync/v5/sources" "github.com/lbryio/ytsync/v5/thumbs" "github.com/lbryio/ytsync/v5/timing" logUtils "github.com/lbryio/ytsync/v5/util" + "github.com/lbryio/ytsync/v5/ytapi" "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/extras/jsonrpc" "github.com/lbryio/lbry.go/v2/extras/stop" "github.com/lbryio/lbry.go/v2/extras/util" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" log "github.com/sirupsen/logrus" - "google.golang.org/api/googleapi/transport" - "google.golang.org/api/youtube/v3" ) const ( @@ -47,52 +39,20 @@ const ( maxReasonLength = 500 ) -type video interface { - Size() *int64 - ID() string - IDAndNum() string - PlaylistPosition() int - PublishedAt() time.Time - Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool, *sync.RWMutex) (*sources.SyncSummary, error) -} - -// sorting videos -type byPublishedAt []video - -func (a byPublishedAt) Len() int { return len(a) } -func (a byPublishedAt) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a byPublishedAt) Less(i, j int) bool { return a[i].PublishedAt().Before(a[j].PublishedAt()) } - // Sync stores the options that control how syncing happens type Sync struct { - APIConfig *sdk.APIConfig - YoutubeChannelID string - LbryChannelName string - MaxTries int - ConcurrentVideos int - Refill int - Manager *SyncManager - LbrycrdString string - AwsS3ID string - AwsS3Secret string - AwsS3Region string - AwsS3Bucket string - Fee *sdk.Fee - daemon *jsonrpc.Client - claimAddress string - videoDirectory string - syncedVideosMux *sync.RWMutex - syncedVideos map[string]sdk.SyncedVideo - grp *stop.Group - lbryChannelID string - namer *namer.Namer - walletMux *sync.RWMutex - queue chan video - transferState int - clientPublishAddress string - publicKey string - defaultAccountID string - MaxVideoLength float64 + DbChannelData *shared.YoutubeChannel + Manager *SyncManager + + daemon *jsonrpc.Client + videoDirectory string + syncedVideosMux *sync.RWMutex + syncedVideos map[string]sdk.SyncedVideo + grp *stop.Group + namer *namer.Namer + walletMux *sync.RWMutex + queue chan ytapi.Video + defaultAccountID string } func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) { @@ -119,118 +79,8 @@ func (s *Sync) IsInterrupted() bool { } } -func (s *Sync) downloadWallet() error { - defaultWalletDir, defaultTempWalletDir, key, err := s.getWalletPaths() - if err != nil { - return errors.Err(err) - } - - creds := credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, "") - s3Session, err := session.NewSession(&aws.Config{Region: aws.String(s.AwsS3Region), Credentials: creds}) - if err != nil { - return errors.Prefix("error starting session: ", err) - } - downloader := s3manager.NewDownloader(s3Session) - out, err := os.Create(defaultTempWalletDir) - if err != nil { - return errors.Prefix("error creating temp wallet: ", err) - } - defer out.Close() - - bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{ - Bucket: aws.String(s.AwsS3Bucket), - Key: key, - }) - if err != nil { - // Casting to the awserr.Error type will allow you to inspect the error - // code returned by the service in code. The error code can be used - // to switch on context specific functionality. In this case a context - // specific error message is printed to the user based on the bucket - // and key existing. - // - // For information on other S3 API error codes see: - // http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html - if aerr, ok := err.(awserr.Error); ok { - code := aerr.Code() - if code == s3.ErrCodeNoSuchKey { - return errors.Err("wallet not on S3") - } - } - return err - } else if bytesWritten == 0 { - return errors.Err("zero bytes written") - } - - err = os.Rename(defaultTempWalletDir, defaultWalletDir) - if err != nil { - return errors.Prefix("error replacing temp wallet for default wallet: ", err) - } - - return nil -} - -func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string, err error) { - - defaultWallet = os.Getenv("HOME") + "/.lbryum/wallets/default_wallet" - tempWallet = os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet" - key = aws.String("/wallets/" + s.YoutubeChannelID) - if logUtils.IsRegTest() { - defaultWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet" - tempWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet" - key = aws.String("/regtest/" + s.YoutubeChannelID) - } - - walletPath := os.Getenv("LBRYNET_WALLETS_DIR") - if walletPath != "" { - defaultWallet = walletPath + "/wallets/default_wallet" - tempWallet = walletPath + "/wallets/tmp_wallet" - } - - if _, err := os.Stat(defaultWallet); !os.IsNotExist(err) { - return "", "", nil, errors.Err("default_wallet already exists") - } - return -} - -func (s *Sync) uploadWallet() error { - defaultWalletDir := logUtils.GetDefaultWalletPath() - key := aws.String("/wallets/" + s.YoutubeChannelID) - if logUtils.IsRegTest() { - key = aws.String("/regtest/" + s.YoutubeChannelID) - } - - if _, err := os.Stat(defaultWalletDir); os.IsNotExist(err) { - return errors.Err("default_wallet does not exist") - } - - creds := credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, "") - s3Session, err := session.NewSession(&aws.Config{Region: aws.String(s.AwsS3Region), Credentials: creds}) - if err != nil { - return err - } - - uploader := s3manager.NewUploader(s3Session) - - file, err := os.Open(defaultWalletDir) - if err != nil { - return err - } - defer file.Close() - - _, err = uploader.Upload(&s3manager.UploadInput{ - Bucket: aws.String(s.AwsS3Bucket), - Key: key, - Body: file, - }) - if err != nil { - return err - } - - return os.Remove(defaultWalletDir) -} - func (s *Sync) setStatusSyncing() error { - syncedVideos, claimNames, err := s.Manager.apiConfig.SetChannelStatus(s.YoutubeChannelID, StatusSyncing, "", nil) + syncedVideos, claimNames, err := s.Manager.ApiConfig.SetChannelStatus(s.DbChannelData.ChannelId, shared.StatusSyncing, "", nil) if err != nil { return err } @@ -241,36 +91,27 @@ func (s *Sync) setStatusSyncing() error { return nil } -func (s *Sync) setExceptions() { - if s.YoutubeChannelID == "UCwjQfNRW6sGYb__pd7d4nUg" { //@FreeTalkLive - s.MaxVideoLength = 9999.0 // skips max length checks - s.Manager.maxVideoSize = 0 - } -} - var stopGroup = stop.New() func (s *Sync) FullCycle() (e error) { if os.Getenv("HOME") == "" { return errors.Err("no $HOME env var found") } - if s.YoutubeChannelID == "" { - return errors.Err("channel ID not provided") - } - - s.setExceptions() defer timing.ClearTimings() s.syncedVideosMux = &sync.RWMutex{} s.walletMux = &sync.RWMutex{} s.grp = stopGroup - s.queue = make(chan video) + s.queue = make(chan ytapi.Video) interruptChan := make(chan os.Signal, 1) signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM) defer signal.Stop(interruptChan) go func() { <-interruptChan + util.SendToSlack("got interrupt, shutting down") log.Println("Got interrupt signal, shutting down (if publishing, will shut down after current publish)") s.grp.Stop() + time.Sleep(5 * time.Second) + debug.PrintStack() // so we can figure out what's not stopping }() err := s.setStatusSyncing() if err != nil { @@ -286,6 +127,10 @@ func (s *Sync) FullCycle() (e error) { } else { log.Println("Starting new wallet") } + err = s.downloadBlockchainDB() + if err != nil { + return errors.Prefix("failure in downloading blockchain.db", err) + } defer s.stopAndUploadWallet(&e) @@ -328,16 +173,18 @@ func (s *Sync) FullCycle() (e error) { func (s *Sync) processTransfers() (e error) { log.Println("Processing transfers") - err := waitConfirmations(s) - if err != nil { - return err + if s.DbChannelData.TransferState != 2 { + err := waitConfirmations(s) + if err != nil { + return err + } } supportAmount, err := abandonSupports(s) if err != nil { return errors.Prefix(fmt.Sprintf("%.6f LBCs were abandoned before failing", supportAmount), err) } if supportAmount > 0 { - logUtils.SendInfoToSlack("(%s) %.6f LBCs were abandoned and should be used as support", s.YoutubeChannelID, supportAmount) + logUtils.SendInfoToSlack("(%s) %.6f LBCs were abandoned and should be used as support", s.DbChannelData.ChannelId, supportAmount) } err = transferVideos(s) if err != nil { @@ -358,19 +205,23 @@ func (s *Sync) processTransfers() (e error) { return err } isTip := true - summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount}, nil) + summary, err := s.daemon.SupportCreate(s.DbChannelData.ChannelClaimID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount}, nil) if err != nil { - if strings.Contains(err.Error(), "tx-size") { //TODO: this is a silly workaround and should be written in an recursive function - summary, err = s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount/2.0), &isTip, nil, []string{defaultAccount}, nil) + if strings.Contains(err.Error(), "tx-size") { //TODO: this is a silly workaround... + _, spendErr := s.daemon.TxoSpend(util.PtrToString("other"), nil, nil, nil, nil, &s.defaultAccountID) + if spendErr != nil { + return errors.Prefix(fmt.Sprintf("something went wrong while tipping the channel for %.6f LBCs", supportAmount), err) + } + err = s.waitForNewBlock() + if err != nil { + return errors.Prefix(fmt.Sprintf("something went wrong while tipping the channel for %.6f LBCs (waiting for new block)", supportAmount), err) + } + summary, err = s.daemon.SupportCreate(s.DbChannelData.ChannelClaimID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount}, nil) if err != nil { return errors.Prefix(fmt.Sprintf("something went wrong while tipping the channel for %.6f LBCs", supportAmount), err) } - summary, err = s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount/2.0), &isTip, nil, []string{defaultAccount}, nil) - if err != nil { - return errors.Err(err) - } } else { - return errors.Err(err) + return errors.Prefix(fmt.Sprintf("something went wrong while tipping the channel for %.6f LBCs", supportAmount), err) } } if len(summary.Outputs) < 1 { @@ -392,7 +243,7 @@ func deleteSyncFolder(videoDirectory string) { } func (s *Sync) shouldTransfer() bool { - return s.transferState >= 1 && s.clientPublishAddress != "" && !s.Manager.SyncFlags.DisableTransfers + return s.DbChannelData.TransferState >= 1 && s.DbChannelData.PublishAddress != "" && !s.Manager.CliFlags.DisableTransfers && s.DbChannelData.TransferState != 3 } func (s *Sync) setChannelTerminationStatus(e *error) { @@ -400,7 +251,7 @@ func (s *Sync) setChannelTerminationStatus(e *error) { if s.shouldTransfer() { if *e == nil { - transferState = util.PtrToInt(TransferStateComplete) + transferState = util.PtrToInt(shared.TransferStateComplete) } } if *e != nil { @@ -413,13 +264,13 @@ func (s *Sync) setChannelTerminationStatus(e *error) { return } failureReason := (*e).Error() - _, _, err := s.Manager.apiConfig.SetChannelStatus(s.YoutubeChannelID, StatusFailed, failureReason, transferState) + _, _, err := s.Manager.ApiConfig.SetChannelStatus(s.DbChannelData.ChannelId, shared.StatusFailed, failureReason, transferState) if err != nil { - msg := fmt.Sprintf("Failed setting failed state for channel %s", s.LbryChannelName) + msg := fmt.Sprintf("Failed setting failed state for channel %s", s.DbChannelData.DesiredChannelName) *e = errors.Prefix(msg+err.Error(), *e) } } else if !s.IsInterrupted() { - _, _, err := s.Manager.apiConfig.SetChannelStatus(s.YoutubeChannelID, StatusSynced, "", transferState) + _, _, err := s.Manager.ApiConfig.SetChannelStatus(s.DbChannelData.ChannelId, shared.StatusSynced, "", transferState) if err != nil { *e = err } @@ -440,7 +291,7 @@ func (s *Sync) waitForDaemonStart() error { if err == nil && status.StartupStatus.Wallet && status.IsRunning { return nil } - if time.Since(beginTime).Minutes() > 60 { + if time.Since(beginTime).Minutes() > 120 { s.grp.Stop() return errors.Err("the daemon is taking too long to start. Something is wrong") } @@ -466,7 +317,14 @@ func (s *Sync) stopAndUploadWallet(e *error) { if err != nil { if *e == nil { e = &err - return + } else { + *e = errors.Prefix("failure uploading wallet", *e) + } + } + err = s.uploadBlockchainDB() + if err != nil { + if *e == nil { + e = &err } else { *e = errors.Prefix("failure uploading wallet", *e) } @@ -515,7 +373,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) { abandonedClaims := false videoIDs := make(map[string]jsonrpc.Claim) for _, c := range claims { - if !isYtsyncClaim(c, s.lbryChannelID) { + if !isYtsyncClaim(c, s.DbChannelData.ChannelClaimID) { continue } tn := c.Value.GetThumbnail().GetUrl() @@ -533,7 +391,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) { claimToAbandon = cl videoIDs[videoID] = c } - if claimToAbandon.Address != s.clientPublishAddress && !s.syncedVideos[videoID].Transferred { + if claimToAbandon.Address != s.DbChannelData.PublishAddress && !s.syncedVideos[videoID].Transferred { log.Debugf("abandoning %+v", claimToAbandon) _, err := s.daemon.StreamAbandon(claimToAbandon.Txid, claimToAbandon.Nout, nil, false) if err != nil { @@ -562,7 +420,7 @@ type ytsyncClaim struct { func (s *Sync) mapFromClaims(claims []jsonrpc.Claim) map[string]ytsyncClaim { videoIDMap := make(map[string]ytsyncClaim, len(claims)) for _, c := range claims { - if !isYtsyncClaim(c, s.lbryChannelID) { + if !isYtsyncClaim(c, s.DbChannelData.ChannelClaimID) { continue } tn := c.Value.GetThumbnail().GetUrl() @@ -602,8 +460,8 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim) claimNameDiffers := claimInDatabase && sv.ClaimName != chainInfo.ClaimName claimMarkedUnpublished := claimInDatabase && !sv.Published _, isOwnClaim := ownClaimsInfo[videoID] - tranferred := !isOwnClaim - transferStatusMismatch := sv.Transferred != tranferred + transferred := !isOwnClaim || s.DbChannelData.TransferState == 3 + transferStatusMismatch := sv.Transferred != transferred if metadataDiffers { log.Debugf("%s: Mismatch in database for metadata. DB: %d - Blockchain: %d", videoID, sv.MetadataVersion, chainInfo.MetadataVersion) @@ -621,7 +479,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim) log.Debugf("%s: Published but is not in database (%s - %s)", videoID, chainInfo.ClaimName, chainInfo.ClaimID) } if transferStatusMismatch { - log.Debugf("%s: is marked as transferred %t on it's actually %t", videoID, sv.Transferred, tranferred) + log.Debugf("%s: is marked as transferred %t on it's actually %t", videoID, sv.Transferred, transferred) } if !claimInDatabase || metadataDiffers || claimIDDiffers || claimNameDiffers || claimMarkedUnpublished || transferStatusMismatch { @@ -631,15 +489,15 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim) } fixed++ log.Debugf("updating %s in the database", videoID) - err = s.Manager.apiConfig.MarkVideoStatus(sdk.VideoStatus{ - ChannelID: s.YoutubeChannelID, + err = s.Manager.ApiConfig.MarkVideoStatus(shared.VideoStatus{ + ChannelID: s.DbChannelData.ChannelId, VideoID: videoID, - Status: VideoStatusPublished, + Status: shared.VideoStatusPublished, ClaimID: chainInfo.ClaimID, ClaimName: chainInfo.ClaimName, Size: util.PtrToInt64(int64(claimSize)), MetaDataVersion: chainInfo.MetadataVersion, - IsTransferred: &tranferred, + IsTransferred: &transferred, }) if err != nil { return count, fixed, 0, err @@ -680,13 +538,13 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim) } _, ok := ownClaimsInfo[vID] if !ok && sv.Published { - log.Debugf("%s: claims to be published but wasn't found in the list of claims and will be removed if --remove-db-unpublished was specified (%t)", vID, s.Manager.SyncFlags.RemoveDBUnpublished) + log.Debugf("%s: claims to be published but wasn't found in the list of claims and will be removed if --remove-db-unpublished was specified (%t)", vID, s.Manager.CliFlags.RemoveDBUnpublished) idsToRemove = append(idsToRemove, vID) } } - if s.Manager.SyncFlags.RemoveDBUnpublished && len(idsToRemove) > 0 { + if s.Manager.CliFlags.RemoveDBUnpublished && len(idsToRemove) > 0 { log.Infof("removing: %s", strings.Join(idsToRemove, ",")) - err := s.Manager.apiConfig.DeleteVideos(idsToRemove) + err := s.Manager.ApiConfig.DeleteVideos(idsToRemove) if err != nil { return count, fixed, len(idsToRemove), err } @@ -717,7 +575,7 @@ func (s *Sync) getClaims(defaultOnly bool) ([]jsonrpc.Claim, error) { } items := make([]jsonrpc.Claim, 0, len(claims.Items)) for _, c := range claims.Items { - if c.SigningChannel != nil && c.SigningChannel.ClaimID == s.lbryChannelID { + if c.SigningChannel != nil && c.SigningChannel.ClaimID == s.DbChannelData.ChannelClaimID { items = append(items, c) } } @@ -774,11 +632,11 @@ func (s *Sync) checkIntegrity() error { } if pubsOnWallet > pubsOnDB { //This case should never happen - logUtils.SendInfoToSlack("We're claiming to have published %d videos but in reality we published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID) + logUtils.SendInfoToSlack("We're claiming to have published %d videos but in reality we published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId) return errors.Err("not all published videos are in the database") } if pubsOnWallet < pubsOnDB { - logUtils.SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID) + logUtils.SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId) } _, err = s.getUnsentSupports() //TODO: use the returned value when it works @@ -811,24 +669,24 @@ func (s *Sync) doSync() error { return err } - if s.transferState < TransferStateComplete { - cert, err := s.daemon.ChannelExport(s.lbryChannelID, nil, nil) + if s.DbChannelData.TransferState < shared.TransferStateComplete { + cert, err := s.daemon.ChannelExport(s.DbChannelData.ChannelClaimID, nil, nil) if err != nil { return errors.Prefix("error getting channel cert", err) } if cert != nil { - err = s.APIConfig.SetChannelCert(string(*cert), s.lbryChannelID) + err = s.Manager.ApiConfig.SetChannelCert(string(*cert), s.DbChannelData.ChannelClaimID) if err != nil { return errors.Prefix("error setting channel cert", err) } } } - if s.Manager.SyncFlags.StopOnError { + if s.Manager.CliFlags.StopOnError { log.Println("Will stop publishing if an error is detected") } - for i := 0; i < s.ConcurrentVideos; i++ { + for i := 0; i < s.Manager.CliFlags.ConcurrentJobs; i++ { s.grp.Add(1) go func(i int) { defer s.grp.Done() @@ -836,7 +694,7 @@ func (s *Sync) doSync() error { }(i) } - if s.LbryChannelName == "@UCBerkeley" { + if s.DbChannelData.DesiredChannelName == "@UCBerkeley" { err = errors.Err("UCB is not supported in this version of YTSYNC") } else { err = s.enqueueYoutubeVideos() @@ -847,7 +705,7 @@ func (s *Sync) doSync() error { } func (s *Sync) startWorker(workerNum int) { - var v video + var v ytapi.Video var more bool for { @@ -872,10 +730,17 @@ func (s *Sync) startWorker(workerNum int) { tryCount := 0 for { + select { // check again inside the loop so this dies faster + case <-s.grp.Ch(): + log.Printf("Stopping worker %d", workerNum) + return + default: + } tryCount++ err := s.processVideo(v) if err != nil { + util.SendToSlack("Tried to process %s. Error: %v", v.ID(), err) logMsg := fmt.Sprintf("error processing video %s: %s", v.ID(), err.Error()) log.Errorln(logMsg) if strings.Contains(strings.ToLower(err.Error()), "interrupted by user") { @@ -891,9 +756,9 @@ func (s *Sync) startWorker(workerNum int) { "Couldn't find private key for id", "You already have a stream claim published under the name", } - if util.SubstringInSlice(err.Error(), fatalErrors) || s.Manager.SyncFlags.StopOnError { + if util.SubstringInSlice(err.Error(), fatalErrors) || s.Manager.CliFlags.StopOnError { s.grp.Stop() - } else if s.MaxTries > 1 { + } else if s.Manager.CliFlags.MaxTries > 1 { errorsNoRetry := []string{ "non 200 status code received", "This video contains content from", @@ -923,7 +788,7 @@ func (s *Sync) startWorker(workerNum int) { } if util.SubstringInSlice(err.Error(), errorsNoRetry) { log.Println("This error should not be retried at all") - } else if tryCount < s.MaxTries { + } else if tryCount < s.Manager.CliFlags.MaxTries { if util.SubstringInSlice(err.Error(), []string{ "txn-mempool-conflict", "too-long-mempool-chain", @@ -972,14 +837,14 @@ func (s *Sync) startWorker(workerNum int) { existingClaimSize = existingClaim.Size } } - videoStatus := VideoStatusFailed + videoStatus := shared.VideoStatusFailed if strings.Contains(err.Error(), "upgrade failed") { - videoStatus = VideoStatusUpgradeFailed + videoStatus = shared.VideoStatusUpgradeFailed } else { s.AppendSyncedVideo(v.ID(), false, err.Error(), existingClaimName, existingClaimID, 0, existingClaimSize) } - err = s.Manager.apiConfig.MarkVideoStatus(sdk.VideoStatus{ - ChannelID: s.YoutubeChannelID, + err = s.Manager.ApiConfig.MarkVideoStatus(shared.VideoStatus{ + ChannelID: s.DbChannelData.ChannelId, VideoID: v.ID(), Status: videoStatus, ClaimID: existingClaimID, @@ -996,107 +861,23 @@ func (s *Sync) startWorker(workerNum int) { } } -var mostRecentlyFailedChannel string - func (s *Sync) enqueueYoutubeVideos() error { - start := time.Now() - defer func(start time.Time) { - timing.TimedComponent("enqueueYoutubeVideos").Add(time.Since(start)) - }(start) - client := &http.Client{ - Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey}, - } + defer func(start time.Time) { timing.TimedComponent("enqueueYoutubeVideos").Add(time.Since(start)) }(time.Now()) - service, err := youtube.New(client) - if err != nil { - return errors.Prefix("error creating YouTube service", err) - } - - response, err := service.Channels.List("contentDetails").Id(s.YoutubeChannelID).Do() - if err != nil { - return errors.Prefix("error getting channels", err) - } - - if len(response.Items) < 1 { - return errors.Err("youtube channel not found") - } - - if response.Items[0].ContentDetails.RelatedPlaylists == nil { - return errors.Err("no related playlists") - } - - playlistID := response.Items[0].ContentDetails.RelatedPlaylists.Uploads - if playlistID == "" { - return errors.Err("no channel playlist") - } - - var videos []video ipPool, err := ip_manager.GetIPPool(s.grp) if err != nil { return err } - playlistMap := make(map[string]*youtube.PlaylistItemSnippet, 50) - nextPageToken := "" - for { - req := service.PlaylistItems.List("snippet"). - PlaylistId(playlistID). - MaxResults(50). - PageToken(nextPageToken) - - playlistResponse, err := req.Do() - if err != nil { - return errors.Prefix("error getting playlist items", err) - } - - if len(playlistResponse.Items) < 1 { - // If there are 50+ videos in a playlist but less than 50 are actually returned by the API, youtube will still redirect - // clients to a next page. Such next page will however be empty. This logic prevents ytsync from failing. - youtubeIsLying := len(videos) > 0 - if youtubeIsLying { - break - } - if s.YoutubeChannelID == mostRecentlyFailedChannel { - return errors.Err("playlist items not found") - } - mostRecentlyFailedChannel = s.YoutubeChannelID - break //return errors.Err("playlist items not found") //TODO: will this work? - } - videoIDs := make([]string, 50) - for i, item := range playlistResponse.Items { - // normally we'd send the video into the channel here, but youtube api doesn't have sorting - // so we have to get ALL the videos, then sort them, then send them in - playlistMap[item.Snippet.ResourceId.VideoId] = item.Snippet - videoIDs[i] = item.Snippet.ResourceId.VideoId - } - req2 := service.Videos.List("snippet,contentDetails,recordingDetails").Id(strings.Join(videoIDs[:], ",")) - - videosListResponse, err := req2.Do() - if err != nil { - return errors.Prefix("error getting videos info", err) - } - for _, item := range videosListResponse.Items { - videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig(), s.grp, ipPool)) - } - - log.Infof("Got info for %d videos from youtube API", len(videos)) - - nextPageToken = playlistResponse.NextPageToken - if nextPageToken == "" || s.Manager.SyncFlags.QuickSync || len(videos) >= s.Manager.videosLimit { - break - } + videos, err := ytapi.GetVideosToSync(s.Manager.ApiConfig, s.DbChannelData.ChannelId, s.syncedVideos, s.Manager.CliFlags.QuickSync, s.Manager.CliFlags.VideosLimit, ytapi.VideoParams{ + VideoDir: s.videoDirectory, + S3Config: *s.Manager.AwsConfigs.GetS3AWSConfig(), + Stopper: s.grp, + IPPool: ipPool, + }, s.DbChannelData.LastUploadedVideo) + if err != nil { + return err } - for k, v := range s.syncedVideos { - if !v.Published { - continue - } - _, ok := playlistMap[k] - if !ok { - videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig(), s.grp, ipPool)) - } - - } - sort.Sort(byPublishedAt(videos)) Enqueue: for _, v := range videos { @@ -1116,7 +897,7 @@ Enqueue: return nil } -func (s *Sync) processVideo(v video) (err error) { +func (s *Sync) processVideo(v ytapi.Video) (err error) { defer func() { if p := recover(); p != nil { logUtils.SendErrorToSlack("Video processing panic! %s", debug.Stack()) @@ -1139,20 +920,9 @@ func (s *Sync) processVideo(v video) (err error) { s.syncedVideosMux.RUnlock() newMetadataVersion := int8(2) alreadyPublished := ok && sv.Published - videoRequiresUpgrade := ok && s.Manager.SyncFlags.UpgradeMetadata && sv.MetadataVersion < newMetadataVersion + videoRequiresUpgrade := ok && s.Manager.CliFlags.UpgradeMetadata && sv.MetadataVersion < newMetadataVersion - neverRetryFailures := []string{ - "Error extracting sts from embedded url response", - "Unable to extract signature tokens", - "the video is too big to sync, skipping for now", - "video is too long to process", - "This video contains content from", - "no compatible format available for this video", - "Watch this video on YouTube.", - "have blocked it on copyright grounds", - "giving up after 0 fragment retries", - "Sign in to confirm your age", - } + neverRetryFailures := shared.NeverRetryFailures if ok && !sv.Published && util.SubstringInSlice(sv.FailureReason, neverRetryFailures) { log.Println(v.ID() + " can't ever be published") return nil @@ -1167,7 +937,7 @@ func (s *Sync) processVideo(v video) (err error) { return nil } - if !videoRequiresUpgrade && v.PlaylistPosition() >= s.Manager.videosLimit { + if !videoRequiresUpgrade && v.PlaylistPosition() >= s.Manager.CliFlags.VideosLimit { log.Println(v.ID() + " is old: skipping") return nil } @@ -1180,13 +950,13 @@ func (s *Sync) processVideo(v video) (err error) { return err } sp := sources.SyncParams{ - ClaimAddress: s.claimAddress, + ClaimAddress: s.DbChannelData.PublishAddress, Amount: publishAmount, - ChannelID: s.lbryChannelID, - MaxVideoSize: s.Manager.maxVideoSize, + ChannelID: s.DbChannelData.ChannelClaimID, + MaxVideoSize: s.DbChannelData.SizeLimit, Namer: s.namer, - MaxVideoLength: s.MaxVideoLength, - Fee: s.Fee, + MaxVideoLength: time.Duration(s.DbChannelData.LengthLimit) * time.Minute, + Fee: s.DbChannelData.Fee, DefaultAccount: da, } @@ -1196,14 +966,14 @@ func (s *Sync) processVideo(v video) (err error) { } s.AppendSyncedVideo(v.ID(), true, "", summary.ClaimName, summary.ClaimID, newMetadataVersion, *v.Size()) - err = s.Manager.apiConfig.MarkVideoStatus(sdk.VideoStatus{ - ChannelID: s.YoutubeChannelID, + err = s.Manager.ApiConfig.MarkVideoStatus(shared.VideoStatus{ + ChannelID: s.DbChannelData.ChannelId, VideoID: v.ID(), - Status: VideoStatusPublished, + Status: shared.VideoStatusPublished, ClaimID: summary.ClaimID, ClaimName: summary.ClaimName, Size: v.Size(), - MetaDataVersion: LatestMetadataVersion, + MetaDataVersion: shared.LatestMetadataVersion, IsTransferred: util.PtrToBool(s.shouldTransfer()), }) if err != nil { @@ -1214,7 +984,7 @@ func (s *Sync) processVideo(v video) (err error) { } func (s *Sync) importPublicKey() error { - if s.publicKey != "" { + if s.DbChannelData.PublicKey != "" { accountsResponse, err := s.daemon.AccountList(1, 50) if err != nil { return errors.Err(err) @@ -1225,13 +995,13 @@ func (s *Sync) importPublicKey() error { } for _, a := range accountsResponse.Items { if *a.Ledger == ledger { - if a.PublicKey == s.publicKey { + if a.PublicKey == s.DbChannelData.PublicKey { return nil } } } - log.Infof("Could not find public key %s in the wallet. Importing it...", s.publicKey) - _, err = s.daemon.AccountAdd(s.LbryChannelName, nil, nil, &s.publicKey, util.PtrToBool(true), nil) + log.Infof("Could not find public key %s in the wallet. Importing it...", s.DbChannelData.PublicKey) + _, err = s.daemon.AccountAdd(s.DbChannelData.DesiredChannelName, nil, nil, &s.DbChannelData.PublicKey, util.PtrToBool(true), nil) return errors.Err(err) } return nil @@ -1243,7 +1013,7 @@ func (s *Sync) getUnsentSupports() (float64, error) { if err != nil { return 0, errors.Err(err) } - if s.transferState == 2 { + if s.DbChannelData.TransferState == 2 { balance, err := s.daemon.AccountBalance(&defaultAccount) if err != nil { return 0, err @@ -1274,8 +1044,8 @@ func (s *Sync) getUnsentSupports() (float64, error) { } } } - if balanceAmount > 10 && sentSupports < 1 { - logUtils.SendErrorToSlack("(%s) this channel has quite some LBCs in it (%.2f) and %.2f LBC in sent tips, it's likely that the tips weren't actually sent or the wallet has unnecessary extra credits in it", s.YoutubeChannelID, balanceAmount, sentSupports) + if balanceAmount > 10 && sentSupports < 1 && s.DbChannelData.TransferState > 1 { + logUtils.SendErrorToSlack("(%s) this channel has quite some LBCs in it (%.2f) and %.2f LBC in sent tips, it's likely that the tips weren't actually sent or the wallet has unnecessary extra credits in it", s.DbChannelData.ChannelId, balanceAmount, sentSupports) return balanceAmount - 10, nil } } @@ -1284,8 +1054,7 @@ func (s *Sync) getUnsentSupports() (float64, error) { // waitForDaemonProcess observes the running processes and returns when the process is no longer running or when the timeout is up func waitForDaemonProcess(timeout time.Duration) error { - then := time.Now() - stopTime := then.Add(time.Duration(timeout * time.Second)) + stopTime := time.Now().Add(timeout * time.Second) for !time.Now().After(stopTime) { wait := 10 * time.Second log.Println("the daemon is still running, waiting for it to exit") diff --git a/sdk/api.go b/sdk/api.go index 903229b..8dbb5ca 100644 --- a/sdk/api.go +++ b/sdk/api.go @@ -13,6 +13,7 @@ import ( "github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/v2/extras/null" + "github.com/lbryio/ytsync/v5/shared" "github.com/lbryio/ytsync/v5/util" @@ -30,58 +31,28 @@ type APIConfig struct { HostName string } -type SyncProperties struct { - SyncFrom int64 - SyncUntil int64 - YoutubeChannelID string -} - -type SyncFlags struct { - StopOnError bool - TakeOverExistingChannel bool - SkipSpaceCheck bool - SyncUpdate bool - SingleRun bool - RemoveDBUnpublished bool - UpgradeMetadata bool - DisableTransfers bool - QuickSync bool -} - -type Fee struct { - Amount string `json:"amount"` - Address string `json:"address"` - Currency string `json:"currency"` -} -type YoutubeChannel struct { - ChannelId string `json:"channel_id"` - TotalVideos uint `json:"total_videos"` - TotalSubscribers uint `json:"total_subscribers"` - DesiredChannelName string `json:"desired_channel_name"` - Fee *Fee `json:"fee"` - ChannelClaimID string `json:"channel_claim_id"` - TransferState int `json:"transfer_state"` - PublishAddress string `json:"publish_address"` - PublicKey string `json:"public_key"` -} - -func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeChannel, error) { +func (a *APIConfig) FetchChannels(status string, cliFlags *shared.SyncFlags) ([]shared.YoutubeChannel, error) { type apiJobsResponse struct { - Success bool `json:"success"` - Error null.String `json:"error"` - Data []YoutubeChannel `json:"data"` + Success bool `json:"success"` + Error null.String `json:"error"` + Data []shared.YoutubeChannel `json:"data"` } endpoint := a.ApiURL + "/yt/jobs" res, err := http.PostForm(endpoint, url.Values{ "auth_token": {a.ApiToken}, "sync_status": {status}, "min_videos": {strconv.Itoa(1)}, - "after": {strconv.Itoa(int(cp.SyncFrom))}, - "before": {strconv.Itoa(int(cp.SyncUntil))}, + "after": {strconv.Itoa(int(cliFlags.SyncFrom))}, + "before": {strconv.Itoa(int(cliFlags.SyncUntil))}, "sync_server": {a.HostName}, - "channel_id": {cp.YoutubeChannelID}, + "channel_id": {cliFlags.ChannelID}, }) if err != nil { + if strings.Contains(err.Error(), "EOF") { + util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint) + time.Sleep(30 * time.Second) + return a.FetchChannels(status, cliFlags) + } return nil, errors.Err(err) } defer res.Body.Close() @@ -90,7 +61,7 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint) log.Debugln(string(body)) time.Sleep(30 * time.Second) - return a.FetchChannels(status, cp) + return a.FetchChannels(status, cliFlags) } var response apiJobsResponse err = json.Unmarshal(body, &response) @@ -126,7 +97,6 @@ func sanitizeFailureReason(s *string) { } func (a *APIConfig) SetChannelCert(certHex string, channelID string) error { - type apiSetChannelCertResponse struct { Success bool `json:"success"` Error null.String `json:"error"` @@ -141,6 +111,11 @@ func (a *APIConfig) SetChannelCert(certHex string, channelID string) error { "auth_token": {a.ApiToken}, }) if err != nil { + if strings.Contains(err.Error(), "EOF") { + util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint) + time.Sleep(30 * time.Second) + return a.SetChannelCert(certHex, channelID) + } return errors.Err(err) } defer res.Body.Close() @@ -184,6 +159,11 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea } res, err := http.PostForm(endpoint, params) if err != nil { + if strings.Contains(err.Error(), "EOF") { + util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint) + time.Sleep(30 * time.Second) + return a.SetChannelStatus(channelID, status, failureReason, transferState) + } return nil, nil, errors.Err(err) } defer res.Body.Close() @@ -229,6 +209,11 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e "channel_claim_id": {channelClaimID}, }) if err != nil { + if strings.Contains(err.Error(), "EOF") { + util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint) + time.Sleep(30 * time.Second) + return a.SetChannelClaimID(channelID, channelClaimID) + } return errors.Err(err) } defer res.Body.Close() @@ -268,6 +253,11 @@ func (a *APIConfig) DeleteVideos(videos []string) error { } res, err := http.PostForm(endpoint, vals) if err != nil { + if strings.Contains(err.Error(), "EOF") { + util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint) + time.Sleep(30 * time.Second) + return a.DeleteVideos(videos) + } return errors.Err(err) } defer res.Body.Close() @@ -297,19 +287,7 @@ func (a *APIConfig) DeleteVideos(videos []string) error { return errors.Err("invalid API response. Status code: %d", res.StatusCode) } -type VideoStatus struct { - ChannelID string - VideoID string - Status string - ClaimID string - ClaimName string - FailureReason string - Size *int64 - MetaDataVersion uint - IsTransferred *bool -} - -func (a *APIConfig) MarkVideoStatus(status VideoStatus) error { +func (a *APIConfig) MarkVideoStatus(status shared.VideoStatus) error { endpoint := a.ApiURL + "/yt/video_status" sanitizeFailureReason(&status.FailureReason) @@ -341,6 +319,11 @@ func (a *APIConfig) MarkVideoStatus(status VideoStatus) error { } res, err := http.PostForm(endpoint, vals) if err != nil { + if strings.Contains(err.Error(), "EOF") { + util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint) + time.Sleep(30 * time.Second) + return a.MarkVideoStatus(status) + } return errors.Err(err) } defer res.Body.Close() @@ -368,3 +351,102 @@ func (a *APIConfig) MarkVideoStatus(status VideoStatus) error { } return errors.Err("invalid API response. Status code: %d", res.StatusCode) } + +func (a *APIConfig) VideoState(videoID string) (string, error) { + endpoint := a.ApiURL + "/yt/video_state" + vals := url.Values{ + "video_id": {videoID}, + "auth_token": {a.ApiToken}, + } + + res, err := http.PostForm(endpoint, vals) + if err != nil { + if strings.Contains(err.Error(), "EOF") { + util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint) + time.Sleep(30 * time.Second) + return a.VideoState(videoID) + } + return "", errors.Err(err) + } + defer res.Body.Close() + body, _ := ioutil.ReadAll(res.Body) + if res.StatusCode == http.StatusNotFound { + return "not_found", nil + } + if res.StatusCode != http.StatusOK { + util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint) + log.Debugln(string(body)) + time.Sleep(30 * time.Second) + return a.VideoState(videoID) + } + var response struct { + Success bool `json:"success"` + Error null.String `json:"error"` + Data null.String `json:"data"` + } + err = json.Unmarshal(body, &response) + if err != nil { + return "", errors.Err(err) + } + if !response.Error.IsNull() { + return "", errors.Err(response.Error.String) + } + if !response.Data.IsNull() { + return response.Data.String, nil + } + return "", errors.Err("invalid API response. Status code: %d", res.StatusCode) +} + +type VideoRelease struct { + ID uint64 `json:"id"` + YoutubeDataID uint64 `json:"youtube_data_id"` + VideoID string `json:"video_id"` + ReleaseTime string `json:"release_time"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +func (a *APIConfig) GetReleasedDate(videoID string) (*VideoRelease, error) { + endpoint := a.ApiURL + "/yt/released" + vals := url.Values{ + "video_id": {videoID}, + "auth_token": {a.ApiToken}, + } + + res, err := http.PostForm(endpoint, vals) + if err != nil { + if strings.Contains(err.Error(), "EOF") { + util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint) + time.Sleep(30 * time.Second) + return a.GetReleasedDate(videoID) + } + return nil, errors.Err(err) + } + defer res.Body.Close() + body, _ := ioutil.ReadAll(res.Body) + if res.StatusCode == http.StatusNotFound { + return nil, nil + } + if res.StatusCode != http.StatusOK { + util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint) + log.Debugln(string(body)) + time.Sleep(30 * time.Second) + return a.GetReleasedDate(videoID) + } + var response struct { + Success bool `json:"success"` + Error null.String `json:"error"` + Data VideoRelease `json:"data"` + } + err = json.Unmarshal(body, &response) + if err != nil { + return nil, errors.Err(err) + } + if !response.Error.IsNull() { + return nil, errors.Err(response.Error.String) + } + if response.Data.ReleaseTime != "" { + return &response.Data, nil + } + return nil, errors.Err("invalid API response. Status code: %d", res.StatusCode) +} diff --git a/shared/shared.go b/shared/shared.go new file mode 100644 index 0000000..28b8077 --- /dev/null +++ b/shared/shared.go @@ -0,0 +1,125 @@ +package shared + +import ( + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" +) + +type Fee struct { + Amount string `json:"amount"` + Address string `json:"address"` + Currency string `json:"currency"` +} +type YoutubeChannel struct { + ChannelId string `json:"channel_id"` + TotalVideos uint `json:"total_videos"` + TotalSubscribers uint `json:"total_subscribers"` + DesiredChannelName string `json:"desired_channel_name"` + Fee *Fee `json:"fee"` + ChannelClaimID string `json:"channel_claim_id"` + TransferState int `json:"transfer_state"` + PublishAddress string `json:"publish_address"` + PublicKey string `json:"public_key"` + LengthLimit int `json:"length_limit"` + SizeLimit int `json:"size_limit"` + LastUploadedVideo string `json:"last_uploaded_video"` +} + +var NeverRetryFailures = []string{ + "Error extracting sts from embedded url response", + "Unable to extract signature tokens", + "the video is too big to sync, skipping for now", + "video is too long to process", + "This video contains content from", + "no compatible format available for this video", + "Watch this video on YouTube.", + "have blocked it on copyright grounds", + "giving up after 0 fragment retries", + "Sign in to confirm your age", +} + +type SyncFlags struct { + StopOnError bool + TakeOverExistingChannel bool + SkipSpaceCheck bool + SyncUpdate bool + SingleRun bool + RemoveDBUnpublished bool + UpgradeMetadata bool + DisableTransfers bool + QuickSync bool + MaxTries int + Refill int + Limit int + SyncStatus string + ChannelID string + SyncFrom int64 + SyncUntil int64 + ConcurrentJobs int + VideosLimit int + MaxVideoSize int + MaxVideoLength time.Duration +} + +func (f *SyncFlags) IsSingleChannelSync() bool { + return f.ChannelID != "" +} + +type VideoStatus struct { + ChannelID string + VideoID string + Status string + ClaimID string + ClaimName string + FailureReason string + Size *int64 + MetaDataVersion uint + IsTransferred *bool +} + +const ( + StatusPending = "pending" // waiting for permission to sync + StatusPendingEmail = "pendingemail" // permission granted but missing email + StatusQueued = "queued" // in sync queue. will be synced soon + StatusPendingUpgrade = "pendingupgrade" // in sync queue. will be synced soon + StatusSyncing = "syncing" // syncing now + StatusSynced = "synced" // done + StatusFailed = "failed" + StatusFinalized = "finalized" // no more changes allowed + StatusAbandoned = "abandoned" // deleted on youtube or banned +) + +var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned} + +const LatestMetadataVersion = 2 + +const ( + VideoStatusPublished = "published" + VideoStatusFailed = "failed" + VideoStatusUpgradeFailed = "upgradefailed" + VideoStatusUnpublished = "unpublished" + VideoStatusTranferFailed = "transferfailed" +) + +const ( + TransferStateNotTouched = iota + TransferStatePending + TransferStateComplete + TransferStateManual +) + +type AwsConfigs struct { + AwsS3ID string + AwsS3Secret string + AwsS3Region string + AwsS3Bucket string +} + +func (a *AwsConfigs) GetS3AWSConfig() *aws.Config { + return &aws.Config{ + Credentials: credentials.NewStaticCredentials(a.AwsS3ID, a.AwsS3Secret, ""), + Region: &a.AwsS3Region, + } +} diff --git a/sources/youtubeVideo.go b/sources/youtubeVideo.go index 26c2f66..6118745 100644 --- a/sources/youtubeVideo.go +++ b/sources/youtubeVideo.go @@ -3,7 +3,6 @@ package sources import ( "fmt" "io/ioutil" - "math" "os" "os/exec" "path/filepath" @@ -13,24 +12,25 @@ import ( "sync" "time" - "github.com/lbryio/lbry.go/v2/extras/errors" - "github.com/lbryio/lbry.go/v2/extras/jsonrpc" - "github.com/lbryio/lbry.go/v2/extras/stop" - "github.com/lbryio/lbry.go/v2/extras/util" - "github.com/lbryio/ytsync/v5/timing" - logUtils "github.com/lbryio/ytsync/v5/util" + "github.com/lbryio/ytsync/v5/downloader/ytdl" + "github.com/lbryio/ytsync/v5/shared" "github.com/lbryio/ytsync/v5/ip_manager" "github.com/lbryio/ytsync/v5/namer" "github.com/lbryio/ytsync/v5/sdk" "github.com/lbryio/ytsync/v5/tags_manager" "github.com/lbryio/ytsync/v5/thumbs" + "github.com/lbryio/ytsync/v5/timing" + logUtils "github.com/lbryio/ytsync/v5/util" + + "github.com/lbryio/lbry.go/v2/extras/errors" + "github.com/lbryio/lbry.go/v2/extras/jsonrpc" + "github.com/lbryio/lbry.go/v2/extras/stop" + "github.com/lbryio/lbry.go/v2/extras/util" - duration "github.com/ChannelMeter/iso8601duration" "github.com/aws/aws-sdk-go/aws" "github.com/shopspring/decimal" log "github.com/sirupsen/logrus" - "google.golang.org/api/youtube/v3" ) type YoutubeVideo struct { @@ -40,10 +40,10 @@ type YoutubeVideo struct { playlistPosition int64 size *int64 maxVideoSize int64 - maxVideoLength float64 + maxVideoLength time.Duration publishedAt time.Time dir string - youtubeInfo *youtube.Video + youtubeInfo *ytdl.YtdlVideo youtubeChannelID string tags []string awsConfig aws.Config @@ -90,22 +90,23 @@ var youtubeCategories = map[string]string{ "44": "trailers", } -func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo { - publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors +func NewYoutubeVideo(directory string, videoData *ytdl.YtdlVideo, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) (*YoutubeVideo, error) { + // youtube-dl returns times in local timezone sometimes. this could break in the future + // maybe we can file a PR to choose the timezone we want from youtube-dl return &YoutubeVideo{ - id: videoData.Id, - title: videoData.Snippet.Title, - description: videoData.Snippet.Description, + id: videoData.ID, + title: videoData.Title, + description: videoData.Description, playlistPosition: playlistPosition, - publishedAt: publishedAt, + publishedAt: videoData.UploadDateForReal, dir: directory, youtubeInfo: videoData, awsConfig: awsConfig, mocked: false, - youtubeChannelID: videoData.Snippet.ChannelId, + youtubeChannelID: videoData.ChannelID, stopGroup: stopGroup, pool: pool, - } + }, nil } func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo { return &YoutubeVideo{ @@ -185,7 +186,7 @@ func (v *YoutubeVideo) download() error { defer func(start time.Time) { timing.TimedComponent("download").Add(time.Since(start)) }(start) - if v.youtubeInfo.Snippet.LiveBroadcastContent != "none" { + if v.youtubeInfo.IsLive != nil { return errors.Err("video is a live stream and hasn't completed yet") } videoPath := v.getFullPath() @@ -208,6 +209,15 @@ func (v *YoutubeVideo) download() error { "480", "320", } + dur := time.Duration(v.youtubeInfo.Duration) * time.Second + if dur.Hours() > 2 { //for videos longer than 2 hours only sync up to 720p + qualities = []string{ + "720", + "480", + "320", + } + } + ytdlArgs := []string{ "--no-progress", "-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"), @@ -218,12 +228,11 @@ func (v *YoutubeVideo) download() error { "-movflags faststart", "--abort-on-unavailable-fragment", "--fragment-retries", - "0", - "--user-agent", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36", + "1", "--cookies", "cookies.txt", } + userAgent := []string{"--user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"} if v.maxVideoSize > 0 { ytdlArgs = append(ytdlArgs, "--max-filesize", @@ -233,7 +242,7 @@ func (v *YoutubeVideo) download() error { if v.maxVideoLength > 0 { ytdlArgs = append(ytdlArgs, "--match-filter", - fmt.Sprintf("duration <= %d", int(math.Round(v.maxVideoLength*3600))), + fmt.Sprintf("duration <= %d", int(v.maxVideoLength.Seconds())), ) } @@ -263,8 +272,10 @@ func (v *YoutubeVideo) download() error { "https://www.youtube.com/watch?v="+v.ID(), ) - for i, quality := range qualities { + for i := 0; i < len(qualities); i++ { + quality := qualities[i] argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+quality+"]+bestaudio[ext!=webm]") + argsWithFilters = append(argsWithFilters, userAgent...) cmd := exec.Command("youtube-dl", argsWithFilters...) log.Printf("Running command youtube-dl %s", strings.Join(argsWithFilters, " ")) @@ -293,6 +304,11 @@ func (v *YoutubeVideo) download() error { return errors.Err(string(errorLog)) } continue //this bypasses the yt throttling IP redistribution... TODO: don't + } else if strings.Contains(string(errorLog), "YouTube said: Unable to extract video data") && !strings.Contains(userAgent[1], "Googlebot") { + i-- //do not lower quality when trying a different user agent + userAgent = []string{"--user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"} + log.Infof("trying different user agent for video %s", v.ID()) + continue } return errors.Err(string(errorLog)) } @@ -370,8 +386,8 @@ func (v *YoutubeVideo) delete(reason string) error { } func (v *YoutubeVideo) triggerThumbnailSave() (err error) { - thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails) - v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig) + thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails) + v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig) return err } @@ -429,8 +445,8 @@ type SyncParams struct { ChannelID string MaxVideoSize int Namer *namer.Namer - MaxVideoLength float64 - Fee *sdk.Fee + MaxVideoLength time.Duration + Fee *shared.Fee DefaultAccount string } @@ -448,13 +464,11 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) { var err error - videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration) - if err != nil { - return nil, errors.Err(err) - } - if videoDuration.ToDuration() > time.Duration(v.maxVideoLength*60)*time.Minute { - log.Infof("%s is %s long and the limit is %s", v.id, videoDuration.ToDuration().String(), (time.Duration(v.maxVideoLength*60) * time.Minute).String()) - logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, videoDuration.ToDuration().String(), (time.Duration(v.maxVideoLength*60) * time.Minute).String()) + + dur := time.Duration(v.youtubeInfo.Duration) * time.Second + + if dur > v.maxVideoLength { + logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, dur.String(), v.maxVideoLength.String()) return nil, errors.Err("video is too long to process") } for { @@ -487,27 +501,30 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo locations = nil tags = nil if !v.mocked { - if v.youtubeInfo.Snippet.DefaultLanguage != "" { - if v.youtubeInfo.Snippet.DefaultLanguage == "iw" { - v.youtubeInfo.Snippet.DefaultLanguage = "he" - } - languages = []string{v.youtubeInfo.Snippet.DefaultLanguage} - } + /* + if v.youtubeInfo.Snippet.DefaultLanguage != "" { + if v.youtubeInfo.Snippet.DefaultLanguage == "iw" { + v.youtubeInfo.Snippet.DefaultLanguage = "he" + } + languages = []string{v.youtubeInfo.Snippet.DefaultLanguage} + }*/ - if v.youtubeInfo.RecordingDetails != nil && v.youtubeInfo.RecordingDetails.Location != nil { + /*if v.youtubeInfo.!= nil && v.youtubeInfo.RecordingDetails.Location != nil { locations = []jsonrpc.Location{{ Latitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Latitude)), Longitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Longitude)), }} - } - tags = v.youtubeInfo.Snippet.Tags + }*/ + tags = v.youtubeInfo.Tags } tags, err := tags_manager.SanitizeTags(tags, v.youtubeChannelID) if err != nil { log.Errorln(err.Error()) } if !v.mocked { - tags = append(tags, youtubeCategories[v.youtubeInfo.Snippet.CategoryId]) + for _, category := range v.youtubeInfo.Categories { + tags = append(tags, youtubeCategories[category]) + } } return languages, locations, tags @@ -532,8 +549,8 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis if v.mocked { return nil, errors.Err("could not find thumbnail for mocked video") } - thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails) - thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig) + thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails) + thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig) } else { thumbnailURL = thumbs.ThumbnailEndpoint + v.ID() } @@ -605,14 +622,9 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis }, nil } - videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration) - if err != nil { - return nil, errors.Err(err) - } - streamCreateOptions.ClaimCreateOptions.Title = &v.title streamCreateOptions.ClaimCreateOptions.Description = util.PtrToString(v.getAbbrevDescription()) - streamCreateOptions.Duration = util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))) + streamCreateOptions.Duration = util.PtrToUint64(uint64(v.youtubeInfo.Duration)) streamCreateOptions.ReleaseTime = util.PtrToInt64(v.publishedAt.Unix()) start := time.Now() pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{ diff --git a/thumbs/uploader.go b/thumbs/uploader.go index 6cf21cd..b61a05e 100644 --- a/thumbs/uploader.go +++ b/thumbs/uploader.go @@ -4,6 +4,9 @@ import ( "io" "net/http" "os" + "strings" + + "github.com/lbryio/ytsync/v5/downloader/ytdl" "github.com/lbryio/lbry.go/v2/extras/errors" @@ -11,7 +14,6 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3/s3manager" log "github.com/sirupsen/logrus" - "google.golang.org/api/youtube/v3" ) type thumbnailUploader struct { @@ -31,7 +33,9 @@ func (u *thumbnailUploader) downloadThumbnail() error { return errors.Err(err) } defer img.Close() - + if strings.HasPrefix(u.originalUrl, "//") { + u.originalUrl = "https:" + u.originalUrl + } resp, err := http.Get(u.originalUrl) if err != nil { return errors.Err(err) @@ -98,15 +102,14 @@ func MirrorThumbnail(url string, name string, s3Config aws.Config) (string, erro return tu.mirroredUrl, nil } -func GetBestThumbnail(thumbnails *youtube.ThumbnailDetails) *youtube.Thumbnail { - if thumbnails.Maxres != nil { - return thumbnails.Maxres - } else if thumbnails.High != nil { - return thumbnails.High - } else if thumbnails.Medium != nil { - return thumbnails.Medium - } else if thumbnails.Standard != nil { - return thumbnails.Standard +func GetBestThumbnail(thumbnails []ytdl.Thumbnail) *ytdl.Thumbnail { + var bestWidth *ytdl.Thumbnail + for _, thumbnail := range thumbnails { + if bestWidth == nil { + bestWidth = &thumbnail + } else if bestWidth.Width < thumbnail.Width { + bestWidth = &thumbnail + } } - return thumbnails.Default + return bestWidth } diff --git a/util/util.go b/util/util.go index 096ec71..d3b1ab1 100644 --- a/util/util.go +++ b/util/util.go @@ -52,7 +52,7 @@ func GetLBRYNetDir() string { } func GetLbryumDir() string { - lbryumDir := os.Getenv("LBRYNET_WALLETS_DIR") + lbryumDir := os.Getenv("LBRYUM_DIR") if lbryumDir == "" { usr, err := user.Current() if err != nil { @@ -255,7 +255,7 @@ func CleanupLbrynet() error { } return errors.Err(err) } - dbSizeLimit := int64(1 * 1024 * 1024 * 1024) + dbSizeLimit := int64(2 * 1024 * 1024 * 1024) if db.Size() > dbSizeLimit { files, err := filepath.Glob(lbryumDir + "/blockchain.db*") if err != nil { @@ -366,9 +366,31 @@ func GetDefaultWalletPath() string { defaultWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet" } - walletPath := os.Getenv("LBRYNET_WALLETS_DIR") + walletPath := os.Getenv("LBRYUM_DIR") if walletPath != "" { defaultWalletDir = walletPath + "/wallets/default_wallet" } return defaultWalletDir } +func GetBlockchainDBPath() string { + lbryumDir := os.Getenv("LBRYUM_DIR") + if lbryumDir == "" { + if IsRegTest() { + lbryumDir = os.Getenv("HOME") + "/.lbryum_regtest" + } else { + lbryumDir = os.Getenv("HOME") + "/.lbryum" + } + } + defaultDB := lbryumDir + "/lbc_mainnet/blockchain.db" + if IsRegTest() { + defaultDB = lbryumDir + "/lbc_regtest/blockchain.db" + } + return defaultDB +} +func GetBlockchainDirectoryName() string { + ledger := "lbc_mainnet" + if IsRegTest() { + ledger = "lbc_regtest" + } + return ledger +} diff --git a/ytapi/ytapi.go b/ytapi/ytapi.go new file mode 100644 index 0000000..32b99f9 --- /dev/null +++ b/ytapi/ytapi.go @@ -0,0 +1,387 @@ +package ytapi + +import ( + "bufio" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/lbryio/ytsync/v5/shared" + logUtils "github.com/lbryio/ytsync/v5/util" + + "github.com/lbryio/ytsync/v5/downloader/ytdl" + + "github.com/lbryio/ytsync/v5/downloader" + "github.com/lbryio/ytsync/v5/ip_manager" + "github.com/lbryio/ytsync/v5/sdk" + "github.com/lbryio/ytsync/v5/sources" + + "github.com/lbryio/lbry.go/v2/extras/errors" + "github.com/lbryio/lbry.go/v2/extras/jsonrpc" + "github.com/lbryio/lbry.go/v2/extras/stop" + "github.com/lbryio/lbry.go/v2/extras/util" + + "github.com/aws/aws-sdk-go/aws" + log "github.com/sirupsen/logrus" +) + +type Video interface { + Size() *int64 + ID() string + IDAndNum() string + PlaylistPosition() int + PublishedAt() time.Time + Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool, *sync.RWMutex) (*sources.SyncSummary, error) +} + +type byPublishedAt []Video + +func (a byPublishedAt) Len() int { return len(a) } +func (a byPublishedAt) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a byPublishedAt) Less(i, j int) bool { return a[i].PublishedAt().Before(a[j].PublishedAt()) } + +type VideoParams struct { + VideoDir string + S3Config aws.Config + Stopper *stop.Group + IPPool *ip_manager.IPPool +} + +var mostRecentlyFailedChannel string // TODO: fix this hack! + +func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams, lastUploadedVideo string) ([]Video, error) { + var videos []Video + if quickSync && maxVideos > 50 { + maxVideos = 50 + } + allVideos, err := downloader.GetPlaylistVideoIDs(channelID, maxVideos, videoParams.Stopper.Ch(), videoParams.IPPool) + if err != nil { + return nil, errors.Err(err) + } + videoIDs := make([]string, 0, len(allVideos)) + for _, video := range allVideos { + sv, ok := syncedVideos[video] + if ok && util.SubstringInSlice(sv.FailureReason, shared.NeverRetryFailures) { + continue + } + videoIDs = append(videoIDs, video) + } + log.Infof("Got info for %d videos from youtube downloader", len(videoIDs)) + + playlistMap := make(map[string]int64) + for i, videoID := range videoIDs { + playlistMap[videoID] = int64(i) + } + //this will ensure that we at least try to sync the video that was marked as last uploaded video in the database. + if lastUploadedVideo != "" { + _, ok := playlistMap[lastUploadedVideo] + if !ok { + playlistMap[lastUploadedVideo] = 0 + videoIDs = append(videoIDs, lastUploadedVideo) + } + } + + if len(videoIDs) < 1 { + if channelID == mostRecentlyFailedChannel { + return nil, errors.Err("playlist items not found") + } + mostRecentlyFailedChannel = channelID + } + + vids, err := getVideos(config, channelID, videoIDs, videoParams.Stopper.Ch(), videoParams.IPPool) + if err != nil { + return nil, err + } + + for _, item := range vids { + positionInList := playlistMap[item.ID] + videoToAdd, err := sources.NewYoutubeVideo(videoParams.VideoDir, item, positionInList, videoParams.S3Config, videoParams.Stopper, videoParams.IPPool) + if err != nil { + return nil, errors.Err(err) + } + videos = append(videos, videoToAdd) + } + + for k, v := range syncedVideos { + if !v.Published { + continue + } + if _, ok := playlistMap[k]; !ok { + videos = append(videos, sources.NewMockedVideo(videoParams.VideoDir, k, channelID, videoParams.S3Config, videoParams.Stopper, videoParams.IPPool)) + } + } + + sort.Sort(byPublishedAt(videos)) + + return videos, nil +} + +// CountVideosInChannel is unused for now... keeping it here just in case +func CountVideosInChannel(channelID string) (int, error) { + url := "https://socialblade.com/youtube/channel/" + channelID + + req, _ := http.NewRequest("GET", url, nil) + + req.Header.Add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36") + req.Header.Add("Accept", "*/*") + req.Header.Add("Host", "socialblade.com") + + res, err := http.DefaultClient.Do(req) + if err != nil { + return 0, errors.Err(err) + } + defer res.Body.Close() + + var line string + scanner := bufio.NewScanner(res.Body) + for scanner.Scan() { + if strings.Contains(scanner.Text(), "youtube-stats-header-uploads") { + line = scanner.Text() + break + } + } + + if err := scanner.Err(); err != nil { + return 0, err + } + if line == "" { + return 0, errors.Err("upload count line not found") + } + + matches := regexp.MustCompile(">([0-9]+)<").FindStringSubmatch(line) + if len(matches) != 2 { + return 0, errors.Err("upload count not found with regex") + } + + num, err := strconv.Atoi(matches[1]) + if err != nil { + return 0, errors.Err(err) + } + + return num, nil +} + +func ChannelInfo(channelID string) (*YoutubeStatsResponse, error) { + url := "https://www.youtube.com/channel/" + channelID + "/about" + + req, _ := http.NewRequest("GET", url, nil) + + req.Header.Add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36") + req.Header.Add("Accept", "*/*") + + res, err := http.DefaultClient.Do(req) + if err != nil { + return nil, errors.Err(err) + } + defer res.Body.Close() + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, errors.Err(err) + } + pageBody := string(body) + dataStartIndex := strings.Index(pageBody, "window[\"ytInitialData\"] = ") + 26 + dataEndIndex := strings.Index(pageBody, "]}}};") + 4 + + data := pageBody[dataStartIndex:dataEndIndex] + var decodedResponse YoutubeStatsResponse + err = json.Unmarshal([]byte(data), &decodedResponse) + if err != nil { + return nil, errors.Err(err) + } + + return &decodedResponse, nil +} + +func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool) ([]*ytdl.YtdlVideo, error) { + var videos []*ytdl.YtdlVideo + for _, videoID := range videoIDs { + if len(videoID) < 5 { + continue + } + select { + case <-stopChan: + return videos, errors.Err("interrupted by user") + default: + } + + //ip, err := ipPool.GetIP(videoID) + //if err != nil { + // return nil, err + //} + //video, err := downloader.GetVideoInformation(videoID, &net.TCPAddr{IP: net.ParseIP(ip)}) + state, err := config.VideoState(videoID) + if err != nil { + return nil, errors.Err(err) + } + if state == "published" { + continue + } + video, err := downloader.GetVideoInformation(config, videoID, stopChan, nil, ipPool) + if err != nil { + errSDK := config.MarkVideoStatus(shared.VideoStatus{ + ChannelID: channelID, + VideoID: videoID, + Status: "failed", + FailureReason: err.Error(), + }) + logUtils.SendErrorToSlack(fmt.Sprintf("Skipping video (%s): %s", videoID, errors.FullTrace(err))) + if errSDK != nil { + return nil, errors.Err(errSDK) + } + } else { + videos = append(videos, video) + } + } + return videos, nil +} + +type YoutubeStatsResponse struct { + Contents struct { + TwoColumnBrowseResultsRenderer struct { + Tabs []struct { + TabRenderer struct { + Title string `json:"title"` + Selected bool `json:"selected"` + Content struct { + SectionListRenderer struct { + Contents []struct { + ItemSectionRenderer struct { + Contents []struct { + ChannelAboutFullMetadataRenderer struct { + Description struct { + SimpleText string `json:"simpleText"` + } `json:"description"` + ViewCountText struct { + SimpleText string `json:"simpleText"` + } `json:"viewCountText"` + JoinedDateText struct { + Runs []struct { + Text string `json:"text"` + } `json:"runs"` + } `json:"joinedDateText"` + CanonicalChannelURL string `json:"canonicalChannelUrl"` + BypassBusinessEmailCaptcha bool `json:"bypassBusinessEmailCaptcha"` + Title struct { + SimpleText string `json:"simpleText"` + } `json:"title"` + Avatar struct { + Thumbnails []struct { + URL string `json:"url"` + Width int `json:"width"` + Height int `json:"height"` + } `json:"thumbnails"` + } `json:"avatar"` + ShowDescription bool `json:"showDescription"` + DescriptionLabel struct { + Runs []struct { + Text string `json:"text"` + } `json:"runs"` + } `json:"descriptionLabel"` + DetailsLabel struct { + Runs []struct { + Text string `json:"text"` + } `json:"runs"` + } `json:"detailsLabel"` + ChannelID string `json:"channelId"` + } `json:"channelAboutFullMetadataRenderer"` + } `json:"contents"` + } `json:"itemSectionRenderer"` + } `json:"contents"` + } `json:"sectionListRenderer"` + } `json:"content"` + } `json:"tabRenderer"` + } `json:"tabs"` + } `json:"twoColumnBrowseResultsRenderer"` + } `json:"contents"` + Header struct { + C4TabbedHeaderRenderer struct { + ChannelID string `json:"channelId"` + Title string `json:"title"` + Avatar struct { + Thumbnails []struct { + URL string `json:"url"` + Width int `json:"width"` + Height int `json:"height"` + } `json:"thumbnails"` + } `json:"avatar"` + Banner struct { + Thumbnails []struct { + URL string `json:"url"` + Width int `json:"width"` + Height int `json:"height"` + } `json:"thumbnails"` + } `json:"banner"` + VisitTracking struct { + RemarketingPing string `json:"remarketingPing"` + } `json:"visitTracking"` + SubscriberCountText struct { + SimpleText string `json:"simpleText"` + } `json:"subscriberCountText"` + } `json:"c4TabbedHeaderRenderer"` + } `json:"header"` + Metadata struct { + ChannelMetadataRenderer struct { + Title string `json:"title"` + Description string `json:"description"` + RssURL string `json:"rssUrl"` + ChannelConversionURL string `json:"channelConversionUrl"` + ExternalID string `json:"externalId"` + Keywords string `json:"keywords"` + OwnerUrls []string `json:"ownerUrls"` + Avatar struct { + Thumbnails []struct { + URL string `json:"url"` + Width int `json:"width"` + Height int `json:"height"` + } `json:"thumbnails"` + } `json:"avatar"` + ChannelURL string `json:"channelUrl"` + IsFamilySafe bool `json:"isFamilySafe"` + VanityChannelURL string `json:"vanityChannelUrl"` + } `json:"channelMetadataRenderer"` + } `json:"metadata"` + Topbar struct { + DesktopTopbarRenderer struct { + CountryCode string `json:"countryCode"` + } `json:"desktopTopbarRenderer"` + } `json:"topbar"` + Microformat struct { + MicroformatDataRenderer struct { + URLCanonical string `json:"urlCanonical"` + Title string `json:"title"` + Description string `json:"description"` + Thumbnail struct { + Thumbnails []struct { + URL string `json:"url"` + Width int `json:"width"` + Height int `json:"height"` + } `json:"thumbnails"` + } `json:"thumbnail"` + SiteName string `json:"siteName"` + AppName string `json:"appName"` + AndroidPackage string `json:"androidPackage"` + IosAppStoreID string `json:"iosAppStoreId"` + IosAppArguments string `json:"iosAppArguments"` + OgType string `json:"ogType"` + URLApplinksWeb string `json:"urlApplinksWeb"` + URLApplinksIos string `json:"urlApplinksIos"` + URLApplinksAndroid string `json:"urlApplinksAndroid"` + URLTwitterIos string `json:"urlTwitterIos"` + URLTwitterAndroid string `json:"urlTwitterAndroid"` + TwitterCardType string `json:"twitterCardType"` + TwitterSiteHandle string `json:"twitterSiteHandle"` + SchemaDotOrgType string `json:"schemaDotOrgType"` + Noindex bool `json:"noindex"` + Unlisted bool `json:"unlisted"` + FamilySafe bool `json:"familySafe"` + Tags []string `json:"tags"` + } `json:"microformatDataRenderer"` + } `json:"microformat"` +} diff --git a/ytapi/ytapi_test.go b/ytapi/ytapi_test.go new file mode 100644 index 0000000..93a612c --- /dev/null +++ b/ytapi/ytapi_test.go @@ -0,0 +1,12 @@ +package ytapi + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestChannelInfo(t *testing.T) { + _, _, err := ChannelInfo("", "UCNQfQvFMPnInwsU_iGYArJQ") + assert.NoError(t, err) +}