LBRYCommunity/ytsync
1
0
Fork 0
Code Issues39 Pull requests8 Projects Releases197 Packages Wiki Activity

Compare commits

..

No commits in common. "master" and "v5.7.10" have entirely different histories.
master ... v5.7.10

27 changed files with 724 additions and 1349 deletions

4
.gitignore vendored
View file

@ -4,7 +4,3 @@ e2e/supporty/supporty
.env
blobsfiles
ytsync_docker
e2e/config.json
e2e/cookies.txt

2
.travis.yml
View file

@ -2,7 +2,7 @@ os: linux
dist: bionic
language: go
go:
- 1.17.x
- 1.16.3
install: true

39
README.md
View file

@ -8,17 +8,23 @@ With the support of said database, the tool is also able to keep all the channel
# Requirements
- lbrynet SDK https://github.com/lbryio/lbry-sdk/releases (We strive to keep the latest release of ytsync compatible with the latest major release of the SDK)
- lbrynet SDK https://github.com/lbryio/lbry/releases (We strive to keep the latest release of ytsync compatible with the latest major release of the SDK)
- a lbrycrd node running (localhost or on a remote machine) with credits in it
- internal-apis (you cannot run this one yourself)
- python3-pip
- yt-dlp (`pip3 install -U yt-dlp`)
- ffmpeg (latest)
# Setup
- make sure daemon is stopped and can be controlled through `systemctl` (find example below)
- extract the ytsync binary anywhere
- create and fill `config.json` using [this example](config.json.example)
- add the environment variables necessary to the tool
- export SLACK_TOKEN="a-token-to-spam-your-slack"
- export SLACK_CHANNEL="youtube-status"
- export YOUTUBE_API_KEY="youtube-api-key"
- export LBRY_WEB_API="https://lbry-api-url-here"
- export LBRY_API_TOKEN="internal-apis-token-for-ytsync-user"
- export LBRYCRD_STRING="tcp://user:password@host:5429"
- export AWS_S3_ID="THE-ID-LIES-HERE"
- export AWS_S3_SECRET="THE-SECRET-LIES-HERE"
- export AWS_S3_REGION="us-east-1"
- export AWS_S3_BUCKET="ytsync-wallets"
## systemd script example
`/etc/systemd/system/lbrynet.service`
@ -49,26 +55,23 @@ Usage:
Flags:
--after int Specify from when to pull jobs [Unix time](Default: 0)
--before int Specify until when to pull jobs [Unix time](Default: current Unix time) (default 1669311891)
--before int Specify until when to pull jobs [Unix time](Default: current Unix time) (default current timestamp)
--channelID string If specified, only this channel will be synced.
--concurrent-jobs int how many jobs to process concurrently (default 1)
-h, --help help for ytsync
--limit int limit the amount of channels to sync
--max-length int Maximum video length to process (in hours) (default 2)
--max-length float Maximum video length to process (in hours) (default 2)
--max-size int Maximum video size to process (in MB) (default 2048)
--max-tries int Number of times to try a publish that fails (default 3)
--no-transfers Skips the transferring process of videos, channels and supports
--quick Look up only the last 50 videos from youtube
--remove-db-unpublished Remove videos from the database that are marked as published but aren't really published
--run-once Whether the process should be stopped after one cycle or not
--skip-space-check Do not perform free space check on startup
--status string Specify which queue to pull from. Overrides --update
--status2 string Specify which secondary queue to pull from.
--stop-on-error If a publish fails, stop all publishing and exit
--takeover-existing-channel If channel exists and we don't own it, take over the channel
--update Update previously synced channels instead of syncing new ones
--upgrade-metadata Upgrade videos if they're on the old metadata version
--videos-limit int how many videos to process per channel (leave 0 for automatic detection)
--videos-limit int how many videos to process per channel (default 1000)
```
## Running from Source
@ -85,17 +88,17 @@ Contributions to this project are welcome, encouraged, and compensated. For more
## Security
We take security seriously. Please contact [security@lbry.io](mailto:security@lbry.io) regarding any security issues. Our PGP key is [here](https://lbry.com/faq/pgp-key) if you need it.
We take security seriously. Please contact [security@lbry.io](mailto:security@lbry.io) regarding any security issues. Our PGP key is [here](https://keybase.io/lbry/key.asc) if you need it.
## Contact
The primary contact for this project is [Niko Storni](https://github.com/nikooo777) (niko@lbry.com).
The primary contact for this project is [Niko Storni](https://github.com/nikooo777) (niko@lbry.io).
## Additional Info and Links
- [https://lbry.com](https://lbry.com) - The live LBRY website
- [Discord Chat](https://chat.lbry.com) - A chat room for the LBRYians
- [Email us](mailto:hello@lbry.com) - LBRY Support email
- [https://lbry.io](https://lbry.io) - The live LBRY website
- [Discord Chat](https://chat.lbry.io) - A chat room for the LBRYians
- [Email us](mailto:hello@lbry.io) - LBRY Support email
- [Twitter](https://twitter.com/@lbryio) - LBRY Twitter page
- [Facebook](https://www.facebook.com/lbryio/) - LBRY Facebook page
- [Reddit](https://reddit.com/r/lbry) - LBRY Reddit page

35
config.json.example
View file

@ -1,35 +0,0 @@
{
"slack_token": "",
"slack_channel": "ytsync-dev",
"internal_apis_endpoint": "http://localhost:15400",
"internal_apis_auth_token": "ytsyntoken",
"lbrycrd_string": "tcp://lbry:lbry@localhost:15200",
"wallet_s3_config": {
"id": "",
"secret": "",
"region": "us-east-1",
"bucket": "ytsync-wallets",
"endpoint": ""
},
"blockchaindb_s3_config": {
"id": "",
"secret": "",
"region": "us-east-1",
"bucket": "blockchaindbs",
"endpoint": ""
},
"thumbnails_s3_config": {
"id": "",
"secret": "",
"region": "us-east-1",
"bucket": "thumbnails.lbry.com",
"endpoint": ""
},
"aws_thumbnails_s3_config": {
"id": "",
"secret": "",
"region": "us-east-1",
"bucket": "thumbnails.lbry.com",
"endpoint": ""
}
}

75
configs/configs.go
View file

@ -1,75 +0,0 @@
package configs
import (
"os"
"regexp"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
log "github.com/sirupsen/logrus"
"github.com/tkanos/gonfig"
)
type S3Configs struct {
ID string `json:"id"`
Secret string `json:"secret"`
Region string `json:"region"`
Bucket string `json:"bucket"`
Endpoint string `json:"endpoint"`
}
type Configs struct {
SlackToken string `json:"slack_token"`
SlackChannel string `json:"slack_channel"`
InternalApisEndpoint string `json:"internal_apis_endpoint"`
InternalApisAuthToken string `json:"internal_apis_auth_token"`
LbrycrdString string `json:"lbrycrd_string"`
WalletS3Config S3Configs `json:"wallet_s3_config"`
BlockchaindbS3Config S3Configs `json:"blockchaindb_s3_config"`
AWSThumbnailsS3Config S3Configs `json:"aws_thumbnails_s3_config"`
ThumbnailsS3Config S3Configs `json:"thumbnails_s3_config"`
}
var Configuration *Configs
func Init(configPath string) error {
if Configuration != nil {
return nil
}
c := Configs{}
err := gonfig.GetConf(configPath, &c)
if err != nil {
return errors.Err(err)
}
Configuration = &c
return nil
}
func (s *S3Configs) GetS3AWSConfig() *aws.Config {
return &aws.Config{
Credentials: credentials.NewStaticCredentials(s.ID, s.Secret, ""),
Region: &s.Region,
Endpoint: &s.Endpoint,
S3ForcePathStyle: aws.Bool(true),
}
}
func (c *Configs) GetHostname() string {
var hostname string
var err error
hostname, err = os.Hostname()
if err != nil {
log.Error("could not detect system hostname")
hostname = "ytsync_unknown"
}
reg, err := regexp.Compile("[^a-zA-Z0-9_]+")
if err == nil {
hostname = reg.ReplaceAllString(hostname, "_")
}
if len(hostname) > 30 {
hostname = hostname[0:30]
}
return hostname
}

93
downloader/downloader.go
View file

@ -36,9 +36,7 @@ func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan,
}
videoIDs := make([]string, 0, maxVideos)
for i, v := range ids {
if v == "" {
continue
}
logrus.Debugf("%d - video id %s", i, v)
if i >= maxVideos {
break
}
@ -49,7 +47,7 @@ func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan,
const releaseTimeFormat = "2006-01-02, 15:04:05 (MST)"
func GetVideoInformation(videoID string, stopChan stop.Chan, pool *ip_manager.IPPool) (*ytdl.YtdlVideo, error) {
func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Chan, ip *net.TCPAddr, pool *ip_manager.IPPool) (*ytdl.YtdlVideo, error) {
args := []string{
"--skip-download",
"--write-info-json",
@ -79,6 +77,50 @@ func GetVideoInformation(videoID string, stopChan stop.Chan, pool *ip_manager.IP
return nil, errors.Err(err)
}
// now get an accurate time
const maxTries = 5
tries := 0
GetTime:
tries++
t, err := getUploadTime(config, videoID, ip, video.UploadDate)
if err != nil {
//slack(":warning: Upload time error: %v", err)
if tries <= maxTries && (errors.Is(err, errNotScraped) || errors.Is(err, errUploadTimeEmpty) || errors.Is(err, errStatusParse) || errors.Is(err, errConnectionIssue)) {
err := triggerScrape(videoID, ip)
if err == nil {
time.Sleep(2 * time.Second) // let them scrape it
goto GetTime
} else {
//slack("triggering scrape returned error: %v", err)
}
} else if !errors.Is(err, errNotScraped) && !errors.Is(err, errUploadTimeEmpty) {
//slack(":warning: Error while trying to get accurate upload time for %s: %v", videoID, err)
if t == "" {
return nil, errors.Err(err)
} else {
t = "" //TODO: get rid of the other piece below?
}
}
// do fallback below
}
//slack("After all that, upload time for %s is %s", videoID, t)
if t != "" {
parsed, err := time.Parse("2006-01-02, 15:04:05 (MST)", t) // this will probably be UTC, but Go's timezone parsing is fucked up. it ignores the timezone in the date
if err != nil {
return nil, errors.Err(err)
}
//slack(":exclamation: Got an accurate time for %s", videoID)
video.UploadDateForReal = parsed
} else { //TODO: this is the piece that isn't needed!
slack(":warning: Could not get accurate time for %s. Falling back to time from upload ytdl: %s.", videoID, video.UploadDate)
// fall back to UploadDate from youtube-dl
video.UploadDateForReal, err = time.Parse("20060102", video.UploadDate)
if err != nil {
return nil, err
}
}
return video, nil
}
@ -168,7 +210,45 @@ func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr, uploa
}
}
if time.Now().AddDate(0, 0, -3).After(ytdlUploadDate) {
return ytdlUploadDate.Format(releaseTimeFormat), nil
}
client := getClient(ip)
req, err := http.NewRequest(http.MethodGet, "https://caa.iti.gr/get_verificationV3?url=https://www.youtube.com/watch?v="+videoID, nil)
if err != nil {
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
}
req.Header.Set("User-Agent", ChromeUA)
res, err := client.Do(req)
if err != nil {
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
}
defer res.Body.Close()
var uploadTime struct {
Time string `json:"video_upload_time"`
Message string `json:"message"`
Status string `json:"status"`
}
err = json.NewDecoder(res.Body).Decode(&uploadTime)
if err != nil {
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
}
if uploadTime.Status == "ERROR1" {
return ytdlUploadDate.Format(releaseTimeFormat), errNotScraped
}
if uploadTime.Status == "" && strings.HasPrefix(uploadTime.Message, "CANNOT_RETRIEVE_REPORT_FOR_VIDEO_") {
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err("cannot retrieve report for video")
}
if uploadTime.Time == "" {
return ytdlUploadDate.Format(releaseTimeFormat), errUploadTimeEmpty
}
return uploadTime.Time, nil
}
func getClient(ip *net.TCPAddr) *http.Client {
@ -194,7 +274,7 @@ func getClient(ip *net.TCPAddr) *http.Client {
const (
GoogleBotUA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
ChromeUA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36"
ChromeUA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"
maxAttempts = 3
extractionError = "YouTube said: Unable to extract video data"
throttledError = "HTTP Error 429"
@ -285,8 +365,7 @@ func runCmd(cmd *exec.Cmd, stopChan stop.Chan) ([]string, error) {
return nil, errors.Err("interrupted by user")
case err := <-done:
if err != nil {
//return nil, errors.Prefix("yt-dlp "+strings.Join(cmd.Args, " ")+" ["+string(errorLog)+"]", err)
return nil, errors.Prefix(string(errorLog), err)
return nil, errors.Prefix("yt-dlp "+strings.Join(cmd.Args, " ")+" ["+string(errorLog)+"]", err)
}
return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
}

24
downloader/downloader_test.go
View file

@ -3,11 +3,7 @@ package downloader
import (
"testing"
"github.com/lbryio/ytsync/v5/ip_manager"
"github.com/lbryio/ytsync/v5/sdk"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
)
@ -23,18 +19,24 @@ func TestGetPlaylistVideoIDs(t *testing.T) {
}
func TestGetVideoInformation(t *testing.T) {
s := stop.New()
ip, err := ip_manager.GetIPPool(s)
assert.NoError(t, err)
video, err := GetVideoInformation("kDGOHNpRjzc", s.Ch(), ip)
assert.NoError(t, err)
assert.NotNil(t, video)
video, err := GetVideoInformation(nil, "zj7pXM9gE5M", nil, nil, nil)
if err != nil {
logrus.Error(err)
}
if video != nil {
logrus.Info(video.ID)
}
}
func Test_getUploadTime(t *testing.T) {
configs := sdk.APIConfig{}
configs := sdk.APIConfig{
YoutubeAPIKey: "",
ApiURL: "https://api.lbry.com",
ApiToken: "Ht4NETrL5oWKyAaZkuSV68BKhtXkiLh5",
HostName: "test",
}
got, err := getUploadTime(&configs, "kDGOHNpRjzc", nil, "20060102")
assert.NoError(t, err)
t.Log(got)
}

253
downloader/ytdl/Video.go
View file

@ -2,136 +2,149 @@ package ytdl
import (
"time"
"github.com/lbryio/ytsync/v5/sdk"
"github.com/sirupsen/logrus"
)
type YtdlVideo struct {
ID string `json:"id"`
Title string `json:"title"`
Thumbnails []Thumbnail `json:"thumbnails"`
Description string `json:"description"`
ChannelID string `json:"channel_id"`
Duration int `json:"duration"`
Categories []string `json:"categories"`
Tags []string `json:"tags"`
IsLive bool `json:"is_live"`
LiveStatus string `json:"live_status"`
ReleaseTimestamp *int64 `json:"release_timestamp"`
uploadDateForReal *time.Time
Availability string `json:"availability"`
ReleaseDate string `json:"release_date"`
UploadDate string `json:"upload_date"`
UploadDateForReal time.Time // you need to manually set this since the value in the API doesn't include the time
Extractor string `json:"extractor"`
Series interface{} `json:"series"`
Format string `json:"format"`
Vbr interface{} `json:"vbr"`
Chapters interface{} `json:"chapters"`
Height int `json:"height"`
LikeCount interface{} `json:"like_count"`
Duration int `json:"duration"`
Fulltitle string `json:"fulltitle"`
PlaylistIndex interface{} `json:"playlist_index"`
Album interface{} `json:"album"`
ViewCount int `json:"view_count"`
Playlist interface{} `json:"playlist"`
Title string `json:"title"`
Filename string `json:"_filename"`
Creator interface{} `json:"creator"`
Ext string `json:"ext"`
ID string `json:"id"`
DislikeCount interface{} `json:"dislike_count"`
AverageRating float64 `json:"average_rating"`
Abr float64 `json:"abr"`
UploaderURL string `json:"uploader_url"`
Categories []string `json:"categories"`
Fps float64 `json:"fps"`
StretchedRatio interface{} `json:"stretched_ratio"`
SeasonNumber interface{} `json:"season_number"`
Annotations interface{} `json:"annotations"`
WebpageURLBasename string `json:"webpage_url_basename"`
Acodec string `json:"acodec"`
DisplayID string `json:"display_id"`
//RequestedFormats []RequestedFormat `json:"requested_formats"`
//AutomaticCaptions struct{} `json:"automatic_captions"`
Description string `json:"description"`
Tags []string `json:"tags"`
Track interface{} `json:"track"`
RequestedSubtitles interface{} `json:"requested_subtitles"`
StartTime interface{} `json:"start_time"`
Uploader string `json:"uploader"`
ExtractorKey string `json:"extractor_key"`
FormatID string `json:"format_id"`
EpisodeNumber interface{} `json:"episode_number"`
UploaderID string `json:"uploader_id"`
//Subtitles struct{} `json:"subtitles"`
ReleaseYear interface{} `json:"release_year"`
Thumbnails []Thumbnail `json:"thumbnails"`
License interface{} `json:"license"`
Artist interface{} `json:"artist"`
AgeLimit int `json:"age_limit"`
ReleaseDate interface{} `json:"release_date"`
AltTitle interface{} `json:"alt_title"`
Thumbnail string `json:"thumbnail"`
ChannelID string `json:"channel_id"`
IsLive interface{} `json:"is_live"`
Width int `json:"width"`
EndTime interface{} `json:"end_time"`
WebpageURL string `json:"webpage_url"`
Formats []Format `json:"formats"`
ChannelURL string `json:"channel_url"`
Resolution interface{} `json:"resolution"`
Vcodec string `json:"vcodec"`
}
//WasLive bool `json:"was_live"`
//Formats interface{} `json:"formats"`
//Thumbnail string `json:"thumbnail"`
//Uploader string `json:"uploader"`
//UploaderID string `json:"uploader_id"`
//UploaderURL string `json:"uploader_url"`
//ChannelURL string `json:"channel_url"`
//ViewCount int `json:"view_count"`
//AverageRating interface{} `json:"average_rating"`
//AgeLimit int `json:"age_limit"`
//WebpageURL string `json:"webpage_url"`
//PlayableInEmbed bool `json:"playable_in_embed"`
//AutomaticCaptions interface{} `json:"automatic_captions"`
//Subtitles interface{} `json:"subtitles"`
//Chapters interface{} `json:"chapters"`
//LikeCount int `json:"like_count"`
//Channel string `json:"channel"`
//ChannelFollowerCount int `json:"channel_follower_count"`
//OriginalURL string `json:"original_url"`
//WebpageURLBasename string `json:"webpage_url_basename"`
//WebpageURLDomain string `json:"webpage_url_domain"`
//Extractor string `json:"extractor"`
//ExtractorKey string `json:"extractor_key"`
//Playlist interface{} `json:"playlist"`
//PlaylistIndex interface{} `json:"playlist_index"`
//DisplayID string `json:"display_id"`
//Fulltitle string `json:"fulltitle"`
//DurationString string `json:"duration_string"`
//RequestedSubtitles interface{} `json:"requested_subtitles"`
//HasDrm bool `json:"__has_drm"`
//RequestedFormats interface{} `json:"requested_formats"`
//Format string `json:"format"`
//FormatID string `json:"format_id"`
//Ext string `json:"ext"`
//Protocol string `json:"protocol"`
//Language interface{} `json:"language"`
//FormatNote string `json:"format_note"`
//FilesizeApprox int `json:"filesize_approx"`
//Tbr float64 `json:"tbr"`
//Width int `json:"width"`
//Height int `json:"height"`
//Resolution string `json:"resolution"`
//Fps int `json:"fps"`
//DynamicRange string `json:"dynamic_range"`
//Vcodec string `json:"vcodec"`
//Vbr float64 `json:"vbr"`
//StretchedRatio interface{} `json:"stretched_ratio"`
//Acodec string `json:"acodec"`
//Abr float64 `json:"abr"`
//Asr int `json:"asr"`
//Epoch int `json:"epoch"`
//Filename string `json:"filename"`
//Urls string `json:"urls"`
//Type string `json:"_type"`
type RequestedFormat struct {
Asr interface{} `json:"asr"`
Tbr float64 `json:"tbr"`
Container string `json:"container"`
Language interface{} `json:"language"`
Format string `json:"format"`
URL string `json:"url"`
Vcodec string `json:"vcodec"`
FormatNote string `json:"format_note"`
Height int `json:"height"`
Width int `json:"width"`
Ext string `json:"ext"`
FragmentBaseURL string `json:"fragment_base_url"`
Filesize interface{} `json:"filesize"`
Fps float64 `json:"fps"`
ManifestURL string `json:"manifest_url"`
Protocol string `json:"protocol"`
FormatID string `json:"format_id"`
HTTPHeaders struct {
AcceptCharset string `json:"Accept-Charset"`
AcceptLanguage string `json:"Accept-Language"`
AcceptEncoding string `json:"Accept-Encoding"`
Accept string `json:"Accept"`
UserAgent string `json:"User-Agent"`
} `json:"http_headers"`
Fragments []struct {
Path string `json:"path"`
Duration float64 `json:"duration,omitempty"`
} `json:"fragments"`
Acodec string `json:"acodec"`
Abr int `json:"abr,omitempty"`
}
type Format struct {
Asr int `json:"asr"`
Filesize int `json:"filesize"`
FormatID string `json:"format_id"`
FormatNote string `json:"format_note"`
Fps interface{} `json:"fps"`
Height interface{} `json:"height"`
Quality int `json:"quality"`
Tbr float64 `json:"tbr"`
URL string `json:"url"`
Width interface{} `json:"width"`
Ext string `json:"ext"`
Vcodec string `json:"vcodec"`
Acodec string `json:"acodec"`
Abr float64 `json:"abr,omitempty"`
DownloaderOptions struct {
HTTPChunkSize int `json:"http_chunk_size"`
} `json:"downloader_options,omitempty"`
Container string `json:"container,omitempty"`
Format string `json:"format"`
Protocol string `json:"protocol"`
HTTPHeaders struct {
UserAgent string `json:"User-Agent"`
AcceptCharset string `json:"Accept-Charset"`
Accept string `json:"Accept"`
AcceptEncoding string `json:"Accept-Encoding"`
AcceptLanguage string `json:"Accept-Language"`
} `json:"http_headers"`
Vbr float64 `json:"vbr,omitempty"`
}
type Thumbnail struct {
URL string `json:"url"`
Preference int `json:"preference"`
Width int `json:"width"`
Resolution string `json:"resolution"`
ID string `json:"id"`
Height int `json:"height,omitempty"`
Width int `json:"width,omitempty"`
Resolution string `json:"resolution,omitempty"`
Height int `json:"height"`
}
func (v *YtdlVideo) GetUploadTime() time.Time {
//priority list:
// release timestamp from yt
// release timestamp from morty
// release date from yt
// upload date from yt
if v.uploadDateForReal != nil {
return *v.uploadDateForReal
}
var ytdlReleaseTimestamp time.Time
if v.ReleaseTimestamp != nil && *v.ReleaseTimestamp > 0 {
ytdlReleaseTimestamp = time.Unix(*v.ReleaseTimestamp, 0).UTC()
}
//get morty timestamp
var mortyReleaseTimestamp time.Time
mortyRelease, err := sdk.GetAPIsConfigs().GetReleasedDate(v.ID)
if err != nil {
logrus.Error(err)
} else if mortyRelease != nil {
mortyReleaseTimestamp, err = time.ParseInLocation(time.RFC3339, mortyRelease.ReleaseTime, time.UTC)
if err != nil {
logrus.Error(err)
}
}
ytdlReleaseDate, err := time.Parse("20060102", v.ReleaseDate)
if err != nil {
logrus.Error(err)
}
ytdlUploadDate, err := time.Parse("20060102", v.UploadDate)
if err != nil {
logrus.Error(err)
}
if !ytdlReleaseTimestamp.IsZero() {
v.uploadDateForReal = &ytdlReleaseTimestamp
} else if !mortyReleaseTimestamp.IsZero() {
v.uploadDateForReal = &mortyReleaseTimestamp
} else if !ytdlReleaseDate.IsZero() {
v.uploadDateForReal = &ytdlReleaseDate
} else {
v.uploadDateForReal = &ytdlUploadDate
}
return *v.uploadDateForReal
type HTTPHeaders struct {
AcceptCharset string `json:"Accept-Charset"`
AcceptLanguage string `json:"Accept-Language"`
AcceptEncoding string `json:"Accept-Encoding"`
Accept string `json:"Accept"`
UserAgent string `json:"User-Agent"`
}

7
e2e/docker-compose.yml
View file

@ -21,7 +21,7 @@ services:
## Wallet Server ##
###################
walletserver:
image: lbry/wallet-server:v0.101.1
image: lbry/wallet-server:latest-release
restart: always
environment:
- DB_DIRECTORY=/database
@ -81,7 +81,6 @@ services:
- walletserver
environment:
- LBRY_STREAMING_SERVER=0.0.0.0:5280
- LBRY_FEE_PER_NAME_CHAR=0
volumes:
- "./persist/.lbrynet:/home/lbrynet"
- ".:/etc/lbry" #Put your daemon_settings.yml here
@ -110,7 +109,7 @@ services:
## Internal APIs ##
###################
internalapis:
image: odyseeteam/internal-apis:master
image: lbry/internal-apis:master
restart: "no"
ports:
- "15400:8080"
@ -128,7 +127,7 @@ services:
## Chainquery ##
################
chainquery:
image: odyseeteam/chainquery:master
image: lbry/chainquery:master
restart: "no"
ports:
- 6300:6300

11
e2e/e2e.sh
View file

@ -14,8 +14,11 @@ export LOCAL_TMP_DIR="/var/tmp:/var/tmp"
touch -a .env && set -o allexport; source ./.env; set +o allexport
echo "LOCAL_TMP_DIR=$LOCAL_TMP_DIR"
# Compose settings - docker only
export SLACK_CHANNEL="ytsync-travis"
export LBRY_API_TOKEN="ytsyntoken"
export LBRY_WEB_API="http://localhost:15400"
export LBRYNET_ADDRESS="http://localhost:15100"
export LBRYCRD_STRING="tcp://lbry:lbry@localhost:15200" #required for supporty
export LBRYCRD_STRING="tcp://lbry:lbry@localhost:15200"
export LBRYNET_USE_DOCKER=true
export REFLECT_BLOBS=false
export CLEAN_ON_STARTUP=true
@ -47,9 +50,9 @@ until curl --output /dev/null --silent --head --fail http://localhost:15400; do
done
echo "successfully started..."
channelToSync="UCMn-zv1SE-2y6vyewscfFqw"
channelName=@whatever"$(date +%s)"
latestVideoID="yPJgjiMbmX0"
channelToSync="UCGyoEsIRjmnmzrsB67DhrOA"
channelName=@Alaminemoh11"$(date +%s)"
latestVideoID="ejWF7Jjdgmc"
#Data Setup for test
./data_setup.sh "$channelName" "$channelToSync" "$latestVideoID"

167
go.mod
View file

@ -1,5 +1,3 @@
go 1.17
module github.com/lbryio/ytsync/v5
replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
@ -8,144 +6,37 @@ replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203
//replace github.com/lbryio/reflector.go => /home/niko/go/src/github.com/lbryio/reflector.go/
require (
github.com/Microsoft/go-winio v0.4.14 // indirect
github.com/abadojack/whatlanggo v1.0.1
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d
github.com/aws/aws-sdk-go v1.44.6
github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb
github.com/aws/aws-sdk-go v1.25.9
github.com/davecgh/go-spew v1.1.1
github.com/docker/docker v20.10.17+incompatible
github.com/lbryio/lbry.go/v2 v2.7.2-0.20220815204100-2adb8af5b68c
github.com/lbryio/reflector.go v1.1.3-0.20220730181028-f5d30b1a6e79
github.com/mitchellh/go-ps v1.0.0
github.com/prometheus/client_golang v1.12.1
github.com/shopspring/decimal v1.3.1
github.com/sirupsen/logrus v1.9.0
github.com/spf13/cobra v1.4.0
github.com/stretchr/testify v1.7.1
github.com/tkanos/gonfig v0.0.0-20210106201359-53e13348de2f
github.com/vbauerster/mpb/v7 v7.4.1
gopkg.in/vansante/go-ffprobe.v2 v2.0.3
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/docker/docker v1.13.1
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
github.com/hashicorp/memberlist v0.1.5 // indirect
github.com/hashicorp/serf v0.8.5 // indirect
github.com/kr/pretty v0.2.1 // indirect
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d
github.com/lbryio/reflector.go v1.1.3-0.20210412225256-4392c9724262
github.com/miekg/dns v1.1.22 // indirect
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 // indirect
github.com/prometheus/client_golang v0.9.3
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5
github.com/sirupsen/logrus v1.4.2
github.com/spf13/cobra v0.0.5
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/testify v1.7.0
github.com/vbauerster/mpb/v7 v7.0.2
google.golang.org/appengine v1.6.5 // indirect
gopkg.in/ini.v1 v1.60.2 // indirect
gopkg.in/vansante/go-ffprobe.v2 v2.0.2
gotest.tools v2.2.0+incompatible
)
require (
github.com/Microsoft/go-winio v0.5.1 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bluele/gcache v0.0.2 // indirect
github.com/brk0v/directio v0.0.0-20190225130936-69406e757cf7 // indirect
github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 // indirect
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d // indirect
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cheekybits/genny v1.0.0 // indirect
github.com/docker/distribution v2.8.1+incompatible // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/ekyoung/gin-nice-recovery v0.0.0-20160510022553-1654dca486db // indirect
github.com/fatih/structs v1.1.0 // indirect
github.com/fsnotify/fsnotify v1.4.9 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/gin-gonic/gin v1.7.7 // indirect
github.com/go-errors/errors v1.1.1 // indirect
github.com/go-ini/ini v1.48.0 // indirect
github.com/go-playground/locales v0.13.0 // indirect
github.com/go-playground/universal-translator v0.17.0 // indirect
github.com/go-playground/validator/v10 v10.4.1 // indirect
github.com/go-sql-driver/mysql v1.6.0 // indirect
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
github.com/gofrs/uuid v3.2.0+incompatible // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/btree v1.0.1 // indirect
github.com/google/go-cmp v0.5.7 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/gorilla/rpc v1.2.0 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
github.com/hashicorp/go-msgpack v0.5.5 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/hashicorp/memberlist v0.3.0 // indirect
github.com/hashicorp/serf v0.9.7 // indirect
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 // indirect
github.com/johntdyer/slackrus v0.0.0-20211215141436-33e4a270affb // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/karrick/godirwalk v1.17.0 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/lbryio/chainquery v1.9.0 // indirect
github.com/lbryio/lbry.go v1.1.2 // indirect
github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6 // indirect
github.com/leodido/go-urn v1.2.0 // indirect
github.com/lucas-clemente/quic-go v0.28.1 // indirect
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 // indirect
github.com/magiconair/properties v1.8.1 // indirect
github.com/marten-seemann/qpack v0.2.1 // indirect
github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
github.com/marten-seemann/qtls-go1-17 v0.1.2 // indirect
github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect
github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1 // indirect
github.com/mattn/go-isatty v0.0.12 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/miekg/dns v1.1.41 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/nxadm/tail v1.4.8 // indirect
github.com/onsi/ginkgo v1.16.4 // indirect
github.com/onsi/gomega v1.17.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
github.com/pelletier/go-toml v1.9.3 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
github.com/slack-go/slack v0.10.3 // indirect
github.com/spf13/afero v1.4.1 // indirect
github.com/spf13/cast v1.4.1 // indirect
github.com/spf13/jwalterweatherman v1.0.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/spf13/viper v1.7.1 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/ugorji/go/codec v1.1.7 // indirect
github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d // indirect
github.com/volatiletech/null v8.0.0+incompatible // indirect
github.com/volatiletech/sqlboiler v3.4.0+incompatible // indirect
github.com/ybbus/jsonrpc v2.1.2+incompatible // indirect
go.uber.org/atomic v1.9.0 // indirect
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
golang.org/x/mod v0.4.2 // indirect
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
golang.org/x/tools v0.1.5 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/ini.v1 v1.60.2 // indirect
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
gotest.tools/v3 v3.2.0 // indirect
)
go 1.13

604
go.sum
View file

File diff suppressed because it is too large Load diff

91
main.go
View file

@ -7,15 +7,14 @@ import (
"os"
"time"
"github.com/lbryio/ytsync/v5/configs"
"github.com/lbryio/ytsync/v5/manager"
"github.com/lbryio/ytsync/v5/shared"
ytUtils "github.com/lbryio/ytsync/v5/util"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/lbryio/ytsync/v5/manager"
"github.com/lbryio/ytsync/v5/sdk"
"github.com/lbryio/ytsync/v5/shared"
ytUtils "github.com/lbryio/ytsync/v5/util"
"github.com/prometheus/client_golang/prometheus/promhttp"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
@ -32,10 +31,6 @@ var (
func main() {
rand.Seed(time.Now().UnixNano())
log.SetLevel(log.DebugLevel)
customFormatter := new(log.TextFormatter)
customFormatter.TimestampFormat = "2006-01-02 15:04:05"
customFormatter.FullTimestamp = true
log.SetFormatter(customFormatter)
http.Handle("/metrics", promhttp.Handler())
go func() {
log.Error(http.ListenAndServe(":2112", nil))
@ -74,15 +69,22 @@ func main() {
}
func ytSync(cmd *cobra.Command, args []string) {
err := configs.Init("./config.json")
var hostname string
slackToken := os.Getenv("SLACK_TOKEN")
if slackToken == "" {
log.Error("A slack token was not present in env vars! Slack messages disabled!")
} else {
var err error
hostname, err = os.Hostname()
if err != nil {
log.Fatalf("could not parse configuration file: %s", errors.FullTrace(err))
log.Error("could not detect system hostname")
hostname = "ytsync-unknown"
}
if len(hostname) > 30 {
hostname = hostname[0:30]
}
if configs.Configuration.SlackToken == "" {
log.Error("A slack token was not present in the config! Slack messages disabled!")
} else {
util.InitSlack(configs.Configuration.SlackToken, configs.Configuration.SlackChannel, configs.Configuration.GetHostname())
util.InitSlack(os.Getenv("SLACK_TOKEN"), os.Getenv("SLACK_CHANNEL"), hostname)
}
if cliFlags.Status != "" && !util.InSlice(cliFlags.Status, shared.SyncStatuses) {
@ -101,33 +103,68 @@ func ytSync(cmd *cobra.Command, args []string) {
}
cliFlags.MaxVideoLength = time.Duration(maxVideoLength) * time.Hour
if configs.Configuration.InternalApisEndpoint == "" {
log.Errorln("An Internal APIs Endpoint was not defined")
apiURL := os.Getenv("LBRY_WEB_API")
apiToken := os.Getenv("LBRY_API_TOKEN")
youtubeAPIKey := os.Getenv("YOUTUBE_API_KEY")
lbrycrdDsn := os.Getenv("LBRYCRD_STRING")
awsS3ID := os.Getenv("AWS_S3_ID")
awsS3Secret := os.Getenv("AWS_S3_SECRET")
awsS3Region := os.Getenv("AWS_S3_REGION")
awsS3Bucket := os.Getenv("AWS_S3_BUCKET")
if apiURL == "" {
log.Errorln("An API URL was not defined. Please set the environment variable LBRY_WEB_API")
return
}
if configs.Configuration.InternalApisAuthToken == "" {
log.Errorln("An Internal APIs auth token was not defined")
if apiToken == "" {
log.Errorln("An API Token was not defined. Please set the environment variable LBRY_API_TOKEN")
return
}
if configs.Configuration.WalletS3Config.ID == "" || configs.Configuration.WalletS3Config.Region == "" || configs.Configuration.WalletS3Config.Bucket == "" || configs.Configuration.WalletS3Config.Secret == "" || configs.Configuration.WalletS3Config.Endpoint == "" {
log.Errorln("Wallet S3 configuration is incomplete")
if youtubeAPIKey == "" {
log.Errorln("A Youtube API key was not defined. Please set the environment variable YOUTUBE_API_KEY")
return
}
if configs.Configuration.BlockchaindbS3Config.ID == "" || configs.Configuration.BlockchaindbS3Config.Region == "" || configs.Configuration.BlockchaindbS3Config.Bucket == "" || configs.Configuration.BlockchaindbS3Config.Secret == "" || configs.Configuration.BlockchaindbS3Config.Endpoint == "" {
log.Errorln("Blockchain DBs S3 configuration is incomplete")
if awsS3ID == "" {
log.Errorln("AWS S3 ID credentials were not defined. Please set the environment variable AWS_S3_ID")
return
}
if configs.Configuration.LbrycrdString == "" {
log.Infoln("Using default (local) lbrycrd instance. Set lbrycrd_string if you want to use something else")
if awsS3Secret == "" {
log.Errorln("AWS S3 Secret credentials were not defined. Please set the environment variable AWS_S3_SECRET")
return
}
if awsS3Region == "" {
log.Errorln("AWS S3 Region was not defined. Please set the environment variable AWS_S3_REGION")
return
}
if awsS3Bucket == "" {
log.Errorln("AWS S3 Bucket was not defined. Please set the environment variable AWS_S3_BUCKET")
return
}
if lbrycrdDsn == "" {
log.Infoln("Using default (local) lbrycrd instance. Set LBRYCRD_STRING if you want to use something else")
}
blobsDir := ytUtils.GetBlobsDir()
apiConfig := &sdk.APIConfig{
YoutubeAPIKey: youtubeAPIKey,
ApiURL: apiURL,
ApiToken: apiToken,
HostName: hostname,
}
awsConfig := &shared.AwsConfigs{
AwsS3ID: awsS3ID,
AwsS3Secret: awsS3Secret,
AwsS3Region: awsS3Region,
AwsS3Bucket: awsS3Bucket,
}
sm := manager.NewSyncManager(
cliFlags,
blobsDir,
lbrycrdDsn,
awsConfig,
apiConfig,
)
err = sm.Start()
err := sm.Start()
if err != nil {
ytUtils.SendErrorToSlack(errors.FullTrace(err))
}

11
manager/manager.go
View file

@ -8,7 +8,6 @@ import (
"time"
"github.com/lbryio/ytsync/v5/blobs_reflector"
"github.com/lbryio/ytsync/v5/configs"
"github.com/lbryio/ytsync/v5/ip_manager"
"github.com/lbryio/ytsync/v5/namer"
"github.com/lbryio/ytsync/v5/sdk"
@ -25,17 +24,19 @@ type SyncManager struct {
CliFlags shared.SyncFlags
ApiConfig *sdk.APIConfig
LbrycrdDsn string
AwsConfigs *shared.AwsConfigs
blobsDir string
channelsToSync []Sync
}
func NewSyncManager(cliFlags shared.SyncFlags, blobsDir string) *SyncManager {
func NewSyncManager(cliFlags shared.SyncFlags, blobsDir, lbrycrdDsn string, awsConfigs *shared.AwsConfigs, apiConfig *sdk.APIConfig) *SyncManager {
return &SyncManager{
CliFlags: cliFlags,
blobsDir: blobsDir,
LbrycrdDsn: configs.Configuration.LbrycrdString,
ApiConfig: sdk.GetAPIsConfigs(),
LbrycrdDsn: lbrycrdDsn,
AwsConfigs: awsConfigs,
ApiConfig: apiConfig,
}
}
func (s *SyncManager) enqueueChannel(channel *shared.YoutubeChannel) {
@ -138,7 +139,7 @@ func (s *SyncManager) Start() error {
"WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR",
"NotEnoughFunds",
"no space left on device",
"there was a problem uploading the wallet",
"failure uploading wallet",
"the channel in the wallet is different than the channel in the database",
"this channel does not belong to this wallet!",
"You already have a stream claim published under the name",

113
manager/s3_storage.go
View file

@ -3,13 +3,6 @@ package manager
import (
"os"
"path/filepath"
"strings"
"time"
"github.com/lbryio/ytsync/v5/configs"
"github.com/lbryio/ytsync/v5/util"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
@ -17,21 +10,24 @@ import (
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
log "github.com/sirupsen/logrus"
"github.com/lbryio/lbry.go/v2/extras/errors"
logUtils "github.com/lbryio/ytsync/v5/util"
)
func (s *Sync) getS3Downloader(config *aws.Config) (*s3manager.Downloader, error) {
s3Session, err := session.NewSession(config)
func (s *Sync) getS3Downloader() (*s3manager.Downloader, error) {
s3Session, err := session.NewSession(s.Manager.AwsConfigs.GetS3AWSConfig())
if err != nil {
return nil, errors.Prefix("error starting session", err)
return nil, errors.Prefix("error starting session: ", err)
}
downloader := s3manager.NewDownloader(s3Session)
return downloader, nil
}
func (s *Sync) getS3Uploader(config *aws.Config) (*s3manager.Uploader, error) {
s3Session, err := session.NewSession(config)
func (s *Sync) getS3Uploader() (*s3manager.Uploader, error) {
s3Session, err := session.NewSession(s.Manager.AwsConfigs.GetS3AWSConfig())
if err != nil {
return nil, errors.Prefix("error starting session", err)
return nil, errors.Prefix("error starting session: ", err)
}
uploader := s3manager.NewUploader(s3Session)
return uploader, nil
@ -42,18 +38,18 @@ func (s *Sync) downloadWallet() error {
if err != nil {
return errors.Err(err)
}
downloader, err := s.getS3Downloader(configs.Configuration.WalletS3Config.GetS3AWSConfig())
downloader, err := s.getS3Downloader()
if err != nil {
return err
}
out, err := os.Create(defaultTempWalletDir)
if err != nil {
return errors.Prefix("error creating temp wallet", err)
return errors.Prefix("error creating temp wallet: ", err)
}
defer out.Close()
bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
Bucket: aws.String(configs.Configuration.WalletS3Config.Bucket),
Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
Key: key,
})
if err != nil {
@ -78,21 +74,21 @@ func (s *Sync) downloadWallet() error {
err = os.Rename(defaultTempWalletDir, defaultWalletDir)
if err != nil {
return errors.Prefix("error replacing temp wallet for default wallet", err)
return errors.Prefix("error replacing temp wallet for default wallet: ", err)
}
return nil
}
func (s *Sync) downloadBlockchainDB() error {
if util.IsRegTest() {
if logUtils.IsRegTest() {
return nil // tests fail if we re-use the same blockchain DB
}
defaultBDBPath, defaultTempBDBPath, key, err := s.getBlockchainDBPaths()
defaultBDBDir, defaultTempBDBDir, key, err := s.getBlockchainDBPaths()
if err != nil {
return errors.Err(err)
}
files, err := filepath.Glob(defaultBDBPath + "*")
files, err := filepath.Glob(defaultBDBDir + "*")
if err != nil {
return errors.Err(err)
}
@ -105,18 +101,18 @@ func (s *Sync) downloadBlockchainDB() error {
if s.DbChannelData.WipeDB {
return nil
}
downloader, err := s.getS3Downloader(configs.Configuration.BlockchaindbS3Config.GetS3AWSConfig())
downloader, err := s.getS3Downloader()
if err != nil {
return errors.Err(err)
}
out, err := os.Create(defaultTempBDBPath)
out, err := os.Create(defaultTempBDBDir)
if err != nil {
return errors.Prefix("error creating temp blockchain DB file", err)
return errors.Prefix("error creating temp wallet: ", err)
}
defer out.Close()
bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
Bucket: aws.String(configs.Configuration.BlockchaindbS3Config.Bucket),
Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
Key: key,
})
if err != nil {
@ -139,16 +135,11 @@ func (s *Sync) downloadBlockchainDB() error {
return errors.Err("zero bytes written")
}
blockchainDbDir := strings.Replace(defaultBDBPath, "blockchain.db", "", -1)
err = util.Untar(defaultTempBDBPath, blockchainDbDir)
err = os.Rename(defaultTempBDBDir, defaultBDBDir)
if err != nil {
return errors.Prefix("error extracting blockchain.db files", err)
return errors.Prefix("error replacing temp blockchain.db for default blockchain.db: ", err)
}
err = os.Remove(defaultTempBDBPath)
if err != nil {
return errors.Err(err)
}
log.Printf("blockchain.db data downloaded and extracted to %s", blockchainDbDir)
log.Printf("blockchain.db downloaded to %s", defaultBDBDir)
return nil
}
@ -156,7 +147,7 @@ func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string,
defaultWallet = os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
tempWallet = os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet"
key = aws.String("/wallets/" + s.DbChannelData.ChannelId)
if util.IsRegTest() {
if logUtils.IsRegTest() {
defaultWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
tempWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet"
key = aws.String("/regtest/" + s.DbChannelData.ChannelId)
@ -177,27 +168,27 @@ func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string,
func (s *Sync) getBlockchainDBPaths() (defaultDB, tempDB string, key *string, err error) {
lbryumDir := os.Getenv("LBRYUM_DIR")
if lbryumDir == "" {
if util.IsRegTest() {
if logUtils.IsRegTest() {
lbryumDir = os.Getenv("HOME") + "/.lbryum_regtest"
} else {
lbryumDir = os.Getenv("HOME") + "/.lbryum"
}
}
defaultDB = lbryumDir + "/lbc_mainnet/blockchain.db"
tempDB = lbryumDir + "/lbc_mainnet/tmp_blockchain.tar"
key = aws.String("/blockchain_dbs/" + s.DbChannelData.ChannelId + ".tar")
if util.IsRegTest() {
tempDB = lbryumDir + "/lbc_mainnet/tmp_blockchain.db"
key = aws.String("/blockchain_dbs/" + s.DbChannelData.ChannelId)
if logUtils.IsRegTest() {
defaultDB = lbryumDir + "/lbc_regtest/blockchain.db"
tempDB = lbryumDir + "/lbc_regtest/tmp_blockchain.tar"
key = aws.String("/regtest_dbs/" + s.DbChannelData.ChannelId + ".tar")
tempDB = lbryumDir + "/lbc_regtest/tmp_blockchain.db"
key = aws.String("/regtest_dbs/" + s.DbChannelData.ChannelId)
}
return
}
func (s *Sync) uploadWallet() error {
defaultWalletDir := util.GetDefaultWalletPath()
defaultWalletDir := logUtils.GetDefaultWalletPath()
key := aws.String("/wallets/" + s.DbChannelData.ChannelId)
if util.IsRegTest() {
if logUtils.IsRegTest() {
key = aws.String("/regtest/" + s.DbChannelData.ChannelId)
}
@ -205,7 +196,7 @@ func (s *Sync) uploadWallet() error {
return errors.Err("default_wallet does not exist")
}
uploader, err := s.getS3Uploader(configs.Configuration.WalletS3Config.GetS3AWSConfig())
uploader, err := s.getS3Uploader()
if err != nil {
return err
}
@ -216,22 +207,13 @@ func (s *Sync) uploadWallet() error {
}
defer file.Close()
start := time.Now()
for time.Since(start) < 30*time.Minute {
_, err = uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(configs.Configuration.WalletS3Config.Bucket),
Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
Key: key,
Body: file,
})
if err != nil {
time.Sleep(30 * time.Second)
continue
}
break
}
if err != nil {
return errors.Prefix("there was a problem uploading the wallet to S3", errors.Err(err))
return err
}
log.Println("wallet uploaded to S3")
@ -247,39 +229,26 @@ func (s *Sync) uploadBlockchainDB() error {
if _, err := os.Stat(defaultBDBDir); os.IsNotExist(err) {
return errors.Err("blockchain.db does not exist")
}
files, err := filepath.Glob(defaultBDBDir + "*")
if err != nil {
return errors.Err(err)
}
tarPath := strings.Replace(defaultBDBDir, "blockchain.db", "", -1) + s.DbChannelData.ChannelId + ".tar"
err = util.CreateTarball(tarPath, files)
uploader, err := s.getS3Uploader()
if err != nil {
return err
}
uploader, err := s.getS3Uploader(configs.Configuration.BlockchaindbS3Config.GetS3AWSConfig())
if err != nil {
return err
}
file, err := os.Open(tarPath)
file, err := os.Open(defaultBDBDir)
if err != nil {
return err
}
defer file.Close()
_, err = uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(configs.Configuration.BlockchaindbS3Config.Bucket),
Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
Key: key,
Body: file,
})
if err != nil {
return err
}
log.Println("blockchain.db files uploaded to S3")
err = os.Remove(tarPath)
if err != nil {
return errors.Err(err)
}
log.Println("blockchain.db uploaded to S3")
return os.Remove(defaultBDBDir)
}

82
manager/setup.go
View file

@ -103,9 +103,6 @@ func (s *Sync) walletSetup() error {
videosOnYoutube = s.Manager.CliFlags.VideosToSync(s.DbChannelData.TotalSubscribers)
}
unallocatedVideos := videosOnYoutube - (publishedCount + failedCount)
if unallocatedVideos < 0 {
unallocatedVideos = 0
}
channelFee := channelClaimAmount
channelAlreadyClaimed := s.DbChannelData.ChannelClaimID != ""
if channelAlreadyClaimed {
@ -113,7 +110,7 @@ func (s *Sync) walletSetup() error {
}
requiredBalance := float64(unallocatedVideos)*(publishAmount+estimatedMaxTxFee) + channelFee
if s.Manager.CliFlags.UpgradeMetadata {
requiredBalance += float64(notUpgradedCount) * estimatedMaxTxFee
requiredBalance += float64(notUpgradedCount) * 0.001
}
refillAmount := 0.0
@ -130,12 +127,6 @@ func (s *Sync) walletSetup() error {
if err != nil {
return errors.Err(err)
}
} else if balance > requiredBalance {
extraLBC := balance - requiredBalance
if extraLBC > 5 {
sendBackAmount := extraLBC - 1
logUtils.SendInfoToSlack("channel %s has %.1f credits which is %.1f more than it requires (%.1f). We should send at least %.1f that back.", s.DbChannelData.ChannelId, balance, extraLBC, requiredBalance, sendBackAmount)
}
}
claimAddress, err := s.daemon.AddressList(nil, nil, 1, 20)
@ -320,12 +311,12 @@ func (s *Sync) waitForNewBlock() error {
func (s *Sync) GenerateRegtestBlock() error {
lbrycrd, err := logUtils.GetLbrycrdClient(s.Manager.LbrycrdDsn)
if err != nil {
return errors.Prefix("error getting lbrycrd client", err)
return errors.Prefix("error getting lbrycrd client: ", err)
}
txs, err := lbrycrd.Generate(1)
if err != nil {
return errors.Prefix("error generating new block", err)
return errors.Prefix("error generating new block: ", err)
}
for _, tx := range txs {
@ -375,12 +366,14 @@ func (s *Sync) ensureChannelOwnership() error {
channelUsesOldMetadata := false
if channelToUse != nil {
channelUsesOldMetadata = channelToUse.Value.GetThumbnail() == nil || (len(channelToUse.Value.GetLanguages()) == 0 && s.DbChannelData.Language != "")
channelUsesOldMetadata = channelToUse.Value.GetThumbnail() == nil
if !channelUsesOldMetadata {
return nil
}
}
channelBidAmount := channelClaimAmount
balanceResp, err := s.daemon.AccountBalance(nil)
if err != nil {
return err
@ -392,8 +385,8 @@ func (s *Sync) ensureChannelOwnership() error {
return errors.Err(err)
}
if balance.LessThan(decimal.NewFromFloat(channelClaimAmount)) {
err = s.addCredits(channelClaimAmount + estimatedMaxTxFee*3)
if balance.LessThan(decimal.NewFromFloat(channelBidAmount)) {
err = s.addCredits(channelBidAmount + 0.3)
if err != nil {
return err
}
@ -414,7 +407,7 @@ func (s *Sync) ensureChannelOwnership() error {
}
thumbnail := channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails)-1].URL
thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail, s.DbChannelData.ChannelId)
thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail, s.DbChannelData.ChannelId, *s.Manager.AwsConfigs.GetS3AWSConfig())
if err != nil {
return err
}
@ -423,6 +416,7 @@ func (s *Sync) ensureChannelOwnership() error {
if channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails != nil {
bURL, err := thumbs.MirrorThumbnail(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails)-1].URL,
"banner-"+s.DbChannelData.ChannelId,
*s.Manager.AwsConfigs.GetS3AWSConfig(),
)
if err != nil {
return err
@ -431,16 +425,18 @@ func (s *Sync) ensureChannelOwnership() error {
}
var languages []string = nil
if s.DbChannelData.Language != "" {
languages = []string{s.DbChannelData.Language}
}
//we don't have this data without the API
//if channelInfo.DefaultLanguage != "" {
// if channelInfo.DefaultLanguage == "iw" {
// channelInfo.DefaultLanguage = "he"
// }
// languages = []string{channelInfo.DefaultLanguage}
//}
var locations []jsonrpc.Location = nil
if channelInfo.Topbar.DesktopTopbarRenderer.CountryCode != "" {
locations = []jsonrpc.Location{{Country: &channelInfo.Topbar.DesktopTopbarRenderer.CountryCode}}
}
var c *jsonrpc.TransactionSummary
var recoveredChannelClaimID string
claimCreateOptions := jsonrpc.ClaimCreateOptions{
Title: &channelInfo.Microformat.MicroformatDataRenderer.Title,
Description: &channelInfo.Metadata.ChannelMetadataRenderer.Description,
@ -450,20 +446,12 @@ func (s *Sync) ensureChannelOwnership() error {
ThumbnailURL: &thumbnailURL,
}
if channelUsesOldMetadata {
da, err := s.getDefaultAccount()
if err != nil {
return err
}
if s.DbChannelData.TransferState <= 1 {
c, err = s.daemon.ChannelUpdate(s.DbChannelData.ChannelClaimID, jsonrpc.ChannelUpdateOptions{
ClearTags: util.PtrToBool(true),
ClearLocations: util.PtrToBool(true),
ClearLanguages: util.PtrToBool(true),
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
AccountID: &da,
FundingAccountIDs: []string{
da,
},
ClaimCreateOptions: claimCreateOptions,
CoverURL: bannerURL,
},
@ -473,50 +461,20 @@ func (s *Sync) ensureChannelOwnership() error {
return nil
}
} else {
c, err = s.daemon.ChannelCreate(s.DbChannelData.DesiredChannelName, channelClaimAmount, jsonrpc.ChannelCreateOptions{
c, err = s.daemon.ChannelCreate(s.DbChannelData.DesiredChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
ClaimCreateOptions: claimCreateOptions,
CoverURL: bannerURL,
})
if err != nil {
claimId, err2 := s.getChannelClaimIDForTimedOutCreation()
if err2 != nil {
err = errors.Prefix(err2.Error(), err)
} else {
recoveredChannelClaimID = claimId
}
}
}
if err != nil {
return err
}
if recoveredChannelClaimID != "" {
s.DbChannelData.ChannelClaimID = recoveredChannelClaimID
} else {
s.DbChannelData.ChannelClaimID = c.Outputs[0].ClaimID
}
return s.Manager.ApiConfig.SetChannelClaimID(s.DbChannelData.ChannelId, s.DbChannelData.ChannelClaimID)
}
//getChannelClaimIDForTimedOutCreation is a raw function that returns the only channel that exists in the wallet
// this is used because the SDK sucks and can't figure out when to return when creating a claim...
func (s *Sync) getChannelClaimIDForTimedOutCreation() (string, error) {
channels, err := s.daemon.ChannelList(nil, 1, 500, nil)
if err != nil {
return "", err
} else if channels == nil {
return "", errors.Err("no channel response")
}
if len((*channels).Items) != 1 {
return "", errors.Err("more than one channel found when trying to recover from SDK failure in creating the channel")
}
desiredChannel := (*channels).Items[0]
if desiredChannel.Name != s.DbChannelData.DesiredChannelName {
return "", errors.Err("the channel found in the wallet has a different name than the one we expected")
}
return desiredChannel.ClaimID, nil
}
func (s *Sync) addCredits(amountToAdd float64) error {
start := time.Now()
defer func(start time.Time) {

4
manager/transfer.go
View file

@ -241,7 +241,7 @@ func transferVideos(s *Sync) error {
},
},
},
Bid: util.PtrToString(fmt.Sprintf("%.5f", publishAmount/2.)),
Bid: util.PtrToString("0.005"), // Todo - Dont hardcode
}
videoStatus := shared.VideoStatus{
ChannelID: s.DbChannelData.ChannelId,
@ -293,7 +293,7 @@ func (s *Sync) streamUpdate(ui *updateInfo) error {
timing.TimedComponent("transferStreamUpdate").Add(time.Since(start))
if updateError != nil {
ui.videoStatus.FailureReason = updateError.Error()
ui.videoStatus.Status = shared.VideoStatusTransferFailed
ui.videoStatus.Status = shared.VideoStatusTranferFailed
ui.videoStatus.IsTransferred = util.PtrToBool(false)
} else {
ui.videoStatus.IsTransferred = util.PtrToBool(len(result.Outputs) != 0)

31
manager/ytsync.go
View file

@ -33,10 +33,11 @@ import (
const (
channelClaimAmount = 0.01
estimatedMaxTxFee = 0.0015
estimatedMaxTxFee = 0.1
minimumAccountBalance = 1.0
minimumRefillAmount = 1
publishAmount = 0.002
publishAmount = 0.01
maxReasonLength = 500
)
// Sync stores the options that control how syncing happens
@ -285,8 +286,6 @@ func (s *Sync) setChannelTerminationStatus(e *error) {
"interrupted during daemon startup",
"interrupted by user",
"use --skip-space-check to ignore",
"failure uploading blockchain DB",
"default_wallet already exists",
}
dbWipeConditions := []string{
"Missing inputs",
@ -336,7 +335,7 @@ func (s *Sync) waitForDaemonStart() error {
}
func (s *Sync) stopAndUploadWallet(e *error) {
log.Println("Stopping daemon")
log.Printf("Stopping daemon")
shutdownErr := logUtils.StopDaemon()
if shutdownErr != nil {
logShutdownError(shutdownErr)
@ -351,17 +350,17 @@ func (s *Sync) stopAndUploadWallet(e *error) {
err := s.uploadWallet()
if err != nil {
if *e == nil {
*e = err
e = &err
} else {
*e = errors.Prefix(fmt.Sprintf("%s + original error", errors.FullTrace(err)), *e)
*e = errors.Prefix("failure uploading wallet", *e)
}
}
err = s.uploadBlockchainDB()
if err != nil {
if *e == nil {
*e = err
e = &err
} else {
*e = errors.Prefix(fmt.Sprintf("failure uploading blockchain DB: %s + original error", errors.FullTrace(err)), *e)
*e = errors.Prefix("failure uploading wallet", *e)
}
}
}
@ -496,7 +495,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
claimMarkedUnpublished := claimInDatabase && !sv.Published
_, isOwnClaim := ownClaimsInfo[videoID]
transferred := !isOwnClaim || s.DbChannelData.TransferState == 3
transferStatusMismatch := claimInDatabase && sv.Transferred != transferred
transferStatusMismatch := sv.Transferred != transferred
if metadataDiffers {
log.Debugf("%s: Mismatch in database for metadata. DB: %d - Blockchain: %d", videoID, sv.MetadataVersion, chainInfo.MetadataVersion)
@ -557,11 +556,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
if sv.Transferred || sv.IsLbryFirst {
_, ok := allClaimsInfo[vID]
if !ok && sv.Published {
searchResponse, err := s.daemon.ClaimSearch(jsonrpc.ClaimSearchArgs{
ClaimID: &sv.ClaimID,
Page: 1,
PageSize: 20,
})
searchResponse, err := s.daemon.ClaimSearch(nil, &sv.ClaimID, nil, nil, 1, 20)
if err != nil {
log.Error(err.Error())
continue
@ -677,8 +672,7 @@ func (s *Sync) checkIntegrity() error {
if pubsOnWallet > pubsOnDB { //This case should never happen
logUtils.SendInfoToSlack("We're claiming to have published %d videos but in reality we published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId)
//we never really done anything about those. it happens when a user updates the channel for a publish to another ytsync channel
//return errors.Err("not all published videos are in the database")
return errors.Err("not all published videos are in the database")
}
if pubsOnWallet < pubsOnDB {
logUtils.SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.DbChannelData.ChannelId)
@ -867,8 +861,9 @@ func (s *Sync) enqueueYoutubeVideos() error {
return err
}
videos, err := ytapi.GetVideosToSync(s.DbChannelData.ChannelId, s.syncedVideos, s.Manager.CliFlags.QuickSync, s.Manager.CliFlags.VideosToSync(s.DbChannelData.TotalSubscribers), ytapi.VideoParams{
videos, err := ytapi.GetVideosToSync(s.Manager.ApiConfig, s.DbChannelData.ChannelId, s.syncedVideos, s.Manager.CliFlags.QuickSync, s.Manager.CliFlags.VideosToSync(s.DbChannelData.TotalSubscribers), ytapi.VideoParams{
VideoDir: s.videoDirectory,
S3Config: *s.Manager.AwsConfigs.GetS3AWSConfig(),
Stopper: s.grp,
IPPool: ipPool,
}, s.DbChannelData.LastUploadedVideo)

18
metrics/metrics.go
View file

@ -1,17 +1,31 @@
package metrics
import (
"github.com/lbryio/ytsync/v5/configs"
"os"
"regexp"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
log "github.com/sirupsen/logrus"
)
var (
Durations = promauto.NewHistogramVec(prometheus.HistogramOpts{
Namespace: "ytsync",
Subsystem: configs.Configuration.GetHostname(),
Subsystem: getHostname(),
Name: "duration",
Help: "The durations of the individual modules",
}, []string{"path"})
)
func getHostname() string {
hostname, err := os.Hostname()
if err != nil {
hostname = "ytsync_unknown"
}
reg, err := regexp.Compile("[^a-zA-Z0-9_]+")
if err != nil {
log.Fatal(err)
}
return reg.ReplaceAllString(hostname, "_")
}

55
sdk/api.go
View file

@ -13,7 +13,6 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/null"
"github.com/lbryio/ytsync/v5/configs"
"github.com/lbryio/ytsync/v5/shared"
"github.com/lbryio/ytsync/v5/util"
@ -26,24 +25,12 @@ const (
)
type APIConfig struct {
YoutubeAPIKey string
ApiURL string
ApiToken string
HostName string
}
var instance *APIConfig
func GetAPIsConfigs() *APIConfig {
if instance == nil {
instance = &APIConfig{
ApiURL: configs.Configuration.InternalApisEndpoint,
ApiToken: configs.Configuration.InternalApisAuthToken,
HostName: configs.Configuration.GetHostname(),
}
}
return instance
}
func (a *APIConfig) FetchChannels(status string, cliFlags *shared.SyncFlags) ([]shared.YoutubeChannel, error) {
type apiJobsResponse struct {
Success bool `json:"success"`
@ -61,10 +48,13 @@ func (a *APIConfig) FetchChannels(status string, cliFlags *shared.SyncFlags) ([]
"channel_id": {cliFlags.ChannelID},
})
if err != nil {
util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.FetchChannels(status, cliFlags)
}
return nil, errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
@ -121,10 +111,13 @@ func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
"auth_token": {a.ApiToken},
})
if err != nil {
util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.SetChannelCert(certHex, channelID)
}
return errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
@ -166,10 +159,13 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
}
res, err := http.PostForm(endpoint, params)
if err != nil {
util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.SetChannelStatus(channelID, status, failureReason, transferState)
}
return nil, nil, errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode >= http.StatusInternalServerError {
@ -213,10 +209,13 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
"channel_claim_id": {channelClaimID},
})
if err != nil {
util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.SetChannelClaimID(channelID, channelClaimID)
}
return errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
@ -254,10 +253,13 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
}
res, err := http.PostForm(endpoint, vals)
if err != nil {
util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.DeleteVideos(videos)
}
return errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
@ -317,10 +319,13 @@ func (a *APIConfig) MarkVideoStatus(status shared.VideoStatus) error {
}
res, err := http.PostForm(endpoint, vals)
if err != nil {
util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s for %s. Waiting to retry", endpoint, status.ClaimName)
time.Sleep(30 * time.Second)
return a.MarkVideoStatus(status)
}
return errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
@ -356,10 +361,13 @@ func (a *APIConfig) VideoState(videoID string) (string, error) {
res, err := http.PostForm(endpoint, vals)
if err != nil {
util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.VideoState(videoID)
}
return "", errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode == http.StatusNotFound {
@ -407,10 +415,13 @@ func (a *APIConfig) GetReleasedDate(videoID string) (*VideoRelease, error) {
res, err := http.PostForm(endpoint, vals)
if err != nil {
util.SendErrorToSlack("error while trying to call %s. Waiting to retry: %s", endpoint, err.Error())
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
time.Sleep(30 * time.Second)
return a.GetReleasedDate(videoID)
}
return nil, errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode == http.StatusNotFound {

30
shared/shared.go
View file

@ -4,6 +4,8 @@ import (
"encoding/json"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/lbryio/lbry.go/v2/extras/errors"
)
@ -26,7 +28,6 @@ type YoutubeChannel struct {
SizeLimit int `json:"size_limit"`
LastUploadedVideo string `json:"last_uploaded_video"`
WipeDB bool `json:"wipe_db"`
Language string `json:"language"`
}
type PublishAddress struct {
@ -78,7 +79,6 @@ var ErrorsNoRetry = []string{
"giving up after 0 fragment retries",
"Sorry about that",
"This video is not available",
"Video unavailable",
"requested format not available",
"interrupted by user",
"Sign in to confirm your age",
@ -91,7 +91,6 @@ var ErrorsNoRetry = []string{
"Premiere will begin shortly",
"cannot unmarshal number 0.0",
"default youtube thumbnail found",
"livestream is likely bugged",
}
var WalletErrors = []string{
"Not enough funds to cover this transaction",
@ -118,8 +117,6 @@ var NeverRetryFailures = []string{
"Playback on other websites has been disabled by the video owner",
"uploader has not made this video available in your country",
"This video has been removed by the uploader",
"Video unavailable",
"Video is not available - hardcoded fix",
}
type SyncFlags struct {
@ -157,7 +154,7 @@ func (f *SyncFlags) VideosToSync(totalSubscribers uint) int {
800: 250,
600: 200,
200: 80,
100: 20,
100: 50,
1: 10,
}
videosToSync := 0
@ -196,10 +193,9 @@ const (
StatusFailed = "failed"
StatusFinalized = "finalized" // no more changes allowed
StatusAbandoned = "abandoned" // deleted on youtube or banned
StatusAgeRestricted = "agerestricted" // one or more videos are age restricted and should be reprocessed with special keys
)
var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned, StatusWipeDb, StatusAgeRestricted}
var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned, StatusWipeDb}
const LatestMetadataVersion = 2
@ -208,14 +204,26 @@ const (
VideoStatusFailed = "failed"
VideoStatusUpgradeFailed = "upgradefailed"
VideoStatusUnpublished = "unpublished"
VideoStatusTransferFailed = "transferfailed"
VideoStatusTranferFailed = "transferfailed"
)
var VideoSyncStatuses = []string{VideoStatusPublished, VideoStatusFailed, VideoStatusUpgradeFailed, VideoStatusUnpublished, VideoStatusTransferFailed}
const (
TransferStateNotTouched = iota
TransferStatePending
TransferStateComplete
TransferStateManual
)
type AwsConfigs struct {
AwsS3ID string
AwsS3Secret string
AwsS3Region string
AwsS3Bucket string
}
func (a *AwsConfigs) GetS3AWSConfig() *aws.Config {
return &aws.Config{
Credentials: credentials.NewStaticCredentials(a.AwsS3ID, a.AwsS3Secret, ""),
Region: &a.AwsS3Region,
}
}

82
sources/youtubeVideo.go
View file

@ -16,12 +16,17 @@ import (
"syscall"
"time"
"github.com/abadojack/whatlanggo"
"github.com/lbryio/ytsync/v5/downloader"
"github.com/lbryio/ytsync/v5/downloader/ytdl"
"github.com/lbryio/ytsync/v5/shared"
"github.com/vbauerster/mpb/v7"
"github.com/vbauerster/mpb/v7/decor"
"gopkg.in/vansante/go-ffprobe.v2"
"github.com/lbryio/ytsync/v5/ip_manager"
"github.com/lbryio/ytsync/v5/namer"
"github.com/lbryio/ytsync/v5/sdk"
"github.com/lbryio/ytsync/v5/shared"
"github.com/lbryio/ytsync/v5/tags_manager"
"github.com/lbryio/ytsync/v5/thumbs"
"github.com/lbryio/ytsync/v5/timing"
@ -32,12 +37,9 @@ import (
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/abadojack/whatlanggo"
"github.com/aws/aws-sdk-go/aws"
"github.com/shopspring/decimal"
log "github.com/sirupsen/logrus"
"github.com/vbauerster/mpb/v7"
"github.com/vbauerster/mpb/v7/decor"
"gopkg.in/vansante/go-ffprobe.v2"
)
type YoutubeVideo struct {
@ -53,6 +55,7 @@ type YoutubeVideo struct {
youtubeInfo *ytdl.YtdlVideo
youtubeChannelID string
tags []string
awsConfig aws.Config
thumbnailURL string
lbryChannelID string
mocked bool
@ -98,7 +101,7 @@ var youtubeCategories = map[string]string{
"44": "trailers",
}
func NewYoutubeVideo(directory string, videoData *ytdl.YtdlVideo, playlistPosition int64, stopGroup *stop.Group, pool *ip_manager.IPPool) (*YoutubeVideo, error) {
func NewYoutubeVideo(directory string, videoData *ytdl.YtdlVideo, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) (*YoutubeVideo, error) {
// youtube-dl returns times in local timezone sometimes. this could break in the future
// maybe we can file a PR to choose the timezone we want from youtube-dl
return &YoutubeVideo{
@ -106,21 +109,22 @@ func NewYoutubeVideo(directory string, videoData *ytdl.YtdlVideo, playlistPositi
title: videoData.Title,
description: videoData.Description,
playlistPosition: playlistPosition,
publishedAt: videoData.GetUploadTime(),
publishedAt: videoData.UploadDateForReal,
dir: directory,
youtubeInfo: videoData,
awsConfig: awsConfig,
mocked: false,
youtubeChannelID: videoData.ChannelID,
stopGroup: stopGroup,
pool: pool,
}, nil
}
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
return &YoutubeVideo{
id: videoID,
playlistPosition: 0,
dir: directory,
awsConfig: awsConfig,
mocked: true,
youtubeChannelID: youtubeChannelID,
stopGroup: stopGroup,
@ -175,7 +179,7 @@ func (v *YoutubeVideo) getFullPath() string {
}
func (v *YoutubeVideo) getAbbrevDescription() string {
maxLength := 6500
maxLength := 2800
description := strings.TrimSpace(v.description)
additionalDescription := "\nhttps://www.youtube.com/watch?v=" + v.id
khanAcademyClaimID := "5fc52291980268b82413ca4c0ace1b8d749f3ffb"
@ -212,7 +216,9 @@ func (v *YoutubeVideo) download() error {
defer func(start time.Time) {
timing.TimedComponent("download").Add(time.Since(start))
}(start)
if v.youtubeInfo.IsLive != nil {
return errors.Err("video is a live stream and hasn't completed yet")
}
videoPath := v.getFullPath()
err := os.Mkdir(v.videoDir(), 0777)
@ -270,10 +276,6 @@ func (v *YoutubeVideo) download() error {
"1",
"--cookies",
"cookies.txt",
"--extractor-args",
"youtube:player_client=android",
//"--concurrent-fragments",
//"2",
"--load-info-json",
metadataPath,
}
@ -317,15 +319,11 @@ func (v *YoutubeVideo) download() error {
sourceAddress,
fmt.Sprintf("https://www.youtube.com/watch?v=%s", v.id),
)
//speedThrottleRetries := 3
for i := 0; i < len(qualities); i++ {
quality := qualities[i]
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][vcodec!*=av01][height<="+quality+"]+bestaudio[ext!=webm][format_id!=258][format_id!=380][format_id!=251][format_id!=256][format_id!=327][format_id!=328]")
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][vcodec!*=av01][height<="+quality+"]+bestaudio[ext!=webm][format_id!=258][format_id!=251][format_id!=256][format_id!=327]")
argsWithFilters = append(argsWithFilters, userAgent...)
//if speedThrottleRetries > 0 {
// speedThrottleRetries--
// argsWithFilters = append(argsWithFilters, "--throttled-rate", "180K")
//}
cmd := exec.Command("yt-dlp", argsWithFilters...)
log.Printf("Running command yt-dlp %s", strings.Join(argsWithFilters, " "))
@ -372,10 +370,6 @@ func (v *YoutubeVideo) download() error {
userAgent = []string{downloader.GoogleBotUA}
log.Infof("trying different user agent for video %s", v.ID())
continue
//} else if strings.Contains(string(errorLog), "yt_dlp.utils.ThrottledDownload") {
// log.Infof("throttled download speed for video %s. Retrying", v.ID())
// i-- //do not lower quality when we're retrying a throttled download
// continue
}
return errors.Err(string(errorLog))
}
@ -514,28 +508,21 @@ func (v *YoutubeVideo) trackProgressBar(argsWithFilters []string, ticker *time.T
bar.Completed()
bar.Abort(true)
}()
origSize := int64(0)
lastUpdate := time.Now()
for {
select {
case <-done.Ch():
return
case <-ticker.C:
var err error
size, err := logUtils.DirSize(v.videoDir())
if err != nil {
log.Errorf("error while getting size of download directory: %s", errors.FullTrace(err))
return
}
if size > origSize {
origSize = size
bar.SetCurrent(size)
if size > int64(videoSize+audioSize) {
bar.SetTotal(size+2048, false)
}
bar.DecoratorEwmaUpdate(time.Since(lastUpdate))
lastUpdate = time.Now()
}
bar.DecoratorEwmaUpdate(400 * time.Millisecond)
}
}
}()
@ -691,7 +678,7 @@ func (v *YoutubeVideo) triggerThumbnailSave() (err error) {
if thumbnail.Width == 0 {
return errors.Err("default youtube thumbnail found")
}
v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID())
v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig)
return err
}
@ -780,19 +767,10 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
var err error
if v.youtubeInfo == nil {
return nil, errors.Err("Video is not available - hardcoded fix")
}
dur := time.Duration(v.youtubeInfo.Duration) * time.Second
minDuration := 7 * time.Second
if v.youtubeInfo.IsLive == true {
return nil, errors.Err("video is a live stream and hasn't completed yet")
}
if v.youtubeInfo.Availability != "public" {
return nil, errors.Err("video is not public")
}
if dur > v.maxVideoLength {
logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, dur.String(), v.maxVideoLength.String())
return nil, errors.Err("video is too long to process")
@ -801,11 +779,6 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
logUtils.SendErrorToSlack("%s is %s long and the minimum is %s", v.id, dur.String(), minDuration.String())
return nil, errors.Err("video is too short to process")
}
buggedLivestream := v.youtubeInfo.LiveStatus == "post_live"
if buggedLivestream && dur >= 2*time.Hour {
return nil, errors.Err("livestream is likely bugged as it was recently published and has a length of %s which is more than 2 hours", dur.String())
}
for {
err = v.download()
if err != nil && strings.Contains(err.Error(), "HTTP Error 429") {
@ -816,6 +789,7 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
break
}
//log.Debugln("Downloaded " + v.id)
ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second)
defer cancelFn()
@ -875,11 +849,7 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
}
func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo) (*SyncSummary, error) {
c, err := daemon.ClaimSearch(jsonrpc.ClaimSearchArgs{
ClaimID: &existingVideoData.ClaimID,
Page: 1,
PageSize: 20,
})
c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil, 1, 20)
if err != nil {
return nil, errors.Err(err)
}
@ -898,7 +868,7 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
return nil, errors.Err("could not find thumbnail for mocked video")
}
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails)
thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID())
thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig)
} else {
thumbnailURL = thumbs.ThumbnailEndpoint + v.ID()
}
@ -915,8 +885,9 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
return nil, errors.Err(err)
}
return v.downloadAndPublish(daemon, params)
}
return nil, errors.Prefix("the video must be republished as we can't get the right size and it doesn't exist on youtube anymore", err)
return nil, errors.Err("the video must be republished as we can't get the right size but it doesn't exist on youtube anymore")
}
}
v.size = util.PtrToInt64(int64(videoSize))
@ -948,7 +919,6 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
Height: util.PtrToUint(720),
Width: util.PtrToUint(1280),
Fee: fee,
ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
}
v.walletLock.RLock()

11
thumbs/uploader.go
View file

@ -6,7 +6,6 @@ import (
"os"
"strings"
"github.com/lbryio/ytsync/v5/configs"
"github.com/lbryio/ytsync/v5/downloader/ytdl"
"github.com/lbryio/lbry.go/v2/extras/errors"
@ -84,11 +83,11 @@ func (u *thumbnailUploader) deleteTmpFile() {
log.Infof("failed to delete local thumbnail file: %s", err.Error())
}
}
func MirrorThumbnail(url string, name string) (string, error) {
func MirrorThumbnail(url string, name string, s3Config aws.Config) (string, error) {
tu := thumbnailUploader{
originalUrl: url,
name: name,
s3Config: *configs.Configuration.AWSThumbnailsS3Config.GetS3AWSConfig(),
s3Config: s3Config,
}
err := tu.downloadThumbnail()
if err != nil {
@ -101,12 +100,14 @@ func MirrorThumbnail(url string, name string) (string, error) {
return "", err
}
//this is our own S3 storage
ownS3Config := s3Config.Copy(&aws.Config{Endpoint: aws.String("s3.lbry.tech")})
tu2 := thumbnailUploader{
originalUrl: url,
name: name,
s3Config: *configs.Configuration.ThumbnailsS3Config.GetS3AWSConfig(),
s3Config: *ownS3Config,
}
//own S3
err = tu2.uploadThumbnail()
if err != nil {
return "", err

109
util/archive.go
View file

@ -1,109 +0,0 @@
package util
import (
"archive/tar"
"io"
"io/fs"
"os"
"path/filepath"
"github.com/lbryio/lbry.go/v2/extras/errors"
)
func CreateTarball(tarballFilePath string, filePaths []string) error {
file, err := os.Create(tarballFilePath)
if err != nil {
return errors.Err("Could not create tarball file '%s', got error '%s'", tarballFilePath, err.Error())
}
defer file.Close()
tarWriter := tar.NewWriter(file)
defer tarWriter.Close()
for _, filePath := range filePaths {
err := addFileToTarWriter(filePath, tarWriter)
if err != nil {
return errors.Err("Could not add file '%s', to tarball, got error '%s'", filePath, err.Error())
}
}
return nil
}
func addFileToTarWriter(filePath string, tarWriter *tar.Writer) error {
file, err := os.Open(filePath)
if err != nil {
return errors.Err("Could not open file '%s', got error '%s'", filePath, err.Error())
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
return errors.Err("Could not get stat for file '%s', got error '%s'", filePath, err.Error())
}
header := &tar.Header{
Name: stat.Name(),
Size: stat.Size(),
Mode: int64(stat.Mode()),
ModTime: stat.ModTime(),
}
err = tarWriter.WriteHeader(header)
if err != nil {
return errors.Err("Could not write header for file '%s', got error '%s'", filePath, err.Error())
}
_, err = io.Copy(tarWriter, file)
if err != nil {
return errors.Err("Could not copy the file '%s' data to the tarball, got error '%s'", filePath, err.Error())
}
return nil
}
func Untar(tarball, target string) error {
reader, err := os.Open(tarball)
if err != nil {
return errors.Err(err)
}
defer reader.Close()
tarReader := tar.NewReader(reader)
for {
header, err := tarReader.Next()
if err == io.EOF {
break
} else if err != nil {
return errors.Err(err)
}
path := filepath.Join(target, header.Name)
info := header.FileInfo()
if info.IsDir() {
if err = os.MkdirAll(path, info.Mode()); err != nil {
return errors.Err(err)
}
continue
}
err = extractFile(path, info, tarReader)
if err != nil {
return err
}
}
return nil
}
func extractFile(path string, info fs.FileInfo, tarReader *tar.Reader) error {
file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
if err != nil {
return errors.Err(err)
}
defer file.Close()
_, err = io.Copy(file, tarReader)
if err != nil {
return errors.Err(err)
}
return nil
}

6
util/log_wrapper.go
View file

@ -14,9 +14,7 @@ func SendErrorToSlack(format string, a ...interface{}) {
message = fmt.Sprintf(format, a...)
}
log.Errorln(message)
log.SetLevel(log.InfoLevel) //I don't want to change the underlying lib so this will do...
err := util.SendToSlack(":sos: ```" + message + "```")
log.SetLevel(log.DebugLevel)
err := util.SendToSlack(":sos: " + message)
if err != nil {
log.Errorln(err)
}
@ -29,9 +27,7 @@ func SendInfoToSlack(format string, a ...interface{}) {
message = fmt.Sprintf(format, a...)
}
log.Infoln(message)
log.SetLevel(log.InfoLevel) //I don't want to change the underlying lib so this will do...
err := util.SendToSlack(":information_source: " + message)
log.SetLevel(log.DebugLevel)
if err != nil {
log.Errorln(err)
}

5
util/util.go
View file

@ -11,7 +11,6 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/lbrycrd"
"github.com/lbryio/ytsync/v5/configs"
"github.com/lbryio/ytsync/v5/timing"
"github.com/docker/docker/api/types"
@ -186,9 +185,9 @@ func CleanForStartup() error {
return errors.Err(err)
}
lbrycrd, err := GetLbrycrdClient(configs.Configuration.LbrycrdString)
lbrycrd, err := GetLbrycrdClient(os.Getenv("LBRYCRD_STRING"))
if err != nil {
return errors.Prefix("error getting lbrycrd client", err)
return errors.Prefix("error getting lbrycrd client: ", err)
}
height, err := lbrycrd.GetBlockCount()
if err != nil {

23
ytapi/ytapi.go
View file

@ -29,6 +29,7 @@ import (
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/aws/aws-sdk-go/aws"
log "github.com/sirupsen/logrus"
)
@ -49,13 +50,14 @@ func (a byPublishedAt) Less(i, j int) bool { return a[i].PublishedAt().Before(a[
type VideoParams struct {
VideoDir string
S3Config aws.Config
Stopper *stop.Group
IPPool *ip_manager.IPPool
}
var mostRecentlyFailedChannel string // TODO: fix this hack!
func GetVideosToSync(channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams, lastUploadedVideo string) ([]Video, error) {
func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams, lastUploadedVideo string) ([]Video, error) {
var videos []Video
if quickSync && maxVideos > 50 {
maxVideos = 50
@ -94,14 +96,14 @@ func GetVideosToSync(channelID string, syncedVideos map[string]sdk.SyncedVideo,
mostRecentlyFailedChannel = channelID
}
vids, err := getVideos(channelID, videoIDs, videoParams.Stopper.Ch(), videoParams.IPPool)
vids, err := getVideos(config, channelID, videoIDs, videoParams.Stopper.Ch(), videoParams.IPPool)
if err != nil {
return nil, err
}
for _, item := range vids {
positionInList := playlistMap[item.ID]
videoToAdd, err := sources.NewYoutubeVideo(videoParams.VideoDir, item, positionInList, videoParams.Stopper, videoParams.IPPool)
videoToAdd, err := sources.NewYoutubeVideo(videoParams.VideoDir, item, positionInList, videoParams.S3Config, videoParams.Stopper, videoParams.IPPool)
if err != nil {
return nil, errors.Err(err)
}
@ -109,12 +111,11 @@ func GetVideosToSync(channelID string, syncedVideos map[string]sdk.SyncedVideo,
}
for k, v := range syncedVideos {
newMetadataVersion := int8(2)
if !v.Published && v.MetadataVersion >= newMetadataVersion {
if !v.Published {
continue
}
if _, ok := playlistMap[k]; !ok {
videos = append(videos, sources.NewMockedVideo(videoParams.VideoDir, k, channelID, videoParams.Stopper, videoParams.IPPool))
videos = append(videos, sources.NewMockedVideo(videoParams.VideoDir, k, channelID, videoParams.S3Config, videoParams.Stopper, videoParams.IPPool))
}
}
@ -204,8 +205,7 @@ func ChannelInfo(channelID string) (*YoutubeStatsResponse, error) {
return &decodedResponse, nil
}
func getVideos(channelID string, videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool) ([]*ytdl.YtdlVideo, error) {
config := sdk.GetAPIsConfigs()
func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool) ([]*ytdl.YtdlVideo, error) {
var videos []*ytdl.YtdlVideo
for _, videoID := range videoIDs {
if len(videoID) < 5 {
@ -217,6 +217,11 @@ func getVideos(channelID string, videoIDs []string, stopChan stop.Chan, ipPool *
default:
}
//ip, err := ipPool.GetIP(videoID)
//if err != nil {
// return nil, err
//}
//video, err := downloader.GetVideoInformation(videoID, &net.TCPAddr{IP: net.ParseIP(ip)})
state, err := config.VideoState(videoID)
if err != nil {
return nil, errors.Err(err)
@ -224,7 +229,7 @@ func getVideos(channelID string, videoIDs []string, stopChan stop.Chan, ipPool *
if state == "published" {
continue
}
video, err := downloader.GetVideoInformation(videoID, stopChan, ipPool)
video, err := downloader.GetVideoInformation(config, videoID, stopChan, nil, ipPool)
if err != nil {
errSDK := config.MarkVideoStatus(shared.VideoStatus{
ChannelID: channelID,