LBRYCommunity/ytsync
1
0
Fork 0
Code Issues 39 Pull requests 8 Projects Releases 197 Packages Wiki Activity

rename selfsync to ytsync

Browse source 
add video marking support
This commit is contained in:
Niko Storni 2018-07-20 19:56:36 -04:00
parent e0ec334af9
commit dfe3e8a078
5 changed files with 77 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
manager.go
View file

@ -117,7 +117,49 @@ func (s SyncManager) setChannelSyncStatus(channelID string, status string) error
if !response.Data.IsNull() && response.Data.String == "ok" {
return nil
}
return errors.Err("invalid API response")
return errors.Err("invalid API response. Status code: %d", res.StatusCode)
}
const (
VideoStatusPublished = "published"
VideoSStatusFailed = "failed"
)
func (s SyncManager) MarkVideoStatus(channelID string, videoID string, status string, claimID string, claimName string, details string) error {
endpoint := s.apiURL + "/yt/track_video"
vals := url.Values{
"youtube_channel_id": {channelID},
"youtube_video_id": {videoID},
"status": {status},
"auth_token": {s.apiToken},
}
if status == VideoStatusPublished {
if claimID == "" || claimName == "" {
return errors.Err("claimID or claimName missing")
}
vals.Add("published_at", strconv.FormatInt(time.Now().Unix(), 10))
vals.Add("claim_id", claimID)
vals.Add("claim_name", claimName)
}
if details != "" {
vals.Add("details", details)
}
res, _ := http.PostForm(endpoint, vals)
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
var response apiSyncUpdateResponse
err := json.Unmarshal(body, &response)
if err != nil {
return err
}
if !response.Error.IsNull() {
return errors.Err(response.Error.String)
}
if !response.Data.IsNull() && response.Data.String == "ok" {
return nil
}
return errors.Err("invalid API response. Status code: %d", res.StatusCode)
}
func (s SyncManager) Start() error {

13
sources/shared.go
View file

@ -16,6 +16,11 @@ import (
var titleRegexp = regexp.MustCompile(`[^a-zA-Z0-9]+`)
type SyncSummary struct {
ClaimID string
ClaimName string
}
func getClaimNameFromTitle(title string, attempt int) string {
suffix := ""
if attempt > 1 {
@ -47,7 +52,7 @@ func getClaimNameFromTitle(title string, attempt int) string {
var publishedNamesMutex sync.RWMutex
var publishedNames = map[string]bool{}
func publishAndRetryExistingNames(daemon *jsonrpc.Client, title, filename string, amount float64, options jsonrpc.PublishOptions) error {
func publishAndRetryExistingNames(daemon *jsonrpc.Client, title, filename string, amount float64, options jsonrpc.PublishOptions) (*SyncSummary, error) {
attempt := 0
for {
attempt++
@ -67,19 +72,19 @@ func publishAndRetryExistingNames(daemon *jsonrpc.Client, title, filename string
name = fmt.Sprintf("%s-%d", hex.EncodeToString(hasher.Sum(nil))[:15], attempt)
}
_, err := daemon.Publish(name, filename, amount, options)
response, err := daemon.Publish(name, filename, amount, options)
if err == nil || strings.Contains(err.Error(), "failed: Multiple claims (") {
publishedNamesMutex.Lock()
publishedNames[name] = true
publishedNamesMutex.Unlock()
if err == nil {
return nil
return &SyncSummary{ClaimID: response.ClaimID, ClaimName: name}, nil
} else {
log.Printf("name exists, retrying (%d attempts so far)\n", attempt)
continue
}
} else {
return err
return nil, err
}
}
}

12
sources/ucbVideo.go
View file

@ -170,7 +170,7 @@ func (v ucbVideo) saveThumbnail() error {
return err
}
func (v ucbVideo) publish(daemon *jsonrpc.Client, claimAddress string, amount float64, channelName string) error {
func (v ucbVideo) publish(daemon *jsonrpc.Client, claimAddress string, amount float64, channelName string) (*SyncSummary, error) {
options := jsonrpc.PublishOptions{
Title: &v.title,
Author: strPtr("UC Berkeley"),
@ -188,11 +188,11 @@ func (v ucbVideo) publish(daemon *jsonrpc.Client, claimAddress string, amount fl
return publishAndRetryExistingNames(daemon, v.title, v.getFilename(), amount, options)
}
func (v ucbVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount float64, channelName string) error {
func (v ucbVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount float64, channelName string) (*SyncSummary, error) {
//download and thumbnail can be done in parallel
err := v.download()
if err != nil {
return errors.Prefix("download error", err)
return nil, errors.Prefix("download error", err)
}
log.Debugln("Downloaded " + v.id)
@ -202,10 +202,10 @@ func (v ucbVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount float
//}
//log.Debugln("Created thumbnail for " + v.id)
err = v.publish(daemon, claimAddress, amount, channelName)
summary, err := v.publish(daemon, claimAddress, amount, channelName)
if err != nil {
return errors.Prefix("publish error", err)
return nil, errors.Prefix("publish error", err)
}
return nil
return summary, nil
}

18
sources/youtubeVideo.go
View file

@ -181,7 +181,7 @@ func (v YoutubeVideo) triggerThumbnailSave() error {
func strPtr(s string) *string { return &s }
func (v YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amount float64, channelName string) error {
func (v YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amount float64, channelName string) (*SyncSummary, error) {
options := jsonrpc.PublishOptions{
Title: &v.title,
Author: &v.channelTitle,
@ -199,38 +199,38 @@ func (v YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amoun
return publishAndRetryExistingNames(daemon, v.title, v.getFilename(), amount, options)
}
func (v YoutubeVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount float64, channelName string) error {
func (v YoutubeVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount float64, channelName string) (*SyncSummary, error) {
//download and thumbnail can be done in parallel
err := v.download()
if err != nil {
return errors.Prefix("download error", err)
return nil, errors.Prefix("download error", err)
}
log.Debugln("Downloaded " + v.id)
fi, err := os.Stat(v.getFilename())
if err != nil {
return err
return nil, err
}
if fi.Size() > 2*1024*1024*1024 {
//delete the video and ignore the error
_ = v.delete()
return errors.Err("video is bigger than 2GB, skipping for now")
return nil, errors.Err("video is bigger than 2GB, skipping for now")
}
err = v.triggerThumbnailSave()
if err != nil {
return errors.Prefix("thumbnail error", err)
return nil, errors.Prefix("thumbnail error", err)
}
log.Debugln("Created thumbnail for " + v.id)
err = v.publish(daemon, claimAddress, amount, channelName)
summary, err := v.publish(daemon, claimAddress, amount, channelName)
//delete the video in all cases (and ignore the error)
_ = v.delete()
if err != nil {
return errors.Prefix("publish error", err)
return nil, errors.Prefix("publish error", err)
}
return nil
return summary, nil
}
// sorting videos

13
ytsync.go
View file

@ -39,7 +39,7 @@ type video interface {
IDAndNum() string
PlaylistPosition() int
PublishedAt() time.Time
Sync(*jsonrpc.Client, string, float64, string) error
Sync(*jsonrpc.Client, string, float64, string) (*sources.SyncSummary, error)
}
// sorting videos
@ -316,6 +316,10 @@ func (s *Sync) startWorker(workerNum int) {
}
util.SendErrorToSlack("Video failed after %d retries, skipping. Stack: %s", tryCount, logMsg)
}
err = s.Manager.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoSStatusFailed, "", "", err.Error())
if err != nil {
util.SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
}
}
break
}
@ -483,11 +487,14 @@ func (s *Sync) processVideo(v video) (err error) {
log.Println(v.ID() + " is old: skipping")
return nil
}
err = v.Sync(s.daemon, s.claimAddress, publishAmount, s.LbryChannelName)
summary, err := v.Sync(s.daemon, s.claimAddress, publishAmount, s.LbryChannelName)
if err != nil {
return err
}
err = s.Manager.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "")
if err != nil {
util.SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
}
err = s.db.SetPublished(v.ID())
if err != nil {
return err