LBRYCommunity/ytsync
1
0
Fork 0
Code Issues 39 Pull requests 8 Projects Releases 197 Packages Wiki Activity

Add marking video status when transfer of claim is complete.

Browse source 
Add error handling when transfer of claim fails.
This commit is contained in:
Mark Beamer Jr 2019-08-20 01:06:51 -04:00 committed by Niko Storni
parent 9a5b6b4e56
commit 45cea808ed
5 changed files with 81 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
e2e/e2e.sh
View file

@ -60,8 +60,10 @@ mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e "UPDATE youtube_data
echo "curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' http://localhost:15400/yt/transfer?auth_token=youtubertoken&address=n1Ygra2pyD6cpESv9GtPM9kDkr4bPeu1Dc"
# Execute the transfer test!
./../bin/ytsync --channelID UCCyr5j8akeu9j4Q7urV0Lqw #Force channel intended...just in case. This channel lines up with the api container
# ALSO CHECK THAT VIDEO IS MARKED TRANSFERRED
transferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT transferred FROM youtube_data WHERE id=1')
if [[ $status != "synced" || $videoStatus != "published" || transferStatus != "1" ]]; then
echo "~~!!!~~~FAILED~~~!!!~~"
echo "Channel Status: $status"
echo "Video Status: $videoStatus"
echo "Transfer Status: $transferStatus"
@ -69,6 +71,4 @@ echo "Transfer Status: $transferStatus"
#docker-compose logs --tail="all" walletserver
#docker-compose logs --tail="all" lbrynet
#docker-compose logs --tail="all" internalapis
echo "List local /var/tmp"
find /var/tmp
exit 1; fi;

1
manager/manager.go
View file

@ -97,6 +97,7 @@ const (
VideoStatusFailed = "failed"
VideoStatusUpgradeFailed = "upgradefailed"
VideoStatusUnpublished = "unpublished"
VideoStatusTranferFailed = "transferfailed"
)
func (s *SyncManager) Start() error {

23
manager/transfer.go
View file

@ -4,13 +4,13 @@ import (
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/jsonrpc"
"github.com/lbryio/lbry.go/extras/util"
"github.com/lbryio/ytsync/sdk"
)
func TransferChannel(channel *Sync) error {
//Transfer channel
for _, video := range channel.syncedVideos {
//Todo - Wait for prior sync to see that the publish is confirmed in lbrycrd?
//Todo - We need to fix the ClaimSearch call in lbry.go for 38.5 lbrynet
c, err := channel.daemon.ClaimSearch(nil, &video.ClaimID, nil, nil)
if err != nil {
errors.Err(err)
@ -28,11 +28,28 @@ func TransferChannel(channel *Sync) error {
Bid: util.PtrToString("0.009"), // Todo - Dont hardcode
}
videoStatus := sdk.VideoStatus{
ChannelID: channel.YoutubeChannelID,
VideoID: video.VideoID,
ClaimID: video.ClaimID,
ClaimName: video.ClaimName,
Status: VideoStatusPublished,
IsTransferred: util.PtrToBool(true),
}
_, err = channel.daemon.StreamUpdate(video.ClaimID, streamUpdateOptions)
if err != nil {
return err
videoStatus.FailureReason = err.Error()
videoStatus.Status = VideoStatusTranferFailed
videoStatus.IsTransferred = util.PtrToBool(false)
}
statusErr := channel.APIConfig.MarkVideoStatus(videoStatus)
if statusErr != nil {
return errors.Err(err)
}
if err != nil {
return errors.Err(err)
}
// Todo - Post to remote db that video is transferred
}
// Todo - Transfer Channel as last step and post back to remote db that channel is transferred.

30
manager/ytsync.go
View file

@ -511,7 +511,15 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int
}
fixed++
log.Debugf("updating %s in the database", videoID)
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", util.PtrToInt64(int64(claimSize)), claimMetadataVersion)
err = s.Manager.apiConfig.MarkVideoStatus(sdk.VideoStatus{
ChannelID: s.YoutubeChannelID,
VideoID: videoID,
Status: VideoStatusPublished,
ClaimID: c.ClaimID,
ClaimName: c.Name,
Size: util.PtrToInt64(int64(claimSize)),
MetaDataVersion: claimMetadataVersion,
})
if err != nil {
return count, fixed, 0, err
}
@ -763,7 +771,15 @@ func (s *Sync) startWorker(workerNum int) {
} else {
s.AppendSyncedVideo(v.ID(), false, err.Error(), existingClaimName, existingClaimID, 0, existingClaimSize)
}
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), videoStatus, existingClaimID, existingClaimName, err.Error(), &existingClaimSize, 0)
err = s.Manager.apiConfig.MarkVideoStatus(sdk.VideoStatus{
ChannelID: s.YoutubeChannelID,
VideoID: v.ID(),
Status: videoStatus,
ClaimID: existingClaimID,
ClaimName: existingClaimName,
FailureReason: err.Error(),
Size: &existingClaimSize,
})
if err != nil {
logUtils.SendErrorToSlack("Failed to mark video on the database: %s", errors.FullTrace(err))
}
@ -954,7 +970,15 @@ func (s *Sync) processVideo(v video) (err error) {
}
s.AppendSyncedVideo(v.ID(), true, "", summary.ClaimName, summary.ClaimID, newMetadataVersion, *v.Size())
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "", v.Size(), 2)
err = s.Manager.apiConfig.MarkVideoStatus(sdk.VideoStatus{
ChannelID: s.YoutubeChannelID,
VideoID: v.ID(),
Status: VideoStatusPublished,
ClaimID: summary.ClaimID,
ClaimName: summary.ClaimName,
Size: v.Size(),
MetaDataVersion: 2,
})
if err != nil {
logUtils.SendErrorToSlack("Failed to mark video on the database: %s", errors.FullTrace(err))
}

47
sdk/api.go
View file

@ -236,32 +236,47 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
return errors.Err("invalid API response. Status code: %d", res.StatusCode)
}
func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status string, claimID string, claimName string, failureReason string, size *int64, metadataVersion uint) error {
type VideoStatus struct {
ChannelID string
VideoID string
Status string
ClaimID string
ClaimName string
FailureReason string
Size *int64
MetaDataVersion uint
IsTransferred *bool
}
func (a *APIConfig) MarkVideoStatus(status VideoStatus) error {
endpoint := a.ApiURL + "/yt/video_status"
sanitizeFailureReason(&failureReason)
sanitizeFailureReason(&status.FailureReason)
vals := url.Values{
"youtube_channel_id": {channelID},
"video_id": {videoID},
"status": {status},
"youtube_channel_id": {status.ChannelID},
"video_id": {status.VideoID},
"status": {status.Status},
"auth_token": {a.ApiToken},
}
if status == VideoStatusPublished || status == VideoStatusUpgradeFailed {
if claimID == "" || claimName == "" {
return errors.Err("claimID (%s) or claimName (%s) missing", claimID, claimName)
if status.Status == VideoStatusPublished || status.Status == VideoStatusUpgradeFailed {
if status.ClaimID == "" || status.ClaimName == "" {
return errors.Err("claimID (%s) or claimName (%s) missing", status.ClaimID, status.ClaimName)
}
vals.Add("published_at", strconv.FormatInt(time.Now().Unix(), 10))
vals.Add("claim_id", claimID)
vals.Add("claim_name", claimName)
if metadataVersion > 0 {
vals.Add("metadata_version", fmt.Sprintf("%d", metadataVersion))
vals.Add("claim_id", status.ClaimID)
vals.Add("claim_name", status.ClaimName)
if status.MetaDataVersion > 0 {
vals.Add("metadata_version", fmt.Sprintf("%d", status.MetaDataVersion))
}
if size != nil {
vals.Add("size", strconv.FormatInt(*size, 10))
if status.Size != nil {
vals.Add("size", strconv.FormatInt(*status.Size, 10))
}
}
if failureReason != "" {
vals.Add("failure_reason", failureReason)
if status.FailureReason != "" {
vals.Add("failure_reason", status.FailureReason)
}
if status.IsTransferred != nil {
vals.Add("transferred", strconv.FormatBool(*status.IsTransferred))
}
res, _ := http.PostForm(endpoint, vals)
defer res.Body.Close()