Add support for new SDK (0.37.*) and support for upgrading channels and claims to new metadata #28
7 changed files with 229 additions and 97 deletions
2
go.sum
2
go.sum
|
|
@ -18,6 +18,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
|
|||
github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
|
||||
github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
|
||||
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
|
||||
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzsP1G42dRafH3vf+al2vQIJU0YHX+1Tw87oco=
|
||||
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
|
||||
github.com/aws/aws-sdk-go v1.17.3 h1:KBXxg7Jh0TxE5zmpNB2DwKmJeDUqh0O6jhy25TuYOmc=
|
||||
github.com/aws/aws-sdk-go v1.17.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
|
|
@ -59,6 +60,7 @@ github.com/go-ini/ini v1.42.0 h1:TWr1wGj35+UiWHlBA8er89seFXxzwFn11spilrrj+38=
|
|||
github.com/go-ini/ini v1.42.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible h1:sUy/in/P6askYr16XJgTKq/0SZhiWsdg4WZGaLsGQkM=
|
||||
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
|
||||
|
|
|
|||
|
|
@ -133,6 +133,7 @@ func (s *SyncManager) Start() error {
|
|||
AwsS3Region: s.awsS3Region,
|
||||
AwsS3Bucket: s.awsS3Bucket,
|
||||
namer: namer.NewNamer(),
|
||||
Fee: channels[0].Fee,
|
||||
}
|
||||
shouldInterruptLoop = true
|
||||
} else {
|
||||
|
|
@ -167,6 +168,7 @@ func (s *SyncManager) Start() error {
|
|||
AwsS3Region: s.awsS3Region,
|
||||
AwsS3Bucket: s.awsS3Bucket,
|
||||
namer: namer.NewNamer(),
|
||||
Fee: c.Fee,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -74,18 +74,17 @@ type Sync struct {
|
|||
AwsS3Secret string
|
||||
AwsS3Region string
|
||||
AwsS3Bucket string
|
||||
|
||||
daemon *jsonrpc.Client
|
||||
claimAddress string
|
||||
videoDirectory string
|
||||
syncedVideosMux *sync.RWMutex
|
||||
syncedVideos map[string]sdk.SyncedVideo
|
||||
grp *stop.Group
|
||||
lbryChannelID string
|
||||
namer *namer.Namer
|
||||
|
||||
walletMux *sync.Mutex
|
||||
queue chan video
|
||||
Fee *sdk.Fee
|
||||
daemon *jsonrpc.Client
|
||||
claimAddress string
|
||||
videoDirectory string
|
||||
syncedVideosMux *sync.RWMutex
|
||||
syncedVideos map[string]sdk.SyncedVideo
|
||||
grp *stop.Group
|
||||
lbryChannelID string
|
||||
namer *namer.Namer
|
||||
walletMux *sync.Mutex
|
||||
queue chan video
|
||||
}
|
||||
|
||||
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string) {
|
||||
|
|
@ -876,6 +875,7 @@ func (s *Sync) processVideo(v video) (err error) {
|
|||
MaxVideoSize: s.Manager.maxVideoSize,
|
||||
Namer: s.namer,
|
||||
MaxVideoLength: s.Manager.maxVideoLength,
|
||||
Fee: s.Fee,
|
||||
}
|
||||
|
||||
summary, err := v.Sync(s.daemon, sp, &sv, isUpgradeSync)
|
||||
|
|
|
|||
13
sdk/api.go
13
sdk/api.go
|
|
@ -35,16 +35,17 @@ type SyncProperties struct {
|
|||
YoutubeChannelID string
|
||||
}
|
||||
|
||||
type Fee struct {
|
||||
Amount string `json:"amount"`
|
||||
Address string `json:"address"`
|
||||
Currency string `json:"currency"`
|
||||
}
|
||||
type YoutubeChannel struct {
|
||||
ChannelId string `json:"channel_id"`
|
||||
TotalVideos uint `json:"total_videos"`
|
||||
DesiredChannelName string `json:"desired_channel_name"`
|
||||
Fee *struct {
|
||||
Amount string `json:"amount"`
|
||||
Address string `json:"address"`
|
||||
Currency string `json:"currency"`
|
||||
} `json:"fee"`
|
||||
ChannelClaimID string `json:"channel_claim_id"`
|
||||
Fee *Fee `json:"fee"`
|
||||
ChannelClaimID string `json:"channel_claim_id"`
|
||||
}
|
||||
|
||||
func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeChannel, error) {
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import (
|
|||
"github.com/ChannelMeter/iso8601duration"
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/nikooo777/ytdl"
|
||||
"github.com/shopspring/decimal"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"google.golang.org/api/youtube/v3"
|
||||
)
|
||||
|
|
@ -50,38 +51,38 @@ type YoutubeVideo struct {
|
|||
const reflectorURL = "http://blobs.lbry.io/"
|
||||
|
||||
var youtubeCategories = map[string]string{
|
||||
"1": "Film & Animation",
|
||||
"2": "Autos & Vehicles",
|
||||
"10": "Music",
|
||||
"15": "Pets & Animals",
|
||||
"17": "Sports",
|
||||
"18": "Short Movies",
|
||||
"19": "Travel & Events",
|
||||
"20": "Gaming",
|
||||
"21": "Videoblogging",
|
||||
"22": "People & Blogs",
|
||||
"23": "Comedy",
|
||||
"24": "Entertainment",
|
||||
"25": "News & Politics",
|
||||
"26": "Howto & Style",
|
||||
"27": "Education",
|
||||
"28": "Science & Technology",
|
||||
"29": "Nonprofits & Activism",
|
||||
"30": "Movies",
|
||||
"31": "Anime/Animation",
|
||||
"32": "Action/Adventure",
|
||||
"33": "Classics",
|
||||
"34": "Comedy",
|
||||
"35": "Documentary",
|
||||
"36": "Drama",
|
||||
"37": "Family",
|
||||
"38": "Foreign",
|
||||
"39": "Horror",
|
||||
"40": "Sci-Fi/Fantasy",
|
||||
"41": "Thriller",
|
||||
"42": "Shorts",
|
||||
"43": "Shows",
|
||||
"44": "Trailers",
|
||||
"1": "film & animation",
|
||||
"2": "autos & vehicles",
|
||||
"10": "music",
|
||||
"15": "pets & animals",
|
||||
"17": "sports",
|
||||
"18": "short movies",
|
||||
"19": "travel & events",
|
||||
"20": "gaming",
|
||||
"21": "videoblogging",
|
||||
"22": "people & blogs",
|
||||
"23": "comedy",
|
||||
"24": "entertainment",
|
||||
"25": "news & politics",
|
||||
"26": "howto & style",
|
||||
"27": "education",
|
||||
"28": "science & technology",
|
||||
"29": "nonprofits & activism",
|
||||
"30": "movies",
|
||||
"31": "anime/animation",
|
||||
"32": "action/adventure",
|
||||
"33": "classics",
|
||||
"34": "comedy",
|
||||
"35": "documentary",
|
||||
"36": "drama",
|
||||
"37": "family",
|
||||
"38": "foreign",
|
||||
"39": "horror",
|
||||
"40": "sci-fi/fantasy",
|
||||
"41": "thriller",
|
||||
"42": "shorts",
|
||||
"43": "shows",
|
||||
"44": "trailers",
|
||||
}
|
||||
|
||||
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config) *YoutubeVideo {
|
||||
|
|
@ -319,19 +320,33 @@ func (v *YoutubeVideo) triggerThumbnailSave() (err error) {
|
|||
return err
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amount float64, namer *namer.Namer) (*SyncSummary, error) {
|
||||
func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
|
||||
languages, locations, tags := v.getMetadata()
|
||||
|
||||
var fee *jsonrpc.Fee
|
||||
if params.Fee != nil {
|
||||
feeAmount, err := decimal.NewFromString(params.Fee.Amount)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
fee = &jsonrpc.Fee{
|
||||
FeeAddress: ¶ms.Fee.Address,
|
||||
FeeAmount: feeAmount,
|
||||
FeeCurrency: jsonrpc.Currency(params.Fee.Currency),
|
||||
}
|
||||
}
|
||||
|
||||
options := jsonrpc.StreamCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
Title: &v.title,
|
||||
Description: util.PtrToString(v.getAbbrevDescription()),
|
||||
ClaimAddress: &claimAddress,
|
||||
ClaimAddress: ¶ms.ClaimAddress,
|
||||
Languages: languages,
|
||||
ThumbnailURL: &v.thumbnailURL,
|
||||
Tags: tags,
|
||||
Locations: locations,
|
||||
},
|
||||
Fee: fee,
|
||||
License: util.PtrToString("Copyrighted (contact publisher)"),
|
||||
ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
|
||||
ChannelID: &v.lbryChannelID,
|
||||
|
|
@ -340,7 +355,7 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amou
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return publishAndRetryExistingNames(daemon, v.title, downloadPath, amount, options, namer)
|
||||
return publishAndRetryExistingNames(daemon, v.title, downloadPath, params.Amount, options, params.Namer)
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) Size() *int64 {
|
||||
|
|
@ -354,6 +369,7 @@ type SyncParams struct {
|
|||
MaxVideoSize int
|
||||
Namer *namer.Namer
|
||||
MaxVideoLength float64
|
||||
Fee *sdk.Fee
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool) (*SyncSummary, error) {
|
||||
|
|
@ -385,7 +401,7 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
|
|||
}
|
||||
log.Debugln("Created thumbnail for " + v.id)
|
||||
|
||||
summary, err := v.publish(daemon, params.ClaimAddress, params.Amount, params.Namer)
|
||||
summary, err := v.publish(daemon, params)
|
||||
//delete the video in all cases (and ignore the error)
|
||||
_ = v.delete()
|
||||
|
||||
|
|
@ -455,21 +471,37 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
|||
return nil, errors.Err("the video must be republished as we can't get the right size")
|
||||
}
|
||||
}
|
||||
var fee *jsonrpc.Fee
|
||||
if params.Fee != nil {
|
||||
feeAmount, err := decimal.NewFromString(params.Fee.Amount)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
fee = &jsonrpc.Fee{
|
||||
FeeAddress: ¶ms.Fee.Address,
|
||||
FeeAmount: feeAmount,
|
||||
FeeCurrency: jsonrpc.Currency(params.Fee.Currency),
|
||||
}
|
||||
}
|
||||
streamCreateOptions := &jsonrpc.StreamCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
Tags: tags,
|
||||
ThumbnailURL: &thumbnailURL,
|
||||
Languages: languages,
|
||||
Locations: locations,
|
||||
},
|
||||
Author: util.PtrToString(""),
|
||||
License: util.PtrToString("Copyrighted (contact publisher)"),
|
||||
ChannelID: &v.lbryChannelID,
|
||||
Height: util.PtrToUint(720),
|
||||
Width: util.PtrToUint(1280),
|
||||
Fee: fee,
|
||||
}
|
||||
|
||||
if v.mocked {
|
||||
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
|
||||
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
Tags: tags,
|
||||
ThumbnailURL: &thumbnailURL,
|
||||
},
|
||||
Author: util.PtrToString(""),
|
||||
License: util.PtrToString("Copyrighted (contact publisher)"),
|
||||
ChannelID: &v.lbryChannelID,
|
||||
Height: util.PtrToUint(720),
|
||||
Width: util.PtrToUint(1280),
|
||||
},
|
||||
FileSize: &videoSize,
|
||||
StreamCreateOptions: streamCreateOptions,
|
||||
FileSize: &videoSize,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -486,28 +518,16 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
|||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
streamCreateOptions.ClaimCreateOptions.Title = &v.title
|
||||
streamCreateOptions.ClaimCreateOptions.Description = util.PtrToString(v.getAbbrevDescription())
|
||||
streamCreateOptions.Duration = util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds())))
|
||||
streamCreateOptions.ReleaseTime = util.PtrToInt64(v.publishedAt.Unix())
|
||||
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
|
||||
ClearLanguages: util.PtrToBool(true),
|
||||
ClearLocations: util.PtrToBool(true),
|
||||
ClearTags: util.PtrToBool(true),
|
||||
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
Title: &v.title,
|
||||
Description: util.PtrToString(v.getAbbrevDescription()),
|
||||
Tags: tags,
|
||||
Languages: languages,
|
||||
Locations: locations,
|
||||
ThumbnailURL: &thumbnailURL,
|
||||
},
|
||||
Author: util.PtrToString(""),
|
||||
License: util.PtrToString("Copyrighted (contact publisher)"),
|
||||
Height: util.PtrToUint(720),
|
||||
Width: util.PtrToUint(1280),
|
||||
ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
|
||||
Duration: util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))),
|
||||
ChannelID: &v.lbryChannelID,
|
||||
},
|
||||
FileSize: &videoSize,
|
||||
ClearLanguages: util.PtrToBool(true),
|
||||
ClearLocations: util.PtrToBool(true),
|
||||
ClearTags: util.PtrToBool(true),
|
||||
StreamCreateOptions: streamCreateOptions,
|
||||
FileSize: &videoSize,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
|
|||
|
|
@ -1,8 +1,11 @@
|
|||
package tagsManager
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -70,17 +73,23 @@ func SanitizeTags(tags []string, youtubeChannelID string) ([]string, error) {
|
|||
|
||||
func normalizeTag(t string) (string, error) {
|
||||
t = strings.ToLower(t)
|
||||
multipleSpaces := regexp.MustCompile(`\s{2,}`)
|
||||
leadingAndTrailingSpaces := regexp.MustCompile(`^\s+|\s$`)
|
||||
hashTags := regexp.MustCompile(`(#\d+\s)|#+`)
|
||||
inParenthesis := regexp.MustCompile(`\([^\)]+\)`)
|
||||
weirdChars := regexp.MustCompile(`[^-\w'& +\/A-Za-zÀ-ÖØ-öø-ÿ]`)
|
||||
startsOrEndsInWeirdChars := regexp.MustCompile(`^[^A-Za-zÀ-ÖØ-öø-ÿ0-9]+|[^A-Za-zÀ-ÖØ-öø-ÿ0-9]+$`)
|
||||
t = hashTags.ReplaceAllString(t, "")
|
||||
t = inParenthesis.ReplaceAllString(t, " ")
|
||||
t = startsOrEndsInWeirdChars.ReplaceAllString(t, "")
|
||||
t = multipleSpaces.ReplaceAllString(t, " ")
|
||||
t = leadingAndTrailingSpaces.ReplaceAllString(t, "")
|
||||
if weirdChars.MatchString(t) {
|
||||
log.Debugf("tag '%s' has weird stuff in it, skipping\n", t)
|
||||
return "", nil
|
||||
|
||||
}
|
||||
return t, nil
|
||||
//r, err := regexp.Compile("/\\([^\\)]+\\)/g")
|
||||
//if err != nil {
|
||||
// return "", errors.Err(err)
|
||||
//}
|
||||
//r2, err := regexp.Compile("/[^\\w-'& \+]/g")
|
||||
//if err != nil {
|
||||
// return "", errors.Err(err)
|
||||
//}
|
||||
//t = r.ReplaceAllString(t, "")
|
||||
//return r2.ReplaceAllString(t, ""), nil
|
||||
}
|
||||
|
||||
type tagsSanitizer struct {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package tagsManager
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
|
|
@ -44,3 +45,100 @@ outer:
|
|||
}
|
||||
|
||||
}
|
||||
func TestNormalizeTag(t *testing.T) {
|
||||
tags := []string{
|
||||
"blockchain",
|
||||
"Switzerland",
|
||||
"news ",
|
||||
" science & Technology ",
|
||||
"economics",
|
||||
"experiments",
|
||||
"this",
|
||||
"in it",
|
||||
"will build the (WOOPS)",
|
||||
"~has",
|
||||
"crypto",
|
||||
"trump",
|
||||
"wall",
|
||||
"expensive",
|
||||
"!currency",
|
||||
" a lot of ",
|
||||
"#",
|
||||
"#whatever",
|
||||
"#123",
|
||||
"#123 Something else",
|
||||
"#123aaa",
|
||||
"!asdasd",
|
||||
"CASA BLANCA",
|
||||
"wwe 2k18 Elimination chamber!",
|
||||
"pero'",
|
||||
"però",
|
||||
"è proprio",
|
||||
"Ep 29",
|
||||
"sctest29 Keddr",
|
||||
"mortal kombat 11 shang tsung",
|
||||
"!asdasd!",
|
||||
}
|
||||
normalizedTags := make([]string, 0, len(tags))
|
||||
for _, tag := range tags {
|
||||
got, err := normalizeTag(tag)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
if got != "" {
|
||||
normalizedTags = append(normalizedTags, got)
|
||||
}
|
||||
fmt.Printf("Got tag: '%s'\n", got)
|
||||
}
|
||||
expected := []string{
|
||||
"blockchain",
|
||||
"switzerland",
|
||||
"news",
|
||||
"science & technology",
|
||||
"economics",
|
||||
"experiments",
|
||||
"this",
|
||||
"in it",
|
||||
"will build the",
|
||||
"has",
|
||||
"crypto",
|
||||
"trump",
|
||||
"wall",
|
||||
"expensive",
|
||||
"currency",
|
||||
"a lot of",
|
||||
"whatever",
|
||||
"123",
|
||||
"something else",
|
||||
"123aaa",
|
||||
"asdasd",
|
||||
"casa blanca",
|
||||
"wwe 2k18 elimination chamber",
|
||||
"pero",
|
||||
"però",
|
||||
"è proprio",
|
||||
"ep 29",
|
||||
"sctest29 keddr",
|
||||
"mortal kombat 11 shang tsung",
|
||||
"asdasd",
|
||||
}
|
||||
if !Equal(normalizedTags, expected) {
|
||||
t.Error("result not as expected")
|
||||
return
|
||||
}
|
||||
|
||||
}
|
||||
func Equal(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
fmt.Printf("expected length %d but got %d", len(b), len(a))
|
||||
return false
|
||||
}
|
||||
for i, v := range a {
|
||||
if v != b[i] {
|
||||
fmt.Printf("expected %s but bot %s\n", b[i], v)
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue