fix bugs with claim names and unicode
update tests and improve logging
This commit is contained in:
parent
9799b0a732
commit
cacd21f840
5 changed files with 33 additions and 9 deletions
|
@ -20,5 +20,5 @@ ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(2,"youtubertok
|
|||
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
|
||||
#Add their youtube channel to be synced
|
||||
ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable,total_views,reviewed,last_uploaded_video,length_limit,size_limit,reward_amount,reward_expiration)
|
||||
VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','$1','$2','СтопХам','queued','2019-08-01 00:00:00','sync',1000,1000,1,1,10000,1,'d1230hX1eMc',60,2048,0,'2019-08-01 00:00:00')"
|
||||
VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','$1','$2','СтопХам','queued','2019-08-01 00:00:00','sync',1000,1000,1,1,10000,1,'$3',60,2048,0,'2019-08-01 00:00:00')"
|
||||
mysql -u lbry -plbry -D lbry -h "127.0.0.1" --default-character-set=utf8 -P 15500 -e "$ADDYTCHANNEL"
|
||||
|
|
|
@ -50,11 +50,12 @@ until curl --output /dev/null --silent --head --fail http://localhost:15400; do
|
|||
done
|
||||
echo "successfully started..."
|
||||
|
||||
channelToSync="UCJIMS94jwjEmvnsNqkH_KUg"
|
||||
channelName=@СтопХам"$(date +%s)"
|
||||
channelToSync="UCGyoEsIRjmnmzrsB67DhrOA"
|
||||
channelName=@Alaminemoh11"$(date +%s)"
|
||||
latestVideoID="ejWF7Jjdgmc"
|
||||
|
||||
#Data Setup for test
|
||||
./data_setup.sh "$channelName" "$channelToSync"
|
||||
./data_setup.sh "$channelName" "$channelToSync" "$latestVideoID"
|
||||
|
||||
# Execute the sync test!
|
||||
./../bin/ytsync --channelID "$channelToSync" --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container
|
||||
|
|
|
@ -72,7 +72,7 @@ func getClaimNameFromTitle(title string, attempt int) string {
|
|||
|
||||
name := chunks[0]
|
||||
if len(name) > maxLen {
|
||||
return name[:maxLen]
|
||||
return truncateUnicode(name, maxLen)
|
||||
}
|
||||
|
||||
for _, chunk := range chunks[1:] {
|
||||
|
@ -91,3 +91,18 @@ func getClaimNameFromTitle(title string, attempt int) string {
|
|||
|
||||
return name + suffix
|
||||
}
|
||||
|
||||
func truncateUnicode(name string, limit int) string {
|
||||
reNameBlacklist := regexp.MustCompile(`(&|>|<|\/|:|\n|\r)*`)
|
||||
name = reNameBlacklist.ReplaceAllString(name, "")
|
||||
result := name
|
||||
chars := 0
|
||||
for i := range name {
|
||||
if chars >= limit {
|
||||
result = name[:i]
|
||||
break
|
||||
}
|
||||
chars++
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
|
|
@ -8,9 +8,17 @@ import (
|
|||
|
||||
func Test_getClaimNameFromTitle(t *testing.T) {
|
||||
name := getClaimNameFromTitle("СтопХам - \"В ожидании ответа\"", 0)
|
||||
assert.Equal(t, name, "стопхам-в-ожидании")
|
||||
assert.Equal(t, "стопхам-в-ожидании", name)
|
||||
name = getClaimNameFromTitle("SADB - \"A Weak Woman With a Strong Hood\"", 0)
|
||||
assert.Equal(t, name, "sadb-a-weak-woman-with-a-strong-hood")
|
||||
assert.Equal(t, "sadb-a-weak-woman-with-a-strong-hood", name)
|
||||
name = getClaimNameFromTitle("錢包整理術 5 Tips、哪種錢包最NG?|有錢人默默在做的「錢包整理術」 ft.@SHIN LI", 0)
|
||||
assert.Equal(t, name, "錢包整理術-5-tips、哪種錢包最")
|
||||
assert.Equal(t, "錢包整理術-5-tips、哪種錢包最", name)
|
||||
name = getClaimNameFromTitle("اسرع-طريقة-لتختيم", 0)
|
||||
assert.Equal(t, "اسرع-طريقة-لتختيم", name)
|
||||
name = getClaimNameFromTitle("شكرا على 380 مشترك😍😍😍😍 لي يريد دعم ادا وصلنا المقطع 40 لايك وراح ادعم قناتين", 0)
|
||||
assert.Equal(t, "شكرا-على-380-مشترك😍😍\xf0\x9f", name)
|
||||
name = getClaimNameFromTitle("test-@", 0)
|
||||
assert.Equal(t, "test", name)
|
||||
name = getClaimNameFromTitle("『あなたはただの空の殻でした』", 0)
|
||||
assert.Equal(t, "『あなたはただの空の殻でした』", name)
|
||||
}
|
||||
|
|
|
@ -320,7 +320,7 @@ func (a *APIConfig) MarkVideoStatus(status shared.VideoStatus) error {
|
|||
res, err := http.PostForm(endpoint, vals)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
util.SendErrorToSlack("EOF error while trying to call %s for %s. Waiting to retry", endpoint, status.ClaimName)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.MarkVideoStatus(status)
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue