fix bugs with claim names and unicode

update tests and improve logging
This commit is contained in:
Niko Storni 2021-03-15 22:50:23 +01:00
parent 9799b0a732
commit cacd21f840
5 changed files with 33 additions and 9 deletions

View file

@ -20,5 +20,5 @@ ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(2,"youtubertok
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
#Add their youtube channel to be synced
ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable,total_views,reviewed,last_uploaded_video,length_limit,size_limit,reward_amount,reward_expiration)
VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','$1','$2','СтопХам','queued','2019-08-01 00:00:00','sync',1000,1000,1,1,10000,1,'d1230hX1eMc',60,2048,0,'2019-08-01 00:00:00')"
VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','$1','$2','СтопХам','queued','2019-08-01 00:00:00','sync',1000,1000,1,1,10000,1,'$3',60,2048,0,'2019-08-01 00:00:00')"
mysql -u lbry -plbry -D lbry -h "127.0.0.1" --default-character-set=utf8 -P 15500 -e "$ADDYTCHANNEL"

View file

@ -50,11 +50,12 @@ until curl --output /dev/null --silent --head --fail http://localhost:15400; do
done
echo "successfully started..."
channelToSync="UCJIMS94jwjEmvnsNqkH_KUg"
channelName=@СтопХам"$(date +%s)"
channelToSync="UCGyoEsIRjmnmzrsB67DhrOA"
channelName=@Alaminemoh11"$(date +%s)"
latestVideoID="ejWF7Jjdgmc"
#Data Setup for test
./data_setup.sh "$channelName" "$channelToSync"
./data_setup.sh "$channelName" "$channelToSync" "$latestVideoID"
# Execute the sync test!
./../bin/ytsync --channelID "$channelToSync" --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container

View file

@ -72,7 +72,7 @@ func getClaimNameFromTitle(title string, attempt int) string {
name := chunks[0]
if len(name) > maxLen {
return name[:maxLen]
return truncateUnicode(name, maxLen)
}
for _, chunk := range chunks[1:] {
@ -91,3 +91,18 @@ func getClaimNameFromTitle(title string, attempt int) string {
return name + suffix
}
func truncateUnicode(name string, limit int) string {
reNameBlacklist := regexp.MustCompile(`(&|>|<|\/|:|\n|\r)*`)
name = reNameBlacklist.ReplaceAllString(name, "")
result := name
chars := 0
for i := range name {
if chars >= limit {
result = name[:i]
break
}
chars++
}
return result
}

View file

@ -8,9 +8,17 @@ import (
func Test_getClaimNameFromTitle(t *testing.T) {
name := getClaimNameFromTitle("СтопХам - \"В ожидании ответа\"", 0)
assert.Equal(t, name, "стопхам-в-ожидании")
assert.Equal(t, "стопхам-в-ожидании", name)
name = getClaimNameFromTitle("SADB - \"A Weak Woman With a Strong Hood\"", 0)
assert.Equal(t, name, "sadb-a-weak-woman-with-a-strong-hood")
assert.Equal(t, "sadb-a-weak-woman-with-a-strong-hood", name)
name = getClaimNameFromTitle("錢包整理術 5 Tips、哪種錢包最NG有錢人默默在做的「錢包整理術」 ft.@SHIN LI", 0)
assert.Equal(t, name, "錢包整理術-5-tips、哪種錢包最")
assert.Equal(t, "錢包整理術-5-tips、哪種錢包最", name)
name = getClaimNameFromTitle("اسرع-طريقة-لتختيم", 0)
assert.Equal(t, "اسرع-طريقة-لتختيم", name)
name = getClaimNameFromTitle("شكرا على 380 مشترك😍😍😍😍 لي يريد دعم ادا وصلنا المقطع 40 لايك وراح ادعم قناتين", 0)
assert.Equal(t, "شكرا-على-380-مشترك😍😍\xf0\x9f", name)
name = getClaimNameFromTitle("test-@", 0)
assert.Equal(t, "test", name)
name = getClaimNameFromTitle("『あなたはただの空の殻でした』", 0)
assert.Equal(t, "『あなたはただの空の殻でした』", name)
}

View file

@ -320,7 +320,7 @@ func (a *APIConfig) MarkVideoStatus(status shared.VideoStatus) error {
res, err := http.PostForm(endpoint, vals)
if err != nil {
if strings.Contains(err.Error(), "EOF") {
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
util.SendErrorToSlack("EOF error while trying to call %s for %s. Waiting to retry", endpoint, status.ClaimName)
time.Sleep(30 * time.Second)
return a.MarkVideoStatus(status)
}