From cacd21f8408134048898ddff94d98286f5733528 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Mon, 15 Mar 2021 22:50:23 +0100 Subject: [PATCH] fix bugs with claim names and unicode update tests and improve logging --- e2e/data_setup.sh | 2 +- e2e/e2e.sh | 7 ++++--- namer/names.go | 17 ++++++++++++++++- namer/names_test.go | 14 +++++++++++--- sdk/api.go | 2 +- 5 files changed, 33 insertions(+), 9 deletions(-) diff --git a/e2e/data_setup.sh b/e2e/data_setup.sh index bbab7e6..266863e 100755 --- a/e2e/data_setup.sh +++ b/e2e/data_setup.sh @@ -20,5 +20,5 @@ ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(2,"youtubertok mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN" #Add their youtube channel to be synced ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable,total_views,reviewed,last_uploaded_video,length_limit,size_limit,reward_amount,reward_expiration) -VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','$1','$2','СтопХам','queued','2019-08-01 00:00:00','sync',1000,1000,1,1,10000,1,'d1230hX1eMc',60,2048,0,'2019-08-01 00:00:00')" +VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','$1','$2','СтопХам','queued','2019-08-01 00:00:00','sync',1000,1000,1,1,10000,1,'$3',60,2048,0,'2019-08-01 00:00:00')" mysql -u lbry -plbry -D lbry -h "127.0.0.1" --default-character-set=utf8 -P 15500 -e "$ADDYTCHANNEL" diff --git a/e2e/e2e.sh b/e2e/e2e.sh index daecc6e..edafec1 100755 --- a/e2e/e2e.sh +++ b/e2e/e2e.sh @@ -50,11 +50,12 @@ until curl --output /dev/null --silent --head --fail http://localhost:15400; do done echo "successfully started..." -channelToSync="UCJIMS94jwjEmvnsNqkH_KUg" -channelName=@СтопХам"$(date +%s)" +channelToSync="UCGyoEsIRjmnmzrsB67DhrOA" +channelName=@Alaminemoh11"$(date +%s)" +latestVideoID="ejWF7Jjdgmc" #Data Setup for test -./data_setup.sh "$channelName" "$channelToSync" +./data_setup.sh "$channelName" "$channelToSync" "$latestVideoID" # Execute the sync test! ./../bin/ytsync --channelID "$channelToSync" --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container diff --git a/namer/names.go b/namer/names.go index c6ca2d7..40bf620 100644 --- a/namer/names.go +++ b/namer/names.go @@ -72,7 +72,7 @@ func getClaimNameFromTitle(title string, attempt int) string { name := chunks[0] if len(name) > maxLen { - return name[:maxLen] + return truncateUnicode(name, maxLen) } for _, chunk := range chunks[1:] { @@ -91,3 +91,18 @@ func getClaimNameFromTitle(title string, attempt int) string { return name + suffix } + +func truncateUnicode(name string, limit int) string { + reNameBlacklist := regexp.MustCompile(`(&|>|<|\/|:|\n|\r)*`) + name = reNameBlacklist.ReplaceAllString(name, "") + result := name + chars := 0 + for i := range name { + if chars >= limit { + result = name[:i] + break + } + chars++ + } + return result +} diff --git a/namer/names_test.go b/namer/names_test.go index 8616d69..02c688d 100644 --- a/namer/names_test.go +++ b/namer/names_test.go @@ -8,9 +8,17 @@ import ( func Test_getClaimNameFromTitle(t *testing.T) { name := getClaimNameFromTitle("СтопХам - \"В ожидании ответа\"", 0) - assert.Equal(t, name, "стопхам-в-ожидании") + assert.Equal(t, "стопхам-в-ожидании", name) name = getClaimNameFromTitle("SADB - \"A Weak Woman With a Strong Hood\"", 0) - assert.Equal(t, name, "sadb-a-weak-woman-with-a-strong-hood") + assert.Equal(t, "sadb-a-weak-woman-with-a-strong-hood", name) name = getClaimNameFromTitle("錢包整理術 5 Tips、哪種錢包最NG？｜有錢人默默在做的「錢包整理術」 ft.@SHIN LI", 0) - assert.Equal(t, name, "錢包整理術-5-tips、哪種錢包最") + assert.Equal(t, "錢包整理術-5-tips、哪種錢包最", name) + name = getClaimNameFromTitle("اسرع-طريقة-لتختيم", 0) + assert.Equal(t, "اسرع-طريقة-لتختيم", name) + name = getClaimNameFromTitle("شكرا على 380 مشترك😍😍😍😍 لي يريد دعم ادا وصلنا المقطع 40 لايك وراح ادعم قناتين", 0) + assert.Equal(t, "شكرا-على-380-مشترك😍😍\xf0\x9f", name) + name = getClaimNameFromTitle("test-@", 0) + assert.Equal(t, "test", name) + name = getClaimNameFromTitle("『あなたはただの空の殻でした』", 0) + assert.Equal(t, "『あなたはただの空の殻でした』", name) } diff --git a/sdk/api.go b/sdk/api.go index 8dbb5ca..f42d298 100644 --- a/sdk/api.go +++ b/sdk/api.go @@ -320,7 +320,7 @@ func (a *APIConfig) MarkVideoStatus(status shared.VideoStatus) error { res, err := http.PostForm(endpoint, vals) if err != nil { if strings.Contains(err.Error(), "EOF") { - util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint) + util.SendErrorToSlack("EOF error while trying to call %s for %s. Waiting to retry", endpoint, status.ClaimName) time.Sleep(30 * time.Second) return a.MarkVideoStatus(status) }