fix bugs with claim names and unicode

update tests and improve logging

e2e/data_setup.sh

@@ -20,5 +20,5 @@ ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(2,"youtubertok
 mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
 #Add their youtube channel to be synced
 ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable,total_views,reviewed,last_uploaded_video,length_limit,size_limit,reward_amount,reward_expiration)
-VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','$1','$2','СтопХам','queued','2019-08-01 00:00:00','sync',1000,1000,1,1,10000,1,'d1230hX1eMc',60,2048,0,'2019-08-01 00:00:00')"
+VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','$1','$2','СтопХам','queued','2019-08-01 00:00:00','sync',1000,1000,1,1,10000,1,'$3',60,2048,0,'2019-08-01 00:00:00')"
 mysql -u lbry -plbry -D lbry -h "127.0.0.1" --default-character-set=utf8 -P 15500 -e "$ADDYTCHANNEL"

e2e/e2e.sh

@@ -50,11 +50,12 @@ until curl --output /dev/null --silent --head --fail http://localhost:15400; do
 done
 echo "successfully started..."
 
-channelToSync="UCJIMS94jwjEmvnsNqkH_KUg"
-channelName=@СтопХам"$(date +%s)"
+channelToSync="UCGyoEsIRjmnmzrsB67DhrOA"
+channelName=@Alaminemoh11"$(date +%s)"
+latestVideoID="ejWF7Jjdgmc"
 
 #Data Setup for test
-./data_setup.sh "$channelName" "$channelToSync"
+./data_setup.sh "$channelName" "$channelToSync" "$latestVideoID"
 
 # Execute the sync test!
 ./../bin/ytsync --channelID "$channelToSync" --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container

namer/names.go

@@ -72,7 +72,7 @@ func getClaimNameFromTitle(title string, attempt int) string {
 
 	name := chunks[0]
 	if len(name) > maxLen {
-		return name[:maxLen]
+		return truncateUnicode(name, maxLen)
 	}
 
 	for _, chunk := range chunks[1:] {
@@ -91,3 +91,18 @@ func getClaimNameFromTitle(title string, attempt int) string {
 
 	return name + suffix
 }
+
+func truncateUnicode(name string, limit int) string {
+	reNameBlacklist := regexp.MustCompile(`(&|>|<|\/|:|\n|\r)*`)
+	name = reNameBlacklist.ReplaceAllString(name, "")
+	result := name
+	chars := 0
+	for i := range name {
+		if chars >= limit {
+			result = name[:i]
+			break
+		}
+		chars++
+	}
+	return result
+}

namer/names_test.go

@@ -8,9 +8,17 @@ import (
 
 func Test_getClaimNameFromTitle(t *testing.T) {
 	name := getClaimNameFromTitle("СтопХам - \"В ожидании ответа\"", 0)
-	assert.Equal(t, name, "стопхам-в-ожидании")
+	assert.Equal(t, "стопхам-в-ожидании", name)
 	name = getClaimNameFromTitle("SADB - \"A Weak Woman With a Strong Hood\"", 0)
-	assert.Equal(t, name, "sadb-a-weak-woman-with-a-strong-hood")
+	assert.Equal(t, "sadb-a-weak-woman-with-a-strong-hood", name)
 	name = getClaimNameFromTitle("錢包整理術 5 Tips、哪種錢包最NG？｜有錢人默默在做的「錢包整理術」 ft.@SHIN LI", 0)
-	assert.Equal(t, name, "錢包整理術-5-tips、哪種錢包最")
+	assert.Equal(t, "錢包整理術-5-tips、哪種錢包最", name)
+	name = getClaimNameFromTitle("اسرع-طريقة-لتختيم", 0)
+	assert.Equal(t, "اسرع-طريقة-لتختيم", name)
+	name = getClaimNameFromTitle("شكرا على 380 مشترك😍😍😍😍 لي يريد دعم ادا وصلنا المقطع 40 لايك وراح ادعم قناتين", 0)
+	assert.Equal(t, "شكرا-على-380-مشترك😍😍\xf0\x9f", name)
+	name = getClaimNameFromTitle("test-@", 0)
+	assert.Equal(t, "test", name)
+	name = getClaimNameFromTitle("『あなたはただの空の殻でした』", 0)
+	assert.Equal(t, "『あなたはただの空の殻でした』", name)
 }

sdk/api.go

@@ -320,7 +320,7 @@ func (a *APIConfig) MarkVideoStatus(status shared.VideoStatus) error {
 	res, err := http.PostForm(endpoint, vals)
 	if err != nil {
 		if strings.Contains(err.Error(), "EOF") {
-			util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
+			util.SendErrorToSlack("EOF error while trying to call %s for %s. Waiting to retry", endpoint, status.ClaimName)
 			time.Sleep(30 * time.Second)
 			return a.MarkVideoStatus(status)
 		}