allow claim names with non latin chars

upgrade to latest lbrynet
improve e2e test

e2e/data_setup.sh

@@ -20,5 +20,5 @@ ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(2,"youtubertok
 mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
 #Add their youtube channel to be synced
 ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable,total_views,reviewed,last_uploaded_video,length_limit,size_limit,reward_amount,reward_expiration)
-VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@test"$(date +%s)"','UCNQfQvFMPnInwsU_iGYArJQ','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',10,10,1,1,10000,1,'7bBV2Z-9wpo',60,2048,0,'2019-08-01 00:00:00')"
-mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL"
+VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','$1','$2','СтопХам','queued','2019-08-01 00:00:00','sync',1000,1000,1,1,10000,1,'d1230hX1eMc',60,2048,0,'2019-08-01 00:00:00')"
+mysql -u lbry -plbry -D lbry -h "127.0.0.1" --default-character-set=utf8 -P 15500 -e "$ADDYTCHANNEL"

e2e/docker-compose.yml

@@ -21,7 +21,7 @@ services:
   ## Wallet Server ##
   ###################
   walletserver:
-    image: lbry/wallet-server:v0.83.0
+    image: lbry/wallet-server:latest-release
     restart: always
     environment:
       - DB_DIRECTORY=/database
@@ -47,7 +47,7 @@ services:
   ## Lbrynet ##
   #############
   lbrynet:
-    image: lbry/lbrynet:v0.83.0
+    image: lbry/lbrynet:v0.90.1
     restart: always
     ports:
       - "15100:5279"
@@ -68,7 +68,7 @@ services:
   ## MySQL ##
   ###########
   mysql:
-    image: mysql/mysql-server:5.7.27
+    image: mysql/mysql-server:5.7.33
     restart: "no"
     ports:
       - "15500:3306"
@@ -99,12 +99,13 @@ services:
     environment:
       - MYSQL_DSN=lbry:lbry@tcp(mysql:3306)/lbry
       - LBRYCRD_CONNECT=rpc://lbry:lbry@lbrycrd:29245
+      - REPLICA_DSN=lbry:lbry@tcp(mysql:3306)/lbry
     entrypoint: wait-for-it -t 0 chainquery:6300 -- wait-for-it -t 0 lbrycrd:29245 -- ./latest serve
   ################
   ## Chainquery ##
   ################
   chainquery:
-    image: lbry/chainquery:v2.0.8
+    image: lbry/chainquery:master
     restart: "no"
     ports:
       - 6300:6300

e2e/e2e.sh

@@ -50,11 +50,14 @@ until curl --output /dev/null --silent --head --fail http://localhost:15400; do
 done
 echo "successfully started..."
 
+channelToSync="UCJIMS94jwjEmvnsNqkH_KUg"
+channelName=@СтопХам"$(date +%s)"
+
 #Data Setup for test
-./data_setup.sh
+./data_setup.sh "$channelName" "$channelToSync"
 
 # Execute the sync test!
-./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container
+./../bin/ytsync --channelID "$channelToSync" --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container
 status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1')
 videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1')
 videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=1')
@@ -62,17 +65,17 @@ videoClaimID2=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SEL
 videoClaimAddress1=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=2')
 videoClaimAddress2=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=3')
 # Create Supports for published claim
-./supporty/supporty @BeamerTest "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 1.0
-./supporty/supporty @BeamerTest "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 2.0
-./supporty/supporty @BeamerTest "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 3.0
-./supporty/supporty @BeamerTest "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 3.0
+./supporty/supporty "$channelName" "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 1.0
+./supporty/supporty "$channelName" "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 2.0
+./supporty/supporty "$channelName" "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 3.0
+./supporty/supporty "$channelName" "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 3.0
 curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"generate","params":[1]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200
 # Reset status for transfer test
 mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e "UPDATE youtube_data SET status = 'queued' WHERE id = 1"
 # Trigger transfer api
 curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http://localhost:15400/yt/transfer?auth_token=youtubertoken&address=n4eYeXAYmHo4YRUDEfsEhucy8y5LKRMcHg&public_key=tpubDA9GDAntyJu4hD3wU7175p7CuV6DWbYXfyb2HedBA3yuBp9HZ4n3QE4Ex6RHCSiEuVp2nKAL1Lzf2ZLo9ApaFgNaJjG6Xo1wB3iEeVbrDZp'
 # Execute the transfer test!
-./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container
+./../bin/ytsync --channelID $channelToSync --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container
 # Check that the channel and the video are marked as transferred and that all supports are spent
 channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transfer_state FROM youtube_data')
 videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transferred FROM synced_video')

e2e/lbrynet/docker-compose.yml

@@ -8,7 +8,7 @@ services:
   ## Lbrynet ##
   #############
   lbrynet:
-    image: lbry/lbrynet:v0.83.0
+    image: lbry/lbrynet:v0.90.1
     restart: "no"
     networks:
       lbry-network:

namer/names.go

@@ -10,7 +10,7 @@ import (
 	"sync"
 )
 
-var titleRegexp = regexp.MustCompile(`[^a-zA-Z0-9]+`)
+var claimNameRegexp = regexp.MustCompile(`[=&#:$@%?;\\"/<>%{}|^~\x60[\]\s]`)
 
 type Namer struct {
 	mu    *sync.Mutex
@@ -68,7 +68,7 @@ func getClaimNameFromTitle(title string, attempt int) string {
 	}
 	maxLen := 40 - len(suffix)
 
-	chunks := strings.Split(strings.ToLower(strings.Trim(titleRegexp.ReplaceAllString(title, "-"), "-")), "-")
+	chunks := strings.Split(strings.ToLower(strings.Trim(claimNameRegexp.ReplaceAllString(title, "-"), "-")), "-")
 
 	name := chunks[0]
 	if len(name) > maxLen {
@@ -76,6 +76,9 @@ func getClaimNameFromTitle(title string, attempt int) string {
 	}
 
 	for _, chunk := range chunks[1:] {
+		if chunk == "" {
+			continue
+		}
 		tmpName := name + "-" + chunk
 		if len(tmpName) > maxLen {
 			if len(name) < 20 {

namer/names_test.go

@@ -0,0 +1,16 @@
+package namer
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_getClaimNameFromTitle(t *testing.T) {
+	name := getClaimNameFromTitle("СтопХам - \"В ожидании ответа\"", 0)
+	assert.Equal(t, name, "стопхам-в-ожидании")
+	name = getClaimNameFromTitle("SADB - \"A Weak Woman With a Strong Hood\"", 0)
+	assert.Equal(t, name, "sadb-a-weak-woman-with-a-strong-hood")
+	name = getClaimNameFromTitle("錢包整理術 5 Tips、哪種錢包最NG？｜有錢人默默在做的「錢包整理術」 ft.@SHIN LI", 0)
+	assert.Equal(t, name, "錢包整理術-5-tips、哪種錢包最")
+}