75 lines
2.2 KiB
Go
75 lines
2.2 KiB
Go
|
//go:build use_icu_normalization
|
|||
|
|
// +build use_icu_normalization
|
|||
|
|
|
|||
|
|
package normalization
|
|||
|
|
|
|||
|
|
import (
|
|||
|
|
"bytes"
|
|||
|
|
"encoding/hex"
|
|||
|
|
"testing"
|
|||
|
|
"unicode/utf8"
|
|||
|
|
|
|||
|
|
"github.com/stretchr/testify/assert"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
func TestNormalizationICU(t *testing.T) {
|
|||
|
|
testNormalization(t, normalizeICU)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func BenchmarkNormalizeICU(b *testing.B) {
|
|||
|
|
benchmarkNormalize(b, normalizeICU)
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
var testStrings = []string{
|
|||
|
|
"Les-Masques-Blancs-Die-Dead-place-Sathonay-28-Août",
|
|||
|
|
"Bez-komentu-výbuch-z-vnútra,-radšej-pozri-video...-",
|
|||
|
|
"၂-နစ်အကြာမှာ",
|
|||
|
|
"ငရဲပြည်မှ-6",
|
|||
|
|
"@happyvision",
|
|||
|
|
"ကမ္ဘာပျက်ကိန်း-9",
|
|||
|
|
"ဝိညာဉ်နား၊-3",
|
|||
|
|
"un-amore-nuovo-o-un-ritorno-cosa-mi-dona",
|
|||
|
|
"è-innamorato-di-me-anche-se-non-lo-dice",
|
|||
|
|
"ပြင်ဆင်ပါ-no.1",
|
|||
|
|
"ပြင်ဆင်ပါ-no.4",
|
|||
|
|
"ပြင်ဆင်ပါ-no.2",
|
|||
|
|
"ပြင်ဆင်ပါ-no.3",
|
|||
|
|
"ငရဲပြည်မှ-5",
|
|||
|
|
"ပြင်ဆင်ပါ-no.6",
|
|||
|
|
"ပြင်ဆင်ပါ-no.5",
|
|||
|
|
"ပြင်ဆင်ပါ-no.7",
|
|||
|
|
"ပြင်ဆင်ပါ-no.8",
|
|||
|
|
"အချိန်-2",
|
|||
|
|
"ဝိညာဉ်နား၊-4",
|
|||
|
|
"ပြင်ဆင်ပါ-no.-13",
|
|||
|
|
"ပြင်ဆင်ပါ-no.15",
|
|||
|
|
"ပြင်ဆင်ပါ-9",
|
|||
|
|
"schilddrüsenhormonsubstitution-nach",
|
|||
|
|
"Linxextremismus-JPzuG_UBtEg",
|
|||
|
|
"Ꮖ-Ꮩ-Ꭺ-N--------Ꭺ-N-Ꮹ-Ꭼ-Ꮮ-Ꭺ-on-Instagram_-“Our-next-destination-is-East-and-Southeast-Asia--selfie--asia”",
|
|||
|
|
"ABCDEFGHIJKLMNOPQRSTUVWXYZ",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestBlock760150_1020105(t *testing.T) {
|
|||
|
|
test, _ := hex.DecodeString("43efbfbd")
|
|||
|
|
assert.True(t, utf8.Valid(test))
|
|||
|
|
a := normalizeGo(test)
|
|||
|
|
b := normalizeICU(test)
|
|||
|
|
assert.Equal(t, a, b)
|
|||
|
|
|
|||
|
|
for i, s := range testStrings {
|
|||
|
|
a = normalizeGo([]byte(s))
|
|||
|
|
b = normalizeICU([]byte(s))
|
|||
|
|
assert.Equal(t, a, b, "%d: %s != %s", i, string(a), string(b))
|
|||
|
|
// t.Logf("%s -> %s", s, string(b))
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func TestBlock1085612(t *testing.T) {
|
|||
|
|
s, err := hex.DecodeString("6eccb7cd9dcc92cd90cc86cc80cc80cd91cd9dcd8acd80cd92cc94cc85cc8fccbdcda0ccbdcd80cda0cd84cc94cc8ccc9acd84cc94cd9bcda0cca7cc99ccaccd99cca9cca7")
|
|||
|
|
assert.NoError(t, err)
|
|||
|
|
a := normalizeICU(s)
|
|||
|
|
b := normalizeGo(s)
|
|||
|
|
assert.Equal(t, a, b, "%s != %s, %v", string(a), string(b), bytes.Equal(b, s))
|
|||
|
|
}
|