lbcd/claimtrie/normalization/case_folder.go
2021-08-19 17:51:07 -04:00

52 lines
1 KiB
Go

package normalization
import (
"bytes"
_ "embed"
"regexp"
"strconv"
"strings"
"unicode/utf8"
)
//go:embed CaseFolding_v11.txt
var v11 string
var foldMap map[rune][]rune
func init() {
foldMap = map[rune][]rune{}
r, _ := regexp.Compile(`([[:xdigit:]]+?); (.); ([[:xdigit:] ]+?);`)
matches := r.FindAllStringSubmatch(v11, 1000000000)
for i := range matches {
if matches[i][2] == "C" || matches[i][2] == "F" {
key, _ := strconv.Unquote(`"\u` + matches[i][1] + `"`)
splits := strings.Split(matches[i][3], " ")
var values []rune
for j := range splits {
value, _ := strconv.Unquote(`"\u` + splits[j] + `"`)
values = append(values, []rune(value)[0])
}
foldMap[[]rune(key)[0]] = values
}
}
}
func CaseFold(name []byte) []byte {
var b bytes.Buffer
b.Grow(len(name))
for _, r := range string(name) {
if r == utf8.RuneError {
return name
}
replacements := foldMap[r]
if len(replacements) > 0 {
for j := range replacements {
b.WriteRune(replacements[j])
}
} else {
b.WriteRune(r)
}
}
return b.Bytes()
}