diff options
Diffstat (limited to 'internal/norm/norm.go')
| -rw-r--r-- | internal/norm/norm.go | 14 |
1 files changed, 11 insertions, 3 deletions
diff --git a/internal/norm/norm.go b/internal/norm/norm.go index e7933ec..77bb604 100644 --- a/internal/norm/norm.go +++ b/internal/norm/norm.go @@ -11,6 +11,11 @@ import ( unorm "golang.org/x/text/unicode/norm" ) +// Version identifies what Text and Name produce. Bump it whenever a change +// could make any string normalise differently: keyword answers cached under +// another version are discarded (spec §6.1). +const Version = 1 + // special holds the letters that do not decompose under Unicode NFD, so // Fold maps them explicitly. var special = map[rune]string{ @@ -18,15 +23,17 @@ var special = map[rune]string{ 'ø': "o", 'Ø': "O", 'đ': "d", 'Đ': "D", 'ħ': "h", 'Ħ': "H", - 'ß': "ss", + 'ß': "ss", 'ẞ': "SS", 'æ': "ae", 'Æ': "AE", 'œ': "oe", 'Œ': "OE", 'ı': "i", } // Fold strips diacritics: Unicode NFD, drop combining marks (category Mn), -// then map the letters that do not decompose. ASCII input is returned -// unchanged without allocating. +// then map the letters that do not decompose. Invalid UTF-8 becomes U+FFFD +// first: left in, an incomplete sequence can keep NFD from decomposing the +// letter after it, and folding the result again would change it. ASCII +// input is returned unchanged without allocating. func Fold(s string) string { ascii := true for i := 0; i < len(s); i++ { @@ -39,6 +46,7 @@ func Fold(s string) string { return s } + s = strings.ToValidUTF8(s, "\ufffd") var b strings.Builder b.Grow(len(s)) for _, r := range unorm.NFD.String(s) { |