plan 8: fuzz decoders; fold ẞ and invalid UTF-8 correctly, refuse non-UTF-8 paths in krino new

1 files changed, 11 insertions, 3 deletions

diff --git a/internal/norm/norm.go b/internal/norm/norm.go
index e7933ec..77bb604 100644
--- a/internal/norm/norm.go
+++ b/internal/norm/norm.go
@@ -11,6 +11,11 @@ import (
 	unorm "golang.org/x/text/unicode/norm"
 )
 
+// Version identifies what Text and Name produce. Bump it whenever a change
+// could make any string normalise differently: keyword answers cached under
+// another version are discarded (spec §6.1).
+const Version = 1
+
 // special holds the letters that do not decompose under Unicode NFD, so
 // Fold maps them explicitly.
 var special = map[rune]string{
@@ -18,15 +23,17 @@ var special = map[rune]string{
 	'ø': "o", 'Ø': "O",
 	'đ': "d", 'Đ': "D",
 	'ħ': "h", 'Ħ': "H",
-	'ß': "ss",
+	'ß': "ss", 'ẞ': "SS",
 	'æ': "ae", 'Æ': "AE",
 	'œ': "oe", 'Œ': "OE",
 	'ı': "i",
 }
 
 // Fold strips diacritics: Unicode NFD, drop combining marks (category Mn),
-// then map the letters that do not decompose. ASCII input is returned
-// unchanged without allocating.
+// then map the letters that do not decompose. Invalid UTF-8 becomes U+FFFD
+// first: left in, an incomplete sequence can keep NFD from decomposing the
+// letter after it, and folding the result again would change it. ASCII
+// input is returned unchanged without allocating.
 func Fold(s string) string {
 	ascii := true
 	for i := 0; i < len(s); i++ {
@@ -39,6 +46,7 @@ func Fold(s string) string {
 		return s
 	}
 
+	s = strings.ToValidUTF8(s, "\ufffd")
 	var b strings.Builder
 	b.Grow(len(s))
 	for _, r := range unorm.NFD.String(s) {