plan 8: fuzz decoders; fold ẞ and invalid UTF-8 correctly, refuse non-UTF-8 paths in krino new

4 files changed, 45 insertions, 3 deletions

diff --git a/internal/norm/fuzz_test.go b/internal/norm/fuzz_test.go
new file mode 100644
index 0000000..fde62a0
--- /dev/null
+++ b/internal/norm/fuzz_test.go
@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package norm
+
+import "testing"
+
+// FuzzTextIdempotent: normalising text twice changes nothing more than
+// normalising it once, under every case and fold setting, and so for names.
+// Keywords are normalised at load and text at match time by the same
+// functions; a second pass that still changed something would mean the two
+// could disagree about what one normalised string is.
+func FuzzTextIdempotent(f *testing.F) {
+	for _, s := range []string{"Spółka Z O.O.", "ẞ straße", "a\u0301", "  tabs\tand\nlines ", "\xff", "İstanbul", "Dž", "Æsir Œuvre"} {
+		f.Add(s)
+	}
+	f.Fuzz(func(t *testing.T, s string) {
+		for _, opt := range [][2]bool{{false, false}, {true, false}, {false, true}, {true, true}} {
+			once := Text(s, opt[0], opt[1])
+			if twice := Text(once, opt[0], opt[1]); twice != once {
+				t.Fatalf("Text(%q, %v, %v) = %q, again %q", s, opt[0], opt[1], once, twice)
+			}
+		}
+		for _, fold := range []bool{false, true} {
+			once := Name(s, fold)
+			if twice := Name(once, fold); twice != once {
+				t.Fatalf("Name(%q, %v) = %q, again %q", s, fold, once, twice)
+			}
+		}
+	})
+}
diff --git a/internal/norm/norm.go b/internal/norm/norm.go
index e7933ec..77bb604 100644
--- a/internal/norm/norm.go
+++ b/internal/norm/norm.go
@@ -11,6 +11,11 @@ import (
 	unorm "golang.org/x/text/unicode/norm"
 )
 
+// Version identifies what Text and Name produce. Bump it whenever a change
+// could make any string normalise differently: keyword answers cached under
+// another version are discarded (spec §6.1).
+const Version = 1
+
 // special holds the letters that do not decompose under Unicode NFD, so
 // Fold maps them explicitly.
 var special = map[rune]string{
@@ -18,15 +23,17 @@ var special = map[rune]string{
 	'ø': "o", 'Ø': "O",
 	'đ': "d", 'Đ': "D",
 	'ħ': "h", 'Ħ': "H",
-	'ß': "ss",
+	'ß': "ss", 'ẞ': "SS",
 	'æ': "ae", 'Æ': "AE",
 	'œ': "oe", 'Œ': "OE",
 	'ı': "i",
 }
 
 // Fold strips diacritics: Unicode NFD, drop combining marks (category Mn),
-// then map the letters that do not decompose. ASCII input is returned
-// unchanged without allocating.
+// then map the letters that do not decompose. Invalid UTF-8 becomes U+FFFD
+// first: left in, an incomplete sequence can keep NFD from decomposing the
+// letter after it, and folding the result again would change it. ASCII
+// input is returned unchanged without allocating.
 func Fold(s string) string {
 	ascii := true
 	for i := 0; i < len(s); i++ {
@@ -39,6 +46,7 @@ func Fold(s string) string {
 		return s
 	}
 
+	s = strings.ToValidUTF8(s, "\ufffd")
 	var b strings.Builder
 	b.Grow(len(s))
 	for _, r := range unorm.NFD.String(s) {
diff --git a/internal/norm/norm_test.go b/internal/norm/norm_test.go
index 2fc35b6..885c6f7 100644
--- a/internal/norm/norm_test.go
+++ b/internal/norm/norm_test.go
@@ -12,6 +12,8 @@ func TestFold(t *testing.T) {
 		"Łódź":              "Lodz",
 		"ZAŻÓŁĆ GĘŚLĄ JAŹŃ": "ZAZOLC GESLA JAZN",
 		"Straße":            "Strasse",
+		"STRAẞE":            "STRASSE",
+		"\xf3Á":             "\ufffdA", // an invalid byte must not stop the next letter folding
 		"Øresund":           "Oresund",
 		"Ærø":               "AEro",
 		"œuvre":             "oeuvre",
diff --git a/internal/norm/testdata/fuzz/FuzzTextIdempotent/d26a0358dee954b3 b/internal/norm/testdata/fuzz/FuzzTextIdempotent/d26a0358dee954b3
new file mode 100644
index 0000000..4b3fbfe
--- /dev/null
+++ b/internal/norm/testdata/fuzz/FuzzTextIdempotent/d26a0358dee954b3
@@ -0,0 +1,2 @@
+go test fuzz v1
+string("\xf3Á")