captures keep the combining marks of a decomposed name; test sources escape invisible characters

3 files changed, 20 insertions, 7 deletions

diff --git a/internal/norm/fuzz_test.go b/internal/norm/fuzz_test.go
index 27deadc..2dc864e 100644
--- a/internal/norm/fuzz_test.go
+++ b/internal/norm/fuzz_test.go
@@ -36,7 +36,7 @@ func FuzzTextIdempotent(f *testing.F) {
 // FuzzFoldMapped: FoldMapped's text is always Fold's, and any part of it
 // leads back to original text that folds to something holding that part.
 func FuzzFoldMapped(f *testing.F) {
-	for _, s := range []string{"Łódź-faktura.pdf", "straße", "áb", "\xff\xfeé", "́a", "Æsir"} {
+	for _, s := range []string{"Łódź-faktura.pdf", "straße", "a\u0301b", "\xff\xfeé", "\u0301a", "Æsir"} {
 		f.Add(s, 0, 2)
 	}
 	f.Fuzz(func(t *testing.T, s string, a, b int) {
diff --git a/internal/norm/norm.go b/internal/norm/norm.go
index bff2127..1c2f61d 100644
--- a/internal/norm/norm.go
+++ b/internal/norm/norm.go
@@ -123,7 +123,9 @@ func FoldMapped(s string) Folded {
 }
 
 // Source returns the original text that Text[a:b] was folded from, widened
-// to whole characters; without a map, Text[a:b] itself.
+// to whole characters and to the characters right after them that fold to
+// nothing - the combining marks of a decomposed name, which belong to the
+// letter before them (plan 11 review L5); without a map, Text[a:b] itself.
 func (f Folded) Source(a, b int) string {
 	switch {
 	case f.same || f.start == nil:
@@ -131,7 +133,15 @@ func (f Folded) Source(a, b int) string {
 	case a >= b:
 		return ""
 	}
-	return f.src[f.start[a]:f.end[b-1]]
+	end := f.end[b-1]
+	for end < len(f.src) {
+		r, w := utf8.DecodeRuneInString(f.src[end:])
+		if r == utf8.RuneError && w == 1 || Fold(f.src[end:end+w]) != "" {
+			break
+		}
+		end += w
+	}
+	return f.src[f.start[a]:end]
 }
 
 // Text puts s into the form content and keywords are compared in: Fold if
diff --git a/internal/norm/norm_test.go b/internal/norm/norm_test.go
index a165b32..dfae915 100644
--- a/internal/norm/norm_test.go
+++ b/internal/norm/norm_test.go
@@ -75,8 +75,9 @@ func TestFoldASCIINoAlloc(t *testing.T) {
 }
 
 // TestFoldMappedSource: a part of the folded text leads back to the original
-// characters it came from, widened to whole characters - so a capture can be
-// written with its diacritics.
+// characters it came from, widened to whole characters and to the combining
+// marks that follow them (a decomposed name, plan 11 review L5) - so a
+// capture can be written with its diacritics.
 func TestFoldMappedSource(t *testing.T) {
 	cases := []struct {
 		in, folded string
@@ -86,8 +87,10 @@ func TestFoldMappedSource(t *testing.T) {
 		{"Łódź-faktura.pdf", "Lodz-faktura.pdf", 0, 4, "Łódź"},
 		{"Łódź-faktura.pdf", "Lodz-faktura.pdf", 5, 12, "faktura"},
 		{"straße", "strasse", 4, 5, "ß"},
-		{"áb", "ab", 0, 1, "a"},
-		{"áb", "ab", 1, 2, "b"},
+		{"a\u0301b", "ab", 0, 1, "a\u0301"},
+		{"e\u0301-x.pdf", "e-x.pdf", 0, 1, "e\u0301"},
+		{"\u0141o\u0301dz\u0301-f.pdf", "Lodz-f.pdf", 0, 4, "\u0141o\u0301dz\u0301"},
+		{"a\u0301b", "ab", 1, 2, "b"},
 		{"\xff\xfeé", "�e", 0, 3, "\xff\xfe"},
 		{"plain.txt", "plain.txt", 0, 5, "plain"},
 		{"Łódź", "Lodz", 2, 2, ""},