captures keep the combining marks of a decomposed name; test sources escape invisible characters

1 files changed, 12 insertions, 2 deletions

diff --git a/internal/norm/norm.go b/internal/norm/norm.go
index bff2127..1c2f61d 100644
--- a/internal/norm/norm.go
+++ b/internal/norm/norm.go
@@ -123,7 +123,9 @@ func FoldMapped(s string) Folded {
 }
 
 // Source returns the original text that Text[a:b] was folded from, widened
-// to whole characters; without a map, Text[a:b] itself.
+// to whole characters and to the characters right after them that fold to
+// nothing - the combining marks of a decomposed name, which belong to the
+// letter before them (plan 11 review L5); without a map, Text[a:b] itself.
 func (f Folded) Source(a, b int) string {
 	switch {
 	case f.same || f.start == nil:
@@ -131,7 +133,15 @@ func (f Folded) Source(a, b int) string {
 	case a >= b:
 		return ""
 	}
-	return f.src[f.start[a]:f.end[b-1]]
+	end := f.end[b-1]
+	for end < len(f.src) {
+		r, w := utf8.DecodeRuneInString(f.src[end:])
+		if r == utf8.RuneError && w == 1 || Fold(f.src[end:end+w]) != "" {
+			break
+		}
+		end += w
+	}
+	return f.src[f.start[a]:end]
 }
 
 // Text puts s into the form content and keywords are compared in: Fold if