diff options
Diffstat (limited to 'internal/norm')
| -rw-r--r-- | internal/norm/fuzz_test.go | 2 | ||||
| -rw-r--r-- | internal/norm/norm.go | 14 | ||||
| -rw-r--r-- | internal/norm/norm_test.go | 11 |
3 files changed, 20 insertions, 7 deletions
diff --git a/internal/norm/fuzz_test.go b/internal/norm/fuzz_test.go index 27deadc..2dc864e 100644 --- a/internal/norm/fuzz_test.go +++ b/internal/norm/fuzz_test.go @@ -36,7 +36,7 @@ func FuzzTextIdempotent(f *testing.F) { // FuzzFoldMapped: FoldMapped's text is always Fold's, and any part of it // leads back to original text that folds to something holding that part. func FuzzFoldMapped(f *testing.F) { - for _, s := range []string{"Łódź-faktura.pdf", "straße", "áb", "\xff\xfeé", "́a", "Æsir"} { + for _, s := range []string{"Łódź-faktura.pdf", "straße", "a\u0301b", "\xff\xfeé", "\u0301a", "Æsir"} { f.Add(s, 0, 2) } f.Fuzz(func(t *testing.T, s string, a, b int) { diff --git a/internal/norm/norm.go b/internal/norm/norm.go index bff2127..1c2f61d 100644 --- a/internal/norm/norm.go +++ b/internal/norm/norm.go @@ -123,7 +123,9 @@ func FoldMapped(s string) Folded { } // Source returns the original text that Text[a:b] was folded from, widened -// to whole characters; without a map, Text[a:b] itself. +// to whole characters and to the characters right after them that fold to +// nothing - the combining marks of a decomposed name, which belong to the +// letter before them (plan 11 review L5); without a map, Text[a:b] itself. func (f Folded) Source(a, b int) string { switch { case f.same || f.start == nil: @@ -131,7 +133,15 @@ func (f Folded) Source(a, b int) string { case a >= b: return "" } - return f.src[f.start[a]:f.end[b-1]] + end := f.end[b-1] + for end < len(f.src) { + r, w := utf8.DecodeRuneInString(f.src[end:]) + if r == utf8.RuneError && w == 1 || Fold(f.src[end:end+w]) != "" { + break + } + end += w + } + return f.src[f.start[a]:end] } // Text puts s into the form content and keywords are compared in: Fold if diff --git a/internal/norm/norm_test.go b/internal/norm/norm_test.go index a165b32..dfae915 100644 --- a/internal/norm/norm_test.go +++ b/internal/norm/norm_test.go @@ -75,8 +75,9 @@ func TestFoldASCIINoAlloc(t *testing.T) { } // TestFoldMappedSource: a part of the folded text leads back to the original -// characters it came from, widened to whole characters - so a capture can be -// written with its diacritics. +// characters it came from, widened to whole characters and to the combining +// marks that follow them (a decomposed name, plan 11 review L5) - so a +// capture can be written with its diacritics. func TestFoldMappedSource(t *testing.T) { cases := []struct { in, folded string @@ -86,8 +87,10 @@ func TestFoldMappedSource(t *testing.T) { {"Łódź-faktura.pdf", "Lodz-faktura.pdf", 0, 4, "Łódź"}, {"Łódź-faktura.pdf", "Lodz-faktura.pdf", 5, 12, "faktura"}, {"straße", "strasse", 4, 5, "ß"}, - {"áb", "ab", 0, 1, "a"}, - {"áb", "ab", 1, 2, "b"}, + {"a\u0301b", "ab", 0, 1, "a\u0301"}, + {"e\u0301-x.pdf", "e-x.pdf", 0, 1, "e\u0301"}, + {"\u0141o\u0301dz\u0301-f.pdf", "Lodz-f.pdf", 0, 4, "\u0141o\u0301dz\u0301"}, + {"a\u0301b", "ab", 1, 2, "b"}, {"\xff\xfeé", "�e", 0, 3, "\xff\xfe"}, {"plain.txt", "plain.txt", 0, 5, "plain"}, {"Łódź", "Lodz", 2, 2, ""}, |
