1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
|
// SPDX-License-Identifier: GPL-3.0-or-later
// Package norm puts text into the form krino compares it in: optionally
// without diacritics, optionally lower case, with white space collapsed.
package norm
import (
"fmt"
"strings"
"unicode"
"unicode/utf8"
unorm "golang.org/x/text/unicode/norm"
)
// Version identifies what Text and Name produce. Bump it whenever a change
// could make any string normalise differently: keyword answers cached under
// another version are discarded (spec §6.1).
const Version = 1
// Fingerprint names everything Text and Name's output depends on: Version,
// and the Unicode tables of the standard library and of x/text's
// normalisation, which a Go or x/text upgrade can change (review cache F4).
func Fingerprint() string {
return fmt.Sprintf("norm%d unicode%s nfd%s", Version, unicode.Version, unorm.Version)
}
// special holds the letters that do not decompose under Unicode NFD, so
// Fold maps them explicitly.
var special = map[rune]string{
'ł': "l", 'Ł': "L",
'ø': "o", 'Ø': "O",
'đ': "d", 'Đ': "D",
'ħ': "h", 'Ħ': "H",
'ß': "ss", 'ẞ': "SS",
'æ': "ae", 'Æ': "AE",
'œ': "oe", 'Œ': "OE",
'ı': "i",
}
// Fold strips diacritics: Unicode NFD, drop combining marks (category Mn),
// then map the letters that do not decompose. Invalid UTF-8 becomes U+FFFD
// first: left in, an incomplete sequence can keep NFD from decomposing the
// letter after it, and folding the result again would change it. ASCII
// input is returned unchanged without allocating.
func Fold(s string) string {
ascii := true
for i := 0; i < len(s); i++ {
if s[i] >= 0x80 {
ascii = false
break
}
}
if ascii {
return s
}
s = strings.ToValidUTF8(s, "\ufffd")
var b strings.Builder
b.Grow(len(s))
for _, r := range unorm.NFD.String(s) {
if unicode.Is(unicode.Mn, r) {
continue
}
if rep, ok := special[r]; ok {
b.WriteString(rep)
continue
}
b.WriteRune(r)
}
return b.String()
}
// Folded is Fold's result with a way back to the text it was folded from.
type Folded struct {
Text string // Fold(src)
src string
same bool // Text is src (ASCII)
start, end []int // per byte of Text, the bounds in src of the character it came from; nil when no map exists
}
// FoldMapped folds s like Fold and keeps, for every byte of the result, the
// original character it came from, so a regex match on the folded text can
// be read back from s with its diacritics. The map is built by folding one
// character at a time (a run of invalid bytes counts as one, as Fold's
// U+FFFD does); in the rare case that differs from Fold's decomposition of
// the whole string (combining marks NFD reorders), there is no map and
// Source returns the folded text.
func FoldMapped(s string) Folded {
text := Fold(s)
if text == s {
return Folded{Text: text, src: s, same: true}
}
var b strings.Builder
start := make([]int, 0, len(text))
end := make([]int, 0, len(text))
for i := 0; i < len(s); {
j, piece := i, ""
if r, w := utf8.DecodeRuneInString(s[i:]); r == utf8.RuneError && w == 1 {
for j < len(s) {
if r, w := utf8.DecodeRuneInString(s[j:]); r != utf8.RuneError || w != 1 {
break
}
j++
}
piece = "\ufffd"
} else {
j = i + w
piece = Fold(s[i:j])
}
b.WriteString(piece)
for range len(piece) {
start = append(start, i)
end = append(end, j)
}
i = j
}
if b.String() != text {
return Folded{Text: text, src: s}
}
return Folded{Text: text, src: s, start: start, end: end}
}
// Source returns the original text that Text[a:b] was folded from, widened
// to whole characters and to the characters right after them that fold to
// nothing - the combining marks of a decomposed name, which belong to the
// letter before them (plan 11 review L5); without a map, Text[a:b] itself.
func (f Folded) Source(a, b int) string {
switch {
case f.same || f.start == nil:
return f.Text[a:b]
case a >= b:
return ""
}
end := f.end[b-1]
for end < len(f.src) {
r, w := utf8.DecodeRuneInString(f.src[end:])
if r == utf8.RuneError && w == 1 || Fold(f.src[end:end+w]) != "" {
break
}
end += w
}
return f.src[f.start[a]:end]
}
// Text puts s into the form content and keywords are compared in: Fold if
// fold, strings.ToLower if ignoreCase, then every run of Unicode white
// space becomes one ASCII space and both ends are trimmed.
func Text(s string, ignoreCase, fold bool) string {
if fold {
s = Fold(s)
}
if ignoreCase {
s = strings.ToLower(s)
}
var b strings.Builder
b.Grow(len(s))
inSpace := false
started := false
for _, r := range s {
if unicode.IsSpace(r) {
if started {
inSpace = true
}
continue
}
if inSpace {
b.WriteByte(' ')
inSpace = false
}
b.WriteRune(r)
started = true
}
return b.String()
}
// Name puts a file name into the form it is matched against: Fold(s) if
// fold, else s. Case is handled by the regex flag, not here.
func Name(s string, fold bool) string {
if fold {
return Fold(s)
}
return s
}
|
