// SPDX-License-Identifier: GPL-3.0-or-later // Package norm puts text into the form krino compares it in: optionally // without diacritics, optionally lower case, with white space collapsed. package norm import ( "fmt" "strings" "unicode" "unicode/utf8" unorm "golang.org/x/text/unicode/norm" ) // Version identifies what Text and Name produce. Bump it whenever a change // could make any string normalise differently: keyword answers cached under // another version are discarded (spec §6.1). const Version = 1 // Fingerprint names everything Text and Name's output depends on: Version, // and the Unicode tables of the standard library and of x/text's // normalisation, which a Go or x/text upgrade can change. func Fingerprint() string { return fmt.Sprintf("norm%d unicode%s nfd%s", Version, unicode.Version, unorm.Version) } // special holds the letters that do not decompose under Unicode NFD, so // Fold maps them explicitly. var special = map[rune]string{ 'ł': "l", 'Ł': "L", 'ø': "o", 'Ø': "O", 'đ': "d", 'Đ': "D", 'ħ': "h", 'Ħ': "H", 'ß': "ss", 'ẞ': "SS", 'æ': "ae", 'Æ': "AE", 'œ': "oe", 'Œ': "OE", 'ı': "i", } // Fold strips diacritics: Unicode NFD, drop combining marks (category Mn), // then map the letters that do not decompose. Invalid UTF-8 becomes U+FFFD // first: left in, an incomplete sequence can keep NFD from decomposing the // letter after it, and folding the result again would change it. ASCII // input is returned unchanged without allocating. func Fold(s string) string { ascii := true for i := 0; i < len(s); i++ { if s[i] >= 0x80 { ascii = false break } } if ascii { return s } s = strings.ToValidUTF8(s, "\ufffd") var b strings.Builder b.Grow(len(s)) for _, r := range unorm.NFD.String(s) { if unicode.Is(unicode.Mn, r) { continue } if rep, ok := special[r]; ok { b.WriteString(rep) continue } b.WriteRune(r) } return b.String() } // Folded is Fold's result with a way back to the text it was folded from. type Folded struct { Text string // Fold(src) src string same bool // Text is src (ASCII) start, end []int // per byte of Text, the bounds in src of the character it came from; nil when no map exists } // FoldMapped folds s like Fold and keeps, for every byte of the result, the // original character it came from, so a regex match on the folded text can // be read back from s with its diacritics. The map is built by folding one // character at a time (a run of invalid bytes counts as one, as Fold's // U+FFFD does); in the rare case that differs from Fold's decomposition of // the whole string (combining marks NFD reorders), there is no map and // Source returns the folded text. func FoldMapped(s string) Folded { text := Fold(s) if text == s { return Folded{Text: text, src: s, same: true} } var b strings.Builder start := make([]int, 0, len(text)) end := make([]int, 0, len(text)) for i := 0; i < len(s); { j, piece := i, "" if r, w := utf8.DecodeRuneInString(s[i:]); r == utf8.RuneError && w == 1 { for j < len(s) { if r, w := utf8.DecodeRuneInString(s[j:]); r != utf8.RuneError || w != 1 { break } j++ } piece = "\ufffd" } else { j = i + w piece = Fold(s[i:j]) } b.WriteString(piece) for range len(piece) { start = append(start, i) end = append(end, j) } i = j } if b.String() != text { return Folded{Text: text, src: s} } return Folded{Text: text, src: s, start: start, end: end} } // Source returns the original text that Text[a:b] was folded from, widened // to whole characters and to the characters right after them that fold to // nothing - the combining marks of a decomposed name, which belong to the // letter before them; without a map, Text[a:b] itself. func (f Folded) Source(a, b int) string { switch { case f.same || f.start == nil: return f.Text[a:b] case a >= b: return "" } end := f.end[b-1] for end < len(f.src) { r, w := utf8.DecodeRuneInString(f.src[end:]) if r == utf8.RuneError && w == 1 || Fold(f.src[end:end+w]) != "" { break } end += w } return f.src[f.start[a]:end] } // Text puts s into the form content and keywords are compared in: Fold if // fold, strings.ToLower if ignoreCase, then every run of Unicode white // space becomes one ASCII space and both ends are trimmed. func Text(s string, ignoreCase, fold bool) string { if fold { s = Fold(s) } if ignoreCase { s = strings.ToLower(s) } var b strings.Builder b.Grow(len(s)) inSpace := false started := false for _, r := range s { if unicode.IsSpace(r) { if started { inSpace = true } continue } if inSpace { b.WriteByte(' ') inSpace = false } b.WriteRune(r) started = true } return b.String() } // Name puts a file name into the form it is matched against: Fold(s) if // fold, else s. Case is handled by the regex flag, not here. func Name(s string, fold bool) string { if fold { return Fold(s) } return s }