krino: matching — scan, ignore, conditions, extraction, duplicates, explain, dry run

2 files changed, 169 insertions, 0 deletions

diff --git a/internal/norm/norm.go b/internal/norm/norm.go
new file mode 100644
index 0000000..e7933ec
--- /dev/null
+++ b/internal/norm/norm.go
@@ -0,0 +1,96 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+// Package norm puts text into the form krino compares it in: optionally
+// without diacritics, optionally lower case, with white space collapsed.
+package norm
+
+import (
+	"strings"
+	"unicode"
+
+	unorm "golang.org/x/text/unicode/norm"
+)
+
+// special holds the letters that do not decompose under Unicode NFD, so
+// Fold maps them explicitly.
+var special = map[rune]string{
+	'ł': "l", 'Ł': "L",
+	'ø': "o", 'Ø': "O",
+	'đ': "d", 'Đ': "D",
+	'ħ': "h", 'Ħ': "H",
+	'ß': "ss",
+	'æ': "ae", 'Æ': "AE",
+	'œ': "oe", 'Œ': "OE",
+	'ı': "i",
+}
+
+// Fold strips diacritics: Unicode NFD, drop combining marks (category Mn),
+// then map the letters that do not decompose. ASCII input is returned
+// unchanged without allocating.
+func Fold(s string) string {
+	ascii := true
+	for i := 0; i < len(s); i++ {
+		if s[i] >= 0x80 {
+			ascii = false
+			break
+		}
+	}
+	if ascii {
+		return s
+	}
+
+	var b strings.Builder
+	b.Grow(len(s))
+	for _, r := range unorm.NFD.String(s) {
+		if unicode.Is(unicode.Mn, r) {
+			continue
+		}
+		if rep, ok := special[r]; ok {
+			b.WriteString(rep)
+			continue
+		}
+		b.WriteRune(r)
+	}
+	return b.String()
+}
+
+// Text puts s into the form content and keywords are compared in: Fold if
+// fold, strings.ToLower if ignoreCase, then every run of Unicode white
+// space becomes one ASCII space and both ends are trimmed.
+func Text(s string, ignoreCase, fold bool) string {
+	if fold {
+		s = Fold(s)
+	}
+	if ignoreCase {
+		s = strings.ToLower(s)
+	}
+
+	var b strings.Builder
+	b.Grow(len(s))
+	inSpace := false
+	started := false
+	for _, r := range s {
+		if unicode.IsSpace(r) {
+			if started {
+				inSpace = true
+			}
+			continue
+		}
+		if inSpace {
+			b.WriteByte(' ')
+			inSpace = false
+		}
+		b.WriteRune(r)
+		started = true
+	}
+	return b.String()
+}
+
+// Name puts a file name into the form it is matched against: Fold(s) if
+// fold, else s. Case is handled by the regex flag, not here.
+func Name(s string, fold bool) string {
+	if fold {
+		return Fold(s)
+	}
+	return s
+}
diff --git a/internal/norm/norm_test.go b/internal/norm/norm_test.go
new file mode 100644
index 0000000..2fc35b6
--- /dev/null
+++ b/internal/norm/norm_test.go
@@ -0,0 +1,73 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package norm
+
+import "testing"
+
+func TestFold(t *testing.T) {
+	tests := map[string]string{
+		"":                "",
+		"plain ASCII 123": "plain ASCII 123",
+		"spółka z ograniczoną odpowiedzialnością": "spolka z ograniczona odpowiedzialnoscia",
+		"Łódź":              "Lodz",
+		"ZAŻÓŁĆ GĘŚLĄ JAŹŃ": "ZAZOLC GESLA JAZN",
+		"Straße":            "Strasse",
+		"Øresund":           "Oresund",
+		"Ærø":               "AEro",
+		"œuvre":             "oeuvre",
+		"déjà vu":           "deja vu",
+		"Đakovo":            "Dakovo",
+		"ħelp":              "help",
+	}
+	for in, want := range tests {
+		if got := Fold(in); got != want {
+			t.Errorf("Fold(%q) = %q, want %q", in, got, want)
+		}
+	}
+}
+
+func TestText(t *testing.T) {
+	tests := []struct {
+		in               string
+		ignoreCase, fold bool
+		want             string
+	}{
+		{"  Hello\n\tWORLD  ", true, false, "hello world"},
+		{"A  B\r\nC", false, false, "A B C"},
+		{"ZAŻÓŁĆ   gęślą", true, true, "zazolc gesla"},
+		{"Spółka", false, true, "Spolka"},
+		{"Faktura\u00A0VAT", true, false, "faktura vat"},
+		{"", true, true, ""},
+		{"   ", true, true, ""},
+	}
+	for _, tt := range tests {
+		if got := Text(tt.in, tt.ignoreCase, tt.fold); got != tt.want {
+			t.Errorf("Text(%q, %v, %v) = %q, want %q", tt.in, tt.ignoreCase, tt.fold, got, tt.want)
+		}
+	}
+}
+
+func TestTextIdempotent(t *testing.T) {
+	for _, s := range []string{"  Łódź\n\nMIASTO ", "a\tb", "Ærø  Œuvre"} {
+		once := Text(s, true, true)
+		if twice := Text(once, true, true); twice != once {
+			t.Errorf("Text not idempotent on %q: %q then %q", s, once, twice)
+		}
+	}
+}
+
+func TestName(t *testing.T) {
+	if got := Name("Spółka.PDF", true); got != "Spolka.PDF" {
+		t.Errorf("Name fold = %q", got)
+	}
+	if got := Name("Spółka.PDF", false); got != "Spółka.PDF" {
+		t.Errorf("Name no fold = %q", got)
+	}
+}
+
+func TestFoldASCIINoAlloc(t *testing.T) {
+	s := "already plain ascii text"
+	if n := testing.AllocsPerRun(100, func() { _ = Fold(s) }); n != 0 {
+		t.Errorf("Fold allocates %v times on ASCII input", n)
+	}
+}