diff options
| author | Lukasz Kasprzak <lukas@labunix.xyz> | 2026-09-14 23:34:51 +0200 |
|---|---|---|
| committer | Lukasz Kasprzak <lukas@labunix.xyz> | 2026-09-14 23:34:51 +0200 |
| commit | 3990586a85e91923c0a3ae1a1f0f61420db555ac (patch) | |
| tree | 041692e7961cb66333b2c7b47d32a25487c6522d | |
| parent | 01b0cfe2055e23cf03870d9a9f60648037b4005e (diff) | |
| download | krino-3990586a85e91923c0a3ae1a1f0f61420db555ac.tar.gz krino-3990586a85e91923c0a3ae1a1f0f61420db555ac.zip | |
captures keep the name's diacritics
| -rw-r--r-- | CHANGELOG.md | 9 | ||||
| -rw-r--r-- | Makefile | 1 | ||||
| -rw-r--r-- | docs/design.md | 8 | ||||
| -rw-r--r-- | internal/cond/eval.go | 30 | ||||
| -rw-r--r-- | internal/cond/eval_test.go | 17 | ||||
| -rw-r--r-- | internal/engine/exclude_test.go | 21 | ||||
| -rw-r--r-- | internal/norm/fuzz_test.go | 36 | ||||
| -rw-r--r-- | internal/norm/norm.go | 64 | ||||
| -rw-r--r-- | internal/norm/norm_test.go | 30 | ||||
| -rw-r--r-- | internal/norm/testdata/fuzz/FuzzFoldMapped/ef66462f0d689aef | 4 | ||||
| -rw-r--r-- | man/krino.conf.5 | 26 |
11 files changed, 216 insertions, 30 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index beedcdc..78dbcd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,15 @@ on (a self-extracting installer) has no content, like an image: it no longer matches a content test, is not set aside by a content exclude, and raises no warning. +- `{1}` … `{9}` keep the name's diacritics: a folded `name` test still + matches `Lodz`, but `Łódź-faktura.pdf` now files under `Łódź`, not `Lodz`. +- A tool's error message is shown even when it writes more than the + capture limit at once. +- A damaged lock file naming a pid beyond 32 bits no longer counts as held + by whatever process the kernel reads it as. +- Tests pass on OpenBSD 7.9 and FreeBSD 15.0 (test binaries run on the + hosts; the fake extraction tools no longer rely on GNU `head -c` or a + `printf` builtin). ## 0.0.7 — 2026-09-14 @@ -92,6 +92,7 @@ FUZZ_TARGETS = \ internal/config:FuzzParseSize \ internal/config:FuzzParseDuration \ internal/norm:FuzzTextIdempotent \ + internal/norm:FuzzFoldMapped \ internal/plan:FuzzExpand \ internal/ignore:FuzzMatch \ internal/kwcache:FuzzLoad \ diff --git a/docs/design.md b/docs/design.md index 4fa5b4a..35b0de0 100644 --- a/docs/design.md +++ b/docs/design.md @@ -435,10 +435,10 @@ while evaluating this rule. Tests of equal cost keep their written order, so every such `name` test in it has at least N groups, and a rule that uses `{N}` with no `name` test at all. -Known limitation: a `name` test matches the folded name when `fold` is on -(the default), and its captures are taken from that same text, so `{1}` -loses diacritics: `(name "^(.+)-faktura")` on `Łódź-faktura.pdf` makes `{1}` -`Lodz`. Set `(fold no)` in the rule to keep them. +With `fold` on, a `name` test matches the folded name, but its captures are +the original name's characters: `(name "^(.+)-faktura")` on +`Łódź-faktura.pdf` makes `{1}` `Łódź`, not `Lodz`. A capture that ends inside +a letter folding to two (`ß` to `ss`) takes the whole letter. ### 7.4 Conflicts diff --git a/internal/cond/eval.go b/internal/cond/eval.go index b0ee828..f241504 100644 --- a/internal/cond/eval.go +++ b/internal/cond/eval.go @@ -164,17 +164,33 @@ func (c *Cond) evalLeaf(n *node, f Facts) (ok bool, reason, warn string, caps [] subj = f.Rel() word = "path" } - subj = norm.Name(subj, c.opt.Fold) + if n.kind == kPath { + subj = norm.Name(subj, c.opt.Fold) + for _, p := range n.patterns { + if p.re.MatchString(subj) { + return true, word + ` "` + p.src + `"`, "", nil + } + } + return false, "", "", nil + } + // A name test matches the folded name, but its captures are read + // back from the original, so {1} keeps the name's diacritics. + folded := norm.Folded{Text: subj} + if c.opt.Fold { + folded = norm.FoldMapped(subj) + } for _, p := range n.patterns { - m := p.re.FindStringSubmatch(subj) - if m == nil { + loc := p.re.FindStringSubmatchIndex(folded.Text) + if loc == nil { continue } - reason = word + ` "` + p.src + `"` - if n.kind == kName { - return true, reason, "", m + caps := make([]string, len(loc)/2) + for g := range caps { + if loc[2*g] >= 0 { + caps[g] = folded.Source(loc[2*g], loc[2*g+1]) + } } - return true, reason, "", nil + return true, word + ` "` + p.src + `"`, "", caps } return false, "", "", nil diff --git a/internal/cond/eval_test.go b/internal/cond/eval_test.go index 88612ac..2e8bcfd 100644 --- a/internal/cond/eval_test.go +++ b/internal/cond/eval_test.go @@ -240,3 +240,20 @@ func TestEvalReportsUnreadableContent(t *testing.T) { t.Errorf("not reached: Match %v Unreadable %v; want true, false", r.Match, r.Unreadable) } } + +// TestCapturesKeepDiacritics: a name test matches the folded name, but its +// captures are the original name's characters, so {1} keeps "Łódź". +func TestCapturesKeepDiacritics(t *testing.T) { + f := &fake{name: "Łódź-Faktura.pdf"} + r := eval(t, `(name "^(.+)-faktura")`, Options{IgnoreCase: true, Fold: true}, f) + if !r.Match { + t.Fatal("no match") + } + if want := []string{"Łódź-Faktura", "Łódź"}; !reflect.DeepEqual(r.Captures, want) { + t.Errorf("captures = %q, want %q", r.Captures, want) + } + r = eval(t, `(name "^(lodz)?(x)?-")`, Options{IgnoreCase: true, Fold: true}, f) + if want := []string{"Łódź-", "Łódź", ""}; !reflect.DeepEqual(r.Captures, want) { + t.Errorf("captures with a group that did not take part = %q, want %q", r.Captures, want) + } +} diff --git a/internal/engine/exclude_test.go b/internal/engine/exclude_test.go index e5231f8..e183268 100644 --- a/internal/engine/exclude_test.go +++ b/internal/engine/exclude_test.go @@ -330,3 +330,24 @@ func TestTextTurningBinaryIsNoText(t *testing.T) { } } } + +// TestPlannedDestinationKeepsDiacritics: a folded name test's capture goes +// into the destination as the file name wrote it. +func TestPlannedDestinationKeepsDiacritics(t *testing.T) { + h, _ := excludeTree(t, map[string]string{"Łódź-faktura.pdf": "x"}) + main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": ` +(path "~/dl") +(rule "city" (when (name "^(.+)-faktura")) (move "Out/{1}")) +`}) + e, errs := Load(main) + if len(errs) > 0 { + t.Fatal(errs) + } + dp, err := e.Plan(context.Background(), e.Dirs[0], plan.NewClaims()) + if err != nil { + t.Fatal(err) + } + if len(dp.Chains) != 1 || !strings.HasSuffix(dp.Chains[0].Steps[0].Dst, "/Out/Łódź/Łódź-faktura.pdf") { + t.Fatalf("chains = %+v; want the move into Out/Łódź", dp.Chains) + } +} diff --git a/internal/norm/fuzz_test.go b/internal/norm/fuzz_test.go index fde62a0..27deadc 100644 --- a/internal/norm/fuzz_test.go +++ b/internal/norm/fuzz_test.go @@ -2,7 +2,11 @@ package norm -import "testing" +import ( + "strings" + "testing" + "unicode/utf8" +) // FuzzTextIdempotent: normalising text twice changes nothing more than // normalising it once, under every case and fold setting, and so for names. @@ -28,3 +32,33 @@ func FuzzTextIdempotent(f *testing.F) { } }) } + +// FuzzFoldMapped: FoldMapped's text is always Fold's, and any part of it +// leads back to original text that folds to something holding that part. +func FuzzFoldMapped(f *testing.F) { + for _, s := range []string{"Łódź-faktura.pdf", "straße", "áb", "\xff\xfeé", "́a", "Æsir"} { + f.Add(s, 0, 2) + } + f.Fuzz(func(t *testing.T, s string, a, b int) { + m := FoldMapped(s) + if m.Text != Fold(s) { + t.Fatalf("FoldMapped(%q).Text = %q, Fold = %q", s, m.Text, Fold(s)) + } + n := len(m.Text) + 1 + a, b = (a%n+n)%n, (b%n+n)%n + if a > b { + a, b = b, a + } + // A regex match starts and ends on whole characters. + for a < len(m.Text) && !utf8.RuneStart(m.Text[a]) { + a-- + } + for b < len(m.Text) && !utf8.RuneStart(m.Text[b]) { + b-- + } + src := m.Source(a, b) + if !strings.Contains(Fold(src), m.Text[a:b]) { + t.Fatalf("FoldMapped(%q).Source(%d, %d) = %q, which folds to %q, not holding %q", s, a, b, src, Fold(src), m.Text[a:b]) + } + }) +} diff --git a/internal/norm/norm.go b/internal/norm/norm.go index 701f765..bff2127 100644 --- a/internal/norm/norm.go +++ b/internal/norm/norm.go @@ -8,6 +8,7 @@ import ( "fmt" "strings" "unicode" + "unicode/utf8" unorm "golang.org/x/text/unicode/norm" ) @@ -70,6 +71,69 @@ func Fold(s string) string { return b.String() } +// Folded is Fold's result with a way back to the text it was folded from. +type Folded struct { + Text string // Fold(src) + + src string + same bool // Text is src (ASCII) + start, end []int // per byte of Text, the bounds in src of the character it came from; nil when no map exists +} + +// FoldMapped folds s like Fold and keeps, for every byte of the result, the +// original character it came from, so a regex match on the folded text can +// be read back from s with its diacritics. The map is built by folding one +// character at a time (a run of invalid bytes counts as one, as Fold's +// U+FFFD does); in the rare case that differs from Fold's decomposition of +// the whole string (combining marks NFD reorders), there is no map and +// Source returns the folded text. +func FoldMapped(s string) Folded { + text := Fold(s) + if text == s { + return Folded{Text: text, src: s, same: true} + } + var b strings.Builder + start := make([]int, 0, len(text)) + end := make([]int, 0, len(text)) + for i := 0; i < len(s); { + j, piece := i, "" + if r, w := utf8.DecodeRuneInString(s[i:]); r == utf8.RuneError && w == 1 { + for j < len(s) { + if r, w := utf8.DecodeRuneInString(s[j:]); r != utf8.RuneError || w != 1 { + break + } + j++ + } + piece = "\ufffd" + } else { + j = i + w + piece = Fold(s[i:j]) + } + b.WriteString(piece) + for range len(piece) { + start = append(start, i) + end = append(end, j) + } + i = j + } + if b.String() != text { + return Folded{Text: text, src: s} + } + return Folded{Text: text, src: s, start: start, end: end} +} + +// Source returns the original text that Text[a:b] was folded from, widened +// to whole characters; without a map, Text[a:b] itself. +func (f Folded) Source(a, b int) string { + switch { + case f.same || f.start == nil: + return f.Text[a:b] + case a >= b: + return "" + } + return f.src[f.start[a]:f.end[b-1]] +} + // Text puts s into the form content and keywords are compared in: Fold if // fold, strings.ToLower if ignoreCase, then every run of Unicode white // space becomes one ASCII space and both ends are trimmed. diff --git a/internal/norm/norm_test.go b/internal/norm/norm_test.go index 885c6f7..a165b32 100644 --- a/internal/norm/norm_test.go +++ b/internal/norm/norm_test.go @@ -73,3 +73,33 @@ func TestFoldASCIINoAlloc(t *testing.T) { t.Errorf("Fold allocates %v times on ASCII input", n) } } + +// TestFoldMappedSource: a part of the folded text leads back to the original +// characters it came from, widened to whole characters - so a capture can be +// written with its diacritics. +func TestFoldMappedSource(t *testing.T) { + cases := []struct { + in, folded string + a, b int + want string + }{ + {"Łódź-faktura.pdf", "Lodz-faktura.pdf", 0, 4, "Łódź"}, + {"Łódź-faktura.pdf", "Lodz-faktura.pdf", 5, 12, "faktura"}, + {"straße", "strasse", 4, 5, "ß"}, + {"áb", "ab", 0, 1, "a"}, + {"áb", "ab", 1, 2, "b"}, + {"\xff\xfeé", "�e", 0, 3, "\xff\xfe"}, + {"plain.txt", "plain.txt", 0, 5, "plain"}, + {"Łódź", "Lodz", 2, 2, ""}, + } + for _, c := range cases { + f := FoldMapped(c.in) + if f.Text != c.folded || f.Text != Fold(c.in) { + t.Errorf("FoldMapped(%q).Text = %q, want %q (Fold)", c.in, f.Text, c.folded) + continue + } + if got := f.Source(c.a, c.b); got != c.want { + t.Errorf("FoldMapped(%q).Source(%d, %d) = %q, want %q", c.in, c.a, c.b, got, c.want) + } + } +} diff --git a/internal/norm/testdata/fuzz/FuzzFoldMapped/ef66462f0d689aef b/internal/norm/testdata/fuzz/FuzzFoldMapped/ef66462f0d689aef new file mode 100644 index 0000000..f77399e --- /dev/null +++ b/internal/norm/testdata/fuzz/FuzzFoldMapped/ef66462f0d689aef @@ -0,0 +1,4 @@ +go test fuzz v1 +string("0\u0381") +int(0) +int(2) diff --git a/man/krino.conf.5 b/man/krino.conf.5 index 28d7034..26da541 100644 --- a/man/krino.conf.5 +++ b/man/krino.conf.5 @@ -666,10 +666,15 @@ With .Ic fold on, the default, a .Ic name -test matches the folded name and its captures come from that text, so +test matches the folded name, but its captures are the original name's +characters: +.Ql (name \(dq^(.+)-faktura\(dq) +on +.Pa \[/L]\['o]d\[u017A]-faktura.pdf +makes .Ic {1} -loses diacritics -.Pq see Sx KNOWN LIMITATIONS . +the name's own +.Ql \[/L]\['o]d\[u017A] . .Pp .Ar FMT is a strftime subset: @@ -801,21 +806,6 @@ Duplicate conditions with different scopes do not share an original, so two such rules can between them move every copy of a group aside; nothing is deleted. .Sh KNOWN LIMITATIONS -Captures -.Ic {1} No ... Ic {9} -come from the folded name when -.Ic fold -is on: -.Ql (name \(dq^(.+)-faktura\(dq) -on -.Pa \[/L]\['o]d\[u017A]-faktura.pdf -makes -.Ic {1} -.Ql Lodz . -Set -.Ql (fold no) -in the rule to keep diacritics. -.Pp A .Ar dest whose |
