aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLukasz Kasprzak <lukas@labunix.xyz>2026-09-14 23:34:51 +0200
committerLukasz Kasprzak <lukas@labunix.xyz>2026-09-14 23:34:51 +0200
commit3990586a85e91923c0a3ae1a1f0f61420db555ac (patch)
tree041692e7961cb66333b2c7b47d32a25487c6522d
parent01b0cfe2055e23cf03870d9a9f60648037b4005e (diff)
downloadkrino-3990586a85e91923c0a3ae1a1f0f61420db555ac.tar.gz
krino-3990586a85e91923c0a3ae1a1f0f61420db555ac.zip
captures keep the name's diacritics
-rw-r--r--CHANGELOG.md9
-rw-r--r--Makefile1
-rw-r--r--docs/design.md8
-rw-r--r--internal/cond/eval.go30
-rw-r--r--internal/cond/eval_test.go17
-rw-r--r--internal/engine/exclude_test.go21
-rw-r--r--internal/norm/fuzz_test.go36
-rw-r--r--internal/norm/norm.go64
-rw-r--r--internal/norm/norm_test.go30
-rw-r--r--internal/norm/testdata/fuzz/FuzzFoldMapped/ef66462f0d689aef4
-rw-r--r--man/krino.conf.526
11 files changed, 216 insertions, 30 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md
index beedcdc..78dbcd8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,15 @@
on (a self-extracting installer) has no content, like an image: it no
longer matches a content test, is not set aside by a content exclude, and
raises no warning.
+- `{1}` … `{9}` keep the name's diacritics: a folded `name` test still
+ matches `Lodz`, but `Łódź-faktura.pdf` now files under `Łódź`, not `Lodz`.
+- A tool's error message is shown even when it writes more than the
+ capture limit at once.
+- A damaged lock file naming a pid beyond 32 bits no longer counts as held
+ by whatever process the kernel reads it as.
+- Tests pass on OpenBSD 7.9 and FreeBSD 15.0 (test binaries run on the
+ hosts; the fake extraction tools no longer rely on GNU `head -c` or a
+ `printf` builtin).
## 0.0.7 — 2026-09-14
diff --git a/Makefile b/Makefile
index 5a1b6fb..1a713b7 100644
--- a/Makefile
+++ b/Makefile
@@ -92,6 +92,7 @@ FUZZ_TARGETS = \
internal/config:FuzzParseSize \
internal/config:FuzzParseDuration \
internal/norm:FuzzTextIdempotent \
+ internal/norm:FuzzFoldMapped \
internal/plan:FuzzExpand \
internal/ignore:FuzzMatch \
internal/kwcache:FuzzLoad \
diff --git a/docs/design.md b/docs/design.md
index 4fa5b4a..35b0de0 100644
--- a/docs/design.md
+++ b/docs/design.md
@@ -435,10 +435,10 @@ while evaluating this rule. Tests of equal cost keep their written order, so
every such `name` test in it has at least N groups, and a rule that uses
`{N}` with no `name` test at all.
-Known limitation: a `name` test matches the folded name when `fold` is on
-(the default), and its captures are taken from that same text, so `{1}`
-loses diacritics: `(name "^(.+)-faktura")` on `Łódź-faktura.pdf` makes `{1}`
-`Lodz`. Set `(fold no)` in the rule to keep them.
+With `fold` on, a `name` test matches the folded name, but its captures are
+the original name's characters: `(name "^(.+)-faktura")` on
+`Łódź-faktura.pdf` makes `{1}` `Łódź`, not `Lodz`. A capture that ends inside
+a letter folding to two (`ß` to `ss`) takes the whole letter.
### 7.4 Conflicts
diff --git a/internal/cond/eval.go b/internal/cond/eval.go
index b0ee828..f241504 100644
--- a/internal/cond/eval.go
+++ b/internal/cond/eval.go
@@ -164,17 +164,33 @@ func (c *Cond) evalLeaf(n *node, f Facts) (ok bool, reason, warn string, caps []
subj = f.Rel()
word = "path"
}
- subj = norm.Name(subj, c.opt.Fold)
+ if n.kind == kPath {
+ subj = norm.Name(subj, c.opt.Fold)
+ for _, p := range n.patterns {
+ if p.re.MatchString(subj) {
+ return true, word + ` "` + p.src + `"`, "", nil
+ }
+ }
+ return false, "", "", nil
+ }
+ // A name test matches the folded name, but its captures are read
+ // back from the original, so {1} keeps the name's diacritics.
+ folded := norm.Folded{Text: subj}
+ if c.opt.Fold {
+ folded = norm.FoldMapped(subj)
+ }
for _, p := range n.patterns {
- m := p.re.FindStringSubmatch(subj)
- if m == nil {
+ loc := p.re.FindStringSubmatchIndex(folded.Text)
+ if loc == nil {
continue
}
- reason = word + ` "` + p.src + `"`
- if n.kind == kName {
- return true, reason, "", m
+ caps := make([]string, len(loc)/2)
+ for g := range caps {
+ if loc[2*g] >= 0 {
+ caps[g] = folded.Source(loc[2*g], loc[2*g+1])
+ }
}
- return true, reason, "", nil
+ return true, word + ` "` + p.src + `"`, "", caps
}
return false, "", "", nil
diff --git a/internal/cond/eval_test.go b/internal/cond/eval_test.go
index 88612ac..2e8bcfd 100644
--- a/internal/cond/eval_test.go
+++ b/internal/cond/eval_test.go
@@ -240,3 +240,20 @@ func TestEvalReportsUnreadableContent(t *testing.T) {
t.Errorf("not reached: Match %v Unreadable %v; want true, false", r.Match, r.Unreadable)
}
}
+
+// TestCapturesKeepDiacritics: a name test matches the folded name, but its
+// captures are the original name's characters, so {1} keeps "Łódź".
+func TestCapturesKeepDiacritics(t *testing.T) {
+ f := &fake{name: "Łódź-Faktura.pdf"}
+ r := eval(t, `(name "^(.+)-faktura")`, Options{IgnoreCase: true, Fold: true}, f)
+ if !r.Match {
+ t.Fatal("no match")
+ }
+ if want := []string{"Łódź-Faktura", "Łódź"}; !reflect.DeepEqual(r.Captures, want) {
+ t.Errorf("captures = %q, want %q", r.Captures, want)
+ }
+ r = eval(t, `(name "^(lodz)?(x)?-")`, Options{IgnoreCase: true, Fold: true}, f)
+ if want := []string{"Łódź-", "Łódź", ""}; !reflect.DeepEqual(r.Captures, want) {
+ t.Errorf("captures with a group that did not take part = %q, want %q", r.Captures, want)
+ }
+}
diff --git a/internal/engine/exclude_test.go b/internal/engine/exclude_test.go
index e5231f8..e183268 100644
--- a/internal/engine/exclude_test.go
+++ b/internal/engine/exclude_test.go
@@ -330,3 +330,24 @@ func TestTextTurningBinaryIsNoText(t *testing.T) {
}
}
}
+
+// TestPlannedDestinationKeepsDiacritics: a folded name test's capture goes
+// into the destination as the file name wrote it.
+func TestPlannedDestinationKeepsDiacritics(t *testing.T) {
+ h, _ := excludeTree(t, map[string]string{"Łódź-faktura.pdf": "x"})
+ main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": `
+(path "~/dl")
+(rule "city" (when (name "^(.+)-faktura")) (move "Out/{1}"))
+`})
+ e, errs := Load(main)
+ if len(errs) > 0 {
+ t.Fatal(errs)
+ }
+ dp, err := e.Plan(context.Background(), e.Dirs[0], plan.NewClaims())
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(dp.Chains) != 1 || !strings.HasSuffix(dp.Chains[0].Steps[0].Dst, "/Out/Łódź/Łódź-faktura.pdf") {
+ t.Fatalf("chains = %+v; want the move into Out/Łódź", dp.Chains)
+ }
+}
diff --git a/internal/norm/fuzz_test.go b/internal/norm/fuzz_test.go
index fde62a0..27deadc 100644
--- a/internal/norm/fuzz_test.go
+++ b/internal/norm/fuzz_test.go
@@ -2,7 +2,11 @@
package norm
-import "testing"
+import (
+ "strings"
+ "testing"
+ "unicode/utf8"
+)
// FuzzTextIdempotent: normalising text twice changes nothing more than
// normalising it once, under every case and fold setting, and so for names.
@@ -28,3 +32,33 @@ func FuzzTextIdempotent(f *testing.F) {
}
})
}
+
+// FuzzFoldMapped: FoldMapped's text is always Fold's, and any part of it
+// leads back to original text that folds to something holding that part.
+func FuzzFoldMapped(f *testing.F) {
+ for _, s := range []string{"Łódź-faktura.pdf", "straße", "áb", "\xff\xfeé", "́a", "Æsir"} {
+ f.Add(s, 0, 2)
+ }
+ f.Fuzz(func(t *testing.T, s string, a, b int) {
+ m := FoldMapped(s)
+ if m.Text != Fold(s) {
+ t.Fatalf("FoldMapped(%q).Text = %q, Fold = %q", s, m.Text, Fold(s))
+ }
+ n := len(m.Text) + 1
+ a, b = (a%n+n)%n, (b%n+n)%n
+ if a > b {
+ a, b = b, a
+ }
+ // A regex match starts and ends on whole characters.
+ for a < len(m.Text) && !utf8.RuneStart(m.Text[a]) {
+ a--
+ }
+ for b < len(m.Text) && !utf8.RuneStart(m.Text[b]) {
+ b--
+ }
+ src := m.Source(a, b)
+ if !strings.Contains(Fold(src), m.Text[a:b]) {
+ t.Fatalf("FoldMapped(%q).Source(%d, %d) = %q, which folds to %q, not holding %q", s, a, b, src, Fold(src), m.Text[a:b])
+ }
+ })
+}
diff --git a/internal/norm/norm.go b/internal/norm/norm.go
index 701f765..bff2127 100644
--- a/internal/norm/norm.go
+++ b/internal/norm/norm.go
@@ -8,6 +8,7 @@ import (
"fmt"
"strings"
"unicode"
+ "unicode/utf8"
unorm "golang.org/x/text/unicode/norm"
)
@@ -70,6 +71,69 @@ func Fold(s string) string {
return b.String()
}
+// Folded is Fold's result with a way back to the text it was folded from.
+type Folded struct {
+ Text string // Fold(src)
+
+ src string
+ same bool // Text is src (ASCII)
+ start, end []int // per byte of Text, the bounds in src of the character it came from; nil when no map exists
+}
+
+// FoldMapped folds s like Fold and keeps, for every byte of the result, the
+// original character it came from, so a regex match on the folded text can
+// be read back from s with its diacritics. The map is built by folding one
+// character at a time (a run of invalid bytes counts as one, as Fold's
+// U+FFFD does); in the rare case that differs from Fold's decomposition of
+// the whole string (combining marks NFD reorders), there is no map and
+// Source returns the folded text.
+func FoldMapped(s string) Folded {
+ text := Fold(s)
+ if text == s {
+ return Folded{Text: text, src: s, same: true}
+ }
+ var b strings.Builder
+ start := make([]int, 0, len(text))
+ end := make([]int, 0, len(text))
+ for i := 0; i < len(s); {
+ j, piece := i, ""
+ if r, w := utf8.DecodeRuneInString(s[i:]); r == utf8.RuneError && w == 1 {
+ for j < len(s) {
+ if r, w := utf8.DecodeRuneInString(s[j:]); r != utf8.RuneError || w != 1 {
+ break
+ }
+ j++
+ }
+ piece = "\ufffd"
+ } else {
+ j = i + w
+ piece = Fold(s[i:j])
+ }
+ b.WriteString(piece)
+ for range len(piece) {
+ start = append(start, i)
+ end = append(end, j)
+ }
+ i = j
+ }
+ if b.String() != text {
+ return Folded{Text: text, src: s}
+ }
+ return Folded{Text: text, src: s, start: start, end: end}
+}
+
+// Source returns the original text that Text[a:b] was folded from, widened
+// to whole characters; without a map, Text[a:b] itself.
+func (f Folded) Source(a, b int) string {
+ switch {
+ case f.same || f.start == nil:
+ return f.Text[a:b]
+ case a >= b:
+ return ""
+ }
+ return f.src[f.start[a]:f.end[b-1]]
+}
+
// Text puts s into the form content and keywords are compared in: Fold if
// fold, strings.ToLower if ignoreCase, then every run of Unicode white
// space becomes one ASCII space and both ends are trimmed.
diff --git a/internal/norm/norm_test.go b/internal/norm/norm_test.go
index 885c6f7..a165b32 100644
--- a/internal/norm/norm_test.go
+++ b/internal/norm/norm_test.go
@@ -73,3 +73,33 @@ func TestFoldASCIINoAlloc(t *testing.T) {
t.Errorf("Fold allocates %v times on ASCII input", n)
}
}
+
+// TestFoldMappedSource: a part of the folded text leads back to the original
+// characters it came from, widened to whole characters - so a capture can be
+// written with its diacritics.
+func TestFoldMappedSource(t *testing.T) {
+ cases := []struct {
+ in, folded string
+ a, b int
+ want string
+ }{
+ {"Łódź-faktura.pdf", "Lodz-faktura.pdf", 0, 4, "Łódź"},
+ {"Łódź-faktura.pdf", "Lodz-faktura.pdf", 5, 12, "faktura"},
+ {"straße", "strasse", 4, 5, "ß"},
+ {"áb", "ab", 0, 1, "a"},
+ {"áb", "ab", 1, 2, "b"},
+ {"\xff\xfeé", "�e", 0, 3, "\xff\xfe"},
+ {"plain.txt", "plain.txt", 0, 5, "plain"},
+ {"Łódź", "Lodz", 2, 2, ""},
+ }
+ for _, c := range cases {
+ f := FoldMapped(c.in)
+ if f.Text != c.folded || f.Text != Fold(c.in) {
+ t.Errorf("FoldMapped(%q).Text = %q, want %q (Fold)", c.in, f.Text, c.folded)
+ continue
+ }
+ if got := f.Source(c.a, c.b); got != c.want {
+ t.Errorf("FoldMapped(%q).Source(%d, %d) = %q, want %q", c.in, c.a, c.b, got, c.want)
+ }
+ }
+}
diff --git a/internal/norm/testdata/fuzz/FuzzFoldMapped/ef66462f0d689aef b/internal/norm/testdata/fuzz/FuzzFoldMapped/ef66462f0d689aef
new file mode 100644
index 0000000..f77399e
--- /dev/null
+++ b/internal/norm/testdata/fuzz/FuzzFoldMapped/ef66462f0d689aef
@@ -0,0 +1,4 @@
+go test fuzz v1
+string("0\u0381")
+int(0)
+int(2)
diff --git a/man/krino.conf.5 b/man/krino.conf.5
index 28d7034..26da541 100644
--- a/man/krino.conf.5
+++ b/man/krino.conf.5
@@ -666,10 +666,15 @@ With
.Ic fold
on, the default, a
.Ic name
-test matches the folded name and its captures come from that text, so
+test matches the folded name, but its captures are the original name's
+characters:
+.Ql (name \(dq^(.+)-faktura\(dq)
+on
+.Pa \[/L]\['o]d\[u017A]-faktura.pdf
+makes
.Ic {1}
-loses diacritics
-.Pq see Sx KNOWN LIMITATIONS .
+the name's own
+.Ql \[/L]\['o]d\[u017A] .
.Pp
.Ar FMT
is a strftime subset:
@@ -801,21 +806,6 @@ Duplicate conditions with different scopes do not share an original, so two
such rules can between them move every copy of a group aside; nothing is
deleted.
.Sh KNOWN LIMITATIONS
-Captures
-.Ic {1} No ... Ic {9}
-come from the folded name when
-.Ic fold
-is on:
-.Ql (name \(dq^(.+)-faktura\(dq)
-on
-.Pa \[/L]\['o]d\[u017A]-faktura.pdf
-makes
-.Ic {1}
-.Ql Lodz .
-Set
-.Ql (fold no)
-in the rule to keep diacritics.
-.Pp
A
.Ar dest
whose