aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLukasz Kasprzak <lukas@labunix.xyz>2026-09-12 01:22:12 +0200
committerLukasz Kasprzak <lukas@labunix.xyz>2026-09-12 01:22:12 +0200
commit3b36a48b7ce5a53a9366f3b31f94311f178e2553 (patch)
treeecbb277ff916b719f2ee45fba017792b85d5faf9
parent42b02c47be9b285099203e44a2570636d4ca6f03 (diff)
downloadkrino-3b36a48b7ce5a53a9366f3b31f94311f178e2553.tar.gz
krino-3b36a48b7ce5a53a9366f3b31f94311f178e2553.zip
krino: matching — scan, ignore, conditions, extraction, duplicates, explain, dry run
-rw-r--r--CHANGELOG.md2
-rw-r--r--cmd/krino/check.go38
-rw-r--r--cmd/krino/commands_test.go6
-rw-r--r--cmd/krino/explain.go70
-rw-r--r--cmd/krino/main.go6
-rw-r--r--cmd/krino/main_test.go1
-rw-r--r--cmd/krino/matching_test.go374
-rw-r--r--cmd/krino/sort.go252
-rw-r--r--cmd/krino/sort_test.go23
-rw-r--r--docs/design.md15
-rw-r--r--go.mod4
-rw-r--r--go.sum2
-rw-r--r--internal/cond/compile.go388
-rw-r--r--internal/cond/compile_test.go150
-rw-r--r--internal/cond/eval.go302
-rw-r--r--internal/cond/eval_test.go222
-rw-r--r--internal/cond/types.go111
-rw-r--r--internal/dup/dup.go398
-rw-r--r--internal/dup/dup_test.go331
-rw-r--r--internal/engine/engine.go170
-rw-r--r--internal/engine/engine_test.go166
-rw-r--r--internal/engine/facts.go204
-rw-r--r--internal/engine/facts_test.go77
-rw-r--r--internal/engine/match.go357
-rw-r--r--internal/engine/match_test.go367
-rw-r--r--internal/extract/extract.go154
-rw-r--r--internal/extract/plain.go282
-rw-r--r--internal/extract/plain_test.go229
-rw-r--r--internal/extract/tools.go246
-rw-r--r--internal/extract/tools_test.go239
-rw-r--r--internal/extract/zipxml.go224
-rw-r--r--internal/extract/zipxml_test.go205
-rw-r--r--internal/ignore/ignore.go324
-rw-r--r--internal/ignore/ignore_test.go58
-rw-r--r--internal/ignore/oracle_test.go94
-rw-r--r--internal/norm/norm.go96
-rw-r--r--internal/norm/norm_test.go73
-rw-r--r--internal/scan/scan.go233
-rw-r--r--internal/scan/scan_test.go263
39 files changed, 6729 insertions, 27 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8b5fb8b..b5adf3e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,4 +2,4 @@
## Unreleased
-- Initial development: config language, `krino init`, `krino new`, `krino check`.
+- Initial development: config language, `krino init`, `krino new`, `krino check`, `krino explain`, and the `-n` dry run.
diff --git a/cmd/krino/check.go b/cmd/krino/check.go
index fcd507d..1820bbe 100644
--- a/cmd/krino/check.go
+++ b/cmd/krino/check.go
@@ -5,40 +5,52 @@ package main
import (
"fmt"
"io"
- "os"
- "krino/internal/config"
+ "krino/internal/engine"
"krino/internal/xdg"
)
func init() { commands["check"] = cmdCheck }
-// cmdCheck validates the configuration and lists each directory's rules.
+// cmdCheck validates the configuration and lists each directory's rules,
+// its current state, and the extractors available for content tests.
func cmdCheck(g *globals, args []string, stdout, stderr io.Writer) int {
fs := flagSet("check", g)
if code, ok := parse(fs, args, stdout, stderr); !ok {
return code
}
- cfg, errs := config.Load(mainFile(g), fs.Args()...)
+ e, errs := engine.Load(mainFile(g), fs.Args()...)
if len(errs) > 0 {
printDiags(stderr, errs)
return 2
}
- fmt.Fprintf(stdout, "config: %s\n", xdg.Abbrev(cfg.Main.File))
- if len(cfg.Dirs) == 0 {
+ r := e.Check()
+
+ fmt.Fprintf(stdout, "config: %s\n", xdg.Abbrev(r.MainFile))
+ fmt.Fprintf(stdout, "log: %s\n", xdg.Abbrev(r.LogFile))
+ if len(r.Dirs) == 0 {
fmt.Fprintln(stdout, "no directories included; add one with: krino new NAME PATH")
}
- for _, d := range cfg.Dirs {
- fmt.Fprintf(stdout, "\n%s %s\n", d.Name, xdg.Abbrev(d.Path))
- if fi, err := os.Stat(d.Path); err != nil || !fi.IsDir() {
- fmt.Fprintf(stdout, " warning: %s is not a directory right now; it will be skipped\n", xdg.Abbrev(d.Path))
+ for _, dr := range r.Dirs {
+ fmt.Fprintf(stdout, "\n%s %s\n", dr.Dir.Name, xdg.Abbrev(dr.Dir.Root))
+ if dr.Missing {
+ fmt.Fprintf(stdout, " warning: %s is not a directory right now; it will be skipped\n", xdg.Abbrev(dr.Dir.Root))
}
- if len(d.Rules) == 0 {
+ if len(dr.Dir.Rules) == 0 {
fmt.Fprintln(stdout, " no rules yet")
}
- for i, r := range d.Rules {
- fmt.Fprintf(stdout, " %2d %-16s %s\n", i+1, r.Name, describeActions(r))
+ for i, rule := range dr.Dir.Rules {
+ fmt.Fprintf(stdout, " %2d %-16s %s\n", i+1, rule.Name, describeActions(rule.Conf))
+ }
+ }
+
+ fmt.Fprintln(stdout, "\nextractors:")
+ for _, tool := range r.Tools {
+ path := "not installed"
+ if tool.Path != "" {
+ path = xdg.Abbrev(tool.Path)
}
+ fmt.Fprintf(stdout, " %-9s %s\n", tool.Name, path)
}
return 0
}
diff --git a/cmd/krino/commands_test.go b/cmd/krino/commands_test.go
index 68a2174..2156da2 100644
--- a/cmd/krino/commands_test.go
+++ b/cmd/krino/commands_test.go
@@ -9,12 +9,14 @@ import (
"testing"
)
-// home gives each test its own HOME with an empty XDG config.
+// home gives each test its own HOME with no XDG overrides.
func home(t *testing.T) string {
t.Helper()
h := t.TempDir()
t.Setenv("HOME", h)
- t.Setenv("XDG_CONFIG_HOME", "")
+ for _, v := range []string{"XDG_CONFIG_HOME", "XDG_STATE_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME"} {
+ t.Setenv(v, "")
+ }
return h
}
diff --git a/cmd/krino/explain.go b/cmd/krino/explain.go
new file mode 100644
index 0000000..a4d0253
--- /dev/null
+++ b/cmd/krino/explain.go
@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package main
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "io"
+ "strings"
+
+ "krino/internal/cond"
+ "krino/internal/engine"
+ "krino/internal/xdg"
+)
+
+func init() { commands["explain"] = cmdExplain }
+
+// cmdExplain evaluates every rule of FILE's directory against it and shows
+// each test's result.
+func cmdExplain(g *globals, args []string, stdout, stderr io.Writer) int {
+ fs := flagSet("explain", g)
+ if code, ok := parse(fs, args, stdout, stderr); !ok {
+ return code
+ }
+ if fs.NArg() != 1 {
+ return usageError(stderr, "usage: krino explain FILE")
+ }
+
+ e, errs := engine.Load(mainFile(g))
+ if len(errs) > 0 {
+ printDiags(stderr, errs)
+ return 2
+ }
+
+ x, err := e.Explain(context.Background(), xdg.Expand(fs.Arg(0)))
+ if err != nil {
+ fmt.Fprintf(stderr, "krino: %v\n", err)
+ return 2
+ }
+
+ fmt.Fprintf(stdout, "%s (directory %s)\n", xdg.Abbrev(x.File.Path), x.Dir.Name)
+ if x.Skip != "" {
+ fmt.Fprintf(stdout, "krino would not look at this file: %s\n", x.Skip)
+ }
+ fmt.Fprintln(stdout)
+ for _, rt := range x.Rules {
+ if rt.Stopped != "" {
+ fmt.Fprintf(stdout, "rule %s: not evaluated, %s\n", rt.Rule.Name, rt.Stopped)
+ continue
+ }
+ status := "no"
+ if rt.Match {
+ status = "MATCH"
+ }
+ fmt.Fprintf(stdout, "rule %s: %s\n", rt.Rule.Name, status)
+ printTrace(stdout, rt.Trace)
+ }
+ return 0
+}
+
+// printTrace writes t's Format output with every line prefixed by two
+// spaces.
+func printTrace(w io.Writer, t *cond.Trace) {
+ var buf bytes.Buffer
+ t.Format(&buf)
+ for _, line := range strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") {
+ fmt.Fprintf(w, " %s\n", line)
+ }
+}
diff --git a/cmd/krino/main.go b/cmd/krino/main.go
index 0048098..9618f6c 100644
--- a/cmd/krino/main.go
+++ b/cmd/krino/main.go
@@ -97,9 +97,3 @@ func parse(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) (int, bool
return 2, false
}
}
-
-// cmdSort plans and applies the included directories. It arrives in plan 3.
-func cmdSort(g *globals, names []string, stdout, stderr io.Writer) int {
- fmt.Fprintln(stderr, "krino: sorting is not implemented yet; try 'krino check'")
- return 2
-}
diff --git a/cmd/krino/main_test.go b/cmd/krino/main_test.go
index 1caab6b..46ceb44 100644
--- a/cmd/krino/main_test.go
+++ b/cmd/krino/main_test.go
@@ -52,6 +52,7 @@ func TestCommandsAreReserved(t *testing.T) {
}
func TestSortNotYet(t *testing.T) {
+ home(t)
code, _, errOut := runCLI(t)
if code != 2 || !strings.Contains(errOut, "not implemented yet") {
t.Fatalf("got %d %q", code, errOut)
diff --git a/cmd/krino/matching_test.go b/cmd/krino/matching_test.go
new file mode 100644
index 0000000..eaf9494
--- /dev/null
+++ b/cmd/krino/matching_test.go
@@ -0,0 +1,374 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package main
+
+import (
+ "bytes"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+)
+
+const dlRules = `
+(path "~/dl")
+(recursive yes)
+(min-age 0s)
+(ignore "*.part")
+(rule "dups" (when (duplicate)) (delete) (stop))
+(rule "acme" (when (type document) (content "acme ltd")) (move "Work/Acme") (stop))
+(rule "images" (when (type image)) (move "Pictures"))
+(rule "rest" (when (not (matched)) (type text)) (move "Other"))
+`
+
+// matchingFixture creates ~/dl, a config for it, and an empty PATH, so no
+// extraction tool exists.
+func matchingFixture(t *testing.T) string {
+ t.Helper()
+ h := home(t)
+ t.Setenv("PATH", t.TempDir())
+ dl := filepath.Join(h, "dl")
+ files := map[string]string{
+ "inv1.txt": "Invoice from ACME LTD", "notes.txt": "shopping list",
+ "photo.jpg": "\xff\xd8 jpeg", "report.pdf": "%PDF same", "report (1).pdf": "%PDF same",
+ "brochure.doc": "\xd0\xcf doc", "movie.mkv": "v", "movie.mkv.part": "p",
+ }
+ old := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+ for n, b := range files {
+ p := filepath.Join(dl, n)
+ os.MkdirAll(filepath.Dir(p), 0o755)
+ os.WriteFile(p, []byte(b), 0o644)
+ os.Chtimes(p, old, old)
+ }
+ os.Chtimes(filepath.Join(dl, "report (1).pdf"), old.Add(time.Hour), old.Add(time.Hour))
+ if code, _, errOut := runCLI(t, "init"); code != 0 {
+ t.Fatal(errOut)
+ }
+ if code, _, errOut := runCLI(t, "new", "dl", dl); code != 0 {
+ t.Fatal(errOut)
+ }
+ if err := os.WriteFile(filepath.Join(h, ".config/krino/dirs/dl.conf"), []byte(dlRules), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ return h
+}
+
+func TestDryRun(t *testing.T) {
+ matchingFixture(t)
+ code, out, errOut := runCLI(t, "-n")
+ if code != 0 {
+ t.Fatalf("exit %d: %s", code, errOut)
+ }
+ for _, want := range []string{
+ "krino: dl ~/dl\n8 scanned · 4 matched · 2 warnings · ",
+ "\n inv1.txt acme: type txt, content \"acme ltd\"\n",
+ "\n notes.txt rest: not matched, type txt\n",
+ "\n report (1).pdf dups: duplicate of report.pdf\n",
+ "\nwarnings\n brochure.doc acme: content unreadable: needs antiword or catdoc, not installed\n",
+ "\nnot matched: 2 · ignored: 1 · busy: 1 (-v lists them)\n",
+ } {
+ if !strings.Contains(out, want) {
+ t.Errorf("output lacks %q:\n%s", want, out)
+ }
+ }
+}
+
+func TestDryRunVerbose(t *testing.T) {
+ matchingFixture(t)
+ _, out, _ := runCLI(t, "-n", "-v")
+ for _, want := range []string{
+ "not matched: 2 · ignored: 1 · busy: 1\n",
+ "\nnot matched\n brochure.doc\n report.pdf\n",
+ "\nskipped\n movie.mkv busy\n movie.mkv.part ignored\n",
+ } {
+ if !strings.Contains(out, want) {
+ t.Errorf("verbose output lacks %q:\n%s", want, out)
+ }
+ }
+}
+
+func TestExplainCommand(t *testing.T) {
+ h := matchingFixture(t)
+ code, out, errOut := runCLI(t, "explain", filepath.Join(h, "dl", "inv1.txt"))
+ if code != 0 {
+ t.Fatalf("exit %d: %s", code, errOut)
+ }
+ for _, want := range []string{
+ "~/dl/inv1.txt (directory dl)\n",
+ "rule dups: no\n no duplicate\n",
+ "rule acme: MATCH\n yes and\n yes type document\n yes content \"acme ltd\"\n",
+ "rule images: not evaluated, stopped by rule acme\n",
+ } {
+ if !strings.Contains(out, want) {
+ t.Errorf("explain output lacks %q:\n%s", want, out)
+ }
+ }
+ _, out, _ = runCLI(t, "explain", "~/dl/movie.mkv")
+ if !strings.Contains(out, "krino would not look at this file: busy\n") {
+ t.Errorf("busy file:\n%s", out)
+ }
+ if code, _, errOut := runCLI(t, "explain"); code != 2 || !strings.Contains(errOut, "usage: krino explain FILE") {
+ t.Errorf("no argument: %d %q", code, errOut)
+ }
+}
+
+func TestSortFlags(t *testing.T) {
+ matchingFixture(t)
+ tests := []struct {
+ args []string
+ want string
+ }{
+ {[]string{"-y", "-n"}, "-y and -n cannot be used together"},
+ {nil, "applying files is not implemented yet; use -n to see what would happen"},
+ {[]string{"-n", "--json"}, "--json is not implemented yet"},
+ }
+ for _, tt := range tests {
+ if code, _, errOut := runCLI(t, tt.args...); code != 2 || !strings.Contains(errOut, tt.want) {
+ t.Errorf("%v: %d %q, want %q", tt.args, code, errOut, tt.want)
+ }
+ }
+}
+
+func TestCheckListsExtractors(t *testing.T) {
+ h := matchingFixture(t)
+ bin := filepath.Join(h, "bin")
+ os.Mkdir(bin, 0o755)
+ os.WriteFile(filepath.Join(bin, "pdftotext"), []byte("#!/bin/sh\n"), 0o755)
+ t.Setenv("PATH", bin)
+ code, out, errOut := runCLI(t, "check")
+ if code != 0 {
+ t.Fatalf("exit %d: %s", code, errOut)
+ }
+ for _, want := range []string{"log: ~/.local/state/krino/krino.log\n", "\nextractors:\n pdftotext ~/bin/pdftotext\n antiword not installed\n"} {
+ if !strings.Contains(out, want) {
+ t.Errorf("check output lacks %q:\n%s", want, out)
+ }
+ }
+ os.WriteFile(filepath.Join(h, ".config/krino/dirs/dl.conf"), []byte(`(path "~/dl") (rule "x" (when (type "pdf")) (stop))`), 0o644)
+ if code, _, errOut := runCLI(t, "check"); code != 2 || !strings.Contains(errOut, `dl.conf:1:37: type names are bare words: write (type pdf)`) {
+ t.Errorf("condition error: %d %q", code, errOut)
+ }
+}
+
+// TestDryRunWarningsSortedByRel is controller ruling 2026-09-12: the
+// warnings section is one Rel-sorted list across matched and unmatched
+// files, not matched files followed by unmatched files - a reader scans it
+// by name and has no way to see which group a file fell into. "cover.pdf"
+// matches "pdfs" but still carries the warning "acme" recorded before it
+// gave up; "brochure.doc" never matches at all. Their Rel order
+// ("brochure.doc" < "cover.pdf") is the reverse of matched-then-unmatched
+// grouping, so a grouped rendering fails this.
+func TestDryRunWarningsSortedByRel(t *testing.T) {
+ h := home(t)
+ t.Setenv("PATH", t.TempDir())
+ dl := filepath.Join(h, "dl")
+ files := map[string]string{
+ "brochure.doc": "\xd0\xcf doc",
+ "cover.pdf": "%PDF fake",
+ }
+ old := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+ for n, b := range files {
+ p := filepath.Join(dl, n)
+ os.MkdirAll(filepath.Dir(p), 0o755)
+ os.WriteFile(p, []byte(b), 0o644)
+ os.Chtimes(p, old, old)
+ }
+ if code, _, errOut := runCLI(t, "init"); code != 0 {
+ t.Fatal(errOut)
+ }
+ if code, _, errOut := runCLI(t, "new", "dl", dl); code != 0 {
+ t.Fatal(errOut)
+ }
+ conf := `
+(path "~/dl")
+(recursive yes)
+(min-age 0s)
+(rule "acme" (when (content "acme ltd")) (move "Work/Acme"))
+(rule "pdfs" (when (type pdf)) (move "PDFs"))
+`
+ if err := os.WriteFile(filepath.Join(h, ".config/krino/dirs/dl.conf"), []byte(conf), 0o644); err != nil {
+ t.Fatal(err)
+ }
+
+ code, out, errOut := runCLI(t, "-n")
+ if code != 0 {
+ t.Fatalf("exit %d: %s", code, errOut)
+ }
+ want := "\nwarnings\n brochure.doc acme: content unreadable: needs antiword or catdoc, not installed\n cover.pdf acme: content unreadable: needs pdftotext, not installed\n"
+ if !strings.Contains(out, want) {
+ t.Errorf("warnings not Rel-sorted across matched and unmatched:\n%s\nwant substring:\n%s", out, want)
+ }
+}
+
+// TestDirectoryWarningAfterHeader: C3. A directory-level warning must be
+// emitted after its own header line, not before it, so on a terminal (both
+// streams sharing one tty, hence stdout and stderr driven into the same
+// buffer here to observe their relative order) it reads as describing the
+// directory just named instead of floating above it.
+func TestDirectoryWarningAfterHeader(t *testing.T) {
+ h := home(t)
+ dl := filepath.Join(h, "dl")
+ if err := os.MkdirAll(dl, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(dl, "only.txt"), []byte("hello"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if code, _, errOut := runCLI(t, "init"); code != 0 {
+ t.Fatal(errOut)
+ }
+ if code, _, errOut := runCLI(t, "new", "dl", dl); code != 0 {
+ t.Fatal(errOut)
+ }
+ conf := `
+(path "~/dl")
+(recursive yes)
+(min-age 0s)
+(rule "r" (when (duplicate "~/missing")) (stop))
+`
+ if err := os.WriteFile(filepath.Join(h, ".config/krino/dirs/dl.conf"), []byte(conf), 0o644); err != nil {
+ t.Fatal(err)
+ }
+
+ var buf bytes.Buffer
+ if code := run([]string{"-n"}, &buf, &buf); code != 0 {
+ t.Fatalf("exit %d: %s", code, buf.String())
+ }
+ out := buf.String()
+ header := strings.Index(out, "krino: dl ~/dl\n")
+ warning := strings.Index(out, "krino: dl: duplicate: ")
+ if header == -1 || warning == -1 || warning < header {
+ t.Fatalf("directory-level warning not after its header line:\n%s", out)
+ }
+}
+
+// TestSortSkipsMissingRootAndContinues: C5, the exit-1 skip path. A
+// directory whose root has vanished since it was configured is skipped
+// with one line on stderr naming it, but every other directory is still
+// processed, with a blank line still separating their two outputs, and
+// the run as a whole exits 1.
+func TestSortSkipsMissingRootAndContinues(t *testing.T) {
+ h := home(t)
+ a, b, gone := filepath.Join(h, "a"), filepath.Join(h, "b"), filepath.Join(h, "gone")
+ for _, p := range []string{a, b, gone} {
+ if err := os.MkdirAll(p, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if code, _, errOut := runCLI(t, "init"); code != 0 {
+ t.Fatal(errOut)
+ }
+ for _, tt := range []struct{ name, path string }{{"a", a}, {"gone", gone}, {"b", b}} {
+ if code, _, errOut := runCLI(t, "new", tt.name, tt.path); code != 0 {
+ t.Fatal(errOut)
+ }
+ }
+ if err := os.RemoveAll(gone); err != nil {
+ t.Fatal(err)
+ }
+
+ code, out, errOut := runCLI(t, "-n")
+ if code != 1 {
+ t.Fatalf("exit = %d, want 1", code)
+ }
+ if want := "krino: skipping gone: ~/gone is not a directory\n"; errOut != want {
+ t.Fatalf("stderr = %q, want %q", errOut, want)
+ }
+ ai := strings.Index(out, "krino: a ~/a\n")
+ bi := strings.Index(out, "\n\nkrino: b ~/b\n")
+ if ai == -1 || bi == -1 || bi < ai {
+ t.Fatalf("directories a and b not both processed with a blank line between them:\n%s", out)
+ }
+}
+
+// TestDirectoryWarningNotCountedInWarningsField: C5. A directory-level
+// warning is printed as "krino: NAME: <warning>" on stderr, but is not one
+// of the per-file warnings the "N warnings" field in the summary line
+// counts.
+func TestDirectoryWarningNotCountedInWarningsField(t *testing.T) {
+ h := home(t)
+ dl := filepath.Join(h, "dl")
+ if err := os.MkdirAll(dl, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(dl, "only.txt"), []byte("hello"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if code, _, errOut := runCLI(t, "init"); code != 0 {
+ t.Fatal(errOut)
+ }
+ if code, _, errOut := runCLI(t, "new", "dl", dl); code != 0 {
+ t.Fatal(errOut)
+ }
+ conf := `
+(path "~/dl")
+(recursive yes)
+(min-age 0s)
+(rule "r" (when (duplicate "~/missing")) (stop))
+`
+ if err := os.WriteFile(filepath.Join(h, ".config/krino/dirs/dl.conf"), []byte(conf), 0o644); err != nil {
+ t.Fatal(err)
+ }
+
+ code, out, errOut := runCLI(t, "-n")
+ if code != 0 {
+ t.Fatalf("exit %d: %s", code, errOut)
+ }
+ if want := "krino: dl: duplicate: "; !strings.Contains(errOut, want) {
+ t.Fatalf("stderr lacks the directory-level warning %q:\n%s", want, errOut)
+ }
+ if want := "0 warnings"; !strings.Contains(out, want) {
+ t.Fatalf("summary line should not count the directory-level warning:\n%s", out)
+ }
+}
+
+// TestLongNameNotPaddedLayoutIntact: C5, the 40-character cap. A file name
+// longer than the 40-character column cap is left unpadded (not truncated,
+// not stretched further), while a short name alongside it is still padded
+// out to the full 40-column cap — the layout stays a clean two-column grid
+// even though one row's first cell overruns it.
+func TestLongNameNotPaddedLayoutIntact(t *testing.T) {
+ h := home(t)
+ dl := filepath.Join(h, "dl")
+ long := strings.Repeat("a", 42) + ".txt" // 46 runes: past the 40-column cap
+ short := "b.txt"
+ old := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+ for _, name := range []string{long, short} {
+ p := filepath.Join(dl, name)
+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(p, []byte("x"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Chtimes(p, old, old); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if code, _, errOut := runCLI(t, "init"); code != 0 {
+ t.Fatal(errOut)
+ }
+ if code, _, errOut := runCLI(t, "new", "dl", dl); code != 0 {
+ t.Fatal(errOut)
+ }
+ conf := `
+(path "~/dl")
+(min-age 0s)
+(rule "r" (when (type txt)) (stop))
+`
+ if err := os.WriteFile(filepath.Join(h, ".config/krino/dirs/dl.conf"), []byte(conf), 0o644); err != nil {
+ t.Fatal(err)
+ }
+
+ code, out, errOut := runCLI(t, "-n")
+ if code != 0 {
+ t.Fatalf("exit %d: %s", code, errOut)
+ }
+ if want := "\n " + long + " r: type txt\n"; !strings.Contains(out, want) {
+ t.Errorf("long name should be unpadded (exactly two trailing spaces before the rule column):\n%s\nwant substring:\n%s", out, want)
+ }
+ if want := "\n " + short + strings.Repeat(" ", 40-len(short)) + " r: type txt\n"; !strings.Contains(out, want) {
+ t.Errorf("short name should still be padded to the 40-column cap:\n%s\nwant substring:\n%s", out, want)
+ }
+}
diff --git a/cmd/krino/sort.go b/cmd/krino/sort.go
new file mode 100644
index 0000000..fe88cf8
--- /dev/null
+++ b/cmd/krino/sort.go
@@ -0,0 +1,252 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "os"
+ "sort"
+ "strings"
+ "unicode/utf8"
+
+ "krino/internal/engine"
+ "krino/internal/scan"
+ "krino/internal/xdg"
+)
+
+// cmdSort plans and applies the included directories. Only -n (dry run) is
+// implemented; applying arrives in plan 3.
+func cmdSort(g *globals, names []string, stdout, stderr io.Writer) int {
+ if g.yes && g.dry {
+ return usageError(stderr, "-y and -n cannot be used together")
+ }
+ if g.json {
+ fmt.Fprintln(stderr, "krino: --json is not implemented yet")
+ return 2
+ }
+ if !g.dry {
+ fmt.Fprintln(stderr, "krino: applying files is not implemented yet; use -n to see what would happen")
+ return 2
+ }
+
+ e, errs := engine.Load(mainFile(g), names...)
+ if len(errs) > 0 {
+ printDiags(stderr, errs)
+ return 2
+ }
+
+ exit := 0
+ printed := false
+ for _, d := range e.Dirs {
+ if fi, err := os.Stat(d.Root); err != nil || !fi.IsDir() {
+ fmt.Fprintf(stderr, "krino: skipping %s: %s is not a directory\n", d.Name, xdg.Abbrev(d.Root))
+ exit = 1
+ continue
+ }
+ r, err := e.Match(context.Background(), d)
+ if err != nil {
+ fmt.Fprintf(stderr, "krino: skipping %s: %v\n", d.Name, err)
+ exit = 1
+ continue
+ }
+ if printed {
+ fmt.Fprintln(stdout)
+ }
+ printed = true
+ fmt.Fprintf(stdout, "krino: %s %s\n", d.Name, xdg.Abbrev(d.Root))
+ // C3: directory-level warnings go to stderr after the header
+ // line above, not before it, so on a terminal they read as
+ // describing the directory just named instead of floating above it.
+ for _, w := range r.Warnings {
+ fmt.Fprintf(stderr, "krino: %s: %s\n", d.Name, w)
+ }
+ printResult(stdout, r, g.verbose)
+ }
+ return exit
+}
+
+// printResult renders one directory's match result the way krino -n shows
+// it, below the header line cmdSort has already written: a summary line,
+// then the matched, warnings and skip-count sections, each present only
+// when it has something to show. Plan 3 reuses this for the outcome of an
+// actual run.
+func printResult(w io.Writer, r *engine.Result, verbose bool) {
+ // C1: spec §8.2's worked example is "266 scanned" against "41 to act
+ // on" and "not acted on: 3 busy · 12 ignored · 210 unmatched", and
+ // 41+3+12+210 = 266 - so scanned counts matched, unmatched and skipped
+ // alike, not just matched plus unmatched.
+ scanned := len(r.Matched) + len(r.Unmatched) + len(r.Skipped)
+ warned := 0
+ for _, fm := range r.Matched {
+ if len(fm.Warnings) > 0 {
+ warned++
+ }
+ }
+ for _, fm := range r.Unmatched {
+ if len(fm.Warnings) > 0 {
+ warned++
+ }
+ }
+ fmt.Fprintf(w, "%d scanned · %d matched · %d warnings · %.2fs\n", scanned, len(r.Matched), warned, r.Elapsed.Seconds())
+
+ if len(r.Matched) > 0 {
+ fmt.Fprintln(w)
+ printMatched(w, r.Matched)
+ }
+
+ if lines := collectWarnings(r); len(lines) > 0 {
+ fmt.Fprintln(w)
+ fmt.Fprintln(w, "warnings")
+ printWarnings(w, lines)
+ }
+
+ if line := skipSummaryLine(r, verbose); line != "" {
+ fmt.Fprintln(w)
+ fmt.Fprintln(w, line)
+ }
+
+ if verbose {
+ if len(r.Unmatched) > 0 {
+ fmt.Fprintln(w)
+ fmt.Fprintln(w, "not matched")
+ for _, fm := range r.Unmatched {
+ fmt.Fprintf(w, " %s\n", fm.File.Rel)
+ }
+ }
+ if len(r.Skipped) > 0 {
+ fmt.Fprintln(w)
+ fmt.Fprintln(w, "skipped")
+ printSkipped(w, r.Skipped)
+ }
+ }
+}
+
+// printMatched lists each matched file, its Rel padded to the widest shown
+// (capped at 40), then each matching rule as "name: reasons", rules joined
+// by "; ".
+func printMatched(w io.Writer, matched []engine.FileMatch) {
+ rels := make([]string, len(matched))
+ for i, fm := range matched {
+ rels[i] = fm.File.Rel
+ }
+ width := relWidth(rels)
+ for _, fm := range matched {
+ parts := make([]string, len(fm.Rules))
+ for i, rm := range fm.Rules {
+ parts[i] = rm.Rule.Name + ": " + strings.Join(rm.Reasons, ", ")
+ }
+ fmt.Fprintf(w, " %s %s\n", padCell(fm.File.Rel, width), strings.Join(parts, "; "))
+ }
+}
+
+// warnLine is one file's warning, for the warnings section.
+type warnLine struct {
+ rel string
+ text string
+}
+
+// collectWarnings gathers every file's warnings into a single list sorted
+// by Rel across matched and unmatched files alike: a reader scans this
+// section by file name and has no way to tell which group a file fell
+// into, so grouping by match state is invisible structure that would only
+// show up as an odd order. A file's own warnings (when it has more than
+// one) stay in the order they were recorded.
+func collectWarnings(r *engine.Result) []warnLine {
+ files := make([]engine.FileMatch, 0, len(r.Matched)+len(r.Unmatched))
+ files = append(files, r.Matched...)
+ files = append(files, r.Unmatched...)
+ sort.Slice(files, func(i, j int) bool { return files[i].File.Rel < files[j].File.Rel })
+
+ var out []warnLine
+ for _, fm := range files {
+ for _, w := range fm.Warnings {
+ out = append(out, warnLine{fm.File.Rel, w})
+ }
+ }
+ return out
+}
+
+// printWarnings lists one line per warning, Rel padded to the widest shown
+// (capped at 40).
+func printWarnings(w io.Writer, lines []warnLine) {
+ rels := make([]string, len(lines))
+ for i, l := range lines {
+ rels[i] = l.rel
+ }
+ width := relWidth(rels)
+ for _, l := range lines {
+ fmt.Fprintf(w, " %s %s\n", padCell(l.rel, width), l.text)
+ }
+}
+
+// printSkipped lists each skipped file, Rel padded to the widest shown
+// (capped at 40), then its reason.
+func printSkipped(w io.Writer, skipped []scan.Skipped) {
+ rels := make([]string, len(skipped))
+ for i, s := range skipped {
+ rels[i] = s.Rel
+ }
+ width := relWidth(rels)
+ for _, s := range skipped {
+ fmt.Fprintf(w, " %s %s\n", padCell(s.Rel, width), s.Reason.String())
+ }
+}
+
+// skipReasonOrder is the order the last line reports skip reasons in,
+// after "not matched".
+var skipReasonOrder = []scan.Reason{scan.Ignored, scan.Busy, scan.TooNew, scan.Symlink, scan.NotRegular, scan.Unreadable}
+
+// skipSummaryLine builds the "not matched: N · ignored: N ..." line, only
+// the non-zero counts, or "" when every count is zero.
+func skipSummaryLine(r *engine.Result, verbose bool) string {
+ counts := map[scan.Reason]int{}
+ for _, s := range r.Skipped {
+ counts[s.Reason]++
+ }
+
+ var parts []string
+ if n := len(r.Unmatched); n > 0 {
+ parts = append(parts, fmt.Sprintf("not matched: %d", n))
+ }
+ for _, reason := range skipReasonOrder {
+ if n := counts[reason]; n > 0 {
+ parts = append(parts, fmt.Sprintf("%s: %d", reason.String(), n))
+ }
+ }
+ if len(parts) == 0 {
+ return ""
+ }
+ line := strings.Join(parts, " · ")
+ if !verbose {
+ line += " (-v lists them)"
+ }
+ return line
+}
+
+// relWidth returns the column width for a list of Rel names: the widest in
+// runes (C4: not bytes, or a name carrying diacritics misaligns its
+// column), capped at 40.
+func relWidth(rels []string) int {
+ w := 0
+ for _, s := range rels {
+ if n := utf8.RuneCountInString(s); n > w {
+ w = n
+ }
+ }
+ if w > 40 {
+ w = 40
+ }
+ return w
+}
+
+// padCell pads s to width w (runes, not bytes) with trailing spaces; s
+// already at or beyond w is left unpadded.
+func padCell(s string, w int) string {
+ n := utf8.RuneCountInString(s)
+ if n >= w {
+ return s
+ }
+ return s + strings.Repeat(" ", w-n)
+}
diff --git a/cmd/krino/sort_test.go b/cmd/krino/sort_test.go
new file mode 100644
index 0000000..a646503
--- /dev/null
+++ b/cmd/krino/sort_test.go
@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package main
+
+import "testing"
+
+// TestRelWidthAndPadCellCountRunes: C4. relWidth and padCell must measure
+// column width in runes, not bytes, or a name carrying diacritics
+// misaligns its column - "próba.txt" is 9 runes but 10 bytes (ó is a
+// two-byte UTF-8 sequence), one column narrower than its byte length
+// would suggest.
+func TestRelWidthAndPadCellCountRunes(t *testing.T) {
+ rels := []string{"a.txt", "próba.txt"}
+ if w := relWidth(rels); w != 9 {
+ t.Fatalf("relWidth(%q) = %d, want 9 (rune count of próba.txt, not its %d bytes)", rels, w, len("próba.txt"))
+ }
+ if got, want := padCell("a.txt", 9), "a.txt "; got != want {
+ t.Fatalf("padCell(%q, 9) = %q, want %q", "a.txt", got, want)
+ }
+ if got, want := padCell("próba.txt", 9), "próba.txt"; got != want {
+ t.Fatalf("padCell(%q, 9) = %q, want %q (already at width: no padding)", "próba.txt", got, want)
+ }
+}
diff --git a/docs/design.md b/docs/design.md
index ed77e8c..888ab34 100644
--- a/docs/design.md
+++ b/docs/design.md
@@ -204,8 +204,8 @@ decompose (ł ø đ ħ ß æ œ and their capitals) to ASCII.
2. Rules are evaluated in order. A matching rule adds its actions to the
file's chain. `(stop)` on a matching rule ends evaluation for that file.
3. Tests have no side effects, so `and` / `or` evaluate their arguments
- cheapest first: `type`, `name`, `path`, `size`, `age`, `matched`, then
- `duplicate`, then `content`. Content is extracted at most once per file,
+ cheapest first: `type`, `size`, `age` and `matched`, then `name` and
+ `path`, then `duplicate`, then `content`. Content is extracted at most once per file,
and only if evaluation reaches a `content` test.
4. If a file's text cannot be extracted, `content` is false and the plan
shows a warning naming the rule that wanted it.
@@ -330,6 +330,11 @@ name has to change, the log records the actual name.
- Always ignored: every rule `DEST` that lies inside the root, the Trash, and
the config directory. For a `DEST` with placeholders, the part before the
first placeholder is ignored: `Work/Acme/{mtime:%Y}` ignores `Work/Acme/`.
+ Known limitation: a `DEST` whose *first* path component is itself a
+ placeholder (`{ext}`, `{mtime:%Y}`) has no static prefix, so nothing can be
+ excluded before the walk, and krino would re-examine its own output; plan 3
+ closes this by treating a file already at its computed destination as a
+ no-op.
- Skipped as busy: files newer than `min-age`, or with a `busy` sibling.
### 8.2 Display
@@ -471,8 +476,10 @@ cmd/krino/ flags, subcommands, exit codes
internal/sexp/ reader: syntax tree with positions and byte offsets
internal/config/ syntax tree → typed config; validation; embedded template and defaults
internal/cond/ condition tree: compile, cost ordering, evaluation, explain trace
-internal/scan/ walking, gitignore matcher, busy detection
-internal/extract/ text extraction and normalisation
+internal/scan/ walking, busy detection
+internal/ignore/ gitignore-compatible path matcher
+internal/extract/ text extraction
+internal/norm/ text normalisation: case, diacritics, white space
internal/dup/ duplicate index
internal/plan/ files × rules → chains; placeholders; conflicts; JSON
internal/apply/ executing approved steps; cross-filesystem moves
diff --git a/go.mod b/go.mod
index aea6085..3f0c254 100644
--- a/go.mod
+++ b/go.mod
@@ -1,3 +1,5 @@
module krino
-go 1.24
+go 1.24.0
+
+require golang.org/x/text v0.34.0
diff --git a/go.sum b/go.sum
new file mode 100644
index 0000000..47c6532
--- /dev/null
+++ b/go.sum
@@ -0,0 +1,2 @@
+golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
+golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
diff --git a/internal/cond/compile.go b/internal/cond/compile.go
new file mode 100644
index 0000000..4fdc75f
--- /dev/null
+++ b/internal/cond/compile.go
@@ -0,0 +1,388 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package cond
+
+import (
+ "errors"
+ "fmt"
+ "regexp"
+ "regexp/syntax"
+ "sort"
+ "strings"
+
+ "krino/internal/config"
+ "krino/internal/norm"
+ "krino/internal/sexp"
+)
+
+// maxDepth is the deepest a condition may nest; the top-level conditions
+// are depth 1.
+const maxDepth = 64
+
+// Compile compiles a rule's (when ...) conditions, applying cost-based
+// reordering. If it returns any diagnostic, the returned *Cond is nil: a
+// caller that ignores errs and calls Eval/Explain on the result panics
+// loudly, instead of silently treating a broken rule as "no conditions"
+// (always true).
+func Compile(file string, when []*sexp.Node, opt Options) (*Cond, []*config.Diag) {
+ c, errs := compile(file, when, opt, true)
+ if len(errs) > 0 {
+ return nil, errs
+ }
+ return c, errs
+}
+
+// compile is Compile with cost reordering switchable, for the property test
+// in Task 8.
+func compile(file string, when []*sexp.Node, opt Options, reorder bool) (*Cond, []*config.Diag) {
+ c := &compiler{file: file, opt: opt, reorder: reorder, cond: &Cond{opt: opt}}
+ var roots []*node
+ for _, n := range when {
+ if nd := c.compileNode(n, 1); nd != nil {
+ roots = append(roots, nd)
+ }
+ }
+ switch len(roots) {
+ case 0:
+ // no conditions: root stays nil, always true
+ case 1:
+ c.cond.root = roots[0]
+ default:
+ // several top-level conditions are an implicit and
+ c.cond.root = c.combine(kAnd, "and", roots, when[0].Pos)
+ }
+ return c.cond, c.errs
+}
+
+// compiler holds the state of one Compile call.
+type compiler struct {
+ file string
+ opt Options
+ reorder bool
+ cond *Cond
+ errs []*config.Diag
+}
+
+// errorf records a diagnostic at n's position; a nil n means the file as a
+// whole (unused here, kept for symmetry with config.diags).
+func (c *compiler) errorf(n *sexp.Node, format string, args ...any) {
+ var pos sexp.Pos
+ if n != nil {
+ pos = n.Pos
+ }
+ c.errs = append(c.errs, &config.Diag{File: c.file, Pos: pos, Msg: fmt.Sprintf(format, args...)})
+}
+
+// compileNode compiles one condition node at the given nesting depth (the
+// top-level conditions are depth 1). It returns nil, having recorded at
+// least one diagnostic, when n does not compile.
+func (c *compiler) compileNode(n *sexp.Node, depth int) *node {
+ if depth > maxDepth {
+ c.errorf(n, "conditions nest deeper than %d levels", maxDepth)
+ return nil
+ }
+ if n.Kind != sexp.List || n.Head() == "" {
+ c.errorf(n, "a condition is a form like (type pdf), not %s", n.String())
+ return nil
+ }
+ switch head := n.Head(); head {
+ case "and":
+ return c.compileCombiner(n, depth, kAnd, "and", "and needs at least one condition")
+ case "or":
+ return c.compileCombiner(n, depth, kOr, "or", "or needs at least one condition")
+ case "not":
+ return c.compileNot(n, depth)
+ case "type":
+ return c.compileType(n)
+ case "name":
+ return c.compileRegex(n, kName, "name")
+ case "path":
+ return c.compileRegex(n, kPath, "path")
+ case "content":
+ return c.compileContent(n)
+ case "size":
+ return c.compileSize(n)
+ case "age":
+ return c.compileAge(n)
+ case "duplicate":
+ return c.compileDuplicate(n)
+ case "matched":
+ return c.compileMatched(n)
+ default:
+ c.errorf(n, "unknown test (%s ...); tests are type, name, path, content, size, age, duplicate, matched, and, or, not", head)
+ return nil
+ }
+}
+
+// compileCombiner compiles (and ...) / (or ...): one or more child
+// conditions, cost-summed and cost-sorted (via combine).
+func (c *compiler) compileCombiner(n *sexp.Node, depth int, k kind, label, emptyMsg string) *node {
+ args := n.Args()
+ if len(args) == 0 {
+ c.errorf(n, "%s", emptyMsg)
+ return nil
+ }
+ children := make([]*node, 0, len(args))
+ for _, a := range args {
+ if ch := c.compileNode(a, depth+1); ch != nil {
+ children = append(children, ch)
+ }
+ }
+ return c.combine(k, label, children, n.Pos)
+}
+
+// combine builds an and/or node from already-compiled children: stable
+// cost-sort when reordering, cost is the sum of the children's.
+func (c *compiler) combine(k kind, label string, children []*node, pos sexp.Pos) *node {
+ if len(children) == 0 {
+ return nil
+ }
+ if c.reorder {
+ sort.SliceStable(children, func(i, j int) bool { return children[i].cost < children[j].cost })
+ }
+ cost := 0
+ for _, ch := range children {
+ cost += ch.cost
+ }
+ return &node{kind: k, pos: pos, label: label, children: children, cost: cost}
+}
+
+// compileNot compiles (not C): exactly one child condition.
+func (c *compiler) compileNot(n *sexp.Node, depth int) *node {
+ args := n.Args()
+ if len(args) != 1 {
+ c.errorf(n, "not takes exactly one condition")
+ return nil
+ }
+ child := c.compileNode(args[0], depth+1)
+ if child == nil {
+ return nil
+ }
+ return &node{kind: kNot, pos: n.Pos, label: "not", children: []*node{child}, cost: child.cost}
+}
+
+// compileType compiles (type T...): symbols, lower-cased; a group name
+// expands to its extensions, anything else is a literal (possibly
+// multi-part) extension.
+func (c *compiler) compileType(n *sexp.Node) *node {
+ args := n.Args()
+ if len(args) == 0 {
+ c.errorf(n, "type needs at least one extension or group, like (type pdf)")
+ return nil
+ }
+ var suffixes []string
+ bad := false
+ for _, a := range args {
+ if a.Kind != sexp.Symbol {
+ c.errorf(a, "type names are bare words: write (type pdf)")
+ bad = true
+ continue
+ }
+ lower := strings.ToLower(a.Text)
+ if exts, ok := groups[lower]; ok {
+ for _, e := range exts {
+ suffixes = append(suffixes, "."+e)
+ }
+ } else {
+ suffixes = append(suffixes, "."+lower)
+ }
+ }
+ if bad {
+ return nil
+ }
+ return &node{kind: kType, pos: n.Pos, label: argsLabel("type", args), cost: costCheap, suffixes: suffixes}
+}
+
+// compileRegex compiles (name "RE"...) / (path "RE"...).
+func (c *compiler) compileRegex(n *sexp.Node, k kind, test string) *node {
+ args := n.Args()
+ if len(args) == 0 {
+ c.errorf(n, "%s needs at least one regex in quotes", test)
+ return nil
+ }
+ var patterns []pattern
+ bad := false
+ for _, a := range args {
+ if a.Kind != sexp.String {
+ c.quotedExampleErr(a, test, "regexes")
+ bad = true
+ continue
+ }
+ src := a.Text
+ pat := src
+ if c.opt.Fold {
+ // E5: folding is applied to the regex source itself, not just
+ // to the text it is matched against - so a fold that expands
+ // one character into several changes the pattern's structure,
+ // not just its literal characters: "ß+" (one letter, a
+ // quantifier on it) becomes "ss+" (a quantifier on only the
+ // second "s") once norm.Fold expands "ß" to "ss", and likewise
+ // "æ" to "ae". A rule relying on repetition or anchoring
+ // around such a letter needs to account for this.
+ pat = norm.Fold(pat)
+ }
+ if c.opt.IgnoreCase {
+ pat = "(?i)" + pat
+ }
+ re, err := regexp.Compile(pat)
+ if err != nil {
+ var se *syntax.Error
+ if errors.As(err, &se) {
+ c.errorf(a, "%s: bad regex %q: %s", test, src, se.Code)
+ } else {
+ c.errorf(a, "%s: bad regex %q: %s", test, src, err)
+ }
+ bad = true
+ continue
+ }
+ patterns = append(patterns, pattern{re: re, src: src})
+ }
+ if bad {
+ return nil
+ }
+ return &node{kind: k, pos: n.Pos, label: argsLabel(test, args), cost: costRegex, patterns: patterns}
+}
+
+// compileContent compiles (content "KW"...): each keyword is normalised;
+// an empty result is an error.
+func (c *compiler) compileContent(n *sexp.Node) *node {
+ args := n.Args()
+ if len(args) == 0 {
+ c.errorf(n, "content needs at least one keyword in quotes")
+ return nil
+ }
+ var keywords []keyword
+ bad := false
+ for _, a := range args {
+ if a.Kind != sexp.String {
+ c.quotedExampleErr(a, "content", "keywords")
+ bad = true
+ continue
+ }
+ normed := norm.Text(a.Text, c.opt.IgnoreCase, c.opt.Fold)
+ if normed == "" {
+ c.errorf(a, "content keyword is empty")
+ bad = true
+ continue
+ }
+ keywords = append(keywords, keyword{norm: normed, src: a.Text})
+ }
+ if bad {
+ return nil
+ }
+ c.cond.UsesContent = true
+ return &node{kind: kContent, pos: n.Pos, label: argsLabel("content", args), cost: costContent, keywords: keywords}
+}
+
+// compileSize compiles (size OP SIZE).
+func (c *compiler) compileSize(n *sexp.Node) *node {
+ args := n.Args()
+ if len(args) != 2 || args[0].Kind != sexp.Symbol || args[1].Kind != sexp.Symbol {
+ c.errorf(n, "size takes an operator and a size, like (size > 10M)")
+ return nil
+ }
+ op := args[0].Text
+ bad := false
+ if !validOp(op) {
+ c.errorf(args[0], "size operator is one of > >= < <= =, not %s", op)
+ bad = true
+ }
+ val, err := config.ParseSize(args[1].Text)
+ if err != nil {
+ c.errorf(args[1], "size: %s", err)
+ bad = true
+ }
+ if bad {
+ return nil
+ }
+ return &node{kind: kSize, pos: n.Pos, label: argsLabel("size", args), cost: costCheap, op: op, sizeVal: val}
+}
+
+// compileAge compiles (age OP DURATION).
+func (c *compiler) compileAge(n *sexp.Node) *node {
+ args := n.Args()
+ if len(args) != 2 || args[0].Kind != sexp.Symbol || args[1].Kind != sexp.Symbol {
+ c.errorf(n, "age takes an operator and a duration, like (age > 30d)")
+ return nil
+ }
+ op := args[0].Text
+ bad := false
+ if !validOp(op) {
+ c.errorf(args[0], "age operator is one of > >= < <= =, not %s", op)
+ bad = true
+ }
+ val, err := config.ParseDuration(args[1].Text)
+ if err != nil {
+ c.errorf(args[1], "age: %s", err)
+ bad = true
+ }
+ if bad {
+ return nil
+ }
+ return &node{kind: kAge, pos: n.Pos, label: argsLabel("age", args), cost: costCheap, op: op, ageVal: val}
+}
+
+// validOp reports whether op is one of the size/age comparison operators.
+func validOp(op string) bool {
+ switch op {
+ case ">", ">=", "<", "<=", "=":
+ return true
+ }
+ return false
+}
+
+// compileDuplicate compiles (duplicate "DIR"...): zero or more directories,
+// stored raw for the engine to resolve. Every compiled test's list is
+// appended to Cond.DupDirs, in order.
+func (c *compiler) compileDuplicate(n *sexp.Node) *node {
+ args := n.Args()
+ dirs := make([]string, 0, len(args))
+ bad := false
+ for _, a := range args {
+ if a.Kind != sexp.String {
+ c.quotedExampleErr(a, "duplicate", "directories")
+ bad = true
+ continue
+ }
+ dirs = append(dirs, a.Text)
+ }
+ if bad {
+ return nil
+ }
+ c.cond.DupDirs = append(c.cond.DupDirs, dirs)
+ return &node{kind: kDuplicate, pos: n.Pos, label: argsLabel("duplicate", args), cost: costDuplicate, dirs: dirs}
+}
+
+// compileMatched compiles (matched): no arguments.
+func (c *compiler) compileMatched(n *sexp.Node) *node {
+ if len(n.Args()) != 0 {
+ c.errorf(n, "matched takes nothing: write (matched)")
+ return nil
+ }
+ return &node{kind: kMatched, pos: n.Pos, label: "matched", cost: costCheap}
+}
+
+// quotedExampleErr records the shared "takes X in quotes: write (test
+// "arg")" diagnostic used by name, path, content and duplicate.
+func (c *compiler) quotedExampleErr(a *sexp.Node, test, noun string) {
+ c.errorf(a, "%s takes %s in quotes: write (%s %s)", test, noun, test, sexp.Quote(a.Text))
+}
+
+// argsLabel renders a leaf test as written: the head followed by each
+// argument, bare symbols verbatim, strings as their decoded text in double
+// quotes without escaping.
+func argsLabel(head string, args []*sexp.Node) string {
+ var b strings.Builder
+ b.WriteString(head)
+ for _, a := range args {
+ b.WriteByte(' ')
+ if a.Kind == sexp.String {
+ b.WriteByte('"')
+ b.WriteString(a.Text)
+ b.WriteByte('"')
+ } else {
+ b.WriteString(a.Text)
+ }
+ }
+ return b.String()
+}
diff --git a/internal/cond/compile_test.go b/internal/cond/compile_test.go
new file mode 100644
index 0000000..e68a4d8
--- /dev/null
+++ b/internal/cond/compile_test.go
@@ -0,0 +1,150 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package cond
+
+import (
+ "reflect"
+ "strings"
+ "testing"
+
+ "krino/internal/sexp"
+)
+
+func nodes(t *testing.T, src string) []*sexp.Node {
+ t.Helper()
+ n, err := sexp.Parse("d.conf", []byte(src))
+ if err != nil {
+ t.Fatalf("parse %q: %v", src, err)
+ }
+ return n
+}
+
+func TestCompileGood(t *testing.T) {
+ for _, src := range []string{
+ ``,
+ `(type pdf)`,
+ `(type document image tar.gz)`,
+ `(and (type pdf) (or (content "acme ltd" "0000000000") (name "\bacme\b")) (not (name "^draft")))`,
+ `(path "^Work/") (size >= 10M) (age < 2w) (matched)`,
+ `(duplicate) (duplicate "Work" "~/docs")`,
+ `(not (not (type pdf)))`,
+ } {
+ if _, errs := Compile("d.conf", nodes(t, src), Options{IgnoreCase: true, Fold: true}); len(errs) > 0 {
+ t.Errorf("%s: %v", src, errs)
+ }
+ }
+}
+
+func TestCompileErrors(t *testing.T) {
+ tests := []struct{ src, want string }{
+ {`pdf`, `d.conf:1:1: a condition is a form like (type pdf), not pdf`},
+ {`("type" pdf)`, `d.conf:1:1: a condition is a form like (type pdf), not ("type" pdf)`},
+ {`(foo 1)`, `d.conf:1:1: unknown test (foo ...); tests are type, name, path, content, size, age, duplicate, matched, and, or, not`},
+ {`(and)`, `d.conf:1:1: and needs at least one condition`},
+ {`(or)`, `d.conf:1:1: or needs at least one condition`},
+ {`(not)`, `d.conf:1:1: not takes exactly one condition`},
+ {`(not (type pdf) (type doc))`, `d.conf:1:1: not takes exactly one condition`},
+ {`(type)`, `d.conf:1:1: type needs at least one extension or group, like (type pdf)`},
+ {`(type "pdf")`, `d.conf:1:7: type names are bare words: write (type pdf)`},
+ {`(name)`, `d.conf:1:1: name needs at least one regex in quotes`},
+ {`(path)`, `d.conf:1:1: path needs at least one regex in quotes`},
+ {`(name x)`, `d.conf:1:7: name takes regexes in quotes: write (name "x")`},
+ {`(name "(abc")`, `d.conf:1:7: name: bad regex "(abc": missing closing )`},
+ {`(path "[z-a]")`, `d.conf:1:7: path: bad regex "[z-a]": invalid character class range`},
+ {`(content)`, `d.conf:1:1: content needs at least one keyword in quotes`},
+ {`(content acme)`, `d.conf:1:10: content takes keywords in quotes: write (content "acme")`},
+ {`(content " ")`, `d.conf:1:10: content keyword is empty`},
+ {`(size 10M)`, `d.conf:1:1: size takes an operator and a size, like (size > 10M)`},
+ {`(age 30d)`, `d.conf:1:1: age takes an operator and a duration, like (age > 30d)`},
+ {`(size >> 10M)`, `d.conf:1:7: size operator is one of > >= < <= =, not >>`},
+ {`(size > 10Q)`, `d.conf:1:9: size: bad size "10Q": want a whole number with an optional K, M, G or T, like 50M`},
+ {`(age > 30)`, `d.conf:1:8: age: bad duration "30": want a whole number followed by s, m, h, d or w, like 30d`},
+ {`(duplicate Work)`, `d.conf:1:12: duplicate takes directories in quotes: write (duplicate "Work")`},
+ {`(matched x)`, `d.conf:1:1: matched takes nothing: write (matched)`},
+ }
+ for _, tt := range tests {
+ _, errs := Compile("d.conf", nodes(t, tt.src), Options{IgnoreCase: true})
+ if len(errs) != 1 || errs[0].Error() != tt.want {
+ t.Errorf("%s:\n got %v\n want %s", tt.src, errs, tt.want)
+ }
+ }
+}
+
+func TestCompileCollectsAllErrors(t *testing.T) {
+ _, errs := Compile("d.conf", nodes(t, `(type "a") (size 1) (matched x)`), Options{})
+ if len(errs) != 3 {
+ t.Fatalf("got %d errors: %v", len(errs), errs)
+ }
+}
+
+func TestDepthLimit(t *testing.T) {
+ deep := func(n int) string { return strings.Repeat("(not ", n) + "(type pdf)" + strings.Repeat(")", n) }
+ if _, errs := Compile("d.conf", nodes(t, deep(63)), Options{}); len(errs) != 0 {
+ t.Fatalf("64 levels rejected: %v", errs)
+ }
+ _, errs := Compile("d.conf", nodes(t, deep(64)), Options{})
+ if len(errs) != 1 || !strings.HasSuffix(errs[0].Error(), "conditions nest deeper than 64 levels") {
+ t.Fatalf("65 levels: %v", errs)
+ }
+}
+
+func TestFlagsAndDupDirs(t *testing.T) {
+ c, _ := Compile("d.conf", nodes(t, `(or (type pdf) (content "x")) (duplicate) (duplicate "Work" "~/docs")`), Options{})
+ if !c.UsesContent {
+ t.Error("UsesContent not set")
+ }
+ if want := [][]string{{}, {"Work", "~/docs"}}; !reflect.DeepEqual(c.DupDirs, want) {
+ t.Errorf("DupDirs = %#v, want %#v", c.DupDirs, want)
+ }
+ c, _ = Compile("d.conf", nodes(t, `(type pdf)`), Options{})
+ if c.UsesContent || len(c.DupDirs) != 0 {
+ t.Errorf("flags set without content/duplicate tests: %+v", c)
+ }
+}
+
+func TestCompileErrorReturnsNilCond(t *testing.T) {
+ c, errs := Compile("d.conf", nodes(t, `(type "pdf")`), Options{})
+ if len(errs) != 1 || c != nil {
+ t.Fatalf("got c=%v errs=%v, want c=nil and exactly one error", c, errs)
+ }
+}
+
+// TestEmptyChildrenGuard is a regression test for combine's empty-children
+// guard: an and/or all of whose children failed to compile must not become
+// a hollow node that evaluates vacuously (and -> true). Without the guard,
+// the inner (and (name "[")) would compile to an empty and, evaluate to
+// true, and the outer or would match on any file.
+func TestEmptyChildrenGuard(t *testing.T) {
+ c, errs := compile("d.conf", nodes(t, `(or (type pdf) (and (name "[")))`), Options{}, true)
+ if len(errs) != 1 {
+ t.Fatalf("got %d errors, want 1: %v", len(errs), errs)
+ }
+ if c.Eval(&fake{name: "x.txt"}).Match {
+ t.Error("hollow and inside or vacuously matched")
+ }
+}
+
+func TestGroupsMatchSpec(t *testing.T) {
+ want := map[string]string{
+ "image": "jpg jpeg png gif webp bmp tif tiff heic heif avif svg ico raw cr2 nef arw dng",
+ "video": "mp4 mkv webm mov avi m4v mpg mpeg wmv flv 3gp",
+ "audio": "mp3 flac ogg opus m4a aac wav wma aiff",
+ "archive": "zip tar gz tgz bz2 tbz2 xz txz zst 7z rar lz lzma cpio",
+ "document": "pdf doc docx odt rtf txt md tex",
+ "spreadsheet": "xls xlsx ods csv tsv",
+ "presentation": "ppt pptx odp",
+ "ebook": "epub mobi azw azw3 fb2 djvu",
+ "code": "go c h cpp hpp py sh js ts rs java rb pl lua html css json yaml yml toml xml sql",
+ "text": "txt md log csv tsv json yaml yml toml xml ini conf",
+ "package": "deb rpm apk appimage exe msi flatpak snap",
+ "font": "ttf otf woff woff2",
+ }
+ if len(groups) != len(want) {
+ t.Fatalf("%d groups, want %d", len(groups), len(want))
+ }
+ for g, exts := range want {
+ if got := strings.Join(groups[g], " "); got != exts {
+ t.Errorf("group %s = %q, want %q", g, got, exts)
+ }
+ }
+}
diff --git a/internal/cond/eval.go b/internal/cond/eval.go
new file mode 100644
index 0000000..e3093af
--- /dev/null
+++ b/internal/cond/eval.go
@@ -0,0 +1,302 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package cond
+
+import (
+ "fmt"
+ "io"
+ "strings"
+ "time"
+
+ "krino/internal/norm"
+)
+
+// Facts is what a condition may ask about one file. Implementations
+// memoise: Eval and Explain may ask the same question more than once.
+type Facts interface {
+ Name() string // base name
+ Rel() string // slash path relative to the root
+ Size() int64
+ ModTime() time.Time
+ Now() time.Time
+ Content(ignoreCase, fold bool) (string, error) // normalised with norm.Text
+ Duplicate(dirs []string) (original string, ok bool, err error)
+ Matched() bool // an earlier rule matched this file
+}
+
+// Result is the outcome of evaluating a Cond against one file's Facts.
+type Result struct {
+ Match bool
+ Captures []string // submatches of the first true, non-negated name test: [0] whole match, [1:] groups
+ Reasons []string // what made it true, e.g. `type pdf`, `content "acme ltd"`, `name "\bacme\b"`
+ Warnings []string // e.g. `content unreadable: needs pdftotext, not installed`
+}
+
+// Trace is the full evaluation of every node, for krino explain.
+type Trace struct {
+ Label string
+ Value bool
+ Err string
+ Children []*Trace
+}
+
+// evalCtx accumulates state across one Eval call: the captures of the
+// first true, non-negated name test, and warnings de-duplicated in
+// first-seen order.
+type evalCtx struct {
+ captures []string
+ warned map[string]bool
+ warnings []string
+}
+
+// warn records msg unless it has already been recorded.
+func (ctx *evalCtx) warn(msg string) {
+ if msg == "" {
+ return
+ }
+ if ctx.warned == nil {
+ ctx.warned = map[string]bool{}
+ }
+ if ctx.warned[msg] {
+ return
+ }
+ ctx.warned[msg] = true
+ ctx.warnings = append(ctx.warnings, msg)
+}
+
+// Eval evaluates c against f. and/or short-circuit in (cost-sorted) order,
+// cheapest tests first, so an unreadable or slow test may never run.
+func (c *Cond) Eval(f Facts) Result {
+ if c.root == nil {
+ return Result{Match: true, Reasons: []string{"no condition"}}
+ }
+ ctx := &evalCtx{}
+ match, reasons := c.eval(c.root, f, ctx, false)
+ return Result{Match: match, Captures: ctx.captures, Reasons: reasons, Warnings: ctx.warnings}
+}
+
+// eval evaluates one node against f, short-circuiting and/or in child
+// (cost-sorted) order. negated tracks whether n is reached under an odd
+// number of enclosing nots, so a matching name test found there does not
+// supply Result.Captures.
+func (c *Cond) eval(n *node, f Facts, ctx *evalCtx, negated bool) (bool, []string) {
+ switch n.kind {
+ case kAnd:
+ var reasons []string
+ for _, ch := range n.children {
+ ok, r := c.eval(ch, f, ctx, negated)
+ if !ok {
+ return false, nil
+ }
+ reasons = append(reasons, r...)
+ }
+ return true, reasons
+ case kOr:
+ for _, ch := range n.children {
+ if ok, r := c.eval(ch, f, ctx, negated); ok {
+ return true, r
+ }
+ }
+ return false, nil
+ case kNot:
+ child := n.children[0]
+ ok, _ := c.eval(child, f, ctx, !negated)
+ if ok {
+ return false, nil
+ }
+ // E4: a negated leaf reads fine as "not " plus the leaf's own
+ // label ("not matched", "not type pdf"), but a negated and/or's
+ // bare label is just the word "and"/"or" - "not and"/"not or"
+ // reaches the user in the reasons column reading as nothing a
+ // person would write, so it is parenthesised instead, the way the
+ // config itself would write a negated combinator.
+ label := child.label
+ if child.kind == kAnd || child.kind == kOr {
+ label = "(" + child.label + " ...)"
+ }
+ return true, []string{"not " + label}
+ default:
+ ok, reason, warn, caps := c.evalLeaf(n, f)
+ if warn != "" {
+ ctx.warn(warn)
+ }
+ if !ok {
+ return false, nil
+ }
+ if caps != nil && !negated && ctx.captures == nil {
+ ctx.captures = caps
+ }
+ return true, []string{reason}
+ }
+}
+
+// evalLeaf evaluates one leaf (non-combinator) node against f: whether it
+// matched, its reason if so, a warning if a fact could not be read (only
+// content and duplicate can fail), and (for a matching name test) its
+// regex captures.
+func (c *Cond) evalLeaf(n *node, f Facts) (ok bool, reason, warn string, caps []string) {
+ switch n.kind {
+ case kType:
+ lower := strings.ToLower(f.Name())
+ for _, suf := range n.suffixes {
+ if strings.HasSuffix(lower, suf) {
+ return true, "type " + strings.TrimPrefix(suf, "."), "", nil
+ }
+ }
+ return false, "", "", nil
+
+ case kName, kPath:
+ subj := f.Name()
+ word := "name"
+ if n.kind == kPath {
+ subj = f.Rel()
+ word = "path"
+ }
+ subj = norm.Name(subj, c.opt.Fold)
+ for _, p := range n.patterns {
+ m := p.re.FindStringSubmatch(subj)
+ if m == nil {
+ continue
+ }
+ reason = word + ` "` + p.src + `"`
+ if n.kind == kName {
+ return true, reason, "", m
+ }
+ return true, reason, "", nil
+ }
+ return false, "", "", nil
+
+ case kContent:
+ text, err := f.Content(c.opt.IgnoreCase, c.opt.Fold)
+ if err != nil {
+ return false, "", "content unreadable: " + err.Error(), nil
+ }
+ for _, kw := range n.keywords {
+ if strings.Contains(text, kw.norm) {
+ return true, `content "` + kw.src + `"`, "", nil
+ }
+ }
+ return false, "", "", nil
+
+ case kSize:
+ if compareInt64(f.Size(), n.op, n.sizeVal) {
+ return true, n.label, "", nil
+ }
+ return false, "", "", nil
+
+ case kAge:
+ if compareDuration(f.Now().Sub(f.ModTime()), n.op, n.ageVal) {
+ return true, n.label, "", nil
+ }
+ return false, "", "", nil
+
+ case kDuplicate:
+ orig, dup, err := f.Duplicate(n.dirs)
+ if err != nil {
+ return false, "", "duplicate check failed: " + err.Error(), nil
+ }
+ if dup {
+ return true, "duplicate of " + orig, "", nil
+ }
+ return false, "", "", nil
+
+ case kMatched:
+ if f.Matched() {
+ return true, "matched", "", nil
+ }
+ return false, "", "", nil
+ }
+ return false, "", "", nil
+}
+
+// compareInt64 applies a size comparison operator (one of > >= < <= =).
+func compareInt64(v int64, op string, want int64) bool {
+ switch op {
+ case ">":
+ return v > want
+ case ">=":
+ return v >= want
+ case "<":
+ return v < want
+ case "<=":
+ return v <= want
+ case "=":
+ return v == want
+ }
+ return false
+}
+
+// compareDuration applies an age comparison operator (one of > >= < <= =).
+func compareDuration(v time.Duration, op string, want time.Duration) bool {
+ switch op {
+ case ">":
+ return v > want
+ case ">=":
+ return v >= want
+ case "<":
+ return v < want
+ case "<=":
+ return v <= want
+ case "=":
+ return v == want
+ }
+ return false
+}
+
+// Explain evaluates every node of c against f with no short-circuit,
+// building the full trace for krino explain.
+func (c *Cond) Explain(f Facts) *Trace {
+ if c.root == nil {
+ return &Trace{Label: "no condition", Value: true}
+ }
+ return c.explain(c.root, f)
+}
+
+// explain visits n and, for and/or/not, every child, in (cost-sorted)
+// order, always - unlike eval, it never short-circuits.
+func (c *Cond) explain(n *node, f Facts) *Trace {
+ switch n.kind {
+ case kAnd, kOr:
+ t := &Trace{Label: n.label}
+ val := n.kind == kAnd // identity: and starts true, or starts false
+ for _, ch := range n.children {
+ ct := c.explain(ch, f)
+ t.Children = append(t.Children, ct)
+ if n.kind == kAnd {
+ val = val && ct.Value
+ } else {
+ val = val || ct.Value
+ }
+ }
+ t.Value = val
+ return t
+ case kNot:
+ ct := c.explain(n.children[0], f)
+ return &Trace{Label: n.label, Value: !ct.Value, Children: []*Trace{ct}}
+ default:
+ ok, _, warn, _ := c.evalLeaf(n, f)
+ return &Trace{Label: n.label, Value: ok, Err: warn}
+ }
+}
+
+// Format writes one line per node: "yes"/"no " padded to three, two
+// spaces, two spaces of indent per depth, the label, and " (Err)" when
+// Err is set.
+func (t *Trace) Format(w io.Writer) {
+ t.format(w, 0)
+}
+
+func (t *Trace) format(w io.Writer, depth int) {
+ word := "no"
+ if t.Value {
+ word = "yes"
+ }
+ fmt.Fprintf(w, "%-3s %s%s", word, strings.Repeat(" ", depth), t.Label)
+ if t.Err != "" {
+ fmt.Fprintf(w, " (%s)", t.Err)
+ }
+ fmt.Fprint(w, "\n")
+ for _, ch := range t.Children {
+ ch.format(w, depth+1)
+ }
+}
diff --git a/internal/cond/eval_test.go b/internal/cond/eval_test.go
new file mode 100644
index 0000000..3607978
--- /dev/null
+++ b/internal/cond/eval_test.go
@@ -0,0 +1,222 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package cond
+
+import (
+ "errors"
+ "math/rand"
+ "reflect"
+ "strings"
+ "testing"
+ "time"
+
+ "krino/internal/norm"
+)
+
+var now = time.Date(2026, 9, 11, 12, 0, 0, 0, time.UTC)
+
+type fake struct {
+ name, rel, raw string
+ rawErr error
+ size int64
+ age time.Duration
+ matched bool
+ dupOrig string
+ dupOK bool
+ contentCalls int
+}
+
+func (f *fake) Name() string { return f.name }
+func (f *fake) Rel() string {
+ if f.rel != "" {
+ return f.rel
+ }
+ return f.name
+}
+func (f *fake) Size() int64 { return f.size }
+func (f *fake) ModTime() time.Time { return now.Add(-f.age) }
+func (f *fake) Now() time.Time { return now }
+func (f *fake) Matched() bool { return f.matched }
+func (f *fake) Content(ic, fold bool) (string, error) {
+ f.contentCalls++
+ if f.rawErr != nil {
+ return "", f.rawErr
+ }
+ return norm.Text(f.raw, ic, fold), nil
+}
+func (f *fake) Duplicate(dirs []string) (string, bool, error) { return f.dupOrig, f.dupOK, nil }
+
+func eval(t *testing.T, src string, opt Options, f Facts) Result {
+ t.Helper()
+ c, errs := Compile("d.conf", nodes(t, src), opt)
+ if len(errs) > 0 {
+ t.Fatalf("%s: %v", src, errs)
+ }
+ return c.Eval(f)
+}
+
+func TestTruth(t *testing.T) {
+ ic := Options{IgnoreCase: true}
+ pdf := &fake{name: "Scan001.PDF", raw: "Invoice from ACME LTD", size: 20 << 20, age: 40 * 24 * time.Hour}
+ tests := []struct {
+ src string
+ want bool
+ }{
+ {``, true},
+ {`(type pdf)`, true},
+ {`(type document)`, true},
+ {`(type jpg)`, false},
+ {`(type pdf) (size > 10M)`, true},
+ {`(type pdf) (size < 10M)`, false},
+ {`(or (type jpg) (content "acme ltd"))`, true},
+ {`(not (content "acme ltd"))`, false},
+ {`(age > 30d) (age <= 41d)`, true},
+ {`(name "^scan\d+")`, true},
+ {`(name "(?-i)^scan")`, false},
+ {`(matched)`, false},
+ {`(and (type pdf) (or (name "^x") (not (name "^y"))))`, true},
+ }
+ for _, tt := range tests {
+ if got := eval(t, tt.src, ic, pdf); got.Match != tt.want {
+ t.Errorf("%s = %v, want %v (%+v)", tt.src, got.Match, tt.want, got)
+ }
+ }
+}
+
+func TestCaseAndFold(t *testing.T) {
+ f := &fake{name: "SPOLKA-umowa.pdf", raw: "Umowa: spółka z o.o."}
+ if !eval(t, `(name "spółka")`, Options{IgnoreCase: true, Fold: true}, f).Match {
+ t.Error("folded, case-ignoring name did not match")
+ }
+ if eval(t, `(name "spółka")`, Options{IgnoreCase: true, Fold: false}, f).Match {
+ t.Error("matched without folding")
+ }
+ if !eval(t, `(content "SPOLKA Z O.O.")`, Options{IgnoreCase: true, Fold: true}, f).Match {
+ t.Error("folded content did not match")
+ }
+ img := &fake{name: "IMG_0001.jpg"}
+ if eval(t, `(name "^img")`, Options{IgnoreCase: false}, img).Match {
+ t.Error("strict case matched")
+ }
+ if !eval(t, `(name "(?i)^img")`, Options{IgnoreCase: false}, img).Match {
+ t.Error("inline (?i) did not override strict case")
+ }
+}
+
+func TestReasonsAndCaptures(t *testing.T) {
+ f := &fake{name: "Screenshot_20260911.png", raw: "acme ltd"}
+ r := eval(t, `(type image) (name "^Screenshot_(\d{4})(\d{2})") (not (name "^x(y)"))`, Options{IgnoreCase: true}, f)
+ if !r.Match {
+ t.Fatal("no match")
+ }
+ if want := []string{"Screenshot_202609", "2026", "09"}; !reflect.DeepEqual(r.Captures, want) {
+ t.Errorf("captures = %q, want %q", r.Captures, want)
+ }
+ if want := []string{`type png`, `name "^Screenshot_(\d{4})(\d{2})"`, `not name "^x(y)"`}; !reflect.DeepEqual(r.Reasons, want) {
+ t.Errorf("reasons = %q, want %q", r.Reasons, want)
+ }
+ r = eval(t, `(or (content "nope" "acme ltd") (type png))`, Options{IgnoreCase: true}, f)
+ if want := []string{`type png`}; !reflect.DeepEqual(r.Reasons, want) {
+ t.Errorf("or reasons = %q, want %q (cheapest true child)", r.Reasons, want)
+ }
+ r = eval(t, `(content "nope" "acme ltd")`, Options{IgnoreCase: true}, f)
+ if want := []string{`content "acme ltd"`}; !reflect.DeepEqual(r.Reasons, want) {
+ t.Errorf("content reasons = %q, want %q", r.Reasons, want)
+ }
+ if r := eval(t, ``, Options{}, f); !reflect.DeepEqual(r.Reasons, []string{"no condition"}) {
+ t.Errorf("empty reasons = %q", r.Reasons)
+ }
+ d := &fake{name: "report (1).pdf", dupOrig: "report.pdf", dupOK: true}
+ if r := eval(t, `(duplicate)`, Options{}, d); !r.Match || r.Reasons[0] != "duplicate of report.pdf" {
+ t.Errorf("duplicate = %+v", r)
+ }
+}
+
+func TestCheapFirstAvoidsContent(t *testing.T) {
+ f := &fake{name: "a.txt", raw: "x"}
+ eval(t, `(and (content "x") (type pdf))`, Options{}, f)
+ if f.contentCalls != 0 {
+ t.Errorf("content read although type was false (%d calls)", f.contentCalls)
+ }
+ g := &fake{name: "a.pdf", raw: "x"}
+ eval(t, `(or (content "x") (type pdf))`, Options{}, g)
+ if g.contentCalls != 0 {
+ t.Errorf("content read although type was true (%d calls)", g.contentCalls)
+ }
+}
+
+func TestContentErrorWarns(t *testing.T) {
+ f := &fake{name: "a.pdf", rawErr: errors.New("needs pdftotext, not installed")}
+ r := eval(t, `(or (content "acme") (content "other"))`, Options{}, f)
+ if r.Match {
+ t.Fatal("matched unreadable content")
+ }
+ if want := []string{"content unreadable: needs pdftotext, not installed"}; !reflect.DeepEqual(r.Warnings, want) {
+ t.Errorf("warnings = %q, want %q (de-duplicated)", r.Warnings, want)
+ }
+}
+
+func TestExplainFormat(t *testing.T) {
+ f := &fake{name: "scan.pdf", rawErr: errors.New("needs pdftotext, not installed")}
+ c, _ := Compile("d.conf", nodes(t, `(type pdf) (or (content "acme ltd") (name "\bacme\b"))`), Options{IgnoreCase: true})
+ var b strings.Builder
+ c.Explain(f).Format(&b)
+ want := "no and\n" +
+ "yes type pdf\n" +
+ "no or\n" +
+ "no name \"\\bacme\\b\"\n" +
+ "no content \"acme ltd\" (content unreadable: needs pdftotext, not installed)\n"
+ if b.String() != want {
+ t.Fatalf("got\n%s\nwant\n%s", b.String(), want)
+ }
+}
+
+// TestReorderKeepsMeaning: random trees give the same answer with and without
+// cost reordering.
+func TestReorderKeepsMeaning(t *testing.T) {
+ leaves := []string{`(type pdf)`, `(type txt)`, `(name "^a")`, `(name "b$")`, `(size > 10)`, `(size < 5)`,
+ `(age > 1d)`, `(content "x")`, `(content "y")`, `(matched)`}
+ rng := rand.New(rand.NewSource(1))
+ var gen func(depth int) string
+ gen = func(depth int) string {
+ if depth == 0 || rng.Intn(3) == 0 {
+ return leaves[rng.Intn(len(leaves))]
+ }
+ switch rng.Intn(3) {
+ case 0:
+ return "(not " + gen(depth-1) + ")"
+ case 1:
+ return "(and " + gen(depth-1) + " " + gen(depth-1) + ")"
+ default:
+ return "(or " + gen(depth-1) + " " + gen(depth-1) + " " + gen(depth-1) + ")"
+ }
+ }
+ names := []string{"a.pdf", "b.txt", "ab.pdf", "c.jpg"}
+ for i := 0; i < 500; i++ {
+ src := gen(4)
+ f := &fake{name: names[rng.Intn(len(names))], raw: []string{"x", "y", "xy", ""}[rng.Intn(4)],
+ size: int64(rng.Intn(20)), age: time.Duration(rng.Intn(72)) * time.Hour, matched: rng.Intn(2) == 0}
+ a, _ := compile("d.conf", nodes(t, src), Options{}, true)
+ b, _ := compile("d.conf", nodes(t, src), Options{}, false)
+ if ra, rb := a.Eval(f), b.Eval(f); ra.Match != rb.Match {
+ t.Fatalf("%s on %+v: reordered %v, original %v", src, f, ra.Match, rb.Match)
+ }
+ }
+}
+
+// TestNegatedCombinatorReason: E4. Negating a combinator must read as
+// something a person would write ("not (and ...)" / "not (or ...)"), not
+// the bare "not and" / "not or"; a negated leaf keeps its own label
+// unchanged ("not matched").
+func TestNegatedCombinatorReason(t *testing.T) {
+ f := &fake{name: "a.pdf"}
+ if r := eval(t, `(not (and (type pdf) (matched)))`, Options{}, f); !r.Match || r.Reasons[0] != "not (and ...)" {
+ t.Errorf("negated and = %+v, want reason %q", r, "not (and ...)")
+ }
+ if r := eval(t, `(not (or (matched) (name "^zzz")))`, Options{}, f); !r.Match || r.Reasons[0] != "not (or ...)" {
+ t.Errorf("negated or = %+v, want reason %q", r, "not (or ...)")
+ }
+ if r := eval(t, `(not (matched))`, Options{}, f); !r.Match || r.Reasons[0] != "not matched" {
+ t.Errorf("negated leaf = %+v, want its own label unchanged: %q", r, "not matched")
+ }
+}
diff --git a/internal/cond/types.go b/internal/cond/types.go
new file mode 100644
index 0000000..81c2f45
--- /dev/null
+++ b/internal/cond/types.go
@@ -0,0 +1,111 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+// Package cond compiles the s-expression conditions of a rule's (when ...)
+// into a tree that Task 8's evaluator walks against one file's facts.
+package cond
+
+import (
+ "regexp"
+ "time"
+
+ "krino/internal/sexp"
+)
+
+// Options carries a rule's resolved case and fold settings into compilation.
+type Options struct {
+ IgnoreCase bool // the rule's resolved case setting is "ignore"
+ Fold bool // the rule's resolved fold setting
+}
+
+// Cond is a compiled condition. A Cond compiled from no conditions (a rule
+// without when) is always true.
+type Cond struct {
+ root *node // nil: always true
+ opt Options // the case/fold settings conditions were compiled with; Task 8 needs them again at eval time
+ UsesContent bool // some content test exists
+ DupDirs [][]string // the raw directory arguments of each duplicate test, in order
+}
+
+// kind is what a compiled node tests, or how it combines its children.
+type kind int
+
+const (
+ kAnd kind = iota
+ kOr
+ kNot
+ kType
+ kName
+ kPath
+ kContent
+ kSize
+ kAge
+ kDuplicate
+ kMatched
+)
+
+// Costs, per the brief's cost order: cheapest first when sorting and/or
+// children. not takes its child's cost; and/or take the sum of theirs.
+const (
+ costCheap = 1 // type, size, age, matched
+ costRegex = 2 // name, path
+ costDuplicate = 5
+ costContent = 10
+)
+
+// pattern is one name/path regex, compiled and paired with the text it was
+// written as (undecorated by (?i) or folding), for labels and reasons.
+type pattern struct {
+ re *regexp.Regexp
+ src string
+}
+
+// keyword is one content keyword, normalised for matching and paired with
+// the text it was written as, for labels and reasons.
+type keyword struct {
+ norm string
+ src string
+}
+
+// node is one compiled condition: a leaf test, or an and/or/not combinator
+// over other nodes. Task 8 evaluates this tree.
+type node struct {
+ kind kind
+ pos sexp.Pos // the position of the node as written, for diagnostics
+ label string // the test as written, e.g. `content "acme ltd" "0000000000"`
+ cost int // this node's evaluation cost; and/or sort children by it
+ children []*node // and, or, not
+
+ // type
+ suffixes []string // leading-dot, lower-case, e.g. ".pdf"
+
+ // name, path
+ patterns []pattern
+
+ // content
+ keywords []keyword
+
+ // size, age (kind tells which is populated)
+ op string
+ sizeVal int64
+ ageVal time.Duration
+
+ // duplicate
+ dirs []string
+}
+
+// groups maps a (type ...) group name to the extensions it expands to,
+// spec Appendix A.
+var groups = map[string][]string{
+ "image": {"jpg", "jpeg", "png", "gif", "webp", "bmp", "tif", "tiff", "heic", "heif", "avif", "svg", "ico", "raw", "cr2", "nef", "arw", "dng"},
+ "video": {"mp4", "mkv", "webm", "mov", "avi", "m4v", "mpg", "mpeg", "wmv", "flv", "3gp"},
+ "audio": {"mp3", "flac", "ogg", "opus", "m4a", "aac", "wav", "wma", "aiff"},
+ "archive": {"zip", "tar", "gz", "tgz", "bz2", "tbz2", "xz", "txz", "zst", "7z", "rar", "lz", "lzma", "cpio"},
+ "document": {"pdf", "doc", "docx", "odt", "rtf", "txt", "md", "tex"},
+ "spreadsheet": {"xls", "xlsx", "ods", "csv", "tsv"},
+ "presentation": {"ppt", "pptx", "odp"},
+ "ebook": {"epub", "mobi", "azw", "azw3", "fb2", "djvu"},
+ "code": {"go", "c", "h", "cpp", "hpp", "py", "sh", "js", "ts", "rs", "java", "rb", "pl", "lua", "html", "css", "json", "yaml", "yml", "toml", "xml", "sql"},
+ "text": {"txt", "md", "log", "csv", "tsv", "json", "yaml", "yml", "toml", "xml", "ini", "conf"},
+ "package": {"deb", "rpm", "apk", "appimage", "exe", "msi", "flatpak", "snap"},
+ "font": {"ttf", "otf", "woff", "woff2"},
+}
diff --git a/internal/dup/dup.go b/internal/dup/dup.go
new file mode 100644
index 0000000..502ef31
--- /dev/null
+++ b/internal/dup/dup.go
@@ -0,0 +1,398 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+// Package dup answers "is this file a duplicate, and of which original?"
+// cheaply: candidates are grouped by size, and only hashed when sizes
+// collide — a partial hash first, a full hash only on a partial collision.
+package dup
+
+import (
+ "crypto/sha256"
+ "encoding/binary"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "sync"
+ "time"
+
+ "krino/internal/scan"
+)
+
+// partialChunk is the size of the head and tail read for the partial hash.
+const partialChunk = 64 << 10 // 64 KiB
+
+// candidate is one file the index knows about: a scanned file, or a file
+// found under one of the extra directories.
+type candidate struct {
+ path string
+ size int64
+ modTime time.Time
+ name string
+ extra bool // found under one of the extra directories, not scanned
+}
+
+// Index finds files with identical content among the scanned files and,
+// optionally, every regular file under some extra directories.
+type Index struct {
+ candidates []candidate
+ bySize map[int64][]int // size -> indexes into candidates
+ scanned map[string]int // scanned file path -> index into candidates
+
+ mu sync.Mutex
+ partial map[string][sha256.Size]byte // memoised partial hash, by path
+ full map[string][sha256.Size]byte // memoised full hash, by path
+
+ candErrs []CandidateError
+ candErrSeen map[string]bool // path already recorded in candErrs
+}
+
+// CandidateError is one candidate (never the file Lookup was asked about)
+// that could not be hashed, so it was skipped rather than comparing it.
+type CandidateError struct {
+ Path string
+ Err error
+}
+
+func (e CandidateError) Error() string { return e.Path + ": " + e.Err.Error() }
+
+// NewIndex stats the extra directories (recursively, symlinks skipped). A
+// missing or unreadable extra directory is reported in the error list and
+// otherwise ignored; an unreadable subdirectory found while walking an
+// otherwise-readable extra directory adds its own error but does not stop
+// the rest of that directory from being indexed. An extra directory that is
+// itself a symlink (A4) is not followed either — filepath.WalkDir Lstats
+// its root, so left unchecked it would be indexed as silently empty — and
+// is reported in the error list instead. Nothing is hashed yet.
+func NewIndex(files []scan.File, extra []string) (*Index, []error) {
+ x := &Index{
+ bySize: make(map[int64][]int),
+ scanned: make(map[string]int, len(files)),
+ partial: make(map[string][sha256.Size]byte),
+ full: make(map[string][sha256.Size]byte),
+ }
+ for _, f := range files {
+ x.scanned[f.Path] = x.add(candidate{path: f.Path, size: f.Size, modTime: f.ModTime, name: f.Name})
+ }
+
+ var errs []error
+ for _, dir := range extra {
+ errs = append(errs, x.addExtraDir(dir)...)
+ }
+ return x, errs
+}
+
+// add appends c to the candidate list and its size group, and returns its
+// index.
+func (x *Index) add(c candidate) int {
+ idx := len(x.candidates)
+ x.candidates = append(x.candidates, c)
+ x.bySize[c.size] = append(x.bySize[c.size], idx)
+ return idx
+}
+
+// addExtraDir walks dir, adding every regular file found (symlinks, both to
+// files and to directories, are skipped: filepath.WalkDir never follows
+// them, so it is enough not to add or descend into one). An error on dir
+// itself (missing, or unreadable) aborts the walk and is the sole error
+// returned; an unreadable subdirectory deeper in the tree adds one error
+// naming it and the walk continues, so files elsewhere in dir are still
+// indexed. A file whose own Info() fails (A3) is dropped from the index the
+// same way: silently if it has simply vanished (fs.ErrNotExist), otherwise
+// with its own error added to errs.
+func (x *Index) addExtraDir(dir string) []error {
+ var errs []error
+ err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
+ if err != nil {
+ if path == dir {
+ return err
+ }
+ errs = append(errs, fmt.Errorf("%s: %w", path, err))
+ return nil
+ }
+ if path == dir && d.Type()&fs.ModeSymlink != 0 {
+ // filepath.WalkDir Lstats its root: a symlinked extra directory
+ // (A4) would otherwise be silently treated as empty rather than
+ // followed, with no sign anything was wrong.
+ errs = append(errs, fmt.Errorf("%s is a symlink; not followed", path))
+ return nil
+ }
+ if d.Type()&fs.ModeSymlink != 0 || d.IsDir() || !d.Type().IsRegular() {
+ return nil
+ }
+ x.addEntry(path, d, &errs)
+ return nil
+ })
+ if err != nil {
+ errs = append(errs, fmt.Errorf("%s: %w", dir, err))
+ }
+ return errs
+}
+
+// addEntry indexes one regular-file entry found while walking an extra
+// directory: A3, a file whose own Info() fails is dropped from the index —
+// silently if it has simply vanished (fs.ErrNotExist), otherwise with its
+// own error appended to errs. Factored out of addExtraDir's WalkDir
+// callback so a test can drive it directly with a fabricated fs.DirEntry,
+// since filepath.WalkDir gives no way to inject a canned Info() failure on
+// a real walk.
+func (x *Index) addEntry(path string, d fs.DirEntry, errs *[]error) {
+ info, err := d.Info()
+ if err != nil {
+ if !errors.Is(err, fs.ErrNotExist) {
+ *errs = append(*errs, fmt.Errorf("%s: %w", path, err))
+ }
+ return
+ }
+ x.add(candidate{path: path, size: info.Size(), modTime: info.ModTime(), name: d.Name(), extra: true})
+}
+
+// Lookup reports whether path (one of the scanned files) duplicates another
+// file, and which file is the original. Safe for concurrent use.
+func (x *Index) Lookup(path string) (original string, dup bool, err error) {
+ idx, ok := x.scanned[path]
+ if !ok {
+ return "", false, fmt.Errorf("dup: %s was not scanned", path)
+ }
+ c := x.candidates[idx]
+ if c.size == 0 {
+ // Empty files are never duplicates, and are never read.
+ return path, false, nil
+ }
+ group := x.bySize[c.size]
+ if len(group) < 2 {
+ // Alone in its size group: not a duplicate, never read.
+ return path, false, nil
+ }
+
+ identical, err := x.identicalTo(idx, group)
+ if err != nil {
+ return "", false, err
+ }
+ if len(identical) < 2 {
+ return path, false, nil
+ }
+ orig := x.candidates[x.original(identical)].path
+ return orig, orig != path, nil
+}
+
+// identicalTo returns the indexes in group (which all share idx's size,
+// idx included) whose content matches candidates[idx]: same partial hash,
+// then, only for those that collide, the same full hash. idx is the file
+// Lookup was asked about; a failure hashing it propagates, since Lookup can
+// answer nothing without it. A failure hashing any other candidate in group
+// only removes that candidate from consideration: a vanished candidate
+// (errors.Is fs.ErrNotExist) is dropped silently, any other failure is
+// recorded on the Index (see recordCandidateError) so the caller can warn
+// about it once matching is done.
+func (x *Index) identicalTo(idx int, group []int) ([]int, error) {
+ idxPartial, err := x.partialHash(x.candidates[idx].path)
+ if err != nil {
+ return nil, err
+ }
+
+ same := []int{idx}
+ var idxFull [sha256.Size]byte
+ haveIdxFull := false
+ for _, j := range group {
+ if j == idx {
+ continue
+ }
+ jPartial, err := x.partialHash(x.candidates[j].path)
+ if err != nil {
+ x.recordCandidateError(x.candidates[j].path, err)
+ continue
+ }
+ if jPartial != idxPartial {
+ continue
+ }
+ if !haveIdxFull {
+ idxFull, err = x.fullHash(x.candidates[idx].path)
+ if err != nil {
+ return nil, err
+ }
+ haveIdxFull = true
+ }
+ jFull, err := x.fullHash(x.candidates[j].path)
+ if err != nil {
+ x.recordCandidateError(x.candidates[j].path, err)
+ continue
+ }
+ if jFull == idxFull {
+ same = append(same, j)
+ }
+ }
+ return same, nil
+}
+
+// recordCandidateError records that path (never the file Lookup was asked
+// about) could not be hashed and so was skipped, unless it simply vanished
+// (fs.ErrNotExist), which is not worth reporting, or was already recorded.
+// Safe for concurrent use.
+func (x *Index) recordCandidateError(path string, err error) {
+ if errors.Is(err, fs.ErrNotExist) {
+ return
+ }
+ x.mu.Lock()
+ defer x.mu.Unlock()
+ if x.candErrSeen == nil {
+ x.candErrSeen = make(map[string]bool)
+ }
+ if x.candErrSeen[path] {
+ return
+ }
+ x.candErrSeen[path] = true
+ x.candErrs = append(x.candErrs, CandidateError{Path: path, Err: err})
+}
+
+// Errors returns every candidate-hashing failure recorded so far,
+// deduplicated by path, in first-recorded order. Safe for concurrent use.
+func (x *Index) Errors() []CandidateError {
+ x.mu.Lock()
+ defer x.mu.Unlock()
+ return append([]CandidateError(nil), x.candErrs...)
+}
+
+// original picks, among a set of identical candidates, the index that is
+// the original. Spec §5.5: one flat comparison, in order — a file under an
+// extra directory beats one that is not; then the oldest by ModTime; then
+// the shortest base name; then the base name that sorts first; then (the
+// final, always-deterministic tie-break) the full path that sorts first.
+// This one chain applies to every pair alike; extra-vs-extra candidates
+// are not a special case broken by path alone.
+func (x *Index) original(idxs []int) int {
+ best := idxs[0]
+ for _, j := range idxs[1:] {
+ if x.preferred(j, best) {
+ best = j
+ }
+ }
+ return best
+}
+
+// preferred reports whether candidate a should be chosen as the original
+// over candidate b.
+func (x *Index) preferred(a, b int) bool {
+ ca, cb := x.candidates[a], x.candidates[b]
+ if ca.extra != cb.extra {
+ return ca.extra
+ }
+ if !ca.modTime.Equal(cb.modTime) {
+ return ca.modTime.Before(cb.modTime)
+ }
+ if len(ca.name) != len(cb.name) {
+ return len(ca.name) < len(cb.name)
+ }
+ if ca.name != cb.name {
+ return ca.name < cb.name
+ }
+ return ca.path < cb.path
+}
+
+// partialHash returns the memoised partial hash for path, computing and
+// storing it on first use. The hash is computed outside the lock; only the
+// memo access is guarded.
+func (x *Index) partialHash(path string) ([sha256.Size]byte, error) {
+ x.mu.Lock()
+ h, ok := x.partial[path]
+ x.mu.Unlock()
+ if ok {
+ return h, nil
+ }
+ h, err := computePartialHash(path)
+ if err != nil {
+ return h, err
+ }
+ x.mu.Lock()
+ x.partial[path] = h
+ x.mu.Unlock()
+ return h, nil
+}
+
+// fullHash returns the memoised full-file hash for path, computing and
+// storing it on first use.
+func (x *Index) fullHash(path string) ([sha256.Size]byte, error) {
+ x.mu.Lock()
+ h, ok := x.full[path]
+ x.mu.Unlock()
+ if ok {
+ return h, nil
+ }
+ h, err := computeFullHash(path)
+ if err != nil {
+ return h, err
+ }
+ x.mu.Lock()
+ x.full[path] = h
+ x.mu.Unlock()
+ return h, nil
+}
+
+// computePartialHash hashes the file's size, its first 64 KiB and its last
+// 64 KiB (the two overlap, or repeat the whole file, when it is smaller
+// than 64 KiB).
+func computePartialHash(path string) ([sha256.Size]byte, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return [sha256.Size]byte{}, err
+ }
+ defer f.Close()
+ info, err := f.Stat()
+ if err != nil {
+ return [sha256.Size]byte{}, err
+ }
+ size := info.Size()
+
+ h := sha256.New()
+ var sizeBuf [8]byte
+ binary.BigEndian.PutUint64(sizeBuf[:], uint64(size))
+ h.Write(sizeBuf[:])
+
+ head, err := readAt(f, 0)
+ if err != nil {
+ return [sha256.Size]byte{}, err
+ }
+ h.Write(head)
+
+ tailOff := size - partialChunk
+ if tailOff < 0 {
+ tailOff = 0
+ }
+ tail, err := readAt(f, tailOff)
+ if err != nil {
+ return [sha256.Size]byte{}, err
+ }
+ h.Write(tail)
+
+ var out [sha256.Size]byte
+ copy(out[:], h.Sum(nil))
+ return out, nil
+}
+
+// readAt reads up to partialChunk bytes starting at off, without disturbing
+// f's current offset.
+func readAt(f *os.File, off int64) ([]byte, error) {
+ buf := make([]byte, partialChunk)
+ n, err := f.ReadAt(buf, off)
+ if err != nil && err != io.EOF {
+ return nil, err
+ }
+ return buf[:n], nil
+}
+
+// computeFullHash hashes the whole file.
+func computeFullHash(path string) ([sha256.Size]byte, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return [sha256.Size]byte{}, err
+ }
+ defer f.Close()
+ h := sha256.New()
+ if _, err := io.Copy(h, f); err != nil {
+ return [sha256.Size]byte{}, err
+ }
+ var out [sha256.Size]byte
+ copy(out[:], h.Sum(nil))
+ return out, nil
+}
diff --git a/internal/dup/dup_test.go b/internal/dup/dup_test.go
new file mode 100644
index 0000000..fb4e64d
--- /dev/null
+++ b/internal/dup/dup_test.go
@@ -0,0 +1,331 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package dup
+
+import (
+ "bytes"
+ "errors"
+ "fmt"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "krino/internal/scan"
+)
+
+var base = time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
+
+// put writes content at dir/name with an mtime `age` hours after base.
+func put(t *testing.T, dir, name string, content []byte, hours int) scan.File {
+ t.Helper()
+ p := filepath.Join(dir, name)
+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(p, content, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ mt := base.Add(time.Duration(hours) * time.Hour)
+ if err := os.Chtimes(p, mt, mt); err != nil {
+ t.Fatal(err)
+ }
+ return scan.File{Path: p, Rel: name, Name: filepath.Base(name), Size: int64(len(content)), ModTime: mt}
+}
+
+func lookup(t *testing.T, x *Index, f scan.File) (string, bool) {
+ t.Helper()
+ orig, dup, err := x.Lookup(f.Path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return orig, dup
+}
+
+func TestDuplicatesInScan(t *testing.T) {
+ d := t.TempDir()
+ a := put(t, d, "report.pdf", []byte("same content"), 1)
+ b := put(t, d, "report (1).pdf", []byte("same content"), 5)
+ c := put(t, d, "other.pdf", []byte("diff content"), 0) // same size, different bytes
+ e1 := put(t, d, "empty1", nil, 0)
+ e2 := put(t, d, "empty2", nil, 1)
+ x, errs := NewIndex([]scan.File{a, b, c, e1, e2}, nil)
+ if len(errs) > 0 {
+ t.Fatal(errs)
+ }
+ if orig, dup := lookup(t, x, b); !dup || orig != a.Path {
+ t.Errorf("copy: dup=%v orig=%s, want dup of %s", dup, orig, a.Path)
+ }
+ if _, dup := lookup(t, x, a); dup {
+ t.Error("the original reported as a duplicate")
+ }
+ if _, dup := lookup(t, x, c); dup {
+ t.Error("same size, different content reported as a duplicate")
+ }
+ if _, dup := lookup(t, x, e2); dup {
+ t.Error("empty files reported as duplicates")
+ }
+}
+
+func TestExtraDirHoldsTheOriginal(t *testing.T) {
+ scanned, filed := t.TempDir(), t.TempDir()
+ dl := put(t, scanned, "invoice.pdf", []byte("invoice 42"), 0) // older than the filed copy
+ put(t, filed, "2026/invoice-42.pdf", []byte("invoice 42"), 9)
+ x, errs := NewIndex([]scan.File{dl}, []string{filed, filepath.Join(filed, "missing")})
+ if len(errs) != 1 {
+ t.Errorf("want one error for the missing extra dir, got %v", errs)
+ }
+ if orig, dup := lookup(t, x, dl); !dup || orig != filepath.Join(filed, "2026/invoice-42.pdf") {
+ t.Errorf("dup=%v orig=%s, want the filed copy as original", dup, orig)
+ }
+}
+
+func TestTieBreaks(t *testing.T) {
+ d := t.TempDir()
+ long := put(t, d, "longer-name.txt", []byte("x"), 0)
+ short := put(t, d, "b.txt", []byte("x"), 0)
+ same := put(t, d, "a.txt", []byte("x"), 0)
+ x, _ := NewIndex([]scan.File{long, short, same}, nil)
+ for _, f := range []scan.File{long, short} {
+ if orig, dup := lookup(t, x, f); !dup || orig != same.Path {
+ t.Errorf("%s: dup=%v orig=%s, want %s (shortest name, then path order)", f.Name, dup, orig, same.Path)
+ }
+ }
+}
+
+func TestPartialHashCollisionResolvedByFullHash(t *testing.T) {
+ d := t.TempDir()
+ head, tail := bytes.Repeat([]byte("h"), 70<<10), bytes.Repeat([]byte("t"), 70<<10)
+ one := append(append(append([]byte{}, head...), []byte("MIDDLE-ONE")...), tail...)
+ two := append(append(append([]byte{}, head...), []byte("MIDDLE-TWO")...), tail...)
+ a := put(t, d, "a.bin", one, 0)
+ b := put(t, d, "b.bin", two, 1)
+ x, _ := NewIndex([]scan.File{a, b}, nil)
+ if _, dup := lookup(t, x, b); dup {
+ t.Error("files differing only in the middle reported as duplicates")
+ }
+}
+
+func TestLookupUnknownPath(t *testing.T) {
+ x, _ := NewIndex(nil, nil)
+ if _, _, err := x.Lookup("/nowhere"); err == nil {
+ t.Fatal("no error for a path that was not scanned")
+ }
+}
+
+// TestTieBreakNameBeforePath: same-length names, same mtime, in directories
+// that sort in the opposite order from the names — the base name decides,
+// not the full path (spec §5.5's flat chain, not a path-only fallback).
+func TestTieBreakNameBeforePath(t *testing.T) {
+ d := t.TempDir()
+ catInZzz := put(t, d, "zzz/cat.txt", []byte("x"), 0)
+ dogInAaa := put(t, d, "aaa/dog.txt", []byte("x"), 0)
+ x, _ := NewIndex([]scan.File{catInZzz, dogInAaa}, nil)
+ for _, f := range []scan.File{catInZzz, dogInAaa} {
+ if orig, dup := lookup(t, x, f); orig != catInZzz.Path || dup != (f.Path != catInZzz.Path) {
+ t.Errorf("%s: orig=%s dup=%v, want %s (name sorts before path)", f.Rel, orig, dup, catInZzz.Path)
+ }
+ }
+}
+
+// TestTieBreakExtraVsExtra: two extra directories hold identical copies;
+// the one under the lexically later directory is older and must still win
+// on ModTime — extra-vs-extra ties are not resolved by path alone.
+func TestTieBreakExtraVsExtra(t *testing.T) {
+ common := t.TempDir()
+ aaa, zzz := filepath.Join(common, "aaa"), filepath.Join(common, "zzz")
+ newer := put(t, aaa, "copy.txt", []byte("invoice 42"), 5) // lexically first, newer
+ older := put(t, zzz, "copy.txt", []byte("invoice 42"), 0) // lexically last, older
+ scanned := t.TempDir()
+ dl := put(t, scanned, "download.txt", []byte("invoice 42"), 3)
+ x, errs := NewIndex([]scan.File{dl}, []string{aaa, zzz})
+ if len(errs) != 0 {
+ t.Fatal(errs)
+ }
+ if orig, dup := lookup(t, x, dl); !dup || orig != older.Path {
+ t.Errorf("dup=%v orig=%s, want %s (older extra copy, despite sorting after %s)", dup, orig, older.Path, newer.Path)
+ }
+}
+
+// TestUnreadableSubdirReported: an unreadable subdirectory under an extra
+// directory is reported as its own error, and the rest of that extra
+// directory is still indexed.
+func TestUnreadableSubdirReported(t *testing.T) {
+ if os.Geteuid() == 0 {
+ t.Skip("permissions are not enforced running as root")
+ }
+ filed := t.TempDir()
+ blocked := filepath.Join(filed, "blocked")
+ if err := os.MkdirAll(blocked, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ put(t, blocked, "secret.pdf", []byte("secret 42"), 0)
+ put(t, filed, "visible.pdf", []byte("visible content"), 0)
+ if err := os.Chmod(blocked, 0o000); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() {
+ if err := os.Chmod(blocked, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ })
+
+ scanned := t.TempDir()
+ dl := put(t, scanned, "visible.pdf", []byte("visible content"), 1)
+
+ x, errs := NewIndex([]scan.File{dl}, []string{filed})
+ if len(errs) != 1 || !strings.Contains(errs[0].Error(), blocked) {
+ t.Fatalf("want one error naming %s, got %v", blocked, errs)
+ }
+ if orig, dup := lookup(t, x, dl); !dup || orig != filepath.Join(filed, "visible.pdf") {
+ t.Errorf("dup=%v orig=%s, want the filed copy (visible.pdf still indexed despite the unreadable sibling)", dup, orig)
+ }
+}
+
+// TestUnreadableCandidateSkipped: three files share a size; one of them
+// (not the subject of either Lookup call) is unreadable. A1: the other two
+// are still reported as a duplicate pair, and the unreadable one is
+// recorded exactly once as a candidate error, not returned as a Lookup
+// error.
+func TestUnreadableCandidateSkipped(t *testing.T) {
+ if os.Geteuid() == 0 {
+ t.Skip("permissions are not enforced running as root")
+ }
+ d := t.TempDir()
+ a := put(t, d, "a.bin", []byte("same content"), 0)
+ b := put(t, d, "b.bin", []byte("same content"), 1)
+ c := put(t, d, "c.bin", []byte("diff content"), 2) // same size, different bytes
+ if err := os.Chmod(c.Path, 0o000); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { os.Chmod(c.Path, 0o644) })
+
+ x, errs := NewIndex([]scan.File{a, b, c}, nil)
+ if len(errs) != 0 {
+ t.Fatalf("NewIndex errors: %v", errs)
+ }
+ if orig, dup := lookup(t, x, b); !dup || orig != a.Path {
+ t.Errorf("a/b duplicate pair broken by unreadable sibling: dup=%v orig=%s", dup, orig)
+ }
+ if orig, dup := lookup(t, x, a); dup {
+ t.Errorf("a reported as a duplicate: orig=%s", orig)
+ }
+ cerrs := x.Errors()
+ if len(cerrs) != 1 {
+ t.Fatalf("got %d candidate errors, want 1: %v", len(cerrs), cerrs)
+ }
+ if cerrs[0].Path != c.Path {
+ t.Errorf("candidate error names %q, want %q", cerrs[0].Path, c.Path)
+ }
+}
+
+// TestVanishedCandidateSkippedSilently: a candidate that vanishes between
+// being indexed and being hashed is dropped with no error recorded at all.
+func TestVanishedCandidateSkippedSilently(t *testing.T) {
+ d := t.TempDir()
+ a := put(t, d, "a.bin", []byte("same content"), 0)
+ b := put(t, d, "b.bin", []byte("same content"), 1)
+ c := put(t, d, "c.bin", []byte("diff content"), 2)
+ x, errs := NewIndex([]scan.File{a, b, c}, nil)
+ if len(errs) != 0 {
+ t.Fatalf("NewIndex errors: %v", errs)
+ }
+ if err := os.Remove(c.Path); err != nil {
+ t.Fatal(err)
+ }
+ if orig, dup := lookup(t, x, b); !dup || orig != a.Path {
+ t.Errorf("a/b duplicate pair broken by vanished sibling: dup=%v orig=%s", dup, orig)
+ }
+ if got := x.Errors(); len(got) != 0 {
+ t.Errorf("vanished candidate recorded as an error: %v", got)
+ }
+}
+
+// TestLookupFailsWhenSubjectUnreadable: Lookup still fails outright when
+// the file it was asked about (not some other candidate) cannot be read.
+func TestLookupFailsWhenSubjectUnreadable(t *testing.T) {
+ if os.Geteuid() == 0 {
+ t.Skip("permissions are not enforced running as root")
+ }
+ d := t.TempDir()
+ a := put(t, d, "a.bin", []byte("same content"), 0)
+ b := put(t, d, "b.bin", []byte("same content"), 1)
+ if err := os.Chmod(a.Path, 0o000); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { os.Chmod(a.Path, 0o644) })
+ x, _ := NewIndex([]scan.File{a, b}, nil)
+ if _, _, err := x.Lookup(a.Path); err == nil {
+ t.Fatal("no error looking up an unreadable subject")
+ }
+}
+
+// fakeDirEntry is an fs.DirEntry whose Info() returns a canned result, for
+// exercising addEntry's Info()-failure handling directly (A3) — a real
+// filepath.WalkDir gives no hook to inject a stat failure deterministically
+// and without root.
+type fakeDirEntry struct {
+ name string
+ info fs.FileInfo
+ infoErr error
+}
+
+func (f fakeDirEntry) Name() string { return f.name }
+func (f fakeDirEntry) IsDir() bool { return false }
+func (f fakeDirEntry) Type() fs.FileMode { return 0 }
+func (f fakeDirEntry) Info() (fs.FileInfo, error) { return f.info, f.infoErr }
+
+// TestAddEntryInfoFailure: A3. A vanished entry's Info() failure
+// (fs.ErrNotExist) is dropped with no error recorded; any other Info()
+// failure is dropped too, but recorded in errs, naming the entry.
+func TestAddEntryInfoFailure(t *testing.T) {
+ x := &Index{bySize: make(map[int64][]int), scanned: make(map[string]int)}
+ var errs []error
+
+ x.addEntry("/extra/vanished.txt", fakeDirEntry{
+ name: "vanished.txt", infoErr: fmt.Errorf("stat vanished.txt: %w", fs.ErrNotExist),
+ }, &errs)
+ if len(errs) != 0 {
+ t.Fatalf("vanished entry recorded an error: %v", errs)
+ }
+ if len(x.candidates) != 0 {
+ t.Fatalf("vanished entry was indexed: %v", x.candidates)
+ }
+
+ x.addEntry("/extra/denied.txt", fakeDirEntry{
+ name: "denied.txt", infoErr: errors.New("permission denied"),
+ }, &errs)
+ if len(errs) != 1 || !strings.Contains(errs[0].Error(), "/extra/denied.txt") {
+ t.Fatalf("want one error naming /extra/denied.txt, got %v", errs)
+ }
+ if len(x.candidates) != 0 {
+ t.Fatalf("denied entry was indexed: %v", x.candidates)
+ }
+}
+
+// TestExtraDirSymlinkNotFollowed: A4. An extra directory that is itself a
+// symlink to a real directory is not silently treated as empty:
+// filepath.WalkDir Lstats its root, so without a check for this the walk
+// would report no error and index nothing, misleading the user into
+// thinking an archive was consulted when it never was.
+func TestExtraDirSymlinkNotFollowed(t *testing.T) {
+ real := t.TempDir()
+ put(t, real, "invoice.pdf", []byte("invoice 42"), 0)
+ link := filepath.Join(t.TempDir(), "link")
+ if err := os.Symlink(real, link); err != nil {
+ t.Fatal(err)
+ }
+
+ scanned := t.TempDir()
+ dl := put(t, scanned, "download.pdf", []byte("invoice 42"), 1)
+
+ x, errs := NewIndex([]scan.File{dl}, []string{link})
+ if len(errs) != 1 || !strings.Contains(errs[0].Error(), link) || !strings.Contains(errs[0].Error(), "symlink") {
+ t.Fatalf("want one error naming %s as a symlink, got %v", link, errs)
+ }
+ if orig, dup := lookup(t, x, dl); dup {
+ t.Errorf("symlinked extra dir was indexed despite the error: dup=%v orig=%s", dup, orig)
+ }
+}
diff --git a/internal/engine/engine.go b/internal/engine/engine.go
new file mode 100644
index 0000000..eec9a60
--- /dev/null
+++ b/internal/engine/engine.go
@@ -0,0 +1,170 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+// Package engine is what every krino front end calls: it loads and compiles
+// the configuration, matches files against rules, and (from plan 3) plans
+// and applies actions. It returns data; front ends only render it.
+package engine
+
+import (
+ "os"
+ "time"
+
+ "krino/internal/cond"
+ "krino/internal/config"
+ "krino/internal/extract"
+ "krino/internal/ignore"
+)
+
+// Engine holds a loaded, compiled configuration: everything a front end
+// needs to check, match and (from plan 3) act.
+type Engine struct {
+ Config *config.Config
+ Dirs []*Dir
+ Extract *extract.Extractor
+ Now func() time.Time // time.Now; tests replace it
+ MainFile string
+}
+
+// Dir is one configured directory, with its ignore matcher and rules
+// compiled.
+type Dir struct {
+ Name string
+ Root string // absolute
+ Conf *config.Dir
+ Settings config.Resolved // built-in, then defaults, then the directory
+ Ignore *ignore.Matcher
+ Rules []*Rule
+
+ // ContentVariants is the distinct (IgnoreCase, Fold) pairs any of
+ // Rules' content tests evaluate under, in first-seen order. B2: when
+ // this holds exactly one variant, facts.Content releases a file's raw
+ // extracted text once that variant's normalised copy exists, since no
+ // other variant will ever be asked for; with more than one, both must
+ // stay memoised, as before.
+ ContentVariants []cond.Options
+}
+
+// Rule is one directory's rule, with its condition compiled.
+type Rule struct {
+ Name string
+ Conf *config.Rule
+ Settings config.Resolved // the rule's own settings over its directory's
+ Cond *cond.Cond
+}
+
+// Load reads and compiles everything. Any problem anywhere returns a nil
+// Engine and every diagnostic: krino never acts on a configuration it only
+// partly understood. Duplicate names are ignored after their first use.
+func Load(mainFile string, names ...string) (*Engine, []*config.Diag) {
+ cfg, errs := config.Load(mainFile, dedupeNames(names)...)
+ if cfg == nil {
+ return nil, errs
+ }
+
+ var dirs []*Dir
+ for _, d := range cfg.Dirs {
+ dir := &Dir{
+ Name: d.Name,
+ Root: d.Path,
+ Conf: d,
+ Settings: cfg.Resolved(d),
+ }
+ if m, err := ignore.New(d.Ignore); err != nil {
+ errs = append(errs, &config.Diag{File: d.File, Msg: err.Error()})
+ } else {
+ dir.Ignore = m
+ }
+ for _, r := range d.Rules {
+ rs := r.Settings.Over(dir.Settings)
+ c, cerrs := cond.Compile(d.File, r.When, cond.Options{
+ IgnoreCase: rs.Case == config.CaseIgnore,
+ Fold: rs.Fold,
+ })
+ if len(cerrs) > 0 {
+ errs = append(errs, cerrs...)
+ continue
+ }
+ dir.Rules = append(dir.Rules, &Rule{Name: r.Name, Conf: r, Settings: rs, Cond: c})
+ }
+ dir.ContentVariants = contentVariants(dir.Rules)
+ dirs = append(dirs, dir)
+ }
+
+ if len(errs) > 0 {
+ return nil, errs
+ }
+ return &Engine{
+ Config: cfg,
+ Dirs: dirs,
+ Extract: extract.New(),
+ Now: time.Now,
+ MainFile: mainFile,
+ }, nil
+}
+
+// dedupeNames returns names with every repeat after its first occurrence
+// removed, order preserved.
+func dedupeNames(names []string) []string {
+ var out []string
+ seen := map[string]bool{}
+ for _, n := range names {
+ if seen[n] {
+ continue
+ }
+ seen[n] = true
+ out = append(out, n)
+ }
+ return out
+}
+
+// contentVariants returns the distinct (IgnoreCase, Fold) pairs any of
+// rules' content tests evaluate under, in first-seen order — B2's per-Dir
+// ContentVariants. A rule whose condition has no content test at all
+// (Cond.UsesContent false) never calls facts.Content, so its resolved
+// case/fold settings contribute no variant here.
+func contentVariants(rules []*Rule) []cond.Options {
+ var out []cond.Options
+ seen := map[cond.Options]bool{}
+ for _, r := range rules {
+ if !r.Cond.UsesContent {
+ continue
+ }
+ opt := cond.Options{IgnoreCase: r.Settings.Case == config.CaseIgnore, Fold: r.Settings.Fold}
+ if seen[opt] {
+ continue
+ }
+ seen[opt] = true
+ out = append(out, opt)
+ }
+ return out
+}
+
+// Report is what Check reports: the files involved and each directory's
+// state.
+type Report struct {
+ MainFile string
+ LogFile string
+ Dirs []DirReport
+ Tools []extract.Tool
+}
+
+// DirReport is one directory's state in a Report.
+type DirReport struct {
+ Dir *Dir
+ Missing bool // the root is not a directory right now
+}
+
+// Check reports the engine's configuration files, each directory's current
+// state and the external tools found for content extraction.
+func (e *Engine) Check() Report {
+ r := Report{
+ MainFile: e.MainFile,
+ LogFile: e.Config.LogFile(),
+ Tools: e.Extract.Tools(),
+ }
+ for _, d := range e.Dirs {
+ fi, err := os.Stat(d.Root)
+ r.Dirs = append(r.Dirs, DirReport{Dir: d, Missing: err != nil || !fi.IsDir()})
+ }
+ return r
+}
diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go
new file mode 100644
index 0000000..5c0536f
--- /dev/null
+++ b/internal/engine/engine_test.go
@@ -0,0 +1,166 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package engine
+
+import (
+ "os"
+ "path/filepath"
+ "reflect"
+ "strings"
+ "testing"
+ "time"
+
+ "krino/internal/cond"
+)
+
+// sandbox gives a test its own HOME with no XDG overrides and returns it.
+func sandbox(t *testing.T) string {
+ t.Helper()
+ h := t.TempDir()
+ t.Setenv("HOME", h)
+ for _, v := range []string{"XDG_CONFIG_HOME", "XDG_STATE_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME"} {
+ t.Setenv(v, "")
+ }
+ return h
+}
+
+// writeConfig writes krino.conf and dirs/<name>.conf files under home/.config/krino.
+func writeConfig(t *testing.T, home, main string, dirs map[string]string) string {
+ t.Helper()
+ cdir := filepath.Join(home, ".config", "krino")
+ if err := os.MkdirAll(filepath.Join(cdir, "dirs"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ mainFile := filepath.Join(cdir, "krino.conf")
+ if err := os.WriteFile(mainFile, []byte(main), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ for n, body := range dirs {
+ if err := os.WriteFile(filepath.Join(cdir, "dirs", n+".conf"), []byte(body), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ return mainFile
+}
+
+// fakeFacts is a minimal cond.Facts for checking compiled rules.
+type fakeFacts struct{ name string }
+
+func (f fakeFacts) Name() string { return f.name }
+func (f fakeFacts) Rel() string { return f.name }
+func (f fakeFacts) Size() int64 { return 1 }
+func (f fakeFacts) ModTime() time.Time { return time.Time{} }
+func (f fakeFacts) Now() time.Time { return time.Time{} }
+func (f fakeFacts) Content(bool, bool) (string, error) { return "", nil }
+func (f fakeFacts) Duplicate([]string) (string, bool, error) { return "", false, nil }
+func (f fakeFacts) Matched() bool { return false }
+
+var _ cond.Facts = fakeFacts{}
+
+func TestLoadCompilesWithRuleSettings(t *testing.T) {
+ h := sandbox(t)
+ os.Mkdir(filepath.Join(h, "dl"), 0o755)
+ main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": `
+(path "~/dl")
+(case strict)
+(ignore "*.part")
+(rule "strict" (when (name "^img")) (stop))
+(rule "loose" (case ignore) (when (name "^img")) (stop))
+`})
+ e, errs := Load(main, "dl", "dl")
+ if len(errs) > 0 {
+ t.Fatal(errs)
+ }
+ if len(e.Dirs) != 1 {
+ t.Fatalf("got %d dirs, want 1 (duplicate name ignored)", len(e.Dirs))
+ }
+ d := e.Dirs[0]
+ if d.Root != filepath.Join(h, "dl") || d.Ignore == nil || !d.Ignore.Match("x.part", false) {
+ t.Fatalf("dir = %+v", d)
+ }
+ img := fakeFacts{name: "IMG_1.jpg"}
+ if d.Rules[0].Cond.Eval(img).Match {
+ t.Error("rule under (case strict) matched IMG against ^img")
+ }
+ if !d.Rules[1].Cond.Eval(img).Match {
+ t.Error("rule-level (case ignore) did not apply at compile time")
+ }
+}
+
+func TestLoadReportsEveryProblem(t *testing.T) {
+ h := sandbox(t)
+ main := writeConfig(t, h, `(include "a" "b")`, map[string]string{
+ "a": `(path "/tmp") (rule "x" (when (type "pdf")) (stop))`,
+ "b": `(path "/tmp") (ignore "[abc") (rule "y" (when (size 1)) (stop))`,
+ })
+ e, errs := Load(main)
+ if e != nil {
+ t.Fatal("engine returned despite errors")
+ }
+ joined := ""
+ for _, d := range errs {
+ joined += d.Error() + "\n"
+ }
+ for _, want := range []string{
+ "a.conf:1:37: type names are bare words: write (type pdf)",
+ `b.conf: bad ignore pattern "[abc": unterminated [`,
+ "b.conf:1:47: size takes an operator and a size, like (size > 10M)",
+ } {
+ if !strings.Contains(joined, want) {
+ t.Errorf("missing %q in:\n%s", want, joined)
+ }
+ }
+}
+
+func TestCheck(t *testing.T) {
+ h := sandbox(t)
+ bin := t.TempDir()
+ os.WriteFile(filepath.Join(bin, "pdftotext"), []byte("#!/bin/sh\n"), 0o755)
+ t.Setenv("PATH", bin)
+ os.Mkdir(filepath.Join(h, "here"), 0o755)
+ main := writeConfig(t, h, `(include "here" "gone")`, map[string]string{
+ "here": `(path "~/here") (rule "r" (stop))`,
+ "gone": `(path "~/gone") (rule "r" (stop))`,
+ })
+ e, errs := Load(main)
+ if len(errs) > 0 {
+ t.Fatal(errs)
+ }
+ r := e.Check()
+ if r.MainFile != main || r.LogFile != filepath.Join(h, ".local", "state", "krino", "krino.log") {
+ t.Errorf("report files = %q %q", r.MainFile, r.LogFile)
+ }
+ if len(r.Dirs) != 2 || r.Dirs[0].Missing || !r.Dirs[1].Missing {
+ t.Errorf("dirs = %+v", r.Dirs)
+ }
+ if r.Tools[0].Name != "pdftotext" || r.Tools[0].Path != filepath.Join(bin, "pdftotext") || r.Tools[1].Path != "" {
+ t.Errorf("tools = %+v", r.Tools)
+ }
+}
+
+// TestContentVariantsComputedAtLoad: B2 plumbing. Load computes each
+// directory's distinct (ignoreCase, fold) content-test variants from its
+// rules' resolved settings: a rule with no content test contributes
+// nothing; two rules sharing a variant fold into one; a rule-level (case
+// ignore) override adds a second.
+func TestContentVariantsComputedAtLoad(t *testing.T) {
+ h := sandbox(t)
+ os.Mkdir(filepath.Join(h, "dl"), 0o755)
+ main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": `
+(path "~/dl")
+(case strict)
+(rule "no-content" (when (type pdf)) (stop))
+(rule "strict-content" (when (content "acme")) (stop))
+(rule "also-strict-content" (when (content "other")) (stop))
+(rule "loose-content" (case ignore) (when (content "acme")) (stop))
+`})
+ e, errs := Load(main, "dl")
+ if len(errs) > 0 {
+ t.Fatal(errs)
+ }
+ got := e.Dirs[0].ContentVariants
+ want := []cond.Options{{IgnoreCase: false, Fold: true}, {IgnoreCase: true, Fold: true}}
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("ContentVariants = %+v, want %+v", got, want)
+ }
+}
diff --git a/internal/engine/facts.go b/internal/engine/facts.go
new file mode 100644
index 0000000..60380f3
--- /dev/null
+++ b/internal/engine/facts.go
@@ -0,0 +1,204 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package engine
+
+import (
+ "context"
+ "path/filepath"
+ "sort"
+ "strings"
+ "sync"
+ "time"
+
+ "krino/internal/cond"
+ "krino/internal/dup"
+ "krino/internal/norm"
+ "krino/internal/scan"
+ "krino/internal/xdg"
+)
+
+// matchRun holds the state shared by every file evaluated during one Match
+// or Explain call: the directory being matched, the full set of scanned
+// files (for duplicate detection) and the lazily built duplicate indexes,
+// one per distinct set of resolved directories a (duplicate ...) test
+// names. mu guards dupOnce, dupIdx and warn, the only fields any goroutine
+// but the one that created the matchRun ever touches.
+type matchRun struct {
+ e *Engine
+ d *Dir
+ ctx context.Context
+ now time.Time
+ files []scan.File
+
+ mu sync.Mutex
+ dupOnce map[string]*sync.Once
+ dupIdx map[string]*dup.Index
+ warn []string
+}
+
+// newMatchRun builds a matchRun over files, the set a (duplicate ...) test
+// with no directories of its own checks against.
+func newMatchRun(e *Engine, d *Dir, ctx context.Context, now time.Time, files []scan.File) *matchRun {
+ return &matchRun{
+ e: e,
+ d: d,
+ ctx: ctx,
+ now: now,
+ files: files,
+ dupOnce: make(map[string]*sync.Once),
+ dupIdx: make(map[string]*dup.Index),
+ }
+}
+
+// warnings returns the directory-level warnings collected so far (from
+// building duplicate indexes), in the order they were recorded.
+func (run *matchRun) warnings() []string {
+ run.mu.Lock()
+ defer run.mu.Unlock()
+ return append([]string(nil), run.warn...)
+}
+
+// drainDupErrors appends every duplicate index's candidate-hashing errors
+// (A1: a candidate other than the file being looked up that could not be
+// hashed) to run.warn, once matching is done and every index has seen every
+// Lookup it is going to see. Candidate paths are abbreviated with
+// xdg.Abbrev, as every other user-visible path is.
+func (run *matchRun) drainDupErrors() {
+ run.mu.Lock()
+ defer run.mu.Unlock()
+ for _, idx := range run.dupIdx {
+ for _, ce := range idx.Errors() {
+ run.warn = append(run.warn, "duplicate: "+xdg.Abbrev(ce.Path)+": "+ce.Err.Error())
+ }
+ }
+}
+
+// dupIndex returns the shared *dup.Index for the resolved, sorted extra
+// directories named by key, building it exactly once across every
+// concurrent caller that asks for the same key.
+func (run *matchRun) dupIndex(key string, dirs []string) *dup.Index {
+ run.mu.Lock()
+ once, ok := run.dupOnce[key]
+ if !ok {
+ once = &sync.Once{}
+ run.dupOnce[key] = once
+ }
+ run.mu.Unlock()
+
+ once.Do(func() {
+ idx, errs := dup.NewIndex(run.files, dirs)
+ run.mu.Lock()
+ run.dupIdx[key] = idx
+ for _, err := range errs {
+ run.warn = append(run.warn, "duplicate: "+err.Error())
+ }
+ run.mu.Unlock()
+ })
+
+ run.mu.Lock()
+ idx := run.dupIdx[key]
+ run.mu.Unlock()
+ return idx
+}
+
+// facts is one file's cond.Facts. It is used by exactly one goroutine, so
+// its own memoised state (content, its normalised variants, and whether an
+// earlier rule matched) needs no locking of its own; only the matchRun it
+// points at is shared.
+type facts struct {
+ run *matchRun
+ file scan.File
+
+ matched bool
+
+ contentDone bool
+ content string
+ contentErr error
+ normCache map[[2]bool]string
+}
+
+var _ cond.Facts = (*facts)(nil)
+
+// newFacts builds the Facts for one scanned file.
+func newFacts(run *matchRun, file scan.File) *facts {
+ return &facts{run: run, file: file, normCache: make(map[[2]bool]string)}
+}
+
+func (f *facts) Name() string { return f.file.Name }
+func (f *facts) Rel() string { return f.file.Rel }
+func (f *facts) Size() int64 { return f.file.Size }
+func (f *facts) ModTime() time.Time { return f.file.ModTime }
+func (f *facts) Now() time.Time { return f.run.now }
+func (f *facts) Matched() bool { return f.matched }
+
+// Content extracts the file's text once, then normalises it per
+// (ignoreCase, fold) variant, memoising each. B2: when the directory's
+// rules use exactly one variant (Dir.ContentVariants), the raw text is
+// released as soon as that variant's normalised copy exists — no other
+// variant will ever be asked for, so there is no reason to keep both the
+// raw text and its normalised copy in memory at once. A directory using
+// more than one variant keeps the raw text for as long as f lives, exactly
+// as before.
+func (f *facts) Content(ignoreCase, fold bool) (string, error) {
+ if !f.contentDone {
+ f.content, f.contentErr = f.run.e.Extract.Text(f.run.ctx, f.file.Path, f.file.Size, f.run.d.Settings.MaxRead)
+ f.contentDone = true
+ }
+ if f.contentErr != nil {
+ return "", f.contentErr
+ }
+ key := [2]bool{ignoreCase, fold}
+ if v, ok := f.normCache[key]; ok {
+ return v, nil
+ }
+ v := norm.Text(f.content, ignoreCase, fold)
+ f.normCache[key] = v
+ if len(f.run.d.ContentVariants) == 1 {
+ f.content = ""
+ }
+ return v, nil
+}
+
+// Duplicate resolves dirs against the directory's root, builds (or reuses)
+// the shared duplicate index for that resolved, sorted set, and looks the
+// file up in it.
+func (f *facts) Duplicate(dirs []string) (string, bool, error) {
+ root := f.run.d.Root
+ resolved := make([]string, len(dirs))
+ for i, raw := range dirs {
+ resolved[i] = resolveDir(raw, root)
+ }
+ sorted := append([]string(nil), resolved...)
+ sort.Strings(sorted)
+ key := strings.Join(sorted, "\x00")
+
+ idx := f.run.dupIndex(key, sorted)
+ orig, isDup, err := idx.Lookup(f.file.Path)
+ if err != nil {
+ return "", false, err
+ }
+ if !isDup {
+ return "", false, nil
+ }
+ return displayOriginal(orig, root), true, nil
+}
+
+// resolveDir expands a leading ~ and joins a relative directory to root,
+// cleaned.
+func resolveDir(raw, root string) string {
+ p := xdg.Expand(raw)
+ if !filepath.IsAbs(p) {
+ p = filepath.Join(root, p)
+ }
+ return filepath.Clean(p)
+}
+
+// displayOriginal reports orig relative to root when it lies inside root,
+// else as an absolute path.
+func displayOriginal(orig, root string) string {
+ rel, err := filepath.Rel(root, orig)
+ if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
+ return orig
+ }
+ return filepath.ToSlash(rel)
+}
diff --git a/internal/engine/facts_test.go b/internal/engine/facts_test.go
new file mode 100644
index 0000000..c511804
--- /dev/null
+++ b/internal/engine/facts_test.go
@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package engine
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "krino/internal/cond"
+ "krino/internal/extract"
+ "krino/internal/scan"
+)
+
+// contentFacts builds a *facts for a real text file, under a Dir whose
+// ContentVariants is variants, for exercising B2's raw-release directly.
+func contentFacts(t *testing.T, body string, variants []cond.Options) *facts {
+ t.Helper()
+ p := filepath.Join(t.TempDir(), "a.txt")
+ if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ fi, err := os.Stat(p)
+ if err != nil {
+ t.Fatal(err)
+ }
+ e := &Engine{Extract: extract.New(), Now: time.Now}
+ d := &Dir{Name: "d", ContentVariants: variants}
+ file := scan.File{Path: p, Rel: "a.txt", Name: "a.txt", Size: fi.Size(), ModTime: fi.ModTime()}
+ run := newMatchRun(e, d, context.Background(), time.Now(), []scan.File{file})
+ return newFacts(run, file)
+}
+
+// TestContentReleasesRawWithOneVariant: B2. A directory whose rules use
+// exactly one (ignoreCase, fold) variant releases the raw extracted text
+// once that variant's normalised copy exists.
+func TestContentReleasesRawWithOneVariant(t *testing.T) {
+ f := contentFacts(t, "Hello World", []cond.Options{{IgnoreCase: true, Fold: false}})
+ const want = "hello world"
+ got, err := f.Content(true, false)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got != want {
+ t.Fatalf("got %q, want %q", got, want)
+ }
+ if f.content != "" {
+ t.Errorf("raw text not released with a single content variant: %q", f.content)
+ }
+ // A second call for the same (already cached) variant must still work
+ // from normCache, without needing the released raw text.
+ if got, err := f.Content(true, false); err != nil || got != want {
+ t.Errorf("second call for the cached variant: got %q, %v, want %q", got, err, want)
+ }
+}
+
+// TestContentKeepsRawWithTwoVariants: B2. A directory whose rules use two
+// distinct variants must not release the raw text after the first: the
+// second variant still needs it, and normalising it correctly (not from an
+// emptied string) is the proof the raw text was kept.
+func TestContentKeepsRawWithTwoVariants(t *testing.T) {
+ f := contentFacts(t, "Hello World", []cond.Options{
+ {IgnoreCase: true, Fold: false},
+ {IgnoreCase: false, Fold: false},
+ })
+ if got, err := f.Content(true, false); err != nil || got != "hello world" {
+ t.Fatalf("first variant: got %q, %v", got, err)
+ }
+ if f.content == "" {
+ t.Fatal("raw text released after only the first of two variants")
+ }
+ if got, err := f.Content(false, false); err != nil || got != "Hello World" {
+ t.Fatalf("second variant: got %q, %v, want the unfolded original (raw text must still be available)", got, err)
+ }
+}
diff --git a/internal/engine/match.go b/internal/engine/match.go
new file mode 100644
index 0000000..0693bd5
--- /dev/null
+++ b/internal/engine/match.go
@@ -0,0 +1,357 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package engine
+
+import (
+ "context"
+ "fmt"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "runtime"
+ "sort"
+ "strings"
+ "sync"
+ "time"
+
+ "krino/internal/cond"
+ "krino/internal/config"
+ "krino/internal/scan"
+ "krino/internal/xdg"
+)
+
+// RuleMatch is one rule that matched a file, and why.
+type RuleMatch struct {
+ Rule *Rule
+ Captures []string
+ Reasons []string
+}
+
+// FileMatch is one file and the rules that did, or did not, match it.
+type FileMatch struct {
+ File scan.File
+ Rules []RuleMatch // matching rules in order, ending at the first with (stop)
+ Warnings []string // "<rule>: <warning>", e.g. "acme: content unreadable: needs pdftotext, not installed"
+}
+
+// Result is everything Match found in one directory.
+type Result struct {
+ Dir *Dir
+ Matched []FileMatch // at least one rule matched; sorted by File.Rel
+ Unmatched []FileMatch // no rule matched (Warnings may say why); sorted by File.Rel
+ Skipped []scan.Skipped
+ Warnings []string // directory-level, sorted; e.g. "duplicate: /x/y does not exist"
+ Elapsed time.Duration
+}
+
+// Match walks d's root and evaluates every rule against every file found,
+// concurrently. Output order never depends on scheduling: results are
+// placed by index into a slice the size of the walk, then split into
+// Matched and Unmatched keeping that (Rel-sorted) order.
+func (e *Engine) Match(ctx context.Context, d *Dir) (*Result, error) {
+ started := time.Now()
+ now := e.Now()
+ excl := e.excludeDirs(d)
+
+ wres, err := scan.Walk(d.Root, walkOptions(d, excl, now))
+ if err != nil {
+ return nil, err
+ }
+
+ run := newMatchRun(e, d, ctx, now, wres.Files)
+ fileMatches := make([]FileMatch, len(wres.Files))
+
+ workers := runtime.GOMAXPROCS(0)
+ if workers < 1 {
+ workers = 1
+ }
+ var wg sync.WaitGroup
+ jobs := make(chan int)
+ for w := 0; w < workers; w++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for i := range jobs {
+ fileMatches[i] = evalFile(run, wres.Files[i])
+ }
+ }()
+ }
+ for i := range wres.Files {
+ jobs <- i
+ }
+ close(jobs)
+ wg.Wait()
+ run.drainDupErrors()
+
+ var matched, unmatched []FileMatch
+ for _, fm := range fileMatches {
+ if len(fm.Rules) > 0 {
+ matched = append(matched, fm)
+ } else {
+ unmatched = append(unmatched, fm)
+ }
+ }
+
+ warnings := run.warnings()
+ sort.Strings(warnings)
+
+ return &Result{
+ Dir: d,
+ Matched: matched,
+ Unmatched: unmatched,
+ Skipped: wres.Skipped,
+ Warnings: warnings,
+ Elapsed: time.Since(started),
+ }, nil
+}
+
+// evalFile evaluates every rule of run.d, in order, against file: matched
+// becomes true after the first matching rule, so a later (not (matched))
+// test sees it, and evaluation stops right after a matching rule whose
+// Stop is set.
+func evalFile(run *matchRun, file scan.File) FileMatch {
+ f := newFacts(run, file)
+ fm := FileMatch{File: file}
+ for _, r := range run.d.Rules {
+ res := r.Cond.Eval(f)
+ for _, w := range res.Warnings {
+ fm.Warnings = append(fm.Warnings, r.Name+": "+w)
+ }
+ if !res.Match {
+ continue
+ }
+ fm.Rules = append(fm.Rules, RuleMatch{Rule: r, Captures: res.Captures, Reasons: res.Reasons})
+ f.matched = true
+ if r.Conf.Stop {
+ break
+ }
+ }
+ return fm
+}
+
+// RuleTrace is one rule's outcome in an Explain call.
+type RuleTrace struct {
+ Rule *Rule
+ Match bool
+ Trace *cond.Trace // nil when not evaluated
+ Stopped string // "stopped by rule acme" when an earlier (stop) ended the search
+}
+
+// Explanation is why (or why not) krino would act on one file.
+type Explanation struct {
+ Dir *Dir
+ File scan.File
+ Skip string // why krino would not look at this file at all; "" when it would
+ Rules []RuleTrace
+}
+
+// Explain reports, for one file, whether krino's ordinary scan would ever
+// reach it and how every rule of its directory evaluates against it. Rules
+// are traced in order even when Skip is set, so a user can see what would
+// match if the file were looked at; a rule reached after an earlier
+// matching (stop) is recorded as Stopped, with no trace.
+func (e *Engine) Explain(ctx context.Context, path string) (*Explanation, error) {
+ abs, err := filepath.Abs(path)
+ if err != nil {
+ return nil, err
+ }
+ abs = filepath.Clean(abs)
+
+ d := e.dirFor(abs)
+ if d == nil {
+ return nil, fmt.Errorf("%s is not inside any included directory", path)
+ }
+
+ info, err := os.Lstat(abs)
+ if err != nil {
+ return nil, err
+ }
+ if info.Mode()&fs.ModeSymlink != 0 || !info.Mode().IsRegular() {
+ return nil, fmt.Errorf("%s is not a regular file", path)
+ }
+
+ rel, err := filepath.Rel(d.Root, abs)
+ if err != nil {
+ return nil, err
+ }
+ rel = filepath.ToSlash(rel)
+
+ sf := scan.File{
+ Path: abs,
+ Rel: rel,
+ Name: filepath.Base(abs),
+ Size: info.Size(),
+ ModTime: info.ModTime(),
+ Mode: info.Mode(),
+ }
+
+ now := e.Now()
+ excl := e.excludeDirs(d)
+ skip := explainSkip(d, sf, excl, now)
+
+ run := newMatchRun(e, d, ctx, now, e.filesForExplain(d, sf, excl, now))
+ f := newFacts(run, sf)
+
+ var rules []RuleTrace
+ stoppedBy := ""
+ for _, r := range d.Rules {
+ if stoppedBy != "" {
+ rules = append(rules, RuleTrace{Rule: r, Stopped: "stopped by rule " + stoppedBy})
+ continue
+ }
+ trace := r.Cond.Explain(f)
+ match := trace.Value
+ if match {
+ f.matched = true
+ }
+ rules = append(rules, RuleTrace{Rule: r, Match: match, Trace: trace})
+ if match && r.Conf.Stop {
+ stoppedBy = r.Name
+ }
+ }
+
+ return &Explanation{Dir: d, File: sf, Skip: skip, Rules: rules}, nil
+}
+
+// filesForExplain returns the file set Explain's duplicate checks run
+// against: the directory's ordinary scan, plus the explained file itself
+// when that scan would not have reached it (it is busy, ignored, too new,
+// excluded, or beyond recursive/max-depth) - so (duplicate ...) always has
+// a real answer for the file being explained, and sees the same siblings
+// Match would.
+func (e *Engine) filesForExplain(d *Dir, sf scan.File, excl []string, now time.Time) []scan.File {
+ wres, err := scan.Walk(d.Root, walkOptions(d, excl, now))
+ if err != nil {
+ return []scan.File{sf}
+ }
+ for _, wf := range wres.Files {
+ if wf.Path == sf.Path {
+ return wres.Files
+ }
+ }
+ return append(append([]scan.File{}, wres.Files...), sf)
+}
+
+// walkOptions builds the scan.Options both Match and Explain's
+// filesForExplain walk d's root with, so the two never drift apart.
+func walkOptions(d *Dir, excl []string, now time.Time) scan.Options {
+ return scan.Options{
+ Recursive: d.Settings.Recursive,
+ MaxDepth: d.Settings.MaxDepth,
+ Ignore: d.Ignore,
+ Exclude: excl,
+ Busy: d.Settings.Busy,
+ MinAge: d.Settings.MinAge,
+ Now: now,
+ }
+}
+
+// dirFor returns the configured Dir whose root most specifically (longest
+// root wins) contains abs, or nil if none does.
+func (e *Engine) dirFor(abs string) *Dir {
+ var best *Dir
+ var bestRoot string
+ for _, d := range e.Dirs {
+ root := filepath.Clean(d.Root)
+ if abs != root && !strings.HasPrefix(abs, root+string(filepath.Separator)) {
+ continue
+ }
+ if best == nil || len(root) > len(bestRoot) {
+ best, bestRoot = d, root
+ }
+ }
+ return best
+}
+
+// explainSkip decides, in priority order, why krino's ordinary scan would
+// not reach sf, or "" if it would.
+func explainSkip(d *Dir, sf scan.File, excl []string, now time.Time) string {
+ segs := strings.Split(sf.Rel, "/")
+ if !d.Settings.Recursive && len(segs) > 1 {
+ return "in a subdirectory, and recursive is off"
+ }
+ if d.Settings.MaxDepth > 0 && len(segs) > d.Settings.MaxDepth {
+ return "deeper than max-depth"
+ }
+ if insideAny(sf.Path, excl) {
+ return "inside a rule destination, which krino never scans"
+ }
+ if d.Ignore != nil && d.Ignore.Match(sf.Rel, false) {
+ return "ignored"
+ }
+ if isBusy(sf.Path, d.Settings.Busy) {
+ return "busy"
+ }
+ if now.Sub(sf.ModTime) < d.Settings.MinAge {
+ return "too new"
+ }
+ return ""
+}
+
+// insideAny reports whether path is dir itself, or inside it, for any dir
+// in dirs.
+func insideAny(path string, dirs []string) bool {
+ for _, dir := range dirs {
+ if path == dir || strings.HasPrefix(path, dir+string(filepath.Separator)) {
+ return true
+ }
+ }
+ return false
+}
+
+// isBusy reports whether path has a sibling named path+suffix, for any
+// configured busy suffix - the mark of an in-progress download.
+func isBusy(path string, suffixes []string) bool {
+ for _, suf := range suffixes {
+ if _, err := os.Lstat(path + suf); err == nil {
+ return true
+ }
+ }
+ return false
+}
+
+// excludeDirs computes the directories Match and Explain never enter: each
+// rule's copy/move destination, the Trash, and the directory holding the
+// main config file - each kept only when it lies strictly inside d's root.
+// A destination with no template placeholder excludes exactly that
+// directory; a destination with a placeholder excludes only the static
+// part before its first "{", cut back to a full path component (its last
+// "/"), since anything from there on varies per file - spec 8.1: "Work/
+// Acme/{mtime:%Y}" excludes "Work/Acme", and "Work/Acme-{mtime:%Y}"
+// (the placeholder mid-segment) excludes "Work".
+func (e *Engine) excludeDirs(d *Dir) []string {
+ root := filepath.Clean(d.Root)
+ var out []string
+ add := func(p string) {
+ if p == "" {
+ return
+ }
+ p = filepath.Clean(p)
+ if strings.HasPrefix(p, root+string(filepath.Separator)) {
+ out = append(out, p)
+ }
+ }
+ for _, r := range d.Rules {
+ for _, a := range r.Conf.Actions {
+ if a.Kind != config.Copy && a.Kind != config.Move {
+ continue
+ }
+ prefix := a.Arg
+ if idx := strings.IndexByte(prefix, '{'); idx >= 0 {
+ prefix = prefix[:idx]
+ if idx2 := strings.LastIndexByte(prefix, '/'); idx2 >= 0 {
+ prefix = prefix[:idx2]
+ } else {
+ // The very first path component is itself templated
+ // (e.g. "{year}-Stuff"): nothing about the destination
+ // is known statically, so there is nothing to exclude.
+ prefix = ""
+ }
+ }
+ add(resolveDir(prefix, root))
+ }
+ }
+ add(filepath.Join(xdg.DataHome(), "Trash"))
+ add(filepath.Dir(e.MainFile))
+ return out
+}
diff --git a/internal/engine/match_test.go b/internal/engine/match_test.go
new file mode 100644
index 0000000..1356074
--- /dev/null
+++ b/internal/engine/match_test.go
@@ -0,0 +1,367 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package engine
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+ "reflect"
+ "strings"
+ "testing"
+ "time"
+)
+
+const dlConf = `
+(path "~/dl")
+(recursive yes)
+(min-age 0s)
+(ignore "*.part")
+(rule "dups" (when (duplicate)) (delete) (stop))
+(rule "acme" (when (type document) (content "acme ltd")) (move "Work/Acme") (stop))
+(rule "images" (when (type image)) (move "Pictures"))
+(rule "rest" (when (not (matched)) (type text)) (move "Other"))
+`
+
+// fixture builds ~/dl and returns a loaded engine. PATH is empty, so no
+// extraction tool exists and PDFs and .doc files are unreadable.
+func fixture(t *testing.T) (*Engine, *Dir, string) {
+ t.Helper()
+ h := sandbox(t)
+ t.Setenv("PATH", t.TempDir())
+ dl := filepath.Join(h, "dl")
+ files := map[string]string{
+ "inv1.txt": "Invoice from ACME LTD, tax 0000000000",
+ "notes.txt": "shopping list",
+ "photo.jpg": "\xff\xd8\xff\xe0 jpeg bytes",
+ "report.pdf": "%PDF same bytes",
+ "report (1).pdf": "%PDF same bytes",
+ "brochure.doc": "\xd0\xcf\x11\xe0 doc bytes",
+ "movie.mkv": "video",
+ "movie.mkv.part": "partial",
+ "Work/Acme/filed.txt": "acme ltd, already filed",
+ "Pictures/old.jpg": "\xff\xd8 old",
+ }
+ old := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+ for name, body := range files {
+ p := filepath.Join(dl, name)
+ os.MkdirAll(filepath.Dir(p), 0o755)
+ if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ os.Chtimes(p, old, old)
+ }
+ newer := old.Add(time.Hour)
+ os.Chtimes(filepath.Join(dl, "report (1).pdf"), newer, newer)
+ main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": dlConf})
+ e, errs := Load(main)
+ if len(errs) > 0 {
+ t.Fatal(errs)
+ }
+ return e, e.Dirs[0], dl
+}
+
+// summary renders a result compactly for comparison.
+func summary(r *Result) []string {
+ var out []string
+ for _, m := range r.Matched {
+ var rs []string
+ for _, rm := range m.Rules {
+ rs = append(rs, rm.Rule.Name+"["+strings.Join(rm.Reasons, "; ")+"]")
+ }
+ out = append(out, "match "+m.File.Rel+" "+strings.Join(rs, " "))
+ }
+ for _, m := range r.Unmatched {
+ out = append(out, "none "+m.File.Rel+" "+strings.Join(m.Warnings, " | "))
+ }
+ for _, s := range r.Skipped {
+ out = append(out, "skip "+s.Rel+" "+s.Reason.String())
+ }
+ return out
+}
+
+func TestMatch(t *testing.T) {
+ e, d, _ := fixture(t)
+ r, err := e.Match(context.Background(), d)
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := []string{
+ `match inv1.txt acme[type txt; content "acme ltd"]`,
+ `match notes.txt rest[not matched; type txt]`,
+ `match photo.jpg images[type jpg]`,
+ `match report (1).pdf dups[duplicate of report.pdf]`,
+ `none brochure.doc acme: content unreadable: needs antiword or catdoc, not installed`,
+ `none report.pdf acme: content unreadable: needs pdftotext, not installed`,
+ `skip movie.mkv busy`,
+ `skip movie.mkv.part ignored`,
+ }
+ if got := summary(r); !reflect.DeepEqual(got, want) {
+ t.Fatalf("got\n%s\nwant\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
+ }
+ again, _ := e.Match(context.Background(), d)
+ if !reflect.DeepEqual(summary(again), summary(r)) {
+ t.Fatal("a second run gave a different result")
+ }
+}
+
+func TestMatchMissingRoot(t *testing.T) {
+ e, d, dl := fixture(t)
+ os.RemoveAll(dl)
+ if _, err := e.Match(context.Background(), d); err == nil {
+ t.Fatal("no error for a missing root")
+ }
+}
+
+func TestExplain(t *testing.T) {
+ e, _, dl := fixture(t)
+ x, err := e.Explain(context.Background(), filepath.Join(dl, "inv1.txt"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ var got []string
+ for _, rt := range x.Rules {
+ got = append(got, fmt.Sprintf("%s match=%v stopped=%q trace=%v", rt.Rule.Name, rt.Match, rt.Stopped, rt.Trace != nil))
+ }
+ want := []string{
+ `dups match=false stopped="" trace=true`,
+ `acme match=true stopped="" trace=true`,
+ `images match=false stopped="stopped by rule acme" trace=false`,
+ `rest match=false stopped="stopped by rule acme" trace=false`,
+ }
+ if !reflect.DeepEqual(got, want) || x.Skip != "" {
+ t.Fatalf("skip=%q\n%s", x.Skip, strings.Join(got, "\n"))
+ }
+ for name, skip := range map[string]string{
+ "movie.mkv": "busy",
+ "movie.mkv.part": "ignored",
+ "Work/Acme/filed.txt": "inside a rule destination, which krino never scans",
+ } {
+ x, err := e.Explain(context.Background(), filepath.Join(dl, name))
+ if err != nil || x.Skip != skip {
+ t.Errorf("%s: skip %q, %v; want %q", name, x.Skip, err, skip)
+ }
+ }
+ if _, err := e.Explain(context.Background(), "/etc/hostname"); err == nil || !strings.HasSuffix(err.Error(), "is not inside any included directory") {
+ t.Errorf("outside: %v", err)
+ }
+}
+
+// TestMatchExcludesOnlyRuleDest checks that a rule destination with no
+// placeholder excludes exactly that directory, not its parent: a sibling
+// subdirectory of the destination's parent must still be scanned.
+func TestMatchExcludesOnlyRuleDest(t *testing.T) {
+ h := sandbox(t)
+ dl := filepath.Join(h, "dl")
+ files := map[string]string{
+ "Work/Acme/filed.txt": "already filed",
+ "Work/Other/keep.txt": "keep me",
+ }
+ for name, body := range files {
+ p := filepath.Join(dl, name)
+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ conf := `
+(path "~/dl")
+(recursive yes)
+(min-age 0s)
+(rule "acme" (when (name "nope")) (move "Work/Acme"))
+`
+ main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": conf})
+ e, errs := Load(main)
+ if len(errs) > 0 {
+ t.Fatal(errs)
+ }
+ r, err := e.Match(context.Background(), e.Dirs[0])
+ if err != nil {
+ t.Fatal(err)
+ }
+ seen := map[string]bool{}
+ for _, m := range r.Matched {
+ seen[m.File.Rel] = true
+ }
+ for _, m := range r.Unmatched {
+ seen[m.File.Rel] = true
+ }
+ for _, s := range r.Skipped {
+ seen[s.Rel] = true
+ }
+ if !seen["Work/Other/keep.txt"] {
+ t.Error("Work/Other/keep.txt should have been scanned: only the rule's own destination (Work/Acme) may be excluded")
+ }
+ if seen["Work/Acme/filed.txt"] {
+ t.Error("Work/Acme/filed.txt should have been excluded as inside the rule's destination")
+ }
+}
+
+// TestMatchWarningsSorted checks that Result.Warnings is sorted, not in
+// whatever order concurrent workers happened to build the duplicate
+// indexes that failed: rule "b" (declared first, with a stop that must
+// not skip rule "a" since it never matches) names a missing directory
+// that sorts after rule "a"'s, and rule "c" names the very same missing
+// directory as rule "a" - which must fold into one warning, not two.
+func TestMatchWarningsSorted(t *testing.T) {
+ h := sandbox(t)
+ dl := filepath.Join(h, "dl")
+ if err := os.MkdirAll(dl, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(dl, "only.txt"), []byte("hello"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ conf := `
+(path "~/dl")
+(recursive yes)
+(min-age 0s)
+(rule "b" (when (duplicate "~/zz-missing")) (stop))
+(rule "a" (when (duplicate "~/aa-missing")) (stop))
+(rule "c" (when (duplicate "~/aa-missing")) (stop))
+`
+ main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": conf})
+ e, errs := Load(main)
+ if len(errs) > 0 {
+ t.Fatal(errs)
+ }
+ r, err := e.Match(context.Background(), e.Dirs[0])
+ if err != nil {
+ t.Fatal(err)
+ }
+ aaDir := filepath.Join(h, "aa-missing")
+ zzDir := filepath.Join(h, "zz-missing")
+ if len(r.Warnings) != 2 {
+ t.Fatalf("got %d warnings, want 2 (the shared aa-missing dir should fold into one):\n%s", len(r.Warnings), strings.Join(r.Warnings, "\n"))
+ }
+ wantPrefix := []string{"duplicate: " + aaDir + ":", "duplicate: " + zzDir + ":"}
+ for i, want := range wantPrefix {
+ if !strings.HasPrefix(r.Warnings[i], want) {
+ t.Errorf("Warnings[%d] = %q, want prefix %q", i, r.Warnings[i], want)
+ }
+ }
+}
+
+// TestExcludeDirs is a table check of excludeDirs's rule-destination
+// handling, in rule order: a plain destination excludes itself exactly; a
+// destination with a placeholder excludes only the static part before it,
+// cut back to a full path component; a destination (after that cut) equal
+// to the root itself, or outside the root, excludes nothing; an absolute
+// destination inside the root excludes that directory; copy counts like
+// move; rename is not a destination at all.
+func TestExcludeDirs(t *testing.T) {
+ h := sandbox(t)
+ root := filepath.Join(h, "root")
+ absDest := filepath.Join(root, "AbsDest")
+ conf := `
+(path "` + root + `")
+(rule "r1" (move "Work/Acme"))
+(rule "r2" (move "Photos/{mtime:%Y}"))
+(rule "r3" (move "Work/Acme-{mtime:%Y}"))
+(rule "r4" (move "{ext}"))
+(rule "r5" (move "."))
+(rule "r6" (move "~/elsewhere"))
+(rule "r7" (move "` + absDest + `"))
+(rule "r8" (copy "Backup"))
+(rule "r9" (rename "x-{name}"))
+`
+ main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": conf})
+ e, errs := Load(main)
+ if len(errs) > 0 {
+ t.Fatal(errs)
+ }
+ got := e.excludeDirs(e.Dirs[0])
+ want := []string{
+ filepath.Join(root, "Work", "Acme"), // r1: no placeholder, exact
+ filepath.Join(root, "Photos"), // r2: cut back to "Photos/"
+ filepath.Join(root, "Work"), // r3: cut back past "Acme-"
+ // r4 "{ext}": nothing before "{" at all -> resolves to the root
+ // itself -> not strictly inside it -> excludes nothing.
+ // r5 ".": no placeholder, resolves to the root itself -> nothing.
+ // r6 "~/elsewhere": outside the root -> nothing.
+ absDest, // r7: absolute, already inside root
+ filepath.Join(root, "Backup"), // r8: copy counts like move
+ // r9 rename "x-{name}": rename is never a destination.
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("excludeDirs =\n%v\nwant\n%v", got, want)
+ }
+}
+
+// TestMatchDrainsDupCandidateErrors: A1 plumbing across the dup/engine
+// boundary. Within one directory's own scan (no extra directories), a
+// candidate that cannot be hashed must not poison the duplicate answer for
+// its size-mates, and must surface exactly once in Result.Warnings, its
+// path abbreviated the way every other user-visible path is.
+func TestMatchDrainsDupCandidateErrors(t *testing.T) {
+ if os.Geteuid() == 0 {
+ t.Skip("permissions are not enforced running as root")
+ }
+ h := sandbox(t)
+ dl := filepath.Join(h, "dl")
+ old := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+ files := map[string]string{
+ "a.txt": "same content", // older: the original
+ "b.txt": "same content", // newer: reported as the duplicate
+ "c.txt": "diff content", // same size as a/b, different bytes
+ }
+ for name, body := range files {
+ p := filepath.Join(dl, name)
+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ os.Chtimes(filepath.Join(dl, "a.txt"), old, old)
+ os.Chtimes(filepath.Join(dl, "b.txt"), old.Add(time.Hour), old.Add(time.Hour))
+ os.Chtimes(filepath.Join(dl, "c.txt"), old, old)
+ cPath := filepath.Join(dl, "c.txt")
+ if err := os.Chmod(cPath, 0o000); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { os.Chmod(cPath, 0o644) })
+
+ conf := `
+(path "~/dl")
+(recursive yes)
+(min-age 0s)
+(rule "dup" (when (duplicate)) (stop))
+`
+ main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": conf})
+ e, errs := Load(main)
+ if len(errs) > 0 {
+ t.Fatal(errs)
+ }
+ r, err := e.Match(context.Background(), e.Dirs[0])
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ var bReasons []string
+ for _, m := range r.Matched {
+ if m.File.Rel == "b.txt" {
+ for _, rm := range m.Rules {
+ bReasons = append(bReasons, rm.Reasons...)
+ }
+ }
+ }
+ if len(bReasons) == 0 || bReasons[0] != "duplicate of a.txt" {
+ t.Errorf("b.txt duplicate pair with a.txt broken by unreadable sibling c.txt: %v", summary(r))
+ }
+
+ want := "duplicate: ~/dl/c.txt: "
+ found := 0
+ for _, w := range r.Warnings {
+ if strings.HasPrefix(w, want) {
+ found++
+ }
+ }
+ if found != 1 {
+ t.Errorf("got %d warnings with prefix %q, want 1; warnings: %v", found, want, r.Warnings)
+ }
+}
diff --git a/internal/extract/extract.go b/internal/extract/extract.go
new file mode 100644
index 0000000..aa3ae93
--- /dev/null
+++ b/internal/extract/extract.go
@@ -0,0 +1,154 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+// Package extract gets text out of files so rules can test their content.
+// Text is returned raw; callers normalise it with norm.Text.
+package extract
+
+import (
+ "context"
+ "errors"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+)
+
+var (
+ // ErrUnsupported is returned when the format carries no text krino
+ // knows how to extract.
+ ErrUnsupported = errors.New("no text in this format")
+ // ErrTooLarge is returned when the file is larger than the configured
+ // max-read; nothing is read in that case.
+ ErrTooLarge = errors.New("larger than max-read")
+)
+
+// ToolMissingError is returned when a format needs an external tool that
+// was not found on this system.
+type ToolMissingError struct{ Tool string }
+
+func (e *ToolMissingError) Error() string {
+ return "needs " + e.Tool + ", not installed"
+}
+
+// Tool is one external extractor and the absolute path it was found at, or
+// "" if it was not found.
+type Tool struct{ Name, Path string }
+
+// toolNames lists the external tools Extractor looks up, in the order
+// Tools() reports them.
+var toolNames = []string{"pdftotext", "antiword", "catdoc", "xls2csv", "catppt"}
+
+// markupExt is the set of markup extensions: tags stripped, entities
+// decoded.
+var markupExt = map[string]bool{
+ "html": true, "htm": true, "xhtml": true, "xml": true, "svg": true,
+}
+
+// plainExt is the set of extensions read as plain text without further
+// inspection.
+var plainExt = map[string]bool{
+ "txt": true, "md": true, "log": true, "csv": true, "tsv": true,
+ "json": true, "yaml": true, "yml": true, "toml": true, "ini": true,
+ "conf": true, "cfg": true, "rtf": true, "tex": true, "go": true,
+ "c": true, "h": true, "cpp": true, "hpp": true, "py": true, "sh": true,
+ "js": true, "ts": true, "rs": true, "java": true, "rb": true,
+ "pl": true, "lua": true, "css": true, "sql": true,
+}
+
+// zipExt is the set of Office/OpenDocument/ebook formats: a zip container
+// plus XML inside it (Task 5).
+var zipExt = map[string]bool{
+ "docx": true, "xlsx": true, "pptx": true,
+ "odt": true, "ods": true, "odp": true,
+ "epub": true,
+}
+
+// toolExt maps an extension to the external tool it needs (Task 6). pdf is
+// handled separately, since it has its own fixed command line.
+var toolExt = map[string]string{
+ "doc": "antiword", // falls back to catdoc
+ "xls": "xls2csv",
+ "ppt": "catppt",
+}
+
+// Extractor gets text out of files, using the external tools it found at
+// construction.
+type Extractor struct {
+ tools map[string]string // name -> absolute path, only tools found
+ Timeout time.Duration // per external tool run
+}
+
+// New builds an Extractor, looking up each external tool once in the
+// process's $PATH, with a 30 s per-tool Timeout.
+func New() *Extractor {
+ return newWithPath(os.Getenv("PATH"))
+}
+
+// newWithPath builds an Extractor looking up tools in the given PATH-style
+// list instead of the process environment, so tests control what is found.
+// It does not use exec.LookPath, which reads the process's own PATH; it
+// walks path itself.
+func newWithPath(path string) *Extractor {
+ dirs := filepath.SplitList(path)
+ tools := make(map[string]string, len(toolNames))
+ for _, name := range toolNames {
+ for _, dir := range dirs {
+ if dir == "" {
+ continue
+ }
+ p := filepath.Join(dir, name)
+ info, err := os.Stat(p)
+ if err != nil || !info.Mode().IsRegular() {
+ continue // no such file, or a directory/FIFO/socket/device
+ }
+ if info.Mode()&0o111 == 0 {
+ continue // not executable
+ }
+ tools[name] = p
+ break
+ }
+ }
+ return &Extractor{tools: tools, Timeout: 30 * time.Second}
+}
+
+// Tools reports every external tool Extractor knows about, in a fixed
+// order, with the path it was found at or "" if it was not found.
+func (e *Extractor) Tools() []Tool {
+ out := make([]Tool, len(toolNames))
+ for i, name := range toolNames {
+ out[i] = Tool{Name: name, Path: e.tools[name]}
+ }
+ return out
+}
+
+// Text returns path's raw text content: it does not normalise (callers use
+// norm.Text). size is the file's size, as already known to the caller;
+// maxRead is the configured ceiling, 0 meaning unlimited. Dispatch is on
+// the lower-cased extension; a format extension with no reader yet
+// implemented returns ErrUnsupported.
+func (e *Extractor) Text(ctx context.Context, path string, size, maxRead int64) (string, error) {
+ if maxRead > 0 && size > maxRead {
+ return "", ErrTooLarge
+ }
+
+ ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(path), "."))
+
+ switch {
+ case ext == "pdf":
+ return e.pdfText(ctx, path, maxRead)
+ case zipExt[ext]:
+ return zipText(ctx, path, ext, maxRead)
+ case toolExt[ext] != "":
+ return e.legacyText(ctx, path, ext, maxRead)
+ case markupExt[ext]:
+ raw, err := readDecoded(path)
+ if err != nil {
+ return "", err
+ }
+ return stripMarkup(raw), nil
+ case plainExt[ext]:
+ return readDecoded(path)
+ default:
+ return sniffText(path)
+ }
+}
diff --git a/internal/extract/plain.go b/internal/extract/plain.go
new file mode 100644
index 0000000..d245a41
--- /dev/null
+++ b/internal/extract/plain.go
@@ -0,0 +1,282 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package extract
+
+import (
+ "bytes"
+ "encoding/binary"
+ "html"
+ "io"
+ "os"
+ "strings"
+ "unicode/utf16"
+ "unicode/utf8"
+)
+
+// sniffSize is how much of an unknown-extension file is inspected to guess
+// whether it is text (design.md §6).
+const sniffSize = 8192
+
+// readDecoded reads path whole and decodes it per the encoding rules: a
+// leading UTF-8 BOM is stripped and the rest used as is; a UTF-16 LE or BE
+// BOM is decoded with unicode/utf16; otherwise valid UTF-8 is used as is,
+// and any other invalid UTF-8 is decoded one byte per Latin-1 code point.
+func readDecoded(path string) (string, error) {
+ data, err := os.ReadFile(path)
+ if err != nil {
+ return "", err
+ }
+ return decode(data), nil
+}
+
+// sniffText decides whether an unknown extension is text, reading at most
+// sniffSize bytes before deciding: a file whose first sniffSize bytes are
+// not a UTF-16 BOM and not valid-UTF8-with-no-NUL is rejected as
+// ErrUnsupported without reading any further (so a large binary file is
+// never read in full just to be rejected). Only once that sample passes is
+// the rest of the file read; a UTF-16 BOM is trusted as plain text from
+// the sample alone (UTF-16 text is full of NUL bytes by design), but a
+// sample that merely looks like UTF-8 must hold for the WHOLE file — no
+// NUL byte anywhere, and no invalid UTF-8 anywhere past the sample — or
+// the file is ErrUnsupported after all; the Latin-1 fallback in decode
+// never applies to a sniffed file, only to a file whose extension already
+// names it as text. D2: when the file continues past the sample (n ==
+// sniffSize), the validity check is run against a trimmed copy with any
+// incomplete trailing rune removed, so a multi-byte rune that happens to
+// straddle byte sniffSize does not make an otherwise-valid file sniff as
+// unsupported; sample itself, used below to build the returned text, is
+// left untouched — the rest of the file (read after the check) supplies
+// the bytes trimming set aside.
+func sniffText(path string) (string, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return "", err
+ }
+ defer f.Close()
+
+ sample := make([]byte, sniffSize)
+ n, err := io.ReadFull(f, sample)
+ if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
+ return "", err
+ }
+ sample = sample[:n]
+
+ check := sample
+ if n == sniffSize {
+ check = trimIncompleteTrailingRune(sample)
+ }
+
+ utf16BOM := hasUTF16BOM(sample)
+ if !utf16BOM && !(utf8.Valid(check) && !bytes.Contains(check, []byte{0})) {
+ return "", ErrUnsupported
+ }
+
+ rest, err := io.ReadAll(f)
+ if err != nil {
+ return "", err
+ }
+ data := append(sample, rest...)
+
+ if utf16BOM {
+ return decode(data), nil
+ }
+ if !utf8.Valid(data) || bytes.Contains(data, []byte{0}) {
+ return "", ErrUnsupported
+ }
+ return decode(data), nil
+}
+
+// trimIncompleteTrailingRune drops an incomplete UTF-8 sequence left
+// dangling at the very end of b — D2's fix for a rune cut off exactly at
+// the sniff sample's boundary. It looks back at most utf8.UTFMax-1 bytes
+// for the start of the trailing rune; if the bytes from there to the end
+// are not a complete encoding (utf8.FullRune), that partial rune is cut,
+// since more bytes to finish it may simply not have been read yet. A
+// sample already ending cleanly (the common case, and every all-ASCII
+// sample) is returned unchanged.
+func trimIncompleteTrailingRune(b []byte) []byte {
+ end := len(b)
+ start := end - 1
+ for start >= 0 && end-start < utf8.UTFMax && !utf8.RuneStart(b[start]) {
+ start--
+ }
+ if start < 0 || utf8.FullRune(b[start:end]) {
+ return b
+ }
+ return b[:start]
+}
+
+// hasUTF16BOM reports whether b begins with a UTF-16 little- or big-endian
+// byte-order mark.
+func hasUTF16BOM(b []byte) bool {
+ return len(b) >= 2 && ((b[0] == 0xFF && b[1] == 0xFE) || (b[0] == 0xFE && b[1] == 0xFF))
+}
+
+// decode applies the encoding rules to a whole file's bytes.
+func decode(data []byte) string {
+ switch {
+ case len(data) >= 2 && data[0] == 0xFF && data[1] == 0xFE:
+ return decodeUTF16(data[2:], binary.LittleEndian)
+ case len(data) >= 2 && data[0] == 0xFE && data[1] == 0xFF:
+ return decodeUTF16(data[2:], binary.BigEndian)
+ case len(data) >= 3 && data[0] == 0xEF && data[1] == 0xBB && data[2] == 0xBF:
+ return string(data[3:])
+ case utf8.Valid(data):
+ return string(data)
+ default:
+ return decodeLatin1(data)
+ }
+}
+
+// decodeUTF16 decodes b (already past the BOM) as UTF-16 in the given byte
+// order; a trailing odd byte with no pair is dropped.
+func decodeUTF16(b []byte, order binary.ByteOrder) string {
+ n := len(b) / 2
+ units := make([]uint16, n)
+ for i := 0; i < n; i++ {
+ units[i] = order.Uint16(b[i*2 : i*2+2])
+ }
+ return string(utf16.Decode(units))
+}
+
+// decodeLatin1 decodes b as Latin-1: each byte is its own Unicode code
+// point.
+func decodeLatin1(b []byte) string {
+ r := make([]rune, len(b))
+ for i, c := range b {
+ r[i] = rune(c)
+ }
+ return string(r)
+}
+
+// stripMarkup turns decoded HTML/XML/SVG text into plain text: a small
+// scanner replaces every <...> tag with a space, dropping the contents of
+// <script> and <style> along with their tags and skipping <!-- ... -->
+// comments outright, then entities are decoded with html.UnescapeString.
+// html.UnescapeString turns &nbsp; into U+00A0 (a non-breaking space, not
+// a plain space); since the source markup used it as ordinary inter-word
+// spacing, it is folded to a regular space here too.
+//
+// A '<' only starts a tag when followed by a letter, '/', '!' or '?' —
+// HTML's own rule for what can open a tag, close tag, comment/doctype, or
+// processing instruction. Anything else (a digit, a space, end of string)
+// is literal text, so "a < b" is not mistaken for markup.
+func stripMarkup(s string) string {
+ var b strings.Builder
+ b.Grow(len(s))
+ i, n := 0, len(s)
+ for i < n {
+ if s[i] != '<' || !startsTag(s, i) {
+ b.WriteByte(s[i])
+ i++
+ continue
+ }
+
+ if strings.HasPrefix(s[i:], "<!--") {
+ end := n
+ if k := strings.Index(s[i+4:], "-->"); k != -1 {
+ end = i + 4 + k + len("-->")
+ }
+ b.WriteByte(' ')
+ i = end
+ continue
+ }
+
+ j := i + 1
+ closing := false
+ if j < n && s[j] == '/' {
+ closing = true
+ j++
+ }
+ nameStart := j
+ for j < n && isTagNameByte(s[j]) {
+ j++
+ }
+ name := strings.ToLower(s[nameStart:j])
+
+ gt := strings.IndexByte(s[j:], '>')
+ if gt == -1 {
+ // D1: an unterminated tag (no closing '>') can no longer be
+ // parsed as markup, but that is no reason to discard the rest
+ // of the file - copy it through as literal text instead of
+ // simply stopping the scan.
+ b.WriteString(s[i:])
+ break
+ }
+ end := j + gt + 1
+
+ if !closing && (name == "script" || name == "style") {
+ if close := indexCloseTag(s[end:], name); close != -1 {
+ end += close
+ if gt2 := strings.IndexByte(s[end:], '>'); gt2 != -1 {
+ end += gt2 + 1
+ } else {
+ end = n
+ }
+ } else {
+ end = n
+ }
+ }
+
+ b.WriteByte(' ')
+ i = end
+ }
+
+ return strings.ReplaceAll(html.UnescapeString(b.String()), string(nbsp), " ")
+}
+
+// startsTag reports whether s[i] == '<' begins a tag-like construct: the
+// next character is a letter, '/', '!' or '?'. A '<' at the very end of s,
+// or followed by anything else (digit, space, punctuation), is literal
+// text instead.
+func startsTag(s string, i int) bool {
+ if i+1 >= len(s) {
+ return false
+ }
+ c := s[i+1]
+ return c == '/' || c == '!' || c == '?' ||
+ (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
+}
+
+// nbsp is U+00A0, NO-BREAK SPACE: what html.UnescapeString decodes &nbsp;
+// to, folded to a regular space since the source markup used it as one.
+const nbsp = rune(0xA0)
+
+// isTagNameByte reports whether c can appear in an HTML/XML tag name.
+func isTagNameByte(c byte) bool {
+ return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '-' || c == ':' || c == '_'
+}
+
+// indexCloseTag returns the index in s of the first ASCII case-insensitive
+// occurrence of "</name", or -1.
+func indexCloseTag(s, name string) int {
+ target := "</" + name
+ tn := len(target)
+ for i := 0; i+tn <= len(s); i++ {
+ if asciiEqualFold(s[i:i+tn], target) {
+ return i
+ }
+ }
+ return -1
+}
+
+// asciiEqualFold reports whether a and b are equal, ASCII letters compared
+// without regard to case.
+func asciiEqualFold(a, b string) bool {
+ if len(a) != len(b) {
+ return false
+ }
+ for i := 0; i < len(a); i++ {
+ ca, cb := a[i], b[i]
+ if 'A' <= ca && ca <= 'Z' {
+ ca += 'a' - 'A'
+ }
+ if 'A' <= cb && cb <= 'Z' {
+ cb += 'a' - 'A'
+ }
+ if ca != cb {
+ return false
+ }
+ }
+ return true
+}
diff --git a/internal/extract/plain_test.go b/internal/extract/plain_test.go
new file mode 100644
index 0000000..045ea26
--- /dev/null
+++ b/internal/extract/plain_test.go
@@ -0,0 +1,229 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package extract
+
+import (
+ "context"
+ "errors"
+ "os"
+ "path/filepath"
+ "strings"
+ "syscall"
+ "testing"
+)
+
+// file writes data to name in a temp dir and returns the path.
+func file(t *testing.T, name string, data []byte) string {
+ t.Helper()
+ p := filepath.Join(t.TempDir(), name)
+ if err := os.WriteFile(p, data, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ return p
+}
+
+func text(t *testing.T, e *Extractor, p string, maxRead int64) (string, error) {
+ t.Helper()
+ fi, err := os.Stat(p)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return e.Text(context.Background(), p, fi.Size(), maxRead)
+}
+
+func TestPlainEncodings(t *testing.T) {
+ e := newWithPath("")
+ utf16le := []byte{0xFF, 0xFE, 'A', 0, 'c', 0, 'm', 0, 'e', 0}
+ tests := []struct {
+ name string
+ data []byte
+ want string
+ }{
+ {"a.txt", []byte("Faktura acme ltd\n"), "Faktura acme ltd\n"},
+ {"bom.txt", []byte("\xEF\xBB\xBFhello"), "hello"},
+ {"latin1.txt", []byte("Gr\xfc\xdfe"), "Grüße"},
+ {"u16.txt", utf16le, "Acme"},
+ {"notes.md", []byte("# Title\nbody"), "# Title\nbody"},
+ {"empty.txt", nil, ""},
+ {"README", []byte("no extension but text"), "no extension but text"},
+ }
+ for _, tt := range tests {
+ got, err := text(t, e, file(t, tt.name, tt.data), 0)
+ if err != nil || got != tt.want {
+ t.Errorf("%s: got %q, %v; want %q", tt.name, got, err, tt.want)
+ }
+ }
+}
+
+func TestMarkup(t *testing.T) {
+ e := newWithPath("")
+ tests := []struct {
+ name string
+ src string
+ want []string // at least one of these must be a substring
+ wantAny bool // if true, want is an alternative set: any one suffices
+ bad []string // none of these may be a substring
+ }{
+ {
+ name: "tags, script/style dropped, entities decoded",
+ src: "<html><head><style>p{color:red}</style><script>var x='acme'</script></head>" +
+ "<body><p>Faktura&nbsp;VAT &amp; co</p><p>acme&#32;ltd</p></body></html>",
+ want: []string{"Faktura VAT & co", "acme ltd"},
+ bad: []string{"color:red", "var x", "<p>"},
+ },
+ {
+ name: "a lone < followed by a digit or space is literal text",
+ src: "<p>price < 500 zl, done</p>",
+ want: []string{"price < 500 zl, done"},
+ wantAny: true,
+ },
+ {
+ name: "an HTML comment is skipped, not its neighbours",
+ src: "<p>a<!-- hidden -->b</p>",
+ want: []string{"a b", "ab"},
+ wantAny: true,
+ bad: []string{"hidden"},
+ },
+ }
+ for _, tt := range tests {
+ got, err := text(t, e, file(t, "page.html", []byte(tt.src)), 0)
+ if err != nil {
+ t.Fatalf("%s: %v", tt.name, err)
+ }
+ if tt.wantAny {
+ ok := false
+ for _, w := range tt.want {
+ if strings.Contains(got, w) {
+ ok = true
+ break
+ }
+ }
+ if !ok {
+ t.Errorf("%s: markup text %q has none of %q", tt.name, got, tt.want)
+ }
+ } else {
+ for _, want := range tt.want {
+ if !strings.Contains(got, want) {
+ t.Errorf("%s: markup text %q lacks %q", tt.name, got, want)
+ }
+ }
+ }
+ for _, bad := range tt.bad {
+ if strings.Contains(got, bad) {
+ t.Errorf("%s: markup text %q still contains %q", tt.name, got, bad)
+ }
+ }
+ }
+}
+
+func TestUnsupportedAndTooLarge(t *testing.T) {
+ e := newWithPath("")
+ png := []byte("\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR")
+ if _, err := text(t, e, file(t, "image.png", png), 0); !errors.Is(err, ErrUnsupported) {
+ t.Errorf("png: %v, want ErrUnsupported", err)
+ }
+ if _, err := text(t, e, file(t, "blob.bin", []byte{1, 2, 0, 3}), 0); !errors.Is(err, ErrUnsupported) {
+ t.Errorf("binary: %v, want ErrUnsupported", err)
+ }
+ if _, err := text(t, e, file(t, "big.txt", []byte(strings.Repeat("x", 100))), 50); !errors.Is(err, ErrTooLarge) {
+ t.Errorf("big: %v, want ErrTooLarge", err)
+ }
+}
+
+func TestToolMissingError(t *testing.T) {
+ err := &ToolMissingError{Tool: "pdftotext"}
+ if err.Error() != "needs pdftotext, not installed" {
+ t.Fatalf("got %q", err.Error())
+ }
+}
+
+func TestToolsListedInOrder(t *testing.T) {
+ var names []string
+ for _, tl := range newWithPath("").Tools() {
+ names = append(names, tl.Name)
+ if tl.Path != "" {
+ t.Errorf("%s found with an empty PATH", tl.Name)
+ }
+ }
+ if strings.Join(names, " ") != "pdftotext antiword catdoc xls2csv catppt" {
+ t.Fatalf("tools = %v", names)
+ }
+}
+
+// TestSniffWholeFileMustBeValid: the first 8 KiB sniffs as plain ASCII text,
+// but the file goes on to hold an invalid UTF-8 byte and a NUL past that
+// sample — sniffText must reject the whole file, not just decode what the
+// sample alone promised (it must not fall back to Latin-1 the way a known
+// text extension would).
+func TestSniffWholeFileMustBeValid(t *testing.T) {
+ e := newWithPath("")
+ data := append([]byte(strings.Repeat("x", 8192)), 0xFF, 0x00)
+ if _, err := text(t, e, file(t, "blob.data", data), 0); !errors.Is(err, ErrUnsupported) {
+ t.Errorf("got %v, want ErrUnsupported", err)
+ }
+}
+
+// TestSniffLargeBinaryRejected: an unrecognised-extension file whose very
+// first byte is invalid UTF-8 is rejected from the sample alone; this only
+// checks the outcome (ErrUnsupported), not that the rest of the megabyte
+// went unread — that efficiency claim isn't something a black-box test can
+// time reliably.
+func TestSniffLargeBinaryRejected(t *testing.T) {
+ e := newWithPath("")
+ data := make([]byte, 1<<20) // 1 MiB, far past the 8 KiB sniff window
+ data[0] = 0xFF // invalid UTF-8 lead byte, visible in the sample
+ if _, err := text(t, e, file(t, "huge.blob", data), 0); !errors.Is(err, ErrUnsupported) {
+ t.Errorf("got %v, want ErrUnsupported", err)
+ }
+}
+
+// TestToolLookupSkipsNonRegular: a FIFO named like a tool, executable bits
+// and all, must never be picked up — only a regular file counts.
+func TestToolLookupSkipsNonRegular(t *testing.T) {
+ dir := t.TempDir()
+ fifo := filepath.Join(dir, "pdftotext")
+ if err := syscall.Mkfifo(fifo, 0o755); err != nil {
+ t.Skipf("mkfifo not available: %v", err)
+ }
+ for _, tl := range newWithPath(dir).Tools() {
+ if tl.Name == "pdftotext" && tl.Path != "" {
+ t.Errorf("pdftotext resolved to a non-regular file: %s", tl.Path)
+ }
+ }
+}
+
+// TestUnterminatedTagKeepsRemainder: D1. An unterminated ordinary tag (no
+// closing '>') must not discard the rest of the file - only the malformed
+// tag markup itself is unrecoverable; whatever follows it is still real
+// content and must still reach the extracted text.
+func TestUnterminatedTagKeepsRemainder(t *testing.T) {
+ e := newWithPath("")
+ src := "<p>before</p><p unterminated text after"
+ got, err := text(t, e, file(t, "broken.html", []byte(src)), 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(got, "before") {
+ t.Errorf("text before the unterminated tag missing: %q", got)
+ }
+ if !strings.Contains(got, "unterminated text after") {
+ t.Errorf("text after the unterminated tag was discarded: %q", got)
+ }
+}
+
+// TestSniffRuneStraddlingSampleBoundary: D2. A multi-byte rune ("ż", two
+// UTF-8 bytes) placed exactly so its lead byte is the sniff sample's last
+// byte and its continuation byte falls just past it must not make an
+// otherwise valid UTF-8 file sniff as unsupported.
+func TestSniffRuneStraddlingSampleBoundary(t *testing.T) {
+ e := newWithPath("")
+ prefix := strings.Repeat("a", sniffSize-1)
+ data := []byte(prefix + "ż" + "bcd")
+ got, err := text(t, e, file(t, "straddle.blob", data), 0)
+ if err != nil {
+ t.Fatalf("valid UTF-8 with a rune straddling the sniff boundary: %v", err)
+ }
+ if want := "żbcd"; !strings.HasSuffix(got, want) {
+ t.Errorf("got tail %q, want it to end in %q", got[len(got)-8:], want)
+ }
+}
diff --git a/internal/extract/tools.go b/internal/extract/tools.go
new file mode 100644
index 0000000..ca6a9b0
--- /dev/null
+++ b/internal/extract/tools.go
@@ -0,0 +1,246 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package extract
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "os/exec"
+ "path/filepath"
+ "time"
+)
+
+// maxToolOutput caps the bytes kept from an external tool's stdout; more
+// than this returns ErrTooLarge. A package var, not a const, so a test can
+// lower it without generating gigabytes of fake output. B1: the directory's
+// max-read may cap a single extraction further still — see budget.
+var maxToolOutput int64 = 64 << 20
+
+// budget returns the smaller of fixed (the package's own default ceiling —
+// maxToolOutput or zipBudget) and maxRead, the directory's configured
+// max-read; maxRead 0 means unlimited, so fixed alone applies. B1: a
+// single extraction's output must never exceed the ceiling the user set,
+// even when that ceiling is below the fixed default.
+func budget(fixed, maxRead int64) int64 {
+ if maxRead > 0 && maxRead < fixed {
+ return maxRead
+ }
+ return fixed
+}
+
+// maxStderr caps the bytes kept from an external tool's stderr — enough
+// for a diagnostic first line. Unlike stdout, crossing this never kills
+// the tool: stderr noise is not grounds to abort an otherwise-working
+// extraction, only grounds to stop remembering more of it.
+const maxStderr = 4 << 10
+
+// maxErrLine caps how much of stderr's first line is folded into the
+// error text run() returns, so a flooding tool cannot make that message
+// itself unbounded.
+const maxErrLine = 200
+
+// boundedWriter keeps at most limit bytes written to it and silently
+// discards the rest, always reporting success to the writer — an
+// io.Writer that returns an error would abort the copy goroutine
+// exec.Cmd runs for Stdout/Stderr, which is not what should happen here:
+// the tool must keep being drained (or be killed outright, via
+// onOverflow) rather than have its pipe start backing up. If onOverflow
+// is set, it fires exactly once, the moment the total ever written first
+// exceeds limit; run() uses it on stdout, and only stdout, to cancel the
+// command immediately rather than let an over-producing tool run until
+// e.Timeout. Write is only ever called by the single copy goroutine
+// exec.Cmd runs per stream, so no locking is needed; run() only reads a
+// boundedWriter's fields after cmd.Run() has returned, which happens
+// strictly after that goroutine has finished (Wait's documented
+// synchronisation), giving the read a safe happens-before.
+type boundedWriter struct {
+ limit int64
+ onOverflow func()
+
+ buf bytes.Buffer
+ total int64
+ overflowed bool
+}
+
+func (w *boundedWriter) Write(p []byte) (int, error) {
+ w.total += int64(len(p))
+ if w.total > w.limit {
+ if !w.overflowed {
+ w.overflowed = true
+ if w.onOverflow != nil {
+ w.onOverflow()
+ }
+ }
+ return len(p), nil
+ }
+ w.buf.Write(p)
+ return len(p), nil
+}
+
+// run executes tool (looked up in e.tools, its absolute path) with args,
+// under a timeout of e.Timeout, and returns its stdout as text, its
+// stdout capped at maxOut bytes (the caller passes budget(maxToolOutput,
+// maxRead), B1). No shell is involved: exec.CommandContext runs the
+// tool's path directly with args passed separately, so nothing in a
+// hostile filename or argument is ever interpreted. The environment is
+// inherited unchanged.
+//
+// os/exec is left to own all the copying — cmd.Stdout and cmd.Stderr are
+// bounded writers, and cmd.Run does the reading — so that cmd.WaitDelay's
+// hang protection actually applies: WaitDelay bounds how long Wait spends
+// on I/O after the process itself has exited (or after ctx is done),
+// forcibly closing the pipes once that grace period elapses. An earlier
+// version of this function read stdout itself, ahead of Wait, which
+// starved WaitDelay of the thing it bounds: once that manual read
+// stopped (at EOF, or at the output cap), Wait was left blocked on
+// whatever was still holding the pipe open — a grandchild the tool
+// backgrounded and left running, or the tool itself blocked writing to a
+// pipe nobody was draining once the cap was hit — with nothing left to
+// force it closed. In both shapes the call could run for the full
+// e.Timeout (or longer) instead of returning promptly.
+//
+// Stdout is capped at maxOut bytes (the caller's budget(maxToolOutput,
+// maxRead), B1): crossing it cancels the command immediately, via the
+// boundedWriter's onOverflow, and run reports ErrTooLarge. Stderr is
+// capped at maxStderr bytes and never
+// cancels anything; only its first line, truncated to maxErrLine bytes,
+// ever reaches an error message, so a tool flooding stderr costs bounded
+// memory and produces a bounded error.
+//
+// Killing the command — by the caller's ctx being cancelled, by
+// e.Timeout expiring, or by the stdout cap being crossed — can leave
+// cmd.Run reporting exec.ErrWaitDelay even though the process's own exit
+// status was clean: SIGKILL forces the pipes closed without giving the
+// child a chance to flush or exit on its own. That alone is not a
+// failure (see the ErrWaitDelay case below); only a genuinely non-zero
+// exit is treated as one.
+//
+// A non-zero exit returns "<tool> failed: <first line of stderr>" (or
+// "<tool> failed: <err>" when stderr was empty), the line capped to
+// maxErrLine bytes. A caller-cancelled ctx returns promptly, with an
+// error wrapping ctx.Err(); e.Timeout expiring on its own returns
+// "<tool> timed out after <Timeout>".
+func (e *Extractor) run(ctx context.Context, maxOut int64, tool string, args ...string) (string, error) {
+ path := e.tools[tool]
+
+ runCtx, cancel := context.WithTimeout(ctx, e.Timeout)
+ defer cancel()
+
+ cmd := exec.CommandContext(runCtx, path, args...)
+ cmd.WaitDelay = time.Second
+
+ stdout := &boundedWriter{limit: maxOut, onOverflow: cancel}
+ stderr := &boundedWriter{limit: maxStderr}
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+
+ err := cmd.Run()
+
+ switch {
+ case stdout.overflowed:
+ // Checked first: killing the tool for overflow also cancels
+ // runCtx, so without this ordering the case below would report
+ // the cancellation as a timeout instead of what it actually was.
+ return "", ErrTooLarge
+ case ctx.Err() != nil:
+ // The caller's own context, not the internal e.Timeout deadline
+ // derived from it — checked before runCtx's, since runCtx
+ // inherits the caller's cancellation too and would otherwise be
+ // indistinguishable from it below.
+ return "", fmt.Errorf("%s: %w", tool, ctx.Err())
+ case runCtx.Err() == context.DeadlineExceeded:
+ return "", fmt.Errorf("%s timed out after %s", tool, e.Timeout)
+ }
+
+ if err != nil {
+ if errors.Is(err, exec.ErrWaitDelay) && cmd.ProcessState != nil && cmd.ProcessState.ExitCode() == 0 {
+ return stdout.buf.String(), nil
+ }
+ if line := truncate(firstLine(stderr.buf.Bytes()), maxErrLine); line != "" {
+ return "", fmt.Errorf("%s failed: %s", tool, line)
+ }
+ return "", fmt.Errorf("%s failed: %s", tool, err)
+ }
+ return stdout.buf.String(), nil
+}
+
+// firstLine returns the first non-empty line of b, trimmed of its
+// trailing newline, or "" if b holds nothing but blank lines.
+func firstLine(b []byte) string {
+ for _, line := range bytes.Split(b, []byte("\n")) {
+ if len(bytes.TrimSpace(line)) > 0 {
+ return string(bytes.TrimRight(line, "\r"))
+ }
+ }
+ return ""
+}
+
+// truncate returns s cut to at most n bytes, so text built from
+// untrusted tool output has a hard, predictable bound on its length
+// regardless of what the tool wrote. It may cut a multi-byte UTF-8
+// sequence in two; a trailing partial rune in a diagnostic error message
+// is an acceptable cost for a byte bound that never slips.
+func truncate(s string, n int) string {
+ if len(s) > n {
+ return s[:n]
+ }
+ return s
+}
+
+// pdfText extracts text from a PDF with pdftotext, its output capped per
+// budget(maxToolOutput, maxRead) (B1).
+func (e *Extractor) pdfText(ctx context.Context, path string, maxRead int64) (string, error) {
+ if e.tools["pdftotext"] == "" {
+ return "", &ToolMissingError{Tool: "pdftotext"}
+ }
+ abs, err := filepath.Abs(path)
+ if err != nil {
+ return "", err
+ }
+ return e.run(ctx, budget(maxToolOutput, maxRead), "pdftotext", "-q", "-enc", "UTF-8", abs, "-")
+}
+
+// legacyText extracts text from a legacy binary Office format (doc xls
+// ppt) with the external tool toolExt names, its output capped per
+// budget(maxToolOutput, maxRead) (B1). .doc prefers antiword, falling
+// back to catdoc if antiword is absent or fails.
+func (e *Extractor) legacyText(ctx context.Context, path, ext string, maxRead int64) (string, error) {
+ abs, err := filepath.Abs(path)
+ if err != nil {
+ return "", err
+ }
+ out := budget(maxToolOutput, maxRead)
+
+ switch ext {
+ case "doc":
+ haveAntiword := e.tools["antiword"] != ""
+ haveCatdoc := e.tools["catdoc"] != ""
+ if !haveAntiword && !haveCatdoc {
+ return "", &ToolMissingError{Tool: "antiword or catdoc"}
+ }
+ if haveAntiword {
+ text, err := e.run(ctx, out, "antiword", "-m", "UTF-8.txt", abs)
+ if err == nil {
+ return text, nil
+ }
+ if !haveCatdoc {
+ return "", err
+ }
+ }
+ return e.run(ctx, out, "catdoc", "-d", "utf-8", abs)
+ case "xls":
+ if e.tools["xls2csv"] == "" {
+ return "", &ToolMissingError{Tool: "xls2csv"}
+ }
+ return e.run(ctx, out, "xls2csv", "-d", "utf-8", abs)
+ case "ppt":
+ if e.tools["catppt"] == "" {
+ return "", &ToolMissingError{Tool: "catppt"}
+ }
+ return e.run(ctx, out, "catppt", "-d", "utf-8", abs)
+ default:
+ return "", errors.New("extract: unreachable: legacyText called with unknown extension " + ext)
+ }
+}
diff --git a/internal/extract/tools_test.go b/internal/extract/tools_test.go
new file mode 100644
index 0000000..4a03f1b
--- /dev/null
+++ b/internal/extract/tools_test.go
@@ -0,0 +1,239 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package extract
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+)
+
+// fakeTool writes an executable shell script named name into dir.
+func fakeTool(t *testing.T, dir, name, body string) {
+ t.Helper()
+ script := "#!/bin/sh\n" + body + "\n"
+ if err := os.WriteFile(filepath.Join(dir, name), []byte(script), 0o755); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestPdfViaFakeTool(t *testing.T) {
+ bin := t.TempDir()
+ argsFile := filepath.Join(t.TempDir(), "args")
+ fakeTool(t, bin, "pdftotext", `printf '%s\n' "$@" > "`+argsFile+`"; echo "acme ltd invoice"`)
+ e := newWithPath(bin)
+ p := file(t, "-leading-dash.pdf", []byte("%PDF-1.4"))
+ got, err := text(t, e, p, 0)
+ if err != nil || strings.TrimSpace(got) != "acme ltd invoice" {
+ t.Fatalf("got %q, %v", got, err)
+ }
+ args, _ := os.ReadFile(argsFile)
+ want := "-q\n-enc\nUTF-8\n" + p + "\n-\n"
+ if string(args) != want {
+ t.Fatalf("pdftotext args:\n%q\nwant\n%q", args, want)
+ }
+ if !filepath.IsAbs(strings.Split(string(args), "\n")[3]) {
+ t.Fatal("path argument is not absolute")
+ }
+}
+
+func TestLegacyFormats(t *testing.T) {
+ bin := t.TempDir()
+ fakeTool(t, bin, "catdoc", `echo "from catdoc"`)
+ fakeTool(t, bin, "xls2csv", `echo "from xls2csv"`)
+ fakeTool(t, bin, "catppt", `echo "from catppt"`)
+ e := newWithPath(bin) // no antiword: .doc falls back to catdoc
+ for ext, want := range map[string]string{"doc": "from catdoc", "xls": "from xls2csv", "ppt": "from catppt"} {
+ got, err := text(t, e, file(t, "f."+ext, []byte("x")), 0)
+ if err != nil || strings.TrimSpace(got) != want {
+ t.Errorf("%s: got %q, %v", ext, got, err)
+ }
+ }
+ fakeTool(t, bin, "antiword", `echo "from antiword"`)
+ e = newWithPath(bin)
+ if got, _ := text(t, e, file(t, "g.doc", []byte("x")), 0); strings.TrimSpace(got) != "from antiword" {
+ t.Errorf("antiword not preferred: %q", got)
+ }
+ fakeTool(t, bin, "antiword", `echo "antiword broke" >&2; exit 1`)
+ e = newWithPath(bin)
+ if got, _ := text(t, e, file(t, "h.doc", []byte("x")), 0); strings.TrimSpace(got) != "from catdoc" {
+ t.Errorf("no fallback to catdoc after antiword failed: %q", got)
+ }
+}
+
+func TestToolErrors(t *testing.T) {
+ e := newWithPath(t.TempDir())
+ var tm *ToolMissingError
+ if _, err := text(t, e, file(t, "a.pdf", []byte("x")), 0); !errors.As(err, &tm) || tm.Error() != "needs pdftotext, not installed" {
+ t.Errorf("missing pdftotext: %v", err)
+ }
+ if _, err := text(t, e, file(t, "a.doc", []byte("x")), 0); err == nil || err.Error() != "needs antiword or catdoc, not installed" {
+ t.Errorf("missing doc tools: %v", err)
+ }
+ bin := t.TempDir()
+ fakeTool(t, bin, "pdftotext", `echo "Syntax Error: broken xref" >&2; exit 3`)
+ e = newWithPath(bin)
+ if _, err := text(t, e, file(t, "b.pdf", []byte("x")), 0); err == nil || err.Error() != "pdftotext failed: Syntax Error: broken xref" {
+ t.Errorf("failing tool: %v", err)
+ }
+ fakeTool(t, bin, "pdftotext", `sleep 5`)
+ e = newWithPath(bin)
+ e.Timeout = 200 * time.Millisecond
+ start := time.Now()
+ _, err := text(t, e, file(t, "c.pdf", []byte("x")), 0)
+ if err == nil || !strings.Contains(err.Error(), "pdftotext timed out after 200ms") {
+ t.Errorf("slow tool: %v", err)
+ }
+ if time.Since(start) > 3*time.Second {
+ t.Errorf("timeout took %v", time.Since(start))
+ }
+}
+
+// minimalPDF returns a valid one-page PDF showing text in Helvetica.
+func minimalPDF(text string) []byte {
+ var b bytes.Buffer
+ var offsets []int
+ obj := func(s string) { offsets = append(offsets, b.Len()); b.WriteString(s) }
+ b.WriteString("%PDF-1.4\n")
+ obj("1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n")
+ obj("2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj\n")
+ obj("3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >> endobj\n")
+ stream := "BT /F1 24 Tf 72 700 Td (" + text + ") Tj ET"
+ obj(fmt.Sprintf("4 0 obj << /Length %d >> stream\n%s\nendstream endobj\n", len(stream), stream))
+ obj("5 0 obj << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> endobj\n")
+ xref := b.Len()
+ fmt.Fprintf(&b, "xref\n0 %d\n0000000000 65535 f \n", len(offsets)+1)
+ for _, o := range offsets {
+ fmt.Fprintf(&b, "%010d 00000 n \n", o)
+ }
+ fmt.Fprintf(&b, "trailer << /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n", len(offsets)+1, xref)
+ return b.Bytes()
+}
+
+func TestRealPdftotext(t *testing.T) {
+ if _, err := exec.LookPath("pdftotext"); err != nil {
+ t.Skip("pdftotext not installed")
+ }
+ e := New()
+ got, err := text(t, e, file(t, "hello.pdf", minimalPDF("Hello acme ltd")), 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(got, "Hello acme ltd") {
+ t.Fatalf("pdftotext gave %q", got)
+ }
+}
+
+// TestOrphanChildDoesNotHangRun: a tool that exits itself but leaves a
+// backgrounded grandchild holding its stdout pipe open must not make
+// run() wait for that grandchild. Only os/exec itself, copying into
+// cmd.Stdout and bounding the post-exit wait via WaitDelay, can end this
+// promptly; run() manually draining a StdoutPipe before calling Wait
+// starves WaitDelay of the thing it bounds, since by the time Wait runs
+// there is nothing left for it to forcibly cut off.
+func TestOrphanChildDoesNotHangRun(t *testing.T) {
+ bin := t.TempDir()
+ fakeTool(t, bin, "pdftotext", "sleep 5 & echo x")
+ e := newWithPath(bin)
+ e.Timeout = 30 * time.Second
+
+ start := time.Now()
+ got, err := e.run(context.Background(), maxToolOutput, "pdftotext")
+ if err != nil {
+ t.Fatalf("orphan: %v", err)
+ }
+ if strings.TrimSpace(got) != "x" {
+ t.Errorf("orphan: got %q, want %q", got, "x")
+ }
+ if d := time.Since(start); d > 3*time.Second {
+ t.Errorf("orphan: took %v, want well under e.Timeout (30s) and under 3s", d)
+ }
+}
+
+// TestStdoutOverflowKillsToolPromptly: a tool that keeps writing past
+// maxToolOutput must be killed the moment the cap is crossed, not left
+// running until e.Timeout expires — once run() stops draining its pipe,
+// an unkilled tool blocks on its own write() and never exits on its own.
+func TestStdoutOverflowKillsToolPromptly(t *testing.T) {
+ old := maxToolOutput
+ maxToolOutput = 1 << 20
+ defer func() { maxToolOutput = old }()
+
+ bin := t.TempDir()
+ fakeTool(t, bin, "pdftotext", "yes aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
+ e := newWithPath(bin)
+ e.Timeout = 30 * time.Second
+
+ start := time.Now()
+ _, err := e.run(context.Background(), maxToolOutput, "pdftotext")
+ if !errors.Is(err, ErrTooLarge) {
+ t.Fatalf("overflow: got %v, want ErrTooLarge", err)
+ }
+ if d := time.Since(start); d > 5*time.Second {
+ t.Errorf("overflow: took %v, want under 5s (well under e.Timeout=30s)", d)
+ }
+}
+
+// TestCallerCancelReturnsPromptly: cancelling the ctx passed to run()
+// (distinct from e.Timeout's own internal deadline, which is untouched
+// here) must stop the tool and return quickly, with an error wrapping
+// the caller's own context.Canceled — not silently absorbed into a
+// generic "<tool> failed: ..." string, and not held open until
+// e.Timeout.
+func TestCallerCancelReturnsPromptly(t *testing.T) {
+ bin := t.TempDir()
+ fakeTool(t, bin, "pdftotext", "sleep 10")
+ e := newWithPath(bin)
+ e.Timeout = 30 * time.Second
+
+ ctx, cancel := context.WithCancel(context.Background())
+ time.AfterFunc(100*time.Millisecond, cancel)
+
+ start := time.Now()
+ _, err := e.run(ctx, maxToolOutput, "pdftotext")
+ if !errors.Is(err, context.Canceled) {
+ t.Fatalf("caller cancel: got %v, want an error wrapping context.Canceled", err)
+ }
+ if d := time.Since(start); d > 3*time.Second {
+ t.Errorf("caller cancel: took %v, want under 3s", d)
+ }
+}
+
+// TestStderrFloodBounded: a tool that floods stderr must not blow up the
+// size of the error message run() produces — only a bounded prefix of
+// its first line may ever reach the returned error text, and capturing
+// it at all must not cost unbounded memory.
+func TestStderrFloodBounded(t *testing.T) {
+ bin := t.TempDir()
+ fakeTool(t, bin, "pdftotext", `head -c 10000000 /dev/zero | tr '\0' x >&2; exit 1`)
+ e := newWithPath(bin)
+
+ _, err := e.run(context.Background(), maxToolOutput, "pdftotext")
+ if err == nil {
+ t.Fatal("stderr flood: want an error")
+ }
+ if max := len("pdftotext failed: ") + 200; len(err.Error()) > max {
+ t.Errorf("stderr flood: message is %d bytes, want <=%d", len(err.Error()), max)
+ }
+}
+
+// TestMaxReadCapsToolOutput: B1. A directory's max-read, when smaller than
+// the fixed maxToolOutput default, caps a single tool's output on its
+// own — proven here with maxToolOutput left at its default, so only
+// budget(maxToolOutput, maxRead) picking the smaller maxRead explains the
+// result.
+func TestMaxReadCapsToolOutput(t *testing.T) {
+ bin := t.TempDir()
+ fakeTool(t, bin, "pdftotext", `yes aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa | head -c 200000`)
+ e := newWithPath(bin)
+ if _, err := text(t, e, file(t, "small.pdf", []byte("%PDF-1.4")), 1024); !errors.Is(err, ErrTooLarge) {
+ t.Fatalf("got %v, want ErrTooLarge (max-read 1024 should have capped a 200000-byte tool output)", err)
+ }
+}
diff --git a/internal/extract/zipxml.go b/internal/extract/zipxml.go
new file mode 100644
index 0000000..1b3ba82
--- /dev/null
+++ b/internal/extract/zipxml.go
@@ -0,0 +1,224 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package extract
+
+import (
+ "archive/zip"
+ "context"
+ "encoding/xml"
+ "errors"
+ "fmt"
+ "io"
+ "path"
+ "strings"
+)
+
+// zipBudget caps the total uncompressed bytes read from one archive's
+// matched entries, guarding against a zip bomb; a test lowers it. B1: the
+// directory's max-read may cap a single archive further still — see
+// budget (tools.go).
+var zipBudget int64 = 64 << 20
+
+// zipPatterns maps a zip-based format's extension to the path.Match
+// patterns (tried in matchEntry) of the entries that carry its text.
+// Invoices often carry the tax number only in a header or footer, which is
+// why docx's header*/footer* entries are included; ODF keeps headers and
+// footers in styles.xml, not content.xml, so both are read.
+var zipPatterns = map[string][]string{
+ "docx": {"word/document.xml", "word/header*.xml", "word/footer*.xml", "word/footnotes.xml"},
+ "xlsx": {"xl/sharedStrings.xml", "xl/worksheets/sheet*.xml"},
+ "pptx": {"ppt/slides/slide*.xml"},
+ "odt": {"content.xml", "styles.xml"},
+ "ods": {"content.xml", "styles.xml"},
+ "odp": {"content.xml", "styles.xml"},
+ "epub": {"*.xhtml", "*.html", "*.htm"},
+}
+
+// zipText extracts text from a zip-based document (docx xlsx pptx odt ods
+// odp epub): it opens the archive and, for each entry matching the
+// format's zipPatterns in archive order, decodes its XML character data
+// into the result with xmlText, separating entries with a newline. A
+// corrupt archive returns the zip package's own error, not
+// ErrUnsupported: the file claims a format it does not have, which the
+// user should see. Reading stops with ErrTooLarge immediately once the
+// entries read from the archive exceed budget(zipBudget, maxRead)
+// uncompressed bytes in total (B1: maxRead, the directory's configured
+// ceiling, may cap this lower than the fixed zipBudget). ctx is checked
+// between entries so a cancelled extraction stops
+// promptly.
+//
+// Any other per-entry failure — the entry won't open, or its XML is
+// malformed — is lenient rather than fatal: whatever text that entry had
+// already yielded (xmlText writes as it walks, so a syntax error partway
+// through still leaves the text read up to that point) is kept, and the
+// archive keeps going to its remaining entries, since one bad part (a
+// corrupt header, say) should not blank out a document's otherwise
+// readable body. The first such error is remembered, wrapped as "<entry
+// name>: <err>", and returned only if no matched entry ever wrote any
+// character data at all — an error report is more useful than silent
+// empty text when nothing could be read. "Wrote any character data" is
+// tracked per entry (via the builder's length just before and after that
+// entry's own xmlText call, not the whole archive's final length): the
+// newline zipText adds to separate a successful entry from the next one
+// would otherwise make an entry that parsed cleanly but held no text of
+// its own (an empty element, say) look like it had produced something,
+// which could then mask a later entry's genuine failure.
+func zipText(ctx context.Context, path, ext string, maxRead int64) (string, error) {
+ zr, err := zip.OpenReader(path)
+ if err != nil {
+ return "", err
+ }
+ defer zr.Close()
+
+ patterns := zipPatterns[ext]
+ var b strings.Builder
+ remaining := budget(zipBudget, maxRead)
+ var firstErr error
+ wroteText := false
+ for _, f := range zr.File {
+ if err := ctx.Err(); err != nil {
+ return "", err
+ }
+ if !matchEntry(patterns, f.Name) {
+ continue
+ }
+ before := b.Len()
+ err := readZipEntry(f, &remaining, &b)
+ // Measured before the separator below is written, so a
+ // separator alone (an entry that parsed but held no character
+ // data) never counts as "wrote text" — only xmlText's own
+ // writes do, whether or not this entry went on to error.
+ if b.Len() > before {
+ wroteText = true
+ }
+ if err != nil {
+ if errors.Is(err, ErrTooLarge) {
+ return "", ErrTooLarge
+ }
+ if firstErr == nil {
+ firstErr = fmt.Errorf("%s: %w", f.Name, err)
+ }
+ continue
+ }
+ b.WriteByte('\n')
+ }
+ if !wroteText && firstErr != nil {
+ return "", firstErr
+ }
+ return b.String(), nil
+}
+
+// matchEntry reports whether name is one of the entries a format reads:
+// each pattern is tried first against the full entry name — which is what
+// the docx/xlsx/pptx/odt directory-qualified patterns need — and, failing
+// that, against name's base name. The base-name fallback is what lets
+// epub's bare "*.xhtml"/"*.html"/"*.htm" find chapters nested at any depth
+// inside the archive; applied to every format, it also means a nested
+// part sharing a matched base name is picked up deliberately, not by
+// accident — e.g. an ODF embedded object's own "Object 1/content.xml"
+// matches odt/ods/odp's bare "content.xml" pattern alongside the
+// document's own content.xml, because an embedded chart's or formula's
+// text is text the document shows its reader.
+func matchEntry(patterns []string, name string) bool {
+ base := path.Base(name)
+ for _, p := range patterns {
+ if ok, _ := path.Match(p, name); ok {
+ return true
+ }
+ if ok, _ := path.Match(p, base); ok {
+ return true
+ }
+ }
+ return false
+}
+
+// readZipEntry opens one matched zip entry, decodes its text into b
+// through a budgetedReader sharing remaining across the whole archive, and
+// reports ErrTooLarge if that budget was exceeded — checked on the reader
+// itself after xmlText returns, since the XML decoder may not pass the
+// reader's own error through unchanged (a truncated entry can look like a
+// cleanly finished document).
+func readZipEntry(f *zip.File, remaining *int64, b *strings.Builder) error {
+ rc, err := f.Open()
+ if err != nil {
+ return err
+ }
+ defer rc.Close()
+
+ br := &budgetedReader{r: rc, remaining: remaining}
+ err = xmlText(br, b)
+ if br.exceeded {
+ return ErrTooLarge
+ }
+ return err
+}
+
+// budgetedReader wraps a zip entry's reader, decrementing remaining — a
+// counter shared across every entry read from one archive — as bytes are
+// read. Once remaining is exhausted it stops reading and reports io.EOF
+// instead, recording that in exceeded so the caller can tell a genuine
+// end of document from a budget cutoff.
+type budgetedReader struct {
+ r io.Reader
+ remaining *int64
+ exceeded bool
+}
+
+func (br *budgetedReader) Read(p []byte) (int, error) {
+ if *br.remaining <= 0 {
+ br.exceeded = true
+ return 0, io.EOF
+ }
+ if int64(len(p)) > *br.remaining {
+ p = p[:*br.remaining]
+ }
+ n, err := br.r.Read(p)
+ *br.remaining -= int64(n)
+ return n, err
+}
+
+// xmlText appends the character data of one XML entry to b, with the
+// separators described above. inSharedCell tracks xlsx <c t="s">.
+func xmlText(r io.Reader, b *strings.Builder) error {
+ dec := xml.NewDecoder(r)
+ dec.Strict = false
+ dec.Entity = xml.HTMLEntity
+ sharedCell, inV := false, false
+ for {
+ tok, err := dec.Token()
+ if err == io.EOF {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ switch t := tok.(type) {
+ case xml.StartElement:
+ switch t.Name.Local {
+ case "s", "tab", "br", "line-break", "cr":
+ b.WriteByte(' ')
+ case "c":
+ sharedCell = false
+ for _, a := range t.Attr {
+ if a.Name.Local == "t" && a.Value == "s" {
+ sharedCell = true
+ }
+ }
+ case "v":
+ inV = true
+ }
+ case xml.EndElement:
+ switch t.Name.Local {
+ case "p", "h", "tc", "tr", "td", "th", "li", "si", "c", "row", "div", "title":
+ b.WriteByte('\n')
+ case "v":
+ inV = false
+ }
+ case xml.CharData:
+ if inV && sharedCell {
+ continue
+ }
+ b.Write(t)
+ }
+ }
+}
diff --git a/internal/extract/zipxml_test.go b/internal/extract/zipxml_test.go
new file mode 100644
index 0000000..14ead80
--- /dev/null
+++ b/internal/extract/zipxml_test.go
@@ -0,0 +1,205 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package extract
+
+import (
+ "archive/zip"
+ "bytes"
+ "errors"
+ "strings"
+ "testing"
+)
+
+// zipFile builds an archive from name -> content pairs and writes it to disk.
+func zipFile(t *testing.T, name string, entries map[string]string) string {
+ t.Helper()
+ var buf bytes.Buffer
+ w := zip.NewWriter(&buf)
+ for n, c := range entries {
+ f, err := w.Create(n)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := f.Write([]byte(c)); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := w.Close(); err != nil {
+ t.Fatal(err)
+ }
+ return file(t, name, buf.Bytes())
+}
+
+func TestZipFormats(t *testing.T) {
+ e := newWithPath("")
+ tests := []struct {
+ name string
+ entries map[string]string
+ want []string
+ }{
+ {"inv.docx", map[string]string{
+ "word/document.xml": `<w:document xmlns:w="w"><w:body><w:p><w:r><w:t>Ac</w:t></w:r><w:r><w:t>me Ltd</w:t></w:r></w:p><w:p><w:r><w:t>Faktura</w:t><w:tab/><w:t>VAT</w:t></w:r></w:p></w:body></w:document>`,
+ "word/footer1.xml": `<w:ftr xmlns:w="w"><w:p><w:r><w:t>NIP 0000000000</w:t></w:r></w:p></w:ftr>`,
+ "word/styles.xml": `<w:styles xmlns:w="w"><w:t>NOT-INCLUDED</w:t></w:styles>`,
+ }, []string{"Acme Ltd", "Faktura VAT", "NIP 0000000000"}},
+ {"sheet.xlsx", map[string]string{
+ "xl/sharedStrings.xml": `<sst><si><t>Invoice</t></si><si><t>acme ltd</t></si></sst>`,
+ "xl/worksheets/sheet1.xml": `<worksheet><sheetData><row><c t="s"><v>0</v></c><c><v>1234567890</v></c><c t="inlineStr"><is><t>inline text</t></is></c></row></sheetData></worksheet>`,
+ }, []string{"Invoice", "acme ltd", "1234567890", "inline text"}},
+ {"deck.pptx", map[string]string{
+ "ppt/slides/slide1.xml": `<p:sld xmlns:p="p" xmlns:a="a"><a:p><a:r><a:t>Quarterly report</a:t></a:r></a:p></p:sld>`,
+ }, []string{"Quarterly report"}},
+ {"letter.odt", map[string]string{
+ "content.xml": `<office:document-content xmlns:office="o" xmlns:text="t"><text:p>Faktura<text:s/>VAT</text:p></office:document-content>`,
+ "styles.xml": `<office:document-styles xmlns:office="o" xmlns:text="t"><text:p>header acme</text:p></office:document-styles>`,
+ }, []string{"Faktura VAT", "header acme"}},
+ {"book.epub", map[string]string{
+ "OEBPS/ch1.xhtml": `<html xmlns="h"><body><p>Chapter one&amp;two</p></body></html>`,
+ "mimetype": `application/epub+zip`,
+ }, []string{"Chapter one&two"}},
+ }
+ for _, tt := range tests {
+ got, err := text(t, e, zipFile(t, tt.name, tt.entries), 0)
+ if err != nil {
+ t.Errorf("%s: %v", tt.name, err)
+ continue
+ }
+ for _, w := range tt.want {
+ if !strings.Contains(got, w) {
+ t.Errorf("%s: text %q lacks %q", tt.name, got, w)
+ }
+ }
+ if strings.Contains(got, "NOT-INCLUDED") {
+ t.Errorf("%s: read an entry it should skip: %q", tt.name, got)
+ }
+ }
+}
+
+func TestSharedStringIndexNotText(t *testing.T) {
+ e := newWithPath("")
+ got, err := text(t, e, zipFile(t, "s.xlsx", map[string]string{
+ "xl/sharedStrings.xml": `<sst><si><t>alpha</t></si></sst>`,
+ "xl/worksheets/sheet1.xml": `<worksheet><sheetData><row><c t="s"><v>987654</v></c></row></sheetData></worksheet>`,
+ }), 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if strings.Contains(got, "987654") {
+ t.Fatalf("shared-string index leaked into text: %q", got)
+ }
+}
+
+func TestZipCorruptAndBudget(t *testing.T) {
+ e := newWithPath("")
+ if _, err := text(t, e, file(t, "broken.docx", []byte("not a zip at all")), 0); err == nil || errors.Is(err, ErrUnsupported) {
+ t.Errorf("corrupt docx: %v, want a zip error", err)
+ }
+ old := zipBudget
+ zipBudget = 1 << 10
+ defer func() { zipBudget = old }()
+ big := `<w:document xmlns:w="w"><w:p><w:t>` + strings.Repeat("x", 4096) + `</w:t></w:p></w:document>`
+ if _, err := text(t, e, zipFile(t, "huge.docx", map[string]string{"word/document.xml": big}), 0); !errors.Is(err, ErrTooLarge) {
+ t.Errorf("over budget: %v, want ErrTooLarge", err)
+ }
+}
+
+// TestOdfEmbeddedObjectIncluded: an ODF embedded object (a chart, a
+// formula) keeps its own content.xml inside a subdirectory such as
+// "Object 1/"; matchEntry's base-name fallback picks it up alongside the
+// document's own content.xml, deliberately — that embedded text is text
+// the document shows its reader.
+func TestOdfEmbeddedObjectIncluded(t *testing.T) {
+ e := newWithPath("")
+ got, err := text(t, e, zipFile(t, "embed.odt", map[string]string{
+ "content.xml": `<office:document-content xmlns:office="o" xmlns:text="t"><text:p>cover page</text:p></office:document-content>`,
+ "Object 1/content.xml": `<office:document-content xmlns:office="o" xmlns:text="t"><text:p>embedded chart acme</text:p></office:document-content>`,
+ }), 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(got, "embedded chart acme") {
+ t.Errorf("embedded object text missing: %q", got)
+ }
+}
+
+// TestZipLenientOnMalformedEntry: a malformed entry must not blank out
+// text already read from a good entry in the same archive, and the text
+// the malformed entry itself yielded before its own error is kept too.
+// The footer's mismatched end tag (</w:xyz> where </w:ftr> was open)
+// genuinely fails to parse under Strict=false — unlike a merely missing
+// end tag, which the decoder synthesises and swallows — but only after
+// "broken" and the newline for </w:p> have already been written; the
+// XML decoder itself confirms this token by token (see fix round 2's
+// report for the trace): CharData "broken", EndElement p, then the
+// error "unexpected end element </xyz>".
+func TestZipLenientOnMalformedEntry(t *testing.T) {
+ e := newWithPath("")
+ got, err := text(t, e, zipFile(t, "partial.docx", map[string]string{
+ "word/document.xml": `<w:document xmlns:w="w"><w:body><w:p><w:r><w:t>good body text</w:t></w:r></w:p></w:body></w:document>`,
+ "word/footer1.xml": `<w:ftr xmlns:w="w"><w:p><w:r><w:t>broken</w:t></w:r></w:p></w:xyz>`,
+ }), 0)
+ if err != nil {
+ t.Fatalf("good entry alongside a malformed one: %v", err)
+ }
+ if !strings.Contains(got, "good body text") {
+ t.Errorf("text from the good entry was discarded: %q", got)
+ }
+ if !strings.Contains(got, "broken") {
+ t.Errorf("text the malformed entry yielded before its own error was discarded: %q", got)
+ }
+}
+
+// TestZipMalformedOnlyEntryErrors: when the only matched entry is
+// malformed, no text survives to return, so the error is reported instead
+// — named after the entry it came from. The malformed attribute syntax
+// breaks the decode before any character data is ever emitted, so there
+// is nothing for the lenient path (TestZipLenientOnMalformedEntry) to
+// keep.
+func TestZipMalformedOnlyEntryErrors(t *testing.T) {
+ e := newWithPath("")
+ _, err := text(t, e, zipFile(t, "bad.docx", map[string]string{
+ "word/document.xml": `<w:document><w:body attr="unterminated><w:p><w:t>oops</w:t></w:p></w:body></w:document>`,
+ }), 0)
+ if err == nil {
+ t.Fatal("malformed only entry: want an error, got nil")
+ }
+ if !strings.Contains(err.Error(), "word/document.xml") {
+ t.Errorf("error %q does not name the entry", err.Error())
+ }
+}
+
+// TestZipNoTextAndFailingSiblingErrors: a well-formed entry that simply
+// has no character data (an empty <w:body/>) must not count as "text was
+// found" and mask a failing sibling's error — the separator newline
+// zipText appends after every successful entry means b.Len() alone
+// cannot answer "did anything write text"; only word/footer1.xml's own
+// contribution (zero, since it fails while still inside its opening tag,
+// before any token is emitted) may be counted, and it contributed
+// nothing either, so the archive as a whole produced no text and the
+// wrapped error must surface.
+func TestZipNoTextAndFailingSiblingErrors(t *testing.T) {
+ e := newWithPath("")
+ _, err := text(t, e, zipFile(t, "empty.docx", map[string]string{
+ "word/document.xml": `<w:document xmlns:w="w"><w:body/></w:document>`,
+ "word/footer1.xml": `<w:ftr xmlns:w="w" a="unterminated><w:p/></w:ftr>`,
+ }), 0)
+ if err == nil {
+ t.Fatal("textless entry plus a failing sibling: want an error, got nil")
+ }
+ if !strings.Contains(err.Error(), "word/footer1.xml") {
+ t.Errorf("error %q does not name the failing entry", err.Error())
+ }
+}
+
+// TestMaxReadCapsZipOutput: B1. A directory's max-read, when smaller than
+// the fixed zipBudget default, caps a single archive's extracted text on
+// its own — zipBudget is left at its default, so only budget(zipBudget,
+// maxRead) picking the smaller maxRead explains the result.
+func TestMaxReadCapsZipOutput(t *testing.T) {
+ e := newWithPath("")
+ big := `<w:document xmlns:w="w"><w:p><w:t>` + strings.Repeat("x", 4096) + `</w:t></w:p></w:document>`
+ p := zipFile(t, "huge.docx", map[string]string{"word/document.xml": big})
+ if _, err := text(t, e, p, 1024); !errors.Is(err, ErrTooLarge) {
+ t.Fatalf("got %v, want ErrTooLarge (max-read 1024 should have capped a 4096-byte entry)", err)
+ }
+}
diff --git a/internal/ignore/ignore.go b/internal/ignore/ignore.go
new file mode 100644
index 0000000..fff6dcc
--- /dev/null
+++ b/internal/ignore/ignore.go
@@ -0,0 +1,324 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+// Package ignore implements a gitignore(5)-compatible path matcher.
+package ignore
+
+import (
+ "errors"
+ "fmt"
+ "regexp"
+ "strings"
+)
+
+// pattern is one compiled gitignore(5) pattern.
+type pattern struct {
+ re *regexp.Regexp
+ negate bool
+ dirOnly bool
+}
+
+// Matcher decides which paths a set of gitignore(5) patterns ignores.
+type Matcher struct {
+ patterns []pattern
+}
+
+// New compiles patterns in gitignore(5) syntax, in order. Blank patterns and
+// patterns starting with "#" are skipped. A nil or empty list gives a
+// matcher that matches nothing.
+func New(patterns []string) (*Matcher, error) {
+ m := &Matcher{}
+ for _, orig := range patterns {
+ p := orig
+ if p == "" || strings.HasPrefix(p, "#") {
+ continue
+ }
+ negate := false
+ switch {
+ case strings.HasPrefix(p, "!"):
+ negate = true
+ p = p[1:]
+ case strings.HasPrefix(p, `\!`), strings.HasPrefix(p, `\#`):
+ p = p[1:]
+ }
+ p = trimTrailingSpaces(p)
+ dirOnly := false
+ if strings.HasSuffix(p, "/") {
+ dirOnly = true
+ p = strings.TrimSuffix(p, "/")
+ }
+ reStr, err := toRegexp(p)
+ if err != nil {
+ return nil, fmt.Errorf("bad ignore pattern %q: %v", orig, err)
+ }
+ re, err := regexp.Compile(reStr)
+ if err != nil {
+ return nil, fmt.Errorf("bad ignore pattern %q: %v", orig, err)
+ }
+ m.patterns = append(m.patterns, pattern{re: re, negate: negate, dirOnly: dirOnly})
+ }
+ return m, nil
+}
+
+// trimTrailingSpaces removes trailing spaces that are not escaped by a
+// preceding backslash, per gitignore(5).
+func trimTrailingSpaces(p string) string {
+ for strings.HasSuffix(p, " ") {
+ n := 0
+ for i := len(p) - 2; i >= 0 && p[i] == '\\'; i-- {
+ n++
+ }
+ if n%2 == 1 {
+ break // the trailing space is escaped: keep it
+ }
+ p = p[:len(p)-1]
+ }
+ return p
+}
+
+// Match reports whether rel is ignored. rel is slash-separated and relative
+// to the root, with no leading slash and no trailing slash, and each of its
+// path components must be "." and ".."-free (as produced by a directory
+// walk, not a raw user-typed path).
+func (m *Matcher) Match(rel string, isDir bool) bool {
+ parts := strings.Split(rel, "/")
+ for i := 1; i < len(parts); i++ {
+ ancestor := strings.Join(parts[:i], "/")
+ if m.matchOne(ancestor, true) {
+ return true
+ }
+ }
+ return m.matchOne(rel, isDir)
+}
+
+// matchOne applies every pattern in order to one path and returns the final
+// ignored state, without considering ancestor directories.
+func (m *Matcher) matchOne(path string, isDir bool) bool {
+ ignored := false
+ for _, pat := range m.patterns {
+ if pat.dirOnly && !isDir {
+ continue
+ }
+ if pat.re.MatchString(path) {
+ ignored = !pat.negate
+ }
+ }
+ return ignored
+}
+
+// toRegexp translates the body of one gitignore pattern (no leading "!",
+// no trailing "/") into a regular expression over slash-separated paths.
+func toRegexp(p string) (string, error) {
+ anchored := strings.Contains(p, "/")
+ p = strings.TrimPrefix(p, "/")
+ var b strings.Builder
+ b.WriteString("^")
+ if !anchored {
+ b.WriteString("(?:.*/)?")
+ }
+ for i := 0; i < len(p); i++ {
+ c := p[i]
+ switch {
+ case strings.HasPrefix(p[i:], "**/") && (i == 0 || p[i-1] == '/'):
+ b.WriteString("(?:.*/)?")
+ i += 2
+ case strings.HasPrefix(p[i:], "/**") && i+3 == len(p):
+ b.WriteString("/.*")
+ i += 2
+ case strings.HasPrefix(p[i:], "**"):
+ b.WriteString("[^/]*")
+ i++
+ case c == '*':
+ b.WriteString("[^/]*")
+ case c == '?':
+ b.WriteString("[^/]")
+ case c == '[':
+ negated := false
+ start := i + 1
+ if start < len(p) && (p[start] == '!' || p[start] == '^') {
+ negated = true
+ start++
+ }
+ items, end, err := parseClassBody(p, start)
+ if err != nil {
+ return "", err
+ }
+ b.WriteString(renderClass(items, negated))
+ i = end
+ case c == '\\' && i+1 < len(p):
+ i++
+ b.WriteString(regexp.QuoteMeta(string(p[i])))
+ default:
+ b.WriteString(regexp.QuoteMeta(string(c)))
+ }
+ }
+ b.WriteString("$")
+ return b.String(), nil
+}
+
+// classItem is one member of a gitignore bracket expression: either a
+// verbatim POSIX bracket subexpression ("[:digit:]", "[.ch.]", "[=e=]"), or
+// a literal byte (lo == hi) or a byte range (lo-hi).
+type classItem struct {
+ posix string
+ lo, hi byte
+}
+
+// parseClassBody scans p starting at start (just past "[", "[!" or "[^")
+// for the members of a bracket expression, up to and including the
+// matching unescaped "]". A "]" appearing immediately at start is a
+// literal member, not the terminator, per glob syntax (e.g. "[]a]" matches
+// "]" or "a"). Inside the class, "\x" escapes x to a literal member,
+// neutralizing any meaning x would otherwise have (closing the class,
+// starting a POSIX subexpression, forming a range) — matching git's
+// wildmatch. It returns the members found and the index of the closing
+// "]", or an error if none is found (including a class left unterminated
+// because its only "]" was escaped).
+func parseClassBody(p string, start int) ([]classItem, int, error) {
+ var items []classItem
+ k := start
+ if k < len(p) && p[k] == ']' {
+ items = append(items, classItem{lo: ']', hi: ']'})
+ k++
+ }
+ for k < len(p) && p[k] != ']' {
+ // "\x" escapes x to a literal member; it can never start a
+ // POSIX subexpression or a range. A backslash with nothing
+ // after it can never close the class either.
+ if p[k] == '\\' {
+ if k+1 >= len(p) {
+ return nil, 0, errors.New("unterminated [")
+ }
+ items = append(items, classItem{lo: p[k+1], hi: p[k+1]})
+ k += 2
+ continue
+ }
+ if p[k] == '[' && k+1 < len(p) && (p[k+1] == ':' || p[k+1] == '.' || p[k+1] == '=') {
+ delim := p[k+1]
+ rest := strings.Index(p[k+2:], string(delim)+"]")
+ if rest < 0 {
+ return nil, 0, errors.New("unterminated [")
+ }
+ stop := k + 2 + rest + 2
+ items = append(items, classItem{posix: p[k:stop]})
+ k = stop
+ continue
+ }
+ // "x-y" is a range unless the "-" is the last character before
+ // the closing "]", in which case it is a literal "-".
+ if k+2 < len(p) && p[k+1] == '-' && p[k+2] != ']' {
+ items = append(items, classItem{lo: p[k], hi: p[k+2]})
+ k += 3
+ continue
+ }
+ items = append(items, classItem{lo: p[k], hi: p[k]})
+ k++
+ }
+ if k >= len(p) {
+ return nil, 0, errors.New("unterminated [")
+ }
+ return items, k, nil
+}
+
+// renderClass turns the members of a bracket expression into a Go regexp
+// character class. Git's bracket expressions never match "/", regardless
+// of negation, so "/" is dropped from a positive class (splitting any range
+// that spans it) and added to a negated one. A positive class left
+// matching nothing (e.g. "[/]") is rendered as a class that can never
+// match any character, since RE2 has no empty class or lookahead.
+func renderClass(items []classItem, negated bool) string {
+ if negated {
+ items = append(items, classItem{lo: '/', hi: '/'})
+ return "[^" + renderClassItems(items) + "]"
+ }
+ items = excludeSlash(items)
+ if len(items) == 0 {
+ return `[^\x00-\x{10FFFF}]`
+ }
+ return "[" + renderClassItems(items) + "]"
+}
+
+// posixSlashFree gives the ASCII definition, minus "/", of every standard
+// POSIX bracket class whose definition otherwise includes it: graph and
+// print (visible characters, "/" among them) and punct (documented in
+// gitignore's own toRegexp derivation as "!-. :-@ [-` {-~", i.e. punct's
+// usual "!-/" range with "/" trimmed to "!-."). The other nine standard
+// classes (alnum, alpha, blank, cntrl, digit, lower, space, upper,
+// xdigit) never contain "/" and so need no rewriting.
+var posixSlashFree = map[string][]classItem{
+ "graph": {{lo: '!', hi: '.'}, {lo: '0', hi: '~'}},
+ "print": {{lo: ' ', hi: '.'}, {lo: '0', hi: '~'}},
+ "punct": {{lo: '!', hi: '.'}, {lo: ':', hi: '@'}, {lo: '[', hi: '`'}, {lo: '{', hi: '~'}},
+}
+
+// posixClassName returns the name inside a "[:name:]" POSIX bracket class,
+// or "", false for anything else (a collating symbol "[.x.]", an
+// equivalence class "[=x=]", or a malformed value).
+func posixClassName(posix string) (string, bool) {
+ if strings.HasPrefix(posix, "[:") && strings.HasSuffix(posix, ":]") {
+ return posix[2 : len(posix)-2], true
+ }
+ return "", false
+}
+
+// excludeSlash removes "/" from a positive class's members, splitting any
+// range that spans it into the parts on either side, and substituting the
+// "/"-free ASCII definition for any POSIX class that would otherwise
+// include it.
+func excludeSlash(items []classItem) []classItem {
+ var out []classItem
+ for _, it := range items {
+ switch {
+ case it.posix != "":
+ if name, ok := posixClassName(it.posix); ok {
+ if repl, hasSlash := posixSlashFree[name]; hasSlash {
+ out = append(out, repl...)
+ continue
+ }
+ }
+ out = append(out, it)
+ case it.lo == '/' && it.hi == '/':
+ // drop the lone literal "/"
+ case it.lo <= '/' && '/' <= it.hi:
+ if it.lo <= '/'-1 {
+ out = append(out, classItem{lo: it.lo, hi: '/' - 1})
+ }
+ if '/'+1 <= it.hi {
+ out = append(out, classItem{lo: '/' + 1, hi: it.hi})
+ }
+ default:
+ out = append(out, it)
+ }
+ }
+ return out
+}
+
+func renderClassItems(items []classItem) string {
+ var b strings.Builder
+ for _, it := range items {
+ switch {
+ case it.posix != "":
+ b.WriteString(it.posix)
+ case it.lo == it.hi:
+ b.WriteString(classChar(it.lo))
+ default:
+ b.WriteString(classChar(it.lo))
+ b.WriteByte('-')
+ b.WriteString(classChar(it.hi))
+ }
+ }
+ return b.String()
+}
+
+// classChar escapes a byte that would otherwise be misread by the Go
+// regexp parser when emitted as a member (or range endpoint) inside a
+// character class: "]" would close the class, "^" would negate it if
+// first, "-" would start a range, "[" could open a POSIX subexpression,
+// and "\" always needs escaping to be literal.
+func classChar(c byte) string {
+ switch c {
+ case '\\', ']', '^', '-', '[':
+ return "\\" + string(c)
+ default:
+ return string(c)
+ }
+}
diff --git a/internal/ignore/ignore_test.go b/internal/ignore/ignore_test.go
new file mode 100644
index 0000000..04b60c4
--- /dev/null
+++ b/internal/ignore/ignore_test.go
@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package ignore
+
+import "testing"
+
+func TestMatch(t *testing.T) {
+ type c struct {
+ rel string
+ isDir bool
+ want bool
+ }
+ tests := []struct {
+ patterns []string
+ cases []c
+ }{
+ {[]string{"*.log"}, []c{{"a.log", false, true}, {"dir/b.log", false, true}, {"a.txt", false, false}}},
+ {[]string{"/a.txt"}, []c{{"a.txt", false, true}, {"dir/a.txt", false, false}}},
+ {[]string{"build/"}, []c{{"build", true, true}, {"build", false, false}, {"src/build", true, true}, {"build/x.bin", false, true}}},
+ {[]string{"doc/*.md"}, []c{{"doc/r.md", false, true}, {"doc/sub/r.md", false, false}, {"x/doc/r.md", false, false}}},
+ {[]string{"**/readme.md"}, []c{{"readme.md", false, true}, {"a/b/readme.md", false, true}}},
+ {[]string{"deep/**"}, []c{{"deep", true, false}, {"deep/a", true, true}, {"deep/a/b/f.tmp", false, true}}},
+ {[]string{"x/**/z.txt"}, []c{{"x/z.txt", false, true}, {"x/y/z.txt", false, true}, {"x/y/w/z.txt", false, true}, {"q/x/z.txt", false, false}}},
+ {[]string{"*.part", "!keep.part"}, []c{{"a.part", false, true}, {"keep.part", false, false}}},
+ {[]string{"dir/", "!dir/c.txt"}, []c{{"dir/c.txt", false, true}}},
+ {[]string{".*"}, []c{{".hidden", false, true}, {"a/.cfg", false, true}, {"visible", false, false}}},
+ {[]string{"a?.txt", "[bc].txt"}, []c{{"a1.txt", false, true}, {"a12.txt", false, false}, {"b.txt", false, true}, {"d.txt", false, false}}},
+ {[]string{"[!a]*.txt"}, []c{{"b.txt", false, true}, {"a.txt", false, false}}},
+ {[]string{"sub"}, []c{{"sub", true, true}, {"x/sub", true, true}, {"x/sub/f", false, true}, {"subx", false, false}}},
+ {[]string{"\\#notes", "\\!bang"}, []c{{"#notes", false, true}, {"!bang", false, true}}},
+ {[]string{"", "# comment"}, []c{{"# comment", false, false}}},
+ {nil, []c{{"anything", false, false}}},
+ {[]string{"x[]a]y"}, []c{{"x]y", false, true}, {"xay", false, true}, {"xby", false, false}}},
+ {[]string{"a[/]b"}, []c{{"a/b", false, false}}},
+ }
+ for _, tt := range tests {
+ m, err := New(tt.patterns)
+ if err != nil {
+ t.Fatalf("New(%q): %v", tt.patterns, err)
+ }
+ for _, cs := range tt.cases {
+ if got := m.Match(cs.rel, cs.isDir); got != cs.want {
+ t.Errorf("patterns %q: Match(%q, dir=%v) = %v, want %v", tt.patterns, cs.rel, cs.isDir, got, cs.want)
+ }
+ }
+ }
+}
+
+func TestBadPattern(t *testing.T) {
+ if _, err := New([]string{"[abc"}); err == nil || err.Error() != `bad ignore pattern "[abc": unterminated [` {
+ t.Fatalf("got %v", err)
+ }
+ // "\]" escapes the only "]" to a literal member, leaving the class
+ // with no terminator.
+ if _, err := New([]string{"a[\\]d"}); err == nil || err.Error() != `bad ignore pattern "a[\\]d": unterminated [` {
+ t.Fatalf("got %v", err)
+ }
+}
diff --git a/internal/ignore/oracle_test.go b/internal/ignore/oracle_test.go
new file mode 100644
index 0000000..a5a589c
--- /dev/null
+++ b/internal/ignore/oracle_test.go
@@ -0,0 +1,94 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package ignore
+
+import (
+ "os"
+ "os/exec"
+ "path/filepath"
+ "sort"
+ "strings"
+ "testing"
+)
+
+var oracleFiles = []string{
+ "a.txt", "b.log", "a1.txt", "abc", ".hidden", "foo.part", "keep.part",
+ "dir/c.txt", "dir/d.log", "dir/.hidden2", "dir/sub/e.txt",
+ "build/out.bin", "src/build/x.go", "doc/readme.md", "doc/sub/readme.md",
+ "deep/a/b/c/file.tmp", "x/z.txt", "x/y/z.txt", "sub/f.txt", "space name.txt",
+ "x]y", "xay", "xby", "a/b", "1.txt", "]a.txt",
+ "a.b", "xyz", "aq", "bq",
+}
+
+var oracleSets = [][]string{
+ {"*.log"}, {"/a.txt"}, {"build/"}, {"doc/*.md"}, {"**/readme.md"},
+ {"deep/**"}, {"x/**/z.txt"}, {"*.part", "!keep.part"}, {".*"},
+ {"a?.txt", "[ab]*.txt"}, {"[!a]*.txt"}, {"dir/", "!dir/c.txt"},
+ {"sub"}, {"space name.txt"}, {"*", "!*.txt"}, {"dir/**/*.txt"},
+ {"x[]a]y"}, {"a[/]b"}, {"[[:digit:]]*.txt"}, {"[!]a]*"},
+ {"a[[:punct:]]b"}, {"x[[:alpha:]]z"}, {"[a\\b]q"},
+}
+
+// TestAgainstGit compares every file and directory of a real tree against
+// `git check-ignore --no-index` for each pattern set.
+func TestAgainstGit(t *testing.T) {
+ if _, err := exec.LookPath("git"); err != nil {
+ t.Skip("git not installed")
+ }
+ root := t.TempDir()
+ t.Setenv("HOME", root)
+ t.Setenv("GIT_CONFIG_GLOBAL", os.DevNull)
+ t.Setenv("GIT_CONFIG_NOSYSTEM", "1")
+ for _, f := range oracleFiles {
+ p := filepath.Join(root, f)
+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(p, nil, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if out, err := exec.Command("git", "-C", root, "init", "-q").CombinedOutput(); err != nil {
+ t.Fatalf("git init: %v %s", err, out)
+ }
+ // every path in the tree, files and directories, relative and slash-separated
+ var paths []string
+ isDir := map[string]bool{}
+ filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error {
+ rel, _ := filepath.Rel(root, p)
+ if rel == "." || rel == ".git" || strings.HasPrefix(rel, ".git"+string(filepath.Separator)) || rel == ".gitignore" {
+ if rel == ".git" {
+ return filepath.SkipDir
+ }
+ return nil
+ }
+ rel = filepath.ToSlash(rel)
+ paths = append(paths, rel)
+ isDir[rel] = d.IsDir()
+ return nil
+ })
+ sort.Strings(paths)
+ for _, set := range oracleSets {
+ if err := os.WriteFile(filepath.Join(root, ".gitignore"), []byte(strings.Join(set, "\n")+"\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ cmd := exec.Command("git", "-C", root, "check-ignore", "--no-index", "--stdin")
+ cmd.Stdin = strings.NewReader(strings.Join(paths, "\n") + "\n")
+ out, _ := cmd.Output() // exit status 1 means "nothing ignored"
+ gitIgnored := map[string]bool{}
+ for _, l := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+ if l != "" {
+ gitIgnored[strings.TrimSuffix(l, "/")] = true
+ }
+ }
+ m, err := New(set)
+ if err != nil {
+ t.Fatalf("New(%q): %v", set, err)
+ }
+ for _, p := range paths {
+ if got := m.Match(p, isDir[p]); got != gitIgnored[p] {
+ t.Errorf("patterns %q, path %q (dir=%v): krino says %v, git says %v", set, p, isDir[p], got, gitIgnored[p])
+ }
+ }
+ }
+}
diff --git a/internal/norm/norm.go b/internal/norm/norm.go
new file mode 100644
index 0000000..e7933ec
--- /dev/null
+++ b/internal/norm/norm.go
@@ -0,0 +1,96 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+// Package norm puts text into the form krino compares it in: optionally
+// without diacritics, optionally lower case, with white space collapsed.
+package norm
+
+import (
+ "strings"
+ "unicode"
+
+ unorm "golang.org/x/text/unicode/norm"
+)
+
+// special holds the letters that do not decompose under Unicode NFD, so
+// Fold maps them explicitly.
+var special = map[rune]string{
+ 'ł': "l", 'Ł': "L",
+ 'ø': "o", 'Ø': "O",
+ 'đ': "d", 'Đ': "D",
+ 'ħ': "h", 'Ħ': "H",
+ 'ß': "ss",
+ 'æ': "ae", 'Æ': "AE",
+ 'œ': "oe", 'Œ': "OE",
+ 'ı': "i",
+}
+
+// Fold strips diacritics: Unicode NFD, drop combining marks (category Mn),
+// then map the letters that do not decompose. ASCII input is returned
+// unchanged without allocating.
+func Fold(s string) string {
+ ascii := true
+ for i := 0; i < len(s); i++ {
+ if s[i] >= 0x80 {
+ ascii = false
+ break
+ }
+ }
+ if ascii {
+ return s
+ }
+
+ var b strings.Builder
+ b.Grow(len(s))
+ for _, r := range unorm.NFD.String(s) {
+ if unicode.Is(unicode.Mn, r) {
+ continue
+ }
+ if rep, ok := special[r]; ok {
+ b.WriteString(rep)
+ continue
+ }
+ b.WriteRune(r)
+ }
+ return b.String()
+}
+
+// Text puts s into the form content and keywords are compared in: Fold if
+// fold, strings.ToLower if ignoreCase, then every run of Unicode white
+// space becomes one ASCII space and both ends are trimmed.
+func Text(s string, ignoreCase, fold bool) string {
+ if fold {
+ s = Fold(s)
+ }
+ if ignoreCase {
+ s = strings.ToLower(s)
+ }
+
+ var b strings.Builder
+ b.Grow(len(s))
+ inSpace := false
+ started := false
+ for _, r := range s {
+ if unicode.IsSpace(r) {
+ if started {
+ inSpace = true
+ }
+ continue
+ }
+ if inSpace {
+ b.WriteByte(' ')
+ inSpace = false
+ }
+ b.WriteRune(r)
+ started = true
+ }
+ return b.String()
+}
+
+// Name puts a file name into the form it is matched against: Fold(s) if
+// fold, else s. Case is handled by the regex flag, not here.
+func Name(s string, fold bool) string {
+ if fold {
+ return Fold(s)
+ }
+ return s
+}
diff --git a/internal/norm/norm_test.go b/internal/norm/norm_test.go
new file mode 100644
index 0000000..2fc35b6
--- /dev/null
+++ b/internal/norm/norm_test.go
@@ -0,0 +1,73 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package norm
+
+import "testing"
+
+func TestFold(t *testing.T) {
+ tests := map[string]string{
+ "": "",
+ "plain ASCII 123": "plain ASCII 123",
+ "spółka z ograniczoną odpowiedzialnością": "spolka z ograniczona odpowiedzialnoscia",
+ "Łódź": "Lodz",
+ "ZAŻÓŁĆ GĘŚLĄ JAŹŃ": "ZAZOLC GESLA JAZN",
+ "Straße": "Strasse",
+ "Øresund": "Oresund",
+ "Ærø": "AEro",
+ "œuvre": "oeuvre",
+ "déjà vu": "deja vu",
+ "Đakovo": "Dakovo",
+ "ħelp": "help",
+ }
+ for in, want := range tests {
+ if got := Fold(in); got != want {
+ t.Errorf("Fold(%q) = %q, want %q", in, got, want)
+ }
+ }
+}
+
+func TestText(t *testing.T) {
+ tests := []struct {
+ in string
+ ignoreCase, fold bool
+ want string
+ }{
+ {" Hello\n\tWORLD ", true, false, "hello world"},
+ {"A B\r\nC", false, false, "A B C"},
+ {"ZAŻÓŁĆ gęślą", true, true, "zazolc gesla"},
+ {"Spółka", false, true, "Spolka"},
+ {"Faktura\u00A0VAT", true, false, "faktura vat"},
+ {"", true, true, ""},
+ {" ", true, true, ""},
+ }
+ for _, tt := range tests {
+ if got := Text(tt.in, tt.ignoreCase, tt.fold); got != tt.want {
+ t.Errorf("Text(%q, %v, %v) = %q, want %q", tt.in, tt.ignoreCase, tt.fold, got, tt.want)
+ }
+ }
+}
+
+func TestTextIdempotent(t *testing.T) {
+ for _, s := range []string{" Łódź\n\nMIASTO ", "a\tb", "Ærø Œuvre"} {
+ once := Text(s, true, true)
+ if twice := Text(once, true, true); twice != once {
+ t.Errorf("Text not idempotent on %q: %q then %q", s, once, twice)
+ }
+ }
+}
+
+func TestName(t *testing.T) {
+ if got := Name("Spółka.PDF", true); got != "Spolka.PDF" {
+ t.Errorf("Name fold = %q", got)
+ }
+ if got := Name("Spółka.PDF", false); got != "Spółka.PDF" {
+ t.Errorf("Name no fold = %q", got)
+ }
+}
+
+func TestFoldASCIINoAlloc(t *testing.T) {
+ s := "already plain ascii text"
+ if n := testing.AllocsPerRun(100, func() { _ = Fold(s) }); n != 0 {
+ t.Errorf("Fold allocates %v times on ASCII input", n)
+ }
+}
diff --git a/internal/scan/scan.go b/internal/scan/scan.go
new file mode 100644
index 0000000..e87b741
--- /dev/null
+++ b/internal/scan/scan.go
@@ -0,0 +1,233 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+// Package scan walks a directory tree and reports the files krino will
+// consider sorting, and why the rest were skipped.
+package scan
+
+import (
+ "errors"
+ "fmt"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "sort"
+ "time"
+
+ "krino/internal/ignore"
+)
+
+// File is a regular file found by Walk.
+type File struct {
+ Path string // absolute, cleaned
+ Rel string // slash-separated, relative to the root
+ Name string // base name
+ Size int64
+ ModTime time.Time
+ Mode fs.FileMode
+}
+
+// Reason is why an entry was not returned as a File.
+type Reason int
+
+const (
+ Ignored Reason = iota // matched an ignore pattern
+ Busy // a sibling NAME<busy-suffix> exists
+ TooNew // modified less than MinAge ago
+ Symlink // a symbolic link (never followed)
+ NotRegular // fifo, socket, device
+ Unreadable // a directory that could not be read
+)
+
+// String names a Reason the way it should read in a report.
+func (r Reason) String() string {
+ switch r {
+ case Ignored:
+ return "ignored"
+ case Busy:
+ return "busy"
+ case TooNew:
+ return "too new"
+ case Symlink:
+ return "symlink"
+ case NotRegular:
+ return "not a regular file"
+ case Unreadable:
+ return "unreadable"
+ default:
+ return fmt.Sprintf("Reason(%d)", int(r))
+ }
+}
+
+// Skipped is one entry Walk did not return as a File, and why.
+type Skipped struct {
+ Rel string
+ Reason Reason
+}
+
+// Options controls how Walk traverses a directory.
+type Options struct {
+ Recursive bool
+ MaxDepth int // 0: unlimited; 1: the root's own entries only
+ Ignore *ignore.Matcher // nil: nothing ignored
+ Exclude []string // absolute directories never entered
+ Busy []string // suffixes, e.g. ".part"
+ MinAge time.Duration
+ Now time.Time
+}
+
+// Result is everything Walk found under a root.
+type Result struct {
+ Files []File // sorted by Rel
+ Skipped []Skipped // sorted by Rel
+}
+
+// Walk lists the files under root that krino will consider, and reports why
+// the rest were skipped. root must exist, be a directory, and be readable;
+// an unreadable subdirectory found during the walk is reported as
+// Unreadable and does not abort the scan.
+func Walk(root string, opt Options) (*Result, error) {
+ root, err := filepath.Abs(root)
+ if err != nil {
+ return nil, err
+ }
+ info, err := os.Stat(root)
+ if err != nil {
+ return nil, err
+ }
+ if !info.IsDir() {
+ return nil, fmt.Errorf("%s is not a directory", root)
+ }
+ entries, err := os.ReadDir(root)
+ if err != nil {
+ return nil, err
+ }
+
+ exclude := make(map[string]bool, len(opt.Exclude))
+ for _, e := range opt.Exclude {
+ exclude[filepath.Clean(e)] = true
+ }
+
+ w := &walker{root: root, opt: opt, exclude: exclude}
+ if err := w.walk(root, "", 1, entries); err != nil {
+ return nil, err
+ }
+
+ sort.Slice(w.result.Files, func(i, j int) bool { return w.result.Files[i].Rel < w.result.Files[j].Rel })
+ sort.Slice(w.result.Skipped, func(i, j int) bool { return w.result.Skipped[i].Rel < w.result.Skipped[j].Rel })
+ return &w.result, nil
+}
+
+// walker accumulates the Result across recursive calls.
+type walker struct {
+ opt Options
+ root string
+ exclude map[string]bool
+ result Result
+}
+
+// walk applies the skip checks to entries, the already-read contents of dir
+// (at relDir relative to the root, dir's own entries at depth). A
+// subdirectory it recurses into is read here, right before recursing, so a
+// ReadDir failure on it can be reported as Unreadable and skipped without
+// aborting the rest of the walk; only a failure reading dir itself (passed
+// in by the caller) would need to propagate, and only Walk's own read of
+// the root works that way.
+func (w *walker) walk(dir, relDir string, depth int, entries []os.DirEntry) error {
+ // The set of names in this directory, built once, for the busy check.
+ names := make(map[string]bool, len(entries))
+ for _, e := range entries {
+ names[e.Name()] = true
+ }
+
+ for _, e := range entries {
+ name := e.Name()
+ rel := name
+ if relDir != "" {
+ rel = relDir + "/" + name
+ }
+ path := filepath.Join(dir, name)
+
+ // A symlink is never followed, whatever it points to; DirEntry's
+ // Type is Lstat-like and does not resolve it.
+ if e.Type()&fs.ModeSymlink != 0 {
+ w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: Symlink})
+ continue
+ }
+
+ if e.IsDir() {
+ // Directories themselves are not reported, with two
+ // exceptions: the symlink case above, and an ignored
+ // directory (C2) — reported once for the directory itself,
+ // not for each file inside it, since pruning it without
+ // descending is the whole point; without this, its contents
+ // would appear in no count and in no -v listing at all.
+ if !w.opt.Recursive {
+ continue
+ }
+ if w.opt.Ignore != nil && w.opt.Ignore.Match(rel, true) {
+ w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: Ignored})
+ continue
+ }
+ if w.exclude[filepath.Clean(path)] {
+ continue
+ }
+ if w.opt.MaxDepth > 0 && depth+1 > w.opt.MaxDepth {
+ continue
+ }
+ subEntries, err := os.ReadDir(path)
+ if err != nil {
+ w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: Unreadable})
+ continue
+ }
+ if err := w.walk(path, rel, depth+1, subEntries); err != nil {
+ return err
+ }
+ continue
+ }
+
+ info, err := e.Info()
+ if err != nil {
+ if errors.Is(err, fs.ErrNotExist) {
+ continue // vanished mid-walk (a download finishing, say): silently skipped
+ }
+ w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: Unreadable})
+ continue
+ }
+ if !info.Mode().IsRegular() {
+ w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: NotRegular})
+ continue
+ }
+ if w.opt.Ignore != nil && w.opt.Ignore.Match(rel, false) {
+ w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: Ignored})
+ continue
+ }
+ if busy := w.isBusy(name, names); busy {
+ w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: Busy})
+ continue
+ }
+ if w.opt.Now.Sub(info.ModTime()) < w.opt.MinAge {
+ w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: TooNew})
+ continue
+ }
+ w.result.Files = append(w.result.Files, File{
+ Path: path,
+ Rel: rel,
+ Name: name,
+ Size: info.Size(),
+ ModTime: info.ModTime(),
+ Mode: info.Mode(),
+ })
+ }
+ return nil
+}
+
+// isBusy reports whether name+suffix, for any configured Busy suffix, is
+// among names — the sibling of an in-progress download.
+func (w *walker) isBusy(name string, names map[string]bool) bool {
+ for _, suffix := range w.opt.Busy {
+ if names[name+suffix] {
+ return true
+ }
+ }
+ return false
+}
diff --git a/internal/scan/scan_test.go b/internal/scan/scan_test.go
new file mode 100644
index 0000000..94f922b
--- /dev/null
+++ b/internal/scan/scan_test.go
@@ -0,0 +1,263 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package scan
+
+import (
+ "errors"
+ "fmt"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "reflect"
+ "syscall"
+ "testing"
+ "time"
+
+ "krino/internal/ignore"
+)
+
+var now = time.Date(2026, 9, 11, 12, 0, 0, 0, time.UTC)
+
+// tree creates files (with an mtime one hour before now) and returns the root.
+func tree(t *testing.T, files ...string) string {
+ t.Helper()
+ root := t.TempDir()
+ for _, f := range files {
+ p := filepath.Join(root, f)
+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(p, []byte(f), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ old := now.Add(-time.Hour)
+ if err := os.Chtimes(p, old, old); err != nil {
+ t.Fatal(err)
+ }
+ }
+ return root
+}
+
+func rels(r *Result) []string {
+ var out []string
+ for _, f := range r.Files {
+ out = append(out, f.Rel)
+ }
+ return out
+}
+
+func skipped(r *Result) map[string]Reason {
+ m := map[string]Reason{}
+ for _, s := range r.Skipped {
+ m[s.Rel] = s.Reason
+ }
+ return m
+}
+
+func TestTopLevelOnly(t *testing.T) {
+ root := tree(t, "a.pdf", "b.txt", "sub/c.txt")
+ r, err := Walk(root, Options{Now: now})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if want := []string{"a.pdf", "b.txt"}; !reflect.DeepEqual(rels(r), want) {
+ t.Fatalf("files = %v, want %v", rels(r), want)
+ }
+ f := r.Files[0]
+ if f.Name != "a.pdf" || f.Size != int64(len("a.pdf")) || f.Path != filepath.Join(root, "a.pdf") || !f.ModTime.Equal(now.Add(-time.Hour)) {
+ t.Fatalf("file = %+v", f)
+ }
+}
+
+func TestRecursiveDepthExcludeIgnore(t *testing.T) {
+ root := tree(t, "a.txt", "one/b.txt", "one/two/c.txt", "Work/filed.pdf", "skip/x.txt", "keep/y.log")
+ m, _ := ignore.New([]string{"skip/", "*.log"})
+ r, err := Walk(root, Options{Recursive: true, Ignore: m, Exclude: []string{filepath.Join(root, "Work")}, Now: now})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if want := []string{"a.txt", "one/b.txt", "one/two/c.txt"}; !reflect.DeepEqual(rels(r), want) {
+ t.Fatalf("files = %v, want %v", rels(r), want)
+ }
+ if got := skipped(r); got["keep/y.log"] != Ignored {
+ t.Fatalf("skipped = %v", got)
+ }
+ r, _ = Walk(root, Options{Recursive: true, MaxDepth: 2, Now: now})
+ for _, rel := range rels(r) {
+ if rel == "one/two/c.txt" {
+ t.Fatalf("MaxDepth 2 reached depth 3: %v", rels(r))
+ }
+ }
+}
+
+func TestBusyTooNewSymlinkFifo(t *testing.T) {
+ root := tree(t, "movie.mkv", "movie.mkv.aria2", "doc.pdf", "doc.pdf.part", "plain.txt")
+ fresh := filepath.Join(root, "fresh.txt")
+ if err := os.WriteFile(fresh, nil, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Chtimes(fresh, now.Add(-30*time.Second), now.Add(-30*time.Second)); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(filepath.Join(root, "plain.txt"), filepath.Join(root, "link.txt")); err != nil {
+ t.Fatal(err)
+ }
+ fifo := filepath.Join(root, "pipe")
+ haveFifo := syscall.Mkfifo(fifo, 0o644) == nil
+ m, _ := ignore.New([]string{"*.part", "*.aria2"})
+ r, err := Walk(root, Options{Ignore: m, Busy: []string{".part", ".aria2"}, MinAge: 2 * time.Minute, Now: now})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if want := []string{"plain.txt"}; !reflect.DeepEqual(rels(r), want) {
+ t.Fatalf("files = %v, want %v", rels(r), want)
+ }
+ want := map[string]Reason{
+ "movie.mkv": Busy, "doc.pdf": Busy, "movie.mkv.aria2": Ignored, "doc.pdf.part": Ignored,
+ "fresh.txt": TooNew, "link.txt": Symlink,
+ }
+ if haveFifo {
+ want["pipe"] = NotRegular
+ }
+ if got := skipped(r); !reflect.DeepEqual(got, want) {
+ t.Fatalf("skipped = %v, want %v", got, want)
+ }
+}
+
+func TestSymlinkedDirNotFollowed(t *testing.T) {
+ root := tree(t, "real/x.txt")
+ if err := os.Symlink(filepath.Join(root, "real"), filepath.Join(root, "alias")); err != nil {
+ t.Fatal(err)
+ }
+ r, err := Walk(root, Options{Recursive: true, Now: now})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if want := []string{"real/x.txt"}; !reflect.DeepEqual(rels(r), want) {
+ t.Fatalf("files = %v, want %v", rels(r), want)
+ }
+ if skipped(r)["alias"] != Symlink {
+ t.Fatalf("skipped = %v", skipped(r))
+ }
+}
+
+func TestReasonString(t *testing.T) {
+ want := map[Reason]string{Ignored: "ignored", Busy: "busy", TooNew: "too new", Symlink: "symlink", NotRegular: "not a regular file", Unreadable: "unreadable"}
+ for r, s := range want {
+ if r.String() != s {
+ t.Errorf("%d = %q, want %q", r, r.String(), s)
+ }
+ }
+}
+
+func TestUnreadableSubdir(t *testing.T) {
+ if os.Geteuid() == 0 {
+ t.Skip("running as root: permissions are not enforced")
+ }
+ root := tree(t, "a.txt", "locked/secret.txt", "one/b.txt")
+ locked := filepath.Join(root, "locked")
+ if err := os.Chmod(locked, 0o000); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() {
+ if err := os.Chmod(locked, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ })
+ r, err := Walk(root, Options{Recursive: true, Now: now})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if want := []string{"a.txt", "one/b.txt"}; !reflect.DeepEqual(rels(r), want) {
+ t.Fatalf("files = %v, want %v", rels(r), want)
+ }
+ if got := skipped(r)["locked"]; got != Unreadable {
+ t.Fatalf("skipped[locked] = %v, want Unreadable", got)
+ }
+}
+
+func TestRootMustBeDirectory(t *testing.T) {
+ root := tree(t, "f")
+ if _, err := Walk(filepath.Join(root, "f"), Options{Now: now}); err == nil {
+ t.Fatal("walking a file succeeded")
+ }
+ if _, err := Walk(filepath.Join(root, "missing"), Options{Now: now}); err == nil {
+ t.Fatal("walking a missing path succeeded")
+ }
+}
+
+// fakeDirEntry is an os.DirEntry whose Info() returns a canned result,
+// simulating a race scan_test cannot otherwise provoke portably: a file
+// renamed or removed between being listed by ReadDir and having Info()
+// called on it (A2), or some other Info() failure.
+type fakeDirEntry struct {
+ name string
+ info fs.FileInfo
+ infoErr error
+}
+
+func (f fakeDirEntry) Name() string { return f.name }
+func (f fakeDirEntry) IsDir() bool { return false }
+func (f fakeDirEntry) Type() fs.FileMode { return 0 }
+func (f fakeDirEntry) Info() (fs.FileInfo, error) { return f.info, f.infoErr }
+
+// TestFileInfoFailureMidWalk: A2. A vanished file's Info() failure
+// (fs.ErrNotExist) is skipped silently, with no trace in Skipped; any
+// other Info() failure is reported as Unreadable; and the walk continues
+// to later entries in either case rather than aborting.
+func TestFileInfoFailureMidWalk(t *testing.T) {
+ root := t.TempDir()
+ okPath := filepath.Join(root, "ok.txt")
+ if err := os.WriteFile(okPath, []byte("ok"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ old := now.Add(-time.Hour)
+ if err := os.Chtimes(okPath, old, old); err != nil {
+ t.Fatal(err)
+ }
+ okInfo, err := os.Lstat(okPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ entries := []os.DirEntry{
+ fakeDirEntry{name: "vanished.txt", infoErr: fmt.Errorf("stat vanished.txt: %w", fs.ErrNotExist)},
+ fakeDirEntry{name: "denied.txt", infoErr: errors.New("stat denied.txt: permission denied")},
+ fakeDirEntry{name: "ok.txt", info: okInfo},
+ }
+
+ w := &walker{root: root, opt: Options{Now: now}}
+ if err := w.walk(root, "", 1, entries); err != nil {
+ t.Fatalf("walk aborted: %v", err)
+ }
+
+ if want := []string{"ok.txt"}; !reflect.DeepEqual(rels(&w.result), want) {
+ t.Fatalf("files = %v, want %v", rels(&w.result), want)
+ }
+ got := skipped(&w.result)
+ if _, vanished := got["vanished.txt"]; vanished {
+ t.Errorf("vanished file recorded as skipped: %v", got)
+ }
+ if got["denied.txt"] != Unreadable {
+ t.Errorf("skipped[denied.txt] = %v, want Unreadable", got["denied.txt"])
+ }
+}
+
+// TestIgnoredDirectoryReportedOnce: C2. An ignored directory with files
+// inside it is reported once, for the directory itself, not once per file
+// — pruning it without descending is the point — so its contents are not
+// simply invisible to every count and to -v.
+func TestIgnoredDirectoryReportedOnce(t *testing.T) {
+ root := tree(t, "keep.txt", "skip/a.txt", "skip/b.txt", "skip/c.txt")
+ m, _ := ignore.New([]string{"skip/"})
+ r, err := Walk(root, Options{Recursive: true, Ignore: m, Now: now})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if want := []string{"keep.txt"}; !reflect.DeepEqual(rels(r), want) {
+ t.Fatalf("files = %v, want %v", rels(r), want)
+ }
+ if want := (map[string]Reason{"skip": Ignored}); !reflect.DeepEqual(skipped(r), want) {
+ t.Fatalf("skipped = %v, want %v (the directory once, not its three files)", skipped(r), want)
+ }
+}