diff options
39 files changed, 6729 insertions, 27 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b5fb8b..b5adf3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,4 +2,4 @@ ## Unreleased -- Initial development: config language, `krino init`, `krino new`, `krino check`. +- Initial development: config language, `krino init`, `krino new`, `krino check`, `krino explain`, and the `-n` dry run. diff --git a/cmd/krino/check.go b/cmd/krino/check.go index fcd507d..1820bbe 100644 --- a/cmd/krino/check.go +++ b/cmd/krino/check.go @@ -5,40 +5,52 @@ package main import ( "fmt" "io" - "os" - "krino/internal/config" + "krino/internal/engine" "krino/internal/xdg" ) func init() { commands["check"] = cmdCheck } -// cmdCheck validates the configuration and lists each directory's rules. +// cmdCheck validates the configuration and lists each directory's rules, +// its current state, and the extractors available for content tests. func cmdCheck(g *globals, args []string, stdout, stderr io.Writer) int { fs := flagSet("check", g) if code, ok := parse(fs, args, stdout, stderr); !ok { return code } - cfg, errs := config.Load(mainFile(g), fs.Args()...) + e, errs := engine.Load(mainFile(g), fs.Args()...) if len(errs) > 0 { printDiags(stderr, errs) return 2 } - fmt.Fprintf(stdout, "config: %s\n", xdg.Abbrev(cfg.Main.File)) - if len(cfg.Dirs) == 0 { + r := e.Check() + + fmt.Fprintf(stdout, "config: %s\n", xdg.Abbrev(r.MainFile)) + fmt.Fprintf(stdout, "log: %s\n", xdg.Abbrev(r.LogFile)) + if len(r.Dirs) == 0 { fmt.Fprintln(stdout, "no directories included; add one with: krino new NAME PATH") } - for _, d := range cfg.Dirs { - fmt.Fprintf(stdout, "\n%s %s\n", d.Name, xdg.Abbrev(d.Path)) - if fi, err := os.Stat(d.Path); err != nil || !fi.IsDir() { - fmt.Fprintf(stdout, " warning: %s is not a directory right now; it will be skipped\n", xdg.Abbrev(d.Path)) + for _, dr := range r.Dirs { + fmt.Fprintf(stdout, "\n%s %s\n", dr.Dir.Name, xdg.Abbrev(dr.Dir.Root)) + if dr.Missing { + fmt.Fprintf(stdout, " warning: %s is not a directory right now; it will be skipped\n", xdg.Abbrev(dr.Dir.Root)) } - if len(d.Rules) == 0 { + if len(dr.Dir.Rules) == 0 { fmt.Fprintln(stdout, " no rules yet") } - for i, r := range d.Rules { - fmt.Fprintf(stdout, " %2d %-16s %s\n", i+1, r.Name, describeActions(r)) + for i, rule := range dr.Dir.Rules { + fmt.Fprintf(stdout, " %2d %-16s %s\n", i+1, rule.Name, describeActions(rule.Conf)) + } + } + + fmt.Fprintln(stdout, "\nextractors:") + for _, tool := range r.Tools { + path := "not installed" + if tool.Path != "" { + path = xdg.Abbrev(tool.Path) } + fmt.Fprintf(stdout, " %-9s %s\n", tool.Name, path) } return 0 } diff --git a/cmd/krino/commands_test.go b/cmd/krino/commands_test.go index 68a2174..2156da2 100644 --- a/cmd/krino/commands_test.go +++ b/cmd/krino/commands_test.go @@ -9,12 +9,14 @@ import ( "testing" ) -// home gives each test its own HOME with an empty XDG config. +// home gives each test its own HOME with no XDG overrides. func home(t *testing.T) string { t.Helper() h := t.TempDir() t.Setenv("HOME", h) - t.Setenv("XDG_CONFIG_HOME", "") + for _, v := range []string{"XDG_CONFIG_HOME", "XDG_STATE_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME"} { + t.Setenv(v, "") + } return h } diff --git a/cmd/krino/explain.go b/cmd/krino/explain.go new file mode 100644 index 0000000..a4d0253 --- /dev/null +++ b/cmd/krino/explain.go @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package main + +import ( + "bytes" + "context" + "fmt" + "io" + "strings" + + "krino/internal/cond" + "krino/internal/engine" + "krino/internal/xdg" +) + +func init() { commands["explain"] = cmdExplain } + +// cmdExplain evaluates every rule of FILE's directory against it and shows +// each test's result. +func cmdExplain(g *globals, args []string, stdout, stderr io.Writer) int { + fs := flagSet("explain", g) + if code, ok := parse(fs, args, stdout, stderr); !ok { + return code + } + if fs.NArg() != 1 { + return usageError(stderr, "usage: krino explain FILE") + } + + e, errs := engine.Load(mainFile(g)) + if len(errs) > 0 { + printDiags(stderr, errs) + return 2 + } + + x, err := e.Explain(context.Background(), xdg.Expand(fs.Arg(0))) + if err != nil { + fmt.Fprintf(stderr, "krino: %v\n", err) + return 2 + } + + fmt.Fprintf(stdout, "%s (directory %s)\n", xdg.Abbrev(x.File.Path), x.Dir.Name) + if x.Skip != "" { + fmt.Fprintf(stdout, "krino would not look at this file: %s\n", x.Skip) + } + fmt.Fprintln(stdout) + for _, rt := range x.Rules { + if rt.Stopped != "" { + fmt.Fprintf(stdout, "rule %s: not evaluated, %s\n", rt.Rule.Name, rt.Stopped) + continue + } + status := "no" + if rt.Match { + status = "MATCH" + } + fmt.Fprintf(stdout, "rule %s: %s\n", rt.Rule.Name, status) + printTrace(stdout, rt.Trace) + } + return 0 +} + +// printTrace writes t's Format output with every line prefixed by two +// spaces. +func printTrace(w io.Writer, t *cond.Trace) { + var buf bytes.Buffer + t.Format(&buf) + for _, line := range strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") { + fmt.Fprintf(w, " %s\n", line) + } +} diff --git a/cmd/krino/main.go b/cmd/krino/main.go index 0048098..9618f6c 100644 --- a/cmd/krino/main.go +++ b/cmd/krino/main.go @@ -97,9 +97,3 @@ func parse(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) (int, bool return 2, false } } - -// cmdSort plans and applies the included directories. It arrives in plan 3. -func cmdSort(g *globals, names []string, stdout, stderr io.Writer) int { - fmt.Fprintln(stderr, "krino: sorting is not implemented yet; try 'krino check'") - return 2 -} diff --git a/cmd/krino/main_test.go b/cmd/krino/main_test.go index 1caab6b..46ceb44 100644 --- a/cmd/krino/main_test.go +++ b/cmd/krino/main_test.go @@ -52,6 +52,7 @@ func TestCommandsAreReserved(t *testing.T) { } func TestSortNotYet(t *testing.T) { + home(t) code, _, errOut := runCLI(t) if code != 2 || !strings.Contains(errOut, "not implemented yet") { t.Fatalf("got %d %q", code, errOut) diff --git a/cmd/krino/matching_test.go b/cmd/krino/matching_test.go new file mode 100644 index 0000000..eaf9494 --- /dev/null +++ b/cmd/krino/matching_test.go @@ -0,0 +1,374 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +const dlRules = ` +(path "~/dl") +(recursive yes) +(min-age 0s) +(ignore "*.part") +(rule "dups" (when (duplicate)) (delete) (stop)) +(rule "acme" (when (type document) (content "acme ltd")) (move "Work/Acme") (stop)) +(rule "images" (when (type image)) (move "Pictures")) +(rule "rest" (when (not (matched)) (type text)) (move "Other")) +` + +// matchingFixture creates ~/dl, a config for it, and an empty PATH, so no +// extraction tool exists. +func matchingFixture(t *testing.T) string { + t.Helper() + h := home(t) + t.Setenv("PATH", t.TempDir()) + dl := filepath.Join(h, "dl") + files := map[string]string{ + "inv1.txt": "Invoice from ACME LTD", "notes.txt": "shopping list", + "photo.jpg": "\xff\xd8 jpeg", "report.pdf": "%PDF same", "report (1).pdf": "%PDF same", + "brochure.doc": "\xd0\xcf doc", "movie.mkv": "v", "movie.mkv.part": "p", + } + old := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for n, b := range files { + p := filepath.Join(dl, n) + os.MkdirAll(filepath.Dir(p), 0o755) + os.WriteFile(p, []byte(b), 0o644) + os.Chtimes(p, old, old) + } + os.Chtimes(filepath.Join(dl, "report (1).pdf"), old.Add(time.Hour), old.Add(time.Hour)) + if code, _, errOut := runCLI(t, "init"); code != 0 { + t.Fatal(errOut) + } + if code, _, errOut := runCLI(t, "new", "dl", dl); code != 0 { + t.Fatal(errOut) + } + if err := os.WriteFile(filepath.Join(h, ".config/krino/dirs/dl.conf"), []byte(dlRules), 0o644); err != nil { + t.Fatal(err) + } + return h +} + +func TestDryRun(t *testing.T) { + matchingFixture(t) + code, out, errOut := runCLI(t, "-n") + if code != 0 { + t.Fatalf("exit %d: %s", code, errOut) + } + for _, want := range []string{ + "krino: dl ~/dl\n8 scanned · 4 matched · 2 warnings · ", + "\n inv1.txt acme: type txt, content \"acme ltd\"\n", + "\n notes.txt rest: not matched, type txt\n", + "\n report (1).pdf dups: duplicate of report.pdf\n", + "\nwarnings\n brochure.doc acme: content unreadable: needs antiword or catdoc, not installed\n", + "\nnot matched: 2 · ignored: 1 · busy: 1 (-v lists them)\n", + } { + if !strings.Contains(out, want) { + t.Errorf("output lacks %q:\n%s", want, out) + } + } +} + +func TestDryRunVerbose(t *testing.T) { + matchingFixture(t) + _, out, _ := runCLI(t, "-n", "-v") + for _, want := range []string{ + "not matched: 2 · ignored: 1 · busy: 1\n", + "\nnot matched\n brochure.doc\n report.pdf\n", + "\nskipped\n movie.mkv busy\n movie.mkv.part ignored\n", + } { + if !strings.Contains(out, want) { + t.Errorf("verbose output lacks %q:\n%s", want, out) + } + } +} + +func TestExplainCommand(t *testing.T) { + h := matchingFixture(t) + code, out, errOut := runCLI(t, "explain", filepath.Join(h, "dl", "inv1.txt")) + if code != 0 { + t.Fatalf("exit %d: %s", code, errOut) + } + for _, want := range []string{ + "~/dl/inv1.txt (directory dl)\n", + "rule dups: no\n no duplicate\n", + "rule acme: MATCH\n yes and\n yes type document\n yes content \"acme ltd\"\n", + "rule images: not evaluated, stopped by rule acme\n", + } { + if !strings.Contains(out, want) { + t.Errorf("explain output lacks %q:\n%s", want, out) + } + } + _, out, _ = runCLI(t, "explain", "~/dl/movie.mkv") + if !strings.Contains(out, "krino would not look at this file: busy\n") { + t.Errorf("busy file:\n%s", out) + } + if code, _, errOut := runCLI(t, "explain"); code != 2 || !strings.Contains(errOut, "usage: krino explain FILE") { + t.Errorf("no argument: %d %q", code, errOut) + } +} + +func TestSortFlags(t *testing.T) { + matchingFixture(t) + tests := []struct { + args []string + want string + }{ + {[]string{"-y", "-n"}, "-y and -n cannot be used together"}, + {nil, "applying files is not implemented yet; use -n to see what would happen"}, + {[]string{"-n", "--json"}, "--json is not implemented yet"}, + } + for _, tt := range tests { + if code, _, errOut := runCLI(t, tt.args...); code != 2 || !strings.Contains(errOut, tt.want) { + t.Errorf("%v: %d %q, want %q", tt.args, code, errOut, tt.want) + } + } +} + +func TestCheckListsExtractors(t *testing.T) { + h := matchingFixture(t) + bin := filepath.Join(h, "bin") + os.Mkdir(bin, 0o755) + os.WriteFile(filepath.Join(bin, "pdftotext"), []byte("#!/bin/sh\n"), 0o755) + t.Setenv("PATH", bin) + code, out, errOut := runCLI(t, "check") + if code != 0 { + t.Fatalf("exit %d: %s", code, errOut) + } + for _, want := range []string{"log: ~/.local/state/krino/krino.log\n", "\nextractors:\n pdftotext ~/bin/pdftotext\n antiword not installed\n"} { + if !strings.Contains(out, want) { + t.Errorf("check output lacks %q:\n%s", want, out) + } + } + os.WriteFile(filepath.Join(h, ".config/krino/dirs/dl.conf"), []byte(`(path "~/dl") (rule "x" (when (type "pdf")) (stop))`), 0o644) + if code, _, errOut := runCLI(t, "check"); code != 2 || !strings.Contains(errOut, `dl.conf:1:37: type names are bare words: write (type pdf)`) { + t.Errorf("condition error: %d %q", code, errOut) + } +} + +// TestDryRunWarningsSortedByRel is controller ruling 2026-09-12: the +// warnings section is one Rel-sorted list across matched and unmatched +// files, not matched files followed by unmatched files - a reader scans it +// by name and has no way to see which group a file fell into. "cover.pdf" +// matches "pdfs" but still carries the warning "acme" recorded before it +// gave up; "brochure.doc" never matches at all. Their Rel order +// ("brochure.doc" < "cover.pdf") is the reverse of matched-then-unmatched +// grouping, so a grouped rendering fails this. +func TestDryRunWarningsSortedByRel(t *testing.T) { + h := home(t) + t.Setenv("PATH", t.TempDir()) + dl := filepath.Join(h, "dl") + files := map[string]string{ + "brochure.doc": "\xd0\xcf doc", + "cover.pdf": "%PDF fake", + } + old := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for n, b := range files { + p := filepath.Join(dl, n) + os.MkdirAll(filepath.Dir(p), 0o755) + os.WriteFile(p, []byte(b), 0o644) + os.Chtimes(p, old, old) + } + if code, _, errOut := runCLI(t, "init"); code != 0 { + t.Fatal(errOut) + } + if code, _, errOut := runCLI(t, "new", "dl", dl); code != 0 { + t.Fatal(errOut) + } + conf := ` +(path "~/dl") +(recursive yes) +(min-age 0s) +(rule "acme" (when (content "acme ltd")) (move "Work/Acme")) +(rule "pdfs" (when (type pdf)) (move "PDFs")) +` + if err := os.WriteFile(filepath.Join(h, ".config/krino/dirs/dl.conf"), []byte(conf), 0o644); err != nil { + t.Fatal(err) + } + + code, out, errOut := runCLI(t, "-n") + if code != 0 { + t.Fatalf("exit %d: %s", code, errOut) + } + want := "\nwarnings\n brochure.doc acme: content unreadable: needs antiword or catdoc, not installed\n cover.pdf acme: content unreadable: needs pdftotext, not installed\n" + if !strings.Contains(out, want) { + t.Errorf("warnings not Rel-sorted across matched and unmatched:\n%s\nwant substring:\n%s", out, want) + } +} + +// TestDirectoryWarningAfterHeader: C3. A directory-level warning must be +// emitted after its own header line, not before it, so on a terminal (both +// streams sharing one tty, hence stdout and stderr driven into the same +// buffer here to observe their relative order) it reads as describing the +// directory just named instead of floating above it. +func TestDirectoryWarningAfterHeader(t *testing.T) { + h := home(t) + dl := filepath.Join(h, "dl") + if err := os.MkdirAll(dl, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dl, "only.txt"), []byte("hello"), 0o644); err != nil { + t.Fatal(err) + } + if code, _, errOut := runCLI(t, "init"); code != 0 { + t.Fatal(errOut) + } + if code, _, errOut := runCLI(t, "new", "dl", dl); code != 0 { + t.Fatal(errOut) + } + conf := ` +(path "~/dl") +(recursive yes) +(min-age 0s) +(rule "r" (when (duplicate "~/missing")) (stop)) +` + if err := os.WriteFile(filepath.Join(h, ".config/krino/dirs/dl.conf"), []byte(conf), 0o644); err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + if code := run([]string{"-n"}, &buf, &buf); code != 0 { + t.Fatalf("exit %d: %s", code, buf.String()) + } + out := buf.String() + header := strings.Index(out, "krino: dl ~/dl\n") + warning := strings.Index(out, "krino: dl: duplicate: ") + if header == -1 || warning == -1 || warning < header { + t.Fatalf("directory-level warning not after its header line:\n%s", out) + } +} + +// TestSortSkipsMissingRootAndContinues: C5, the exit-1 skip path. A +// directory whose root has vanished since it was configured is skipped +// with one line on stderr naming it, but every other directory is still +// processed, with a blank line still separating their two outputs, and +// the run as a whole exits 1. +func TestSortSkipsMissingRootAndContinues(t *testing.T) { + h := home(t) + a, b, gone := filepath.Join(h, "a"), filepath.Join(h, "b"), filepath.Join(h, "gone") + for _, p := range []string{a, b, gone} { + if err := os.MkdirAll(p, 0o755); err != nil { + t.Fatal(err) + } + } + if code, _, errOut := runCLI(t, "init"); code != 0 { + t.Fatal(errOut) + } + for _, tt := range []struct{ name, path string }{{"a", a}, {"gone", gone}, {"b", b}} { + if code, _, errOut := runCLI(t, "new", tt.name, tt.path); code != 0 { + t.Fatal(errOut) + } + } + if err := os.RemoveAll(gone); err != nil { + t.Fatal(err) + } + + code, out, errOut := runCLI(t, "-n") + if code != 1 { + t.Fatalf("exit = %d, want 1", code) + } + if want := "krino: skipping gone: ~/gone is not a directory\n"; errOut != want { + t.Fatalf("stderr = %q, want %q", errOut, want) + } + ai := strings.Index(out, "krino: a ~/a\n") + bi := strings.Index(out, "\n\nkrino: b ~/b\n") + if ai == -1 || bi == -1 || bi < ai { + t.Fatalf("directories a and b not both processed with a blank line between them:\n%s", out) + } +} + +// TestDirectoryWarningNotCountedInWarningsField: C5. A directory-level +// warning is printed as "krino: NAME: <warning>" on stderr, but is not one +// of the per-file warnings the "N warnings" field in the summary line +// counts. +func TestDirectoryWarningNotCountedInWarningsField(t *testing.T) { + h := home(t) + dl := filepath.Join(h, "dl") + if err := os.MkdirAll(dl, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dl, "only.txt"), []byte("hello"), 0o644); err != nil { + t.Fatal(err) + } + if code, _, errOut := runCLI(t, "init"); code != 0 { + t.Fatal(errOut) + } + if code, _, errOut := runCLI(t, "new", "dl", dl); code != 0 { + t.Fatal(errOut) + } + conf := ` +(path "~/dl") +(recursive yes) +(min-age 0s) +(rule "r" (when (duplicate "~/missing")) (stop)) +` + if err := os.WriteFile(filepath.Join(h, ".config/krino/dirs/dl.conf"), []byte(conf), 0o644); err != nil { + t.Fatal(err) + } + + code, out, errOut := runCLI(t, "-n") + if code != 0 { + t.Fatalf("exit %d: %s", code, errOut) + } + if want := "krino: dl: duplicate: "; !strings.Contains(errOut, want) { + t.Fatalf("stderr lacks the directory-level warning %q:\n%s", want, errOut) + } + if want := "0 warnings"; !strings.Contains(out, want) { + t.Fatalf("summary line should not count the directory-level warning:\n%s", out) + } +} + +// TestLongNameNotPaddedLayoutIntact: C5, the 40-character cap. A file name +// longer than the 40-character column cap is left unpadded (not truncated, +// not stretched further), while a short name alongside it is still padded +// out to the full 40-column cap — the layout stays a clean two-column grid +// even though one row's first cell overruns it. +func TestLongNameNotPaddedLayoutIntact(t *testing.T) { + h := home(t) + dl := filepath.Join(h, "dl") + long := strings.Repeat("a", 42) + ".txt" // 46 runes: past the 40-column cap + short := "b.txt" + old := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for _, name := range []string{long, short} { + p := filepath.Join(dl, name) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(p, old, old); err != nil { + t.Fatal(err) + } + } + if code, _, errOut := runCLI(t, "init"); code != 0 { + t.Fatal(errOut) + } + if code, _, errOut := runCLI(t, "new", "dl", dl); code != 0 { + t.Fatal(errOut) + } + conf := ` +(path "~/dl") +(min-age 0s) +(rule "r" (when (type txt)) (stop)) +` + if err := os.WriteFile(filepath.Join(h, ".config/krino/dirs/dl.conf"), []byte(conf), 0o644); err != nil { + t.Fatal(err) + } + + code, out, errOut := runCLI(t, "-n") + if code != 0 { + t.Fatalf("exit %d: %s", code, errOut) + } + if want := "\n " + long + " r: type txt\n"; !strings.Contains(out, want) { + t.Errorf("long name should be unpadded (exactly two trailing spaces before the rule column):\n%s\nwant substring:\n%s", out, want) + } + if want := "\n " + short + strings.Repeat(" ", 40-len(short)) + " r: type txt\n"; !strings.Contains(out, want) { + t.Errorf("short name should still be padded to the 40-column cap:\n%s\nwant substring:\n%s", out, want) + } +} diff --git a/cmd/krino/sort.go b/cmd/krino/sort.go new file mode 100644 index 0000000..fe88cf8 --- /dev/null +++ b/cmd/krino/sort.go @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package main + +import ( + "context" + "fmt" + "io" + "os" + "sort" + "strings" + "unicode/utf8" + + "krino/internal/engine" + "krino/internal/scan" + "krino/internal/xdg" +) + +// cmdSort plans and applies the included directories. Only -n (dry run) is +// implemented; applying arrives in plan 3. +func cmdSort(g *globals, names []string, stdout, stderr io.Writer) int { + if g.yes && g.dry { + return usageError(stderr, "-y and -n cannot be used together") + } + if g.json { + fmt.Fprintln(stderr, "krino: --json is not implemented yet") + return 2 + } + if !g.dry { + fmt.Fprintln(stderr, "krino: applying files is not implemented yet; use -n to see what would happen") + return 2 + } + + e, errs := engine.Load(mainFile(g), names...) + if len(errs) > 0 { + printDiags(stderr, errs) + return 2 + } + + exit := 0 + printed := false + for _, d := range e.Dirs { + if fi, err := os.Stat(d.Root); err != nil || !fi.IsDir() { + fmt.Fprintf(stderr, "krino: skipping %s: %s is not a directory\n", d.Name, xdg.Abbrev(d.Root)) + exit = 1 + continue + } + r, err := e.Match(context.Background(), d) + if err != nil { + fmt.Fprintf(stderr, "krino: skipping %s: %v\n", d.Name, err) + exit = 1 + continue + } + if printed { + fmt.Fprintln(stdout) + } + printed = true + fmt.Fprintf(stdout, "krino: %s %s\n", d.Name, xdg.Abbrev(d.Root)) + // C3: directory-level warnings go to stderr after the header + // line above, not before it, so on a terminal they read as + // describing the directory just named instead of floating above it. + for _, w := range r.Warnings { + fmt.Fprintf(stderr, "krino: %s: %s\n", d.Name, w) + } + printResult(stdout, r, g.verbose) + } + return exit +} + +// printResult renders one directory's match result the way krino -n shows +// it, below the header line cmdSort has already written: a summary line, +// then the matched, warnings and skip-count sections, each present only +// when it has something to show. Plan 3 reuses this for the outcome of an +// actual run. +func printResult(w io.Writer, r *engine.Result, verbose bool) { + // C1: spec §8.2's worked example is "266 scanned" against "41 to act + // on" and "not acted on: 3 busy · 12 ignored · 210 unmatched", and + // 41+3+12+210 = 266 - so scanned counts matched, unmatched and skipped + // alike, not just matched plus unmatched. + scanned := len(r.Matched) + len(r.Unmatched) + len(r.Skipped) + warned := 0 + for _, fm := range r.Matched { + if len(fm.Warnings) > 0 { + warned++ + } + } + for _, fm := range r.Unmatched { + if len(fm.Warnings) > 0 { + warned++ + } + } + fmt.Fprintf(w, "%d scanned · %d matched · %d warnings · %.2fs\n", scanned, len(r.Matched), warned, r.Elapsed.Seconds()) + + if len(r.Matched) > 0 { + fmt.Fprintln(w) + printMatched(w, r.Matched) + } + + if lines := collectWarnings(r); len(lines) > 0 { + fmt.Fprintln(w) + fmt.Fprintln(w, "warnings") + printWarnings(w, lines) + } + + if line := skipSummaryLine(r, verbose); line != "" { + fmt.Fprintln(w) + fmt.Fprintln(w, line) + } + + if verbose { + if len(r.Unmatched) > 0 { + fmt.Fprintln(w) + fmt.Fprintln(w, "not matched") + for _, fm := range r.Unmatched { + fmt.Fprintf(w, " %s\n", fm.File.Rel) + } + } + if len(r.Skipped) > 0 { + fmt.Fprintln(w) + fmt.Fprintln(w, "skipped") + printSkipped(w, r.Skipped) + } + } +} + +// printMatched lists each matched file, its Rel padded to the widest shown +// (capped at 40), then each matching rule as "name: reasons", rules joined +// by "; ". +func printMatched(w io.Writer, matched []engine.FileMatch) { + rels := make([]string, len(matched)) + for i, fm := range matched { + rels[i] = fm.File.Rel + } + width := relWidth(rels) + for _, fm := range matched { + parts := make([]string, len(fm.Rules)) + for i, rm := range fm.Rules { + parts[i] = rm.Rule.Name + ": " + strings.Join(rm.Reasons, ", ") + } + fmt.Fprintf(w, " %s %s\n", padCell(fm.File.Rel, width), strings.Join(parts, "; ")) + } +} + +// warnLine is one file's warning, for the warnings section. +type warnLine struct { + rel string + text string +} + +// collectWarnings gathers every file's warnings into a single list sorted +// by Rel across matched and unmatched files alike: a reader scans this +// section by file name and has no way to tell which group a file fell +// into, so grouping by match state is invisible structure that would only +// show up as an odd order. A file's own warnings (when it has more than +// one) stay in the order they were recorded. +func collectWarnings(r *engine.Result) []warnLine { + files := make([]engine.FileMatch, 0, len(r.Matched)+len(r.Unmatched)) + files = append(files, r.Matched...) + files = append(files, r.Unmatched...) + sort.Slice(files, func(i, j int) bool { return files[i].File.Rel < files[j].File.Rel }) + + var out []warnLine + for _, fm := range files { + for _, w := range fm.Warnings { + out = append(out, warnLine{fm.File.Rel, w}) + } + } + return out +} + +// printWarnings lists one line per warning, Rel padded to the widest shown +// (capped at 40). +func printWarnings(w io.Writer, lines []warnLine) { + rels := make([]string, len(lines)) + for i, l := range lines { + rels[i] = l.rel + } + width := relWidth(rels) + for _, l := range lines { + fmt.Fprintf(w, " %s %s\n", padCell(l.rel, width), l.text) + } +} + +// printSkipped lists each skipped file, Rel padded to the widest shown +// (capped at 40), then its reason. +func printSkipped(w io.Writer, skipped []scan.Skipped) { + rels := make([]string, len(skipped)) + for i, s := range skipped { + rels[i] = s.Rel + } + width := relWidth(rels) + for _, s := range skipped { + fmt.Fprintf(w, " %s %s\n", padCell(s.Rel, width), s.Reason.String()) + } +} + +// skipReasonOrder is the order the last line reports skip reasons in, +// after "not matched". +var skipReasonOrder = []scan.Reason{scan.Ignored, scan.Busy, scan.TooNew, scan.Symlink, scan.NotRegular, scan.Unreadable} + +// skipSummaryLine builds the "not matched: N · ignored: N ..." line, only +// the non-zero counts, or "" when every count is zero. +func skipSummaryLine(r *engine.Result, verbose bool) string { + counts := map[scan.Reason]int{} + for _, s := range r.Skipped { + counts[s.Reason]++ + } + + var parts []string + if n := len(r.Unmatched); n > 0 { + parts = append(parts, fmt.Sprintf("not matched: %d", n)) + } + for _, reason := range skipReasonOrder { + if n := counts[reason]; n > 0 { + parts = append(parts, fmt.Sprintf("%s: %d", reason.String(), n)) + } + } + if len(parts) == 0 { + return "" + } + line := strings.Join(parts, " · ") + if !verbose { + line += " (-v lists them)" + } + return line +} + +// relWidth returns the column width for a list of Rel names: the widest in +// runes (C4: not bytes, or a name carrying diacritics misaligns its +// column), capped at 40. +func relWidth(rels []string) int { + w := 0 + for _, s := range rels { + if n := utf8.RuneCountInString(s); n > w { + w = n + } + } + if w > 40 { + w = 40 + } + return w +} + +// padCell pads s to width w (runes, not bytes) with trailing spaces; s +// already at or beyond w is left unpadded. +func padCell(s string, w int) string { + n := utf8.RuneCountInString(s) + if n >= w { + return s + } + return s + strings.Repeat(" ", w-n) +} diff --git a/cmd/krino/sort_test.go b/cmd/krino/sort_test.go new file mode 100644 index 0000000..a646503 --- /dev/null +++ b/cmd/krino/sort_test.go @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package main + +import "testing" + +// TestRelWidthAndPadCellCountRunes: C4. relWidth and padCell must measure +// column width in runes, not bytes, or a name carrying diacritics +// misaligns its column - "próba.txt" is 9 runes but 10 bytes (ó is a +// two-byte UTF-8 sequence), one column narrower than its byte length +// would suggest. +func TestRelWidthAndPadCellCountRunes(t *testing.T) { + rels := []string{"a.txt", "próba.txt"} + if w := relWidth(rels); w != 9 { + t.Fatalf("relWidth(%q) = %d, want 9 (rune count of próba.txt, not its %d bytes)", rels, w, len("próba.txt")) + } + if got, want := padCell("a.txt", 9), "a.txt "; got != want { + t.Fatalf("padCell(%q, 9) = %q, want %q", "a.txt", got, want) + } + if got, want := padCell("próba.txt", 9), "próba.txt"; got != want { + t.Fatalf("padCell(%q, 9) = %q, want %q (already at width: no padding)", "próba.txt", got, want) + } +} diff --git a/docs/design.md b/docs/design.md index ed77e8c..888ab34 100644 --- a/docs/design.md +++ b/docs/design.md @@ -204,8 +204,8 @@ decompose (ł ø đ ħ ß æ œ and their capitals) to ASCII. 2. Rules are evaluated in order. A matching rule adds its actions to the file's chain. `(stop)` on a matching rule ends evaluation for that file. 3. Tests have no side effects, so `and` / `or` evaluate their arguments - cheapest first: `type`, `name`, `path`, `size`, `age`, `matched`, then - `duplicate`, then `content`. Content is extracted at most once per file, + cheapest first: `type`, `size`, `age` and `matched`, then `name` and + `path`, then `duplicate`, then `content`. Content is extracted at most once per file, and only if evaluation reaches a `content` test. 4. If a file's text cannot be extracted, `content` is false and the plan shows a warning naming the rule that wanted it. @@ -330,6 +330,11 @@ name has to change, the log records the actual name. - Always ignored: every rule `DEST` that lies inside the root, the Trash, and the config directory. For a `DEST` with placeholders, the part before the first placeholder is ignored: `Work/Acme/{mtime:%Y}` ignores `Work/Acme/`. + Known limitation: a `DEST` whose *first* path component is itself a + placeholder (`{ext}`, `{mtime:%Y}`) has no static prefix, so nothing can be + excluded before the walk, and krino would re-examine its own output; plan 3 + closes this by treating a file already at its computed destination as a + no-op. - Skipped as busy: files newer than `min-age`, or with a `busy` sibling. ### 8.2 Display @@ -471,8 +476,10 @@ cmd/krino/ flags, subcommands, exit codes internal/sexp/ reader: syntax tree with positions and byte offsets internal/config/ syntax tree → typed config; validation; embedded template and defaults internal/cond/ condition tree: compile, cost ordering, evaluation, explain trace -internal/scan/ walking, gitignore matcher, busy detection -internal/extract/ text extraction and normalisation +internal/scan/ walking, busy detection +internal/ignore/ gitignore-compatible path matcher +internal/extract/ text extraction +internal/norm/ text normalisation: case, diacritics, white space internal/dup/ duplicate index internal/plan/ files × rules → chains; placeholders; conflicts; JSON internal/apply/ executing approved steps; cross-filesystem moves @@ -1,3 +1,5 @@ module krino -go 1.24 +go 1.24.0 + +require golang.org/x/text v0.34.0 @@ -0,0 +1,2 @@ +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= diff --git a/internal/cond/compile.go b/internal/cond/compile.go new file mode 100644 index 0000000..4fdc75f --- /dev/null +++ b/internal/cond/compile.go @@ -0,0 +1,388 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package cond + +import ( + "errors" + "fmt" + "regexp" + "regexp/syntax" + "sort" + "strings" + + "krino/internal/config" + "krino/internal/norm" + "krino/internal/sexp" +) + +// maxDepth is the deepest a condition may nest; the top-level conditions +// are depth 1. +const maxDepth = 64 + +// Compile compiles a rule's (when ...) conditions, applying cost-based +// reordering. If it returns any diagnostic, the returned *Cond is nil: a +// caller that ignores errs and calls Eval/Explain on the result panics +// loudly, instead of silently treating a broken rule as "no conditions" +// (always true). +func Compile(file string, when []*sexp.Node, opt Options) (*Cond, []*config.Diag) { + c, errs := compile(file, when, opt, true) + if len(errs) > 0 { + return nil, errs + } + return c, errs +} + +// compile is Compile with cost reordering switchable, for the property test +// in Task 8. +func compile(file string, when []*sexp.Node, opt Options, reorder bool) (*Cond, []*config.Diag) { + c := &compiler{file: file, opt: opt, reorder: reorder, cond: &Cond{opt: opt}} + var roots []*node + for _, n := range when { + if nd := c.compileNode(n, 1); nd != nil { + roots = append(roots, nd) + } + } + switch len(roots) { + case 0: + // no conditions: root stays nil, always true + case 1: + c.cond.root = roots[0] + default: + // several top-level conditions are an implicit and + c.cond.root = c.combine(kAnd, "and", roots, when[0].Pos) + } + return c.cond, c.errs +} + +// compiler holds the state of one Compile call. +type compiler struct { + file string + opt Options + reorder bool + cond *Cond + errs []*config.Diag +} + +// errorf records a diagnostic at n's position; a nil n means the file as a +// whole (unused here, kept for symmetry with config.diags). +func (c *compiler) errorf(n *sexp.Node, format string, args ...any) { + var pos sexp.Pos + if n != nil { + pos = n.Pos + } + c.errs = append(c.errs, &config.Diag{File: c.file, Pos: pos, Msg: fmt.Sprintf(format, args...)}) +} + +// compileNode compiles one condition node at the given nesting depth (the +// top-level conditions are depth 1). It returns nil, having recorded at +// least one diagnostic, when n does not compile. +func (c *compiler) compileNode(n *sexp.Node, depth int) *node { + if depth > maxDepth { + c.errorf(n, "conditions nest deeper than %d levels", maxDepth) + return nil + } + if n.Kind != sexp.List || n.Head() == "" { + c.errorf(n, "a condition is a form like (type pdf), not %s", n.String()) + return nil + } + switch head := n.Head(); head { + case "and": + return c.compileCombiner(n, depth, kAnd, "and", "and needs at least one condition") + case "or": + return c.compileCombiner(n, depth, kOr, "or", "or needs at least one condition") + case "not": + return c.compileNot(n, depth) + case "type": + return c.compileType(n) + case "name": + return c.compileRegex(n, kName, "name") + case "path": + return c.compileRegex(n, kPath, "path") + case "content": + return c.compileContent(n) + case "size": + return c.compileSize(n) + case "age": + return c.compileAge(n) + case "duplicate": + return c.compileDuplicate(n) + case "matched": + return c.compileMatched(n) + default: + c.errorf(n, "unknown test (%s ...); tests are type, name, path, content, size, age, duplicate, matched, and, or, not", head) + return nil + } +} + +// compileCombiner compiles (and ...) / (or ...): one or more child +// conditions, cost-summed and cost-sorted (via combine). +func (c *compiler) compileCombiner(n *sexp.Node, depth int, k kind, label, emptyMsg string) *node { + args := n.Args() + if len(args) == 0 { + c.errorf(n, "%s", emptyMsg) + return nil + } + children := make([]*node, 0, len(args)) + for _, a := range args { + if ch := c.compileNode(a, depth+1); ch != nil { + children = append(children, ch) + } + } + return c.combine(k, label, children, n.Pos) +} + +// combine builds an and/or node from already-compiled children: stable +// cost-sort when reordering, cost is the sum of the children's. +func (c *compiler) combine(k kind, label string, children []*node, pos sexp.Pos) *node { + if len(children) == 0 { + return nil + } + if c.reorder { + sort.SliceStable(children, func(i, j int) bool { return children[i].cost < children[j].cost }) + } + cost := 0 + for _, ch := range children { + cost += ch.cost + } + return &node{kind: k, pos: pos, label: label, children: children, cost: cost} +} + +// compileNot compiles (not C): exactly one child condition. +func (c *compiler) compileNot(n *sexp.Node, depth int) *node { + args := n.Args() + if len(args) != 1 { + c.errorf(n, "not takes exactly one condition") + return nil + } + child := c.compileNode(args[0], depth+1) + if child == nil { + return nil + } + return &node{kind: kNot, pos: n.Pos, label: "not", children: []*node{child}, cost: child.cost} +} + +// compileType compiles (type T...): symbols, lower-cased; a group name +// expands to its extensions, anything else is a literal (possibly +// multi-part) extension. +func (c *compiler) compileType(n *sexp.Node) *node { + args := n.Args() + if len(args) == 0 { + c.errorf(n, "type needs at least one extension or group, like (type pdf)") + return nil + } + var suffixes []string + bad := false + for _, a := range args { + if a.Kind != sexp.Symbol { + c.errorf(a, "type names are bare words: write (type pdf)") + bad = true + continue + } + lower := strings.ToLower(a.Text) + if exts, ok := groups[lower]; ok { + for _, e := range exts { + suffixes = append(suffixes, "."+e) + } + } else { + suffixes = append(suffixes, "."+lower) + } + } + if bad { + return nil + } + return &node{kind: kType, pos: n.Pos, label: argsLabel("type", args), cost: costCheap, suffixes: suffixes} +} + +// compileRegex compiles (name "RE"...) / (path "RE"...). +func (c *compiler) compileRegex(n *sexp.Node, k kind, test string) *node { + args := n.Args() + if len(args) == 0 { + c.errorf(n, "%s needs at least one regex in quotes", test) + return nil + } + var patterns []pattern + bad := false + for _, a := range args { + if a.Kind != sexp.String { + c.quotedExampleErr(a, test, "regexes") + bad = true + continue + } + src := a.Text + pat := src + if c.opt.Fold { + // E5: folding is applied to the regex source itself, not just + // to the text it is matched against - so a fold that expands + // one character into several changes the pattern's structure, + // not just its literal characters: "ß+" (one letter, a + // quantifier on it) becomes "ss+" (a quantifier on only the + // second "s") once norm.Fold expands "ß" to "ss", and likewise + // "æ" to "ae". A rule relying on repetition or anchoring + // around such a letter needs to account for this. + pat = norm.Fold(pat) + } + if c.opt.IgnoreCase { + pat = "(?i)" + pat + } + re, err := regexp.Compile(pat) + if err != nil { + var se *syntax.Error + if errors.As(err, &se) { + c.errorf(a, "%s: bad regex %q: %s", test, src, se.Code) + } else { + c.errorf(a, "%s: bad regex %q: %s", test, src, err) + } + bad = true + continue + } + patterns = append(patterns, pattern{re: re, src: src}) + } + if bad { + return nil + } + return &node{kind: k, pos: n.Pos, label: argsLabel(test, args), cost: costRegex, patterns: patterns} +} + +// compileContent compiles (content "KW"...): each keyword is normalised; +// an empty result is an error. +func (c *compiler) compileContent(n *sexp.Node) *node { + args := n.Args() + if len(args) == 0 { + c.errorf(n, "content needs at least one keyword in quotes") + return nil + } + var keywords []keyword + bad := false + for _, a := range args { + if a.Kind != sexp.String { + c.quotedExampleErr(a, "content", "keywords") + bad = true + continue + } + normed := norm.Text(a.Text, c.opt.IgnoreCase, c.opt.Fold) + if normed == "" { + c.errorf(a, "content keyword is empty") + bad = true + continue + } + keywords = append(keywords, keyword{norm: normed, src: a.Text}) + } + if bad { + return nil + } + c.cond.UsesContent = true + return &node{kind: kContent, pos: n.Pos, label: argsLabel("content", args), cost: costContent, keywords: keywords} +} + +// compileSize compiles (size OP SIZE). +func (c *compiler) compileSize(n *sexp.Node) *node { + args := n.Args() + if len(args) != 2 || args[0].Kind != sexp.Symbol || args[1].Kind != sexp.Symbol { + c.errorf(n, "size takes an operator and a size, like (size > 10M)") + return nil + } + op := args[0].Text + bad := false + if !validOp(op) { + c.errorf(args[0], "size operator is one of > >= < <= =, not %s", op) + bad = true + } + val, err := config.ParseSize(args[1].Text) + if err != nil { + c.errorf(args[1], "size: %s", err) + bad = true + } + if bad { + return nil + } + return &node{kind: kSize, pos: n.Pos, label: argsLabel("size", args), cost: costCheap, op: op, sizeVal: val} +} + +// compileAge compiles (age OP DURATION). +func (c *compiler) compileAge(n *sexp.Node) *node { + args := n.Args() + if len(args) != 2 || args[0].Kind != sexp.Symbol || args[1].Kind != sexp.Symbol { + c.errorf(n, "age takes an operator and a duration, like (age > 30d)") + return nil + } + op := args[0].Text + bad := false + if !validOp(op) { + c.errorf(args[0], "age operator is one of > >= < <= =, not %s", op) + bad = true + } + val, err := config.ParseDuration(args[1].Text) + if err != nil { + c.errorf(args[1], "age: %s", err) + bad = true + } + if bad { + return nil + } + return &node{kind: kAge, pos: n.Pos, label: argsLabel("age", args), cost: costCheap, op: op, ageVal: val} +} + +// validOp reports whether op is one of the size/age comparison operators. +func validOp(op string) bool { + switch op { + case ">", ">=", "<", "<=", "=": + return true + } + return false +} + +// compileDuplicate compiles (duplicate "DIR"...): zero or more directories, +// stored raw for the engine to resolve. Every compiled test's list is +// appended to Cond.DupDirs, in order. +func (c *compiler) compileDuplicate(n *sexp.Node) *node { + args := n.Args() + dirs := make([]string, 0, len(args)) + bad := false + for _, a := range args { + if a.Kind != sexp.String { + c.quotedExampleErr(a, "duplicate", "directories") + bad = true + continue + } + dirs = append(dirs, a.Text) + } + if bad { + return nil + } + c.cond.DupDirs = append(c.cond.DupDirs, dirs) + return &node{kind: kDuplicate, pos: n.Pos, label: argsLabel("duplicate", args), cost: costDuplicate, dirs: dirs} +} + +// compileMatched compiles (matched): no arguments. +func (c *compiler) compileMatched(n *sexp.Node) *node { + if len(n.Args()) != 0 { + c.errorf(n, "matched takes nothing: write (matched)") + return nil + } + return &node{kind: kMatched, pos: n.Pos, label: "matched", cost: costCheap} +} + +// quotedExampleErr records the shared "takes X in quotes: write (test +// "arg")" diagnostic used by name, path, content and duplicate. +func (c *compiler) quotedExampleErr(a *sexp.Node, test, noun string) { + c.errorf(a, "%s takes %s in quotes: write (%s %s)", test, noun, test, sexp.Quote(a.Text)) +} + +// argsLabel renders a leaf test as written: the head followed by each +// argument, bare symbols verbatim, strings as their decoded text in double +// quotes without escaping. +func argsLabel(head string, args []*sexp.Node) string { + var b strings.Builder + b.WriteString(head) + for _, a := range args { + b.WriteByte(' ') + if a.Kind == sexp.String { + b.WriteByte('"') + b.WriteString(a.Text) + b.WriteByte('"') + } else { + b.WriteString(a.Text) + } + } + return b.String() +} diff --git a/internal/cond/compile_test.go b/internal/cond/compile_test.go new file mode 100644 index 0000000..e68a4d8 --- /dev/null +++ b/internal/cond/compile_test.go @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package cond + +import ( + "reflect" + "strings" + "testing" + + "krino/internal/sexp" +) + +func nodes(t *testing.T, src string) []*sexp.Node { + t.Helper() + n, err := sexp.Parse("d.conf", []byte(src)) + if err != nil { + t.Fatalf("parse %q: %v", src, err) + } + return n +} + +func TestCompileGood(t *testing.T) { + for _, src := range []string{ + ``, + `(type pdf)`, + `(type document image tar.gz)`, + `(and (type pdf) (or (content "acme ltd" "0000000000") (name "\bacme\b")) (not (name "^draft")))`, + `(path "^Work/") (size >= 10M) (age < 2w) (matched)`, + `(duplicate) (duplicate "Work" "~/docs")`, + `(not (not (type pdf)))`, + } { + if _, errs := Compile("d.conf", nodes(t, src), Options{IgnoreCase: true, Fold: true}); len(errs) > 0 { + t.Errorf("%s: %v", src, errs) + } + } +} + +func TestCompileErrors(t *testing.T) { + tests := []struct{ src, want string }{ + {`pdf`, `d.conf:1:1: a condition is a form like (type pdf), not pdf`}, + {`("type" pdf)`, `d.conf:1:1: a condition is a form like (type pdf), not ("type" pdf)`}, + {`(foo 1)`, `d.conf:1:1: unknown test (foo ...); tests are type, name, path, content, size, age, duplicate, matched, and, or, not`}, + {`(and)`, `d.conf:1:1: and needs at least one condition`}, + {`(or)`, `d.conf:1:1: or needs at least one condition`}, + {`(not)`, `d.conf:1:1: not takes exactly one condition`}, + {`(not (type pdf) (type doc))`, `d.conf:1:1: not takes exactly one condition`}, + {`(type)`, `d.conf:1:1: type needs at least one extension or group, like (type pdf)`}, + {`(type "pdf")`, `d.conf:1:7: type names are bare words: write (type pdf)`}, + {`(name)`, `d.conf:1:1: name needs at least one regex in quotes`}, + {`(path)`, `d.conf:1:1: path needs at least one regex in quotes`}, + {`(name x)`, `d.conf:1:7: name takes regexes in quotes: write (name "x")`}, + {`(name "(abc")`, `d.conf:1:7: name: bad regex "(abc": missing closing )`}, + {`(path "[z-a]")`, `d.conf:1:7: path: bad regex "[z-a]": invalid character class range`}, + {`(content)`, `d.conf:1:1: content needs at least one keyword in quotes`}, + {`(content acme)`, `d.conf:1:10: content takes keywords in quotes: write (content "acme")`}, + {`(content " ")`, `d.conf:1:10: content keyword is empty`}, + {`(size 10M)`, `d.conf:1:1: size takes an operator and a size, like (size > 10M)`}, + {`(age 30d)`, `d.conf:1:1: age takes an operator and a duration, like (age > 30d)`}, + {`(size >> 10M)`, `d.conf:1:7: size operator is one of > >= < <= =, not >>`}, + {`(size > 10Q)`, `d.conf:1:9: size: bad size "10Q": want a whole number with an optional K, M, G or T, like 50M`}, + {`(age > 30)`, `d.conf:1:8: age: bad duration "30": want a whole number followed by s, m, h, d or w, like 30d`}, + {`(duplicate Work)`, `d.conf:1:12: duplicate takes directories in quotes: write (duplicate "Work")`}, + {`(matched x)`, `d.conf:1:1: matched takes nothing: write (matched)`}, + } + for _, tt := range tests { + _, errs := Compile("d.conf", nodes(t, tt.src), Options{IgnoreCase: true}) + if len(errs) != 1 || errs[0].Error() != tt.want { + t.Errorf("%s:\n got %v\n want %s", tt.src, errs, tt.want) + } + } +} + +func TestCompileCollectsAllErrors(t *testing.T) { + _, errs := Compile("d.conf", nodes(t, `(type "a") (size 1) (matched x)`), Options{}) + if len(errs) != 3 { + t.Fatalf("got %d errors: %v", len(errs), errs) + } +} + +func TestDepthLimit(t *testing.T) { + deep := func(n int) string { return strings.Repeat("(not ", n) + "(type pdf)" + strings.Repeat(")", n) } + if _, errs := Compile("d.conf", nodes(t, deep(63)), Options{}); len(errs) != 0 { + t.Fatalf("64 levels rejected: %v", errs) + } + _, errs := Compile("d.conf", nodes(t, deep(64)), Options{}) + if len(errs) != 1 || !strings.HasSuffix(errs[0].Error(), "conditions nest deeper than 64 levels") { + t.Fatalf("65 levels: %v", errs) + } +} + +func TestFlagsAndDupDirs(t *testing.T) { + c, _ := Compile("d.conf", nodes(t, `(or (type pdf) (content "x")) (duplicate) (duplicate "Work" "~/docs")`), Options{}) + if !c.UsesContent { + t.Error("UsesContent not set") + } + if want := [][]string{{}, {"Work", "~/docs"}}; !reflect.DeepEqual(c.DupDirs, want) { + t.Errorf("DupDirs = %#v, want %#v", c.DupDirs, want) + } + c, _ = Compile("d.conf", nodes(t, `(type pdf)`), Options{}) + if c.UsesContent || len(c.DupDirs) != 0 { + t.Errorf("flags set without content/duplicate tests: %+v", c) + } +} + +func TestCompileErrorReturnsNilCond(t *testing.T) { + c, errs := Compile("d.conf", nodes(t, `(type "pdf")`), Options{}) + if len(errs) != 1 || c != nil { + t.Fatalf("got c=%v errs=%v, want c=nil and exactly one error", c, errs) + } +} + +// TestEmptyChildrenGuard is a regression test for combine's empty-children +// guard: an and/or all of whose children failed to compile must not become +// a hollow node that evaluates vacuously (and -> true). Without the guard, +// the inner (and (name "[")) would compile to an empty and, evaluate to +// true, and the outer or would match on any file. +func TestEmptyChildrenGuard(t *testing.T) { + c, errs := compile("d.conf", nodes(t, `(or (type pdf) (and (name "[")))`), Options{}, true) + if len(errs) != 1 { + t.Fatalf("got %d errors, want 1: %v", len(errs), errs) + } + if c.Eval(&fake{name: "x.txt"}).Match { + t.Error("hollow and inside or vacuously matched") + } +} + +func TestGroupsMatchSpec(t *testing.T) { + want := map[string]string{ + "image": "jpg jpeg png gif webp bmp tif tiff heic heif avif svg ico raw cr2 nef arw dng", + "video": "mp4 mkv webm mov avi m4v mpg mpeg wmv flv 3gp", + "audio": "mp3 flac ogg opus m4a aac wav wma aiff", + "archive": "zip tar gz tgz bz2 tbz2 xz txz zst 7z rar lz lzma cpio", + "document": "pdf doc docx odt rtf txt md tex", + "spreadsheet": "xls xlsx ods csv tsv", + "presentation": "ppt pptx odp", + "ebook": "epub mobi azw azw3 fb2 djvu", + "code": "go c h cpp hpp py sh js ts rs java rb pl lua html css json yaml yml toml xml sql", + "text": "txt md log csv tsv json yaml yml toml xml ini conf", + "package": "deb rpm apk appimage exe msi flatpak snap", + "font": "ttf otf woff woff2", + } + if len(groups) != len(want) { + t.Fatalf("%d groups, want %d", len(groups), len(want)) + } + for g, exts := range want { + if got := strings.Join(groups[g], " "); got != exts { + t.Errorf("group %s = %q, want %q", g, got, exts) + } + } +} diff --git a/internal/cond/eval.go b/internal/cond/eval.go new file mode 100644 index 0000000..e3093af --- /dev/null +++ b/internal/cond/eval.go @@ -0,0 +1,302 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package cond + +import ( + "fmt" + "io" + "strings" + "time" + + "krino/internal/norm" +) + +// Facts is what a condition may ask about one file. Implementations +// memoise: Eval and Explain may ask the same question more than once. +type Facts interface { + Name() string // base name + Rel() string // slash path relative to the root + Size() int64 + ModTime() time.Time + Now() time.Time + Content(ignoreCase, fold bool) (string, error) // normalised with norm.Text + Duplicate(dirs []string) (original string, ok bool, err error) + Matched() bool // an earlier rule matched this file +} + +// Result is the outcome of evaluating a Cond against one file's Facts. +type Result struct { + Match bool + Captures []string // submatches of the first true, non-negated name test: [0] whole match, [1:] groups + Reasons []string // what made it true, e.g. `type pdf`, `content "acme ltd"`, `name "\bacme\b"` + Warnings []string // e.g. `content unreadable: needs pdftotext, not installed` +} + +// Trace is the full evaluation of every node, for krino explain. +type Trace struct { + Label string + Value bool + Err string + Children []*Trace +} + +// evalCtx accumulates state across one Eval call: the captures of the +// first true, non-negated name test, and warnings de-duplicated in +// first-seen order. +type evalCtx struct { + captures []string + warned map[string]bool + warnings []string +} + +// warn records msg unless it has already been recorded. +func (ctx *evalCtx) warn(msg string) { + if msg == "" { + return + } + if ctx.warned == nil { + ctx.warned = map[string]bool{} + } + if ctx.warned[msg] { + return + } + ctx.warned[msg] = true + ctx.warnings = append(ctx.warnings, msg) +} + +// Eval evaluates c against f. and/or short-circuit in (cost-sorted) order, +// cheapest tests first, so an unreadable or slow test may never run. +func (c *Cond) Eval(f Facts) Result { + if c.root == nil { + return Result{Match: true, Reasons: []string{"no condition"}} + } + ctx := &evalCtx{} + match, reasons := c.eval(c.root, f, ctx, false) + return Result{Match: match, Captures: ctx.captures, Reasons: reasons, Warnings: ctx.warnings} +} + +// eval evaluates one node against f, short-circuiting and/or in child +// (cost-sorted) order. negated tracks whether n is reached under an odd +// number of enclosing nots, so a matching name test found there does not +// supply Result.Captures. +func (c *Cond) eval(n *node, f Facts, ctx *evalCtx, negated bool) (bool, []string) { + switch n.kind { + case kAnd: + var reasons []string + for _, ch := range n.children { + ok, r := c.eval(ch, f, ctx, negated) + if !ok { + return false, nil + } + reasons = append(reasons, r...) + } + return true, reasons + case kOr: + for _, ch := range n.children { + if ok, r := c.eval(ch, f, ctx, negated); ok { + return true, r + } + } + return false, nil + case kNot: + child := n.children[0] + ok, _ := c.eval(child, f, ctx, !negated) + if ok { + return false, nil + } + // E4: a negated leaf reads fine as "not " plus the leaf's own + // label ("not matched", "not type pdf"), but a negated and/or's + // bare label is just the word "and"/"or" - "not and"/"not or" + // reaches the user in the reasons column reading as nothing a + // person would write, so it is parenthesised instead, the way the + // config itself would write a negated combinator. + label := child.label + if child.kind == kAnd || child.kind == kOr { + label = "(" + child.label + " ...)" + } + return true, []string{"not " + label} + default: + ok, reason, warn, caps := c.evalLeaf(n, f) + if warn != "" { + ctx.warn(warn) + } + if !ok { + return false, nil + } + if caps != nil && !negated && ctx.captures == nil { + ctx.captures = caps + } + return true, []string{reason} + } +} + +// evalLeaf evaluates one leaf (non-combinator) node against f: whether it +// matched, its reason if so, a warning if a fact could not be read (only +// content and duplicate can fail), and (for a matching name test) its +// regex captures. +func (c *Cond) evalLeaf(n *node, f Facts) (ok bool, reason, warn string, caps []string) { + switch n.kind { + case kType: + lower := strings.ToLower(f.Name()) + for _, suf := range n.suffixes { + if strings.HasSuffix(lower, suf) { + return true, "type " + strings.TrimPrefix(suf, "."), "", nil + } + } + return false, "", "", nil + + case kName, kPath: + subj := f.Name() + word := "name" + if n.kind == kPath { + subj = f.Rel() + word = "path" + } + subj = norm.Name(subj, c.opt.Fold) + for _, p := range n.patterns { + m := p.re.FindStringSubmatch(subj) + if m == nil { + continue + } + reason = word + ` "` + p.src + `"` + if n.kind == kName { + return true, reason, "", m + } + return true, reason, "", nil + } + return false, "", "", nil + + case kContent: + text, err := f.Content(c.opt.IgnoreCase, c.opt.Fold) + if err != nil { + return false, "", "content unreadable: " + err.Error(), nil + } + for _, kw := range n.keywords { + if strings.Contains(text, kw.norm) { + return true, `content "` + kw.src + `"`, "", nil + } + } + return false, "", "", nil + + case kSize: + if compareInt64(f.Size(), n.op, n.sizeVal) { + return true, n.label, "", nil + } + return false, "", "", nil + + case kAge: + if compareDuration(f.Now().Sub(f.ModTime()), n.op, n.ageVal) { + return true, n.label, "", nil + } + return false, "", "", nil + + case kDuplicate: + orig, dup, err := f.Duplicate(n.dirs) + if err != nil { + return false, "", "duplicate check failed: " + err.Error(), nil + } + if dup { + return true, "duplicate of " + orig, "", nil + } + return false, "", "", nil + + case kMatched: + if f.Matched() { + return true, "matched", "", nil + } + return false, "", "", nil + } + return false, "", "", nil +} + +// compareInt64 applies a size comparison operator (one of > >= < <= =). +func compareInt64(v int64, op string, want int64) bool { + switch op { + case ">": + return v > want + case ">=": + return v >= want + case "<": + return v < want + case "<=": + return v <= want + case "=": + return v == want + } + return false +} + +// compareDuration applies an age comparison operator (one of > >= < <= =). +func compareDuration(v time.Duration, op string, want time.Duration) bool { + switch op { + case ">": + return v > want + case ">=": + return v >= want + case "<": + return v < want + case "<=": + return v <= want + case "=": + return v == want + } + return false +} + +// Explain evaluates every node of c against f with no short-circuit, +// building the full trace for krino explain. +func (c *Cond) Explain(f Facts) *Trace { + if c.root == nil { + return &Trace{Label: "no condition", Value: true} + } + return c.explain(c.root, f) +} + +// explain visits n and, for and/or/not, every child, in (cost-sorted) +// order, always - unlike eval, it never short-circuits. +func (c *Cond) explain(n *node, f Facts) *Trace { + switch n.kind { + case kAnd, kOr: + t := &Trace{Label: n.label} + val := n.kind == kAnd // identity: and starts true, or starts false + for _, ch := range n.children { + ct := c.explain(ch, f) + t.Children = append(t.Children, ct) + if n.kind == kAnd { + val = val && ct.Value + } else { + val = val || ct.Value + } + } + t.Value = val + return t + case kNot: + ct := c.explain(n.children[0], f) + return &Trace{Label: n.label, Value: !ct.Value, Children: []*Trace{ct}} + default: + ok, _, warn, _ := c.evalLeaf(n, f) + return &Trace{Label: n.label, Value: ok, Err: warn} + } +} + +// Format writes one line per node: "yes"/"no " padded to three, two +// spaces, two spaces of indent per depth, the label, and " (Err)" when +// Err is set. +func (t *Trace) Format(w io.Writer) { + t.format(w, 0) +} + +func (t *Trace) format(w io.Writer, depth int) { + word := "no" + if t.Value { + word = "yes" + } + fmt.Fprintf(w, "%-3s %s%s", word, strings.Repeat(" ", depth), t.Label) + if t.Err != "" { + fmt.Fprintf(w, " (%s)", t.Err) + } + fmt.Fprint(w, "\n") + for _, ch := range t.Children { + ch.format(w, depth+1) + } +} diff --git a/internal/cond/eval_test.go b/internal/cond/eval_test.go new file mode 100644 index 0000000..3607978 --- /dev/null +++ b/internal/cond/eval_test.go @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package cond + +import ( + "errors" + "math/rand" + "reflect" + "strings" + "testing" + "time" + + "krino/internal/norm" +) + +var now = time.Date(2026, 9, 11, 12, 0, 0, 0, time.UTC) + +type fake struct { + name, rel, raw string + rawErr error + size int64 + age time.Duration + matched bool + dupOrig string + dupOK bool + contentCalls int +} + +func (f *fake) Name() string { return f.name } +func (f *fake) Rel() string { + if f.rel != "" { + return f.rel + } + return f.name +} +func (f *fake) Size() int64 { return f.size } +func (f *fake) ModTime() time.Time { return now.Add(-f.age) } +func (f *fake) Now() time.Time { return now } +func (f *fake) Matched() bool { return f.matched } +func (f *fake) Content(ic, fold bool) (string, error) { + f.contentCalls++ + if f.rawErr != nil { + return "", f.rawErr + } + return norm.Text(f.raw, ic, fold), nil +} +func (f *fake) Duplicate(dirs []string) (string, bool, error) { return f.dupOrig, f.dupOK, nil } + +func eval(t *testing.T, src string, opt Options, f Facts) Result { + t.Helper() + c, errs := Compile("d.conf", nodes(t, src), opt) + if len(errs) > 0 { + t.Fatalf("%s: %v", src, errs) + } + return c.Eval(f) +} + +func TestTruth(t *testing.T) { + ic := Options{IgnoreCase: true} + pdf := &fake{name: "Scan001.PDF", raw: "Invoice from ACME LTD", size: 20 << 20, age: 40 * 24 * time.Hour} + tests := []struct { + src string + want bool + }{ + {``, true}, + {`(type pdf)`, true}, + {`(type document)`, true}, + {`(type jpg)`, false}, + {`(type pdf) (size > 10M)`, true}, + {`(type pdf) (size < 10M)`, false}, + {`(or (type jpg) (content "acme ltd"))`, true}, + {`(not (content "acme ltd"))`, false}, + {`(age > 30d) (age <= 41d)`, true}, + {`(name "^scan\d+")`, true}, + {`(name "(?-i)^scan")`, false}, + {`(matched)`, false}, + {`(and (type pdf) (or (name "^x") (not (name "^y"))))`, true}, + } + for _, tt := range tests { + if got := eval(t, tt.src, ic, pdf); got.Match != tt.want { + t.Errorf("%s = %v, want %v (%+v)", tt.src, got.Match, tt.want, got) + } + } +} + +func TestCaseAndFold(t *testing.T) { + f := &fake{name: "SPOLKA-umowa.pdf", raw: "Umowa: spółka z o.o."} + if !eval(t, `(name "spółka")`, Options{IgnoreCase: true, Fold: true}, f).Match { + t.Error("folded, case-ignoring name did not match") + } + if eval(t, `(name "spółka")`, Options{IgnoreCase: true, Fold: false}, f).Match { + t.Error("matched without folding") + } + if !eval(t, `(content "SPOLKA Z O.O.")`, Options{IgnoreCase: true, Fold: true}, f).Match { + t.Error("folded content did not match") + } + img := &fake{name: "IMG_0001.jpg"} + if eval(t, `(name "^img")`, Options{IgnoreCase: false}, img).Match { + t.Error("strict case matched") + } + if !eval(t, `(name "(?i)^img")`, Options{IgnoreCase: false}, img).Match { + t.Error("inline (?i) did not override strict case") + } +} + +func TestReasonsAndCaptures(t *testing.T) { + f := &fake{name: "Screenshot_20260911.png", raw: "acme ltd"} + r := eval(t, `(type image) (name "^Screenshot_(\d{4})(\d{2})") (not (name "^x(y)"))`, Options{IgnoreCase: true}, f) + if !r.Match { + t.Fatal("no match") + } + if want := []string{"Screenshot_202609", "2026", "09"}; !reflect.DeepEqual(r.Captures, want) { + t.Errorf("captures = %q, want %q", r.Captures, want) + } + if want := []string{`type png`, `name "^Screenshot_(\d{4})(\d{2})"`, `not name "^x(y)"`}; !reflect.DeepEqual(r.Reasons, want) { + t.Errorf("reasons = %q, want %q", r.Reasons, want) + } + r = eval(t, `(or (content "nope" "acme ltd") (type png))`, Options{IgnoreCase: true}, f) + if want := []string{`type png`}; !reflect.DeepEqual(r.Reasons, want) { + t.Errorf("or reasons = %q, want %q (cheapest true child)", r.Reasons, want) + } + r = eval(t, `(content "nope" "acme ltd")`, Options{IgnoreCase: true}, f) + if want := []string{`content "acme ltd"`}; !reflect.DeepEqual(r.Reasons, want) { + t.Errorf("content reasons = %q, want %q", r.Reasons, want) + } + if r := eval(t, ``, Options{}, f); !reflect.DeepEqual(r.Reasons, []string{"no condition"}) { + t.Errorf("empty reasons = %q", r.Reasons) + } + d := &fake{name: "report (1).pdf", dupOrig: "report.pdf", dupOK: true} + if r := eval(t, `(duplicate)`, Options{}, d); !r.Match || r.Reasons[0] != "duplicate of report.pdf" { + t.Errorf("duplicate = %+v", r) + } +} + +func TestCheapFirstAvoidsContent(t *testing.T) { + f := &fake{name: "a.txt", raw: "x"} + eval(t, `(and (content "x") (type pdf))`, Options{}, f) + if f.contentCalls != 0 { + t.Errorf("content read although type was false (%d calls)", f.contentCalls) + } + g := &fake{name: "a.pdf", raw: "x"} + eval(t, `(or (content "x") (type pdf))`, Options{}, g) + if g.contentCalls != 0 { + t.Errorf("content read although type was true (%d calls)", g.contentCalls) + } +} + +func TestContentErrorWarns(t *testing.T) { + f := &fake{name: "a.pdf", rawErr: errors.New("needs pdftotext, not installed")} + r := eval(t, `(or (content "acme") (content "other"))`, Options{}, f) + if r.Match { + t.Fatal("matched unreadable content") + } + if want := []string{"content unreadable: needs pdftotext, not installed"}; !reflect.DeepEqual(r.Warnings, want) { + t.Errorf("warnings = %q, want %q (de-duplicated)", r.Warnings, want) + } +} + +func TestExplainFormat(t *testing.T) { + f := &fake{name: "scan.pdf", rawErr: errors.New("needs pdftotext, not installed")} + c, _ := Compile("d.conf", nodes(t, `(type pdf) (or (content "acme ltd") (name "\bacme\b"))`), Options{IgnoreCase: true}) + var b strings.Builder + c.Explain(f).Format(&b) + want := "no and\n" + + "yes type pdf\n" + + "no or\n" + + "no name \"\\bacme\\b\"\n" + + "no content \"acme ltd\" (content unreadable: needs pdftotext, not installed)\n" + if b.String() != want { + t.Fatalf("got\n%s\nwant\n%s", b.String(), want) + } +} + +// TestReorderKeepsMeaning: random trees give the same answer with and without +// cost reordering. +func TestReorderKeepsMeaning(t *testing.T) { + leaves := []string{`(type pdf)`, `(type txt)`, `(name "^a")`, `(name "b$")`, `(size > 10)`, `(size < 5)`, + `(age > 1d)`, `(content "x")`, `(content "y")`, `(matched)`} + rng := rand.New(rand.NewSource(1)) + var gen func(depth int) string + gen = func(depth int) string { + if depth == 0 || rng.Intn(3) == 0 { + return leaves[rng.Intn(len(leaves))] + } + switch rng.Intn(3) { + case 0: + return "(not " + gen(depth-1) + ")" + case 1: + return "(and " + gen(depth-1) + " " + gen(depth-1) + ")" + default: + return "(or " + gen(depth-1) + " " + gen(depth-1) + " " + gen(depth-1) + ")" + } + } + names := []string{"a.pdf", "b.txt", "ab.pdf", "c.jpg"} + for i := 0; i < 500; i++ { + src := gen(4) + f := &fake{name: names[rng.Intn(len(names))], raw: []string{"x", "y", "xy", ""}[rng.Intn(4)], + size: int64(rng.Intn(20)), age: time.Duration(rng.Intn(72)) * time.Hour, matched: rng.Intn(2) == 0} + a, _ := compile("d.conf", nodes(t, src), Options{}, true) + b, _ := compile("d.conf", nodes(t, src), Options{}, false) + if ra, rb := a.Eval(f), b.Eval(f); ra.Match != rb.Match { + t.Fatalf("%s on %+v: reordered %v, original %v", src, f, ra.Match, rb.Match) + } + } +} + +// TestNegatedCombinatorReason: E4. Negating a combinator must read as +// something a person would write ("not (and ...)" / "not (or ...)"), not +// the bare "not and" / "not or"; a negated leaf keeps its own label +// unchanged ("not matched"). +func TestNegatedCombinatorReason(t *testing.T) { + f := &fake{name: "a.pdf"} + if r := eval(t, `(not (and (type pdf) (matched)))`, Options{}, f); !r.Match || r.Reasons[0] != "not (and ...)" { + t.Errorf("negated and = %+v, want reason %q", r, "not (and ...)") + } + if r := eval(t, `(not (or (matched) (name "^zzz")))`, Options{}, f); !r.Match || r.Reasons[0] != "not (or ...)" { + t.Errorf("negated or = %+v, want reason %q", r, "not (or ...)") + } + if r := eval(t, `(not (matched))`, Options{}, f); !r.Match || r.Reasons[0] != "not matched" { + t.Errorf("negated leaf = %+v, want its own label unchanged: %q", r, "not matched") + } +} diff --git a/internal/cond/types.go b/internal/cond/types.go new file mode 100644 index 0000000..81c2f45 --- /dev/null +++ b/internal/cond/types.go @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +// Package cond compiles the s-expression conditions of a rule's (when ...) +// into a tree that Task 8's evaluator walks against one file's facts. +package cond + +import ( + "regexp" + "time" + + "krino/internal/sexp" +) + +// Options carries a rule's resolved case and fold settings into compilation. +type Options struct { + IgnoreCase bool // the rule's resolved case setting is "ignore" + Fold bool // the rule's resolved fold setting +} + +// Cond is a compiled condition. A Cond compiled from no conditions (a rule +// without when) is always true. +type Cond struct { + root *node // nil: always true + opt Options // the case/fold settings conditions were compiled with; Task 8 needs them again at eval time + UsesContent bool // some content test exists + DupDirs [][]string // the raw directory arguments of each duplicate test, in order +} + +// kind is what a compiled node tests, or how it combines its children. +type kind int + +const ( + kAnd kind = iota + kOr + kNot + kType + kName + kPath + kContent + kSize + kAge + kDuplicate + kMatched +) + +// Costs, per the brief's cost order: cheapest first when sorting and/or +// children. not takes its child's cost; and/or take the sum of theirs. +const ( + costCheap = 1 // type, size, age, matched + costRegex = 2 // name, path + costDuplicate = 5 + costContent = 10 +) + +// pattern is one name/path regex, compiled and paired with the text it was +// written as (undecorated by (?i) or folding), for labels and reasons. +type pattern struct { + re *regexp.Regexp + src string +} + +// keyword is one content keyword, normalised for matching and paired with +// the text it was written as, for labels and reasons. +type keyword struct { + norm string + src string +} + +// node is one compiled condition: a leaf test, or an and/or/not combinator +// over other nodes. Task 8 evaluates this tree. +type node struct { + kind kind + pos sexp.Pos // the position of the node as written, for diagnostics + label string // the test as written, e.g. `content "acme ltd" "0000000000"` + cost int // this node's evaluation cost; and/or sort children by it + children []*node // and, or, not + + // type + suffixes []string // leading-dot, lower-case, e.g. ".pdf" + + // name, path + patterns []pattern + + // content + keywords []keyword + + // size, age (kind tells which is populated) + op string + sizeVal int64 + ageVal time.Duration + + // duplicate + dirs []string +} + +// groups maps a (type ...) group name to the extensions it expands to, +// spec Appendix A. +var groups = map[string][]string{ + "image": {"jpg", "jpeg", "png", "gif", "webp", "bmp", "tif", "tiff", "heic", "heif", "avif", "svg", "ico", "raw", "cr2", "nef", "arw", "dng"}, + "video": {"mp4", "mkv", "webm", "mov", "avi", "m4v", "mpg", "mpeg", "wmv", "flv", "3gp"}, + "audio": {"mp3", "flac", "ogg", "opus", "m4a", "aac", "wav", "wma", "aiff"}, + "archive": {"zip", "tar", "gz", "tgz", "bz2", "tbz2", "xz", "txz", "zst", "7z", "rar", "lz", "lzma", "cpio"}, + "document": {"pdf", "doc", "docx", "odt", "rtf", "txt", "md", "tex"}, + "spreadsheet": {"xls", "xlsx", "ods", "csv", "tsv"}, + "presentation": {"ppt", "pptx", "odp"}, + "ebook": {"epub", "mobi", "azw", "azw3", "fb2", "djvu"}, + "code": {"go", "c", "h", "cpp", "hpp", "py", "sh", "js", "ts", "rs", "java", "rb", "pl", "lua", "html", "css", "json", "yaml", "yml", "toml", "xml", "sql"}, + "text": {"txt", "md", "log", "csv", "tsv", "json", "yaml", "yml", "toml", "xml", "ini", "conf"}, + "package": {"deb", "rpm", "apk", "appimage", "exe", "msi", "flatpak", "snap"}, + "font": {"ttf", "otf", "woff", "woff2"}, +} diff --git a/internal/dup/dup.go b/internal/dup/dup.go new file mode 100644 index 0000000..502ef31 --- /dev/null +++ b/internal/dup/dup.go @@ -0,0 +1,398 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +// Package dup answers "is this file a duplicate, and of which original?" +// cheaply: candidates are grouped by size, and only hashed when sizes +// collide — a partial hash first, a full hash only on a partial collision. +package dup + +import ( + "crypto/sha256" + "encoding/binary" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "sync" + "time" + + "krino/internal/scan" +) + +// partialChunk is the size of the head and tail read for the partial hash. +const partialChunk = 64 << 10 // 64 KiB + +// candidate is one file the index knows about: a scanned file, or a file +// found under one of the extra directories. +type candidate struct { + path string + size int64 + modTime time.Time + name string + extra bool // found under one of the extra directories, not scanned +} + +// Index finds files with identical content among the scanned files and, +// optionally, every regular file under some extra directories. +type Index struct { + candidates []candidate + bySize map[int64][]int // size -> indexes into candidates + scanned map[string]int // scanned file path -> index into candidates + + mu sync.Mutex + partial map[string][sha256.Size]byte // memoised partial hash, by path + full map[string][sha256.Size]byte // memoised full hash, by path + + candErrs []CandidateError + candErrSeen map[string]bool // path already recorded in candErrs +} + +// CandidateError is one candidate (never the file Lookup was asked about) +// that could not be hashed, so it was skipped rather than comparing it. +type CandidateError struct { + Path string + Err error +} + +func (e CandidateError) Error() string { return e.Path + ": " + e.Err.Error() } + +// NewIndex stats the extra directories (recursively, symlinks skipped). A +// missing or unreadable extra directory is reported in the error list and +// otherwise ignored; an unreadable subdirectory found while walking an +// otherwise-readable extra directory adds its own error but does not stop +// the rest of that directory from being indexed. An extra directory that is +// itself a symlink (A4) is not followed either — filepath.WalkDir Lstats +// its root, so left unchecked it would be indexed as silently empty — and +// is reported in the error list instead. Nothing is hashed yet. +func NewIndex(files []scan.File, extra []string) (*Index, []error) { + x := &Index{ + bySize: make(map[int64][]int), + scanned: make(map[string]int, len(files)), + partial: make(map[string][sha256.Size]byte), + full: make(map[string][sha256.Size]byte), + } + for _, f := range files { + x.scanned[f.Path] = x.add(candidate{path: f.Path, size: f.Size, modTime: f.ModTime, name: f.Name}) + } + + var errs []error + for _, dir := range extra { + errs = append(errs, x.addExtraDir(dir)...) + } + return x, errs +} + +// add appends c to the candidate list and its size group, and returns its +// index. +func (x *Index) add(c candidate) int { + idx := len(x.candidates) + x.candidates = append(x.candidates, c) + x.bySize[c.size] = append(x.bySize[c.size], idx) + return idx +} + +// addExtraDir walks dir, adding every regular file found (symlinks, both to +// files and to directories, are skipped: filepath.WalkDir never follows +// them, so it is enough not to add or descend into one). An error on dir +// itself (missing, or unreadable) aborts the walk and is the sole error +// returned; an unreadable subdirectory deeper in the tree adds one error +// naming it and the walk continues, so files elsewhere in dir are still +// indexed. A file whose own Info() fails (A3) is dropped from the index the +// same way: silently if it has simply vanished (fs.ErrNotExist), otherwise +// with its own error added to errs. +func (x *Index) addExtraDir(dir string) []error { + var errs []error + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + if path == dir { + return err + } + errs = append(errs, fmt.Errorf("%s: %w", path, err)) + return nil + } + if path == dir && d.Type()&fs.ModeSymlink != 0 { + // filepath.WalkDir Lstats its root: a symlinked extra directory + // (A4) would otherwise be silently treated as empty rather than + // followed, with no sign anything was wrong. + errs = append(errs, fmt.Errorf("%s is a symlink; not followed", path)) + return nil + } + if d.Type()&fs.ModeSymlink != 0 || d.IsDir() || !d.Type().IsRegular() { + return nil + } + x.addEntry(path, d, &errs) + return nil + }) + if err != nil { + errs = append(errs, fmt.Errorf("%s: %w", dir, err)) + } + return errs +} + +// addEntry indexes one regular-file entry found while walking an extra +// directory: A3, a file whose own Info() fails is dropped from the index — +// silently if it has simply vanished (fs.ErrNotExist), otherwise with its +// own error appended to errs. Factored out of addExtraDir's WalkDir +// callback so a test can drive it directly with a fabricated fs.DirEntry, +// since filepath.WalkDir gives no way to inject a canned Info() failure on +// a real walk. +func (x *Index) addEntry(path string, d fs.DirEntry, errs *[]error) { + info, err := d.Info() + if err != nil { + if !errors.Is(err, fs.ErrNotExist) { + *errs = append(*errs, fmt.Errorf("%s: %w", path, err)) + } + return + } + x.add(candidate{path: path, size: info.Size(), modTime: info.ModTime(), name: d.Name(), extra: true}) +} + +// Lookup reports whether path (one of the scanned files) duplicates another +// file, and which file is the original. Safe for concurrent use. +func (x *Index) Lookup(path string) (original string, dup bool, err error) { + idx, ok := x.scanned[path] + if !ok { + return "", false, fmt.Errorf("dup: %s was not scanned", path) + } + c := x.candidates[idx] + if c.size == 0 { + // Empty files are never duplicates, and are never read. + return path, false, nil + } + group := x.bySize[c.size] + if len(group) < 2 { + // Alone in its size group: not a duplicate, never read. + return path, false, nil + } + + identical, err := x.identicalTo(idx, group) + if err != nil { + return "", false, err + } + if len(identical) < 2 { + return path, false, nil + } + orig := x.candidates[x.original(identical)].path + return orig, orig != path, nil +} + +// identicalTo returns the indexes in group (which all share idx's size, +// idx included) whose content matches candidates[idx]: same partial hash, +// then, only for those that collide, the same full hash. idx is the file +// Lookup was asked about; a failure hashing it propagates, since Lookup can +// answer nothing without it. A failure hashing any other candidate in group +// only removes that candidate from consideration: a vanished candidate +// (errors.Is fs.ErrNotExist) is dropped silently, any other failure is +// recorded on the Index (see recordCandidateError) so the caller can warn +// about it once matching is done. +func (x *Index) identicalTo(idx int, group []int) ([]int, error) { + idxPartial, err := x.partialHash(x.candidates[idx].path) + if err != nil { + return nil, err + } + + same := []int{idx} + var idxFull [sha256.Size]byte + haveIdxFull := false + for _, j := range group { + if j == idx { + continue + } + jPartial, err := x.partialHash(x.candidates[j].path) + if err != nil { + x.recordCandidateError(x.candidates[j].path, err) + continue + } + if jPartial != idxPartial { + continue + } + if !haveIdxFull { + idxFull, err = x.fullHash(x.candidates[idx].path) + if err != nil { + return nil, err + } + haveIdxFull = true + } + jFull, err := x.fullHash(x.candidates[j].path) + if err != nil { + x.recordCandidateError(x.candidates[j].path, err) + continue + } + if jFull == idxFull { + same = append(same, j) + } + } + return same, nil +} + +// recordCandidateError records that path (never the file Lookup was asked +// about) could not be hashed and so was skipped, unless it simply vanished +// (fs.ErrNotExist), which is not worth reporting, or was already recorded. +// Safe for concurrent use. +func (x *Index) recordCandidateError(path string, err error) { + if errors.Is(err, fs.ErrNotExist) { + return + } + x.mu.Lock() + defer x.mu.Unlock() + if x.candErrSeen == nil { + x.candErrSeen = make(map[string]bool) + } + if x.candErrSeen[path] { + return + } + x.candErrSeen[path] = true + x.candErrs = append(x.candErrs, CandidateError{Path: path, Err: err}) +} + +// Errors returns every candidate-hashing failure recorded so far, +// deduplicated by path, in first-recorded order. Safe for concurrent use. +func (x *Index) Errors() []CandidateError { + x.mu.Lock() + defer x.mu.Unlock() + return append([]CandidateError(nil), x.candErrs...) +} + +// original picks, among a set of identical candidates, the index that is +// the original. Spec §5.5: one flat comparison, in order — a file under an +// extra directory beats one that is not; then the oldest by ModTime; then +// the shortest base name; then the base name that sorts first; then (the +// final, always-deterministic tie-break) the full path that sorts first. +// This one chain applies to every pair alike; extra-vs-extra candidates +// are not a special case broken by path alone. +func (x *Index) original(idxs []int) int { + best := idxs[0] + for _, j := range idxs[1:] { + if x.preferred(j, best) { + best = j + } + } + return best +} + +// preferred reports whether candidate a should be chosen as the original +// over candidate b. +func (x *Index) preferred(a, b int) bool { + ca, cb := x.candidates[a], x.candidates[b] + if ca.extra != cb.extra { + return ca.extra + } + if !ca.modTime.Equal(cb.modTime) { + return ca.modTime.Before(cb.modTime) + } + if len(ca.name) != len(cb.name) { + return len(ca.name) < len(cb.name) + } + if ca.name != cb.name { + return ca.name < cb.name + } + return ca.path < cb.path +} + +// partialHash returns the memoised partial hash for path, computing and +// storing it on first use. The hash is computed outside the lock; only the +// memo access is guarded. +func (x *Index) partialHash(path string) ([sha256.Size]byte, error) { + x.mu.Lock() + h, ok := x.partial[path] + x.mu.Unlock() + if ok { + return h, nil + } + h, err := computePartialHash(path) + if err != nil { + return h, err + } + x.mu.Lock() + x.partial[path] = h + x.mu.Unlock() + return h, nil +} + +// fullHash returns the memoised full-file hash for path, computing and +// storing it on first use. +func (x *Index) fullHash(path string) ([sha256.Size]byte, error) { + x.mu.Lock() + h, ok := x.full[path] + x.mu.Unlock() + if ok { + return h, nil + } + h, err := computeFullHash(path) + if err != nil { + return h, err + } + x.mu.Lock() + x.full[path] = h + x.mu.Unlock() + return h, nil +} + +// computePartialHash hashes the file's size, its first 64 KiB and its last +// 64 KiB (the two overlap, or repeat the whole file, when it is smaller +// than 64 KiB). +func computePartialHash(path string) ([sha256.Size]byte, error) { + f, err := os.Open(path) + if err != nil { + return [sha256.Size]byte{}, err + } + defer f.Close() + info, err := f.Stat() + if err != nil { + return [sha256.Size]byte{}, err + } + size := info.Size() + + h := sha256.New() + var sizeBuf [8]byte + binary.BigEndian.PutUint64(sizeBuf[:], uint64(size)) + h.Write(sizeBuf[:]) + + head, err := readAt(f, 0) + if err != nil { + return [sha256.Size]byte{}, err + } + h.Write(head) + + tailOff := size - partialChunk + if tailOff < 0 { + tailOff = 0 + } + tail, err := readAt(f, tailOff) + if err != nil { + return [sha256.Size]byte{}, err + } + h.Write(tail) + + var out [sha256.Size]byte + copy(out[:], h.Sum(nil)) + return out, nil +} + +// readAt reads up to partialChunk bytes starting at off, without disturbing +// f's current offset. +func readAt(f *os.File, off int64) ([]byte, error) { + buf := make([]byte, partialChunk) + n, err := f.ReadAt(buf, off) + if err != nil && err != io.EOF { + return nil, err + } + return buf[:n], nil +} + +// computeFullHash hashes the whole file. +func computeFullHash(path string) ([sha256.Size]byte, error) { + f, err := os.Open(path) + if err != nil { + return [sha256.Size]byte{}, err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return [sha256.Size]byte{}, err + } + var out [sha256.Size]byte + copy(out[:], h.Sum(nil)) + return out, nil +} diff --git a/internal/dup/dup_test.go b/internal/dup/dup_test.go new file mode 100644 index 0000000..fb4e64d --- /dev/null +++ b/internal/dup/dup_test.go @@ -0,0 +1,331 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package dup + +import ( + "bytes" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "krino/internal/scan" +) + +var base = time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) + +// put writes content at dir/name with an mtime `age` hours after base. +func put(t *testing.T, dir, name string, content []byte, hours int) scan.File { + t.Helper() + p := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, content, 0o644); err != nil { + t.Fatal(err) + } + mt := base.Add(time.Duration(hours) * time.Hour) + if err := os.Chtimes(p, mt, mt); err != nil { + t.Fatal(err) + } + return scan.File{Path: p, Rel: name, Name: filepath.Base(name), Size: int64(len(content)), ModTime: mt} +} + +func lookup(t *testing.T, x *Index, f scan.File) (string, bool) { + t.Helper() + orig, dup, err := x.Lookup(f.Path) + if err != nil { + t.Fatal(err) + } + return orig, dup +} + +func TestDuplicatesInScan(t *testing.T) { + d := t.TempDir() + a := put(t, d, "report.pdf", []byte("same content"), 1) + b := put(t, d, "report (1).pdf", []byte("same content"), 5) + c := put(t, d, "other.pdf", []byte("diff content"), 0) // same size, different bytes + e1 := put(t, d, "empty1", nil, 0) + e2 := put(t, d, "empty2", nil, 1) + x, errs := NewIndex([]scan.File{a, b, c, e1, e2}, nil) + if len(errs) > 0 { + t.Fatal(errs) + } + if orig, dup := lookup(t, x, b); !dup || orig != a.Path { + t.Errorf("copy: dup=%v orig=%s, want dup of %s", dup, orig, a.Path) + } + if _, dup := lookup(t, x, a); dup { + t.Error("the original reported as a duplicate") + } + if _, dup := lookup(t, x, c); dup { + t.Error("same size, different content reported as a duplicate") + } + if _, dup := lookup(t, x, e2); dup { + t.Error("empty files reported as duplicates") + } +} + +func TestExtraDirHoldsTheOriginal(t *testing.T) { + scanned, filed := t.TempDir(), t.TempDir() + dl := put(t, scanned, "invoice.pdf", []byte("invoice 42"), 0) // older than the filed copy + put(t, filed, "2026/invoice-42.pdf", []byte("invoice 42"), 9) + x, errs := NewIndex([]scan.File{dl}, []string{filed, filepath.Join(filed, "missing")}) + if len(errs) != 1 { + t.Errorf("want one error for the missing extra dir, got %v", errs) + } + if orig, dup := lookup(t, x, dl); !dup || orig != filepath.Join(filed, "2026/invoice-42.pdf") { + t.Errorf("dup=%v orig=%s, want the filed copy as original", dup, orig) + } +} + +func TestTieBreaks(t *testing.T) { + d := t.TempDir() + long := put(t, d, "longer-name.txt", []byte("x"), 0) + short := put(t, d, "b.txt", []byte("x"), 0) + same := put(t, d, "a.txt", []byte("x"), 0) + x, _ := NewIndex([]scan.File{long, short, same}, nil) + for _, f := range []scan.File{long, short} { + if orig, dup := lookup(t, x, f); !dup || orig != same.Path { + t.Errorf("%s: dup=%v orig=%s, want %s (shortest name, then path order)", f.Name, dup, orig, same.Path) + } + } +} + +func TestPartialHashCollisionResolvedByFullHash(t *testing.T) { + d := t.TempDir() + head, tail := bytes.Repeat([]byte("h"), 70<<10), bytes.Repeat([]byte("t"), 70<<10) + one := append(append(append([]byte{}, head...), []byte("MIDDLE-ONE")...), tail...) + two := append(append(append([]byte{}, head...), []byte("MIDDLE-TWO")...), tail...) + a := put(t, d, "a.bin", one, 0) + b := put(t, d, "b.bin", two, 1) + x, _ := NewIndex([]scan.File{a, b}, nil) + if _, dup := lookup(t, x, b); dup { + t.Error("files differing only in the middle reported as duplicates") + } +} + +func TestLookupUnknownPath(t *testing.T) { + x, _ := NewIndex(nil, nil) + if _, _, err := x.Lookup("/nowhere"); err == nil { + t.Fatal("no error for a path that was not scanned") + } +} + +// TestTieBreakNameBeforePath: same-length names, same mtime, in directories +// that sort in the opposite order from the names — the base name decides, +// not the full path (spec §5.5's flat chain, not a path-only fallback). +func TestTieBreakNameBeforePath(t *testing.T) { + d := t.TempDir() + catInZzz := put(t, d, "zzz/cat.txt", []byte("x"), 0) + dogInAaa := put(t, d, "aaa/dog.txt", []byte("x"), 0) + x, _ := NewIndex([]scan.File{catInZzz, dogInAaa}, nil) + for _, f := range []scan.File{catInZzz, dogInAaa} { + if orig, dup := lookup(t, x, f); orig != catInZzz.Path || dup != (f.Path != catInZzz.Path) { + t.Errorf("%s: orig=%s dup=%v, want %s (name sorts before path)", f.Rel, orig, dup, catInZzz.Path) + } + } +} + +// TestTieBreakExtraVsExtra: two extra directories hold identical copies; +// the one under the lexically later directory is older and must still win +// on ModTime — extra-vs-extra ties are not resolved by path alone. +func TestTieBreakExtraVsExtra(t *testing.T) { + common := t.TempDir() + aaa, zzz := filepath.Join(common, "aaa"), filepath.Join(common, "zzz") + newer := put(t, aaa, "copy.txt", []byte("invoice 42"), 5) // lexically first, newer + older := put(t, zzz, "copy.txt", []byte("invoice 42"), 0) // lexically last, older + scanned := t.TempDir() + dl := put(t, scanned, "download.txt", []byte("invoice 42"), 3) + x, errs := NewIndex([]scan.File{dl}, []string{aaa, zzz}) + if len(errs) != 0 { + t.Fatal(errs) + } + if orig, dup := lookup(t, x, dl); !dup || orig != older.Path { + t.Errorf("dup=%v orig=%s, want %s (older extra copy, despite sorting after %s)", dup, orig, older.Path, newer.Path) + } +} + +// TestUnreadableSubdirReported: an unreadable subdirectory under an extra +// directory is reported as its own error, and the rest of that extra +// directory is still indexed. +func TestUnreadableSubdirReported(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("permissions are not enforced running as root") + } + filed := t.TempDir() + blocked := filepath.Join(filed, "blocked") + if err := os.MkdirAll(blocked, 0o755); err != nil { + t.Fatal(err) + } + put(t, blocked, "secret.pdf", []byte("secret 42"), 0) + put(t, filed, "visible.pdf", []byte("visible content"), 0) + if err := os.Chmod(blocked, 0o000); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.Chmod(blocked, 0o755); err != nil { + t.Fatal(err) + } + }) + + scanned := t.TempDir() + dl := put(t, scanned, "visible.pdf", []byte("visible content"), 1) + + x, errs := NewIndex([]scan.File{dl}, []string{filed}) + if len(errs) != 1 || !strings.Contains(errs[0].Error(), blocked) { + t.Fatalf("want one error naming %s, got %v", blocked, errs) + } + if orig, dup := lookup(t, x, dl); !dup || orig != filepath.Join(filed, "visible.pdf") { + t.Errorf("dup=%v orig=%s, want the filed copy (visible.pdf still indexed despite the unreadable sibling)", dup, orig) + } +} + +// TestUnreadableCandidateSkipped: three files share a size; one of them +// (not the subject of either Lookup call) is unreadable. A1: the other two +// are still reported as a duplicate pair, and the unreadable one is +// recorded exactly once as a candidate error, not returned as a Lookup +// error. +func TestUnreadableCandidateSkipped(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("permissions are not enforced running as root") + } + d := t.TempDir() + a := put(t, d, "a.bin", []byte("same content"), 0) + b := put(t, d, "b.bin", []byte("same content"), 1) + c := put(t, d, "c.bin", []byte("diff content"), 2) // same size, different bytes + if err := os.Chmod(c.Path, 0o000); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.Chmod(c.Path, 0o644) }) + + x, errs := NewIndex([]scan.File{a, b, c}, nil) + if len(errs) != 0 { + t.Fatalf("NewIndex errors: %v", errs) + } + if orig, dup := lookup(t, x, b); !dup || orig != a.Path { + t.Errorf("a/b duplicate pair broken by unreadable sibling: dup=%v orig=%s", dup, orig) + } + if orig, dup := lookup(t, x, a); dup { + t.Errorf("a reported as a duplicate: orig=%s", orig) + } + cerrs := x.Errors() + if len(cerrs) != 1 { + t.Fatalf("got %d candidate errors, want 1: %v", len(cerrs), cerrs) + } + if cerrs[0].Path != c.Path { + t.Errorf("candidate error names %q, want %q", cerrs[0].Path, c.Path) + } +} + +// TestVanishedCandidateSkippedSilently: a candidate that vanishes between +// being indexed and being hashed is dropped with no error recorded at all. +func TestVanishedCandidateSkippedSilently(t *testing.T) { + d := t.TempDir() + a := put(t, d, "a.bin", []byte("same content"), 0) + b := put(t, d, "b.bin", []byte("same content"), 1) + c := put(t, d, "c.bin", []byte("diff content"), 2) + x, errs := NewIndex([]scan.File{a, b, c}, nil) + if len(errs) != 0 { + t.Fatalf("NewIndex errors: %v", errs) + } + if err := os.Remove(c.Path); err != nil { + t.Fatal(err) + } + if orig, dup := lookup(t, x, b); !dup || orig != a.Path { + t.Errorf("a/b duplicate pair broken by vanished sibling: dup=%v orig=%s", dup, orig) + } + if got := x.Errors(); len(got) != 0 { + t.Errorf("vanished candidate recorded as an error: %v", got) + } +} + +// TestLookupFailsWhenSubjectUnreadable: Lookup still fails outright when +// the file it was asked about (not some other candidate) cannot be read. +func TestLookupFailsWhenSubjectUnreadable(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("permissions are not enforced running as root") + } + d := t.TempDir() + a := put(t, d, "a.bin", []byte("same content"), 0) + b := put(t, d, "b.bin", []byte("same content"), 1) + if err := os.Chmod(a.Path, 0o000); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.Chmod(a.Path, 0o644) }) + x, _ := NewIndex([]scan.File{a, b}, nil) + if _, _, err := x.Lookup(a.Path); err == nil { + t.Fatal("no error looking up an unreadable subject") + } +} + +// fakeDirEntry is an fs.DirEntry whose Info() returns a canned result, for +// exercising addEntry's Info()-failure handling directly (A3) — a real +// filepath.WalkDir gives no hook to inject a stat failure deterministically +// and without root. +type fakeDirEntry struct { + name string + info fs.FileInfo + infoErr error +} + +func (f fakeDirEntry) Name() string { return f.name } +func (f fakeDirEntry) IsDir() bool { return false } +func (f fakeDirEntry) Type() fs.FileMode { return 0 } +func (f fakeDirEntry) Info() (fs.FileInfo, error) { return f.info, f.infoErr } + +// TestAddEntryInfoFailure: A3. A vanished entry's Info() failure +// (fs.ErrNotExist) is dropped with no error recorded; any other Info() +// failure is dropped too, but recorded in errs, naming the entry. +func TestAddEntryInfoFailure(t *testing.T) { + x := &Index{bySize: make(map[int64][]int), scanned: make(map[string]int)} + var errs []error + + x.addEntry("/extra/vanished.txt", fakeDirEntry{ + name: "vanished.txt", infoErr: fmt.Errorf("stat vanished.txt: %w", fs.ErrNotExist), + }, &errs) + if len(errs) != 0 { + t.Fatalf("vanished entry recorded an error: %v", errs) + } + if len(x.candidates) != 0 { + t.Fatalf("vanished entry was indexed: %v", x.candidates) + } + + x.addEntry("/extra/denied.txt", fakeDirEntry{ + name: "denied.txt", infoErr: errors.New("permission denied"), + }, &errs) + if len(errs) != 1 || !strings.Contains(errs[0].Error(), "/extra/denied.txt") { + t.Fatalf("want one error naming /extra/denied.txt, got %v", errs) + } + if len(x.candidates) != 0 { + t.Fatalf("denied entry was indexed: %v", x.candidates) + } +} + +// TestExtraDirSymlinkNotFollowed: A4. An extra directory that is itself a +// symlink to a real directory is not silently treated as empty: +// filepath.WalkDir Lstats its root, so without a check for this the walk +// would report no error and index nothing, misleading the user into +// thinking an archive was consulted when it never was. +func TestExtraDirSymlinkNotFollowed(t *testing.T) { + real := t.TempDir() + put(t, real, "invoice.pdf", []byte("invoice 42"), 0) + link := filepath.Join(t.TempDir(), "link") + if err := os.Symlink(real, link); err != nil { + t.Fatal(err) + } + + scanned := t.TempDir() + dl := put(t, scanned, "download.pdf", []byte("invoice 42"), 1) + + x, errs := NewIndex([]scan.File{dl}, []string{link}) + if len(errs) != 1 || !strings.Contains(errs[0].Error(), link) || !strings.Contains(errs[0].Error(), "symlink") { + t.Fatalf("want one error naming %s as a symlink, got %v", link, errs) + } + if orig, dup := lookup(t, x, dl); dup { + t.Errorf("symlinked extra dir was indexed despite the error: dup=%v orig=%s", dup, orig) + } +} diff --git a/internal/engine/engine.go b/internal/engine/engine.go new file mode 100644 index 0000000..eec9a60 --- /dev/null +++ b/internal/engine/engine.go @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +// Package engine is what every krino front end calls: it loads and compiles +// the configuration, matches files against rules, and (from plan 3) plans +// and applies actions. It returns data; front ends only render it. +package engine + +import ( + "os" + "time" + + "krino/internal/cond" + "krino/internal/config" + "krino/internal/extract" + "krino/internal/ignore" +) + +// Engine holds a loaded, compiled configuration: everything a front end +// needs to check, match and (from plan 3) act. +type Engine struct { + Config *config.Config + Dirs []*Dir + Extract *extract.Extractor + Now func() time.Time // time.Now; tests replace it + MainFile string +} + +// Dir is one configured directory, with its ignore matcher and rules +// compiled. +type Dir struct { + Name string + Root string // absolute + Conf *config.Dir + Settings config.Resolved // built-in, then defaults, then the directory + Ignore *ignore.Matcher + Rules []*Rule + + // ContentVariants is the distinct (IgnoreCase, Fold) pairs any of + // Rules' content tests evaluate under, in first-seen order. B2: when + // this holds exactly one variant, facts.Content releases a file's raw + // extracted text once that variant's normalised copy exists, since no + // other variant will ever be asked for; with more than one, both must + // stay memoised, as before. + ContentVariants []cond.Options +} + +// Rule is one directory's rule, with its condition compiled. +type Rule struct { + Name string + Conf *config.Rule + Settings config.Resolved // the rule's own settings over its directory's + Cond *cond.Cond +} + +// Load reads and compiles everything. Any problem anywhere returns a nil +// Engine and every diagnostic: krino never acts on a configuration it only +// partly understood. Duplicate names are ignored after their first use. +func Load(mainFile string, names ...string) (*Engine, []*config.Diag) { + cfg, errs := config.Load(mainFile, dedupeNames(names)...) + if cfg == nil { + return nil, errs + } + + var dirs []*Dir + for _, d := range cfg.Dirs { + dir := &Dir{ + Name: d.Name, + Root: d.Path, + Conf: d, + Settings: cfg.Resolved(d), + } + if m, err := ignore.New(d.Ignore); err != nil { + errs = append(errs, &config.Diag{File: d.File, Msg: err.Error()}) + } else { + dir.Ignore = m + } + for _, r := range d.Rules { + rs := r.Settings.Over(dir.Settings) + c, cerrs := cond.Compile(d.File, r.When, cond.Options{ + IgnoreCase: rs.Case == config.CaseIgnore, + Fold: rs.Fold, + }) + if len(cerrs) > 0 { + errs = append(errs, cerrs...) + continue + } + dir.Rules = append(dir.Rules, &Rule{Name: r.Name, Conf: r, Settings: rs, Cond: c}) + } + dir.ContentVariants = contentVariants(dir.Rules) + dirs = append(dirs, dir) + } + + if len(errs) > 0 { + return nil, errs + } + return &Engine{ + Config: cfg, + Dirs: dirs, + Extract: extract.New(), + Now: time.Now, + MainFile: mainFile, + }, nil +} + +// dedupeNames returns names with every repeat after its first occurrence +// removed, order preserved. +func dedupeNames(names []string) []string { + var out []string + seen := map[string]bool{} + for _, n := range names { + if seen[n] { + continue + } + seen[n] = true + out = append(out, n) + } + return out +} + +// contentVariants returns the distinct (IgnoreCase, Fold) pairs any of +// rules' content tests evaluate under, in first-seen order — B2's per-Dir +// ContentVariants. A rule whose condition has no content test at all +// (Cond.UsesContent false) never calls facts.Content, so its resolved +// case/fold settings contribute no variant here. +func contentVariants(rules []*Rule) []cond.Options { + var out []cond.Options + seen := map[cond.Options]bool{} + for _, r := range rules { + if !r.Cond.UsesContent { + continue + } + opt := cond.Options{IgnoreCase: r.Settings.Case == config.CaseIgnore, Fold: r.Settings.Fold} + if seen[opt] { + continue + } + seen[opt] = true + out = append(out, opt) + } + return out +} + +// Report is what Check reports: the files involved and each directory's +// state. +type Report struct { + MainFile string + LogFile string + Dirs []DirReport + Tools []extract.Tool +} + +// DirReport is one directory's state in a Report. +type DirReport struct { + Dir *Dir + Missing bool // the root is not a directory right now +} + +// Check reports the engine's configuration files, each directory's current +// state and the external tools found for content extraction. +func (e *Engine) Check() Report { + r := Report{ + MainFile: e.MainFile, + LogFile: e.Config.LogFile(), + Tools: e.Extract.Tools(), + } + for _, d := range e.Dirs { + fi, err := os.Stat(d.Root) + r.Dirs = append(r.Dirs, DirReport{Dir: d, Missing: err != nil || !fi.IsDir()}) + } + return r +} diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go new file mode 100644 index 0000000..5c0536f --- /dev/null +++ b/internal/engine/engine_test.go @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package engine + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "krino/internal/cond" +) + +// sandbox gives a test its own HOME with no XDG overrides and returns it. +func sandbox(t *testing.T) string { + t.Helper() + h := t.TempDir() + t.Setenv("HOME", h) + for _, v := range []string{"XDG_CONFIG_HOME", "XDG_STATE_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME"} { + t.Setenv(v, "") + } + return h +} + +// writeConfig writes krino.conf and dirs/<name>.conf files under home/.config/krino. +func writeConfig(t *testing.T, home, main string, dirs map[string]string) string { + t.Helper() + cdir := filepath.Join(home, ".config", "krino") + if err := os.MkdirAll(filepath.Join(cdir, "dirs"), 0o755); err != nil { + t.Fatal(err) + } + mainFile := filepath.Join(cdir, "krino.conf") + if err := os.WriteFile(mainFile, []byte(main), 0o644); err != nil { + t.Fatal(err) + } + for n, body := range dirs { + if err := os.WriteFile(filepath.Join(cdir, "dirs", n+".conf"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + return mainFile +} + +// fakeFacts is a minimal cond.Facts for checking compiled rules. +type fakeFacts struct{ name string } + +func (f fakeFacts) Name() string { return f.name } +func (f fakeFacts) Rel() string { return f.name } +func (f fakeFacts) Size() int64 { return 1 } +func (f fakeFacts) ModTime() time.Time { return time.Time{} } +func (f fakeFacts) Now() time.Time { return time.Time{} } +func (f fakeFacts) Content(bool, bool) (string, error) { return "", nil } +func (f fakeFacts) Duplicate([]string) (string, bool, error) { return "", false, nil } +func (f fakeFacts) Matched() bool { return false } + +var _ cond.Facts = fakeFacts{} + +func TestLoadCompilesWithRuleSettings(t *testing.T) { + h := sandbox(t) + os.Mkdir(filepath.Join(h, "dl"), 0o755) + main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": ` +(path "~/dl") +(case strict) +(ignore "*.part") +(rule "strict" (when (name "^img")) (stop)) +(rule "loose" (case ignore) (when (name "^img")) (stop)) +`}) + e, errs := Load(main, "dl", "dl") + if len(errs) > 0 { + t.Fatal(errs) + } + if len(e.Dirs) != 1 { + t.Fatalf("got %d dirs, want 1 (duplicate name ignored)", len(e.Dirs)) + } + d := e.Dirs[0] + if d.Root != filepath.Join(h, "dl") || d.Ignore == nil || !d.Ignore.Match("x.part", false) { + t.Fatalf("dir = %+v", d) + } + img := fakeFacts{name: "IMG_1.jpg"} + if d.Rules[0].Cond.Eval(img).Match { + t.Error("rule under (case strict) matched IMG against ^img") + } + if !d.Rules[1].Cond.Eval(img).Match { + t.Error("rule-level (case ignore) did not apply at compile time") + } +} + +func TestLoadReportsEveryProblem(t *testing.T) { + h := sandbox(t) + main := writeConfig(t, h, `(include "a" "b")`, map[string]string{ + "a": `(path "/tmp") (rule "x" (when (type "pdf")) (stop))`, + "b": `(path "/tmp") (ignore "[abc") (rule "y" (when (size 1)) (stop))`, + }) + e, errs := Load(main) + if e != nil { + t.Fatal("engine returned despite errors") + } + joined := "" + for _, d := range errs { + joined += d.Error() + "\n" + } + for _, want := range []string{ + "a.conf:1:37: type names are bare words: write (type pdf)", + `b.conf: bad ignore pattern "[abc": unterminated [`, + "b.conf:1:47: size takes an operator and a size, like (size > 10M)", + } { + if !strings.Contains(joined, want) { + t.Errorf("missing %q in:\n%s", want, joined) + } + } +} + +func TestCheck(t *testing.T) { + h := sandbox(t) + bin := t.TempDir() + os.WriteFile(filepath.Join(bin, "pdftotext"), []byte("#!/bin/sh\n"), 0o755) + t.Setenv("PATH", bin) + os.Mkdir(filepath.Join(h, "here"), 0o755) + main := writeConfig(t, h, `(include "here" "gone")`, map[string]string{ + "here": `(path "~/here") (rule "r" (stop))`, + "gone": `(path "~/gone") (rule "r" (stop))`, + }) + e, errs := Load(main) + if len(errs) > 0 { + t.Fatal(errs) + } + r := e.Check() + if r.MainFile != main || r.LogFile != filepath.Join(h, ".local", "state", "krino", "krino.log") { + t.Errorf("report files = %q %q", r.MainFile, r.LogFile) + } + if len(r.Dirs) != 2 || r.Dirs[0].Missing || !r.Dirs[1].Missing { + t.Errorf("dirs = %+v", r.Dirs) + } + if r.Tools[0].Name != "pdftotext" || r.Tools[0].Path != filepath.Join(bin, "pdftotext") || r.Tools[1].Path != "" { + t.Errorf("tools = %+v", r.Tools) + } +} + +// TestContentVariantsComputedAtLoad: B2 plumbing. Load computes each +// directory's distinct (ignoreCase, fold) content-test variants from its +// rules' resolved settings: a rule with no content test contributes +// nothing; two rules sharing a variant fold into one; a rule-level (case +// ignore) override adds a second. +func TestContentVariantsComputedAtLoad(t *testing.T) { + h := sandbox(t) + os.Mkdir(filepath.Join(h, "dl"), 0o755) + main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": ` +(path "~/dl") +(case strict) +(rule "no-content" (when (type pdf)) (stop)) +(rule "strict-content" (when (content "acme")) (stop)) +(rule "also-strict-content" (when (content "other")) (stop)) +(rule "loose-content" (case ignore) (when (content "acme")) (stop)) +`}) + e, errs := Load(main, "dl") + if len(errs) > 0 { + t.Fatal(errs) + } + got := e.Dirs[0].ContentVariants + want := []cond.Options{{IgnoreCase: false, Fold: true}, {IgnoreCase: true, Fold: true}} + if !reflect.DeepEqual(got, want) { + t.Fatalf("ContentVariants = %+v, want %+v", got, want) + } +} diff --git a/internal/engine/facts.go b/internal/engine/facts.go new file mode 100644 index 0000000..60380f3 --- /dev/null +++ b/internal/engine/facts.go @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package engine + +import ( + "context" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "krino/internal/cond" + "krino/internal/dup" + "krino/internal/norm" + "krino/internal/scan" + "krino/internal/xdg" +) + +// matchRun holds the state shared by every file evaluated during one Match +// or Explain call: the directory being matched, the full set of scanned +// files (for duplicate detection) and the lazily built duplicate indexes, +// one per distinct set of resolved directories a (duplicate ...) test +// names. mu guards dupOnce, dupIdx and warn, the only fields any goroutine +// but the one that created the matchRun ever touches. +type matchRun struct { + e *Engine + d *Dir + ctx context.Context + now time.Time + files []scan.File + + mu sync.Mutex + dupOnce map[string]*sync.Once + dupIdx map[string]*dup.Index + warn []string +} + +// newMatchRun builds a matchRun over files, the set a (duplicate ...) test +// with no directories of its own checks against. +func newMatchRun(e *Engine, d *Dir, ctx context.Context, now time.Time, files []scan.File) *matchRun { + return &matchRun{ + e: e, + d: d, + ctx: ctx, + now: now, + files: files, + dupOnce: make(map[string]*sync.Once), + dupIdx: make(map[string]*dup.Index), + } +} + +// warnings returns the directory-level warnings collected so far (from +// building duplicate indexes), in the order they were recorded. +func (run *matchRun) warnings() []string { + run.mu.Lock() + defer run.mu.Unlock() + return append([]string(nil), run.warn...) +} + +// drainDupErrors appends every duplicate index's candidate-hashing errors +// (A1: a candidate other than the file being looked up that could not be +// hashed) to run.warn, once matching is done and every index has seen every +// Lookup it is going to see. Candidate paths are abbreviated with +// xdg.Abbrev, as every other user-visible path is. +func (run *matchRun) drainDupErrors() { + run.mu.Lock() + defer run.mu.Unlock() + for _, idx := range run.dupIdx { + for _, ce := range idx.Errors() { + run.warn = append(run.warn, "duplicate: "+xdg.Abbrev(ce.Path)+": "+ce.Err.Error()) + } + } +} + +// dupIndex returns the shared *dup.Index for the resolved, sorted extra +// directories named by key, building it exactly once across every +// concurrent caller that asks for the same key. +func (run *matchRun) dupIndex(key string, dirs []string) *dup.Index { + run.mu.Lock() + once, ok := run.dupOnce[key] + if !ok { + once = &sync.Once{} + run.dupOnce[key] = once + } + run.mu.Unlock() + + once.Do(func() { + idx, errs := dup.NewIndex(run.files, dirs) + run.mu.Lock() + run.dupIdx[key] = idx + for _, err := range errs { + run.warn = append(run.warn, "duplicate: "+err.Error()) + } + run.mu.Unlock() + }) + + run.mu.Lock() + idx := run.dupIdx[key] + run.mu.Unlock() + return idx +} + +// facts is one file's cond.Facts. It is used by exactly one goroutine, so +// its own memoised state (content, its normalised variants, and whether an +// earlier rule matched) needs no locking of its own; only the matchRun it +// points at is shared. +type facts struct { + run *matchRun + file scan.File + + matched bool + + contentDone bool + content string + contentErr error + normCache map[[2]bool]string +} + +var _ cond.Facts = (*facts)(nil) + +// newFacts builds the Facts for one scanned file. +func newFacts(run *matchRun, file scan.File) *facts { + return &facts{run: run, file: file, normCache: make(map[[2]bool]string)} +} + +func (f *facts) Name() string { return f.file.Name } +func (f *facts) Rel() string { return f.file.Rel } +func (f *facts) Size() int64 { return f.file.Size } +func (f *facts) ModTime() time.Time { return f.file.ModTime } +func (f *facts) Now() time.Time { return f.run.now } +func (f *facts) Matched() bool { return f.matched } + +// Content extracts the file's text once, then normalises it per +// (ignoreCase, fold) variant, memoising each. B2: when the directory's +// rules use exactly one variant (Dir.ContentVariants), the raw text is +// released as soon as that variant's normalised copy exists — no other +// variant will ever be asked for, so there is no reason to keep both the +// raw text and its normalised copy in memory at once. A directory using +// more than one variant keeps the raw text for as long as f lives, exactly +// as before. +func (f *facts) Content(ignoreCase, fold bool) (string, error) { + if !f.contentDone { + f.content, f.contentErr = f.run.e.Extract.Text(f.run.ctx, f.file.Path, f.file.Size, f.run.d.Settings.MaxRead) + f.contentDone = true + } + if f.contentErr != nil { + return "", f.contentErr + } + key := [2]bool{ignoreCase, fold} + if v, ok := f.normCache[key]; ok { + return v, nil + } + v := norm.Text(f.content, ignoreCase, fold) + f.normCache[key] = v + if len(f.run.d.ContentVariants) == 1 { + f.content = "" + } + return v, nil +} + +// Duplicate resolves dirs against the directory's root, builds (or reuses) +// the shared duplicate index for that resolved, sorted set, and looks the +// file up in it. +func (f *facts) Duplicate(dirs []string) (string, bool, error) { + root := f.run.d.Root + resolved := make([]string, len(dirs)) + for i, raw := range dirs { + resolved[i] = resolveDir(raw, root) + } + sorted := append([]string(nil), resolved...) + sort.Strings(sorted) + key := strings.Join(sorted, "\x00") + + idx := f.run.dupIndex(key, sorted) + orig, isDup, err := idx.Lookup(f.file.Path) + if err != nil { + return "", false, err + } + if !isDup { + return "", false, nil + } + return displayOriginal(orig, root), true, nil +} + +// resolveDir expands a leading ~ and joins a relative directory to root, +// cleaned. +func resolveDir(raw, root string) string { + p := xdg.Expand(raw) + if !filepath.IsAbs(p) { + p = filepath.Join(root, p) + } + return filepath.Clean(p) +} + +// displayOriginal reports orig relative to root when it lies inside root, +// else as an absolute path. +func displayOriginal(orig, root string) string { + rel, err := filepath.Rel(root, orig) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return orig + } + return filepath.ToSlash(rel) +} diff --git a/internal/engine/facts_test.go b/internal/engine/facts_test.go new file mode 100644 index 0000000..c511804 --- /dev/null +++ b/internal/engine/facts_test.go @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package engine + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "krino/internal/cond" + "krino/internal/extract" + "krino/internal/scan" +) + +// contentFacts builds a *facts for a real text file, under a Dir whose +// ContentVariants is variants, for exercising B2's raw-release directly. +func contentFacts(t *testing.T, body string, variants []cond.Options) *facts { + t.Helper() + p := filepath.Join(t.TempDir(), "a.txt") + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + fi, err := os.Stat(p) + if err != nil { + t.Fatal(err) + } + e := &Engine{Extract: extract.New(), Now: time.Now} + d := &Dir{Name: "d", ContentVariants: variants} + file := scan.File{Path: p, Rel: "a.txt", Name: "a.txt", Size: fi.Size(), ModTime: fi.ModTime()} + run := newMatchRun(e, d, context.Background(), time.Now(), []scan.File{file}) + return newFacts(run, file) +} + +// TestContentReleasesRawWithOneVariant: B2. A directory whose rules use +// exactly one (ignoreCase, fold) variant releases the raw extracted text +// once that variant's normalised copy exists. +func TestContentReleasesRawWithOneVariant(t *testing.T) { + f := contentFacts(t, "Hello World", []cond.Options{{IgnoreCase: true, Fold: false}}) + const want = "hello world" + got, err := f.Content(true, false) + if err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("got %q, want %q", got, want) + } + if f.content != "" { + t.Errorf("raw text not released with a single content variant: %q", f.content) + } + // A second call for the same (already cached) variant must still work + // from normCache, without needing the released raw text. + if got, err := f.Content(true, false); err != nil || got != want { + t.Errorf("second call for the cached variant: got %q, %v, want %q", got, err, want) + } +} + +// TestContentKeepsRawWithTwoVariants: B2. A directory whose rules use two +// distinct variants must not release the raw text after the first: the +// second variant still needs it, and normalising it correctly (not from an +// emptied string) is the proof the raw text was kept. +func TestContentKeepsRawWithTwoVariants(t *testing.T) { + f := contentFacts(t, "Hello World", []cond.Options{ + {IgnoreCase: true, Fold: false}, + {IgnoreCase: false, Fold: false}, + }) + if got, err := f.Content(true, false); err != nil || got != "hello world" { + t.Fatalf("first variant: got %q, %v", got, err) + } + if f.content == "" { + t.Fatal("raw text released after only the first of two variants") + } + if got, err := f.Content(false, false); err != nil || got != "Hello World" { + t.Fatalf("second variant: got %q, %v, want the unfolded original (raw text must still be available)", got, err) + } +} diff --git a/internal/engine/match.go b/internal/engine/match.go new file mode 100644 index 0000000..0693bd5 --- /dev/null +++ b/internal/engine/match.go @@ -0,0 +1,357 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package engine + +import ( + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "sync" + "time" + + "krino/internal/cond" + "krino/internal/config" + "krino/internal/scan" + "krino/internal/xdg" +) + +// RuleMatch is one rule that matched a file, and why. +type RuleMatch struct { + Rule *Rule + Captures []string + Reasons []string +} + +// FileMatch is one file and the rules that did, or did not, match it. +type FileMatch struct { + File scan.File + Rules []RuleMatch // matching rules in order, ending at the first with (stop) + Warnings []string // "<rule>: <warning>", e.g. "acme: content unreadable: needs pdftotext, not installed" +} + +// Result is everything Match found in one directory. +type Result struct { + Dir *Dir + Matched []FileMatch // at least one rule matched; sorted by File.Rel + Unmatched []FileMatch // no rule matched (Warnings may say why); sorted by File.Rel + Skipped []scan.Skipped + Warnings []string // directory-level, sorted; e.g. "duplicate: /x/y does not exist" + Elapsed time.Duration +} + +// Match walks d's root and evaluates every rule against every file found, +// concurrently. Output order never depends on scheduling: results are +// placed by index into a slice the size of the walk, then split into +// Matched and Unmatched keeping that (Rel-sorted) order. +func (e *Engine) Match(ctx context.Context, d *Dir) (*Result, error) { + started := time.Now() + now := e.Now() + excl := e.excludeDirs(d) + + wres, err := scan.Walk(d.Root, walkOptions(d, excl, now)) + if err != nil { + return nil, err + } + + run := newMatchRun(e, d, ctx, now, wres.Files) + fileMatches := make([]FileMatch, len(wres.Files)) + + workers := runtime.GOMAXPROCS(0) + if workers < 1 { + workers = 1 + } + var wg sync.WaitGroup + jobs := make(chan int) + for w := 0; w < workers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := range jobs { + fileMatches[i] = evalFile(run, wres.Files[i]) + } + }() + } + for i := range wres.Files { + jobs <- i + } + close(jobs) + wg.Wait() + run.drainDupErrors() + + var matched, unmatched []FileMatch + for _, fm := range fileMatches { + if len(fm.Rules) > 0 { + matched = append(matched, fm) + } else { + unmatched = append(unmatched, fm) + } + } + + warnings := run.warnings() + sort.Strings(warnings) + + return &Result{ + Dir: d, + Matched: matched, + Unmatched: unmatched, + Skipped: wres.Skipped, + Warnings: warnings, + Elapsed: time.Since(started), + }, nil +} + +// evalFile evaluates every rule of run.d, in order, against file: matched +// becomes true after the first matching rule, so a later (not (matched)) +// test sees it, and evaluation stops right after a matching rule whose +// Stop is set. +func evalFile(run *matchRun, file scan.File) FileMatch { + f := newFacts(run, file) + fm := FileMatch{File: file} + for _, r := range run.d.Rules { + res := r.Cond.Eval(f) + for _, w := range res.Warnings { + fm.Warnings = append(fm.Warnings, r.Name+": "+w) + } + if !res.Match { + continue + } + fm.Rules = append(fm.Rules, RuleMatch{Rule: r, Captures: res.Captures, Reasons: res.Reasons}) + f.matched = true + if r.Conf.Stop { + break + } + } + return fm +} + +// RuleTrace is one rule's outcome in an Explain call. +type RuleTrace struct { + Rule *Rule + Match bool + Trace *cond.Trace // nil when not evaluated + Stopped string // "stopped by rule acme" when an earlier (stop) ended the search +} + +// Explanation is why (or why not) krino would act on one file. +type Explanation struct { + Dir *Dir + File scan.File + Skip string // why krino would not look at this file at all; "" when it would + Rules []RuleTrace +} + +// Explain reports, for one file, whether krino's ordinary scan would ever +// reach it and how every rule of its directory evaluates against it. Rules +// are traced in order even when Skip is set, so a user can see what would +// match if the file were looked at; a rule reached after an earlier +// matching (stop) is recorded as Stopped, with no trace. +func (e *Engine) Explain(ctx context.Context, path string) (*Explanation, error) { + abs, err := filepath.Abs(path) + if err != nil { + return nil, err + } + abs = filepath.Clean(abs) + + d := e.dirFor(abs) + if d == nil { + return nil, fmt.Errorf("%s is not inside any included directory", path) + } + + info, err := os.Lstat(abs) + if err != nil { + return nil, err + } + if info.Mode()&fs.ModeSymlink != 0 || !info.Mode().IsRegular() { + return nil, fmt.Errorf("%s is not a regular file", path) + } + + rel, err := filepath.Rel(d.Root, abs) + if err != nil { + return nil, err + } + rel = filepath.ToSlash(rel) + + sf := scan.File{ + Path: abs, + Rel: rel, + Name: filepath.Base(abs), + Size: info.Size(), + ModTime: info.ModTime(), + Mode: info.Mode(), + } + + now := e.Now() + excl := e.excludeDirs(d) + skip := explainSkip(d, sf, excl, now) + + run := newMatchRun(e, d, ctx, now, e.filesForExplain(d, sf, excl, now)) + f := newFacts(run, sf) + + var rules []RuleTrace + stoppedBy := "" + for _, r := range d.Rules { + if stoppedBy != "" { + rules = append(rules, RuleTrace{Rule: r, Stopped: "stopped by rule " + stoppedBy}) + continue + } + trace := r.Cond.Explain(f) + match := trace.Value + if match { + f.matched = true + } + rules = append(rules, RuleTrace{Rule: r, Match: match, Trace: trace}) + if match && r.Conf.Stop { + stoppedBy = r.Name + } + } + + return &Explanation{Dir: d, File: sf, Skip: skip, Rules: rules}, nil +} + +// filesForExplain returns the file set Explain's duplicate checks run +// against: the directory's ordinary scan, plus the explained file itself +// when that scan would not have reached it (it is busy, ignored, too new, +// excluded, or beyond recursive/max-depth) - so (duplicate ...) always has +// a real answer for the file being explained, and sees the same siblings +// Match would. +func (e *Engine) filesForExplain(d *Dir, sf scan.File, excl []string, now time.Time) []scan.File { + wres, err := scan.Walk(d.Root, walkOptions(d, excl, now)) + if err != nil { + return []scan.File{sf} + } + for _, wf := range wres.Files { + if wf.Path == sf.Path { + return wres.Files + } + } + return append(append([]scan.File{}, wres.Files...), sf) +} + +// walkOptions builds the scan.Options both Match and Explain's +// filesForExplain walk d's root with, so the two never drift apart. +func walkOptions(d *Dir, excl []string, now time.Time) scan.Options { + return scan.Options{ + Recursive: d.Settings.Recursive, + MaxDepth: d.Settings.MaxDepth, + Ignore: d.Ignore, + Exclude: excl, + Busy: d.Settings.Busy, + MinAge: d.Settings.MinAge, + Now: now, + } +} + +// dirFor returns the configured Dir whose root most specifically (longest +// root wins) contains abs, or nil if none does. +func (e *Engine) dirFor(abs string) *Dir { + var best *Dir + var bestRoot string + for _, d := range e.Dirs { + root := filepath.Clean(d.Root) + if abs != root && !strings.HasPrefix(abs, root+string(filepath.Separator)) { + continue + } + if best == nil || len(root) > len(bestRoot) { + best, bestRoot = d, root + } + } + return best +} + +// explainSkip decides, in priority order, why krino's ordinary scan would +// not reach sf, or "" if it would. +func explainSkip(d *Dir, sf scan.File, excl []string, now time.Time) string { + segs := strings.Split(sf.Rel, "/") + if !d.Settings.Recursive && len(segs) > 1 { + return "in a subdirectory, and recursive is off" + } + if d.Settings.MaxDepth > 0 && len(segs) > d.Settings.MaxDepth { + return "deeper than max-depth" + } + if insideAny(sf.Path, excl) { + return "inside a rule destination, which krino never scans" + } + if d.Ignore != nil && d.Ignore.Match(sf.Rel, false) { + return "ignored" + } + if isBusy(sf.Path, d.Settings.Busy) { + return "busy" + } + if now.Sub(sf.ModTime) < d.Settings.MinAge { + return "too new" + } + return "" +} + +// insideAny reports whether path is dir itself, or inside it, for any dir +// in dirs. +func insideAny(path string, dirs []string) bool { + for _, dir := range dirs { + if path == dir || strings.HasPrefix(path, dir+string(filepath.Separator)) { + return true + } + } + return false +} + +// isBusy reports whether path has a sibling named path+suffix, for any +// configured busy suffix - the mark of an in-progress download. +func isBusy(path string, suffixes []string) bool { + for _, suf := range suffixes { + if _, err := os.Lstat(path + suf); err == nil { + return true + } + } + return false +} + +// excludeDirs computes the directories Match and Explain never enter: each +// rule's copy/move destination, the Trash, and the directory holding the +// main config file - each kept only when it lies strictly inside d's root. +// A destination with no template placeholder excludes exactly that +// directory; a destination with a placeholder excludes only the static +// part before its first "{", cut back to a full path component (its last +// "/"), since anything from there on varies per file - spec 8.1: "Work/ +// Acme/{mtime:%Y}" excludes "Work/Acme", and "Work/Acme-{mtime:%Y}" +// (the placeholder mid-segment) excludes "Work". +func (e *Engine) excludeDirs(d *Dir) []string { + root := filepath.Clean(d.Root) + var out []string + add := func(p string) { + if p == "" { + return + } + p = filepath.Clean(p) + if strings.HasPrefix(p, root+string(filepath.Separator)) { + out = append(out, p) + } + } + for _, r := range d.Rules { + for _, a := range r.Conf.Actions { + if a.Kind != config.Copy && a.Kind != config.Move { + continue + } + prefix := a.Arg + if idx := strings.IndexByte(prefix, '{'); idx >= 0 { + prefix = prefix[:idx] + if idx2 := strings.LastIndexByte(prefix, '/'); idx2 >= 0 { + prefix = prefix[:idx2] + } else { + // The very first path component is itself templated + // (e.g. "{year}-Stuff"): nothing about the destination + // is known statically, so there is nothing to exclude. + prefix = "" + } + } + add(resolveDir(prefix, root)) + } + } + add(filepath.Join(xdg.DataHome(), "Trash")) + add(filepath.Dir(e.MainFile)) + return out +} diff --git a/internal/engine/match_test.go b/internal/engine/match_test.go new file mode 100644 index 0000000..1356074 --- /dev/null +++ b/internal/engine/match_test.go @@ -0,0 +1,367 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package engine + +import ( + "context" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" +) + +const dlConf = ` +(path "~/dl") +(recursive yes) +(min-age 0s) +(ignore "*.part") +(rule "dups" (when (duplicate)) (delete) (stop)) +(rule "acme" (when (type document) (content "acme ltd")) (move "Work/Acme") (stop)) +(rule "images" (when (type image)) (move "Pictures")) +(rule "rest" (when (not (matched)) (type text)) (move "Other")) +` + +// fixture builds ~/dl and returns a loaded engine. PATH is empty, so no +// extraction tool exists and PDFs and .doc files are unreadable. +func fixture(t *testing.T) (*Engine, *Dir, string) { + t.Helper() + h := sandbox(t) + t.Setenv("PATH", t.TempDir()) + dl := filepath.Join(h, "dl") + files := map[string]string{ + "inv1.txt": "Invoice from ACME LTD, tax 0000000000", + "notes.txt": "shopping list", + "photo.jpg": "\xff\xd8\xff\xe0 jpeg bytes", + "report.pdf": "%PDF same bytes", + "report (1).pdf": "%PDF same bytes", + "brochure.doc": "\xd0\xcf\x11\xe0 doc bytes", + "movie.mkv": "video", + "movie.mkv.part": "partial", + "Work/Acme/filed.txt": "acme ltd, already filed", + "Pictures/old.jpg": "\xff\xd8 old", + } + old := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for name, body := range files { + p := filepath.Join(dl, name) + os.MkdirAll(filepath.Dir(p), 0o755) + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + os.Chtimes(p, old, old) + } + newer := old.Add(time.Hour) + os.Chtimes(filepath.Join(dl, "report (1).pdf"), newer, newer) + main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": dlConf}) + e, errs := Load(main) + if len(errs) > 0 { + t.Fatal(errs) + } + return e, e.Dirs[0], dl +} + +// summary renders a result compactly for comparison. +func summary(r *Result) []string { + var out []string + for _, m := range r.Matched { + var rs []string + for _, rm := range m.Rules { + rs = append(rs, rm.Rule.Name+"["+strings.Join(rm.Reasons, "; ")+"]") + } + out = append(out, "match "+m.File.Rel+" "+strings.Join(rs, " ")) + } + for _, m := range r.Unmatched { + out = append(out, "none "+m.File.Rel+" "+strings.Join(m.Warnings, " | ")) + } + for _, s := range r.Skipped { + out = append(out, "skip "+s.Rel+" "+s.Reason.String()) + } + return out +} + +func TestMatch(t *testing.T) { + e, d, _ := fixture(t) + r, err := e.Match(context.Background(), d) + if err != nil { + t.Fatal(err) + } + want := []string{ + `match inv1.txt acme[type txt; content "acme ltd"]`, + `match notes.txt rest[not matched; type txt]`, + `match photo.jpg images[type jpg]`, + `match report (1).pdf dups[duplicate of report.pdf]`, + `none brochure.doc acme: content unreadable: needs antiword or catdoc, not installed`, + `none report.pdf acme: content unreadable: needs pdftotext, not installed`, + `skip movie.mkv busy`, + `skip movie.mkv.part ignored`, + } + if got := summary(r); !reflect.DeepEqual(got, want) { + t.Fatalf("got\n%s\nwant\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } + again, _ := e.Match(context.Background(), d) + if !reflect.DeepEqual(summary(again), summary(r)) { + t.Fatal("a second run gave a different result") + } +} + +func TestMatchMissingRoot(t *testing.T) { + e, d, dl := fixture(t) + os.RemoveAll(dl) + if _, err := e.Match(context.Background(), d); err == nil { + t.Fatal("no error for a missing root") + } +} + +func TestExplain(t *testing.T) { + e, _, dl := fixture(t) + x, err := e.Explain(context.Background(), filepath.Join(dl, "inv1.txt")) + if err != nil { + t.Fatal(err) + } + var got []string + for _, rt := range x.Rules { + got = append(got, fmt.Sprintf("%s match=%v stopped=%q trace=%v", rt.Rule.Name, rt.Match, rt.Stopped, rt.Trace != nil)) + } + want := []string{ + `dups match=false stopped="" trace=true`, + `acme match=true stopped="" trace=true`, + `images match=false stopped="stopped by rule acme" trace=false`, + `rest match=false stopped="stopped by rule acme" trace=false`, + } + if !reflect.DeepEqual(got, want) || x.Skip != "" { + t.Fatalf("skip=%q\n%s", x.Skip, strings.Join(got, "\n")) + } + for name, skip := range map[string]string{ + "movie.mkv": "busy", + "movie.mkv.part": "ignored", + "Work/Acme/filed.txt": "inside a rule destination, which krino never scans", + } { + x, err := e.Explain(context.Background(), filepath.Join(dl, name)) + if err != nil || x.Skip != skip { + t.Errorf("%s: skip %q, %v; want %q", name, x.Skip, err, skip) + } + } + if _, err := e.Explain(context.Background(), "/etc/hostname"); err == nil || !strings.HasSuffix(err.Error(), "is not inside any included directory") { + t.Errorf("outside: %v", err) + } +} + +// TestMatchExcludesOnlyRuleDest checks that a rule destination with no +// placeholder excludes exactly that directory, not its parent: a sibling +// subdirectory of the destination's parent must still be scanned. +func TestMatchExcludesOnlyRuleDest(t *testing.T) { + h := sandbox(t) + dl := filepath.Join(h, "dl") + files := map[string]string{ + "Work/Acme/filed.txt": "already filed", + "Work/Other/keep.txt": "keep me", + } + for name, body := range files { + p := filepath.Join(dl, name) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + conf := ` +(path "~/dl") +(recursive yes) +(min-age 0s) +(rule "acme" (when (name "nope")) (move "Work/Acme")) +` + main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": conf}) + e, errs := Load(main) + if len(errs) > 0 { + t.Fatal(errs) + } + r, err := e.Match(context.Background(), e.Dirs[0]) + if err != nil { + t.Fatal(err) + } + seen := map[string]bool{} + for _, m := range r.Matched { + seen[m.File.Rel] = true + } + for _, m := range r.Unmatched { + seen[m.File.Rel] = true + } + for _, s := range r.Skipped { + seen[s.Rel] = true + } + if !seen["Work/Other/keep.txt"] { + t.Error("Work/Other/keep.txt should have been scanned: only the rule's own destination (Work/Acme) may be excluded") + } + if seen["Work/Acme/filed.txt"] { + t.Error("Work/Acme/filed.txt should have been excluded as inside the rule's destination") + } +} + +// TestMatchWarningsSorted checks that Result.Warnings is sorted, not in +// whatever order concurrent workers happened to build the duplicate +// indexes that failed: rule "b" (declared first, with a stop that must +// not skip rule "a" since it never matches) names a missing directory +// that sorts after rule "a"'s, and rule "c" names the very same missing +// directory as rule "a" - which must fold into one warning, not two. +func TestMatchWarningsSorted(t *testing.T) { + h := sandbox(t) + dl := filepath.Join(h, "dl") + if err := os.MkdirAll(dl, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dl, "only.txt"), []byte("hello"), 0o644); err != nil { + t.Fatal(err) + } + conf := ` +(path "~/dl") +(recursive yes) +(min-age 0s) +(rule "b" (when (duplicate "~/zz-missing")) (stop)) +(rule "a" (when (duplicate "~/aa-missing")) (stop)) +(rule "c" (when (duplicate "~/aa-missing")) (stop)) +` + main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": conf}) + e, errs := Load(main) + if len(errs) > 0 { + t.Fatal(errs) + } + r, err := e.Match(context.Background(), e.Dirs[0]) + if err != nil { + t.Fatal(err) + } + aaDir := filepath.Join(h, "aa-missing") + zzDir := filepath.Join(h, "zz-missing") + if len(r.Warnings) != 2 { + t.Fatalf("got %d warnings, want 2 (the shared aa-missing dir should fold into one):\n%s", len(r.Warnings), strings.Join(r.Warnings, "\n")) + } + wantPrefix := []string{"duplicate: " + aaDir + ":", "duplicate: " + zzDir + ":"} + for i, want := range wantPrefix { + if !strings.HasPrefix(r.Warnings[i], want) { + t.Errorf("Warnings[%d] = %q, want prefix %q", i, r.Warnings[i], want) + } + } +} + +// TestExcludeDirs is a table check of excludeDirs's rule-destination +// handling, in rule order: a plain destination excludes itself exactly; a +// destination with a placeholder excludes only the static part before it, +// cut back to a full path component; a destination (after that cut) equal +// to the root itself, or outside the root, excludes nothing; an absolute +// destination inside the root excludes that directory; copy counts like +// move; rename is not a destination at all. +func TestExcludeDirs(t *testing.T) { + h := sandbox(t) + root := filepath.Join(h, "root") + absDest := filepath.Join(root, "AbsDest") + conf := ` +(path "` + root + `") +(rule "r1" (move "Work/Acme")) +(rule "r2" (move "Photos/{mtime:%Y}")) +(rule "r3" (move "Work/Acme-{mtime:%Y}")) +(rule "r4" (move "{ext}")) +(rule "r5" (move ".")) +(rule "r6" (move "~/elsewhere")) +(rule "r7" (move "` + absDest + `")) +(rule "r8" (copy "Backup")) +(rule "r9" (rename "x-{name}")) +` + main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": conf}) + e, errs := Load(main) + if len(errs) > 0 { + t.Fatal(errs) + } + got := e.excludeDirs(e.Dirs[0]) + want := []string{ + filepath.Join(root, "Work", "Acme"), // r1: no placeholder, exact + filepath.Join(root, "Photos"), // r2: cut back to "Photos/" + filepath.Join(root, "Work"), // r3: cut back past "Acme-" + // r4 "{ext}": nothing before "{" at all -> resolves to the root + // itself -> not strictly inside it -> excludes nothing. + // r5 ".": no placeholder, resolves to the root itself -> nothing. + // r6 "~/elsewhere": outside the root -> nothing. + absDest, // r7: absolute, already inside root + filepath.Join(root, "Backup"), // r8: copy counts like move + // r9 rename "x-{name}": rename is never a destination. + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("excludeDirs =\n%v\nwant\n%v", got, want) + } +} + +// TestMatchDrainsDupCandidateErrors: A1 plumbing across the dup/engine +// boundary. Within one directory's own scan (no extra directories), a +// candidate that cannot be hashed must not poison the duplicate answer for +// its size-mates, and must surface exactly once in Result.Warnings, its +// path abbreviated the way every other user-visible path is. +func TestMatchDrainsDupCandidateErrors(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("permissions are not enforced running as root") + } + h := sandbox(t) + dl := filepath.Join(h, "dl") + old := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + files := map[string]string{ + "a.txt": "same content", // older: the original + "b.txt": "same content", // newer: reported as the duplicate + "c.txt": "diff content", // same size as a/b, different bytes + } + for name, body := range files { + p := filepath.Join(dl, name) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + os.Chtimes(filepath.Join(dl, "a.txt"), old, old) + os.Chtimes(filepath.Join(dl, "b.txt"), old.Add(time.Hour), old.Add(time.Hour)) + os.Chtimes(filepath.Join(dl, "c.txt"), old, old) + cPath := filepath.Join(dl, "c.txt") + if err := os.Chmod(cPath, 0o000); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.Chmod(cPath, 0o644) }) + + conf := ` +(path "~/dl") +(recursive yes) +(min-age 0s) +(rule "dup" (when (duplicate)) (stop)) +` + main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": conf}) + e, errs := Load(main) + if len(errs) > 0 { + t.Fatal(errs) + } + r, err := e.Match(context.Background(), e.Dirs[0]) + if err != nil { + t.Fatal(err) + } + + var bReasons []string + for _, m := range r.Matched { + if m.File.Rel == "b.txt" { + for _, rm := range m.Rules { + bReasons = append(bReasons, rm.Reasons...) + } + } + } + if len(bReasons) == 0 || bReasons[0] != "duplicate of a.txt" { + t.Errorf("b.txt duplicate pair with a.txt broken by unreadable sibling c.txt: %v", summary(r)) + } + + want := "duplicate: ~/dl/c.txt: " + found := 0 + for _, w := range r.Warnings { + if strings.HasPrefix(w, want) { + found++ + } + } + if found != 1 { + t.Errorf("got %d warnings with prefix %q, want 1; warnings: %v", found, want, r.Warnings) + } +} diff --git a/internal/extract/extract.go b/internal/extract/extract.go new file mode 100644 index 0000000..aa3ae93 --- /dev/null +++ b/internal/extract/extract.go @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +// Package extract gets text out of files so rules can test their content. +// Text is returned raw; callers normalise it with norm.Text. +package extract + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "time" +) + +var ( + // ErrUnsupported is returned when the format carries no text krino + // knows how to extract. + ErrUnsupported = errors.New("no text in this format") + // ErrTooLarge is returned when the file is larger than the configured + // max-read; nothing is read in that case. + ErrTooLarge = errors.New("larger than max-read") +) + +// ToolMissingError is returned when a format needs an external tool that +// was not found on this system. +type ToolMissingError struct{ Tool string } + +func (e *ToolMissingError) Error() string { + return "needs " + e.Tool + ", not installed" +} + +// Tool is one external extractor and the absolute path it was found at, or +// "" if it was not found. +type Tool struct{ Name, Path string } + +// toolNames lists the external tools Extractor looks up, in the order +// Tools() reports them. +var toolNames = []string{"pdftotext", "antiword", "catdoc", "xls2csv", "catppt"} + +// markupExt is the set of markup extensions: tags stripped, entities +// decoded. +var markupExt = map[string]bool{ + "html": true, "htm": true, "xhtml": true, "xml": true, "svg": true, +} + +// plainExt is the set of extensions read as plain text without further +// inspection. +var plainExt = map[string]bool{ + "txt": true, "md": true, "log": true, "csv": true, "tsv": true, + "json": true, "yaml": true, "yml": true, "toml": true, "ini": true, + "conf": true, "cfg": true, "rtf": true, "tex": true, "go": true, + "c": true, "h": true, "cpp": true, "hpp": true, "py": true, "sh": true, + "js": true, "ts": true, "rs": true, "java": true, "rb": true, + "pl": true, "lua": true, "css": true, "sql": true, +} + +// zipExt is the set of Office/OpenDocument/ebook formats: a zip container +// plus XML inside it (Task 5). +var zipExt = map[string]bool{ + "docx": true, "xlsx": true, "pptx": true, + "odt": true, "ods": true, "odp": true, + "epub": true, +} + +// toolExt maps an extension to the external tool it needs (Task 6). pdf is +// handled separately, since it has its own fixed command line. +var toolExt = map[string]string{ + "doc": "antiword", // falls back to catdoc + "xls": "xls2csv", + "ppt": "catppt", +} + +// Extractor gets text out of files, using the external tools it found at +// construction. +type Extractor struct { + tools map[string]string // name -> absolute path, only tools found + Timeout time.Duration // per external tool run +} + +// New builds an Extractor, looking up each external tool once in the +// process's $PATH, with a 30 s per-tool Timeout. +func New() *Extractor { + return newWithPath(os.Getenv("PATH")) +} + +// newWithPath builds an Extractor looking up tools in the given PATH-style +// list instead of the process environment, so tests control what is found. +// It does not use exec.LookPath, which reads the process's own PATH; it +// walks path itself. +func newWithPath(path string) *Extractor { + dirs := filepath.SplitList(path) + tools := make(map[string]string, len(toolNames)) + for _, name := range toolNames { + for _, dir := range dirs { + if dir == "" { + continue + } + p := filepath.Join(dir, name) + info, err := os.Stat(p) + if err != nil || !info.Mode().IsRegular() { + continue // no such file, or a directory/FIFO/socket/device + } + if info.Mode()&0o111 == 0 { + continue // not executable + } + tools[name] = p + break + } + } + return &Extractor{tools: tools, Timeout: 30 * time.Second} +} + +// Tools reports every external tool Extractor knows about, in a fixed +// order, with the path it was found at or "" if it was not found. +func (e *Extractor) Tools() []Tool { + out := make([]Tool, len(toolNames)) + for i, name := range toolNames { + out[i] = Tool{Name: name, Path: e.tools[name]} + } + return out +} + +// Text returns path's raw text content: it does not normalise (callers use +// norm.Text). size is the file's size, as already known to the caller; +// maxRead is the configured ceiling, 0 meaning unlimited. Dispatch is on +// the lower-cased extension; a format extension with no reader yet +// implemented returns ErrUnsupported. +func (e *Extractor) Text(ctx context.Context, path string, size, maxRead int64) (string, error) { + if maxRead > 0 && size > maxRead { + return "", ErrTooLarge + } + + ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(path), ".")) + + switch { + case ext == "pdf": + return e.pdfText(ctx, path, maxRead) + case zipExt[ext]: + return zipText(ctx, path, ext, maxRead) + case toolExt[ext] != "": + return e.legacyText(ctx, path, ext, maxRead) + case markupExt[ext]: + raw, err := readDecoded(path) + if err != nil { + return "", err + } + return stripMarkup(raw), nil + case plainExt[ext]: + return readDecoded(path) + default: + return sniffText(path) + } +} diff --git a/internal/extract/plain.go b/internal/extract/plain.go new file mode 100644 index 0000000..d245a41 --- /dev/null +++ b/internal/extract/plain.go @@ -0,0 +1,282 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package extract + +import ( + "bytes" + "encoding/binary" + "html" + "io" + "os" + "strings" + "unicode/utf16" + "unicode/utf8" +) + +// sniffSize is how much of an unknown-extension file is inspected to guess +// whether it is text (design.md §6). +const sniffSize = 8192 + +// readDecoded reads path whole and decodes it per the encoding rules: a +// leading UTF-8 BOM is stripped and the rest used as is; a UTF-16 LE or BE +// BOM is decoded with unicode/utf16; otherwise valid UTF-8 is used as is, +// and any other invalid UTF-8 is decoded one byte per Latin-1 code point. +func readDecoded(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + return decode(data), nil +} + +// sniffText decides whether an unknown extension is text, reading at most +// sniffSize bytes before deciding: a file whose first sniffSize bytes are +// not a UTF-16 BOM and not valid-UTF8-with-no-NUL is rejected as +// ErrUnsupported without reading any further (so a large binary file is +// never read in full just to be rejected). Only once that sample passes is +// the rest of the file read; a UTF-16 BOM is trusted as plain text from +// the sample alone (UTF-16 text is full of NUL bytes by design), but a +// sample that merely looks like UTF-8 must hold for the WHOLE file — no +// NUL byte anywhere, and no invalid UTF-8 anywhere past the sample — or +// the file is ErrUnsupported after all; the Latin-1 fallback in decode +// never applies to a sniffed file, only to a file whose extension already +// names it as text. D2: when the file continues past the sample (n == +// sniffSize), the validity check is run against a trimmed copy with any +// incomplete trailing rune removed, so a multi-byte rune that happens to +// straddle byte sniffSize does not make an otherwise-valid file sniff as +// unsupported; sample itself, used below to build the returned text, is +// left untouched — the rest of the file (read after the check) supplies +// the bytes trimming set aside. +func sniffText(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + + sample := make([]byte, sniffSize) + n, err := io.ReadFull(f, sample) + if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF { + return "", err + } + sample = sample[:n] + + check := sample + if n == sniffSize { + check = trimIncompleteTrailingRune(sample) + } + + utf16BOM := hasUTF16BOM(sample) + if !utf16BOM && !(utf8.Valid(check) && !bytes.Contains(check, []byte{0})) { + return "", ErrUnsupported + } + + rest, err := io.ReadAll(f) + if err != nil { + return "", err + } + data := append(sample, rest...) + + if utf16BOM { + return decode(data), nil + } + if !utf8.Valid(data) || bytes.Contains(data, []byte{0}) { + return "", ErrUnsupported + } + return decode(data), nil +} + +// trimIncompleteTrailingRune drops an incomplete UTF-8 sequence left +// dangling at the very end of b — D2's fix for a rune cut off exactly at +// the sniff sample's boundary. It looks back at most utf8.UTFMax-1 bytes +// for the start of the trailing rune; if the bytes from there to the end +// are not a complete encoding (utf8.FullRune), that partial rune is cut, +// since more bytes to finish it may simply not have been read yet. A +// sample already ending cleanly (the common case, and every all-ASCII +// sample) is returned unchanged. +func trimIncompleteTrailingRune(b []byte) []byte { + end := len(b) + start := end - 1 + for start >= 0 && end-start < utf8.UTFMax && !utf8.RuneStart(b[start]) { + start-- + } + if start < 0 || utf8.FullRune(b[start:end]) { + return b + } + return b[:start] +} + +// hasUTF16BOM reports whether b begins with a UTF-16 little- or big-endian +// byte-order mark. +func hasUTF16BOM(b []byte) bool { + return len(b) >= 2 && ((b[0] == 0xFF && b[1] == 0xFE) || (b[0] == 0xFE && b[1] == 0xFF)) +} + +// decode applies the encoding rules to a whole file's bytes. +func decode(data []byte) string { + switch { + case len(data) >= 2 && data[0] == 0xFF && data[1] == 0xFE: + return decodeUTF16(data[2:], binary.LittleEndian) + case len(data) >= 2 && data[0] == 0xFE && data[1] == 0xFF: + return decodeUTF16(data[2:], binary.BigEndian) + case len(data) >= 3 && data[0] == 0xEF && data[1] == 0xBB && data[2] == 0xBF: + return string(data[3:]) + case utf8.Valid(data): + return string(data) + default: + return decodeLatin1(data) + } +} + +// decodeUTF16 decodes b (already past the BOM) as UTF-16 in the given byte +// order; a trailing odd byte with no pair is dropped. +func decodeUTF16(b []byte, order binary.ByteOrder) string { + n := len(b) / 2 + units := make([]uint16, n) + for i := 0; i < n; i++ { + units[i] = order.Uint16(b[i*2 : i*2+2]) + } + return string(utf16.Decode(units)) +} + +// decodeLatin1 decodes b as Latin-1: each byte is its own Unicode code +// point. +func decodeLatin1(b []byte) string { + r := make([]rune, len(b)) + for i, c := range b { + r[i] = rune(c) + } + return string(r) +} + +// stripMarkup turns decoded HTML/XML/SVG text into plain text: a small +// scanner replaces every <...> tag with a space, dropping the contents of +// <script> and <style> along with their tags and skipping <!-- ... --> +// comments outright, then entities are decoded with html.UnescapeString. +// html.UnescapeString turns into U+00A0 (a non-breaking space, not +// a plain space); since the source markup used it as ordinary inter-word +// spacing, it is folded to a regular space here too. +// +// A '<' only starts a tag when followed by a letter, '/', '!' or '?' — +// HTML's own rule for what can open a tag, close tag, comment/doctype, or +// processing instruction. Anything else (a digit, a space, end of string) +// is literal text, so "a < b" is not mistaken for markup. +func stripMarkup(s string) string { + var b strings.Builder + b.Grow(len(s)) + i, n := 0, len(s) + for i < n { + if s[i] != '<' || !startsTag(s, i) { + b.WriteByte(s[i]) + i++ + continue + } + + if strings.HasPrefix(s[i:], "<!--") { + end := n + if k := strings.Index(s[i+4:], "-->"); k != -1 { + end = i + 4 + k + len("-->") + } + b.WriteByte(' ') + i = end + continue + } + + j := i + 1 + closing := false + if j < n && s[j] == '/' { + closing = true + j++ + } + nameStart := j + for j < n && isTagNameByte(s[j]) { + j++ + } + name := strings.ToLower(s[nameStart:j]) + + gt := strings.IndexByte(s[j:], '>') + if gt == -1 { + // D1: an unterminated tag (no closing '>') can no longer be + // parsed as markup, but that is no reason to discard the rest + // of the file - copy it through as literal text instead of + // simply stopping the scan. + b.WriteString(s[i:]) + break + } + end := j + gt + 1 + + if !closing && (name == "script" || name == "style") { + if close := indexCloseTag(s[end:], name); close != -1 { + end += close + if gt2 := strings.IndexByte(s[end:], '>'); gt2 != -1 { + end += gt2 + 1 + } else { + end = n + } + } else { + end = n + } + } + + b.WriteByte(' ') + i = end + } + + return strings.ReplaceAll(html.UnescapeString(b.String()), string(nbsp), " ") +} + +// startsTag reports whether s[i] == '<' begins a tag-like construct: the +// next character is a letter, '/', '!' or '?'. A '<' at the very end of s, +// or followed by anything else (digit, space, punctuation), is literal +// text instead. +func startsTag(s string, i int) bool { + if i+1 >= len(s) { + return false + } + c := s[i+1] + return c == '/' || c == '!' || c == '?' || + (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') +} + +// nbsp is U+00A0, NO-BREAK SPACE: what html.UnescapeString decodes +// to, folded to a regular space since the source markup used it as one. +const nbsp = rune(0xA0) + +// isTagNameByte reports whether c can appear in an HTML/XML tag name. +func isTagNameByte(c byte) bool { + return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '-' || c == ':' || c == '_' +} + +// indexCloseTag returns the index in s of the first ASCII case-insensitive +// occurrence of "</name", or -1. +func indexCloseTag(s, name string) int { + target := "</" + name + tn := len(target) + for i := 0; i+tn <= len(s); i++ { + if asciiEqualFold(s[i:i+tn], target) { + return i + } + } + return -1 +} + +// asciiEqualFold reports whether a and b are equal, ASCII letters compared +// without regard to case. +func asciiEqualFold(a, b string) bool { + if len(a) != len(b) { + return false + } + for i := 0; i < len(a); i++ { + ca, cb := a[i], b[i] + if 'A' <= ca && ca <= 'Z' { + ca += 'a' - 'A' + } + if 'A' <= cb && cb <= 'Z' { + cb += 'a' - 'A' + } + if ca != cb { + return false + } + } + return true +} diff --git a/internal/extract/plain_test.go b/internal/extract/plain_test.go new file mode 100644 index 0000000..045ea26 --- /dev/null +++ b/internal/extract/plain_test.go @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package extract + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "syscall" + "testing" +) + +// file writes data to name in a temp dir and returns the path. +func file(t *testing.T, name string, data []byte) string { + t.Helper() + p := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(p, data, 0o644); err != nil { + t.Fatal(err) + } + return p +} + +func text(t *testing.T, e *Extractor, p string, maxRead int64) (string, error) { + t.Helper() + fi, err := os.Stat(p) + if err != nil { + t.Fatal(err) + } + return e.Text(context.Background(), p, fi.Size(), maxRead) +} + +func TestPlainEncodings(t *testing.T) { + e := newWithPath("") + utf16le := []byte{0xFF, 0xFE, 'A', 0, 'c', 0, 'm', 0, 'e', 0} + tests := []struct { + name string + data []byte + want string + }{ + {"a.txt", []byte("Faktura acme ltd\n"), "Faktura acme ltd\n"}, + {"bom.txt", []byte("\xEF\xBB\xBFhello"), "hello"}, + {"latin1.txt", []byte("Gr\xfc\xdfe"), "Grüße"}, + {"u16.txt", utf16le, "Acme"}, + {"notes.md", []byte("# Title\nbody"), "# Title\nbody"}, + {"empty.txt", nil, ""}, + {"README", []byte("no extension but text"), "no extension but text"}, + } + for _, tt := range tests { + got, err := text(t, e, file(t, tt.name, tt.data), 0) + if err != nil || got != tt.want { + t.Errorf("%s: got %q, %v; want %q", tt.name, got, err, tt.want) + } + } +} + +func TestMarkup(t *testing.T) { + e := newWithPath("") + tests := []struct { + name string + src string + want []string // at least one of these must be a substring + wantAny bool // if true, want is an alternative set: any one suffices + bad []string // none of these may be a substring + }{ + { + name: "tags, script/style dropped, entities decoded", + src: "<html><head><style>p{color:red}</style><script>var x='acme'</script></head>" + + "<body><p>Faktura VAT & co</p><p>acme ltd</p></body></html>", + want: []string{"Faktura VAT & co", "acme ltd"}, + bad: []string{"color:red", "var x", "<p>"}, + }, + { + name: "a lone < followed by a digit or space is literal text", + src: "<p>price < 500 zl, done</p>", + want: []string{"price < 500 zl, done"}, + wantAny: true, + }, + { + name: "an HTML comment is skipped, not its neighbours", + src: "<p>a<!-- hidden -->b</p>", + want: []string{"a b", "ab"}, + wantAny: true, + bad: []string{"hidden"}, + }, + } + for _, tt := range tests { + got, err := text(t, e, file(t, "page.html", []byte(tt.src)), 0) + if err != nil { + t.Fatalf("%s: %v", tt.name, err) + } + if tt.wantAny { + ok := false + for _, w := range tt.want { + if strings.Contains(got, w) { + ok = true + break + } + } + if !ok { + t.Errorf("%s: markup text %q has none of %q", tt.name, got, tt.want) + } + } else { + for _, want := range tt.want { + if !strings.Contains(got, want) { + t.Errorf("%s: markup text %q lacks %q", tt.name, got, want) + } + } + } + for _, bad := range tt.bad { + if strings.Contains(got, bad) { + t.Errorf("%s: markup text %q still contains %q", tt.name, got, bad) + } + } + } +} + +func TestUnsupportedAndTooLarge(t *testing.T) { + e := newWithPath("") + png := []byte("\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR") + if _, err := text(t, e, file(t, "image.png", png), 0); !errors.Is(err, ErrUnsupported) { + t.Errorf("png: %v, want ErrUnsupported", err) + } + if _, err := text(t, e, file(t, "blob.bin", []byte{1, 2, 0, 3}), 0); !errors.Is(err, ErrUnsupported) { + t.Errorf("binary: %v, want ErrUnsupported", err) + } + if _, err := text(t, e, file(t, "big.txt", []byte(strings.Repeat("x", 100))), 50); !errors.Is(err, ErrTooLarge) { + t.Errorf("big: %v, want ErrTooLarge", err) + } +} + +func TestToolMissingError(t *testing.T) { + err := &ToolMissingError{Tool: "pdftotext"} + if err.Error() != "needs pdftotext, not installed" { + t.Fatalf("got %q", err.Error()) + } +} + +func TestToolsListedInOrder(t *testing.T) { + var names []string + for _, tl := range newWithPath("").Tools() { + names = append(names, tl.Name) + if tl.Path != "" { + t.Errorf("%s found with an empty PATH", tl.Name) + } + } + if strings.Join(names, " ") != "pdftotext antiword catdoc xls2csv catppt" { + t.Fatalf("tools = %v", names) + } +} + +// TestSniffWholeFileMustBeValid: the first 8 KiB sniffs as plain ASCII text, +// but the file goes on to hold an invalid UTF-8 byte and a NUL past that +// sample — sniffText must reject the whole file, not just decode what the +// sample alone promised (it must not fall back to Latin-1 the way a known +// text extension would). +func TestSniffWholeFileMustBeValid(t *testing.T) { + e := newWithPath("") + data := append([]byte(strings.Repeat("x", 8192)), 0xFF, 0x00) + if _, err := text(t, e, file(t, "blob.data", data), 0); !errors.Is(err, ErrUnsupported) { + t.Errorf("got %v, want ErrUnsupported", err) + } +} + +// TestSniffLargeBinaryRejected: an unrecognised-extension file whose very +// first byte is invalid UTF-8 is rejected from the sample alone; this only +// checks the outcome (ErrUnsupported), not that the rest of the megabyte +// went unread — that efficiency claim isn't something a black-box test can +// time reliably. +func TestSniffLargeBinaryRejected(t *testing.T) { + e := newWithPath("") + data := make([]byte, 1<<20) // 1 MiB, far past the 8 KiB sniff window + data[0] = 0xFF // invalid UTF-8 lead byte, visible in the sample + if _, err := text(t, e, file(t, "huge.blob", data), 0); !errors.Is(err, ErrUnsupported) { + t.Errorf("got %v, want ErrUnsupported", err) + } +} + +// TestToolLookupSkipsNonRegular: a FIFO named like a tool, executable bits +// and all, must never be picked up — only a regular file counts. +func TestToolLookupSkipsNonRegular(t *testing.T) { + dir := t.TempDir() + fifo := filepath.Join(dir, "pdftotext") + if err := syscall.Mkfifo(fifo, 0o755); err != nil { + t.Skipf("mkfifo not available: %v", err) + } + for _, tl := range newWithPath(dir).Tools() { + if tl.Name == "pdftotext" && tl.Path != "" { + t.Errorf("pdftotext resolved to a non-regular file: %s", tl.Path) + } + } +} + +// TestUnterminatedTagKeepsRemainder: D1. An unterminated ordinary tag (no +// closing '>') must not discard the rest of the file - only the malformed +// tag markup itself is unrecoverable; whatever follows it is still real +// content and must still reach the extracted text. +func TestUnterminatedTagKeepsRemainder(t *testing.T) { + e := newWithPath("") + src := "<p>before</p><p unterminated text after" + got, err := text(t, e, file(t, "broken.html", []byte(src)), 0) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(got, "before") { + t.Errorf("text before the unterminated tag missing: %q", got) + } + if !strings.Contains(got, "unterminated text after") { + t.Errorf("text after the unterminated tag was discarded: %q", got) + } +} + +// TestSniffRuneStraddlingSampleBoundary: D2. A multi-byte rune ("ż", two +// UTF-8 bytes) placed exactly so its lead byte is the sniff sample's last +// byte and its continuation byte falls just past it must not make an +// otherwise valid UTF-8 file sniff as unsupported. +func TestSniffRuneStraddlingSampleBoundary(t *testing.T) { + e := newWithPath("") + prefix := strings.Repeat("a", sniffSize-1) + data := []byte(prefix + "ż" + "bcd") + got, err := text(t, e, file(t, "straddle.blob", data), 0) + if err != nil { + t.Fatalf("valid UTF-8 with a rune straddling the sniff boundary: %v", err) + } + if want := "żbcd"; !strings.HasSuffix(got, want) { + t.Errorf("got tail %q, want it to end in %q", got[len(got)-8:], want) + } +} diff --git a/internal/extract/tools.go b/internal/extract/tools.go new file mode 100644 index 0000000..ca6a9b0 --- /dev/null +++ b/internal/extract/tools.go @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package extract + +import ( + "bytes" + "context" + "errors" + "fmt" + "os/exec" + "path/filepath" + "time" +) + +// maxToolOutput caps the bytes kept from an external tool's stdout; more +// than this returns ErrTooLarge. A package var, not a const, so a test can +// lower it without generating gigabytes of fake output. B1: the directory's +// max-read may cap a single extraction further still — see budget. +var maxToolOutput int64 = 64 << 20 + +// budget returns the smaller of fixed (the package's own default ceiling — +// maxToolOutput or zipBudget) and maxRead, the directory's configured +// max-read; maxRead 0 means unlimited, so fixed alone applies. B1: a +// single extraction's output must never exceed the ceiling the user set, +// even when that ceiling is below the fixed default. +func budget(fixed, maxRead int64) int64 { + if maxRead > 0 && maxRead < fixed { + return maxRead + } + return fixed +} + +// maxStderr caps the bytes kept from an external tool's stderr — enough +// for a diagnostic first line. Unlike stdout, crossing this never kills +// the tool: stderr noise is not grounds to abort an otherwise-working +// extraction, only grounds to stop remembering more of it. +const maxStderr = 4 << 10 + +// maxErrLine caps how much of stderr's first line is folded into the +// error text run() returns, so a flooding tool cannot make that message +// itself unbounded. +const maxErrLine = 200 + +// boundedWriter keeps at most limit bytes written to it and silently +// discards the rest, always reporting success to the writer — an +// io.Writer that returns an error would abort the copy goroutine +// exec.Cmd runs for Stdout/Stderr, which is not what should happen here: +// the tool must keep being drained (or be killed outright, via +// onOverflow) rather than have its pipe start backing up. If onOverflow +// is set, it fires exactly once, the moment the total ever written first +// exceeds limit; run() uses it on stdout, and only stdout, to cancel the +// command immediately rather than let an over-producing tool run until +// e.Timeout. Write is only ever called by the single copy goroutine +// exec.Cmd runs per stream, so no locking is needed; run() only reads a +// boundedWriter's fields after cmd.Run() has returned, which happens +// strictly after that goroutine has finished (Wait's documented +// synchronisation), giving the read a safe happens-before. +type boundedWriter struct { + limit int64 + onOverflow func() + + buf bytes.Buffer + total int64 + overflowed bool +} + +func (w *boundedWriter) Write(p []byte) (int, error) { + w.total += int64(len(p)) + if w.total > w.limit { + if !w.overflowed { + w.overflowed = true + if w.onOverflow != nil { + w.onOverflow() + } + } + return len(p), nil + } + w.buf.Write(p) + return len(p), nil +} + +// run executes tool (looked up in e.tools, its absolute path) with args, +// under a timeout of e.Timeout, and returns its stdout as text, its +// stdout capped at maxOut bytes (the caller passes budget(maxToolOutput, +// maxRead), B1). No shell is involved: exec.CommandContext runs the +// tool's path directly with args passed separately, so nothing in a +// hostile filename or argument is ever interpreted. The environment is +// inherited unchanged. +// +// os/exec is left to own all the copying — cmd.Stdout and cmd.Stderr are +// bounded writers, and cmd.Run does the reading — so that cmd.WaitDelay's +// hang protection actually applies: WaitDelay bounds how long Wait spends +// on I/O after the process itself has exited (or after ctx is done), +// forcibly closing the pipes once that grace period elapses. An earlier +// version of this function read stdout itself, ahead of Wait, which +// starved WaitDelay of the thing it bounds: once that manual read +// stopped (at EOF, or at the output cap), Wait was left blocked on +// whatever was still holding the pipe open — a grandchild the tool +// backgrounded and left running, or the tool itself blocked writing to a +// pipe nobody was draining once the cap was hit — with nothing left to +// force it closed. In both shapes the call could run for the full +// e.Timeout (or longer) instead of returning promptly. +// +// Stdout is capped at maxOut bytes (the caller's budget(maxToolOutput, +// maxRead), B1): crossing it cancels the command immediately, via the +// boundedWriter's onOverflow, and run reports ErrTooLarge. Stderr is +// capped at maxStderr bytes and never +// cancels anything; only its first line, truncated to maxErrLine bytes, +// ever reaches an error message, so a tool flooding stderr costs bounded +// memory and produces a bounded error. +// +// Killing the command — by the caller's ctx being cancelled, by +// e.Timeout expiring, or by the stdout cap being crossed — can leave +// cmd.Run reporting exec.ErrWaitDelay even though the process's own exit +// status was clean: SIGKILL forces the pipes closed without giving the +// child a chance to flush or exit on its own. That alone is not a +// failure (see the ErrWaitDelay case below); only a genuinely non-zero +// exit is treated as one. +// +// A non-zero exit returns "<tool> failed: <first line of stderr>" (or +// "<tool> failed: <err>" when stderr was empty), the line capped to +// maxErrLine bytes. A caller-cancelled ctx returns promptly, with an +// error wrapping ctx.Err(); e.Timeout expiring on its own returns +// "<tool> timed out after <Timeout>". +func (e *Extractor) run(ctx context.Context, maxOut int64, tool string, args ...string) (string, error) { + path := e.tools[tool] + + runCtx, cancel := context.WithTimeout(ctx, e.Timeout) + defer cancel() + + cmd := exec.CommandContext(runCtx, path, args...) + cmd.WaitDelay = time.Second + + stdout := &boundedWriter{limit: maxOut, onOverflow: cancel} + stderr := &boundedWriter{limit: maxStderr} + cmd.Stdout = stdout + cmd.Stderr = stderr + + err := cmd.Run() + + switch { + case stdout.overflowed: + // Checked first: killing the tool for overflow also cancels + // runCtx, so without this ordering the case below would report + // the cancellation as a timeout instead of what it actually was. + return "", ErrTooLarge + case ctx.Err() != nil: + // The caller's own context, not the internal e.Timeout deadline + // derived from it — checked before runCtx's, since runCtx + // inherits the caller's cancellation too and would otherwise be + // indistinguishable from it below. + return "", fmt.Errorf("%s: %w", tool, ctx.Err()) + case runCtx.Err() == context.DeadlineExceeded: + return "", fmt.Errorf("%s timed out after %s", tool, e.Timeout) + } + + if err != nil { + if errors.Is(err, exec.ErrWaitDelay) && cmd.ProcessState != nil && cmd.ProcessState.ExitCode() == 0 { + return stdout.buf.String(), nil + } + if line := truncate(firstLine(stderr.buf.Bytes()), maxErrLine); line != "" { + return "", fmt.Errorf("%s failed: %s", tool, line) + } + return "", fmt.Errorf("%s failed: %s", tool, err) + } + return stdout.buf.String(), nil +} + +// firstLine returns the first non-empty line of b, trimmed of its +// trailing newline, or "" if b holds nothing but blank lines. +func firstLine(b []byte) string { + for _, line := range bytes.Split(b, []byte("\n")) { + if len(bytes.TrimSpace(line)) > 0 { + return string(bytes.TrimRight(line, "\r")) + } + } + return "" +} + +// truncate returns s cut to at most n bytes, so text built from +// untrusted tool output has a hard, predictable bound on its length +// regardless of what the tool wrote. It may cut a multi-byte UTF-8 +// sequence in two; a trailing partial rune in a diagnostic error message +// is an acceptable cost for a byte bound that never slips. +func truncate(s string, n int) string { + if len(s) > n { + return s[:n] + } + return s +} + +// pdfText extracts text from a PDF with pdftotext, its output capped per +// budget(maxToolOutput, maxRead) (B1). +func (e *Extractor) pdfText(ctx context.Context, path string, maxRead int64) (string, error) { + if e.tools["pdftotext"] == "" { + return "", &ToolMissingError{Tool: "pdftotext"} + } + abs, err := filepath.Abs(path) + if err != nil { + return "", err + } + return e.run(ctx, budget(maxToolOutput, maxRead), "pdftotext", "-q", "-enc", "UTF-8", abs, "-") +} + +// legacyText extracts text from a legacy binary Office format (doc xls +// ppt) with the external tool toolExt names, its output capped per +// budget(maxToolOutput, maxRead) (B1). .doc prefers antiword, falling +// back to catdoc if antiword is absent or fails. +func (e *Extractor) legacyText(ctx context.Context, path, ext string, maxRead int64) (string, error) { + abs, err := filepath.Abs(path) + if err != nil { + return "", err + } + out := budget(maxToolOutput, maxRead) + + switch ext { + case "doc": + haveAntiword := e.tools["antiword"] != "" + haveCatdoc := e.tools["catdoc"] != "" + if !haveAntiword && !haveCatdoc { + return "", &ToolMissingError{Tool: "antiword or catdoc"} + } + if haveAntiword { + text, err := e.run(ctx, out, "antiword", "-m", "UTF-8.txt", abs) + if err == nil { + return text, nil + } + if !haveCatdoc { + return "", err + } + } + return e.run(ctx, out, "catdoc", "-d", "utf-8", abs) + case "xls": + if e.tools["xls2csv"] == "" { + return "", &ToolMissingError{Tool: "xls2csv"} + } + return e.run(ctx, out, "xls2csv", "-d", "utf-8", abs) + case "ppt": + if e.tools["catppt"] == "" { + return "", &ToolMissingError{Tool: "catppt"} + } + return e.run(ctx, out, "catppt", "-d", "utf-8", abs) + default: + return "", errors.New("extract: unreachable: legacyText called with unknown extension " + ext) + } +} diff --git a/internal/extract/tools_test.go b/internal/extract/tools_test.go new file mode 100644 index 0000000..4a03f1b --- /dev/null +++ b/internal/extract/tools_test.go @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package extract + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +// fakeTool writes an executable shell script named name into dir. +func fakeTool(t *testing.T, dir, name, body string) { + t.Helper() + script := "#!/bin/sh\n" + body + "\n" + if err := os.WriteFile(filepath.Join(dir, name), []byte(script), 0o755); err != nil { + t.Fatal(err) + } +} + +func TestPdfViaFakeTool(t *testing.T) { + bin := t.TempDir() + argsFile := filepath.Join(t.TempDir(), "args") + fakeTool(t, bin, "pdftotext", `printf '%s\n' "$@" > "`+argsFile+`"; echo "acme ltd invoice"`) + e := newWithPath(bin) + p := file(t, "-leading-dash.pdf", []byte("%PDF-1.4")) + got, err := text(t, e, p, 0) + if err != nil || strings.TrimSpace(got) != "acme ltd invoice" { + t.Fatalf("got %q, %v", got, err) + } + args, _ := os.ReadFile(argsFile) + want := "-q\n-enc\nUTF-8\n" + p + "\n-\n" + if string(args) != want { + t.Fatalf("pdftotext args:\n%q\nwant\n%q", args, want) + } + if !filepath.IsAbs(strings.Split(string(args), "\n")[3]) { + t.Fatal("path argument is not absolute") + } +} + +func TestLegacyFormats(t *testing.T) { + bin := t.TempDir() + fakeTool(t, bin, "catdoc", `echo "from catdoc"`) + fakeTool(t, bin, "xls2csv", `echo "from xls2csv"`) + fakeTool(t, bin, "catppt", `echo "from catppt"`) + e := newWithPath(bin) // no antiword: .doc falls back to catdoc + for ext, want := range map[string]string{"doc": "from catdoc", "xls": "from xls2csv", "ppt": "from catppt"} { + got, err := text(t, e, file(t, "f."+ext, []byte("x")), 0) + if err != nil || strings.TrimSpace(got) != want { + t.Errorf("%s: got %q, %v", ext, got, err) + } + } + fakeTool(t, bin, "antiword", `echo "from antiword"`) + e = newWithPath(bin) + if got, _ := text(t, e, file(t, "g.doc", []byte("x")), 0); strings.TrimSpace(got) != "from antiword" { + t.Errorf("antiword not preferred: %q", got) + } + fakeTool(t, bin, "antiword", `echo "antiword broke" >&2; exit 1`) + e = newWithPath(bin) + if got, _ := text(t, e, file(t, "h.doc", []byte("x")), 0); strings.TrimSpace(got) != "from catdoc" { + t.Errorf("no fallback to catdoc after antiword failed: %q", got) + } +} + +func TestToolErrors(t *testing.T) { + e := newWithPath(t.TempDir()) + var tm *ToolMissingError + if _, err := text(t, e, file(t, "a.pdf", []byte("x")), 0); !errors.As(err, &tm) || tm.Error() != "needs pdftotext, not installed" { + t.Errorf("missing pdftotext: %v", err) + } + if _, err := text(t, e, file(t, "a.doc", []byte("x")), 0); err == nil || err.Error() != "needs antiword or catdoc, not installed" { + t.Errorf("missing doc tools: %v", err) + } + bin := t.TempDir() + fakeTool(t, bin, "pdftotext", `echo "Syntax Error: broken xref" >&2; exit 3`) + e = newWithPath(bin) + if _, err := text(t, e, file(t, "b.pdf", []byte("x")), 0); err == nil || err.Error() != "pdftotext failed: Syntax Error: broken xref" { + t.Errorf("failing tool: %v", err) + } + fakeTool(t, bin, "pdftotext", `sleep 5`) + e = newWithPath(bin) + e.Timeout = 200 * time.Millisecond + start := time.Now() + _, err := text(t, e, file(t, "c.pdf", []byte("x")), 0) + if err == nil || !strings.Contains(err.Error(), "pdftotext timed out after 200ms") { + t.Errorf("slow tool: %v", err) + } + if time.Since(start) > 3*time.Second { + t.Errorf("timeout took %v", time.Since(start)) + } +} + +// minimalPDF returns a valid one-page PDF showing text in Helvetica. +func minimalPDF(text string) []byte { + var b bytes.Buffer + var offsets []int + obj := func(s string) { offsets = append(offsets, b.Len()); b.WriteString(s) } + b.WriteString("%PDF-1.4\n") + obj("1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n") + obj("2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj\n") + obj("3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >> endobj\n") + stream := "BT /F1 24 Tf 72 700 Td (" + text + ") Tj ET" + obj(fmt.Sprintf("4 0 obj << /Length %d >> stream\n%s\nendstream endobj\n", len(stream), stream)) + obj("5 0 obj << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> endobj\n") + xref := b.Len() + fmt.Fprintf(&b, "xref\n0 %d\n0000000000 65535 f \n", len(offsets)+1) + for _, o := range offsets { + fmt.Fprintf(&b, "%010d 00000 n \n", o) + } + fmt.Fprintf(&b, "trailer << /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n", len(offsets)+1, xref) + return b.Bytes() +} + +func TestRealPdftotext(t *testing.T) { + if _, err := exec.LookPath("pdftotext"); err != nil { + t.Skip("pdftotext not installed") + } + e := New() + got, err := text(t, e, file(t, "hello.pdf", minimalPDF("Hello acme ltd")), 0) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(got, "Hello acme ltd") { + t.Fatalf("pdftotext gave %q", got) + } +} + +// TestOrphanChildDoesNotHangRun: a tool that exits itself but leaves a +// backgrounded grandchild holding its stdout pipe open must not make +// run() wait for that grandchild. Only os/exec itself, copying into +// cmd.Stdout and bounding the post-exit wait via WaitDelay, can end this +// promptly; run() manually draining a StdoutPipe before calling Wait +// starves WaitDelay of the thing it bounds, since by the time Wait runs +// there is nothing left for it to forcibly cut off. +func TestOrphanChildDoesNotHangRun(t *testing.T) { + bin := t.TempDir() + fakeTool(t, bin, "pdftotext", "sleep 5 & echo x") + e := newWithPath(bin) + e.Timeout = 30 * time.Second + + start := time.Now() + got, err := e.run(context.Background(), maxToolOutput, "pdftotext") + if err != nil { + t.Fatalf("orphan: %v", err) + } + if strings.TrimSpace(got) != "x" { + t.Errorf("orphan: got %q, want %q", got, "x") + } + if d := time.Since(start); d > 3*time.Second { + t.Errorf("orphan: took %v, want well under e.Timeout (30s) and under 3s", d) + } +} + +// TestStdoutOverflowKillsToolPromptly: a tool that keeps writing past +// maxToolOutput must be killed the moment the cap is crossed, not left +// running until e.Timeout expires — once run() stops draining its pipe, +// an unkilled tool blocks on its own write() and never exits on its own. +func TestStdoutOverflowKillsToolPromptly(t *testing.T) { + old := maxToolOutput + maxToolOutput = 1 << 20 + defer func() { maxToolOutput = old }() + + bin := t.TempDir() + fakeTool(t, bin, "pdftotext", "yes aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + e := newWithPath(bin) + e.Timeout = 30 * time.Second + + start := time.Now() + _, err := e.run(context.Background(), maxToolOutput, "pdftotext") + if !errors.Is(err, ErrTooLarge) { + t.Fatalf("overflow: got %v, want ErrTooLarge", err) + } + if d := time.Since(start); d > 5*time.Second { + t.Errorf("overflow: took %v, want under 5s (well under e.Timeout=30s)", d) + } +} + +// TestCallerCancelReturnsPromptly: cancelling the ctx passed to run() +// (distinct from e.Timeout's own internal deadline, which is untouched +// here) must stop the tool and return quickly, with an error wrapping +// the caller's own context.Canceled — not silently absorbed into a +// generic "<tool> failed: ..." string, and not held open until +// e.Timeout. +func TestCallerCancelReturnsPromptly(t *testing.T) { + bin := t.TempDir() + fakeTool(t, bin, "pdftotext", "sleep 10") + e := newWithPath(bin) + e.Timeout = 30 * time.Second + + ctx, cancel := context.WithCancel(context.Background()) + time.AfterFunc(100*time.Millisecond, cancel) + + start := time.Now() + _, err := e.run(ctx, maxToolOutput, "pdftotext") + if !errors.Is(err, context.Canceled) { + t.Fatalf("caller cancel: got %v, want an error wrapping context.Canceled", err) + } + if d := time.Since(start); d > 3*time.Second { + t.Errorf("caller cancel: took %v, want under 3s", d) + } +} + +// TestStderrFloodBounded: a tool that floods stderr must not blow up the +// size of the error message run() produces — only a bounded prefix of +// its first line may ever reach the returned error text, and capturing +// it at all must not cost unbounded memory. +func TestStderrFloodBounded(t *testing.T) { + bin := t.TempDir() + fakeTool(t, bin, "pdftotext", `head -c 10000000 /dev/zero | tr '\0' x >&2; exit 1`) + e := newWithPath(bin) + + _, err := e.run(context.Background(), maxToolOutput, "pdftotext") + if err == nil { + t.Fatal("stderr flood: want an error") + } + if max := len("pdftotext failed: ") + 200; len(err.Error()) > max { + t.Errorf("stderr flood: message is %d bytes, want <=%d", len(err.Error()), max) + } +} + +// TestMaxReadCapsToolOutput: B1. A directory's max-read, when smaller than +// the fixed maxToolOutput default, caps a single tool's output on its +// own — proven here with maxToolOutput left at its default, so only +// budget(maxToolOutput, maxRead) picking the smaller maxRead explains the +// result. +func TestMaxReadCapsToolOutput(t *testing.T) { + bin := t.TempDir() + fakeTool(t, bin, "pdftotext", `yes aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa | head -c 200000`) + e := newWithPath(bin) + if _, err := text(t, e, file(t, "small.pdf", []byte("%PDF-1.4")), 1024); !errors.Is(err, ErrTooLarge) { + t.Fatalf("got %v, want ErrTooLarge (max-read 1024 should have capped a 200000-byte tool output)", err) + } +} diff --git a/internal/extract/zipxml.go b/internal/extract/zipxml.go new file mode 100644 index 0000000..1b3ba82 --- /dev/null +++ b/internal/extract/zipxml.go @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package extract + +import ( + "archive/zip" + "context" + "encoding/xml" + "errors" + "fmt" + "io" + "path" + "strings" +) + +// zipBudget caps the total uncompressed bytes read from one archive's +// matched entries, guarding against a zip bomb; a test lowers it. B1: the +// directory's max-read may cap a single archive further still — see +// budget (tools.go). +var zipBudget int64 = 64 << 20 + +// zipPatterns maps a zip-based format's extension to the path.Match +// patterns (tried in matchEntry) of the entries that carry its text. +// Invoices often carry the tax number only in a header or footer, which is +// why docx's header*/footer* entries are included; ODF keeps headers and +// footers in styles.xml, not content.xml, so both are read. +var zipPatterns = map[string][]string{ + "docx": {"word/document.xml", "word/header*.xml", "word/footer*.xml", "word/footnotes.xml"}, + "xlsx": {"xl/sharedStrings.xml", "xl/worksheets/sheet*.xml"}, + "pptx": {"ppt/slides/slide*.xml"}, + "odt": {"content.xml", "styles.xml"}, + "ods": {"content.xml", "styles.xml"}, + "odp": {"content.xml", "styles.xml"}, + "epub": {"*.xhtml", "*.html", "*.htm"}, +} + +// zipText extracts text from a zip-based document (docx xlsx pptx odt ods +// odp epub): it opens the archive and, for each entry matching the +// format's zipPatterns in archive order, decodes its XML character data +// into the result with xmlText, separating entries with a newline. A +// corrupt archive returns the zip package's own error, not +// ErrUnsupported: the file claims a format it does not have, which the +// user should see. Reading stops with ErrTooLarge immediately once the +// entries read from the archive exceed budget(zipBudget, maxRead) +// uncompressed bytes in total (B1: maxRead, the directory's configured +// ceiling, may cap this lower than the fixed zipBudget). ctx is checked +// between entries so a cancelled extraction stops +// promptly. +// +// Any other per-entry failure — the entry won't open, or its XML is +// malformed — is lenient rather than fatal: whatever text that entry had +// already yielded (xmlText writes as it walks, so a syntax error partway +// through still leaves the text read up to that point) is kept, and the +// archive keeps going to its remaining entries, since one bad part (a +// corrupt header, say) should not blank out a document's otherwise +// readable body. The first such error is remembered, wrapped as "<entry +// name>: <err>", and returned only if no matched entry ever wrote any +// character data at all — an error report is more useful than silent +// empty text when nothing could be read. "Wrote any character data" is +// tracked per entry (via the builder's length just before and after that +// entry's own xmlText call, not the whole archive's final length): the +// newline zipText adds to separate a successful entry from the next one +// would otherwise make an entry that parsed cleanly but held no text of +// its own (an empty element, say) look like it had produced something, +// which could then mask a later entry's genuine failure. +func zipText(ctx context.Context, path, ext string, maxRead int64) (string, error) { + zr, err := zip.OpenReader(path) + if err != nil { + return "", err + } + defer zr.Close() + + patterns := zipPatterns[ext] + var b strings.Builder + remaining := budget(zipBudget, maxRead) + var firstErr error + wroteText := false + for _, f := range zr.File { + if err := ctx.Err(); err != nil { + return "", err + } + if !matchEntry(patterns, f.Name) { + continue + } + before := b.Len() + err := readZipEntry(f, &remaining, &b) + // Measured before the separator below is written, so a + // separator alone (an entry that parsed but held no character + // data) never counts as "wrote text" — only xmlText's own + // writes do, whether or not this entry went on to error. + if b.Len() > before { + wroteText = true + } + if err != nil { + if errors.Is(err, ErrTooLarge) { + return "", ErrTooLarge + } + if firstErr == nil { + firstErr = fmt.Errorf("%s: %w", f.Name, err) + } + continue + } + b.WriteByte('\n') + } + if !wroteText && firstErr != nil { + return "", firstErr + } + return b.String(), nil +} + +// matchEntry reports whether name is one of the entries a format reads: +// each pattern is tried first against the full entry name — which is what +// the docx/xlsx/pptx/odt directory-qualified patterns need — and, failing +// that, against name's base name. The base-name fallback is what lets +// epub's bare "*.xhtml"/"*.html"/"*.htm" find chapters nested at any depth +// inside the archive; applied to every format, it also means a nested +// part sharing a matched base name is picked up deliberately, not by +// accident — e.g. an ODF embedded object's own "Object 1/content.xml" +// matches odt/ods/odp's bare "content.xml" pattern alongside the +// document's own content.xml, because an embedded chart's or formula's +// text is text the document shows its reader. +func matchEntry(patterns []string, name string) bool { + base := path.Base(name) + for _, p := range patterns { + if ok, _ := path.Match(p, name); ok { + return true + } + if ok, _ := path.Match(p, base); ok { + return true + } + } + return false +} + +// readZipEntry opens one matched zip entry, decodes its text into b +// through a budgetedReader sharing remaining across the whole archive, and +// reports ErrTooLarge if that budget was exceeded — checked on the reader +// itself after xmlText returns, since the XML decoder may not pass the +// reader's own error through unchanged (a truncated entry can look like a +// cleanly finished document). +func readZipEntry(f *zip.File, remaining *int64, b *strings.Builder) error { + rc, err := f.Open() + if err != nil { + return err + } + defer rc.Close() + + br := &budgetedReader{r: rc, remaining: remaining} + err = xmlText(br, b) + if br.exceeded { + return ErrTooLarge + } + return err +} + +// budgetedReader wraps a zip entry's reader, decrementing remaining — a +// counter shared across every entry read from one archive — as bytes are +// read. Once remaining is exhausted it stops reading and reports io.EOF +// instead, recording that in exceeded so the caller can tell a genuine +// end of document from a budget cutoff. +type budgetedReader struct { + r io.Reader + remaining *int64 + exceeded bool +} + +func (br *budgetedReader) Read(p []byte) (int, error) { + if *br.remaining <= 0 { + br.exceeded = true + return 0, io.EOF + } + if int64(len(p)) > *br.remaining { + p = p[:*br.remaining] + } + n, err := br.r.Read(p) + *br.remaining -= int64(n) + return n, err +} + +// xmlText appends the character data of one XML entry to b, with the +// separators described above. inSharedCell tracks xlsx <c t="s">. +func xmlText(r io.Reader, b *strings.Builder) error { + dec := xml.NewDecoder(r) + dec.Strict = false + dec.Entity = xml.HTMLEntity + sharedCell, inV := false, false + for { + tok, err := dec.Token() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + switch t := tok.(type) { + case xml.StartElement: + switch t.Name.Local { + case "s", "tab", "br", "line-break", "cr": + b.WriteByte(' ') + case "c": + sharedCell = false + for _, a := range t.Attr { + if a.Name.Local == "t" && a.Value == "s" { + sharedCell = true + } + } + case "v": + inV = true + } + case xml.EndElement: + switch t.Name.Local { + case "p", "h", "tc", "tr", "td", "th", "li", "si", "c", "row", "div", "title": + b.WriteByte('\n') + case "v": + inV = false + } + case xml.CharData: + if inV && sharedCell { + continue + } + b.Write(t) + } + } +} diff --git a/internal/extract/zipxml_test.go b/internal/extract/zipxml_test.go new file mode 100644 index 0000000..14ead80 --- /dev/null +++ b/internal/extract/zipxml_test.go @@ -0,0 +1,205 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package extract + +import ( + "archive/zip" + "bytes" + "errors" + "strings" + "testing" +) + +// zipFile builds an archive from name -> content pairs and writes it to disk. +func zipFile(t *testing.T, name string, entries map[string]string) string { + t.Helper() + var buf bytes.Buffer + w := zip.NewWriter(&buf) + for n, c := range entries { + f, err := w.Create(n) + if err != nil { + t.Fatal(err) + } + if _, err := f.Write([]byte(c)); err != nil { + t.Fatal(err) + } + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return file(t, name, buf.Bytes()) +} + +func TestZipFormats(t *testing.T) { + e := newWithPath("") + tests := []struct { + name string + entries map[string]string + want []string + }{ + {"inv.docx", map[string]string{ + "word/document.xml": `<w:document xmlns:w="w"><w:body><w:p><w:r><w:t>Ac</w:t></w:r><w:r><w:t>me Ltd</w:t></w:r></w:p><w:p><w:r><w:t>Faktura</w:t><w:tab/><w:t>VAT</w:t></w:r></w:p></w:body></w:document>`, + "word/footer1.xml": `<w:ftr xmlns:w="w"><w:p><w:r><w:t>NIP 0000000000</w:t></w:r></w:p></w:ftr>`, + "word/styles.xml": `<w:styles xmlns:w="w"><w:t>NOT-INCLUDED</w:t></w:styles>`, + }, []string{"Acme Ltd", "Faktura VAT", "NIP 0000000000"}}, + {"sheet.xlsx", map[string]string{ + "xl/sharedStrings.xml": `<sst><si><t>Invoice</t></si><si><t>acme ltd</t></si></sst>`, + "xl/worksheets/sheet1.xml": `<worksheet><sheetData><row><c t="s"><v>0</v></c><c><v>1234567890</v></c><c t="inlineStr"><is><t>inline text</t></is></c></row></sheetData></worksheet>`, + }, []string{"Invoice", "acme ltd", "1234567890", "inline text"}}, + {"deck.pptx", map[string]string{ + "ppt/slides/slide1.xml": `<p:sld xmlns:p="p" xmlns:a="a"><a:p><a:r><a:t>Quarterly report</a:t></a:r></a:p></p:sld>`, + }, []string{"Quarterly report"}}, + {"letter.odt", map[string]string{ + "content.xml": `<office:document-content xmlns:office="o" xmlns:text="t"><text:p>Faktura<text:s/>VAT</text:p></office:document-content>`, + "styles.xml": `<office:document-styles xmlns:office="o" xmlns:text="t"><text:p>header acme</text:p></office:document-styles>`, + }, []string{"Faktura VAT", "header acme"}}, + {"book.epub", map[string]string{ + "OEBPS/ch1.xhtml": `<html xmlns="h"><body><p>Chapter one&two</p></body></html>`, + "mimetype": `application/epub+zip`, + }, []string{"Chapter one&two"}}, + } + for _, tt := range tests { + got, err := text(t, e, zipFile(t, tt.name, tt.entries), 0) + if err != nil { + t.Errorf("%s: %v", tt.name, err) + continue + } + for _, w := range tt.want { + if !strings.Contains(got, w) { + t.Errorf("%s: text %q lacks %q", tt.name, got, w) + } + } + if strings.Contains(got, "NOT-INCLUDED") { + t.Errorf("%s: read an entry it should skip: %q", tt.name, got) + } + } +} + +func TestSharedStringIndexNotText(t *testing.T) { + e := newWithPath("") + got, err := text(t, e, zipFile(t, "s.xlsx", map[string]string{ + "xl/sharedStrings.xml": `<sst><si><t>alpha</t></si></sst>`, + "xl/worksheets/sheet1.xml": `<worksheet><sheetData><row><c t="s"><v>987654</v></c></row></sheetData></worksheet>`, + }), 0) + if err != nil { + t.Fatal(err) + } + if strings.Contains(got, "987654") { + t.Fatalf("shared-string index leaked into text: %q", got) + } +} + +func TestZipCorruptAndBudget(t *testing.T) { + e := newWithPath("") + if _, err := text(t, e, file(t, "broken.docx", []byte("not a zip at all")), 0); err == nil || errors.Is(err, ErrUnsupported) { + t.Errorf("corrupt docx: %v, want a zip error", err) + } + old := zipBudget + zipBudget = 1 << 10 + defer func() { zipBudget = old }() + big := `<w:document xmlns:w="w"><w:p><w:t>` + strings.Repeat("x", 4096) + `</w:t></w:p></w:document>` + if _, err := text(t, e, zipFile(t, "huge.docx", map[string]string{"word/document.xml": big}), 0); !errors.Is(err, ErrTooLarge) { + t.Errorf("over budget: %v, want ErrTooLarge", err) + } +} + +// TestOdfEmbeddedObjectIncluded: an ODF embedded object (a chart, a +// formula) keeps its own content.xml inside a subdirectory such as +// "Object 1/"; matchEntry's base-name fallback picks it up alongside the +// document's own content.xml, deliberately — that embedded text is text +// the document shows its reader. +func TestOdfEmbeddedObjectIncluded(t *testing.T) { + e := newWithPath("") + got, err := text(t, e, zipFile(t, "embed.odt", map[string]string{ + "content.xml": `<office:document-content xmlns:office="o" xmlns:text="t"><text:p>cover page</text:p></office:document-content>`, + "Object 1/content.xml": `<office:document-content xmlns:office="o" xmlns:text="t"><text:p>embedded chart acme</text:p></office:document-content>`, + }), 0) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(got, "embedded chart acme") { + t.Errorf("embedded object text missing: %q", got) + } +} + +// TestZipLenientOnMalformedEntry: a malformed entry must not blank out +// text already read from a good entry in the same archive, and the text +// the malformed entry itself yielded before its own error is kept too. +// The footer's mismatched end tag (</w:xyz> where </w:ftr> was open) +// genuinely fails to parse under Strict=false — unlike a merely missing +// end tag, which the decoder synthesises and swallows — but only after +// "broken" and the newline for </w:p> have already been written; the +// XML decoder itself confirms this token by token (see fix round 2's +// report for the trace): CharData "broken", EndElement p, then the +// error "unexpected end element </xyz>". +func TestZipLenientOnMalformedEntry(t *testing.T) { + e := newWithPath("") + got, err := text(t, e, zipFile(t, "partial.docx", map[string]string{ + "word/document.xml": `<w:document xmlns:w="w"><w:body><w:p><w:r><w:t>good body text</w:t></w:r></w:p></w:body></w:document>`, + "word/footer1.xml": `<w:ftr xmlns:w="w"><w:p><w:r><w:t>broken</w:t></w:r></w:p></w:xyz>`, + }), 0) + if err != nil { + t.Fatalf("good entry alongside a malformed one: %v", err) + } + if !strings.Contains(got, "good body text") { + t.Errorf("text from the good entry was discarded: %q", got) + } + if !strings.Contains(got, "broken") { + t.Errorf("text the malformed entry yielded before its own error was discarded: %q", got) + } +} + +// TestZipMalformedOnlyEntryErrors: when the only matched entry is +// malformed, no text survives to return, so the error is reported instead +// — named after the entry it came from. The malformed attribute syntax +// breaks the decode before any character data is ever emitted, so there +// is nothing for the lenient path (TestZipLenientOnMalformedEntry) to +// keep. +func TestZipMalformedOnlyEntryErrors(t *testing.T) { + e := newWithPath("") + _, err := text(t, e, zipFile(t, "bad.docx", map[string]string{ + "word/document.xml": `<w:document><w:body attr="unterminated><w:p><w:t>oops</w:t></w:p></w:body></w:document>`, + }), 0) + if err == nil { + t.Fatal("malformed only entry: want an error, got nil") + } + if !strings.Contains(err.Error(), "word/document.xml") { + t.Errorf("error %q does not name the entry", err.Error()) + } +} + +// TestZipNoTextAndFailingSiblingErrors: a well-formed entry that simply +// has no character data (an empty <w:body/>) must not count as "text was +// found" and mask a failing sibling's error — the separator newline +// zipText appends after every successful entry means b.Len() alone +// cannot answer "did anything write text"; only word/footer1.xml's own +// contribution (zero, since it fails while still inside its opening tag, +// before any token is emitted) may be counted, and it contributed +// nothing either, so the archive as a whole produced no text and the +// wrapped error must surface. +func TestZipNoTextAndFailingSiblingErrors(t *testing.T) { + e := newWithPath("") + _, err := text(t, e, zipFile(t, "empty.docx", map[string]string{ + "word/document.xml": `<w:document xmlns:w="w"><w:body/></w:document>`, + "word/footer1.xml": `<w:ftr xmlns:w="w" a="unterminated><w:p/></w:ftr>`, + }), 0) + if err == nil { + t.Fatal("textless entry plus a failing sibling: want an error, got nil") + } + if !strings.Contains(err.Error(), "word/footer1.xml") { + t.Errorf("error %q does not name the failing entry", err.Error()) + } +} + +// TestMaxReadCapsZipOutput: B1. A directory's max-read, when smaller than +// the fixed zipBudget default, caps a single archive's extracted text on +// its own — zipBudget is left at its default, so only budget(zipBudget, +// maxRead) picking the smaller maxRead explains the result. +func TestMaxReadCapsZipOutput(t *testing.T) { + e := newWithPath("") + big := `<w:document xmlns:w="w"><w:p><w:t>` + strings.Repeat("x", 4096) + `</w:t></w:p></w:document>` + p := zipFile(t, "huge.docx", map[string]string{"word/document.xml": big}) + if _, err := text(t, e, p, 1024); !errors.Is(err, ErrTooLarge) { + t.Fatalf("got %v, want ErrTooLarge (max-read 1024 should have capped a 4096-byte entry)", err) + } +} diff --git a/internal/ignore/ignore.go b/internal/ignore/ignore.go new file mode 100644 index 0000000..fff6dcc --- /dev/null +++ b/internal/ignore/ignore.go @@ -0,0 +1,324 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +// Package ignore implements a gitignore(5)-compatible path matcher. +package ignore + +import ( + "errors" + "fmt" + "regexp" + "strings" +) + +// pattern is one compiled gitignore(5) pattern. +type pattern struct { + re *regexp.Regexp + negate bool + dirOnly bool +} + +// Matcher decides which paths a set of gitignore(5) patterns ignores. +type Matcher struct { + patterns []pattern +} + +// New compiles patterns in gitignore(5) syntax, in order. Blank patterns and +// patterns starting with "#" are skipped. A nil or empty list gives a +// matcher that matches nothing. +func New(patterns []string) (*Matcher, error) { + m := &Matcher{} + for _, orig := range patterns { + p := orig + if p == "" || strings.HasPrefix(p, "#") { + continue + } + negate := false + switch { + case strings.HasPrefix(p, "!"): + negate = true + p = p[1:] + case strings.HasPrefix(p, `\!`), strings.HasPrefix(p, `\#`): + p = p[1:] + } + p = trimTrailingSpaces(p) + dirOnly := false + if strings.HasSuffix(p, "/") { + dirOnly = true + p = strings.TrimSuffix(p, "/") + } + reStr, err := toRegexp(p) + if err != nil { + return nil, fmt.Errorf("bad ignore pattern %q: %v", orig, err) + } + re, err := regexp.Compile(reStr) + if err != nil { + return nil, fmt.Errorf("bad ignore pattern %q: %v", orig, err) + } + m.patterns = append(m.patterns, pattern{re: re, negate: negate, dirOnly: dirOnly}) + } + return m, nil +} + +// trimTrailingSpaces removes trailing spaces that are not escaped by a +// preceding backslash, per gitignore(5). +func trimTrailingSpaces(p string) string { + for strings.HasSuffix(p, " ") { + n := 0 + for i := len(p) - 2; i >= 0 && p[i] == '\\'; i-- { + n++ + } + if n%2 == 1 { + break // the trailing space is escaped: keep it + } + p = p[:len(p)-1] + } + return p +} + +// Match reports whether rel is ignored. rel is slash-separated and relative +// to the root, with no leading slash and no trailing slash, and each of its +// path components must be "." and ".."-free (as produced by a directory +// walk, not a raw user-typed path). +func (m *Matcher) Match(rel string, isDir bool) bool { + parts := strings.Split(rel, "/") + for i := 1; i < len(parts); i++ { + ancestor := strings.Join(parts[:i], "/") + if m.matchOne(ancestor, true) { + return true + } + } + return m.matchOne(rel, isDir) +} + +// matchOne applies every pattern in order to one path and returns the final +// ignored state, without considering ancestor directories. +func (m *Matcher) matchOne(path string, isDir bool) bool { + ignored := false + for _, pat := range m.patterns { + if pat.dirOnly && !isDir { + continue + } + if pat.re.MatchString(path) { + ignored = !pat.negate + } + } + return ignored +} + +// toRegexp translates the body of one gitignore pattern (no leading "!", +// no trailing "/") into a regular expression over slash-separated paths. +func toRegexp(p string) (string, error) { + anchored := strings.Contains(p, "/") + p = strings.TrimPrefix(p, "/") + var b strings.Builder + b.WriteString("^") + if !anchored { + b.WriteString("(?:.*/)?") + } + for i := 0; i < len(p); i++ { + c := p[i] + switch { + case strings.HasPrefix(p[i:], "**/") && (i == 0 || p[i-1] == '/'): + b.WriteString("(?:.*/)?") + i += 2 + case strings.HasPrefix(p[i:], "/**") && i+3 == len(p): + b.WriteString("/.*") + i += 2 + case strings.HasPrefix(p[i:], "**"): + b.WriteString("[^/]*") + i++ + case c == '*': + b.WriteString("[^/]*") + case c == '?': + b.WriteString("[^/]") + case c == '[': + negated := false + start := i + 1 + if start < len(p) && (p[start] == '!' || p[start] == '^') { + negated = true + start++ + } + items, end, err := parseClassBody(p, start) + if err != nil { + return "", err + } + b.WriteString(renderClass(items, negated)) + i = end + case c == '\\' && i+1 < len(p): + i++ + b.WriteString(regexp.QuoteMeta(string(p[i]))) + default: + b.WriteString(regexp.QuoteMeta(string(c))) + } + } + b.WriteString("$") + return b.String(), nil +} + +// classItem is one member of a gitignore bracket expression: either a +// verbatim POSIX bracket subexpression ("[:digit:]", "[.ch.]", "[=e=]"), or +// a literal byte (lo == hi) or a byte range (lo-hi). +type classItem struct { + posix string + lo, hi byte +} + +// parseClassBody scans p starting at start (just past "[", "[!" or "[^") +// for the members of a bracket expression, up to and including the +// matching unescaped "]". A "]" appearing immediately at start is a +// literal member, not the terminator, per glob syntax (e.g. "[]a]" matches +// "]" or "a"). Inside the class, "\x" escapes x to a literal member, +// neutralizing any meaning x would otherwise have (closing the class, +// starting a POSIX subexpression, forming a range) — matching git's +// wildmatch. It returns the members found and the index of the closing +// "]", or an error if none is found (including a class left unterminated +// because its only "]" was escaped). +func parseClassBody(p string, start int) ([]classItem, int, error) { + var items []classItem + k := start + if k < len(p) && p[k] == ']' { + items = append(items, classItem{lo: ']', hi: ']'}) + k++ + } + for k < len(p) && p[k] != ']' { + // "\x" escapes x to a literal member; it can never start a + // POSIX subexpression or a range. A backslash with nothing + // after it can never close the class either. + if p[k] == '\\' { + if k+1 >= len(p) { + return nil, 0, errors.New("unterminated [") + } + items = append(items, classItem{lo: p[k+1], hi: p[k+1]}) + k += 2 + continue + } + if p[k] == '[' && k+1 < len(p) && (p[k+1] == ':' || p[k+1] == '.' || p[k+1] == '=') { + delim := p[k+1] + rest := strings.Index(p[k+2:], string(delim)+"]") + if rest < 0 { + return nil, 0, errors.New("unterminated [") + } + stop := k + 2 + rest + 2 + items = append(items, classItem{posix: p[k:stop]}) + k = stop + continue + } + // "x-y" is a range unless the "-" is the last character before + // the closing "]", in which case it is a literal "-". + if k+2 < len(p) && p[k+1] == '-' && p[k+2] != ']' { + items = append(items, classItem{lo: p[k], hi: p[k+2]}) + k += 3 + continue + } + items = append(items, classItem{lo: p[k], hi: p[k]}) + k++ + } + if k >= len(p) { + return nil, 0, errors.New("unterminated [") + } + return items, k, nil +} + +// renderClass turns the members of a bracket expression into a Go regexp +// character class. Git's bracket expressions never match "/", regardless +// of negation, so "/" is dropped from a positive class (splitting any range +// that spans it) and added to a negated one. A positive class left +// matching nothing (e.g. "[/]") is rendered as a class that can never +// match any character, since RE2 has no empty class or lookahead. +func renderClass(items []classItem, negated bool) string { + if negated { + items = append(items, classItem{lo: '/', hi: '/'}) + return "[^" + renderClassItems(items) + "]" + } + items = excludeSlash(items) + if len(items) == 0 { + return `[^\x00-\x{10FFFF}]` + } + return "[" + renderClassItems(items) + "]" +} + +// posixSlashFree gives the ASCII definition, minus "/", of every standard +// POSIX bracket class whose definition otherwise includes it: graph and +// print (visible characters, "/" among them) and punct (documented in +// gitignore's own toRegexp derivation as "!-. :-@ [-` {-~", i.e. punct's +// usual "!-/" range with "/" trimmed to "!-."). The other nine standard +// classes (alnum, alpha, blank, cntrl, digit, lower, space, upper, +// xdigit) never contain "/" and so need no rewriting. +var posixSlashFree = map[string][]classItem{ + "graph": {{lo: '!', hi: '.'}, {lo: '0', hi: '~'}}, + "print": {{lo: ' ', hi: '.'}, {lo: '0', hi: '~'}}, + "punct": {{lo: '!', hi: '.'}, {lo: ':', hi: '@'}, {lo: '[', hi: '`'}, {lo: '{', hi: '~'}}, +} + +// posixClassName returns the name inside a "[:name:]" POSIX bracket class, +// or "", false for anything else (a collating symbol "[.x.]", an +// equivalence class "[=x=]", or a malformed value). +func posixClassName(posix string) (string, bool) { + if strings.HasPrefix(posix, "[:") && strings.HasSuffix(posix, ":]") { + return posix[2 : len(posix)-2], true + } + return "", false +} + +// excludeSlash removes "/" from a positive class's members, splitting any +// range that spans it into the parts on either side, and substituting the +// "/"-free ASCII definition for any POSIX class that would otherwise +// include it. +func excludeSlash(items []classItem) []classItem { + var out []classItem + for _, it := range items { + switch { + case it.posix != "": + if name, ok := posixClassName(it.posix); ok { + if repl, hasSlash := posixSlashFree[name]; hasSlash { + out = append(out, repl...) + continue + } + } + out = append(out, it) + case it.lo == '/' && it.hi == '/': + // drop the lone literal "/" + case it.lo <= '/' && '/' <= it.hi: + if it.lo <= '/'-1 { + out = append(out, classItem{lo: it.lo, hi: '/' - 1}) + } + if '/'+1 <= it.hi { + out = append(out, classItem{lo: '/' + 1, hi: it.hi}) + } + default: + out = append(out, it) + } + } + return out +} + +func renderClassItems(items []classItem) string { + var b strings.Builder + for _, it := range items { + switch { + case it.posix != "": + b.WriteString(it.posix) + case it.lo == it.hi: + b.WriteString(classChar(it.lo)) + default: + b.WriteString(classChar(it.lo)) + b.WriteByte('-') + b.WriteString(classChar(it.hi)) + } + } + return b.String() +} + +// classChar escapes a byte that would otherwise be misread by the Go +// regexp parser when emitted as a member (or range endpoint) inside a +// character class: "]" would close the class, "^" would negate it if +// first, "-" would start a range, "[" could open a POSIX subexpression, +// and "\" always needs escaping to be literal. +func classChar(c byte) string { + switch c { + case '\\', ']', '^', '-', '[': + return "\\" + string(c) + default: + return string(c) + } +} diff --git a/internal/ignore/ignore_test.go b/internal/ignore/ignore_test.go new file mode 100644 index 0000000..04b60c4 --- /dev/null +++ b/internal/ignore/ignore_test.go @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package ignore + +import "testing" + +func TestMatch(t *testing.T) { + type c struct { + rel string + isDir bool + want bool + } + tests := []struct { + patterns []string + cases []c + }{ + {[]string{"*.log"}, []c{{"a.log", false, true}, {"dir/b.log", false, true}, {"a.txt", false, false}}}, + {[]string{"/a.txt"}, []c{{"a.txt", false, true}, {"dir/a.txt", false, false}}}, + {[]string{"build/"}, []c{{"build", true, true}, {"build", false, false}, {"src/build", true, true}, {"build/x.bin", false, true}}}, + {[]string{"doc/*.md"}, []c{{"doc/r.md", false, true}, {"doc/sub/r.md", false, false}, {"x/doc/r.md", false, false}}}, + {[]string{"**/readme.md"}, []c{{"readme.md", false, true}, {"a/b/readme.md", false, true}}}, + {[]string{"deep/**"}, []c{{"deep", true, false}, {"deep/a", true, true}, {"deep/a/b/f.tmp", false, true}}}, + {[]string{"x/**/z.txt"}, []c{{"x/z.txt", false, true}, {"x/y/z.txt", false, true}, {"x/y/w/z.txt", false, true}, {"q/x/z.txt", false, false}}}, + {[]string{"*.part", "!keep.part"}, []c{{"a.part", false, true}, {"keep.part", false, false}}}, + {[]string{"dir/", "!dir/c.txt"}, []c{{"dir/c.txt", false, true}}}, + {[]string{".*"}, []c{{".hidden", false, true}, {"a/.cfg", false, true}, {"visible", false, false}}}, + {[]string{"a?.txt", "[bc].txt"}, []c{{"a1.txt", false, true}, {"a12.txt", false, false}, {"b.txt", false, true}, {"d.txt", false, false}}}, + {[]string{"[!a]*.txt"}, []c{{"b.txt", false, true}, {"a.txt", false, false}}}, + {[]string{"sub"}, []c{{"sub", true, true}, {"x/sub", true, true}, {"x/sub/f", false, true}, {"subx", false, false}}}, + {[]string{"\\#notes", "\\!bang"}, []c{{"#notes", false, true}, {"!bang", false, true}}}, + {[]string{"", "# comment"}, []c{{"# comment", false, false}}}, + {nil, []c{{"anything", false, false}}}, + {[]string{"x[]a]y"}, []c{{"x]y", false, true}, {"xay", false, true}, {"xby", false, false}}}, + {[]string{"a[/]b"}, []c{{"a/b", false, false}}}, + } + for _, tt := range tests { + m, err := New(tt.patterns) + if err != nil { + t.Fatalf("New(%q): %v", tt.patterns, err) + } + for _, cs := range tt.cases { + if got := m.Match(cs.rel, cs.isDir); got != cs.want { + t.Errorf("patterns %q: Match(%q, dir=%v) = %v, want %v", tt.patterns, cs.rel, cs.isDir, got, cs.want) + } + } + } +} + +func TestBadPattern(t *testing.T) { + if _, err := New([]string{"[abc"}); err == nil || err.Error() != `bad ignore pattern "[abc": unterminated [` { + t.Fatalf("got %v", err) + } + // "\]" escapes the only "]" to a literal member, leaving the class + // with no terminator. + if _, err := New([]string{"a[\\]d"}); err == nil || err.Error() != `bad ignore pattern "a[\\]d": unterminated [` { + t.Fatalf("got %v", err) + } +} diff --git a/internal/ignore/oracle_test.go b/internal/ignore/oracle_test.go new file mode 100644 index 0000000..a5a589c --- /dev/null +++ b/internal/ignore/oracle_test.go @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package ignore + +import ( + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "testing" +) + +var oracleFiles = []string{ + "a.txt", "b.log", "a1.txt", "abc", ".hidden", "foo.part", "keep.part", + "dir/c.txt", "dir/d.log", "dir/.hidden2", "dir/sub/e.txt", + "build/out.bin", "src/build/x.go", "doc/readme.md", "doc/sub/readme.md", + "deep/a/b/c/file.tmp", "x/z.txt", "x/y/z.txt", "sub/f.txt", "space name.txt", + "x]y", "xay", "xby", "a/b", "1.txt", "]a.txt", + "a.b", "xyz", "aq", "bq", +} + +var oracleSets = [][]string{ + {"*.log"}, {"/a.txt"}, {"build/"}, {"doc/*.md"}, {"**/readme.md"}, + {"deep/**"}, {"x/**/z.txt"}, {"*.part", "!keep.part"}, {".*"}, + {"a?.txt", "[ab]*.txt"}, {"[!a]*.txt"}, {"dir/", "!dir/c.txt"}, + {"sub"}, {"space name.txt"}, {"*", "!*.txt"}, {"dir/**/*.txt"}, + {"x[]a]y"}, {"a[/]b"}, {"[[:digit:]]*.txt"}, {"[!]a]*"}, + {"a[[:punct:]]b"}, {"x[[:alpha:]]z"}, {"[a\\b]q"}, +} + +// TestAgainstGit compares every file and directory of a real tree against +// `git check-ignore --no-index` for each pattern set. +func TestAgainstGit(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + root := t.TempDir() + t.Setenv("HOME", root) + t.Setenv("GIT_CONFIG_GLOBAL", os.DevNull) + t.Setenv("GIT_CONFIG_NOSYSTEM", "1") + for _, f := range oracleFiles { + p := filepath.Join(root, f) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, nil, 0o644); err != nil { + t.Fatal(err) + } + } + if out, err := exec.Command("git", "-C", root, "init", "-q").CombinedOutput(); err != nil { + t.Fatalf("git init: %v %s", err, out) + } + // every path in the tree, files and directories, relative and slash-separated + var paths []string + isDir := map[string]bool{} + filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error { + rel, _ := filepath.Rel(root, p) + if rel == "." || rel == ".git" || strings.HasPrefix(rel, ".git"+string(filepath.Separator)) || rel == ".gitignore" { + if rel == ".git" { + return filepath.SkipDir + } + return nil + } + rel = filepath.ToSlash(rel) + paths = append(paths, rel) + isDir[rel] = d.IsDir() + return nil + }) + sort.Strings(paths) + for _, set := range oracleSets { + if err := os.WriteFile(filepath.Join(root, ".gitignore"), []byte(strings.Join(set, "\n")+"\n"), 0o644); err != nil { + t.Fatal(err) + } + cmd := exec.Command("git", "-C", root, "check-ignore", "--no-index", "--stdin") + cmd.Stdin = strings.NewReader(strings.Join(paths, "\n") + "\n") + out, _ := cmd.Output() // exit status 1 means "nothing ignored" + gitIgnored := map[string]bool{} + for _, l := range strings.Split(strings.TrimSpace(string(out)), "\n") { + if l != "" { + gitIgnored[strings.TrimSuffix(l, "/")] = true + } + } + m, err := New(set) + if err != nil { + t.Fatalf("New(%q): %v", set, err) + } + for _, p := range paths { + if got := m.Match(p, isDir[p]); got != gitIgnored[p] { + t.Errorf("patterns %q, path %q (dir=%v): krino says %v, git says %v", set, p, isDir[p], got, gitIgnored[p]) + } + } + } +} diff --git a/internal/norm/norm.go b/internal/norm/norm.go new file mode 100644 index 0000000..e7933ec --- /dev/null +++ b/internal/norm/norm.go @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +// Package norm puts text into the form krino compares it in: optionally +// without diacritics, optionally lower case, with white space collapsed. +package norm + +import ( + "strings" + "unicode" + + unorm "golang.org/x/text/unicode/norm" +) + +// special holds the letters that do not decompose under Unicode NFD, so +// Fold maps them explicitly. +var special = map[rune]string{ + 'ł': "l", 'Ł': "L", + 'ø': "o", 'Ø': "O", + 'đ': "d", 'Đ': "D", + 'ħ': "h", 'Ħ': "H", + 'ß': "ss", + 'æ': "ae", 'Æ': "AE", + 'œ': "oe", 'Œ': "OE", + 'ı': "i", +} + +// Fold strips diacritics: Unicode NFD, drop combining marks (category Mn), +// then map the letters that do not decompose. ASCII input is returned +// unchanged without allocating. +func Fold(s string) string { + ascii := true + for i := 0; i < len(s); i++ { + if s[i] >= 0x80 { + ascii = false + break + } + } + if ascii { + return s + } + + var b strings.Builder + b.Grow(len(s)) + for _, r := range unorm.NFD.String(s) { + if unicode.Is(unicode.Mn, r) { + continue + } + if rep, ok := special[r]; ok { + b.WriteString(rep) + continue + } + b.WriteRune(r) + } + return b.String() +} + +// Text puts s into the form content and keywords are compared in: Fold if +// fold, strings.ToLower if ignoreCase, then every run of Unicode white +// space becomes one ASCII space and both ends are trimmed. +func Text(s string, ignoreCase, fold bool) string { + if fold { + s = Fold(s) + } + if ignoreCase { + s = strings.ToLower(s) + } + + var b strings.Builder + b.Grow(len(s)) + inSpace := false + started := false + for _, r := range s { + if unicode.IsSpace(r) { + if started { + inSpace = true + } + continue + } + if inSpace { + b.WriteByte(' ') + inSpace = false + } + b.WriteRune(r) + started = true + } + return b.String() +} + +// Name puts a file name into the form it is matched against: Fold(s) if +// fold, else s. Case is handled by the regex flag, not here. +func Name(s string, fold bool) string { + if fold { + return Fold(s) + } + return s +} diff --git a/internal/norm/norm_test.go b/internal/norm/norm_test.go new file mode 100644 index 0000000..2fc35b6 --- /dev/null +++ b/internal/norm/norm_test.go @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package norm + +import "testing" + +func TestFold(t *testing.T) { + tests := map[string]string{ + "": "", + "plain ASCII 123": "plain ASCII 123", + "spółka z ograniczoną odpowiedzialnością": "spolka z ograniczona odpowiedzialnoscia", + "Łódź": "Lodz", + "ZAŻÓŁĆ GĘŚLĄ JAŹŃ": "ZAZOLC GESLA JAZN", + "Straße": "Strasse", + "Øresund": "Oresund", + "Ærø": "AEro", + "œuvre": "oeuvre", + "déjà vu": "deja vu", + "Đakovo": "Dakovo", + "ħelp": "help", + } + for in, want := range tests { + if got := Fold(in); got != want { + t.Errorf("Fold(%q) = %q, want %q", in, got, want) + } + } +} + +func TestText(t *testing.T) { + tests := []struct { + in string + ignoreCase, fold bool + want string + }{ + {" Hello\n\tWORLD ", true, false, "hello world"}, + {"A B\r\nC", false, false, "A B C"}, + {"ZAŻÓŁĆ gęślą", true, true, "zazolc gesla"}, + {"Spółka", false, true, "Spolka"}, + {"Faktura\u00A0VAT", true, false, "faktura vat"}, + {"", true, true, ""}, + {" ", true, true, ""}, + } + for _, tt := range tests { + if got := Text(tt.in, tt.ignoreCase, tt.fold); got != tt.want { + t.Errorf("Text(%q, %v, %v) = %q, want %q", tt.in, tt.ignoreCase, tt.fold, got, tt.want) + } + } +} + +func TestTextIdempotent(t *testing.T) { + for _, s := range []string{" Łódź\n\nMIASTO ", "a\tb", "Ærø Œuvre"} { + once := Text(s, true, true) + if twice := Text(once, true, true); twice != once { + t.Errorf("Text not idempotent on %q: %q then %q", s, once, twice) + } + } +} + +func TestName(t *testing.T) { + if got := Name("Spółka.PDF", true); got != "Spolka.PDF" { + t.Errorf("Name fold = %q", got) + } + if got := Name("Spółka.PDF", false); got != "Spółka.PDF" { + t.Errorf("Name no fold = %q", got) + } +} + +func TestFoldASCIINoAlloc(t *testing.T) { + s := "already plain ascii text" + if n := testing.AllocsPerRun(100, func() { _ = Fold(s) }); n != 0 { + t.Errorf("Fold allocates %v times on ASCII input", n) + } +} diff --git a/internal/scan/scan.go b/internal/scan/scan.go new file mode 100644 index 0000000..e87b741 --- /dev/null +++ b/internal/scan/scan.go @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +// Package scan walks a directory tree and reports the files krino will +// consider sorting, and why the rest were skipped. +package scan + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "time" + + "krino/internal/ignore" +) + +// File is a regular file found by Walk. +type File struct { + Path string // absolute, cleaned + Rel string // slash-separated, relative to the root + Name string // base name + Size int64 + ModTime time.Time + Mode fs.FileMode +} + +// Reason is why an entry was not returned as a File. +type Reason int + +const ( + Ignored Reason = iota // matched an ignore pattern + Busy // a sibling NAME<busy-suffix> exists + TooNew // modified less than MinAge ago + Symlink // a symbolic link (never followed) + NotRegular // fifo, socket, device + Unreadable // a directory that could not be read +) + +// String names a Reason the way it should read in a report. +func (r Reason) String() string { + switch r { + case Ignored: + return "ignored" + case Busy: + return "busy" + case TooNew: + return "too new" + case Symlink: + return "symlink" + case NotRegular: + return "not a regular file" + case Unreadable: + return "unreadable" + default: + return fmt.Sprintf("Reason(%d)", int(r)) + } +} + +// Skipped is one entry Walk did not return as a File, and why. +type Skipped struct { + Rel string + Reason Reason +} + +// Options controls how Walk traverses a directory. +type Options struct { + Recursive bool + MaxDepth int // 0: unlimited; 1: the root's own entries only + Ignore *ignore.Matcher // nil: nothing ignored + Exclude []string // absolute directories never entered + Busy []string // suffixes, e.g. ".part" + MinAge time.Duration + Now time.Time +} + +// Result is everything Walk found under a root. +type Result struct { + Files []File // sorted by Rel + Skipped []Skipped // sorted by Rel +} + +// Walk lists the files under root that krino will consider, and reports why +// the rest were skipped. root must exist, be a directory, and be readable; +// an unreadable subdirectory found during the walk is reported as +// Unreadable and does not abort the scan. +func Walk(root string, opt Options) (*Result, error) { + root, err := filepath.Abs(root) + if err != nil { + return nil, err + } + info, err := os.Stat(root) + if err != nil { + return nil, err + } + if !info.IsDir() { + return nil, fmt.Errorf("%s is not a directory", root) + } + entries, err := os.ReadDir(root) + if err != nil { + return nil, err + } + + exclude := make(map[string]bool, len(opt.Exclude)) + for _, e := range opt.Exclude { + exclude[filepath.Clean(e)] = true + } + + w := &walker{root: root, opt: opt, exclude: exclude} + if err := w.walk(root, "", 1, entries); err != nil { + return nil, err + } + + sort.Slice(w.result.Files, func(i, j int) bool { return w.result.Files[i].Rel < w.result.Files[j].Rel }) + sort.Slice(w.result.Skipped, func(i, j int) bool { return w.result.Skipped[i].Rel < w.result.Skipped[j].Rel }) + return &w.result, nil +} + +// walker accumulates the Result across recursive calls. +type walker struct { + opt Options + root string + exclude map[string]bool + result Result +} + +// walk applies the skip checks to entries, the already-read contents of dir +// (at relDir relative to the root, dir's own entries at depth). A +// subdirectory it recurses into is read here, right before recursing, so a +// ReadDir failure on it can be reported as Unreadable and skipped without +// aborting the rest of the walk; only a failure reading dir itself (passed +// in by the caller) would need to propagate, and only Walk's own read of +// the root works that way. +func (w *walker) walk(dir, relDir string, depth int, entries []os.DirEntry) error { + // The set of names in this directory, built once, for the busy check. + names := make(map[string]bool, len(entries)) + for _, e := range entries { + names[e.Name()] = true + } + + for _, e := range entries { + name := e.Name() + rel := name + if relDir != "" { + rel = relDir + "/" + name + } + path := filepath.Join(dir, name) + + // A symlink is never followed, whatever it points to; DirEntry's + // Type is Lstat-like and does not resolve it. + if e.Type()&fs.ModeSymlink != 0 { + w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: Symlink}) + continue + } + + if e.IsDir() { + // Directories themselves are not reported, with two + // exceptions: the symlink case above, and an ignored + // directory (C2) — reported once for the directory itself, + // not for each file inside it, since pruning it without + // descending is the whole point; without this, its contents + // would appear in no count and in no -v listing at all. + if !w.opt.Recursive { + continue + } + if w.opt.Ignore != nil && w.opt.Ignore.Match(rel, true) { + w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: Ignored}) + continue + } + if w.exclude[filepath.Clean(path)] { + continue + } + if w.opt.MaxDepth > 0 && depth+1 > w.opt.MaxDepth { + continue + } + subEntries, err := os.ReadDir(path) + if err != nil { + w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: Unreadable}) + continue + } + if err := w.walk(path, rel, depth+1, subEntries); err != nil { + return err + } + continue + } + + info, err := e.Info() + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + continue // vanished mid-walk (a download finishing, say): silently skipped + } + w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: Unreadable}) + continue + } + if !info.Mode().IsRegular() { + w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: NotRegular}) + continue + } + if w.opt.Ignore != nil && w.opt.Ignore.Match(rel, false) { + w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: Ignored}) + continue + } + if busy := w.isBusy(name, names); busy { + w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: Busy}) + continue + } + if w.opt.Now.Sub(info.ModTime()) < w.opt.MinAge { + w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: TooNew}) + continue + } + w.result.Files = append(w.result.Files, File{ + Path: path, + Rel: rel, + Name: name, + Size: info.Size(), + ModTime: info.ModTime(), + Mode: info.Mode(), + }) + } + return nil +} + +// isBusy reports whether name+suffix, for any configured Busy suffix, is +// among names — the sibling of an in-progress download. +func (w *walker) isBusy(name string, names map[string]bool) bool { + for _, suffix := range w.opt.Busy { + if names[name+suffix] { + return true + } + } + return false +} diff --git a/internal/scan/scan_test.go b/internal/scan/scan_test.go new file mode 100644 index 0000000..94f922b --- /dev/null +++ b/internal/scan/scan_test.go @@ -0,0 +1,263 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package scan + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "reflect" + "syscall" + "testing" + "time" + + "krino/internal/ignore" +) + +var now = time.Date(2026, 9, 11, 12, 0, 0, 0, time.UTC) + +// tree creates files (with an mtime one hour before now) and returns the root. +func tree(t *testing.T, files ...string) string { + t.Helper() + root := t.TempDir() + for _, f := range files { + p := filepath.Join(root, f) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(f), 0o644); err != nil { + t.Fatal(err) + } + old := now.Add(-time.Hour) + if err := os.Chtimes(p, old, old); err != nil { + t.Fatal(err) + } + } + return root +} + +func rels(r *Result) []string { + var out []string + for _, f := range r.Files { + out = append(out, f.Rel) + } + return out +} + +func skipped(r *Result) map[string]Reason { + m := map[string]Reason{} + for _, s := range r.Skipped { + m[s.Rel] = s.Reason + } + return m +} + +func TestTopLevelOnly(t *testing.T) { + root := tree(t, "a.pdf", "b.txt", "sub/c.txt") + r, err := Walk(root, Options{Now: now}) + if err != nil { + t.Fatal(err) + } + if want := []string{"a.pdf", "b.txt"}; !reflect.DeepEqual(rels(r), want) { + t.Fatalf("files = %v, want %v", rels(r), want) + } + f := r.Files[0] + if f.Name != "a.pdf" || f.Size != int64(len("a.pdf")) || f.Path != filepath.Join(root, "a.pdf") || !f.ModTime.Equal(now.Add(-time.Hour)) { + t.Fatalf("file = %+v", f) + } +} + +func TestRecursiveDepthExcludeIgnore(t *testing.T) { + root := tree(t, "a.txt", "one/b.txt", "one/two/c.txt", "Work/filed.pdf", "skip/x.txt", "keep/y.log") + m, _ := ignore.New([]string{"skip/", "*.log"}) + r, err := Walk(root, Options{Recursive: true, Ignore: m, Exclude: []string{filepath.Join(root, "Work")}, Now: now}) + if err != nil { + t.Fatal(err) + } + if want := []string{"a.txt", "one/b.txt", "one/two/c.txt"}; !reflect.DeepEqual(rels(r), want) { + t.Fatalf("files = %v, want %v", rels(r), want) + } + if got := skipped(r); got["keep/y.log"] != Ignored { + t.Fatalf("skipped = %v", got) + } + r, _ = Walk(root, Options{Recursive: true, MaxDepth: 2, Now: now}) + for _, rel := range rels(r) { + if rel == "one/two/c.txt" { + t.Fatalf("MaxDepth 2 reached depth 3: %v", rels(r)) + } + } +} + +func TestBusyTooNewSymlinkFifo(t *testing.T) { + root := tree(t, "movie.mkv", "movie.mkv.aria2", "doc.pdf", "doc.pdf.part", "plain.txt") + fresh := filepath.Join(root, "fresh.txt") + if err := os.WriteFile(fresh, nil, 0o644); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(fresh, now.Add(-30*time.Second), now.Add(-30*time.Second)); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(root, "plain.txt"), filepath.Join(root, "link.txt")); err != nil { + t.Fatal(err) + } + fifo := filepath.Join(root, "pipe") + haveFifo := syscall.Mkfifo(fifo, 0o644) == nil + m, _ := ignore.New([]string{"*.part", "*.aria2"}) + r, err := Walk(root, Options{Ignore: m, Busy: []string{".part", ".aria2"}, MinAge: 2 * time.Minute, Now: now}) + if err != nil { + t.Fatal(err) + } + if want := []string{"plain.txt"}; !reflect.DeepEqual(rels(r), want) { + t.Fatalf("files = %v, want %v", rels(r), want) + } + want := map[string]Reason{ + "movie.mkv": Busy, "doc.pdf": Busy, "movie.mkv.aria2": Ignored, "doc.pdf.part": Ignored, + "fresh.txt": TooNew, "link.txt": Symlink, + } + if haveFifo { + want["pipe"] = NotRegular + } + if got := skipped(r); !reflect.DeepEqual(got, want) { + t.Fatalf("skipped = %v, want %v", got, want) + } +} + +func TestSymlinkedDirNotFollowed(t *testing.T) { + root := tree(t, "real/x.txt") + if err := os.Symlink(filepath.Join(root, "real"), filepath.Join(root, "alias")); err != nil { + t.Fatal(err) + } + r, err := Walk(root, Options{Recursive: true, Now: now}) + if err != nil { + t.Fatal(err) + } + if want := []string{"real/x.txt"}; !reflect.DeepEqual(rels(r), want) { + t.Fatalf("files = %v, want %v", rels(r), want) + } + if skipped(r)["alias"] != Symlink { + t.Fatalf("skipped = %v", skipped(r)) + } +} + +func TestReasonString(t *testing.T) { + want := map[Reason]string{Ignored: "ignored", Busy: "busy", TooNew: "too new", Symlink: "symlink", NotRegular: "not a regular file", Unreadable: "unreadable"} + for r, s := range want { + if r.String() != s { + t.Errorf("%d = %q, want %q", r, r.String(), s) + } + } +} + +func TestUnreadableSubdir(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("running as root: permissions are not enforced") + } + root := tree(t, "a.txt", "locked/secret.txt", "one/b.txt") + locked := filepath.Join(root, "locked") + if err := os.Chmod(locked, 0o000); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.Chmod(locked, 0o755); err != nil { + t.Fatal(err) + } + }) + r, err := Walk(root, Options{Recursive: true, Now: now}) + if err != nil { + t.Fatal(err) + } + if want := []string{"a.txt", "one/b.txt"}; !reflect.DeepEqual(rels(r), want) { + t.Fatalf("files = %v, want %v", rels(r), want) + } + if got := skipped(r)["locked"]; got != Unreadable { + t.Fatalf("skipped[locked] = %v, want Unreadable", got) + } +} + +func TestRootMustBeDirectory(t *testing.T) { + root := tree(t, "f") + if _, err := Walk(filepath.Join(root, "f"), Options{Now: now}); err == nil { + t.Fatal("walking a file succeeded") + } + if _, err := Walk(filepath.Join(root, "missing"), Options{Now: now}); err == nil { + t.Fatal("walking a missing path succeeded") + } +} + +// fakeDirEntry is an os.DirEntry whose Info() returns a canned result, +// simulating a race scan_test cannot otherwise provoke portably: a file +// renamed or removed between being listed by ReadDir and having Info() +// called on it (A2), or some other Info() failure. +type fakeDirEntry struct { + name string + info fs.FileInfo + infoErr error +} + +func (f fakeDirEntry) Name() string { return f.name } +func (f fakeDirEntry) IsDir() bool { return false } +func (f fakeDirEntry) Type() fs.FileMode { return 0 } +func (f fakeDirEntry) Info() (fs.FileInfo, error) { return f.info, f.infoErr } + +// TestFileInfoFailureMidWalk: A2. A vanished file's Info() failure +// (fs.ErrNotExist) is skipped silently, with no trace in Skipped; any +// other Info() failure is reported as Unreadable; and the walk continues +// to later entries in either case rather than aborting. +func TestFileInfoFailureMidWalk(t *testing.T) { + root := t.TempDir() + okPath := filepath.Join(root, "ok.txt") + if err := os.WriteFile(okPath, []byte("ok"), 0o644); err != nil { + t.Fatal(err) + } + old := now.Add(-time.Hour) + if err := os.Chtimes(okPath, old, old); err != nil { + t.Fatal(err) + } + okInfo, err := os.Lstat(okPath) + if err != nil { + t.Fatal(err) + } + + entries := []os.DirEntry{ + fakeDirEntry{name: "vanished.txt", infoErr: fmt.Errorf("stat vanished.txt: %w", fs.ErrNotExist)}, + fakeDirEntry{name: "denied.txt", infoErr: errors.New("stat denied.txt: permission denied")}, + fakeDirEntry{name: "ok.txt", info: okInfo}, + } + + w := &walker{root: root, opt: Options{Now: now}} + if err := w.walk(root, "", 1, entries); err != nil { + t.Fatalf("walk aborted: %v", err) + } + + if want := []string{"ok.txt"}; !reflect.DeepEqual(rels(&w.result), want) { + t.Fatalf("files = %v, want %v", rels(&w.result), want) + } + got := skipped(&w.result) + if _, vanished := got["vanished.txt"]; vanished { + t.Errorf("vanished file recorded as skipped: %v", got) + } + if got["denied.txt"] != Unreadable { + t.Errorf("skipped[denied.txt] = %v, want Unreadable", got["denied.txt"]) + } +} + +// TestIgnoredDirectoryReportedOnce: C2. An ignored directory with files +// inside it is reported once, for the directory itself, not once per file +// — pruning it without descending is the point — so its contents are not +// simply invisible to every count and to -v. +func TestIgnoredDirectoryReportedOnce(t *testing.T) { + root := tree(t, "keep.txt", "skip/a.txt", "skip/b.txt", "skip/c.txt") + m, _ := ignore.New([]string{"skip/"}) + r, err := Walk(root, Options{Recursive: true, Ignore: m, Now: now}) + if err != nil { + t.Fatal(err) + } + if want := []string{"keep.txt"}; !reflect.DeepEqual(rels(r), want) { + t.Fatalf("files = %v, want %v", rels(r), want) + } + if want := (map[string]Reason{"skip": Ignored}); !reflect.DeepEqual(skipped(r), want) { + t.Fatalf("skipped = %v, want %v (the directory once, not its three files)", skipped(r), want) + } +} |