From 3b36a48b7ce5a53a9366f3b31f94311f178e2553 Mon Sep 17 00:00:00 2001 From: Lukasz Kasprzak Date: Sat, 12 Sep 2026 01:22:12 +0200 Subject: krino: matching — scan, ignore, conditions, extraction, duplicates, explain, dry run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/engine/engine.go | 170 +++++++++++++++++++ internal/engine/engine_test.go | 166 +++++++++++++++++++ internal/engine/facts.go | 204 +++++++++++++++++++++++ internal/engine/facts_test.go | 77 +++++++++ internal/engine/match.go | 357 +++++++++++++++++++++++++++++++++++++++ internal/engine/match_test.go | 367 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 1341 insertions(+) create mode 100644 internal/engine/engine.go create mode 100644 internal/engine/engine_test.go create mode 100644 internal/engine/facts.go create mode 100644 internal/engine/facts_test.go create mode 100644 internal/engine/match.go create mode 100644 internal/engine/match_test.go (limited to 'internal/engine') diff --git a/internal/engine/engine.go b/internal/engine/engine.go new file mode 100644 index 0000000..eec9a60 --- /dev/null +++ b/internal/engine/engine.go @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +// Package engine is what every krino front end calls: it loads and compiles +// the configuration, matches files against rules, and (from plan 3) plans +// and applies actions. It returns data; front ends only render it. +package engine + +import ( + "os" + "time" + + "krino/internal/cond" + "krino/internal/config" + "krino/internal/extract" + "krino/internal/ignore" +) + +// Engine holds a loaded, compiled configuration: everything a front end +// needs to check, match and (from plan 3) act. +type Engine struct { + Config *config.Config + Dirs []*Dir + Extract *extract.Extractor + Now func() time.Time // time.Now; tests replace it + MainFile string +} + +// Dir is one configured directory, with its ignore matcher and rules +// compiled. +type Dir struct { + Name string + Root string // absolute + Conf *config.Dir + Settings config.Resolved // built-in, then defaults, then the directory + Ignore *ignore.Matcher + Rules []*Rule + + // ContentVariants is the distinct (IgnoreCase, Fold) pairs any of + // Rules' content tests evaluate under, in first-seen order. B2: when + // this holds exactly one variant, facts.Content releases a file's raw + // extracted text once that variant's normalised copy exists, since no + // other variant will ever be asked for; with more than one, both must + // stay memoised, as before. + ContentVariants []cond.Options +} + +// Rule is one directory's rule, with its condition compiled. +type Rule struct { + Name string + Conf *config.Rule + Settings config.Resolved // the rule's own settings over its directory's + Cond *cond.Cond +} + +// Load reads and compiles everything. Any problem anywhere returns a nil +// Engine and every diagnostic: krino never acts on a configuration it only +// partly understood. Duplicate names are ignored after their first use. +func Load(mainFile string, names ...string) (*Engine, []*config.Diag) { + cfg, errs := config.Load(mainFile, dedupeNames(names)...) + if cfg == nil { + return nil, errs + } + + var dirs []*Dir + for _, d := range cfg.Dirs { + dir := &Dir{ + Name: d.Name, + Root: d.Path, + Conf: d, + Settings: cfg.Resolved(d), + } + if m, err := ignore.New(d.Ignore); err != nil { + errs = append(errs, &config.Diag{File: d.File, Msg: err.Error()}) + } else { + dir.Ignore = m + } + for _, r := range d.Rules { + rs := r.Settings.Over(dir.Settings) + c, cerrs := cond.Compile(d.File, r.When, cond.Options{ + IgnoreCase: rs.Case == config.CaseIgnore, + Fold: rs.Fold, + }) + if len(cerrs) > 0 { + errs = append(errs, cerrs...) + continue + } + dir.Rules = append(dir.Rules, &Rule{Name: r.Name, Conf: r, Settings: rs, Cond: c}) + } + dir.ContentVariants = contentVariants(dir.Rules) + dirs = append(dirs, dir) + } + + if len(errs) > 0 { + return nil, errs + } + return &Engine{ + Config: cfg, + Dirs: dirs, + Extract: extract.New(), + Now: time.Now, + MainFile: mainFile, + }, nil +} + +// dedupeNames returns names with every repeat after its first occurrence +// removed, order preserved. +func dedupeNames(names []string) []string { + var out []string + seen := map[string]bool{} + for _, n := range names { + if seen[n] { + continue + } + seen[n] = true + out = append(out, n) + } + return out +} + +// contentVariants returns the distinct (IgnoreCase, Fold) pairs any of +// rules' content tests evaluate under, in first-seen order — B2's per-Dir +// ContentVariants. A rule whose condition has no content test at all +// (Cond.UsesContent false) never calls facts.Content, so its resolved +// case/fold settings contribute no variant here. +func contentVariants(rules []*Rule) []cond.Options { + var out []cond.Options + seen := map[cond.Options]bool{} + for _, r := range rules { + if !r.Cond.UsesContent { + continue + } + opt := cond.Options{IgnoreCase: r.Settings.Case == config.CaseIgnore, Fold: r.Settings.Fold} + if seen[opt] { + continue + } + seen[opt] = true + out = append(out, opt) + } + return out +} + +// Report is what Check reports: the files involved and each directory's +// state. +type Report struct { + MainFile string + LogFile string + Dirs []DirReport + Tools []extract.Tool +} + +// DirReport is one directory's state in a Report. +type DirReport struct { + Dir *Dir + Missing bool // the root is not a directory right now +} + +// Check reports the engine's configuration files, each directory's current +// state and the external tools found for content extraction. +func (e *Engine) Check() Report { + r := Report{ + MainFile: e.MainFile, + LogFile: e.Config.LogFile(), + Tools: e.Extract.Tools(), + } + for _, d := range e.Dirs { + fi, err := os.Stat(d.Root) + r.Dirs = append(r.Dirs, DirReport{Dir: d, Missing: err != nil || !fi.IsDir()}) + } + return r +} diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go new file mode 100644 index 0000000..5c0536f --- /dev/null +++ b/internal/engine/engine_test.go @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package engine + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "krino/internal/cond" +) + +// sandbox gives a test its own HOME with no XDG overrides and returns it. +func sandbox(t *testing.T) string { + t.Helper() + h := t.TempDir() + t.Setenv("HOME", h) + for _, v := range []string{"XDG_CONFIG_HOME", "XDG_STATE_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME"} { + t.Setenv(v, "") + } + return h +} + +// writeConfig writes krino.conf and dirs/.conf files under home/.config/krino. +func writeConfig(t *testing.T, home, main string, dirs map[string]string) string { + t.Helper() + cdir := filepath.Join(home, ".config", "krino") + if err := os.MkdirAll(filepath.Join(cdir, "dirs"), 0o755); err != nil { + t.Fatal(err) + } + mainFile := filepath.Join(cdir, "krino.conf") + if err := os.WriteFile(mainFile, []byte(main), 0o644); err != nil { + t.Fatal(err) + } + for n, body := range dirs { + if err := os.WriteFile(filepath.Join(cdir, "dirs", n+".conf"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + return mainFile +} + +// fakeFacts is a minimal cond.Facts for checking compiled rules. +type fakeFacts struct{ name string } + +func (f fakeFacts) Name() string { return f.name } +func (f fakeFacts) Rel() string { return f.name } +func (f fakeFacts) Size() int64 { return 1 } +func (f fakeFacts) ModTime() time.Time { return time.Time{} } +func (f fakeFacts) Now() time.Time { return time.Time{} } +func (f fakeFacts) Content(bool, bool) (string, error) { return "", nil } +func (f fakeFacts) Duplicate([]string) (string, bool, error) { return "", false, nil } +func (f fakeFacts) Matched() bool { return false } + +var _ cond.Facts = fakeFacts{} + +func TestLoadCompilesWithRuleSettings(t *testing.T) { + h := sandbox(t) + os.Mkdir(filepath.Join(h, "dl"), 0o755) + main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": ` +(path "~/dl") +(case strict) +(ignore "*.part") +(rule "strict" (when (name "^img")) (stop)) +(rule "loose" (case ignore) (when (name "^img")) (stop)) +`}) + e, errs := Load(main, "dl", "dl") + if len(errs) > 0 { + t.Fatal(errs) + } + if len(e.Dirs) != 1 { + t.Fatalf("got %d dirs, want 1 (duplicate name ignored)", len(e.Dirs)) + } + d := e.Dirs[0] + if d.Root != filepath.Join(h, "dl") || d.Ignore == nil || !d.Ignore.Match("x.part", false) { + t.Fatalf("dir = %+v", d) + } + img := fakeFacts{name: "IMG_1.jpg"} + if d.Rules[0].Cond.Eval(img).Match { + t.Error("rule under (case strict) matched IMG against ^img") + } + if !d.Rules[1].Cond.Eval(img).Match { + t.Error("rule-level (case ignore) did not apply at compile time") + } +} + +func TestLoadReportsEveryProblem(t *testing.T) { + h := sandbox(t) + main := writeConfig(t, h, `(include "a" "b")`, map[string]string{ + "a": `(path "/tmp") (rule "x" (when (type "pdf")) (stop))`, + "b": `(path "/tmp") (ignore "[abc") (rule "y" (when (size 1)) (stop))`, + }) + e, errs := Load(main) + if e != nil { + t.Fatal("engine returned despite errors") + } + joined := "" + for _, d := range errs { + joined += d.Error() + "\n" + } + for _, want := range []string{ + "a.conf:1:37: type names are bare words: write (type pdf)", + `b.conf: bad ignore pattern "[abc": unterminated [`, + "b.conf:1:47: size takes an operator and a size, like (size > 10M)", + } { + if !strings.Contains(joined, want) { + t.Errorf("missing %q in:\n%s", want, joined) + } + } +} + +func TestCheck(t *testing.T) { + h := sandbox(t) + bin := t.TempDir() + os.WriteFile(filepath.Join(bin, "pdftotext"), []byte("#!/bin/sh\n"), 0o755) + t.Setenv("PATH", bin) + os.Mkdir(filepath.Join(h, "here"), 0o755) + main := writeConfig(t, h, `(include "here" "gone")`, map[string]string{ + "here": `(path "~/here") (rule "r" (stop))`, + "gone": `(path "~/gone") (rule "r" (stop))`, + }) + e, errs := Load(main) + if len(errs) > 0 { + t.Fatal(errs) + } + r := e.Check() + if r.MainFile != main || r.LogFile != filepath.Join(h, ".local", "state", "krino", "krino.log") { + t.Errorf("report files = %q %q", r.MainFile, r.LogFile) + } + if len(r.Dirs) != 2 || r.Dirs[0].Missing || !r.Dirs[1].Missing { + t.Errorf("dirs = %+v", r.Dirs) + } + if r.Tools[0].Name != "pdftotext" || r.Tools[0].Path != filepath.Join(bin, "pdftotext") || r.Tools[1].Path != "" { + t.Errorf("tools = %+v", r.Tools) + } +} + +// TestContentVariantsComputedAtLoad: B2 plumbing. Load computes each +// directory's distinct (ignoreCase, fold) content-test variants from its +// rules' resolved settings: a rule with no content test contributes +// nothing; two rules sharing a variant fold into one; a rule-level (case +// ignore) override adds a second. +func TestContentVariantsComputedAtLoad(t *testing.T) { + h := sandbox(t) + os.Mkdir(filepath.Join(h, "dl"), 0o755) + main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": ` +(path "~/dl") +(case strict) +(rule "no-content" (when (type pdf)) (stop)) +(rule "strict-content" (when (content "acme")) (stop)) +(rule "also-strict-content" (when (content "other")) (stop)) +(rule "loose-content" (case ignore) (when (content "acme")) (stop)) +`}) + e, errs := Load(main, "dl") + if len(errs) > 0 { + t.Fatal(errs) + } + got := e.Dirs[0].ContentVariants + want := []cond.Options{{IgnoreCase: false, Fold: true}, {IgnoreCase: true, Fold: true}} + if !reflect.DeepEqual(got, want) { + t.Fatalf("ContentVariants = %+v, want %+v", got, want) + } +} diff --git a/internal/engine/facts.go b/internal/engine/facts.go new file mode 100644 index 0000000..60380f3 --- /dev/null +++ b/internal/engine/facts.go @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package engine + +import ( + "context" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "krino/internal/cond" + "krino/internal/dup" + "krino/internal/norm" + "krino/internal/scan" + "krino/internal/xdg" +) + +// matchRun holds the state shared by every file evaluated during one Match +// or Explain call: the directory being matched, the full set of scanned +// files (for duplicate detection) and the lazily built duplicate indexes, +// one per distinct set of resolved directories a (duplicate ...) test +// names. mu guards dupOnce, dupIdx and warn, the only fields any goroutine +// but the one that created the matchRun ever touches. +type matchRun struct { + e *Engine + d *Dir + ctx context.Context + now time.Time + files []scan.File + + mu sync.Mutex + dupOnce map[string]*sync.Once + dupIdx map[string]*dup.Index + warn []string +} + +// newMatchRun builds a matchRun over files, the set a (duplicate ...) test +// with no directories of its own checks against. +func newMatchRun(e *Engine, d *Dir, ctx context.Context, now time.Time, files []scan.File) *matchRun { + return &matchRun{ + e: e, + d: d, + ctx: ctx, + now: now, + files: files, + dupOnce: make(map[string]*sync.Once), + dupIdx: make(map[string]*dup.Index), + } +} + +// warnings returns the directory-level warnings collected so far (from +// building duplicate indexes), in the order they were recorded. +func (run *matchRun) warnings() []string { + run.mu.Lock() + defer run.mu.Unlock() + return append([]string(nil), run.warn...) +} + +// drainDupErrors appends every duplicate index's candidate-hashing errors +// (A1: a candidate other than the file being looked up that could not be +// hashed) to run.warn, once matching is done and every index has seen every +// Lookup it is going to see. Candidate paths are abbreviated with +// xdg.Abbrev, as every other user-visible path is. +func (run *matchRun) drainDupErrors() { + run.mu.Lock() + defer run.mu.Unlock() + for _, idx := range run.dupIdx { + for _, ce := range idx.Errors() { + run.warn = append(run.warn, "duplicate: "+xdg.Abbrev(ce.Path)+": "+ce.Err.Error()) + } + } +} + +// dupIndex returns the shared *dup.Index for the resolved, sorted extra +// directories named by key, building it exactly once across every +// concurrent caller that asks for the same key. +func (run *matchRun) dupIndex(key string, dirs []string) *dup.Index { + run.mu.Lock() + once, ok := run.dupOnce[key] + if !ok { + once = &sync.Once{} + run.dupOnce[key] = once + } + run.mu.Unlock() + + once.Do(func() { + idx, errs := dup.NewIndex(run.files, dirs) + run.mu.Lock() + run.dupIdx[key] = idx + for _, err := range errs { + run.warn = append(run.warn, "duplicate: "+err.Error()) + } + run.mu.Unlock() + }) + + run.mu.Lock() + idx := run.dupIdx[key] + run.mu.Unlock() + return idx +} + +// facts is one file's cond.Facts. It is used by exactly one goroutine, so +// its own memoised state (content, its normalised variants, and whether an +// earlier rule matched) needs no locking of its own; only the matchRun it +// points at is shared. +type facts struct { + run *matchRun + file scan.File + + matched bool + + contentDone bool + content string + contentErr error + normCache map[[2]bool]string +} + +var _ cond.Facts = (*facts)(nil) + +// newFacts builds the Facts for one scanned file. +func newFacts(run *matchRun, file scan.File) *facts { + return &facts{run: run, file: file, normCache: make(map[[2]bool]string)} +} + +func (f *facts) Name() string { return f.file.Name } +func (f *facts) Rel() string { return f.file.Rel } +func (f *facts) Size() int64 { return f.file.Size } +func (f *facts) ModTime() time.Time { return f.file.ModTime } +func (f *facts) Now() time.Time { return f.run.now } +func (f *facts) Matched() bool { return f.matched } + +// Content extracts the file's text once, then normalises it per +// (ignoreCase, fold) variant, memoising each. B2: when the directory's +// rules use exactly one variant (Dir.ContentVariants), the raw text is +// released as soon as that variant's normalised copy exists — no other +// variant will ever be asked for, so there is no reason to keep both the +// raw text and its normalised copy in memory at once. A directory using +// more than one variant keeps the raw text for as long as f lives, exactly +// as before. +func (f *facts) Content(ignoreCase, fold bool) (string, error) { + if !f.contentDone { + f.content, f.contentErr = f.run.e.Extract.Text(f.run.ctx, f.file.Path, f.file.Size, f.run.d.Settings.MaxRead) + f.contentDone = true + } + if f.contentErr != nil { + return "", f.contentErr + } + key := [2]bool{ignoreCase, fold} + if v, ok := f.normCache[key]; ok { + return v, nil + } + v := norm.Text(f.content, ignoreCase, fold) + f.normCache[key] = v + if len(f.run.d.ContentVariants) == 1 { + f.content = "" + } + return v, nil +} + +// Duplicate resolves dirs against the directory's root, builds (or reuses) +// the shared duplicate index for that resolved, sorted set, and looks the +// file up in it. +func (f *facts) Duplicate(dirs []string) (string, bool, error) { + root := f.run.d.Root + resolved := make([]string, len(dirs)) + for i, raw := range dirs { + resolved[i] = resolveDir(raw, root) + } + sorted := append([]string(nil), resolved...) + sort.Strings(sorted) + key := strings.Join(sorted, "\x00") + + idx := f.run.dupIndex(key, sorted) + orig, isDup, err := idx.Lookup(f.file.Path) + if err != nil { + return "", false, err + } + if !isDup { + return "", false, nil + } + return displayOriginal(orig, root), true, nil +} + +// resolveDir expands a leading ~ and joins a relative directory to root, +// cleaned. +func resolveDir(raw, root string) string { + p := xdg.Expand(raw) + if !filepath.IsAbs(p) { + p = filepath.Join(root, p) + } + return filepath.Clean(p) +} + +// displayOriginal reports orig relative to root when it lies inside root, +// else as an absolute path. +func displayOriginal(orig, root string) string { + rel, err := filepath.Rel(root, orig) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return orig + } + return filepath.ToSlash(rel) +} diff --git a/internal/engine/facts_test.go b/internal/engine/facts_test.go new file mode 100644 index 0000000..c511804 --- /dev/null +++ b/internal/engine/facts_test.go @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package engine + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "krino/internal/cond" + "krino/internal/extract" + "krino/internal/scan" +) + +// contentFacts builds a *facts for a real text file, under a Dir whose +// ContentVariants is variants, for exercising B2's raw-release directly. +func contentFacts(t *testing.T, body string, variants []cond.Options) *facts { + t.Helper() + p := filepath.Join(t.TempDir(), "a.txt") + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + fi, err := os.Stat(p) + if err != nil { + t.Fatal(err) + } + e := &Engine{Extract: extract.New(), Now: time.Now} + d := &Dir{Name: "d", ContentVariants: variants} + file := scan.File{Path: p, Rel: "a.txt", Name: "a.txt", Size: fi.Size(), ModTime: fi.ModTime()} + run := newMatchRun(e, d, context.Background(), time.Now(), []scan.File{file}) + return newFacts(run, file) +} + +// TestContentReleasesRawWithOneVariant: B2. A directory whose rules use +// exactly one (ignoreCase, fold) variant releases the raw extracted text +// once that variant's normalised copy exists. +func TestContentReleasesRawWithOneVariant(t *testing.T) { + f := contentFacts(t, "Hello World", []cond.Options{{IgnoreCase: true, Fold: false}}) + const want = "hello world" + got, err := f.Content(true, false) + if err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("got %q, want %q", got, want) + } + if f.content != "" { + t.Errorf("raw text not released with a single content variant: %q", f.content) + } + // A second call for the same (already cached) variant must still work + // from normCache, without needing the released raw text. + if got, err := f.Content(true, false); err != nil || got != want { + t.Errorf("second call for the cached variant: got %q, %v, want %q", got, err, want) + } +} + +// TestContentKeepsRawWithTwoVariants: B2. A directory whose rules use two +// distinct variants must not release the raw text after the first: the +// second variant still needs it, and normalising it correctly (not from an +// emptied string) is the proof the raw text was kept. +func TestContentKeepsRawWithTwoVariants(t *testing.T) { + f := contentFacts(t, "Hello World", []cond.Options{ + {IgnoreCase: true, Fold: false}, + {IgnoreCase: false, Fold: false}, + }) + if got, err := f.Content(true, false); err != nil || got != "hello world" { + t.Fatalf("first variant: got %q, %v", got, err) + } + if f.content == "" { + t.Fatal("raw text released after only the first of two variants") + } + if got, err := f.Content(false, false); err != nil || got != "Hello World" { + t.Fatalf("second variant: got %q, %v, want the unfolded original (raw text must still be available)", got, err) + } +} diff --git a/internal/engine/match.go b/internal/engine/match.go new file mode 100644 index 0000000..0693bd5 --- /dev/null +++ b/internal/engine/match.go @@ -0,0 +1,357 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package engine + +import ( + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "sync" + "time" + + "krino/internal/cond" + "krino/internal/config" + "krino/internal/scan" + "krino/internal/xdg" +) + +// RuleMatch is one rule that matched a file, and why. +type RuleMatch struct { + Rule *Rule + Captures []string + Reasons []string +} + +// FileMatch is one file and the rules that did, or did not, match it. +type FileMatch struct { + File scan.File + Rules []RuleMatch // matching rules in order, ending at the first with (stop) + Warnings []string // ": ", e.g. "acme: content unreadable: needs pdftotext, not installed" +} + +// Result is everything Match found in one directory. +type Result struct { + Dir *Dir + Matched []FileMatch // at least one rule matched; sorted by File.Rel + Unmatched []FileMatch // no rule matched (Warnings may say why); sorted by File.Rel + Skipped []scan.Skipped + Warnings []string // directory-level, sorted; e.g. "duplicate: /x/y does not exist" + Elapsed time.Duration +} + +// Match walks d's root and evaluates every rule against every file found, +// concurrently. Output order never depends on scheduling: results are +// placed by index into a slice the size of the walk, then split into +// Matched and Unmatched keeping that (Rel-sorted) order. +func (e *Engine) Match(ctx context.Context, d *Dir) (*Result, error) { + started := time.Now() + now := e.Now() + excl := e.excludeDirs(d) + + wres, err := scan.Walk(d.Root, walkOptions(d, excl, now)) + if err != nil { + return nil, err + } + + run := newMatchRun(e, d, ctx, now, wres.Files) + fileMatches := make([]FileMatch, len(wres.Files)) + + workers := runtime.GOMAXPROCS(0) + if workers < 1 { + workers = 1 + } + var wg sync.WaitGroup + jobs := make(chan int) + for w := 0; w < workers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := range jobs { + fileMatches[i] = evalFile(run, wres.Files[i]) + } + }() + } + for i := range wres.Files { + jobs <- i + } + close(jobs) + wg.Wait() + run.drainDupErrors() + + var matched, unmatched []FileMatch + for _, fm := range fileMatches { + if len(fm.Rules) > 0 { + matched = append(matched, fm) + } else { + unmatched = append(unmatched, fm) + } + } + + warnings := run.warnings() + sort.Strings(warnings) + + return &Result{ + Dir: d, + Matched: matched, + Unmatched: unmatched, + Skipped: wres.Skipped, + Warnings: warnings, + Elapsed: time.Since(started), + }, nil +} + +// evalFile evaluates every rule of run.d, in order, against file: matched +// becomes true after the first matching rule, so a later (not (matched)) +// test sees it, and evaluation stops right after a matching rule whose +// Stop is set. +func evalFile(run *matchRun, file scan.File) FileMatch { + f := newFacts(run, file) + fm := FileMatch{File: file} + for _, r := range run.d.Rules { + res := r.Cond.Eval(f) + for _, w := range res.Warnings { + fm.Warnings = append(fm.Warnings, r.Name+": "+w) + } + if !res.Match { + continue + } + fm.Rules = append(fm.Rules, RuleMatch{Rule: r, Captures: res.Captures, Reasons: res.Reasons}) + f.matched = true + if r.Conf.Stop { + break + } + } + return fm +} + +// RuleTrace is one rule's outcome in an Explain call. +type RuleTrace struct { + Rule *Rule + Match bool + Trace *cond.Trace // nil when not evaluated + Stopped string // "stopped by rule acme" when an earlier (stop) ended the search +} + +// Explanation is why (or why not) krino would act on one file. +type Explanation struct { + Dir *Dir + File scan.File + Skip string // why krino would not look at this file at all; "" when it would + Rules []RuleTrace +} + +// Explain reports, for one file, whether krino's ordinary scan would ever +// reach it and how every rule of its directory evaluates against it. Rules +// are traced in order even when Skip is set, so a user can see what would +// match if the file were looked at; a rule reached after an earlier +// matching (stop) is recorded as Stopped, with no trace. +func (e *Engine) Explain(ctx context.Context, path string) (*Explanation, error) { + abs, err := filepath.Abs(path) + if err != nil { + return nil, err + } + abs = filepath.Clean(abs) + + d := e.dirFor(abs) + if d == nil { + return nil, fmt.Errorf("%s is not inside any included directory", path) + } + + info, err := os.Lstat(abs) + if err != nil { + return nil, err + } + if info.Mode()&fs.ModeSymlink != 0 || !info.Mode().IsRegular() { + return nil, fmt.Errorf("%s is not a regular file", path) + } + + rel, err := filepath.Rel(d.Root, abs) + if err != nil { + return nil, err + } + rel = filepath.ToSlash(rel) + + sf := scan.File{ + Path: abs, + Rel: rel, + Name: filepath.Base(abs), + Size: info.Size(), + ModTime: info.ModTime(), + Mode: info.Mode(), + } + + now := e.Now() + excl := e.excludeDirs(d) + skip := explainSkip(d, sf, excl, now) + + run := newMatchRun(e, d, ctx, now, e.filesForExplain(d, sf, excl, now)) + f := newFacts(run, sf) + + var rules []RuleTrace + stoppedBy := "" + for _, r := range d.Rules { + if stoppedBy != "" { + rules = append(rules, RuleTrace{Rule: r, Stopped: "stopped by rule " + stoppedBy}) + continue + } + trace := r.Cond.Explain(f) + match := trace.Value + if match { + f.matched = true + } + rules = append(rules, RuleTrace{Rule: r, Match: match, Trace: trace}) + if match && r.Conf.Stop { + stoppedBy = r.Name + } + } + + return &Explanation{Dir: d, File: sf, Skip: skip, Rules: rules}, nil +} + +// filesForExplain returns the file set Explain's duplicate checks run +// against: the directory's ordinary scan, plus the explained file itself +// when that scan would not have reached it (it is busy, ignored, too new, +// excluded, or beyond recursive/max-depth) - so (duplicate ...) always has +// a real answer for the file being explained, and sees the same siblings +// Match would. +func (e *Engine) filesForExplain(d *Dir, sf scan.File, excl []string, now time.Time) []scan.File { + wres, err := scan.Walk(d.Root, walkOptions(d, excl, now)) + if err != nil { + return []scan.File{sf} + } + for _, wf := range wres.Files { + if wf.Path == sf.Path { + return wres.Files + } + } + return append(append([]scan.File{}, wres.Files...), sf) +} + +// walkOptions builds the scan.Options both Match and Explain's +// filesForExplain walk d's root with, so the two never drift apart. +func walkOptions(d *Dir, excl []string, now time.Time) scan.Options { + return scan.Options{ + Recursive: d.Settings.Recursive, + MaxDepth: d.Settings.MaxDepth, + Ignore: d.Ignore, + Exclude: excl, + Busy: d.Settings.Busy, + MinAge: d.Settings.MinAge, + Now: now, + } +} + +// dirFor returns the configured Dir whose root most specifically (longest +// root wins) contains abs, or nil if none does. +func (e *Engine) dirFor(abs string) *Dir { + var best *Dir + var bestRoot string + for _, d := range e.Dirs { + root := filepath.Clean(d.Root) + if abs != root && !strings.HasPrefix(abs, root+string(filepath.Separator)) { + continue + } + if best == nil || len(root) > len(bestRoot) { + best, bestRoot = d, root + } + } + return best +} + +// explainSkip decides, in priority order, why krino's ordinary scan would +// not reach sf, or "" if it would. +func explainSkip(d *Dir, sf scan.File, excl []string, now time.Time) string { + segs := strings.Split(sf.Rel, "/") + if !d.Settings.Recursive && len(segs) > 1 { + return "in a subdirectory, and recursive is off" + } + if d.Settings.MaxDepth > 0 && len(segs) > d.Settings.MaxDepth { + return "deeper than max-depth" + } + if insideAny(sf.Path, excl) { + return "inside a rule destination, which krino never scans" + } + if d.Ignore != nil && d.Ignore.Match(sf.Rel, false) { + return "ignored" + } + if isBusy(sf.Path, d.Settings.Busy) { + return "busy" + } + if now.Sub(sf.ModTime) < d.Settings.MinAge { + return "too new" + } + return "" +} + +// insideAny reports whether path is dir itself, or inside it, for any dir +// in dirs. +func insideAny(path string, dirs []string) bool { + for _, dir := range dirs { + if path == dir || strings.HasPrefix(path, dir+string(filepath.Separator)) { + return true + } + } + return false +} + +// isBusy reports whether path has a sibling named path+suffix, for any +// configured busy suffix - the mark of an in-progress download. +func isBusy(path string, suffixes []string) bool { + for _, suf := range suffixes { + if _, err := os.Lstat(path + suf); err == nil { + return true + } + } + return false +} + +// excludeDirs computes the directories Match and Explain never enter: each +// rule's copy/move destination, the Trash, and the directory holding the +// main config file - each kept only when it lies strictly inside d's root. +// A destination with no template placeholder excludes exactly that +// directory; a destination with a placeholder excludes only the static +// part before its first "{", cut back to a full path component (its last +// "/"), since anything from there on varies per file - spec 8.1: "Work/ +// Acme/{mtime:%Y}" excludes "Work/Acme", and "Work/Acme-{mtime:%Y}" +// (the placeholder mid-segment) excludes "Work". +func (e *Engine) excludeDirs(d *Dir) []string { + root := filepath.Clean(d.Root) + var out []string + add := func(p string) { + if p == "" { + return + } + p = filepath.Clean(p) + if strings.HasPrefix(p, root+string(filepath.Separator)) { + out = append(out, p) + } + } + for _, r := range d.Rules { + for _, a := range r.Conf.Actions { + if a.Kind != config.Copy && a.Kind != config.Move { + continue + } + prefix := a.Arg + if idx := strings.IndexByte(prefix, '{'); idx >= 0 { + prefix = prefix[:idx] + if idx2 := strings.LastIndexByte(prefix, '/'); idx2 >= 0 { + prefix = prefix[:idx2] + } else { + // The very first path component is itself templated + // (e.g. "{year}-Stuff"): nothing about the destination + // is known statically, so there is nothing to exclude. + prefix = "" + } + } + add(resolveDir(prefix, root)) + } + } + add(filepath.Join(xdg.DataHome(), "Trash")) + add(filepath.Dir(e.MainFile)) + return out +} diff --git a/internal/engine/match_test.go b/internal/engine/match_test.go new file mode 100644 index 0000000..1356074 --- /dev/null +++ b/internal/engine/match_test.go @@ -0,0 +1,367 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package engine + +import ( + "context" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" +) + +const dlConf = ` +(path "~/dl") +(recursive yes) +(min-age 0s) +(ignore "*.part") +(rule "dups" (when (duplicate)) (delete) (stop)) +(rule "acme" (when (type document) (content "acme ltd")) (move "Work/Acme") (stop)) +(rule "images" (when (type image)) (move "Pictures")) +(rule "rest" (when (not (matched)) (type text)) (move "Other")) +` + +// fixture builds ~/dl and returns a loaded engine. PATH is empty, so no +// extraction tool exists and PDFs and .doc files are unreadable. +func fixture(t *testing.T) (*Engine, *Dir, string) { + t.Helper() + h := sandbox(t) + t.Setenv("PATH", t.TempDir()) + dl := filepath.Join(h, "dl") + files := map[string]string{ + "inv1.txt": "Invoice from ACME LTD, tax 0000000000", + "notes.txt": "shopping list", + "photo.jpg": "\xff\xd8\xff\xe0 jpeg bytes", + "report.pdf": "%PDF same bytes", + "report (1).pdf": "%PDF same bytes", + "brochure.doc": "\xd0\xcf\x11\xe0 doc bytes", + "movie.mkv": "video", + "movie.mkv.part": "partial", + "Work/Acme/filed.txt": "acme ltd, already filed", + "Pictures/old.jpg": "\xff\xd8 old", + } + old := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for name, body := range files { + p := filepath.Join(dl, name) + os.MkdirAll(filepath.Dir(p), 0o755) + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + os.Chtimes(p, old, old) + } + newer := old.Add(time.Hour) + os.Chtimes(filepath.Join(dl, "report (1).pdf"), newer, newer) + main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": dlConf}) + e, errs := Load(main) + if len(errs) > 0 { + t.Fatal(errs) + } + return e, e.Dirs[0], dl +} + +// summary renders a result compactly for comparison. +func summary(r *Result) []string { + var out []string + for _, m := range r.Matched { + var rs []string + for _, rm := range m.Rules { + rs = append(rs, rm.Rule.Name+"["+strings.Join(rm.Reasons, "; ")+"]") + } + out = append(out, "match "+m.File.Rel+" "+strings.Join(rs, " ")) + } + for _, m := range r.Unmatched { + out = append(out, "none "+m.File.Rel+" "+strings.Join(m.Warnings, " | ")) + } + for _, s := range r.Skipped { + out = append(out, "skip "+s.Rel+" "+s.Reason.String()) + } + return out +} + +func TestMatch(t *testing.T) { + e, d, _ := fixture(t) + r, err := e.Match(context.Background(), d) + if err != nil { + t.Fatal(err) + } + want := []string{ + `match inv1.txt acme[type txt; content "acme ltd"]`, + `match notes.txt rest[not matched; type txt]`, + `match photo.jpg images[type jpg]`, + `match report (1).pdf dups[duplicate of report.pdf]`, + `none brochure.doc acme: content unreadable: needs antiword or catdoc, not installed`, + `none report.pdf acme: content unreadable: needs pdftotext, not installed`, + `skip movie.mkv busy`, + `skip movie.mkv.part ignored`, + } + if got := summary(r); !reflect.DeepEqual(got, want) { + t.Fatalf("got\n%s\nwant\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } + again, _ := e.Match(context.Background(), d) + if !reflect.DeepEqual(summary(again), summary(r)) { + t.Fatal("a second run gave a different result") + } +} + +func TestMatchMissingRoot(t *testing.T) { + e, d, dl := fixture(t) + os.RemoveAll(dl) + if _, err := e.Match(context.Background(), d); err == nil { + t.Fatal("no error for a missing root") + } +} + +func TestExplain(t *testing.T) { + e, _, dl := fixture(t) + x, err := e.Explain(context.Background(), filepath.Join(dl, "inv1.txt")) + if err != nil { + t.Fatal(err) + } + var got []string + for _, rt := range x.Rules { + got = append(got, fmt.Sprintf("%s match=%v stopped=%q trace=%v", rt.Rule.Name, rt.Match, rt.Stopped, rt.Trace != nil)) + } + want := []string{ + `dups match=false stopped="" trace=true`, + `acme match=true stopped="" trace=true`, + `images match=false stopped="stopped by rule acme" trace=false`, + `rest match=false stopped="stopped by rule acme" trace=false`, + } + if !reflect.DeepEqual(got, want) || x.Skip != "" { + t.Fatalf("skip=%q\n%s", x.Skip, strings.Join(got, "\n")) + } + for name, skip := range map[string]string{ + "movie.mkv": "busy", + "movie.mkv.part": "ignored", + "Work/Acme/filed.txt": "inside a rule destination, which krino never scans", + } { + x, err := e.Explain(context.Background(), filepath.Join(dl, name)) + if err != nil || x.Skip != skip { + t.Errorf("%s: skip %q, %v; want %q", name, x.Skip, err, skip) + } + } + if _, err := e.Explain(context.Background(), "/etc/hostname"); err == nil || !strings.HasSuffix(err.Error(), "is not inside any included directory") { + t.Errorf("outside: %v", err) + } +} + +// TestMatchExcludesOnlyRuleDest checks that a rule destination with no +// placeholder excludes exactly that directory, not its parent: a sibling +// subdirectory of the destination's parent must still be scanned. +func TestMatchExcludesOnlyRuleDest(t *testing.T) { + h := sandbox(t) + dl := filepath.Join(h, "dl") + files := map[string]string{ + "Work/Acme/filed.txt": "already filed", + "Work/Other/keep.txt": "keep me", + } + for name, body := range files { + p := filepath.Join(dl, name) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + conf := ` +(path "~/dl") +(recursive yes) +(min-age 0s) +(rule "acme" (when (name "nope")) (move "Work/Acme")) +` + main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": conf}) + e, errs := Load(main) + if len(errs) > 0 { + t.Fatal(errs) + } + r, err := e.Match(context.Background(), e.Dirs[0]) + if err != nil { + t.Fatal(err) + } + seen := map[string]bool{} + for _, m := range r.Matched { + seen[m.File.Rel] = true + } + for _, m := range r.Unmatched { + seen[m.File.Rel] = true + } + for _, s := range r.Skipped { + seen[s.Rel] = true + } + if !seen["Work/Other/keep.txt"] { + t.Error("Work/Other/keep.txt should have been scanned: only the rule's own destination (Work/Acme) may be excluded") + } + if seen["Work/Acme/filed.txt"] { + t.Error("Work/Acme/filed.txt should have been excluded as inside the rule's destination") + } +} + +// TestMatchWarningsSorted checks that Result.Warnings is sorted, not in +// whatever order concurrent workers happened to build the duplicate +// indexes that failed: rule "b" (declared first, with a stop that must +// not skip rule "a" since it never matches) names a missing directory +// that sorts after rule "a"'s, and rule "c" names the very same missing +// directory as rule "a" - which must fold into one warning, not two. +func TestMatchWarningsSorted(t *testing.T) { + h := sandbox(t) + dl := filepath.Join(h, "dl") + if err := os.MkdirAll(dl, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dl, "only.txt"), []byte("hello"), 0o644); err != nil { + t.Fatal(err) + } + conf := ` +(path "~/dl") +(recursive yes) +(min-age 0s) +(rule "b" (when (duplicate "~/zz-missing")) (stop)) +(rule "a" (when (duplicate "~/aa-missing")) (stop)) +(rule "c" (when (duplicate "~/aa-missing")) (stop)) +` + main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": conf}) + e, errs := Load(main) + if len(errs) > 0 { + t.Fatal(errs) + } + r, err := e.Match(context.Background(), e.Dirs[0]) + if err != nil { + t.Fatal(err) + } + aaDir := filepath.Join(h, "aa-missing") + zzDir := filepath.Join(h, "zz-missing") + if len(r.Warnings) != 2 { + t.Fatalf("got %d warnings, want 2 (the shared aa-missing dir should fold into one):\n%s", len(r.Warnings), strings.Join(r.Warnings, "\n")) + } + wantPrefix := []string{"duplicate: " + aaDir + ":", "duplicate: " + zzDir + ":"} + for i, want := range wantPrefix { + if !strings.HasPrefix(r.Warnings[i], want) { + t.Errorf("Warnings[%d] = %q, want prefix %q", i, r.Warnings[i], want) + } + } +} + +// TestExcludeDirs is a table check of excludeDirs's rule-destination +// handling, in rule order: a plain destination excludes itself exactly; a +// destination with a placeholder excludes only the static part before it, +// cut back to a full path component; a destination (after that cut) equal +// to the root itself, or outside the root, excludes nothing; an absolute +// destination inside the root excludes that directory; copy counts like +// move; rename is not a destination at all. +func TestExcludeDirs(t *testing.T) { + h := sandbox(t) + root := filepath.Join(h, "root") + absDest := filepath.Join(root, "AbsDest") + conf := ` +(path "` + root + `") +(rule "r1" (move "Work/Acme")) +(rule "r2" (move "Photos/{mtime:%Y}")) +(rule "r3" (move "Work/Acme-{mtime:%Y}")) +(rule "r4" (move "{ext}")) +(rule "r5" (move ".")) +(rule "r6" (move "~/elsewhere")) +(rule "r7" (move "` + absDest + `")) +(rule "r8" (copy "Backup")) +(rule "r9" (rename "x-{name}")) +` + main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": conf}) + e, errs := Load(main) + if len(errs) > 0 { + t.Fatal(errs) + } + got := e.excludeDirs(e.Dirs[0]) + want := []string{ + filepath.Join(root, "Work", "Acme"), // r1: no placeholder, exact + filepath.Join(root, "Photos"), // r2: cut back to "Photos/" + filepath.Join(root, "Work"), // r3: cut back past "Acme-" + // r4 "{ext}": nothing before "{" at all -> resolves to the root + // itself -> not strictly inside it -> excludes nothing. + // r5 ".": no placeholder, resolves to the root itself -> nothing. + // r6 "~/elsewhere": outside the root -> nothing. + absDest, // r7: absolute, already inside root + filepath.Join(root, "Backup"), // r8: copy counts like move + // r9 rename "x-{name}": rename is never a destination. + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("excludeDirs =\n%v\nwant\n%v", got, want) + } +} + +// TestMatchDrainsDupCandidateErrors: A1 plumbing across the dup/engine +// boundary. Within one directory's own scan (no extra directories), a +// candidate that cannot be hashed must not poison the duplicate answer for +// its size-mates, and must surface exactly once in Result.Warnings, its +// path abbreviated the way every other user-visible path is. +func TestMatchDrainsDupCandidateErrors(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("permissions are not enforced running as root") + } + h := sandbox(t) + dl := filepath.Join(h, "dl") + old := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + files := map[string]string{ + "a.txt": "same content", // older: the original + "b.txt": "same content", // newer: reported as the duplicate + "c.txt": "diff content", // same size as a/b, different bytes + } + for name, body := range files { + p := filepath.Join(dl, name) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + os.Chtimes(filepath.Join(dl, "a.txt"), old, old) + os.Chtimes(filepath.Join(dl, "b.txt"), old.Add(time.Hour), old.Add(time.Hour)) + os.Chtimes(filepath.Join(dl, "c.txt"), old, old) + cPath := filepath.Join(dl, "c.txt") + if err := os.Chmod(cPath, 0o000); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.Chmod(cPath, 0o644) }) + + conf := ` +(path "~/dl") +(recursive yes) +(min-age 0s) +(rule "dup" (when (duplicate)) (stop)) +` + main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": conf}) + e, errs := Load(main) + if len(errs) > 0 { + t.Fatal(errs) + } + r, err := e.Match(context.Background(), e.Dirs[0]) + if err != nil { + t.Fatal(err) + } + + var bReasons []string + for _, m := range r.Matched { + if m.File.Rel == "b.txt" { + for _, rm := range m.Rules { + bReasons = append(bReasons, rm.Reasons...) + } + } + } + if len(bReasons) == 0 || bReasons[0] != "duplicate of a.txt" { + t.Errorf("b.txt duplicate pair with a.txt broken by unreadable sibling c.txt: %v", summary(r)) + } + + want := "duplicate: ~/dl/c.txt: " + found := 0 + for _, w := range r.Warnings { + if strings.HasPrefix(w, want) { + found++ + } + } + if found != 1 { + t.Errorf("got %d warnings with prefix %q, want 1; warnings: %v", found, want, r.Warnings) + } +} -- cgit v1.3