// SPDX-License-Identifier: GPL-3.0-or-later package engine import ( "archive/zip" "bytes" "context" "os" "path/filepath" "strings" "testing" "time" "git.labunix.xyz/krino/internal/plan" "git.labunix.xyz/krino/internal/scan" ) // excludeTree creates ~/dl with files (name -> content), all old enough to // be scanned, and returns home and dl. func excludeTree(t *testing.T, files map[string]string) (home, dl string) { t.Helper() home = sandbox(t) t.Setenv("PATH", t.TempDir()) dl = filepath.Join(home, "dl") old := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) for name, body := range files { p := filepath.Join(dl, name) if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(p, []byte(body), 0o644); err != nil { t.Fatal(err) } if err := os.Chtimes(p, old, old); err != nil { t.Fatal(err) } } return home, dl } // TestExcludeSetsFilesAside: an (exclude ...) in krino.conf applies to the // directory, and the directory's own excludes apply too - by type, by name // regex and by content - so those files get no actions from any rule and // are reported as excluded, with the form that matched. func TestExcludeSetsFilesAside(t *testing.T) { h, _ := excludeTree(t, map[string]string{ "a.iso": "disk image", "draft-1.pdf": "%PDF draft", "secret.txt": "this is poufne material", "keep.txt": "ordinary notes", }) main := writeConfig(t, h, "(include \"dl\")\n(exclude (type iso))\n", map[string]string{"dl": ` (path "~/dl") (exclude (name "^draft")) (exclude (type txt) (content "poufne")) (rule "all" (move "Out")) `}) e, errs := Load(main) if len(errs) > 0 { t.Fatal(errs) } dp, err := e.Plan(context.Background(), e.Dirs[0], plan.NewClaims()) if err != nil { t.Fatal(err) } want := map[string]string{ "a.iso": "(exclude (type iso))", "draft-1.pdf": `(exclude (name "^draft"))`, "secret.txt": `(exclude (type txt) (content "poufne"))`, "keep.txt": "", } for _, fm := range dp.Result.Matched { w, ok := want[fm.File.Rel] if !ok { t.Errorf("unexpected matched file %s", fm.File.Rel) continue } if fm.Excluded != w { t.Errorf("%s: Excluded = %q, want %q", fm.File.Rel, fm.Excluded, w) } if w != "" && len(fm.Rules) != 0 { t.Errorf("%s: excluded but matched rules %v", fm.File.Rel, fm.Rules) } } if len(dp.Result.Matched) != 4 || len(dp.Result.Unmatched) != 0 { t.Errorf("matched %d, unmatched %d; want every file matched (3 excluded, 1 by the rule)", len(dp.Result.Matched), len(dp.Result.Unmatched)) } for _, c := range dp.Chains { if acting := len(c.Steps) > 0; acting != (c.File.Rel == "keep.txt") { t.Errorf("%s: steps %+v", c.File.Rel, c.Steps) } } } // TestExcludeNeedsEveryCondition: within one form, every condition must // hold, as in when. func TestExcludeNeedsEveryCondition(t *testing.T) { h, _ := excludeTree(t, map[string]string{"draft.txt": "x", "draft.pdf": "%PDF x"}) main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": ` (path "~/dl") (exclude (type pdf) (name "^draft")) (rule "all" (move "Out")) `}) e, errs := Load(main) if len(errs) > 0 { t.Fatal(errs) } r, err := e.Match(context.Background(), e.Dirs[0]) if err != nil { t.Fatal(err) } for _, fm := range r.Matched { if excluded := fm.Excluded != ""; excluded != (fm.File.Rel == "draft.pdf") { t.Errorf("%s: Excluded = %q", fm.File.Rel, fm.Excluded) } } } // TestExcludeContentAndRuleContentKeepTheirOwnSettings: an exclude reading // content under the directory's settings first must not change how a rule // with different case/fold settings sees the same text. func TestExcludeContentAndRuleContentKeepTheirOwnSettings(t *testing.T) { h, _ := excludeTree(t, map[string]string{"a.txt": "Invoice ACME"}) main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": ` (path "~/dl") (exclude (content "never present")) (rule "strict" (case strict) (when (content "ACME")) (move "Out")) `}) e, errs := Load(main) if len(errs) > 0 { t.Fatal(errs) } r, err := e.Match(context.Background(), e.Dirs[0]) if err != nil { t.Fatal(err) } if len(r.Matched) != 1 || len(r.Matched[0].Rules) != 1 { t.Fatalf("a.txt should match the strict content rule after the exclude read its text: %+v", r.Matched) } } // TestLoadReportsBadExcludeOnce: a condition error inside a krino.conf // exclude is reported once, not once per directory. func TestLoadReportsBadExcludeOnce(t *testing.T) { h := sandbox(t) main := writeConfig(t, h, "(include \"a\" \"b\")\n(exclude (size big))\n", map[string]string{ "a": "(path \"/tmp\")", "b": "(path \"/tmp\")", }) _, errs := Load(main) if len(errs) != 1 || !strings.Contains(errs[0].Error(), "size") { t.Errorf("errs = %v, want exactly one error about size", errs) } } // TestMaxSizeSkipsTooBig: a directory's max-size skips larger files before // any rule, as too big. func TestMaxSizeSkipsTooBig(t *testing.T) { h, _ := excludeTree(t, map[string]string{"small.txt": "x", "big.txt": strings.Repeat("x", 2048)}) main := writeConfig(t, h, "(include \"dl\")\n(defaults (max-size 1K))\n", map[string]string{"dl": ` (path "~/dl") (rule "all" (move "Out")) `}) e, errs := Load(main) if len(errs) > 0 { t.Fatal(errs) } r, err := e.Match(context.Background(), e.Dirs[0]) if err != nil { t.Fatal(err) } if len(r.Skipped) != 1 || r.Skipped[0].Rel != "big.txt" || r.Skipped[0].Reason != scan.TooBig { t.Errorf("skipped = %+v, want big.txt too big", r.Skipped) } if len(r.Matched) != 1 || r.Matched[0].File.Rel != "small.txt" { t.Errorf("matched = %+v, want small.txt", r.Matched) } } // TestExplainReportsExclusionAndSize: explain names the exclude that sets a // file aside, traces every exclude, and says a file is too big. func TestExplainReportsExclusionAndSize(t *testing.T) { h, dl := excludeTree(t, map[string]string{"a.iso": "disk", "big.txt": strings.Repeat("x", 2048)}) main := writeConfig(t, h, "(include \"dl\")\n(exclude (type iso))\n", map[string]string{"dl": ` (path "~/dl") (max-size 1K) (exclude (name "^nothing")) (rule "all" (move "Out")) `}) e, errs := Load(main) if len(errs) > 0 { t.Fatal(errs) } x, err := e.Explain(context.Background(), filepath.Join(dl, "a.iso")) if err != nil { t.Fatal(err) } if x.Excluded != "(exclude (type iso))" { t.Errorf("Excluded = %q", x.Excluded) } if len(x.Excludes) != 2 || !x.Excludes[0].Match || x.Excludes[1].Match || x.Excludes[0].Trace == nil { t.Errorf("Excludes = %+v, want the global one matching, the directory's one not", x.Excludes) } big, err := e.Explain(context.Background(), filepath.Join(dl, "big.txt")) if err != nil { t.Fatal(err) } if big.Skip != "too big" { t.Errorf("Skip = %q, want too big", big.Skip) } } // TestExcludeFailsClosedOnUnreadableContent: an exclude meant to protect // files holds when its content test cannot read a file (over max-read), so // no rule acts on a file krino could not check. func TestExcludeFailsClosedOnUnreadableContent(t *testing.T) { big := "confidential " + strings.Repeat("x", 2048) h, dl := excludeTree(t, map[string]string{"big.txt": big, "small.txt": "nothing to hide"}) main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": ` (path "~/dl") (max-read 1K) (exclude (type txt) (content "confidential")) (rule "old" (delete)) `}) e, errs := Load(main) if len(errs) > 0 { t.Fatal(errs) } r, err := e.Match(context.Background(), e.Dirs[0]) if err != nil { t.Fatal(err) } for _, fm := range r.Matched { switch fm.File.Rel { case "big.txt": if !strings.HasSuffix(fm.Excluded, "(content unreadable)") || len(fm.Rules) != 0 { t.Errorf("big.txt: Excluded %q, rules %d; want set aside as unreadable", fm.Excluded, len(fm.Rules)) } case "small.txt": if fm.Excluded != "" || len(fm.Rules) != 1 { t.Errorf("small.txt: Excluded %q, rules %d; want the rule", fm.Excluded, len(fm.Rules)) } } } x, err := e.Explain(context.Background(), filepath.Join(dl, "big.txt")) if err != nil { t.Fatal(err) } if !strings.HasSuffix(x.Excluded, "(content unreadable)") { t.Errorf("explain: Excluded %q", x.Excluded) } } // TestLoadChecksMainExcludesWithoutDirectories: a mistake in krino.conf's // (exclude ...) is reported even before any directory is included. func TestLoadChecksMainExcludesWithoutDirectories(t *testing.T) { h := sandbox(t) main := writeConfig(t, h, "(include)\n(exclude (bogus 1))\n", nil) if _, errs := Load(main); len(errs) == 0 { t.Error("a broken krino.conf exclude was not reported") } // Checked with the defaults' case and fold, as a directory would compile // them: a keyword of a lone combining mark is empty only when folded. main = writeConfig(t, h, "(include)\n(defaults (fold no))\n(exclude (content \"\u0301\"))\n", nil) if _, errs := Load(main); len(errs) != 0 { t.Errorf("checked with fold on, though the defaults say no: %v", errs) } } // TestNoTextFormatIsNoMatch: a file whose format has no text cannot contain // a keyword, so a content exclude without a type does not set it aside, and // no "content unreadable" warning is raised - while a real read failure (over // max-read) still fails closed. func TestNoTextFormatIsNoMatch(t *testing.T) { big := "confidential " + strings.Repeat("x", 2048) h, dl := excludeTree(t, map[string]string{"photo.jpg": "\xff\xd8\xff\x00\x01binary", "big.txt": big}) main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": ` (path "~/dl") (max-read 1K) (exclude (content "confidential")) (rule "pics" (when (type jpg)) (move "Pictures")) `}) e, errs := Load(main) if len(errs) > 0 { t.Fatal(errs) } r, err := e.Match(context.Background(), e.Dirs[0]) if err != nil { t.Fatal(err) } for _, fm := range r.Matched { switch fm.File.Rel { case "photo.jpg": if fm.Excluded != "" || len(fm.Rules) != 1 || len(fm.Warnings) != 0 { t.Errorf("photo.jpg: Excluded %q, rules %d, warnings %v; want the pics rule and no warning", fm.Excluded, len(fm.Rules), fm.Warnings) } case "big.txt": if !strings.HasSuffix(fm.Excluded, "(content unreadable)") { t.Errorf("big.txt: Excluded %q; a real read failure must still fail closed", fm.Excluded) } } } x, err := e.Explain(context.Background(), filepath.Join(dl, "photo.jpg")) if err != nil { t.Fatal(err) } if x.Excluded != "" { t.Errorf("explain: Excluded %q, want none", x.Excluded) } } // TestTextTurningBinaryIsNoText: a file with no known extension whose first // 8 KiB read as text but which holds a NUL further on - a self-extracting // installer - counts as having no text, like an image: a content exclude // does not set it aside and there is no warning. func TestTextTurningBinaryIsNoText(t *testing.T) { mixed := "confidential " + strings.Repeat("x", 9000) + "\x00tail" h, _ := excludeTree(t, map[string]string{"mixed": mixed}) main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": ` (path "~/dl") (exclude (content "confidential")) (rule "all" (move "Out")) `}) e, errs := Load(main) if len(errs) > 0 { t.Fatal(errs) } r, err := e.Match(context.Background(), e.Dirs[0]) if err != nil { t.Fatal(err) } for _, fm := range r.Matched { if fm.File.Rel == "mixed" && (fm.Excluded != "" || len(fm.Rules) != 1 || len(fm.Warnings) != 0) { t.Errorf("mixed: Excluded %q, rules %d, warnings %v; want the rule and no warning", fm.Excluded, len(fm.Rules), fm.Warnings) } } } // TestPlannedDestinationKeepsDiacritics: a folded name test's capture goes // into the destination as the file name wrote it. func TestPlannedDestinationKeepsDiacritics(t *testing.T) { h, _ := excludeTree(t, map[string]string{"Łódź-faktura.pdf": "x"}) main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": ` (path "~/dl") (rule "city" (when (name "^(.+)-faktura")) (move "Out/{1}")) `}) e, errs := Load(main) if len(errs) > 0 { t.Fatal(errs) } dp, err := e.Plan(context.Background(), e.Dirs[0], plan.NewClaims()) if err != nil { t.Fatal(err) } if len(dp.Chains) != 1 || !strings.HasSuffix(dp.Chains[0].Steps[0].Dst, "/Out/Łódź/Łódź-faktura.pdf") { t.Fatalf("chains = %+v; want the move into Out/Łódź", dp.Chains) } } // partDocx writes a docx whose body reads and whose footer uses a // compression method Go cannot read: a partly readable document. func partDocx(t *testing.T, path, body string) { t.Helper() var buf bytes.Buffer w := zip.NewWriter(&buf) f, err := w.Create("word/document.xml") if err != nil { t.Fatal(err) } f.Write([]byte(`` + body + ``)) raw, err := w.CreateRaw(&zip.FileHeader{Name: "word/footer1.xml", Method: 12}) if err != nil { t.Fatal(err) } raw.Write([]byte("BZh9 not really bzip2 confidential")) if err := w.Close(); err != nil { t.Fatal(err) } if err := os.WriteFile(path, buf.Bytes(), 0o644); err != nil { t.Fatal(err) } old := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) os.Chtimes(path, old, old) } // TestPartlyReadableDocument: a docx with an unreadable part answers a // keyword it holds in the readable part, but a keyword not found there is // unknown: a content exclude sets the file aside, a rule warns. func TestPartlyReadableDocument(t *testing.T) { h, dl := excludeTree(t, map[string]string{}) os.MkdirAll(dl, 0o755) partDocx(t, filepath.Join(dl, "part.docx"), "good body text") main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": ` (path "~/dl") (exclude (content "confidential")) (rule "all" (move "Out")) `}) e, errs := Load(main) if len(errs) > 0 { t.Fatal(errs) } r, err := e.Match(context.Background(), e.Dirs[0]) if err != nil { t.Fatal(err) } if len(r.Matched) != 1 || !strings.HasSuffix(r.Matched[0].Excluded, "(content unreadable)") { t.Fatalf("matched = %+v; want part.docx set aside as unreadable", r.Matched) } os.WriteFile(filepath.Join(h, ".config", "krino", "dirs", "dl.conf"), []byte(` (path "~/dl") (rule "body" (when (content "good body")) (move "Docs")) (rule "other" (when (content "nowhere")) (move "Other")) `), 0o644) e, errs = Load(main) if len(errs) > 0 { t.Fatal(errs) } r, err = e.Match(context.Background(), e.Dirs[0]) if err != nil { t.Fatal(err) } fm := r.Matched[0] if len(fm.Rules) != 1 || fm.Rules[0].Rule.Name != "body" { t.Errorf("rules = %+v; want only body, found in the readable part", fm.Rules) } if len(fm.Warnings) == 0 { t.Errorf("no warning for the keyword the unreadable part might hold") } } // TestExplainLeavesTheCacheAlone: explain never writes the keyword cache - // not even to remove one a directory without content tests no longer uses // (explain runs without the directory's lock). func TestExplainLeavesTheCacheAlone(t *testing.T) { h, dl := excludeTree(t, map[string]string{"a.txt": "x"}) main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": "(path \"~/dl\")\n(rule \"all\" (move \"Out\"))\n"}) e, errs := Load(main) if len(errs) > 0 { t.Fatal(errs) } e.CacheDir = filepath.Join(h, "cache") os.MkdirAll(e.CacheDir, 0o700) cache := filepath.Join(e.CacheDir, "dl.cache") os.WriteFile(cache, []byte("old"), 0o600) if _, err := e.Explain(context.Background(), filepath.Join(dl, "a.txt")); err != nil { t.Fatal(err) } if _, err := os.Stat(cache); err != nil { t.Errorf("explain removed the cache: %v", err) } } // TestPartlyReadableDocumentIsNotCached: a partial read's answers never // enter the keyword cache, so a second run with the cache still sets the // file aside. func TestPartlyReadableDocumentIsNotCached(t *testing.T) { h, dl := excludeTree(t, map[string]string{}) os.MkdirAll(dl, 0o755) partDocx(t, filepath.Join(dl, "part.docx"), "good body text") main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": ` (path "~/dl") (exclude (content "confidential")) (rule "all" (move "Out")) `}) for run := 1; run <= 2; run++ { e, errs := Load(main) if len(errs) > 0 { t.Fatal(errs) } e.CacheDir = filepath.Join(h, "cache") r, err := e.Match(context.Background(), e.Dirs[0]) if err != nil { t.Fatal(err) } if len(r.Matched) != 1 || !strings.HasSuffix(r.Matched[0].Excluded, "(content unreadable)") { t.Fatalf("run %d: matched = %+v; want part.docx set aside", run, r.Matched) } } } // TestExplainAgreesWithMatchOnADuplicateExclude: a (duplicate) test inside // an exclude needs the directory's other files, as one inside a rule does, // so explain sets aside exactly the files Match does. func TestExplainAgreesWithMatchOnADuplicateExclude(t *testing.T) { h, dl := excludeTree(t, map[string]string{"a.txt": "same", "b.txt": "same", "c.txt": "other"}) main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": ` (path "~/dl") (exclude (not (duplicate))) (rule "all" (move "Out")) `}) e, errs := Load(main) if len(errs) > 0 { t.Fatal(errs) } r, err := e.Match(context.Background(), e.Dirs[0]) if err != nil { t.Fatal(err) } for _, fm := range append(append([]FileMatch{}, r.Matched...), r.Unmatched...) { x, err := e.Explain(context.Background(), filepath.Join(dl, fm.File.Rel)) if err != nil { t.Fatal(err) } if x.Excluded != fm.Excluded { t.Errorf("%s: explain sets it aside as %q, Match as %q", fm.File.Rel, x.Excluded, fm.Excluded) } } } // TestNotMatchedAfterAnUnknownRuleDoesNotAct: a partly read document an // earlier content rule could not decide is not caught by a later // (not (matched)) catch-all. func TestNotMatchedAfterAnUnknownRuleDoesNotAct(t *testing.T) { h, dl := excludeTree(t, map[string]string{}) os.MkdirAll(dl, 0o755) partDocx(t, filepath.Join(dl, "part.docx"), "good body text") main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": ` (path "~/dl") (rule "acme" (when (content "acme ltd")) (move "Acme")) (rule "rest" (when (not (matched))) (move "Unsorted")) `}) e, errs := Load(main) if len(errs) > 0 { t.Fatal(errs) } r, err := e.Match(context.Background(), e.Dirs[0]) if err != nil { t.Fatal(err) } for _, fm := range r.Matched { if fm.File.Rel == "part.docx" { t.Errorf("part.docx matched %d rules; the catch-all must not take a file an earlier rule could not decide", len(fm.Rules)) } } x, err := e.Explain(context.Background(), filepath.Join(dl, "part.docx")) if err != nil { t.Fatal(err) } for _, rt := range x.Rules { if rt.Match { t.Errorf("explain: rule %s matches", rt.Rule.Name) } } } // TestDuplicateExcludeFailsClosed: an exclude whose duplicate lookup fails // holds, marked as such. func TestDuplicateExcludeFailsClosed(t *testing.T) { if os.Getuid() == 0 { t.Skip("root reads a chmod 000 file") } h, dl := excludeTree(t, map[string]string{"a.txt": "same size", "b.txt": "same size"}) os.Chmod(filepath.Join(dl, "a.txt"), 0) t.Cleanup(func() { os.Chmod(filepath.Join(dl, "a.txt"), 0o644) }) main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": ` (path "~/dl") (exclude (duplicate)) (rule "all" (move "Out")) `}) e, errs := Load(main) if len(errs) > 0 { t.Fatal(errs) } r, err := e.Match(context.Background(), e.Dirs[0]) if err != nil { t.Fatal(err) } for _, fm := range r.Matched { if fm.File.Rel == "a.txt" && fm.Excluded != "(exclude (duplicate)) (duplicate check failed)" { t.Errorf("a.txt: Excluded %q, rules %d; want set aside as duplicate check failed", fm.Excluded, len(fm.Rules)) } } } // TestUndecidedStopRuleStops: a (stop) rule whose condition cannot be // decided ends the search for that file, so a later rule does not act on a // file the stop rule was written to keep. func TestUndecidedStopRuleStops(t *testing.T) { big := "confidential " + strings.Repeat("x", 2048) h, dl := excludeTree(t, map[string]string{"big.txt": big, "small.txt": "nothing"}) main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": ` (path "~/dl") (max-read 1K) (rule "keep" (when (content "confidential")) (stop)) (rule "all" (move "Out")) `}) e, errs := Load(main) if len(errs) > 0 { t.Fatal(errs) } r, err := e.Match(context.Background(), e.Dirs[0]) if err != nil { t.Fatal(err) } for _, fm := range append(append([]FileMatch{}, r.Matched...), r.Unmatched...) { switch fm.File.Rel { case "big.txt": if len(fm.Rules) != 0 || len(fm.Warnings) == 0 { t.Errorf("big.txt: rules %d, warnings %v; want no rule and the warning", len(fm.Rules), fm.Warnings) } case "small.txt": if len(fm.Rules) != 1 { t.Errorf("small.txt: rules %d; want the move", len(fm.Rules)) } } } x, err := e.Explain(context.Background(), filepath.Join(dl, "big.txt")) if err != nil { t.Fatal(err) } if len(x.Rules) != 2 || x.Rules[1].Stopped != "stopped by rule keep, which could not be decided" { t.Errorf("explain rules = %+v; want all stopped by the undecided keep", x.Rules) } }