diff options
| author | Lukasz Kasprzak <lukas@labunix.xyz> | 2026-09-14 14:08:36 +0200 |
|---|---|---|
| committer | Lukasz Kasprzak <lukas@labunix.xyz> | 2026-09-14 14:08:36 +0200 |
| commit | 1d3f2d1e4c59867024470d3444e12698b7ebb22e (patch) | |
| tree | 4fa14f455c72f9b206a680dcc1b0f4c1f3708158 /internal | |
| parent | 07c24054cab965800983ef40f53a05c2db131ede (diff) | |
| download | krino-836b2a4ec7f218eb8256cf936ff0070f588fb58d.tar.gz krino-836b2a4ec7f218eb8256cf936ff0070f588fb58d.zip | |
0.0.4: max-size, exclude forms, --min-agev0.0.4
Diffstat (limited to 'internal')
| -rw-r--r-- | internal/config/dir.go | 38 | ||||
| -rw-r--r-- | internal/config/dir_test.go | 36 | ||||
| -rw-r--r-- | internal/config/main.go | 7 | ||||
| -rw-r--r-- | internal/config/main_test.go | 18 | ||||
| -rw-r--r-- | internal/config/settings.go | 15 | ||||
| -rw-r--r-- | internal/config/settings_test.go | 16 | ||||
| -rw-r--r-- | internal/config/skel/krino.conf | 7 | ||||
| -rw-r--r-- | internal/config/skel/template.conf | 6 | ||||
| -rw-r--r-- | internal/engine/engine.go | 47 | ||||
| -rw-r--r-- | internal/engine/exclude_test.go | 209 | ||||
| -rw-r--r-- | internal/engine/match.go | 49 | ||||
| -rw-r--r-- | internal/scan/scan.go | 8 | ||||
| -rw-r--r-- | internal/scan/scan_test.go | 32 |
13 files changed, 471 insertions, 17 deletions
diff --git a/internal/config/dir.go b/internal/config/dir.go index 74c538a..461d912 100644 --- a/internal/config/dir.go +++ b/internal/config/dir.go @@ -19,9 +19,41 @@ type Dir struct { PathText string // as written in the file Settings Settings Ignore []string // gitignore patterns, in order + Excludes []*Exclude Rules []*Rule } +// Exclude is one (exclude COND...) form: a file for which every condition +// holds is set aside before any rule runs. Forms may repeat, so a file is +// excluded when any one form matches it. +type Exclude struct { + Pos sexp.Pos + Text string // the form as written, whitespace collapsed, for check and explain + When []*sexp.Node // the conditions, all of which must hold +} + +// parseExclude reads an (exclude COND...) form from src; nil when it has no +// usable condition. +func parseExclude(n *sexp.Node, src []byte, d *diags) *Exclude { + conds := n.Args() + if len(conds) == 0 { + d.at(n, "(exclude) needs a condition, like (exclude (type iso))") + return nil + } + bad := false + for _, c := range conds { + if c.Kind != sexp.List { + d.at(c, "exclude: a condition is a form like (type pdf), not %s", c) + bad = true + } + } + if bad { + return nil + } + text := strings.Join(strings.Fields(string(src[n.Pos.Offset:n.End.Offset])), " ") + return &Exclude{Pos: n.Pos, Text: text, When: conds} +} + // Rule is a named condition with the actions it performs. type Rule struct { Name string @@ -97,6 +129,10 @@ func ParseDir(name, file string, src []byte) (*Dir, []*Diag) { } dir.Ignore = append(dir.Ignore, a.Text) } + case head == "exclude": + if x := parseExclude(n, src, d); x != nil { + dir.Excludes = append(dir.Excludes, x) + } case head == "rule": r := parseRule(n, d) if r == nil { @@ -111,7 +147,7 @@ func ParseDir(name, file string, src []byte) (*Dir, []*Diag) { case isSetting(head): dir.Settings.parse(n, d, seen) default: - d.at(n, "unknown form (%s ...); a directory file has path, ignore, rule and settings like (recursive yes)", head) + d.at(n, "unknown form (%s ...); a directory file has path, ignore, exclude, rule and settings like (recursive yes)", head) } } if pathNode == nil { diff --git a/internal/config/dir_test.go b/internal/config/dir_test.go index 7ddfa2a..080905c 100644 --- a/internal/config/dir_test.go +++ b/internal/config/dir_test.go @@ -4,6 +4,7 @@ package config import ( "reflect" + "strings" "testing" ) @@ -111,7 +112,7 @@ func TestParseDirErrors(t *testing.T) { {`(path "/a") (rule "x" (delete) (move "y"))`, `d.conf:1:32: rule "x": (move "y") after delete would never run`}, {`(path "/a") (rule "x" (stop now))`, `d.conf:1:23: rule "x": stop takes nothing: write (stop)`}, {`(path "/a") (rule "x" (fly "y"))`, `d.conf:1:23: rule "x": unknown form (fly ...); a rule has when, copy, move, rename, delete, stop, case, fold and on-conflict`}, - {`(path "/a") (sort "x")`, `d.conf:1:13: unknown form (sort ...); a directory file has path, ignore, rule and settings like (recursive yes)`}, + {`(path "/a") (sort "x")`, `d.conf:1:13: unknown form (sort ...); a directory file has path, ignore, exclude, rule and settings like (recursive yes)`}, } for _, tt := range tests { _, errs := ParseDir("d", "d.conf", []byte(tt.src)) @@ -120,3 +121,36 @@ func TestParseDirErrors(t *testing.T) { } } } + +// TestParseExclude: (exclude COND...) may repeat; each form keeps its +// conditions (all must hold, as in when) and its text as written, with the +// whitespace collapsed, for check and explain to show. +func TestParseExclude(t *testing.T) { + src := `(path "/tmp") +(exclude (type iso img)) +(exclude (name "^draft") + (content "poufne")) +` + dir, errs := ParseDir("dl", "dl.conf", []byte(src)) + if len(errs) != 0 { + t.Fatal(errs) + } + if len(dir.Excludes) != 2 || len(dir.Excludes[0].When) != 1 || len(dir.Excludes[1].When) != 2 { + t.Fatalf("Excludes = %+v", dir.Excludes) + } + if got, want := dir.Excludes[1].Text, `(exclude (name "^draft") (content "poufne"))`; got != want { + t.Errorf("Text = %q, want %q", got, want) + } + if dir.Excludes[1].Pos.Line != 3 { + t.Errorf("Pos = %+v, want line 3", dir.Excludes[1].Pos) + } + for _, tt := range []struct{ src, want string }{ + {"(path \"/tmp\")\n(exclude)\n", "(exclude) needs a condition"}, + {"(path \"/tmp\")\n(exclude iso)\n", "exclude: a condition is a form like (type pdf), not iso"}, + } { + _, errs := ParseDir("dl", "dl.conf", []byte(tt.src)) + if len(errs) != 1 || !strings.Contains(errs[0].Msg, tt.want) { + t.Errorf("%q: errs %v, want one containing %q", tt.src, errs, tt.want) + } + } +} diff --git a/internal/config/main.go b/internal/config/main.go index ee6c3df..2764c8a 100644 --- a/internal/config/main.go +++ b/internal/config/main.go @@ -21,6 +21,7 @@ type Main struct { IncludeNode *sexp.Node Log string // absolute; empty means the default Defaults Settings + Excludes []*Exclude // apply to every directory, before its own } // nameRE is what a directory name may look like: it becomes a file name. @@ -87,8 +88,12 @@ func ParseMain(file string, src []byte) (*Main, []*Diag) { } m.Defaults.parse(a, d, dseen) } + case "exclude": + if x := parseExclude(n, src, d); x != nil { + m.Excludes = append(m.Excludes, x) + } default: - d.at(n, "unknown form (%s ...); krino.conf has include, log and defaults", head) + d.at(n, "unknown form (%s ...); krino.conf has include, log, defaults and exclude", head) } } return m, d.list diff --git a/internal/config/main_test.go b/internal/config/main_test.go index a82dfcd..be39faf 100644 --- a/internal/config/main_test.go +++ b/internal/config/main_test.go @@ -4,6 +4,7 @@ package config import ( "reflect" + "strings" "testing" "time" ) @@ -54,7 +55,7 @@ func TestParseMainErrors(t *testing.T) { {`(log "rel/x")`, `k:1:6: log path must be absolute or start with ~`}, {`(defaults (recursive maybe))`, `k:1:22: recursive is yes or no, not maybe`}, {`(defaults (rule "x"))`, `k:1:11: defaults holds settings like (min-age 2m); got (rule "x")`}, - {`(inlcude "a")`, `k:1:1: unknown form (inlcude ...); krino.conf has include, log and defaults`}, + {`(inlcude "a")`, `k:1:1: unknown form (inlcude ...); krino.conf has include, log, defaults and exclude`}, {`include`, `k:1:1: expected a form like (include ...), got include`}, {`(include "a"`, `k:1:1: "(" never closed: (include "a")`}, } @@ -65,3 +66,18 @@ func TestParseMainErrors(t *testing.T) { } } } + +// TestParseMainExclude: krino.conf may hold (exclude ...) forms, which +// apply to every directory. +func TestParseMainExclude(t *testing.T) { + m, errs := ParseMain("krino.conf", []byte("(include \"a\")\n(exclude (type iso))\n(exclude (name \"[.]asc$\"))\n")) + if len(errs) != 0 { + t.Fatal(errs) + } + if len(m.Excludes) != 2 || m.Excludes[0].Text != "(exclude (type iso))" { + t.Fatalf("Excludes = %+v", m.Excludes) + } + if _, errs := ParseMain("krino.conf", []byte("(exclude)")); len(errs) != 1 || !strings.Contains(errs[0].Msg, "(exclude) needs a condition") { + t.Errorf("(exclude): errs %v", errs) + } +} diff --git a/internal/config/settings.go b/internal/config/settings.go index 97355a5..efc935b 100644 --- a/internal/config/settings.go +++ b/internal/config/settings.go @@ -34,6 +34,7 @@ type Settings struct { MaxDepth *int MinAge *time.Duration MaxRead *int64 + MaxSize *int64 Busy *[]string OnConflict *Conflict } @@ -46,6 +47,7 @@ type Resolved struct { MaxDepth int // 0 means unlimited MinAge time.Duration MaxRead int64 + MaxSize int64 // 0 means unlimited Busy []string OnConflict Conflict } @@ -83,6 +85,9 @@ func (s Settings) Over(base Resolved) Resolved { if s.MaxRead != nil { r.MaxRead = *s.MaxRead } + if s.MaxSize != nil { + r.MaxSize = *s.MaxSize + } if s.Busy != nil { r.Busy = *s.Busy } @@ -92,7 +97,7 @@ func (s Settings) Over(base Resolved) Resolved { return r } -var settingNames = []string{"case", "fold", "recursive", "max-depth", "min-age", "max-read", "busy", "on-conflict"} +var settingNames = []string{"case", "fold", "recursive", "max-depth", "min-age", "max-read", "max-size", "busy", "on-conflict"} // ruleSettings are the settings a rule may override. var ruleSettings = map[string]bool{"case": true, "fold": true, "on-conflict": true} @@ -106,6 +111,7 @@ var settingHint = map[string]string{ "max-depth": "a number, like (max-depth 3)", "min-age": "a duration, like (min-age 2m)", "max-read": "a size, like (max-read 50M)", + "max-size": "a size, like (max-size 1G)", "on-conflict": "(on-conflict suffix), skip or overwrite", } @@ -183,6 +189,13 @@ func (s *Settings) parse(n *sexp.Node, d *diags, seen map[string]*sexp.Node) { return } s.MaxRead = &size + case "max-size": + size, err := ParseSize(v) + if err != nil { + d.at(at, "max-size: %v", err) + return + } + s.MaxSize = &size case "on-conflict": c, ok := map[string]Conflict{"suffix": ConflictSuffix, "skip": ConflictSkip, "overwrite": ConflictOverwrite}[v] if !ok { diff --git a/internal/config/settings_test.go b/internal/config/settings_test.go index e46c880..341c0f3 100644 --- a/internal/config/settings_test.go +++ b/internal/config/settings_test.go @@ -4,6 +4,7 @@ package config import ( "reflect" + "strings" "testing" "time" @@ -27,12 +28,12 @@ func parseSettings(t *testing.T, src string) (Settings, []*Diag) { func TestSettingsResolve(t *testing.T) { s, errs := parseSettings(t, `(case strict) (fold no) (recursive yes) (max-depth 3) - (min-age 5m) (max-read 1G) (busy ".tmp") (on-conflict skip)`) + (min-age 5m) (max-read 1G) (max-size 2G) (busy ".tmp") (on-conflict skip)`) if len(errs) > 0 { t.Fatal(errs) } want := Resolved{Case: CaseStrict, Fold: false, Recursive: true, MaxDepth: 3, - MinAge: 5 * time.Minute, MaxRead: 1 << 30, Busy: []string{".tmp"}, OnConflict: ConflictSkip} + MinAge: 5 * time.Minute, MaxRead: 1 << 30, MaxSize: 2 << 30, Busy: []string{".tmp"}, OnConflict: ConflictSkip} if got := s.Over(Builtin()); !reflect.DeepEqual(got, want) { t.Fatalf("got %+v\nwant %+v", got, want) } @@ -83,3 +84,14 @@ func TestSettingErrors(t *testing.T) { } } } + +// TestMaxSizeErrors: max-size takes a size like max-read does, and is not a +// rule-level setting. +func TestMaxSizeErrors(t *testing.T) { + if _, errs := parseSettings(t, "(max-size big)"); len(errs) != 1 || !strings.Contains(errs[0].Msg, "max-size") { + t.Errorf("(max-size big): errs %v, want one max-size error", errs) + } + if _, errs := parseSettings(t, "(max-size 10M)"); len(errs) != 0 { + t.Errorf("(max-size 10M): errs %v, want none", errs) + } +} diff --git a/internal/config/skel/krino.conf b/internal/config/skel/krino.conf index dbd883b..7008dbc 100644 --- a/internal/config/skel/krino.conf +++ b/internal/config/skel/krino.conf @@ -19,5 +19,12 @@ ;; (recursive no) ;; (min-age 2m) ; skip files modified in the last 2 minutes ;; (max-read 50M) ; no content extraction above this size +;; (max-size 2G) ; skip files larger than this entirely ;; (busy ".part" ".aria2" ".crdownload") ;; (on-conflict suffix)) ; suffix | skip | overwrite + +;; Files no rule in any directory may touch. The conditions in one form +;; must all hold; a file matching any form is set aside. Some examples: +;; (exclude (type iso)) ; by type or extension +;; (exclude (name "^keep-")) ; by name, a regex +;; (exclude (type pdf) (content "confidential")) ; by content diff --git a/internal/config/skel/template.conf b/internal/config/skel/template.conf index c3af09d..d366716 100644 --- a/internal/config/skel/template.conf +++ b/internal/config/skel/template.conf @@ -15,10 +15,16 @@ ;; (fold yes) ; yes: "spolka" matches "spółka" ;; (min-age 2m) ; skip files modified in the last 2 minutes ;; (max-read 50M) ; no content extraction above this size +;; (max-size 2G) ; skip files larger than this entirely ;; (on-conflict suffix) ; suffix | skip | overwrite ;; Files and directories to leave alone, in .gitignore syntax. (ignore "*.part" "*.crdownload" "*.aria2" ".*") +;; Files no rule here may touch, tested before any rule. The conditions in +;; one form must all hold; a file matching any form is set aside. +;; (exclude (type iso img)) ; by extension +;; (exclude (name "^keep-")) ; by name, a regex +;; (exclude (type pdf) (content "confidential")) ; by content ;; Rules run top to bottom. Every rule that matches a file adds its actions ;; to that file; (stop) ends the search for it. Some examples: diff --git a/internal/engine/engine.go b/internal/engine/engine.go index b5f2106..8cffbd1 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -46,6 +46,11 @@ type Dir struct { // stay memoised, as before. ContentVariants []cond.Options + // Excludes are the (exclude ...) forms that apply here, compiled with the + // directory's settings: krino.conf's first, then the directory's own. A + // file any of them matches is set aside before any rule runs. + Excludes []*Exclude + // DupScopes is every distinct directory list the duplicate tests of // Rules use, in first-seen order, the plain (duplicate) as an empty // list. Spec §5.5 rule 2 looks a file up under each of them before any @@ -53,6 +58,12 @@ type Dir struct { DupScopes [][]string } +// Exclude is one compiled (exclude ...) form. +type Exclude struct { + Text string // the form as written, for check, explain and the plan + Cond *cond.Cond +} + // Rule is one directory's rule, with its condition compiled. type Rule struct { Name string @@ -71,6 +82,9 @@ func Load(mainFile string, names ...string) (*Engine, []*config.Diag) { } var dirs []*Dir + // A mistake inside a krino.conf exclude is compiled once per directory, + // but reported once. + reported := map[string]bool{} for _, d := range cfg.Dirs { dir := &Dir{ Name: d.Name, @@ -103,7 +117,25 @@ func Load(mainFile string, names ...string) (*Engine, []*config.Diag) { } dir.Rules = append(dir.Rules, &Rule{Name: r.Name, Conf: r, Settings: rs, Cond: c}) } - dir.ContentVariants = contentVariants(dir.Rules) + dirOpt := cond.Options{IgnoreCase: dir.Settings.Case == config.CaseIgnore, Fold: dir.Settings.Fold} + for _, src := range []struct { + file string + excludes []*config.Exclude + }{{cfg.Main.File, cfg.Main.Excludes}, {d.File, d.Excludes}} { + for _, x := range src.excludes { + c, cerrs := cond.Compile(src.file, x.When, dirOpt) + for _, ce := range cerrs { + if key := ce.Error(); !reported[key] { + reported[key] = true + errs = append(errs, ce) + } + } + if len(cerrs) == 0 { + dir.Excludes = append(dir.Excludes, &Exclude{Text: x.Text, Cond: c}) + } + } + } + dir.ContentVariants = contentVariants(dir.Rules, dir.Excludes, dirOpt) dir.DupScopes = dupScopes(dir.Rules) dirs = append(dirs, dir) } @@ -184,14 +216,21 @@ func dedupeNames(names []string) []string { return out } -// contentVariants returns the distinct (IgnoreCase, Fold) pairs any of -// rules' content tests evaluate under, in first-seen order — B2's per-Dir +// contentVariants returns the distinct (IgnoreCase, Fold) pairs any content +// test evaluates under - each exclude's (under the directory's settings, +// dirOpt) and each rule's - in first-seen order — B2's per-Dir // ContentVariants. A rule whose condition has no content test at all // (Cond.UsesContent false) never calls facts.Content, so its resolved // case/fold settings contribute no variant here. -func contentVariants(rules []*Rule) []cond.Options { +func contentVariants(rules []*Rule, excludes []*Exclude, dirOpt cond.Options) []cond.Options { var out []cond.Options seen := map[cond.Options]bool{} + for _, x := range excludes { + if x.Cond.UsesContent && !seen[dirOpt] { + seen[dirOpt] = true + out = append(out, dirOpt) + } + } for _, r := range rules { if !r.Cond.UsesContent { continue diff --git a/internal/engine/exclude_test.go b/internal/engine/exclude_test.go new file mode 100644 index 0000000..40b3548 --- /dev/null +++ b/internal/engine/exclude_test.go @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package engine + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "krino/internal/plan" + "krino/internal/scan" +) + +// excludeTree creates ~/dl with files (name -> content), all old enough to +// be scanned, and returns home and dl. +func excludeTree(t *testing.T, files map[string]string) (home, dl string) { + t.Helper() + home = sandbox(t) + t.Setenv("PATH", t.TempDir()) + dl = filepath.Join(home, "dl") + old := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for name, body := range files { + p := filepath.Join(dl, name) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(p, old, old); err != nil { + t.Fatal(err) + } + } + return home, dl +} + +// TestExcludeSetsFilesAside: an (exclude ...) in krino.conf applies to the +// directory, and the directory's own excludes apply too - by type, by name +// regex and by content - so those files get no actions from any rule and +// are reported as excluded, with the form that matched. +func TestExcludeSetsFilesAside(t *testing.T) { + h, _ := excludeTree(t, map[string]string{ + "a.iso": "disk image", + "draft-1.pdf": "%PDF draft", + "secret.txt": "this is poufne material", + "keep.txt": "ordinary notes", + }) + main := writeConfig(t, h, "(include \"dl\")\n(exclude (type iso))\n", map[string]string{"dl": ` +(path "~/dl") +(exclude (name "^draft")) +(exclude (type txt) (content "poufne")) +(rule "all" (move "Out")) +`}) + e, errs := Load(main) + if len(errs) > 0 { + t.Fatal(errs) + } + dp, err := e.Plan(context.Background(), e.Dirs[0], plan.NewClaims()) + if err != nil { + t.Fatal(err) + } + want := map[string]string{ + "a.iso": "(exclude (type iso))", + "draft-1.pdf": `(exclude (name "^draft"))`, + "secret.txt": `(exclude (type txt) (content "poufne"))`, + "keep.txt": "", + } + for _, fm := range dp.Result.Matched { + w, ok := want[fm.File.Rel] + if !ok { + t.Errorf("unexpected matched file %s", fm.File.Rel) + continue + } + if fm.Excluded != w { + t.Errorf("%s: Excluded = %q, want %q", fm.File.Rel, fm.Excluded, w) + } + if w != "" && len(fm.Rules) != 0 { + t.Errorf("%s: excluded but matched rules %v", fm.File.Rel, fm.Rules) + } + } + if len(dp.Result.Matched) != 4 || len(dp.Result.Unmatched) != 0 { + t.Errorf("matched %d, unmatched %d; want every file matched (3 excluded, 1 by the rule)", len(dp.Result.Matched), len(dp.Result.Unmatched)) + } + for _, c := range dp.Chains { + if acting := len(c.Steps) > 0; acting != (c.File.Rel == "keep.txt") { + t.Errorf("%s: steps %+v", c.File.Rel, c.Steps) + } + } +} + +// TestExcludeNeedsEveryCondition: within one form, every condition must +// hold, as in when. +func TestExcludeNeedsEveryCondition(t *testing.T) { + h, _ := excludeTree(t, map[string]string{"draft.txt": "x", "draft.pdf": "%PDF x"}) + main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": ` +(path "~/dl") +(exclude (type pdf) (name "^draft")) +(rule "all" (move "Out")) +`}) + e, errs := Load(main) + if len(errs) > 0 { + t.Fatal(errs) + } + r, err := e.Match(context.Background(), e.Dirs[0]) + if err != nil { + t.Fatal(err) + } + for _, fm := range r.Matched { + if excluded := fm.Excluded != ""; excluded != (fm.File.Rel == "draft.pdf") { + t.Errorf("%s: Excluded = %q", fm.File.Rel, fm.Excluded) + } + } +} + +// TestExcludeContentKeepsRuleContentVariants: an exclude reading content +// under the directory's settings must not release the raw text a rule +// with different case/fold settings still needs. +func TestExcludeContentKeepsRuleContentVariants(t *testing.T) { + h, _ := excludeTree(t, map[string]string{"a.txt": "Invoice ACME"}) + main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": ` +(path "~/dl") +(exclude (content "never present")) +(rule "strict" (case strict) (when (content "ACME")) (move "Out")) +`}) + e, errs := Load(main) + if len(errs) > 0 { + t.Fatal(errs) + } + r, err := e.Match(context.Background(), e.Dirs[0]) + if err != nil { + t.Fatal(err) + } + if len(r.Matched) != 1 || len(r.Matched[0].Rules) != 1 { + t.Fatalf("a.txt should match the strict content rule after the exclude read its text: %+v", r.Matched) + } +} + +// TestLoadReportsBadExcludeOnce: a condition error inside a krino.conf +// exclude is reported once, not once per directory. +func TestLoadReportsBadExcludeOnce(t *testing.T) { + h := sandbox(t) + main := writeConfig(t, h, "(include \"a\" \"b\")\n(exclude (size big))\n", map[string]string{ + "a": "(path \"/tmp\")", "b": "(path \"/tmp\")", + }) + _, errs := Load(main) + if len(errs) != 1 || !strings.Contains(errs[0].Error(), "size") { + t.Errorf("errs = %v, want exactly one error about size", errs) + } +} + +// TestMaxSizeSkipsTooBig: a directory's max-size skips larger files before +// any rule, as too big. +func TestMaxSizeSkipsTooBig(t *testing.T) { + h, _ := excludeTree(t, map[string]string{"small.txt": "x", "big.txt": strings.Repeat("x", 2048)}) + main := writeConfig(t, h, "(include \"dl\")\n(defaults (max-size 1K))\n", map[string]string{"dl": ` +(path "~/dl") +(rule "all" (move "Out")) +`}) + e, errs := Load(main) + if len(errs) > 0 { + t.Fatal(errs) + } + r, err := e.Match(context.Background(), e.Dirs[0]) + if err != nil { + t.Fatal(err) + } + if len(r.Skipped) != 1 || r.Skipped[0].Rel != "big.txt" || r.Skipped[0].Reason != scan.TooBig { + t.Errorf("skipped = %+v, want big.txt too big", r.Skipped) + } + if len(r.Matched) != 1 || r.Matched[0].File.Rel != "small.txt" { + t.Errorf("matched = %+v, want small.txt", r.Matched) + } +} + +// TestExplainReportsExclusionAndSize: explain names the exclude that sets a +// file aside, traces every exclude, and says a file is too big. +func TestExplainReportsExclusionAndSize(t *testing.T) { + h, dl := excludeTree(t, map[string]string{"a.iso": "disk", "big.txt": strings.Repeat("x", 2048)}) + main := writeConfig(t, h, "(include \"dl\")\n(exclude (type iso))\n", map[string]string{"dl": ` +(path "~/dl") +(max-size 1K) +(exclude (name "^nothing")) +(rule "all" (move "Out")) +`}) + e, errs := Load(main) + if len(errs) > 0 { + t.Fatal(errs) + } + x, err := e.Explain(context.Background(), filepath.Join(dl, "a.iso")) + if err != nil { + t.Fatal(err) + } + if x.Excluded != "(exclude (type iso))" { + t.Errorf("Excluded = %q", x.Excluded) + } + if len(x.Excludes) != 2 || !x.Excludes[0].Match || x.Excludes[1].Match || x.Excludes[0].Trace == nil { + t.Errorf("Excludes = %+v, want the global one matching, the directory's one not", x.Excludes) + } + big, err := e.Explain(context.Background(), filepath.Join(dl, "big.txt")) + if err != nil { + t.Fatal(err) + } + if big.Skip != "too big" { + t.Errorf("Skip = %q, want too big", big.Skip) + } +} diff --git a/internal/engine/match.go b/internal/engine/match.go index ffee7f0..8daea9d 100644 --- a/internal/engine/match.go +++ b/internal/engine/match.go @@ -34,6 +34,10 @@ type FileMatch struct { Rules []RuleMatch // matching rules in order, ending at the first with (stop) Warnings []string // "<rule>: <warning>", e.g. "acme: content unreadable: needs pdftotext, not installed" + // Excluded is the (exclude ...) form that set this file aside before any + // rule ran, as written; "" when none did. An excluded file has no Rules. + Excluded string + // NoDelete is non-empty when no delete step may run for this file (spec // §5.5 rule 2), and says why: the file is a duplicate under a scope its // directory's rules use, or that check failed. Only set for a file some @@ -92,7 +96,7 @@ func (e *Engine) Match(ctx context.Context, d *Dir) (*Result, error) { var matched, unmatched []FileMatch for _, fm := range fileMatches { - if len(fm.Rules) > 0 { + if len(fm.Rules) > 0 || fm.Excluded != "" { matched = append(matched, fm) } else { unmatched = append(unmatched, fm) @@ -119,6 +123,16 @@ func (e *Engine) Match(ctx context.Context, d *Dir) (*Result, error) { func evalFile(run *matchRun, file scan.File) FileMatch { f := newFacts(run, file) fm := FileMatch{File: file} + for _, x := range run.d.Excludes { + res := x.Cond.Eval(f) + for _, w := range res.Warnings { + fm.Warnings = append(fm.Warnings, "exclude: "+w) + } + if res.Match { + fm.Excluded = x.Text + return fm + } + } for _, r := range run.d.Rules { res := r.Cond.Eval(f) for _, w := range res.Warnings { @@ -180,12 +194,21 @@ type RuleTrace struct { Stopped string // "stopped by rule acme" when an earlier (stop) ended the search } +// ExcludeTrace is one (exclude ...) form's outcome in an Explain call. +type ExcludeTrace struct { + Text string + Match bool + Trace *cond.Trace +} + // Explanation is why (or why not) krino would act on one file. type Explanation struct { - Dir *Dir - File scan.File - Skip string // why krino would not look at this file at all; "" when it would - Rules []RuleTrace + Dir *Dir + File scan.File + Skip string // why krino would not look at this file at all; "" when it would + Excludes []ExcludeTrace + Excluded string // the first exclude that matches, which sets the file aside; "" when none does + Rules []RuleTrace } // Explain reports, for one file, whether krino's ordinary scan would ever @@ -235,6 +258,16 @@ func (e *Engine) Explain(ctx context.Context, path string) (*Explanation, error) run := newMatchRun(e, d, ctx, now, e.filesForExplain(d, sf, excl, now)) f := newFacts(run, sf) + var excludes []ExcludeTrace + excluded := "" + for _, x := range d.Excludes { + trace := x.Cond.Explain(f) + excludes = append(excludes, ExcludeTrace{Text: x.Text, Match: trace.Value, Trace: trace}) + if trace.Value && excluded == "" { + excluded = x.Text + } + } + var rules []RuleTrace stoppedBy := "" for _, r := range d.Rules { @@ -253,7 +286,7 @@ func (e *Engine) Explain(ctx context.Context, path string) (*Explanation, error) } } - return &Explanation{Dir: d, File: sf, Skip: skip, Rules: rules}, nil + return &Explanation{Dir: d, File: sf, Skip: skip, Excludes: excludes, Excluded: excluded, Rules: rules}, nil } // filesForExplain returns the file set Explain's duplicate checks run @@ -285,6 +318,7 @@ func walkOptions(d *Dir, excl []string, now time.Time) scan.Options { Exclude: excl, Busy: d.Settings.Busy, MinAge: d.Settings.MinAge, + MaxSize: d.Settings.MaxSize, Now: now, } } @@ -328,6 +362,9 @@ func explainSkip(d *Dir, sf scan.File, excl []string, now time.Time) string { if now.Sub(sf.ModTime) < d.Settings.MinAge { return "too new" } + if d.Settings.MaxSize > 0 && sf.Size > d.Settings.MaxSize { + return "too big" + } return "" } diff --git a/internal/scan/scan.go b/internal/scan/scan.go index e87b741..7fe9e92 100644 --- a/internal/scan/scan.go +++ b/internal/scan/scan.go @@ -36,6 +36,7 @@ const ( Symlink // a symbolic link (never followed) NotRegular // fifo, socket, device Unreadable // a directory that could not be read + TooBig // larger than MaxSize ) // String names a Reason the way it should read in a report. @@ -53,6 +54,8 @@ func (r Reason) String() string { return "not a regular file" case Unreadable: return "unreadable" + case TooBig: + return "too big" default: return fmt.Sprintf("Reason(%d)", int(r)) } @@ -72,6 +75,7 @@ type Options struct { Exclude []string // absolute directories never entered Busy []string // suffixes, e.g. ".part" MinAge time.Duration + MaxSize int64 // bytes; 0: no limit Now time.Time } @@ -209,6 +213,10 @@ func (w *walker) walk(dir, relDir string, depth int, entries []os.DirEntry) erro w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: TooNew}) continue } + if w.opt.MaxSize > 0 && info.Size() > w.opt.MaxSize { + w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: TooBig}) + continue + } w.result.Files = append(w.result.Files, File{ Path: path, Rel: rel, diff --git a/internal/scan/scan_test.go b/internal/scan/scan_test.go index 94f922b..a245893 100644 --- a/internal/scan/scan_test.go +++ b/internal/scan/scan_test.go @@ -261,3 +261,35 @@ func TestIgnoredDirectoryReportedOnce(t *testing.T) { t.Fatalf("skipped = %v, want %v (the directory once, not its three files)", skipped(r), want) } } + +// TestTooBig: with MaxSize set, a file larger than it is skipped as too +// big; a file of exactly MaxSize is kept; MaxSize 0 means no limit. +func TestTooBig(t *testing.T) { + root := tree(t) + for name, size := range map[string]int{"small.bin": 10, "exact.bin": 100, "large.bin": 101} { + p := filepath.Join(root, name) + if err := os.WriteFile(p, make([]byte, size), 0o644); err != nil { + t.Fatal(err) + } + old := now.Add(-time.Hour) + if err := os.Chtimes(p, old, old); err != nil { + t.Fatal(err) + } + } + r, err := Walk(root, Options{MaxSize: 100, Now: now}) + if err != nil { + t.Fatal(err) + } + if want := []string{"exact.bin", "small.bin"}; !reflect.DeepEqual(rels(r), want) { + t.Fatalf("files = %v, want %v", rels(r), want) + } + if got := skipped(r); !reflect.DeepEqual(got, map[string]Reason{"large.bin": TooBig}) { + t.Fatalf("skipped = %v", got) + } + if TooBig.String() != "too big" { + t.Errorf("TooBig reads %q", TooBig.String()) + } + if r, _ := Walk(root, Options{Now: now}); len(rels(r)) != 3 { + t.Errorf("MaxSize 0 skipped files: %v", skipped(r)) + } +} |
