diff options
| author | Lukasz Kasprzak <lukas@labunix.xyz> | 2026-09-14 15:16:55 +0200 |
|---|---|---|
| committer | Lukasz Kasprzak <lukas@labunix.xyz> | 2026-09-14 15:16:55 +0200 |
| commit | 0b5d0eb92c5be2f0ddb2fa73990f31e5654e57fe (patch) | |
| tree | 358e331945b8206ed4a72703e3aedfe8ea7cdcdb /internal/engine | |
| parent | 1d3f2d1e4c59867024470d3444e12698b7ebb22e (diff) | |
| download | krino-0.0.5.tar.gz krino-0.0.5.zip | |
krino: 0.0.5 — keyword cache, t and d in reviewv0.0.5
Diffstat (limited to 'internal/engine')
| -rw-r--r-- | internal/engine/cache_test.go | 224 | ||||
| -rw-r--r-- | internal/engine/engine.go | 58 | ||||
| -rw-r--r-- | internal/engine/engine_test.go | 33 | ||||
| -rw-r--r-- | internal/engine/exclude_test.go | 8 | ||||
| -rw-r--r-- | internal/engine/facts.go | 111 | ||||
| -rw-r--r-- | internal/engine/facts_test.go | 69 | ||||
| -rw-r--r-- | internal/engine/match.go | 45 |
7 files changed, 391 insertions, 157 deletions
diff --git a/internal/engine/cache_test.go b/internal/engine/cache_test.go new file mode 100644 index 0000000..eee2860 --- /dev/null +++ b/internal/engine/cache_test.go @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package engine + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// countingPDFTree makes ~/dl holding the given .pdf files, and a fake +// pdftotext first on PATH that prints the file's own bytes as its text and +// counts its runs in a file; a file whose name contains "fail" makes it +// exit 1. +func countingPDFTree(t *testing.T, files map[string]string) (home, dl string, runs func() int) { + t.Helper() + home, dl = excludeTree(t, files) + bin := t.TempDir() + count := filepath.Join(t.TempDir(), "count") + script := "#!/bin/sh\necho x >> '" + count + "'\ncase \"$4\" in *fail*) exit 1;; esac\nexec /bin/cat \"$4\"\n" + if err := os.WriteFile(filepath.Join(bin, "pdftotext"), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", bin) + runs = func() int { + b, _ := os.ReadFile(count) + return strings.Count(string(b), "x") + } + return home, dl, runs +} + +// cachedMatch loads the configuration afresh, as a new krino process +// would, and matches dl with the cache under ~/.cache/krino. +func cachedMatch(t *testing.T, home, main string) *Result { + t.Helper() + e, errs := Load(main) + if len(errs) > 0 { + t.Fatal(errs) + } + e.CacheDir = filepath.Join(home, ".cache", "krino") + r, err := e.Match(context.Background(), e.Dirs[0]) + if err != nil { + t.Fatal(err) + } + return r +} + +func matchedNames(r *Result) string { + var names []string + for _, fm := range r.Matched { + names = append(names, fm.File.Rel) + } + return strings.Join(names, " ") +} + +const acmeRules = ` +(path "~/dl") +(rule "acme" (when (content "acme")) (move "Acme")) +` + +// TestCacheSkipsExtractionOnRerun: the second run over unchanged files +// extracts nothing and matches the same files. +func TestCacheSkipsExtractionOnRerun(t *testing.T) { + home, _, runs := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME", "b.pdf": "nothing here"}) + main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules}) + + first := cachedMatch(t, home, main) + if runs() != 2 || matchedNames(first) != "a.pdf" { + t.Fatalf("first run: %d extractions, matched %q", runs(), matchedNames(first)) + } + second := cachedMatch(t, home, main) + if runs() != 2 { + t.Errorf("second run extracted again: %d runs in all", runs()) + } + if matchedNames(second) != "a.pdf" { + t.Errorf("second run matched %q, want a.pdf", matchedNames(second)) + } +} + +// TestCacheRereadsChangedFile: a file whose modification time changed is +// extracted again; the other is not. +func TestCacheRereadsChangedFile(t *testing.T) { + home, dl, runs := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME", "b.pdf": "nothing here"}) + main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules}) + cachedMatch(t, home, main) + + later := time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC) + if err := os.WriteFile(filepath.Join(dl, "b.pdf"), []byte("now ACME too"), 0o644); err != nil { + t.Fatal(err) + } + os.Chtimes(filepath.Join(dl, "b.pdf"), later, later) + r := cachedMatch(t, home, main) + if runs() != 3 { + t.Errorf("%d extractions in all, want 3: only b.pdf read again", runs()) + } + if matchedNames(r) != "a.pdf b.pdf" { + t.Errorf("matched %q, want both", matchedNames(r)) + } +} + +// TestCacheRereadsForNewKeyword: a keyword no entry was checked against +// makes the files that reach it be read again, once. +func TestCacheRereadsForNewKeyword(t *testing.T) { + home, _, runs := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME", "b.pdf": "nothing here"}) + main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules}) + cachedMatch(t, home, main) + + writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules + `(rule "nothing" (when (content "nothing")) (move "Nothing"))` + "\n"}) + r := cachedMatch(t, home, main) + if runs() != 4 { + t.Errorf("%d extractions in all, want 4: both read again for the new keyword", runs()) + } + if matchedNames(r) != "a.pdf b.pdf" { + t.Errorf("matched %q, want both", matchedNames(r)) + } + cachedMatch(t, home, main) + if runs() != 4 { + t.Errorf("third run extracted again: %d runs in all", runs()) + } +} + +// TestCacheOffWithoutCacheDir: an engine with no CacheDir reads every time +// and writes no cache. +func TestCacheOffWithoutCacheDir(t *testing.T) { + home, _, runs := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME"}) + main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules}) + for i := 0; i < 2; i++ { + e, errs := Load(main) + if len(errs) > 0 { + t.Fatal(errs) + } + if _, err := e.Match(context.Background(), e.Dirs[0]); err != nil { + t.Fatal(err) + } + } + if runs() != 2 { + t.Errorf("%d extractions, want one per run", runs()) + } + if _, err := os.Stat(filepath.Join(home, ".cache")); !os.IsNotExist(err) { + t.Errorf("a cache was written with no CacheDir: %v", err) + } +} + +// TestCacheDoesNotStoreFailures: a file the tool fails on is tried again +// on every run, and warned about every time. +func TestCacheDoesNotStoreFailures(t *testing.T) { + home, _, runs := countingPDFTree(t, map[string]string{"fail.pdf": "Invoice ACME"}) + main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules}) + for i := 1; i <= 2; i++ { + r := cachedMatch(t, home, main) + if runs() != i { + t.Errorf("run %d: %d extractions in all, want %d", i, runs(), i) + } + if len(r.Unmatched) != 1 || len(r.Unmatched[0].Warnings) == 0 { + t.Errorf("run %d: want fail.pdf unmatched with a warning: %+v", i, r.Unmatched) + } + } +} + +// TestCacheRespectsMaxRead: a file over max-read is not read, even when an +// earlier run cached its answers. +func TestCacheRespectsMaxRead(t *testing.T) { + home, _, _ := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME"}) + main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules}) + cachedMatch(t, home, main) + + writeConfig(t, home, `(include "dl")`, map[string]string{"dl": "(max-read 1)\n" + acmeRules}) + r := cachedMatch(t, home, main) + if len(r.Matched) != 0 || len(r.Unmatched) != 1 || !strings.Contains(strings.Join(r.Unmatched[0].Warnings, " "), "larger than max-read") { + t.Errorf("want a.pdf unmatched as larger than max-read: matched %+v unmatched %+v", r.Matched, r.Unmatched) + } +} + +// TestCacheHoldsNoText: the cache file holds the keywords and answers, not +// the extracted text or the file's name. +func TestCacheHoldsNoText(t *testing.T) { + home, _, _ := countingPDFTree(t, map[string]string{"secret-name.pdf": "Invoice ACME confidential"}) + main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules}) + cachedMatch(t, home, main) + b, err := os.ReadFile(filepath.Join(home, ".cache", "krino", "dl.cache")) + if err != nil { + t.Fatal(err) + } + for _, leak := range []string{"Invoice", "invoice", "confidential", "secret-name"} { + if strings.Contains(string(b), leak) { + t.Errorf("cache holds %q:\n%s", leak, b) + } + } +} + +// TestExplainUsesCacheWithoutWriting: explain answers from the cache a run +// wrote, and never writes one itself. +func TestExplainUsesCacheWithoutWriting(t *testing.T) { + home, dl, runs := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME"}) + main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules}) + explain := func() { + t.Helper() + e, errs := Load(main) + if len(errs) > 0 { + t.Fatal(errs) + } + e.CacheDir = filepath.Join(home, ".cache", "krino") + x, err := e.Explain(context.Background(), filepath.Join(dl, "a.pdf")) + if err != nil { + t.Fatal(err) + } + if len(x.Rules) != 1 || !x.Rules[0].Match { + t.Errorf("explain: %+v", x.Rules) + } + } + explain() + if _, err := os.Stat(filepath.Join(home, ".cache")); !os.IsNotExist(err) { + t.Errorf("explain wrote a cache: %v", err) + } + cachedMatch(t, home, main) + before := runs() + explain() + if runs() != before { + t.Errorf("explain extracted although the run cached a.pdf") + } +} diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 8cffbd1..b50728e 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -8,6 +8,7 @@ package engine import ( "fmt" "os" + "sort" "strings" "time" @@ -26,6 +27,11 @@ type Engine struct { Extract *extract.Extractor Now func() time.Time // time.Now; tests replace it MainFile string + + // CacheDir holds each directory's keyword cache (spec §6.1), as + // NAME.cache; "" means no cache is read or written. Load leaves it + // empty: the command line sets it. + CacheDir string } // Dir is one configured directory, with its ignore matcher and rules @@ -38,13 +44,11 @@ type Dir struct { Ignore *ignore.Matcher Rules []*Rule - // ContentVariants is the distinct (IgnoreCase, Fold) pairs any of - // Rules' content tests evaluate under, in first-seen order. B2: when - // this holds exactly one variant, facts.Content releases a file's raw - // extracted text once that variant's normalised copy exists, since no - // other variant will ever be asked for; with more than one, both must - // stay memoised, as before. - ContentVariants []cond.Options + // ContentKeywords is every content keyword the directory's excludes + // and rules test, once each, sorted by Key. When a file's text is + // extracted, every one of them is answered at once, so one extraction + // serves every content test and fills the keyword cache. + ContentKeywords []cond.Keyword // Excludes are the (exclude ...) forms that apply here, compiled with the // directory's settings: krino.conf's first, then the directory's own. A @@ -135,7 +139,7 @@ func Load(mainFile string, names ...string) (*Engine, []*config.Diag) { } } } - dir.ContentVariants = contentVariants(dir.Rules, dir.Excludes, dirOpt) + dir.ContentKeywords = contentKeywords(dir.Rules, dir.Excludes) dir.DupScopes = dupScopes(dir.Rules) dirs = append(dirs, dir) } @@ -216,32 +220,26 @@ func dedupeNames(names []string) []string { return out } -// contentVariants returns the distinct (IgnoreCase, Fold) pairs any content -// test evaluates under - each exclude's (under the directory's settings, -// dirOpt) and each rule's - in first-seen order — B2's per-Dir -// ContentVariants. A rule whose condition has no content test at all -// (Cond.UsesContent false) never calls facts.Content, so its resolved -// case/fold settings contribute no variant here. -func contentVariants(rules []*Rule, excludes []*Exclude, dirOpt cond.Options) []cond.Options { - var out []cond.Options - seen := map[cond.Options]bool{} - for _, x := range excludes { - if x.Cond.UsesContent && !seen[dirOpt] { - seen[dirOpt] = true - out = append(out, dirOpt) +// contentKeywords returns every content keyword excludes and rules test, +// each once, sorted by Key: Dir.ContentKeywords. +func contentKeywords(rules []*Rule, excludes []*Exclude) []cond.Keyword { + seen := map[string]bool{} + var out []cond.Keyword + add := func(c *cond.Cond) { + for _, k := range c.Keywords { + if !seen[k.Key()] { + seen[k.Key()] = true + out = append(out, k) + } } } + for _, x := range excludes { + add(x.Cond) + } for _, r := range rules { - if !r.Cond.UsesContent { - continue - } - opt := cond.Options{IgnoreCase: r.Settings.Case == config.CaseIgnore, Fold: r.Settings.Fold} - if seen[opt] { - continue - } - seen[opt] = true - out = append(out, opt) + add(r.Cond) } + sort.Slice(out, func(i, j int) bool { return out[i].Key() < out[j].Key() }) return out } diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go index 18b35e7..c38c1fc 100644 --- a/internal/engine/engine_test.go +++ b/internal/engine/engine_test.go @@ -46,14 +46,14 @@ func writeConfig(t *testing.T, home, main string, dirs map[string]string) string // fakeFacts is a minimal cond.Facts for checking compiled rules. type fakeFacts struct{ name string } -func (f fakeFacts) Name() string { return f.name } -func (f fakeFacts) Rel() string { return f.name } -func (f fakeFacts) Size() int64 { return 1 } -func (f fakeFacts) ModTime() time.Time { return time.Time{} } -func (f fakeFacts) Now() time.Time { return time.Time{} } -func (f fakeFacts) Content(bool, bool) (string, error) { return "", nil } -func (f fakeFacts) Duplicate([]string) (string, bool, error) { return "", false, nil } -func (f fakeFacts) Matched() bool { return false } +func (f fakeFacts) Name() string { return f.name } +func (f fakeFacts) Rel() string { return f.name } +func (f fakeFacts) Size() int64 { return 1 } +func (f fakeFacts) ModTime() time.Time { return time.Time{} } +func (f fakeFacts) Now() time.Time { return time.Time{} } +func (f fakeFacts) ContentContains(cond.Options, []string) (int, error) { return -1, nil } +func (f fakeFacts) Duplicate([]string) (string, bool, error) { return "", false, nil } +func (f fakeFacts) Matched() bool { return false } var _ cond.Facts = fakeFacts{} @@ -187,12 +187,10 @@ func TestLoadAcceptsSuppliedCaptures(t *testing.T) { } } -// TestContentVariantsComputedAtLoad: B2 plumbing. Load computes each -// directory's distinct (ignoreCase, fold) content-test variants from its -// rules' resolved settings: a rule with no content test contributes -// nothing; two rules sharing a variant fold into one; a rule-level (case -// ignore) override adds a second. -func TestContentVariantsComputedAtLoad(t *testing.T) { +// TestContentKeywordsComputedAtLoad: Load collects every content keyword +// of a directory's rules, once per (options, normalised keyword), sorted by +// key: a rule-level (case ignore) makes the same word a second keyword. +func TestContentKeywordsComputedAtLoad(t *testing.T) { h := sandbox(t) os.Mkdir(filepath.Join(h, "dl"), 0o755) main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": ` @@ -207,10 +205,11 @@ func TestContentVariantsComputedAtLoad(t *testing.T) { if len(errs) > 0 { t.Fatal(errs) } - got := e.Dirs[0].ContentVariants - want := []cond.Options{{IgnoreCase: false, Fold: true}, {IgnoreCase: true, Fold: true}} + got := e.Dirs[0].ContentKeywords + strict, loose := cond.Options{IgnoreCase: false, Fold: true}, cond.Options{IgnoreCase: true, Fold: true} + want := []cond.Keyword{{Opt: loose, Norm: "acme"}, {Opt: strict, Norm: "acme"}, {Opt: strict, Norm: "other"}} if !reflect.DeepEqual(got, want) { - t.Fatalf("ContentVariants = %+v, want %+v", got, want) + t.Fatalf("ContentKeywords = %+v, want %+v", got, want) } } diff --git a/internal/engine/exclude_test.go b/internal/engine/exclude_test.go index 40b3548..8b26450 100644 --- a/internal/engine/exclude_test.go +++ b/internal/engine/exclude_test.go @@ -115,10 +115,10 @@ func TestExcludeNeedsEveryCondition(t *testing.T) { } } -// TestExcludeContentKeepsRuleContentVariants: an exclude reading content -// under the directory's settings must not release the raw text a rule -// with different case/fold settings still needs. -func TestExcludeContentKeepsRuleContentVariants(t *testing.T) { +// TestExcludeContentAndRuleContentKeepTheirOwnSettings: an exclude reading +// content under the directory's settings first must not change how a rule +// with different case/fold settings sees the same text. +func TestExcludeContentAndRuleContentKeepTheirOwnSettings(t *testing.T) { h, _ := excludeTree(t, map[string]string{"a.txt": "Invoice ACME"}) main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": ` (path "~/dl") diff --git a/internal/engine/facts.go b/internal/engine/facts.go index 5790e95..9035245 100644 --- a/internal/engine/facts.go +++ b/internal/engine/facts.go @@ -12,6 +12,8 @@ import ( "krino/internal/cond" "krino/internal/dup" + "krino/internal/extract" + "krino/internal/kwcache" "krino/internal/norm" "krino/internal/plan" "krino/internal/scan" @@ -30,6 +32,7 @@ type matchRun struct { ctx context.Context now time.Time files []scan.File + cache *kwcache.Cache // nil: no keyword cache mu sync.Mutex dupOnce map[string]*sync.Once @@ -103,26 +106,25 @@ func (run *matchRun) dupIndex(key string, dirs []string) *dup.Index { } // facts is one file's cond.Facts. It is used by exactly one goroutine, so -// its own memoised state (content, its normalised variants, and whether an -// earlier rule matched) needs no locking of its own; only the matchRun it -// points at is shared. +// its own memoised state (the keyword answers, and whether an earlier rule +// matched) needs no locking of its own; only the matchRun it points at is +// shared. type facts struct { run *matchRun file scan.File matched bool - contentDone bool - content string - contentErr error - normCache map[[2]bool]string + contentDone bool // extraction was attempted + contentErr error // why it failed + answers map[string]bool // by cond.KeywordKey, once extracted } var _ cond.Facts = (*facts)(nil) // newFacts builds the Facts for one scanned file. func newFacts(run *matchRun, file scan.File) *facts { - return &facts{run: run, file: file, normCache: make(map[[2]bool]string)} + return &facts{run: run, file: file} } func (f *facts) Name() string { return f.file.Name } @@ -132,32 +134,85 @@ func (f *facts) ModTime() time.Time { return f.file.ModTime } func (f *facts) Now() time.Time { return f.run.now } func (f *facts) Matched() bool { return f.matched } -// Content extracts the file's text once, then normalises it per -// (ignoreCase, fold) variant, memoising each. B2: when the directory's -// rules use exactly one variant (Dir.ContentVariants), the raw text is -// released as soon as that variant's normalised copy exists — no other -// variant will ever be asked for, so there is no reason to keep both the -// raw text and its normalised copy in memory at once. A directory using -// more than one variant keeps the raw text for as long as f lives, exactly -// as before. -func (f *facts) Content(ignoreCase, fold bool) (string, error) { +// ContentContains answers a content test (spec §6.1). A file above +// max-read is never read, cached or not. Before the file has been +// extracted this run, the keyword cache answers when it knows every one of +// keywords for this file as it is now; otherwise the text is extracted +// once, every keyword of the directory (and of this test) is answered from +// it and stored in the cache, and the text itself is dropped. A failed +// extraction is not cached: the next run tries again. +func (f *facts) ContentContains(opt cond.Options, keywords []string) (int, error) { + if max := f.run.d.Settings.MaxRead; max > 0 && f.file.Size > max { + return -1, extract.ErrTooLarge + } + keys := make([]string, len(keywords)) + for i, kw := range keywords { + keys[i] = cond.KeywordKey(opt, kw) + } if !f.contentDone { - f.content, f.contentErr = f.run.e.Extract.Text(f.run.ctx, f.file.Path, f.file.Size, f.run.d.Settings.MaxRead) - f.contentDone = true + if id, ok := f.cacheID(); ok { + if hits, ok := f.run.cache.Lookup(id, keys); ok { + for i, hit := range hits { + if hit { + return i, nil + } + } + return -1, nil + } + } + f.extract(opt, keywords) } if f.contentErr != nil { - return "", f.contentErr + return -1, f.contentErr + } + for i, k := range keys { + if f.answers[k] { + return i, nil + } + } + return -1, nil +} + +// extract reads the file's text and answers every keyword of the directory, +// plus the asking test's (opt, keywords), from it. +func (f *facts) extract(opt cond.Options, keywords []string) { + f.contentDone = true + text, err := f.run.e.Extract.Text(f.run.ctx, f.file.Path, f.file.Size, f.run.d.Settings.MaxRead) + if err != nil { + f.contentErr = err + return + } + all := append([]cond.Keyword(nil), f.run.d.ContentKeywords...) + for _, kw := range keywords { + all = append(all, cond.Keyword{Opt: opt, Norm: kw}) + } + normed := map[cond.Options]string{} + f.answers = make(map[string]bool, len(all)) + for _, k := range all { + t, ok := normed[k.Opt] + if !ok { + t = norm.Text(text, k.Opt.IgnoreCase, k.Opt.Fold) + normed[k.Opt] = t + } + f.answers[k.Key()] = strings.Contains(t, k.Norm) } - key := [2]bool{ignoreCase, fold} - if v, ok := f.normCache[key]; ok { - return v, nil + if id, ok := f.cacheID(); ok { + f.run.cache.Store(id, f.answers) } - v := norm.Text(f.content, ignoreCase, fold) - f.normCache[key] = v - if len(f.run.d.ContentVariants) == 1 { - f.content = "" +} + +// cacheID is the file's keyword cache identity; ok is false when there is +// no cache, or the platform gave the file no inode. +func (f *facts) cacheID() (kwcache.ID, bool) { + if f.run.cache == nil || f.file.Ino == 0 { + return kwcache.ID{}, false } - return v, nil + return fileCacheID(f.file), true +} + +// fileCacheID is file's kwcache.ID. +func fileCacheID(file scan.File) kwcache.ID { + return kwcache.ID{Dev: file.Dev, Ino: file.Ino, Size: file.Size, MTime: file.ModTime.UnixNano()} } // Duplicate resolves dirs against the directory's root, builds (or reuses) diff --git a/internal/engine/facts_test.go b/internal/engine/facts_test.go index 7d844a0..9739fa6 100644 --- a/internal/engine/facts_test.go +++ b/internal/engine/facts_test.go @@ -3,79 +3,10 @@ package engine import ( - "context" - "os" "path/filepath" "testing" - "time" - - "krino/internal/cond" - "krino/internal/extract" - "krino/internal/scan" ) -// contentFacts builds a *facts for a real text file, under a Dir whose -// ContentVariants is variants, for exercising B2's raw-release directly. -func contentFacts(t *testing.T, body string, variants []cond.Options) *facts { - t.Helper() - p := filepath.Join(t.TempDir(), "a.txt") - if err := os.WriteFile(p, []byte(body), 0o644); err != nil { - t.Fatal(err) - } - fi, err := os.Stat(p) - if err != nil { - t.Fatal(err) - } - e := &Engine{Extract: extract.New(), Now: time.Now} - d := &Dir{Name: "d", ContentVariants: variants} - file := scan.File{Path: p, Rel: "a.txt", Name: "a.txt", Size: fi.Size(), ModTime: fi.ModTime()} - run := newMatchRun(e, d, context.Background(), time.Now(), []scan.File{file}) - return newFacts(run, file) -} - -// TestContentReleasesRawWithOneVariant: B2. A directory whose rules use -// exactly one (ignoreCase, fold) variant releases the raw extracted text -// once that variant's normalised copy exists. -func TestContentReleasesRawWithOneVariant(t *testing.T) { - f := contentFacts(t, "Hello World", []cond.Options{{IgnoreCase: true, Fold: false}}) - const want = "hello world" - got, err := f.Content(true, false) - if err != nil { - t.Fatal(err) - } - if got != want { - t.Fatalf("got %q, want %q", got, want) - } - if f.content != "" { - t.Errorf("raw text not released with a single content variant: %q", f.content) - } - // A second call for the same (already cached) variant must still work - // from normCache, without needing the released raw text. - if got, err := f.Content(true, false); err != nil || got != want { - t.Errorf("second call for the cached variant: got %q, %v, want %q", got, err, want) - } -} - -// TestContentKeepsRawWithTwoVariants: B2. A directory whose rules use two -// distinct variants must not release the raw text after the first: the -// second variant still needs it, and normalising it correctly (not from an -// emptied string) is the proof the raw text was kept. -func TestContentKeepsRawWithTwoVariants(t *testing.T) { - f := contentFacts(t, "Hello World", []cond.Options{ - {IgnoreCase: true, Fold: false}, - {IgnoreCase: false, Fold: false}, - }) - if got, err := f.Content(true, false); err != nil || got != "hello world" { - t.Fatalf("first variant: got %q, %v", got, err) - } - if f.content == "" { - t.Fatal("raw text released after only the first of two variants") - } - if got, err := f.Content(false, false); err != nil || got != "Hello World" { - t.Fatalf("second variant: got %q, %v, want the unfolded original (raw text must still be available)", got, err) - } -} - // TestDisplayOriginalAbbreviatesHome: a duplicate's original is shown // root-relative inside the root, and home-abbreviated outside it, the way // every other user-visible path is; a path outside $HOME stays absolute. diff --git a/internal/engine/match.go b/internal/engine/match.go index 8daea9d..1a8077a 100644 --- a/internal/engine/match.go +++ b/internal/engine/match.go @@ -16,6 +16,7 @@ import ( "krino/internal/cond" "krino/internal/config" + "krino/internal/kwcache" "krino/internal/plan" "krino/internal/scan" "krino/internal/xdg" @@ -70,6 +71,7 @@ func (e *Engine) Match(ctx context.Context, d *Dir) (*Result, error) { } run := newMatchRun(e, d, ctx, now, wres.Files) + cacheWarn := e.openCache(run) fileMatches := make([]FileMatch, len(wres.Files)) workers := runtime.GOMAXPROCS(0) @@ -103,7 +105,18 @@ func (e *Engine) Match(ctx context.Context, d *Dir) (*Result, error) { } } - warnings := run.warnings() + warnings := append(run.warnings(), cacheWarn...) + if run.cache != nil { + ids := make([]kwcache.ID, 0, len(wres.Files)) + for _, f := range wres.Files { + if f.Ino != 0 { + ids = append(ids, fileCacheID(f)) + } + } + if err := run.cache.Save(e.cacheFile(d), ids); err != nil { + warnings = append(warnings, "cache: "+err.Error()) + } + } sort.Strings(warnings) return &Result{ @@ -242,20 +255,14 @@ func (e *Engine) Explain(ctx context.Context, path string) (*Explanation, error) } rel = filepath.ToSlash(rel) - sf := scan.File{ - Path: abs, - Rel: rel, - Name: filepath.Base(abs), - Size: info.Size(), - ModTime: info.ModTime(), - Mode: info.Mode(), - } + sf := scan.NewFile(abs, rel, info) now := e.Now() excl := e.excludeDirs(d) skip := explainSkip(d, sf, excl, now) run := newMatchRun(e, d, ctx, now, e.filesForExplain(d, sf, excl, now)) + e.openCache(run) // read only: Explain never writes the cache f := newFacts(run, sf) var excludes []ExcludeTrace @@ -289,6 +296,26 @@ func (e *Engine) Explain(ctx context.Context, path string) (*Explanation, error) return &Explanation{Dir: d, File: sf, Skip: skip, Excludes: excludes, Excluded: excluded, Rules: rules}, nil } +// cacheFile is d's keyword cache file. +func (e *Engine) cacheFile(d *Dir) string { + return filepath.Join(e.CacheDir, d.Name+".cache") +} + +// openCache loads run's directory keyword cache into run.cache, when the +// engine has a CacheDir and the directory has content tests at all. A cache +// that cannot be read is replaced by an empty one, reported as a warning. +func (e *Engine) openCache(run *matchRun) []string { + if e.CacheDir == "" || len(run.d.ContentKeywords) == 0 { + return nil + } + c, err := kwcache.Load(e.cacheFile(run.d), e.Extract.Fingerprint()) + run.cache = c + if err != nil { + return []string{"cache: " + err.Error() + " (starting a new one)"} + } + return nil +} + // filesForExplain returns the file set Explain's duplicate checks run // against: the directory's ordinary scan, plus the explained file itself // when that scan would not have reached it (it is busy, ignored, too new, |
