summaryrefslogtreecommitdiff
path: root/internal
diff options
context:
space:
mode:
authorLukasz Kasprzak <lukas@labunix.xyz>2026-09-14 15:16:55 +0200
committerLukasz Kasprzak <lukas@labunix.xyz>2026-09-14 15:16:55 +0200
commit0b5d0eb92c5be2f0ddb2fa73990f31e5654e57fe (patch)
tree358e331945b8206ed4a72703e3aedfe8ea7cdcdb /internal
parent1d3f2d1e4c59867024470d3444e12698b7ebb22e (diff)
downloadkrino-0b5d0eb92c5be2f0ddb2fa73990f31e5654e57fe.tar.gz
krino-0b5d0eb92c5be2f0ddb2fa73990f31e5654e57fe.zip
krino: 0.0.5 — keyword cache, t and d in reviewv0.0.5
Diffstat (limited to 'internal')
-rw-r--r--internal/cond/compile.go1
-rw-r--r--internal/cond/compile_test.go21
-rw-r--r--internal/cond/eval.go17
-rw-r--r--internal/cond/eval_test.go12
-rw-r--r--internal/cond/types.go24
-rw-r--r--internal/engine/cache_test.go224
-rw-r--r--internal/engine/engine.go58
-rw-r--r--internal/engine/engine_test.go33
-rw-r--r--internal/engine/exclude_test.go8
-rw-r--r--internal/engine/facts.go111
-rw-r--r--internal/engine/facts_test.go69
-rw-r--r--internal/engine/match.go45
-rw-r--r--internal/extract/extract.go26
-rw-r--r--internal/extract/tools_test.go24
-rw-r--r--internal/kwcache/kwcache.go231
-rw-r--r--internal/kwcache/kwcache_test.go180
-rw-r--r--internal/scan/fileid_other.go10
-rw-r--r--internal/scan/fileid_unix.go19
-rw-r--r--internal/scan/scan.go29
-rw-r--r--internal/scan/scan_test.go26
-rw-r--r--internal/xdg/xdg.go3
21 files changed, 997 insertions, 174 deletions
diff --git a/internal/cond/compile.go b/internal/cond/compile.go
index 590d61a..94118ac 100644
--- a/internal/cond/compile.go
+++ b/internal/cond/compile.go
@@ -266,6 +266,7 @@ func (c *compiler) compileContent(n *sexp.Node) *node {
continue
}
keywords = append(keywords, keyword{norm: normed, src: a.Text})
+ c.cond.Keywords = append(c.cond.Keywords, Keyword{Opt: c.opt, Norm: normed})
}
if bad {
return nil
diff --git a/internal/cond/compile_test.go b/internal/cond/compile_test.go
index 2a07ad6..8432365 100644
--- a/internal/cond/compile_test.go
+++ b/internal/cond/compile_test.go
@@ -165,3 +165,24 @@ func TestGroupsMatchSpec(t *testing.T) {
}
}
}
+
+// TestCompileCollectsKeywords: every content keyword is recorded as
+// compared, normalised under the compile options, and its key names those
+// options.
+func TestCompileCollectsKeywords(t *testing.T) {
+ opt := Options{IgnoreCase: true, Fold: true}
+ c, errs := Compile("d.conf", nodes(t, `(or (content "Spółka" "x") (not (content "NIP")))`), opt)
+ if len(errs) > 0 {
+ t.Fatal(errs)
+ }
+ want := []Keyword{{Opt: opt, Norm: "spolka"}, {Opt: opt, Norm: "x"}, {Opt: opt, Norm: "nip"}}
+ if !reflect.DeepEqual(c.Keywords, want) {
+ t.Errorf("Keywords = %+v, want %+v", c.Keywords, want)
+ }
+ if got := want[0].Key(); got != "if:spolka" {
+ t.Errorf("Key = %q", got)
+ }
+ if got := KeywordKey(Options{}, "NIP"); got != "sn:NIP" {
+ t.Errorf("strict, unfolded key = %q", got)
+ }
+}
diff --git a/internal/cond/eval.go b/internal/cond/eval.go
index e3093af..3cec110 100644
--- a/internal/cond/eval.go
+++ b/internal/cond/eval.go
@@ -19,7 +19,10 @@ type Facts interface {
Size() int64
ModTime() time.Time
Now() time.Time
- Content(ignoreCase, fold bool) (string, error) // normalised with norm.Text
+ // ContentContains reports the index of the first of keywords, each
+ // normalised under opt with norm.Text, that the file's text contains, or
+ // -1 when it contains none.
+ ContentContains(opt Options, keywords []string) (int, error)
Duplicate(dirs []string) (original string, ok bool, err error)
Matched() bool // an earlier rule matched this file
}
@@ -167,14 +170,16 @@ func (c *Cond) evalLeaf(n *node, f Facts) (ok bool, reason, warn string, caps []
return false, "", "", nil
case kContent:
- text, err := f.Content(c.opt.IgnoreCase, c.opt.Fold)
+ norms := make([]string, len(n.keywords))
+ for i, kw := range n.keywords {
+ norms[i] = kw.norm
+ }
+ i, err := f.ContentContains(c.opt, norms)
if err != nil {
return false, "", "content unreadable: " + err.Error(), nil
}
- for _, kw := range n.keywords {
- if strings.Contains(text, kw.norm) {
- return true, `content "` + kw.src + `"`, "", nil
- }
+ if i >= 0 {
+ return true, `content "` + n.keywords[i].src + `"`, "", nil
}
return false, "", "", nil
diff --git a/internal/cond/eval_test.go b/internal/cond/eval_test.go
index 3607978..8891de7 100644
--- a/internal/cond/eval_test.go
+++ b/internal/cond/eval_test.go
@@ -37,12 +37,18 @@ func (f *fake) Size() int64 { return f.size }
func (f *fake) ModTime() time.Time { return now.Add(-f.age) }
func (f *fake) Now() time.Time { return now }
func (f *fake) Matched() bool { return f.matched }
-func (f *fake) Content(ic, fold bool) (string, error) {
+func (f *fake) ContentContains(opt Options, keywords []string) (int, error) {
f.contentCalls++
if f.rawErr != nil {
- return "", f.rawErr
+ return -1, f.rawErr
}
- return norm.Text(f.raw, ic, fold), nil
+ text := norm.Text(f.raw, opt.IgnoreCase, opt.Fold)
+ for i, kw := range keywords {
+ if strings.Contains(text, kw) {
+ return i, nil
+ }
+ }
+ return -1, nil
}
func (f *fake) Duplicate(dirs []string) (string, bool, error) { return f.dupOrig, f.dupOK, nil }
diff --git a/internal/cond/types.go b/internal/cond/types.go
index 9455f63..28a4fc9 100644
--- a/internal/cond/types.go
+++ b/internal/cond/types.go
@@ -24,6 +24,30 @@ type Cond struct {
opt Options // the case/fold settings conditions were compiled with; Task 8 needs them again at eval time
UsesContent bool // some content test exists
DupDirs [][]string // the raw directory arguments of each duplicate test, in order
+ Keywords []Keyword // every content keyword, as compiled, in order
+}
+
+// Keyword is one content keyword as a content test compares it: normalised
+// under the options it was compiled with.
+type Keyword struct {
+ Opt Options
+ Norm string
+}
+
+// Key is the keyword's identity across runs, for the keyword cache: its
+// options and its normalised text.
+func (k Keyword) Key() string { return KeywordKey(k.Opt, k.Norm) }
+
+// KeywordKey is Keyword.Key for opt and an already normalised keyword.
+func KeywordKey(opt Options, norm string) string {
+ b := []byte("sn:")
+ if opt.IgnoreCase {
+ b[0] = 'i'
+ }
+ if opt.Fold {
+ b[1] = 'f'
+ }
+ return string(b) + norm
}
// kind is what a compiled node tests, or how it combines its children.
diff --git a/internal/engine/cache_test.go b/internal/engine/cache_test.go
new file mode 100644
index 0000000..eee2860
--- /dev/null
+++ b/internal/engine/cache_test.go
@@ -0,0 +1,224 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package engine
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+)
+
+// countingPDFTree makes ~/dl holding the given .pdf files, and a fake
+// pdftotext first on PATH that prints the file's own bytes as its text and
+// counts its runs in a file; a file whose name contains "fail" makes it
+// exit 1.
+func countingPDFTree(t *testing.T, files map[string]string) (home, dl string, runs func() int) {
+ t.Helper()
+ home, dl = excludeTree(t, files)
+ bin := t.TempDir()
+ count := filepath.Join(t.TempDir(), "count")
+ script := "#!/bin/sh\necho x >> '" + count + "'\ncase \"$4\" in *fail*) exit 1;; esac\nexec /bin/cat \"$4\"\n"
+ if err := os.WriteFile(filepath.Join(bin, "pdftotext"), []byte(script), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ t.Setenv("PATH", bin)
+ runs = func() int {
+ b, _ := os.ReadFile(count)
+ return strings.Count(string(b), "x")
+ }
+ return home, dl, runs
+}
+
+// cachedMatch loads the configuration afresh, as a new krino process
+// would, and matches dl with the cache under ~/.cache/krino.
+func cachedMatch(t *testing.T, home, main string) *Result {
+ t.Helper()
+ e, errs := Load(main)
+ if len(errs) > 0 {
+ t.Fatal(errs)
+ }
+ e.CacheDir = filepath.Join(home, ".cache", "krino")
+ r, err := e.Match(context.Background(), e.Dirs[0])
+ if err != nil {
+ t.Fatal(err)
+ }
+ return r
+}
+
+func matchedNames(r *Result) string {
+ var names []string
+ for _, fm := range r.Matched {
+ names = append(names, fm.File.Rel)
+ }
+ return strings.Join(names, " ")
+}
+
+const acmeRules = `
+(path "~/dl")
+(rule "acme" (when (content "acme")) (move "Acme"))
+`
+
+// TestCacheSkipsExtractionOnRerun: the second run over unchanged files
+// extracts nothing and matches the same files.
+func TestCacheSkipsExtractionOnRerun(t *testing.T) {
+ home, _, runs := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME", "b.pdf": "nothing here"})
+ main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules})
+
+ first := cachedMatch(t, home, main)
+ if runs() != 2 || matchedNames(first) != "a.pdf" {
+ t.Fatalf("first run: %d extractions, matched %q", runs(), matchedNames(first))
+ }
+ second := cachedMatch(t, home, main)
+ if runs() != 2 {
+ t.Errorf("second run extracted again: %d runs in all", runs())
+ }
+ if matchedNames(second) != "a.pdf" {
+ t.Errorf("second run matched %q, want a.pdf", matchedNames(second))
+ }
+}
+
+// TestCacheRereadsChangedFile: a file whose modification time changed is
+// extracted again; the other is not.
+func TestCacheRereadsChangedFile(t *testing.T) {
+ home, dl, runs := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME", "b.pdf": "nothing here"})
+ main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules})
+ cachedMatch(t, home, main)
+
+ later := time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC)
+ if err := os.WriteFile(filepath.Join(dl, "b.pdf"), []byte("now ACME too"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ os.Chtimes(filepath.Join(dl, "b.pdf"), later, later)
+ r := cachedMatch(t, home, main)
+ if runs() != 3 {
+ t.Errorf("%d extractions in all, want 3: only b.pdf read again", runs())
+ }
+ if matchedNames(r) != "a.pdf b.pdf" {
+ t.Errorf("matched %q, want both", matchedNames(r))
+ }
+}
+
+// TestCacheRereadsForNewKeyword: a keyword no entry was checked against
+// makes the files that reach it be read again, once.
+func TestCacheRereadsForNewKeyword(t *testing.T) {
+ home, _, runs := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME", "b.pdf": "nothing here"})
+ main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules})
+ cachedMatch(t, home, main)
+
+ writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules + `(rule "nothing" (when (content "nothing")) (move "Nothing"))` + "\n"})
+ r := cachedMatch(t, home, main)
+ if runs() != 4 {
+ t.Errorf("%d extractions in all, want 4: both read again for the new keyword", runs())
+ }
+ if matchedNames(r) != "a.pdf b.pdf" {
+ t.Errorf("matched %q, want both", matchedNames(r))
+ }
+ cachedMatch(t, home, main)
+ if runs() != 4 {
+ t.Errorf("third run extracted again: %d runs in all", runs())
+ }
+}
+
+// TestCacheOffWithoutCacheDir: an engine with no CacheDir reads every time
+// and writes no cache.
+func TestCacheOffWithoutCacheDir(t *testing.T) {
+ home, _, runs := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME"})
+ main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules})
+ for i := 0; i < 2; i++ {
+ e, errs := Load(main)
+ if len(errs) > 0 {
+ t.Fatal(errs)
+ }
+ if _, err := e.Match(context.Background(), e.Dirs[0]); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if runs() != 2 {
+ t.Errorf("%d extractions, want one per run", runs())
+ }
+ if _, err := os.Stat(filepath.Join(home, ".cache")); !os.IsNotExist(err) {
+ t.Errorf("a cache was written with no CacheDir: %v", err)
+ }
+}
+
+// TestCacheDoesNotStoreFailures: a file the tool fails on is tried again
+// on every run, and warned about every time.
+func TestCacheDoesNotStoreFailures(t *testing.T) {
+ home, _, runs := countingPDFTree(t, map[string]string{"fail.pdf": "Invoice ACME"})
+ main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules})
+ for i := 1; i <= 2; i++ {
+ r := cachedMatch(t, home, main)
+ if runs() != i {
+ t.Errorf("run %d: %d extractions in all, want %d", i, runs(), i)
+ }
+ if len(r.Unmatched) != 1 || len(r.Unmatched[0].Warnings) == 0 {
+ t.Errorf("run %d: want fail.pdf unmatched with a warning: %+v", i, r.Unmatched)
+ }
+ }
+}
+
+// TestCacheRespectsMaxRead: a file over max-read is not read, even when an
+// earlier run cached its answers.
+func TestCacheRespectsMaxRead(t *testing.T) {
+ home, _, _ := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME"})
+ main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules})
+ cachedMatch(t, home, main)
+
+ writeConfig(t, home, `(include "dl")`, map[string]string{"dl": "(max-read 1)\n" + acmeRules})
+ r := cachedMatch(t, home, main)
+ if len(r.Matched) != 0 || len(r.Unmatched) != 1 || !strings.Contains(strings.Join(r.Unmatched[0].Warnings, " "), "larger than max-read") {
+ t.Errorf("want a.pdf unmatched as larger than max-read: matched %+v unmatched %+v", r.Matched, r.Unmatched)
+ }
+}
+
+// TestCacheHoldsNoText: the cache file holds the keywords and answers, not
+// the extracted text or the file's name.
+func TestCacheHoldsNoText(t *testing.T) {
+ home, _, _ := countingPDFTree(t, map[string]string{"secret-name.pdf": "Invoice ACME confidential"})
+ main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules})
+ cachedMatch(t, home, main)
+ b, err := os.ReadFile(filepath.Join(home, ".cache", "krino", "dl.cache"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, leak := range []string{"Invoice", "invoice", "confidential", "secret-name"} {
+ if strings.Contains(string(b), leak) {
+ t.Errorf("cache holds %q:\n%s", leak, b)
+ }
+ }
+}
+
+// TestExplainUsesCacheWithoutWriting: explain answers from the cache a run
+// wrote, and never writes one itself.
+func TestExplainUsesCacheWithoutWriting(t *testing.T) {
+ home, dl, runs := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME"})
+ main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules})
+ explain := func() {
+ t.Helper()
+ e, errs := Load(main)
+ if len(errs) > 0 {
+ t.Fatal(errs)
+ }
+ e.CacheDir = filepath.Join(home, ".cache", "krino")
+ x, err := e.Explain(context.Background(), filepath.Join(dl, "a.pdf"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(x.Rules) != 1 || !x.Rules[0].Match {
+ t.Errorf("explain: %+v", x.Rules)
+ }
+ }
+ explain()
+ if _, err := os.Stat(filepath.Join(home, ".cache")); !os.IsNotExist(err) {
+ t.Errorf("explain wrote a cache: %v", err)
+ }
+ cachedMatch(t, home, main)
+ before := runs()
+ explain()
+ if runs() != before {
+ t.Errorf("explain extracted although the run cached a.pdf")
+ }
+}
diff --git a/internal/engine/engine.go b/internal/engine/engine.go
index 8cffbd1..b50728e 100644
--- a/internal/engine/engine.go
+++ b/internal/engine/engine.go
@@ -8,6 +8,7 @@ package engine
import (
"fmt"
"os"
+ "sort"
"strings"
"time"
@@ -26,6 +27,11 @@ type Engine struct {
Extract *extract.Extractor
Now func() time.Time // time.Now; tests replace it
MainFile string
+
+ // CacheDir holds each directory's keyword cache (spec §6.1), as
+ // NAME.cache; "" means no cache is read or written. Load leaves it
+ // empty: the command line sets it.
+ CacheDir string
}
// Dir is one configured directory, with its ignore matcher and rules
@@ -38,13 +44,11 @@ type Dir struct {
Ignore *ignore.Matcher
Rules []*Rule
- // ContentVariants is the distinct (IgnoreCase, Fold) pairs any of
- // Rules' content tests evaluate under, in first-seen order. B2: when
- // this holds exactly one variant, facts.Content releases a file's raw
- // extracted text once that variant's normalised copy exists, since no
- // other variant will ever be asked for; with more than one, both must
- // stay memoised, as before.
- ContentVariants []cond.Options
+ // ContentKeywords is every content keyword the directory's excludes
+ // and rules test, once each, sorted by Key. When a file's text is
+ // extracted, every one of them is answered at once, so one extraction
+ // serves every content test and fills the keyword cache.
+ ContentKeywords []cond.Keyword
// Excludes are the (exclude ...) forms that apply here, compiled with the
// directory's settings: krino.conf's first, then the directory's own. A
@@ -135,7 +139,7 @@ func Load(mainFile string, names ...string) (*Engine, []*config.Diag) {
}
}
}
- dir.ContentVariants = contentVariants(dir.Rules, dir.Excludes, dirOpt)
+ dir.ContentKeywords = contentKeywords(dir.Rules, dir.Excludes)
dir.DupScopes = dupScopes(dir.Rules)
dirs = append(dirs, dir)
}
@@ -216,32 +220,26 @@ func dedupeNames(names []string) []string {
return out
}
-// contentVariants returns the distinct (IgnoreCase, Fold) pairs any content
-// test evaluates under - each exclude's (under the directory's settings,
-// dirOpt) and each rule's - in first-seen order — B2's per-Dir
-// ContentVariants. A rule whose condition has no content test at all
-// (Cond.UsesContent false) never calls facts.Content, so its resolved
-// case/fold settings contribute no variant here.
-func contentVariants(rules []*Rule, excludes []*Exclude, dirOpt cond.Options) []cond.Options {
- var out []cond.Options
- seen := map[cond.Options]bool{}
- for _, x := range excludes {
- if x.Cond.UsesContent && !seen[dirOpt] {
- seen[dirOpt] = true
- out = append(out, dirOpt)
+// contentKeywords returns every content keyword excludes and rules test,
+// each once, sorted by Key: Dir.ContentKeywords.
+func contentKeywords(rules []*Rule, excludes []*Exclude) []cond.Keyword {
+ seen := map[string]bool{}
+ var out []cond.Keyword
+ add := func(c *cond.Cond) {
+ for _, k := range c.Keywords {
+ if !seen[k.Key()] {
+ seen[k.Key()] = true
+ out = append(out, k)
+ }
}
}
+ for _, x := range excludes {
+ add(x.Cond)
+ }
for _, r := range rules {
- if !r.Cond.UsesContent {
- continue
- }
- opt := cond.Options{IgnoreCase: r.Settings.Case == config.CaseIgnore, Fold: r.Settings.Fold}
- if seen[opt] {
- continue
- }
- seen[opt] = true
- out = append(out, opt)
+ add(r.Cond)
}
+ sort.Slice(out, func(i, j int) bool { return out[i].Key() < out[j].Key() })
return out
}
diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go
index 18b35e7..c38c1fc 100644
--- a/internal/engine/engine_test.go
+++ b/internal/engine/engine_test.go
@@ -46,14 +46,14 @@ func writeConfig(t *testing.T, home, main string, dirs map[string]string) string
// fakeFacts is a minimal cond.Facts for checking compiled rules.
type fakeFacts struct{ name string }
-func (f fakeFacts) Name() string { return f.name }
-func (f fakeFacts) Rel() string { return f.name }
-func (f fakeFacts) Size() int64 { return 1 }
-func (f fakeFacts) ModTime() time.Time { return time.Time{} }
-func (f fakeFacts) Now() time.Time { return time.Time{} }
-func (f fakeFacts) Content(bool, bool) (string, error) { return "", nil }
-func (f fakeFacts) Duplicate([]string) (string, bool, error) { return "", false, nil }
-func (f fakeFacts) Matched() bool { return false }
+func (f fakeFacts) Name() string { return f.name }
+func (f fakeFacts) Rel() string { return f.name }
+func (f fakeFacts) Size() int64 { return 1 }
+func (f fakeFacts) ModTime() time.Time { return time.Time{} }
+func (f fakeFacts) Now() time.Time { return time.Time{} }
+func (f fakeFacts) ContentContains(cond.Options, []string) (int, error) { return -1, nil }
+func (f fakeFacts) Duplicate([]string) (string, bool, error) { return "", false, nil }
+func (f fakeFacts) Matched() bool { return false }
var _ cond.Facts = fakeFacts{}
@@ -187,12 +187,10 @@ func TestLoadAcceptsSuppliedCaptures(t *testing.T) {
}
}
-// TestContentVariantsComputedAtLoad: B2 plumbing. Load computes each
-// directory's distinct (ignoreCase, fold) content-test variants from its
-// rules' resolved settings: a rule with no content test contributes
-// nothing; two rules sharing a variant fold into one; a rule-level (case
-// ignore) override adds a second.
-func TestContentVariantsComputedAtLoad(t *testing.T) {
+// TestContentKeywordsComputedAtLoad: Load collects every content keyword
+// of a directory's rules, once per (options, normalised keyword), sorted by
+// key: a rule-level (case ignore) makes the same word a second keyword.
+func TestContentKeywordsComputedAtLoad(t *testing.T) {
h := sandbox(t)
os.Mkdir(filepath.Join(h, "dl"), 0o755)
main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": `
@@ -207,10 +205,11 @@ func TestContentVariantsComputedAtLoad(t *testing.T) {
if len(errs) > 0 {
t.Fatal(errs)
}
- got := e.Dirs[0].ContentVariants
- want := []cond.Options{{IgnoreCase: false, Fold: true}, {IgnoreCase: true, Fold: true}}
+ got := e.Dirs[0].ContentKeywords
+ strict, loose := cond.Options{IgnoreCase: false, Fold: true}, cond.Options{IgnoreCase: true, Fold: true}
+ want := []cond.Keyword{{Opt: loose, Norm: "acme"}, {Opt: strict, Norm: "acme"}, {Opt: strict, Norm: "other"}}
if !reflect.DeepEqual(got, want) {
- t.Fatalf("ContentVariants = %+v, want %+v", got, want)
+ t.Fatalf("ContentKeywords = %+v, want %+v", got, want)
}
}
diff --git a/internal/engine/exclude_test.go b/internal/engine/exclude_test.go
index 40b3548..8b26450 100644
--- a/internal/engine/exclude_test.go
+++ b/internal/engine/exclude_test.go
@@ -115,10 +115,10 @@ func TestExcludeNeedsEveryCondition(t *testing.T) {
}
}
-// TestExcludeContentKeepsRuleContentVariants: an exclude reading content
-// under the directory's settings must not release the raw text a rule
-// with different case/fold settings still needs.
-func TestExcludeContentKeepsRuleContentVariants(t *testing.T) {
+// TestExcludeContentAndRuleContentKeepTheirOwnSettings: an exclude reading
+// content under the directory's settings first must not change how a rule
+// with different case/fold settings sees the same text.
+func TestExcludeContentAndRuleContentKeepTheirOwnSettings(t *testing.T) {
h, _ := excludeTree(t, map[string]string{"a.txt": "Invoice ACME"})
main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": `
(path "~/dl")
diff --git a/internal/engine/facts.go b/internal/engine/facts.go
index 5790e95..9035245 100644
--- a/internal/engine/facts.go
+++ b/internal/engine/facts.go
@@ -12,6 +12,8 @@ import (
"krino/internal/cond"
"krino/internal/dup"
+ "krino/internal/extract"
+ "krino/internal/kwcache"
"krino/internal/norm"
"krino/internal/plan"
"krino/internal/scan"
@@ -30,6 +32,7 @@ type matchRun struct {
ctx context.Context
now time.Time
files []scan.File
+ cache *kwcache.Cache // nil: no keyword cache
mu sync.Mutex
dupOnce map[string]*sync.Once
@@ -103,26 +106,25 @@ func (run *matchRun) dupIndex(key string, dirs []string) *dup.Index {
}
// facts is one file's cond.Facts. It is used by exactly one goroutine, so
-// its own memoised state (content, its normalised variants, and whether an
-// earlier rule matched) needs no locking of its own; only the matchRun it
-// points at is shared.
+// its own memoised state (the keyword answers, and whether an earlier rule
+// matched) needs no locking of its own; only the matchRun it points at is
+// shared.
type facts struct {
run *matchRun
file scan.File
matched bool
- contentDone bool
- content string
- contentErr error
- normCache map[[2]bool]string
+ contentDone bool // extraction was attempted
+ contentErr error // why it failed
+ answers map[string]bool // by cond.KeywordKey, once extracted
}
var _ cond.Facts = (*facts)(nil)
// newFacts builds the Facts for one scanned file.
func newFacts(run *matchRun, file scan.File) *facts {
- return &facts{run: run, file: file, normCache: make(map[[2]bool]string)}
+ return &facts{run: run, file: file}
}
func (f *facts) Name() string { return f.file.Name }
@@ -132,32 +134,85 @@ func (f *facts) ModTime() time.Time { return f.file.ModTime }
func (f *facts) Now() time.Time { return f.run.now }
func (f *facts) Matched() bool { return f.matched }
-// Content extracts the file's text once, then normalises it per
-// (ignoreCase, fold) variant, memoising each. B2: when the directory's
-// rules use exactly one variant (Dir.ContentVariants), the raw text is
-// released as soon as that variant's normalised copy exists — no other
-// variant will ever be asked for, so there is no reason to keep both the
-// raw text and its normalised copy in memory at once. A directory using
-// more than one variant keeps the raw text for as long as f lives, exactly
-// as before.
-func (f *facts) Content(ignoreCase, fold bool) (string, error) {
+// ContentContains answers a content test (spec §6.1). A file above
+// max-read is never read, cached or not. Before the file has been
+// extracted this run, the keyword cache answers when it knows every one of
+// keywords for this file as it is now; otherwise the text is extracted
+// once, every keyword of the directory (and of this test) is answered from
+// it and stored in the cache, and the text itself is dropped. A failed
+// extraction is not cached: the next run tries again.
+func (f *facts) ContentContains(opt cond.Options, keywords []string) (int, error) {
+ if max := f.run.d.Settings.MaxRead; max > 0 && f.file.Size > max {
+ return -1, extract.ErrTooLarge
+ }
+ keys := make([]string, len(keywords))
+ for i, kw := range keywords {
+ keys[i] = cond.KeywordKey(opt, kw)
+ }
if !f.contentDone {
- f.content, f.contentErr = f.run.e.Extract.Text(f.run.ctx, f.file.Path, f.file.Size, f.run.d.Settings.MaxRead)
- f.contentDone = true
+ if id, ok := f.cacheID(); ok {
+ if hits, ok := f.run.cache.Lookup(id, keys); ok {
+ for i, hit := range hits {
+ if hit {
+ return i, nil
+ }
+ }
+ return -1, nil
+ }
+ }
+ f.extract(opt, keywords)
}
if f.contentErr != nil {
- return "", f.contentErr
+ return -1, f.contentErr
+ }
+ for i, k := range keys {
+ if f.answers[k] {
+ return i, nil
+ }
+ }
+ return -1, nil
+}
+
+// extract reads the file's text and answers every keyword of the directory,
+// plus the asking test's (opt, keywords), from it.
+func (f *facts) extract(opt cond.Options, keywords []string) {
+ f.contentDone = true
+ text, err := f.run.e.Extract.Text(f.run.ctx, f.file.Path, f.file.Size, f.run.d.Settings.MaxRead)
+ if err != nil {
+ f.contentErr = err
+ return
+ }
+ all := append([]cond.Keyword(nil), f.run.d.ContentKeywords...)
+ for _, kw := range keywords {
+ all = append(all, cond.Keyword{Opt: opt, Norm: kw})
+ }
+ normed := map[cond.Options]string{}
+ f.answers = make(map[string]bool, len(all))
+ for _, k := range all {
+ t, ok := normed[k.Opt]
+ if !ok {
+ t = norm.Text(text, k.Opt.IgnoreCase, k.Opt.Fold)
+ normed[k.Opt] = t
+ }
+ f.answers[k.Key()] = strings.Contains(t, k.Norm)
}
- key := [2]bool{ignoreCase, fold}
- if v, ok := f.normCache[key]; ok {
- return v, nil
+ if id, ok := f.cacheID(); ok {
+ f.run.cache.Store(id, f.answers)
}
- v := norm.Text(f.content, ignoreCase, fold)
- f.normCache[key] = v
- if len(f.run.d.ContentVariants) == 1 {
- f.content = ""
+}
+
+// cacheID is the file's keyword cache identity; ok is false when there is
+// no cache, or the platform gave the file no inode.
+func (f *facts) cacheID() (kwcache.ID, bool) {
+ if f.run.cache == nil || f.file.Ino == 0 {
+ return kwcache.ID{}, false
}
- return v, nil
+ return fileCacheID(f.file), true
+}
+
+// fileCacheID is file's kwcache.ID.
+func fileCacheID(file scan.File) kwcache.ID {
+ return kwcache.ID{Dev: file.Dev, Ino: file.Ino, Size: file.Size, MTime: file.ModTime.UnixNano()}
}
// Duplicate resolves dirs against the directory's root, builds (or reuses)
diff --git a/internal/engine/facts_test.go b/internal/engine/facts_test.go
index 7d844a0..9739fa6 100644
--- a/internal/engine/facts_test.go
+++ b/internal/engine/facts_test.go
@@ -3,79 +3,10 @@
package engine
import (
- "context"
- "os"
"path/filepath"
"testing"
- "time"
-
- "krino/internal/cond"
- "krino/internal/extract"
- "krino/internal/scan"
)
-// contentFacts builds a *facts for a real text file, under a Dir whose
-// ContentVariants is variants, for exercising B2's raw-release directly.
-func contentFacts(t *testing.T, body string, variants []cond.Options) *facts {
- t.Helper()
- p := filepath.Join(t.TempDir(), "a.txt")
- if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
- t.Fatal(err)
- }
- fi, err := os.Stat(p)
- if err != nil {
- t.Fatal(err)
- }
- e := &Engine{Extract: extract.New(), Now: time.Now}
- d := &Dir{Name: "d", ContentVariants: variants}
- file := scan.File{Path: p, Rel: "a.txt", Name: "a.txt", Size: fi.Size(), ModTime: fi.ModTime()}
- run := newMatchRun(e, d, context.Background(), time.Now(), []scan.File{file})
- return newFacts(run, file)
-}
-
-// TestContentReleasesRawWithOneVariant: B2. A directory whose rules use
-// exactly one (ignoreCase, fold) variant releases the raw extracted text
-// once that variant's normalised copy exists.
-func TestContentReleasesRawWithOneVariant(t *testing.T) {
- f := contentFacts(t, "Hello World", []cond.Options{{IgnoreCase: true, Fold: false}})
- const want = "hello world"
- got, err := f.Content(true, false)
- if err != nil {
- t.Fatal(err)
- }
- if got != want {
- t.Fatalf("got %q, want %q", got, want)
- }
- if f.content != "" {
- t.Errorf("raw text not released with a single content variant: %q", f.content)
- }
- // A second call for the same (already cached) variant must still work
- // from normCache, without needing the released raw text.
- if got, err := f.Content(true, false); err != nil || got != want {
- t.Errorf("second call for the cached variant: got %q, %v, want %q", got, err, want)
- }
-}
-
-// TestContentKeepsRawWithTwoVariants: B2. A directory whose rules use two
-// distinct variants must not release the raw text after the first: the
-// second variant still needs it, and normalising it correctly (not from an
-// emptied string) is the proof the raw text was kept.
-func TestContentKeepsRawWithTwoVariants(t *testing.T) {
- f := contentFacts(t, "Hello World", []cond.Options{
- {IgnoreCase: true, Fold: false},
- {IgnoreCase: false, Fold: false},
- })
- if got, err := f.Content(true, false); err != nil || got != "hello world" {
- t.Fatalf("first variant: got %q, %v", got, err)
- }
- if f.content == "" {
- t.Fatal("raw text released after only the first of two variants")
- }
- if got, err := f.Content(false, false); err != nil || got != "Hello World" {
- t.Fatalf("second variant: got %q, %v, want the unfolded original (raw text must still be available)", got, err)
- }
-}
-
// TestDisplayOriginalAbbreviatesHome: a duplicate's original is shown
// root-relative inside the root, and home-abbreviated outside it, the way
// every other user-visible path is; a path outside $HOME stays absolute.
diff --git a/internal/engine/match.go b/internal/engine/match.go
index 8daea9d..1a8077a 100644
--- a/internal/engine/match.go
+++ b/internal/engine/match.go
@@ -16,6 +16,7 @@ import (
"krino/internal/cond"
"krino/internal/config"
+ "krino/internal/kwcache"
"krino/internal/plan"
"krino/internal/scan"
"krino/internal/xdg"
@@ -70,6 +71,7 @@ func (e *Engine) Match(ctx context.Context, d *Dir) (*Result, error) {
}
run := newMatchRun(e, d, ctx, now, wres.Files)
+ cacheWarn := e.openCache(run)
fileMatches := make([]FileMatch, len(wres.Files))
workers := runtime.GOMAXPROCS(0)
@@ -103,7 +105,18 @@ func (e *Engine) Match(ctx context.Context, d *Dir) (*Result, error) {
}
}
- warnings := run.warnings()
+ warnings := append(run.warnings(), cacheWarn...)
+ if run.cache != nil {
+ ids := make([]kwcache.ID, 0, len(wres.Files))
+ for _, f := range wres.Files {
+ if f.Ino != 0 {
+ ids = append(ids, fileCacheID(f))
+ }
+ }
+ if err := run.cache.Save(e.cacheFile(d), ids); err != nil {
+ warnings = append(warnings, "cache: "+err.Error())
+ }
+ }
sort.Strings(warnings)
return &Result{
@@ -242,20 +255,14 @@ func (e *Engine) Explain(ctx context.Context, path string) (*Explanation, error)
}
rel = filepath.ToSlash(rel)
- sf := scan.File{
- Path: abs,
- Rel: rel,
- Name: filepath.Base(abs),
- Size: info.Size(),
- ModTime: info.ModTime(),
- Mode: info.Mode(),
- }
+ sf := scan.NewFile(abs, rel, info)
now := e.Now()
excl := e.excludeDirs(d)
skip := explainSkip(d, sf, excl, now)
run := newMatchRun(e, d, ctx, now, e.filesForExplain(d, sf, excl, now))
+ e.openCache(run) // read only: Explain never writes the cache
f := newFacts(run, sf)
var excludes []ExcludeTrace
@@ -289,6 +296,26 @@ func (e *Engine) Explain(ctx context.Context, path string) (*Explanation, error)
return &Explanation{Dir: d, File: sf, Skip: skip, Excludes: excludes, Excluded: excluded, Rules: rules}, nil
}
+// cacheFile is d's keyword cache file.
+func (e *Engine) cacheFile(d *Dir) string {
+ return filepath.Join(e.CacheDir, d.Name+".cache")
+}
+
+// openCache loads run's directory keyword cache into run.cache, when the
+// engine has a CacheDir and the directory has content tests at all. A cache
+// that cannot be read is replaced by an empty one, reported as a warning.
+func (e *Engine) openCache(run *matchRun) []string {
+ if e.CacheDir == "" || len(run.d.ContentKeywords) == 0 {
+ return nil
+ }
+ c, err := kwcache.Load(e.cacheFile(run.d), e.Extract.Fingerprint())
+ run.cache = c
+ if err != nil {
+ return []string{"cache: " + err.Error() + " (starting a new one)"}
+ }
+ return nil
+}
+
// filesForExplain returns the file set Explain's duplicate checks run
// against: the directory's ordinary scan, plus the explained file itself
// when that scan would not have reached it (it is busy, ignored, too new,
diff --git a/internal/extract/extract.go b/internal/extract/extract.go
index aa3ae93..9768db1 100644
--- a/internal/extract/extract.go
+++ b/internal/extract/extract.go
@@ -7,12 +7,18 @@ package extract
import (
"context"
"errors"
+ "fmt"
"os"
"path/filepath"
"strings"
"time"
)
+// Version is the version of the text this package extracts. Bump it
+// whenever a change could make any format's text differ, so every keyword
+// cache built from the old text is discarded (Fingerprint).
+const Version = 1
+
var (
// ErrUnsupported is returned when the format carries no text krino
// knows how to extract.
@@ -111,6 +117,26 @@ func newWithPath(path string) *Extractor {
return &Extractor{tools: tools, Timeout: 30 * time.Second}
}
+// Fingerprint identifies what text this Extractor would produce: Version,
+// and each external tool found with its path, size and modification time.
+// A keyword cache built under another fingerprint is discarded, so
+// installing, removing or upgrading a tool invalidates it.
+func (e *Extractor) Fingerprint() string {
+ var b strings.Builder
+ fmt.Fprintf(&b, "v%d", Version)
+ for _, name := range toolNames {
+ p := e.tools[name]
+ if p == "" {
+ continue
+ }
+ fmt.Fprintf(&b, " %s=%s", name, p)
+ if fi, err := os.Stat(p); err == nil {
+ fmt.Fprintf(&b, ":%d:%d", fi.Size(), fi.ModTime().UnixNano())
+ }
+ }
+ return b.String()
+}
+
// Tools reports every external tool Extractor knows about, in a fixed
// order, with the path it was found at or "" if it was not found.
func (e *Extractor) Tools() []Tool {
diff --git a/internal/extract/tools_test.go b/internal/extract/tools_test.go
index 4a03f1b..b481e8a 100644
--- a/internal/extract/tools_test.go
+++ b/internal/extract/tools_test.go
@@ -237,3 +237,27 @@ func TestMaxReadCapsToolOutput(t *testing.T) {
t.Fatalf("got %v, want ErrTooLarge (max-read 1024 should have capped a 200000-byte tool output)", err)
}
}
+
+// TestFingerprintFollowsTools: the fingerprint changes when a tool is
+// installed or replaced, and not otherwise.
+func TestFingerprintFollowsTools(t *testing.T) {
+ bin := t.TempDir()
+ none := newWithPath(bin).Fingerprint()
+ if none != newWithPath(bin).Fingerprint() {
+ t.Fatal("fingerprint not stable")
+ }
+ tool := filepath.Join(bin, "pdftotext")
+ if err := os.WriteFile(tool, []byte("#!/bin/sh\n"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ installed := newWithPath(bin).Fingerprint()
+ if installed == none {
+ t.Error("installing pdftotext did not change the fingerprint")
+ }
+ if err := os.WriteFile(tool, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if newWithPath(bin).Fingerprint() == installed {
+ t.Error("replacing pdftotext did not change the fingerprint")
+ }
+}
diff --git a/internal/kwcache/kwcache.go b/internal/kwcache/kwcache.go
new file mode 100644
index 0000000..b61cfb1
--- /dev/null
+++ b/internal/kwcache/kwcache.go
@@ -0,0 +1,231 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+// Package kwcache remembers which content keywords a file's extracted text
+// contains, so a file that has not changed is not extracted again (spec
+// §6.1). It stores answers only, never the text, and knows a file by its
+// ID, never by its name.
+package kwcache
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+ "sync"
+)
+
+// version is the on-disk format; a file of any other version loads empty.
+const version = 1
+
+// ID is how a file is recognised: the same inode with the same size and
+// modification time is taken to hold the same content. A move or rename
+// within one filesystem keeps it.
+type ID struct {
+ Dev, Ino uint64
+ Size int64
+ MTime int64 // Unix nanoseconds
+}
+
+// entry is what is known about one file: every keyword it was checked
+// against, and those its text contains.
+type entry struct {
+ checked []string // sorted
+ hits map[string]bool
+}
+
+// Cache holds one directory's answers: those loaded from disk and those
+// stored during this run. It is safe for concurrent use.
+type Cache struct {
+ fingerprint string
+ existed bool // Load found a file, so Save must rewrite it even when empty
+
+ mu sync.Mutex
+ old map[ID]entry
+ cur map[ID]entry
+}
+
+// New returns an empty cache for extractor fingerprint.
+func New(fingerprint string) *Cache {
+ return &Cache{fingerprint: fingerprint, old: map[ID]entry{}, cur: map[ID]entry{}}
+}
+
+type diskFile struct {
+ Dev uint64 `json:"dev"`
+ Ino uint64 `json:"ino"`
+ Size int64 `json:"size"`
+ MTime int64 `json:"mtime"`
+ Set int `json:"keywords"` // index into diskCache.Keywords
+ Hits []int `json:"hits"` // indices into that keyword list
+}
+
+type diskCache struct {
+ Version int `json:"version"`
+ Fingerprint string `json:"fingerprint"`
+ Keywords [][]string `json:"keywords"`
+ Files []diskFile `json:"files"`
+}
+
+// Load reads the cache at path. A missing file, another format version or
+// another fingerprint is an empty cache and no error; an unreadable file
+// is an empty cache and an error. The cache returned is never nil.
+func Load(path, fingerprint string) (*Cache, error) {
+ c := New(fingerprint)
+ data, err := os.ReadFile(path)
+ if errors.Is(err, fs.ErrNotExist) {
+ return c, nil
+ }
+ c.existed = err == nil
+ if err != nil {
+ return c, err
+ }
+ var d diskCache
+ if err := json.Unmarshal(data, &d); err != nil {
+ return c, fmt.Errorf("%s: %v", path, err)
+ }
+ if d.Version != version || d.Fingerprint != fingerprint {
+ return c, nil
+ }
+ for i, set := range d.Keywords {
+ if !sort.StringsAreSorted(set) {
+ return New(fingerprint), fmt.Errorf("%s: keyword list %d is not sorted", path, i)
+ }
+ }
+ for _, f := range d.Files {
+ if f.Set < 0 || f.Set >= len(d.Keywords) {
+ return New(fingerprint), fmt.Errorf("%s: bad keyword list %d", path, f.Set)
+ }
+ set := d.Keywords[f.Set]
+ hits := make(map[string]bool, len(f.Hits))
+ for _, h := range f.Hits {
+ if h < 0 || h >= len(set) {
+ return New(fingerprint), fmt.Errorf("%s: bad keyword %d", path, h)
+ }
+ hits[set[h]] = true
+ }
+ c.old[ID{Dev: f.Dev, Ino: f.Ino, Size: f.Size, MTime: f.MTime}] = entry{checked: set, hits: hits}
+ }
+ c.existed = true
+ return c, nil
+}
+
+// Lookup reports, for each of keys, whether the text of the file id
+// contains it. ok is false unless an entry for id exists and was checked
+// against every one of keys.
+func (c *Cache) Lookup(id ID, keys []string) (hits []bool, ok bool) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ e, found := c.cur[id]
+ if !found {
+ e, found = c.old[id]
+ }
+ if !found {
+ return nil, false
+ }
+ hits = make([]bool, len(keys))
+ for i, k := range keys {
+ j := sort.SearchStrings(e.checked, k)
+ if j == len(e.checked) || e.checked[j] != k {
+ return nil, false
+ }
+ hits[i] = e.hits[k]
+ }
+ return hits, true
+}
+
+// Store records answers, every keyword the file id was checked against
+// and whether its text contains it, replacing anything known about id.
+func (c *Cache) Store(id ID, answers map[string]bool) {
+ e := entry{checked: make([]string, 0, len(answers)), hits: map[string]bool{}}
+ for k, hit := range answers {
+ e.checked = append(e.checked, k)
+ if hit {
+ e.hits[k] = true
+ }
+ }
+ sort.Strings(e.checked)
+ c.mu.Lock()
+ c.cur[id] = e
+ c.mu.Unlock()
+}
+
+// Save writes the entries of the files in present, from this run or loaded,
+// to path, and drops every other: the cache only ever describes files
+// still in the directory. The directory is created 0700 and the file
+// written 0600 under a temporary name, then renamed into place. A cache
+// with nothing to write and no file on disk writes nothing.
+func (c *Cache) Save(path string, present []ID) error {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+
+ d := diskCache{Version: version, Fingerprint: c.fingerprint, Keywords: [][]string{}, Files: []diskFile{}}
+ sets := map[string]int{}
+ seen := map[ID]bool{}
+ for _, id := range present {
+ if seen[id] {
+ continue
+ }
+ seen[id] = true
+ e, ok := c.cur[id]
+ if !ok {
+ e, ok = c.old[id]
+ }
+ if !ok {
+ continue
+ }
+ key := strings.Join(e.checked, "\x00")
+ set, ok := sets[key]
+ if !ok {
+ set = len(d.Keywords)
+ sets[key] = set
+ d.Keywords = append(d.Keywords, e.checked)
+ }
+ f := diskFile{Dev: id.Dev, Ino: id.Ino, Size: id.Size, MTime: id.MTime, Set: set, Hits: []int{}}
+ for i, k := range e.checked {
+ if e.hits[k] {
+ f.Hits = append(f.Hits, i)
+ }
+ }
+ d.Files = append(d.Files, f)
+ }
+ if len(d.Files) == 0 && !c.existed {
+ return nil
+ }
+ sort.Slice(d.Files, func(i, j int) bool {
+ if d.Files[i].Dev != d.Files[j].Dev {
+ return d.Files[i].Dev < d.Files[j].Dev
+ }
+ return d.Files[i].Ino < d.Files[j].Ino
+ })
+
+ data, err := json.Marshal(d)
+ if err != nil {
+ return err
+ }
+ dir := filepath.Dir(path)
+ if err := os.MkdirAll(dir, 0o700); err != nil {
+ return err
+ }
+ tmp, err := os.CreateTemp(dir, ".kwcache-*")
+ if err != nil {
+ return err
+ }
+ if _, err := tmp.Write(data); err != nil {
+ tmp.Close()
+ os.Remove(tmp.Name())
+ return err
+ }
+ if err := tmp.Close(); err != nil {
+ os.Remove(tmp.Name())
+ return err
+ }
+ if err := os.Rename(tmp.Name(), path); err != nil {
+ os.Remove(tmp.Name())
+ return err
+ }
+ c.existed = true
+ return nil
+}
diff --git a/internal/kwcache/kwcache_test.go b/internal/kwcache/kwcache_test.go
new file mode 100644
index 0000000..8395b2d
--- /dev/null
+++ b/internal/kwcache/kwcache_test.go
@@ -0,0 +1,180 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package kwcache
+
+import (
+ "os"
+ "path/filepath"
+ "reflect"
+ "testing"
+)
+
+var (
+ a = ID{Dev: 1, Ino: 10, Size: 100, MTime: 1000}
+ b = ID{Dev: 1, Ino: 11, Size: 200, MTime: 2000}
+)
+
+func saved(t *testing.T, c *Cache, present ...ID) string {
+ t.Helper()
+ path := filepath.Join(t.TempDir(), "sub", "dl.cache")
+ if err := c.Save(path, present); err != nil {
+ t.Fatal(err)
+ }
+ return path
+}
+
+// TestLookupAfterReload: answers stored and saved come back after a Load
+// with the same fingerprint, for any keywords the entry covers.
+func TestLookupAfterReload(t *testing.T) {
+ c := New("fp")
+ c.Store(a, map[string]bool{"k:acme": true, "k:faktura": false, "k:nip": true})
+ path := saved(t, c, a)
+
+ got, err := Load(path, "fp")
+ if err != nil {
+ t.Fatal(err)
+ }
+ hits, ok := got.Lookup(a, []string{"k:faktura", "k:nip"})
+ if !ok || !reflect.DeepEqual(hits, []bool{false, true}) {
+ t.Errorf("Lookup = %v, %v; want [false true], true", hits, ok)
+ }
+ if _, ok := got.Lookup(a, []string{"k:acme", "k:new"}); ok {
+ t.Error("a keyword the entry was never checked against must be a miss")
+ }
+ if _, ok := got.Lookup(b, []string{"k:acme"}); ok {
+ t.Error("a file never stored must be a miss")
+ }
+}
+
+// TestLookupBeforeSave: what was stored in this run answers at once.
+func TestLookupBeforeSave(t *testing.T) {
+ c := New("fp")
+ c.Store(a, map[string]bool{"k:acme": true})
+ if hits, ok := c.Lookup(a, []string{"k:acme"}); !ok || !hits[0] {
+ t.Errorf("Lookup = %v, %v", hits, ok)
+ }
+}
+
+// TestChangedFileMisses: a different size or modification time is a
+// different ID, so nothing stored for the old one answers.
+func TestChangedFileMisses(t *testing.T) {
+ c := New("fp")
+ c.Store(a, map[string]bool{"k:acme": true})
+ for _, id := range []ID{{Dev: 1, Ino: 10, Size: 101, MTime: 1000}, {Dev: 1, Ino: 10, Size: 100, MTime: 1001}} {
+ if _, ok := c.Lookup(id, []string{"k:acme"}); ok {
+ t.Errorf("%+v answered from %+v's entry", id, a)
+ }
+ }
+}
+
+// TestFingerprintMismatchDiscardsAll: a cache written under a different
+// extractor fingerprint loads empty, without error.
+func TestFingerprintMismatchDiscardsAll(t *testing.T) {
+ c := New("old")
+ c.Store(a, map[string]bool{"k:acme": true})
+ path := saved(t, c, a)
+ got, err := Load(path, "new")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, ok := got.Lookup(a, []string{"k:acme"}); ok {
+ t.Error("entry survived a fingerprint change")
+ }
+}
+
+// TestSaveKeepsOnlyPresentFiles: entries for files not passed to Save, from
+// this run or an earlier one, are dropped.
+func TestSaveKeepsOnlyPresentFiles(t *testing.T) {
+ c := New("fp")
+ c.Store(a, map[string]bool{"k:acme": true})
+ c.Store(b, map[string]bool{"k:acme": false})
+ path := saved(t, c, a, b)
+
+ next, err := Load(path, "fp")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := next.Save(path, []ID{b}); err != nil {
+ t.Fatal(err)
+ }
+ last, err := Load(path, "fp")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, ok := last.Lookup(a, []string{"k:acme"}); ok {
+ t.Error("a is gone from the directory but its entry was kept")
+ }
+ if hits, ok := last.Lookup(b, []string{"k:acme"}); !ok || hits[0] {
+ t.Errorf("b's entry lost across a reload: %v, %v", hits, ok)
+ }
+}
+
+// TestStoreReplacesLoadedEntry: a file read again this run replaces what
+// an earlier run stored for it.
+func TestStoreReplacesLoadedEntry(t *testing.T) {
+ c := New("fp")
+ c.Store(a, map[string]bool{"k:acme": true})
+ path := saved(t, c, a)
+ next, _ := Load(path, "fp")
+ next.Store(a, map[string]bool{"k:acme": false, "k:new": true})
+ if hits, ok := next.Lookup(a, []string{"k:acme", "k:new"}); !ok || hits[0] || !hits[1] {
+ t.Errorf("Lookup = %v, %v; want the new entry", hits, ok)
+ }
+}
+
+// TestSavePermissions: the directory is private and the file readable by
+// its owner only, with no temporary file left behind.
+func TestSavePermissions(t *testing.T) {
+ c := New("fp")
+ c.Store(a, map[string]bool{"k:acme": true})
+ path := saved(t, c, a)
+ fi, err := os.Stat(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if perm := fi.Mode().Perm(); perm != 0o600 {
+ t.Errorf("file mode %o, want 600", perm)
+ }
+ di, err := os.Stat(filepath.Dir(path))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if perm := di.Mode().Perm(); perm != 0o700 {
+ t.Errorf("directory mode %o, want 700", perm)
+ }
+ entries, _ := os.ReadDir(filepath.Dir(path))
+ if len(entries) != 1 {
+ t.Errorf("directory holds %d entries, want only the cache", len(entries))
+ }
+}
+
+// TestLoadMissingAndCorrupt: no file is an empty cache and no error; an
+// unreadable one is an empty cache and an error saying so.
+func TestLoadMissingAndCorrupt(t *testing.T) {
+ dir := t.TempDir()
+ c, err := Load(filepath.Join(dir, "none.cache"), "fp")
+ if err != nil || c == nil {
+ t.Fatalf("missing: %v, %v", c, err)
+ }
+ bad := filepath.Join(dir, "bad.cache")
+ os.WriteFile(bad, []byte("{not json"), 0o600)
+ c, err = Load(bad, "fp")
+ if err == nil || c == nil {
+ t.Fatalf("corrupt: want an empty cache and an error, got %v, %v", c, err)
+ }
+ if _, ok := c.Lookup(a, []string{"k:acme"}); ok {
+ t.Error("corrupt cache answered")
+ }
+}
+
+// TestSaveWithNothingWritesNothing: a cache that never held an entry, and
+// has no file yet, creates none.
+func TestSaveWithNothingWritesNothing(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "sub", "dl.cache")
+ if err := New("fp").Save(path, []ID{a}); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := os.Stat(filepath.Dir(path)); !os.IsNotExist(err) {
+ t.Errorf("Save created %s for an empty cache", filepath.Dir(path))
+ }
+}
diff --git a/internal/scan/fileid_other.go b/internal/scan/fileid_other.go
new file mode 100644
index 0000000..503a9b4
--- /dev/null
+++ b/internal/scan/fileid_other.go
@@ -0,0 +1,10 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+//go:build !unix
+
+package scan
+
+import "io/fs"
+
+// fileID reports no identity where the platform has no inodes.
+func fileID(fs.FileInfo) (dev, ino uint64) { return 0, 0 }
diff --git a/internal/scan/fileid_unix.go b/internal/scan/fileid_unix.go
new file mode 100644
index 0000000..e29066a
--- /dev/null
+++ b/internal/scan/fileid_unix.go
@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+//go:build unix
+
+package scan
+
+import (
+ "io/fs"
+ "syscall"
+)
+
+// fileID returns the device and inode info was read from.
+func fileID(info fs.FileInfo) (dev, ino uint64) {
+ st, ok := info.Sys().(*syscall.Stat_t)
+ if !ok {
+ return 0, 0
+ }
+ return uint64(st.Dev), uint64(st.Ino)
+}
diff --git a/internal/scan/scan.go b/internal/scan/scan.go
index 7fe9e92..77cca66 100644
--- a/internal/scan/scan.go
+++ b/internal/scan/scan.go
@@ -24,6 +24,26 @@ type File struct {
Size int64
ModTime time.Time
Mode fs.FileMode
+
+ // Dev and Ino identify the file on its filesystem; both are 0 where the
+ // platform reports neither.
+ Dev, Ino uint64
+}
+
+// NewFile builds the File for path, at rel under the root, from its Lstat
+// info.
+func NewFile(path, rel string, info fs.FileInfo) File {
+ dev, ino := fileID(info)
+ return File{
+ Path: path,
+ Rel: rel,
+ Name: filepath.Base(path),
+ Size: info.Size(),
+ ModTime: info.ModTime(),
+ Mode: info.Mode(),
+ Dev: dev,
+ Ino: ino,
+ }
}
// Reason is why an entry was not returned as a File.
@@ -217,14 +237,7 @@ func (w *walker) walk(dir, relDir string, depth int, entries []os.DirEntry) erro
w.result.Skipped = append(w.result.Skipped, Skipped{Rel: rel, Reason: TooBig})
continue
}
- w.result.Files = append(w.result.Files, File{
- Path: path,
- Rel: rel,
- Name: name,
- Size: info.Size(),
- ModTime: info.ModTime(),
- Mode: info.Mode(),
- })
+ w.result.Files = append(w.result.Files, NewFile(path, rel, info))
}
return nil
}
diff --git a/internal/scan/scan_test.go b/internal/scan/scan_test.go
index a245893..e9aeb73 100644
--- a/internal/scan/scan_test.go
+++ b/internal/scan/scan_test.go
@@ -293,3 +293,29 @@ func TestTooBig(t *testing.T) {
t.Errorf("MaxSize 0 skipped files: %v", skipped(r))
}
}
+
+// TestFilesCarryInode: a walked file carries its device and inode, which a
+// rename keeps.
+func TestFilesCarryInode(t *testing.T) {
+ root := tree(t)
+ p := filepath.Join(root, "a.txt")
+ if err := os.WriteFile(p, []byte("x"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ old := now.Add(-time.Hour)
+ os.Chtimes(p, old, old)
+ first, err := Walk(root, Options{Now: now})
+ if err != nil || len(first.Files) != 1 {
+ t.Fatalf("walk: %v %+v", err, first)
+ }
+ if first.Files[0].Ino == 0 {
+ t.Fatal("no inode on a unix filesystem")
+ }
+ if err := os.Rename(p, filepath.Join(root, "b.txt")); err != nil {
+ t.Fatal(err)
+ }
+ second, _ := Walk(root, Options{Now: now})
+ if len(second.Files) != 1 || second.Files[0].Ino != first.Files[0].Ino || second.Files[0].Dev != first.Files[0].Dev {
+ t.Errorf("rename changed the identity: %+v then %+v", first.Files[0], second.Files)
+ }
+}
diff --git a/internal/xdg/xdg.go b/internal/xdg/xdg.go
index ed34838..be13650 100644
--- a/internal/xdg/xdg.go
+++ b/internal/xdg/xdg.go
@@ -18,6 +18,9 @@ func StateHome() string { return base("XDG_STATE_HOME", filepath.Join(".local",
// DataHome is $XDG_DATA_HOME, or ~/.local/share.
func DataHome() string { return base("XDG_DATA_HOME", filepath.Join(".local", "share")) }
+// CacheHome is $XDG_CACHE_HOME, or ~/.cache.
+func CacheHome() string { return base("XDG_CACHE_HOME", ".cache") }
+
// base follows the XDG rule that a relative value is invalid and ignored.
func base(env, fallback string) string {
if v := os.Getenv(env); filepath.IsAbs(v) {