krino: 0.0.5 — keyword cache, t and d in reviewv0.0.5

1 files changed, 224 insertions, 0 deletions

diff --git a/internal/engine/cache_test.go b/internal/engine/cache_test.go
new file mode 100644
index 0000000..eee2860
--- /dev/null
+++ b/internal/engine/cache_test.go
@@ -0,0 +1,224 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package engine
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+// countingPDFTree makes ~/dl holding the given .pdf files, and a fake
+// pdftotext first on PATH that prints the file's own bytes as its text and
+// counts its runs in a file; a file whose name contains "fail" makes it
+// exit 1.
+func countingPDFTree(t *testing.T, files map[string]string) (home, dl string, runs func() int) {
+	t.Helper()
+	home, dl = excludeTree(t, files)
+	bin := t.TempDir()
+	count := filepath.Join(t.TempDir(), "count")
+	script := "#!/bin/sh\necho x >> '" + count + "'\ncase \"$4\" in *fail*) exit 1;; esac\nexec /bin/cat \"$4\"\n"
+	if err := os.WriteFile(filepath.Join(bin, "pdftotext"), []byte(script), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("PATH", bin)
+	runs = func() int {
+		b, _ := os.ReadFile(count)
+		return strings.Count(string(b), "x")
+	}
+	return home, dl, runs
+}
+
+// cachedMatch loads the configuration afresh, as a new krino process
+// would, and matches dl with the cache under ~/.cache/krino.
+func cachedMatch(t *testing.T, home, main string) *Result {
+	t.Helper()
+	e, errs := Load(main)
+	if len(errs) > 0 {
+		t.Fatal(errs)
+	}
+	e.CacheDir = filepath.Join(home, ".cache", "krino")
+	r, err := e.Match(context.Background(), e.Dirs[0])
+	if err != nil {
+		t.Fatal(err)
+	}
+	return r
+}
+
+func matchedNames(r *Result) string {
+	var names []string
+	for _, fm := range r.Matched {
+		names = append(names, fm.File.Rel)
+	}
+	return strings.Join(names, " ")
+}
+
+const acmeRules = `
+(path "~/dl")
+(rule "acme" (when (content "acme")) (move "Acme"))
+`
+
+// TestCacheSkipsExtractionOnRerun: the second run over unchanged files
+// extracts nothing and matches the same files.
+func TestCacheSkipsExtractionOnRerun(t *testing.T) {
+	home, _, runs := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME", "b.pdf": "nothing here"})
+	main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules})
+
+	first := cachedMatch(t, home, main)
+	if runs() != 2 || matchedNames(first) != "a.pdf" {
+		t.Fatalf("first run: %d extractions, matched %q", runs(), matchedNames(first))
+	}
+	second := cachedMatch(t, home, main)
+	if runs() != 2 {
+		t.Errorf("second run extracted again: %d runs in all", runs())
+	}
+	if matchedNames(second) != "a.pdf" {
+		t.Errorf("second run matched %q, want a.pdf", matchedNames(second))
+	}
+}
+
+// TestCacheRereadsChangedFile: a file whose modification time changed is
+// extracted again; the other is not.
+func TestCacheRereadsChangedFile(t *testing.T) {
+	home, dl, runs := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME", "b.pdf": "nothing here"})
+	main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules})
+	cachedMatch(t, home, main)
+
+	later := time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC)
+	if err := os.WriteFile(filepath.Join(dl, "b.pdf"), []byte("now ACME too"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	os.Chtimes(filepath.Join(dl, "b.pdf"), later, later)
+	r := cachedMatch(t, home, main)
+	if runs() != 3 {
+		t.Errorf("%d extractions in all, want 3: only b.pdf read again", runs())
+	}
+	if matchedNames(r) != "a.pdf b.pdf" {
+		t.Errorf("matched %q, want both", matchedNames(r))
+	}
+}
+
+// TestCacheRereadsForNewKeyword: a keyword no entry was checked against
+// makes the files that reach it be read again, once.
+func TestCacheRereadsForNewKeyword(t *testing.T) {
+	home, _, runs := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME", "b.pdf": "nothing here"})
+	main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules})
+	cachedMatch(t, home, main)
+
+	writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules + `(rule "nothing" (when (content "nothing")) (move "Nothing"))` + "\n"})
+	r := cachedMatch(t, home, main)
+	if runs() != 4 {
+		t.Errorf("%d extractions in all, want 4: both read again for the new keyword", runs())
+	}
+	if matchedNames(r) != "a.pdf b.pdf" {
+		t.Errorf("matched %q, want both", matchedNames(r))
+	}
+	cachedMatch(t, home, main)
+	if runs() != 4 {
+		t.Errorf("third run extracted again: %d runs in all", runs())
+	}
+}
+
+// TestCacheOffWithoutCacheDir: an engine with no CacheDir reads every time
+// and writes no cache.
+func TestCacheOffWithoutCacheDir(t *testing.T) {
+	home, _, runs := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME"})
+	main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules})
+	for i := 0; i < 2; i++ {
+		e, errs := Load(main)
+		if len(errs) > 0 {
+			t.Fatal(errs)
+		}
+		if _, err := e.Match(context.Background(), e.Dirs[0]); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if runs() != 2 {
+		t.Errorf("%d extractions, want one per run", runs())
+	}
+	if _, err := os.Stat(filepath.Join(home, ".cache")); !os.IsNotExist(err) {
+		t.Errorf("a cache was written with no CacheDir: %v", err)
+	}
+}
+
+// TestCacheDoesNotStoreFailures: a file the tool fails on is tried again
+// on every run, and warned about every time.
+func TestCacheDoesNotStoreFailures(t *testing.T) {
+	home, _, runs := countingPDFTree(t, map[string]string{"fail.pdf": "Invoice ACME"})
+	main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules})
+	for i := 1; i <= 2; i++ {
+		r := cachedMatch(t, home, main)
+		if runs() != i {
+			t.Errorf("run %d: %d extractions in all, want %d", i, runs(), i)
+		}
+		if len(r.Unmatched) != 1 || len(r.Unmatched[0].Warnings) == 0 {
+			t.Errorf("run %d: want fail.pdf unmatched with a warning: %+v", i, r.Unmatched)
+		}
+	}
+}
+
+// TestCacheRespectsMaxRead: a file over max-read is not read, even when an
+// earlier run cached its answers.
+func TestCacheRespectsMaxRead(t *testing.T) {
+	home, _, _ := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME"})
+	main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules})
+	cachedMatch(t, home, main)
+
+	writeConfig(t, home, `(include "dl")`, map[string]string{"dl": "(max-read 1)\n" + acmeRules})
+	r := cachedMatch(t, home, main)
+	if len(r.Matched) != 0 || len(r.Unmatched) != 1 || !strings.Contains(strings.Join(r.Unmatched[0].Warnings, " "), "larger than max-read") {
+		t.Errorf("want a.pdf unmatched as larger than max-read: matched %+v unmatched %+v", r.Matched, r.Unmatched)
+	}
+}
+
+// TestCacheHoldsNoText: the cache file holds the keywords and answers, not
+// the extracted text or the file's name.
+func TestCacheHoldsNoText(t *testing.T) {
+	home, _, _ := countingPDFTree(t, map[string]string{"secret-name.pdf": "Invoice ACME confidential"})
+	main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules})
+	cachedMatch(t, home, main)
+	b, err := os.ReadFile(filepath.Join(home, ".cache", "krino", "dl.cache"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, leak := range []string{"Invoice", "invoice", "confidential", "secret-name"} {
+		if strings.Contains(string(b), leak) {
+			t.Errorf("cache holds %q:\n%s", leak, b)
+		}
+	}
+}
+
+// TestExplainUsesCacheWithoutWriting: explain answers from the cache a run
+// wrote, and never writes one itself.
+func TestExplainUsesCacheWithoutWriting(t *testing.T) {
+	home, dl, runs := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME"})
+	main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules})
+	explain := func() {
+		t.Helper()
+		e, errs := Load(main)
+		if len(errs) > 0 {
+			t.Fatal(errs)
+		}
+		e.CacheDir = filepath.Join(home, ".cache", "krino")
+		x, err := e.Explain(context.Background(), filepath.Join(dl, "a.pdf"))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(x.Rules) != 1 || !x.Rules[0].Match {
+			t.Errorf("explain: %+v", x.Rules)
+		}
+	}
+	explain()
+	if _, err := os.Stat(filepath.Join(home, ".cache")); !os.IsNotExist(err) {
+		t.Errorf("explain wrote a cache: %v", err)
+	}
+	cachedMatch(t, home, main)
+	before := runs()
+	explain()
+	if runs() != before {
+		t.Errorf("explain extracted although the run cached a.pdf")
+	}
+}