// SPDX-License-Identifier: GPL-3.0-or-later package engine import ( "context" "os" "path/filepath" "runtime" "strings" "testing" "time" "unicode" "git.labunix.xyz/krino/internal/config" "git.labunix.xyz/krino/internal/extract" ) // countingPDFTree makes ~/dl holding the given .pdf files, and a fake // pdftotext first on PATH that prints the file's own bytes as its text and // counts its runs in a file; a file whose name contains "fail" makes it // exit 1. func countingPDFTree(t *testing.T, files map[string]string) (home, dl string, runs func() int) { t.Helper() home, dl = excludeTree(t, files) bin := t.TempDir() count := filepath.Join(t.TempDir(), "count") script := "#!/bin/sh\necho x >> '" + count + "'\ncase \"$4\" in *fail*) exit 1;; esac\nexec /bin/cat \"$4\"\n" if err := os.WriteFile(filepath.Join(bin, "pdftotext"), []byte(script), 0o755); err != nil { t.Fatal(err) } t.Setenv("PATH", bin) runs = func() int { b, _ := os.ReadFile(count) return strings.Count(string(b), "x") } return home, dl, runs } // cachedMatch loads the configuration afresh, as a new krino process // would, and matches dl with the cache under ~/.cache/krino. func cachedMatch(t *testing.T, home, main string) *Result { t.Helper() e, errs := Load(main) if len(errs) > 0 { t.Fatal(errs) } e.CacheDir = filepath.Join(home, ".cache", "krino") r, err := e.Match(context.Background(), e.Dirs[0]) if err != nil { t.Fatal(err) } return r } func matchedNames(r *Result) string { var names []string for _, fm := range r.Matched { names = append(names, fm.File.Rel) } return strings.Join(names, " ") } const acmeRules = ` (path "~/dl") (rule "acme" (when (content "acme")) (move "Acme")) ` // TestCacheSkipsExtractionOnRerun: the second run over unchanged files // extracts nothing and matches the same files. func TestCacheSkipsExtractionOnRerun(t *testing.T) { home, _, runs := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME", "b.pdf": "nothing here"}) main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules}) first := cachedMatch(t, home, main) if runs() != 2 || matchedNames(first) != "a.pdf" { t.Fatalf("first run: %d extractions, matched %q", runs(), matchedNames(first)) } second := cachedMatch(t, home, main) if runs() != 2 { t.Errorf("second run extracted again: %d runs in all", runs()) } if matchedNames(second) != "a.pdf" { t.Errorf("second run matched %q, want a.pdf", matchedNames(second)) } } // TestCacheRereadsChangedFile: a file whose modification time changed is // extracted again; the other is not. func TestCacheRereadsChangedFile(t *testing.T) { home, dl, runs := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME", "b.pdf": "nothing here"}) main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules}) cachedMatch(t, home, main) later := time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC) if err := os.WriteFile(filepath.Join(dl, "b.pdf"), []byte("now ACME too"), 0o644); err != nil { t.Fatal(err) } os.Chtimes(filepath.Join(dl, "b.pdf"), later, later) r := cachedMatch(t, home, main) if runs() != 3 { t.Errorf("%d extractions in all, want 3: only b.pdf read again", runs()) } if matchedNames(r) != "a.pdf b.pdf" { t.Errorf("matched %q, want both", matchedNames(r)) } } // TestCacheRereadsForNewKeyword: a keyword no entry was checked against // makes the files that reach it be read again, once. func TestCacheRereadsForNewKeyword(t *testing.T) { home, _, runs := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME", "b.pdf": "nothing here"}) main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules}) cachedMatch(t, home, main) writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules + `(rule "nothing" (when (content "nothing")) (move "Nothing"))` + "\n"}) r := cachedMatch(t, home, main) if runs() != 4 { t.Errorf("%d extractions in all, want 4: both read again for the new keyword", runs()) } if matchedNames(r) != "a.pdf b.pdf" { t.Errorf("matched %q, want both", matchedNames(r)) } cachedMatch(t, home, main) if runs() != 4 { t.Errorf("third run extracted again: %d runs in all", runs()) } } // TestCacheOffWithoutCacheDir: an engine with no CacheDir reads every time // and writes no cache. func TestCacheOffWithoutCacheDir(t *testing.T) { home, _, runs := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME"}) main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules}) for i := 0; i < 2; i++ { e, errs := Load(main) if len(errs) > 0 { t.Fatal(errs) } if _, err := e.Match(context.Background(), e.Dirs[0]); err != nil { t.Fatal(err) } } if runs() != 2 { t.Errorf("%d extractions, want one per run", runs()) } if _, err := os.Stat(filepath.Join(home, ".cache")); !os.IsNotExist(err) { t.Errorf("a cache was written with no CacheDir: %v", err) } } // TestCacheDoesNotStoreFailures: a file the tool fails on is tried again // on every run, and warned about every time. func TestCacheDoesNotStoreFailures(t *testing.T) { home, _, runs := countingPDFTree(t, map[string]string{"fail.pdf": "Invoice ACME"}) main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules}) for i := 1; i <= 2; i++ { r := cachedMatch(t, home, main) if runs() != i { t.Errorf("run %d: %d extractions in all, want %d", i, runs(), i) } if len(r.Unmatched) != 1 || len(r.Unmatched[0].Warnings) == 0 { t.Errorf("run %d: want fail.pdf unmatched with a warning: %+v", i, r.Unmatched) } } } // TestCacheRespectsMaxRead: a file over max-read is not read, even when an // earlier run cached its answers. func TestCacheRespectsMaxRead(t *testing.T) { home, _, _ := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME"}) main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules}) cachedMatch(t, home, main) writeConfig(t, home, `(include "dl")`, map[string]string{"dl": "(max-read 1)\n" + acmeRules}) r := cachedMatch(t, home, main) if len(r.Matched) != 0 || len(r.Unmatched) != 1 || !strings.Contains(strings.Join(r.Unmatched[0].Warnings, " "), "larger than max-read") { t.Errorf("want a.pdf unmatched as larger than max-read: matched %+v unmatched %+v", r.Matched, r.Unmatched) } } // TestCacheHoldsNoText: the cache file holds the keywords and answers, not // the extracted text or the file's name. func TestCacheHoldsNoText(t *testing.T) { home, _, _ := countingPDFTree(t, map[string]string{"secret-name.pdf": "Invoice ACME confidential"}) main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules}) cachedMatch(t, home, main) b, err := os.ReadFile(filepath.Join(home, ".cache", "krino", "dl.cache")) if err != nil { t.Fatal(err) } for _, leak := range []string{"Invoice", "invoice", "confidential", "secret-name"} { if strings.Contains(string(b), leak) { t.Errorf("cache holds %q:\n%s", leak, b) } } } // TestExplainUsesCacheWithoutWriting: explain answers from the cache a run // wrote, and never writes one itself. func TestExplainUsesCacheWithoutWriting(t *testing.T) { home, dl, runs := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME"}) main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules}) explain := func() { t.Helper() e, errs := Load(main) if len(errs) > 0 { t.Fatal(errs) } e.CacheDir = filepath.Join(home, ".cache", "krino") x, err := e.Explain(context.Background(), filepath.Join(dl, "a.pdf")) if err != nil { t.Fatal(err) } if len(x.Rules) != 1 || !x.Rules[0].Match { t.Errorf("explain: %+v", x.Rules) } } explain() if _, err := os.Stat(filepath.Join(home, ".cache")); !os.IsNotExist(err) { t.Errorf("explain wrote a cache: %v", err) } cachedMatch(t, home, main) before := runs() explain() if runs() != before { t.Errorf("explain extracted although the run cached a.pdf") } } // TestCacheRereadsARenamedExtension: the extension picks the extractor, so // a file renamed from .html to .txt - same inode, size and mtime - is read // again, not answered from what the markup reader found (review M6). func TestCacheRereadsARenamedExtension(t *testing.T) { home, dl := excludeTree(t, map[string]string{"page.html": "
ACME
Ltd"}) main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": "(path \"~/dl\")\n(rule \"acme\" (when (content \"acme ltd\")) (move \"Acme\"))\n"}) if r := cachedMatch(t, home, main); matchedNames(r) != "page.html" { t.Fatalf("as html: matched %q", matchedNames(r)) } if err := os.Rename(filepath.Join(dl, "page.html"), filepath.Join(dl, "page.txt")); err != nil { t.Fatal(err) } if r := cachedMatch(t, home, main); matchedNames(r) != "" { t.Errorf("as txt, answered from the html read: matched %q", matchedNames(r)) } } // TestCacheFingerprintCoversMaxReadAndTables: answers depend on max-read // (it caps what is read), on the Go release and on the Unicode tables, so // all are in the fingerprint (review cache F2, F4). func TestCacheFingerprintCoversMaxReadAndTables(t *testing.T) { e := &Engine{Extract: extract.New()} small := e.cacheFingerprint(&Dir{Settings: config.Resolved{MaxRead: 1 << 10}}) large := e.cacheFingerprint(&Dir{Settings: config.Resolved{MaxRead: 1 << 20}}) if small == large { t.Error("max-read does not change the fingerprint") } for _, want := range []string{runtime.Version(), unicode.Version} { if !strings.Contains(small, want) { t.Errorf("fingerprint %q lacks %q", small, want) } } } // TestCacheRemovedWhenNoContentTestsRemain: once a directory has no content // test, its cache - holding keywords it no longer uses - is removed (review // cache F6). func TestCacheRemovedWhenNoContentTestsRemain(t *testing.T) { home, _, _ := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME"}) main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules}) cachedMatch(t, home, main) file := filepath.Join(home, ".cache", "krino", "dl.cache") if _, err := os.Stat(file); err != nil { t.Fatal(err) } writeConfig(t, home, `(include "dl")`, map[string]string{"dl": "(path \"~/dl\")\n(rule \"all\" (move \"Out\"))\n"}) cachedMatch(t, home, main) if _, err := os.Stat(file); !os.IsNotExist(err) { t.Errorf("the cache of a directory without content tests is still there: %v", err) } }