plan 9: keyword cache keys on extension, max-read and Unicode tables, trims removed keywords

7 files changed, 178 insertions, 24 deletions

diff --git a/internal/engine/cache_test.go b/internal/engine/cache_test.go
index eee2860..8c52d48 100644
--- a/internal/engine/cache_test.go
+++ b/internal/engine/cache_test.go
@@ -6,9 +6,14 @@ import (
 	"context"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"testing"
 	"time"
+	"unicode"
+
+	"krino/internal/config"
+	"krino/internal/extract"
 )
 
 // countingPDFTree makes ~/dl holding the given .pdf files, and a fake
@@ -222,3 +227,55 @@ func TestExplainUsesCacheWithoutWriting(t *testing.T) {
 		t.Errorf("explain extracted although the run cached a.pdf")
 	}
 }
+
+// TestCacheRereadsARenamedExtension: the extension picks the extractor, so
+// a file renamed from .html to .txt - same inode, size and mtime - is read
+// again, not answered from what the markup reader found (review M6).
+func TestCacheRereadsARenamedExtension(t *testing.T) {
+	home, dl := excludeTree(t, map[string]string{"page.html": "<p>ACME</p>&nbsp;Ltd"})
+	main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": "(path \"~/dl\")\n(rule \"acme\" (when (content \"acme ltd\")) (move \"Acme\"))\n"})
+	if r := cachedMatch(t, home, main); matchedNames(r) != "page.html" {
+		t.Fatalf("as html: matched %q", matchedNames(r))
+	}
+	if err := os.Rename(filepath.Join(dl, "page.html"), filepath.Join(dl, "page.txt")); err != nil {
+		t.Fatal(err)
+	}
+	if r := cachedMatch(t, home, main); matchedNames(r) != "" {
+		t.Errorf("as txt, answered from the html read: matched %q", matchedNames(r))
+	}
+}
+
+// TestCacheFingerprintCoversMaxReadAndTables: answers depend on max-read
+// (it caps what is read), on the Go release and on the Unicode tables, so
+// all are in the fingerprint (review cache F2, F4).
+func TestCacheFingerprintCoversMaxReadAndTables(t *testing.T) {
+	e := &Engine{Extract: extract.New()}
+	small := e.cacheFingerprint(&Dir{Settings: config.Resolved{MaxRead: 1 << 10}})
+	large := e.cacheFingerprint(&Dir{Settings: config.Resolved{MaxRead: 1 << 20}})
+	if small == large {
+		t.Error("max-read does not change the fingerprint")
+	}
+	for _, want := range []string{runtime.Version(), unicode.Version} {
+		if !strings.Contains(small, want) {
+			t.Errorf("fingerprint %q lacks %q", small, want)
+		}
+	}
+}
+
+// TestCacheRemovedWhenNoContentTestsRemain: once a directory has no content
+// test, its cache - holding keywords it no longer uses - is removed (review
+// cache F6).
+func TestCacheRemovedWhenNoContentTestsRemain(t *testing.T) {
+	home, _, _ := countingPDFTree(t, map[string]string{"a.pdf": "Invoice ACME"})
+	main := writeConfig(t, home, `(include "dl")`, map[string]string{"dl": acmeRules})
+	cachedMatch(t, home, main)
+	file := filepath.Join(home, ".cache", "krino", "dl.cache")
+	if _, err := os.Stat(file); err != nil {
+		t.Fatal(err)
+	}
+	writeConfig(t, home, `(include "dl")`, map[string]string{"dl": "(path \"~/dl\")\n(rule \"all\" (move \"Out\"))\n"})
+	cachedMatch(t, home, main)
+	if _, err := os.Stat(file); !os.IsNotExist(err) {
+		t.Errorf("the cache of a directory without content tests is still there: %v", err)
+	}
+}
diff --git a/internal/engine/facts.go b/internal/engine/facts.go
index 9035245..487604b 100644
--- a/internal/engine/facts.go
+++ b/internal/engine/facts.go
@@ -212,7 +212,7 @@ func (f *facts) cacheID() (kwcache.ID, bool) {
 
 // fileCacheID is file's kwcache.ID.
 func fileCacheID(file scan.File) kwcache.ID {
-	return kwcache.ID{Dev: file.Dev, Ino: file.Ino, Size: file.Size, MTime: file.ModTime.UnixNano()}
+	return kwcache.ID{Dev: file.Dev, Ino: file.Ino, Size: file.Size, MTime: file.ModTime.UnixNano(), Ext: strings.ToLower(filepath.Ext(file.Name))}
 }
 
 // Duplicate resolves dirs against the directory's root, builds (or reuses)
diff --git a/internal/engine/match.go b/internal/engine/match.go
index b922829..08d75af 100644
--- a/internal/engine/match.go
+++ b/internal/engine/match.go
@@ -114,7 +114,11 @@ func (e *Engine) Match(ctx context.Context, d *Dir) (*Result, error) {
 				ids = append(ids, fileCacheID(f))
 			}
 		}
-		if err := run.cache.Save(e.cacheFile(d), ids); err != nil {
+		keys := make([]string, len(d.ContentKeywords))
+		for i, k := range d.ContentKeywords {
+			keys[i] = k.Key()
+		}
+		if err := run.cache.Save(e.cacheFile(d), ids, keys); err != nil {
 			warnings = append(warnings, "cache: "+err.Error())
 		}
 	}
@@ -297,11 +301,12 @@ func (e *Engine) Explain(ctx context.Context, path string) (*Explanation, error)
 	return &Explanation{Dir: d, File: sf, Skip: skip, Excludes: excludes, Excluded: excluded, Rules: rules}, nil
 }
 
-// cacheFingerprint identifies what a cached keyword answer depends on
-// besides the file: the extractor (its version and tools) and the
-// normalisation version. A cache written under any other is discarded.
-func (e *Engine) cacheFingerprint() string {
-	return fmt.Sprintf("%s norm%d", e.Extract.Fingerprint(), norm.Version)
+// cacheFingerprint identifies what a cached keyword answer of d depends on
+// besides the file: the extractor (its version and tools), normalisation
+// and its Unicode tables, the Go release, and d's max-read, which caps what
+// is read. A cache written under any other is discarded.
+func (e *Engine) cacheFingerprint(d *Dir) string {
+	return fmt.Sprintf("%s %s %s max-read=%d", e.Extract.Fingerprint(), norm.Fingerprint(), runtime.Version(), d.Settings.MaxRead)
 }
 
 // cacheFile is d's keyword cache file.
@@ -313,10 +318,18 @@ func (e *Engine) cacheFile(d *Dir) string {
 // engine has a CacheDir and the directory has content tests at all. A cache
 // that cannot be read is replaced by an empty one, reported as a warning.
 func (e *Engine) openCache(run *matchRun) []string {
-	if e.CacheDir == "" || len(run.d.ContentKeywords) == 0 {
+	if e.CacheDir == "" {
+		return nil
+	}
+	if len(run.d.ContentKeywords) == 0 {
+		// No content test left: a cache from an earlier configuration only
+		// holds keywords this directory no longer uses (review cache F6).
+		if err := os.Remove(e.cacheFile(run.d)); err != nil && !os.IsNotExist(err) {
+			return []string{"cache: " + err.Error()}
+		}
 		return nil
 	}
-	c, err := kwcache.Load(e.cacheFile(run.d), e.cacheFingerprint())
+	c, err := kwcache.Load(e.cacheFile(run.d), e.cacheFingerprint(run.d))
 	run.cache = c
 	if err != nil {
 		return []string{"cache: " + err.Error() + " (starting a new one)"}
diff --git a/internal/kwcache/fuzz_test.go b/internal/kwcache/fuzz_test.go
index 0b0eced..8ac6c95 100644
--- a/internal/kwcache/fuzz_test.go
+++ b/internal/kwcache/fuzz_test.go
@@ -15,7 +15,7 @@ func FuzzLoad(f *testing.F) {
 	seed := New("fp")
 	seed.Store(a, map[string]bool{"k:acme": true, "k:x": false})
 	path := filepath.Join(f.TempDir(), "seed.cache")
-	if err := seed.Save(path, []ID{a}); err != nil {
+	if err := seed.Save(path, []ID{a}, allKeys); err != nil {
 		f.Fatal(err)
 	}
 	good, err := os.ReadFile(path)
@@ -43,7 +43,7 @@ func FuzzLoad(f *testing.F) {
 		if err != nil && ok {
 			t.Fatalf("a cache that failed to load (%v) answered", err)
 		}
-		if err := c.Save(filepath.Join(t.TempDir(), "out.cache"), []ID{a, b}); err != nil {
+		if err := c.Save(filepath.Join(t.TempDir(), "out.cache"), []ID{a, b}, allKeys); err != nil {
 			t.Fatal(err)
 		}
 	})
diff --git a/internal/kwcache/kwcache.go b/internal/kwcache/kwcache.go
index b61cfb1..c9af7b8 100644
--- a/internal/kwcache/kwcache.go
+++ b/internal/kwcache/kwcache.go
@@ -19,15 +19,18 @@ import (
 )
 
 // version is the on-disk format; a file of any other version loads empty.
-const version = 1
+// 2: the extension is part of a file's ID.
+const version = 2
 
 // ID is how a file is recognised: the same inode with the same size and
-// modification time is taken to hold the same content. A move or rename
-// within one filesystem keeps it.
+// modification time is taken to hold the same content, and the same
+// extension is read by the same extractor. A move within one filesystem
+// keeps it; a rename that changes the extension does not (review M6).
 type ID struct {
 	Dev, Ino uint64
 	Size     int64
-	MTime    int64 // Unix nanoseconds
+	MTime    int64  // Unix nanoseconds
+	Ext      string // lower case, with its dot; "" for none
 }
 
 // entry is what is known about one file: every keyword it was checked
@@ -58,6 +61,7 @@ type diskFile struct {
 	Ino   uint64 `json:"ino"`
 	Size  int64  `json:"size"`
 	MTime int64  `json:"mtime"`
+	Ext   string `json:"ext"`
 	Set   int    `json:"keywords"` // index into diskCache.Keywords
 	Hits  []int  `json:"hits"`     // indices into that keyword list
 }
@@ -106,7 +110,7 @@ func Load(path, fingerprint string) (*Cache, error) {
 			}
 			hits[set[h]] = true
 		}
-		c.old[ID{Dev: f.Dev, Ino: f.Ino, Size: f.Size, MTime: f.MTime}] = entry{checked: set, hits: hits}
+		c.old[ID{Dev: f.Dev, Ino: f.Ino, Size: f.Size, MTime: f.MTime, Ext: f.Ext}] = entry{checked: set, hits: hits}
 	}
 	c.existed = true
 	return c, nil
@@ -154,14 +158,22 @@ func (c *Cache) Store(id ID, answers map[string]bool) {
 
 // Save writes the entries of the files in present, from this run or loaded,
 // to path, and drops every other: the cache only ever describes files
-// still in the directory. The directory is created 0700 and the file
-// written 0600 under a temporary name, then renamed into place. A cache
-// with nothing to write and no file on disk writes nothing.
-func (c *Cache) Save(path string, present []ID) error {
+// still in the directory. Each entry keeps only the keywords in keywords -
+// the directory's current ones - so a keyword removed from the
+// configuration leaves the cache too, and an entry left with none is
+// dropped. The directory is made private (0700, tightened if it already
+// existed) and the file written 0600 under a temporary name, then renamed
+// into place. A cache with nothing to write and no file on disk writes
+// nothing.
+func (c *Cache) Save(path string, present []ID, keywords []string) error {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
 	d := diskCache{Version: version, Fingerprint: c.fingerprint, Keywords: [][]string{}, Files: []diskFile{}}
+	current := make(map[string]bool, len(keywords))
+	for _, k := range keywords {
+		current[k] = true
+	}
 	sets := map[string]int{}
 	seen := map[ID]bool{}
 	for _, id := range present {
@@ -176,6 +188,10 @@ func (c *Cache) Save(path string, present []ID) error {
 		if !ok {
 			continue
 		}
+		e = trimmed(e, current)
+		if len(e.checked) == 0 {
+			continue
+		}
 		key := strings.Join(e.checked, "\x00")
 		set, ok := sets[key]
 		if !ok {
@@ -183,7 +199,7 @@ func (c *Cache) Save(path string, present []ID) error {
 			sets[key] = set
 			d.Keywords = append(d.Keywords, e.checked)
 		}
-		f := diskFile{Dev: id.Dev, Ino: id.Ino, Size: id.Size, MTime: id.MTime, Set: set, Hits: []int{}}
+		f := diskFile{Dev: id.Dev, Ino: id.Ino, Size: id.Size, MTime: id.MTime, Ext: id.Ext, Set: set, Hits: []int{}}
 		for i, k := range e.checked {
 			if e.hits[k] {
 				f.Hits = append(f.Hits, i)
@@ -209,6 +225,9 @@ func (c *Cache) Save(path string, present []ID) error {
 	if err := os.MkdirAll(dir, 0o700); err != nil {
 		return err
 	}
+	if err := os.Chmod(dir, 0o700); err != nil {
+		return err
+	}
 	tmp, err := os.CreateTemp(dir, ".kwcache-*")
 	if err != nil {
 		return err
@@ -229,3 +248,17 @@ func (c *Cache) Save(path string, present []ID) error {
 	c.existed = true
 	return nil
 }
+
+// trimmed returns e keeping only the keywords in current.
+func trimmed(e entry, current map[string]bool) entry {
+	out := entry{hits: map[string]bool{}}
+	for _, k := range e.checked {
+		if current[k] {
+			out.checked = append(out.checked, k)
+			if e.hits[k] {
+				out.hits[k] = true
+			}
+		}
+	}
+	return out
+}
diff --git a/internal/kwcache/kwcache_test.go b/internal/kwcache/kwcache_test.go
index 8395b2d..077ecd3 100644
--- a/internal/kwcache/kwcache_test.go
+++ b/internal/kwcache/kwcache_test.go
@@ -6,9 +6,13 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
+	"strings"
 	"testing"
 )
 
+// allKeys is every keyword the tests store, for Save calls that keep them all.
+var allKeys = []string{"k:acme", "k:faktura", "k:nip", "k:new", "k:x"}
+
 var (
 	a = ID{Dev: 1, Ino: 10, Size: 100, MTime: 1000}
 	b = ID{Dev: 1, Ino: 11, Size: 200, MTime: 2000}
@@ -17,7 +21,7 @@ var (
 func saved(t *testing.T, c *Cache, present ...ID) string {
 	t.Helper()
 	path := filepath.Join(t.TempDir(), "sub", "dl.cache")
-	if err := c.Save(path, present); err != nil {
+	if err := c.Save(path, present, allKeys); err != nil {
 		t.Fatal(err)
 	}
 	return path
@@ -94,7 +98,7 @@ func TestSaveKeepsOnlyPresentFiles(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := next.Save(path, []ID{b}); err != nil {
+	if err := next.Save(path, []ID{b}, allKeys); err != nil {
 		t.Fatal(err)
 	}
 	last, err := Load(path, "fp")
@@ -171,10 +175,49 @@ func TestLoadMissingAndCorrupt(t *testing.T) {
 // has no file yet, creates none.
 func TestSaveWithNothingWritesNothing(t *testing.T) {
 	path := filepath.Join(t.TempDir(), "sub", "dl.cache")
-	if err := New("fp").Save(path, []ID{a}); err != nil {
+	if err := New("fp").Save(path, []ID{a}, allKeys); err != nil {
 		t.Fatal(err)
 	}
 	if _, err := os.Stat(filepath.Dir(path)); !os.IsNotExist(err) {
 		t.Errorf("Save created %s for an empty cache", filepath.Dir(path))
 	}
 }
+
+// TestSaveTrimsToCurrentKeywords: a keyword no longer in the configuration
+// is not kept in the cache file (review cache F6).
+func TestSaveTrimsToCurrentKeywords(t *testing.T) {
+	c := New("fp")
+	c.Store(a, map[string]bool{"k:keep": true, "k:gone client": true})
+	path := filepath.Join(t.TempDir(), "dl.cache")
+	if err := c.Save(path, []ID{a}, []string{"k:keep"}); err != nil {
+		t.Fatal(err)
+	}
+	b, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Contains(string(b), "gone client") {
+		t.Errorf("a removed keyword is still stored:\n%s", b)
+	}
+	got, _ := Load(path, "fp")
+	if hits, ok := got.Lookup(a, []string{"k:keep"}); !ok || !hits[0] {
+		t.Errorf("the kept keyword's answer was lost: %v %v", hits, ok)
+	}
+}
+
+// TestSaveTightensAnExistingDirectory: the cache directory is private even
+// when it already existed with wider permissions (review cache F7).
+func TestSaveTightensAnExistingDirectory(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "krino")
+	if err := os.Mkdir(dir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	c := New("fp")
+	c.Store(a, map[string]bool{"k:acme": true})
+	if err := c.Save(filepath.Join(dir, "dl.cache"), []ID{a}, []string{"k:acme"}); err != nil {
+		t.Fatal(err)
+	}
+	if fi, err := os.Stat(dir); err != nil || fi.Mode().Perm() != 0o700 {
+		t.Errorf("directory mode %v, %v; want 0700", fi.Mode().Perm(), err)
+	}
+}
diff --git a/internal/norm/norm.go b/internal/norm/norm.go
index 77bb604..701f765 100644
--- a/internal/norm/norm.go
+++ b/internal/norm/norm.go
@@ -5,6 +5,7 @@
 package norm
 
 import (
+	"fmt"
 	"strings"
 	"unicode"
 
@@ -16,6 +17,13 @@ import (
 // another version are discarded (spec §6.1).
 const Version = 1
 
+// Fingerprint names everything Text and Name's output depends on: Version,
+// and the Unicode tables of the standard library and of x/text's
+// normalisation, which a Go or x/text upgrade can change (review cache F4).
+func Fingerprint() string {
+	return fmt.Sprintf("norm%d unicode%s nfd%s", Version, unicode.Version, unorm.Version)
+}
+
 // special holds the letters that do not decompose under Unicode NFD, so
 // Fold maps them explicitly.
 var special = map[rune]string{