From aa24cfb344b1b3eaef7217d996359023cd72ba28 Mon Sep 17 00:00:00 2001 From: Lukasz Kasprzak Date: Mon, 14 Sep 2026 20:10:01 +0200 Subject: plan 8: fuzz decoders; fold ẞ and invalid UTF-8 correctly, refuse non-UTF-8 paths in krino new MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 9 ++++ Makefile | 11 ++++- docs/design.md | 5 +- internal/config/fuzz_test.go | 51 ++++++++++++++++++++ internal/config/skel.go | 6 +++ internal/config/skel_test.go | 4 +- internal/engine/match.go | 10 +++- internal/extract/fuzz_test.go | 56 ++++++++++++++++++++++ internal/ignore/fuzz_test.go | 44 +++++++++++++++++ internal/kwcache/fuzz_test.go | 50 +++++++++++++++++++ internal/norm/fuzz_test.go | 30 ++++++++++++ internal/norm/norm.go | 14 ++++-- internal/norm/norm_test.go | 2 + .../fuzz/FuzzTextIdempotent/d26a0358dee954b3 | 2 + internal/plan/fuzz_test.go | 33 +++++++++++++ internal/sexp/fuzz_test.go | 28 ++++++++++- man/krino.conf.5 | 2 +- 17 files changed, 347 insertions(+), 10 deletions(-) create mode 100644 internal/config/fuzz_test.go create mode 100644 internal/extract/fuzz_test.go create mode 100644 internal/ignore/fuzz_test.go create mode 100644 internal/kwcache/fuzz_test.go create mode 100644 internal/norm/fuzz_test.go create mode 100644 internal/norm/testdata/fuzz/FuzzTextIdempotent/d26a0358dee954b3 create mode 100644 internal/plan/fuzz_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 74ff1e3..b856f3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## Unreleased +- Folding maps the capital sharp s (ẞ) to "SS", as the other capitals + already were: "STRAẞE" in a document now matches the keyword "straße". +- Folding replaces invalid UTF-8 with U+FFFD first, so a stray invalid byte + no longer stops the letter after it from losing its accent. +- `krino new` refuses a directory whose path is not valid UTF-8 with that + reason, instead of reporting a broken template. +- The keyword cache is also discarded when normalisation changes, so answers + cached before these fixes are recomputed once. + - Builds need Go 1.25 or newer and use the Go 1.26.8 toolchain, which an older `go` downloads itself. `golang.org/x/text` is updated to v0.41.0. This fixes an infinite loop a crafted file's text could cause diff --git a/Makefile b/Makefile index 3cfc8f1..4713ecb 100644 --- a/Makefile +++ b/Makefile @@ -87,7 +87,16 @@ FUZZ_TARGETS = \ internal/journal:FuzzParseLine \ internal/journal:FuzzEntryRoundTrip \ internal/trash:FuzzPercentRoundTrip \ - internal/trash:FuzzParsePath + internal/trash:FuzzParsePath \ + internal/sexp:FuzzQuoteRoundTrip \ + internal/config:FuzzParseSize \ + internal/config:FuzzParseDuration \ + internal/norm:FuzzTextIdempotent \ + internal/plan:FuzzExpand \ + internal/ignore:FuzzMatch \ + internal/kwcache:FuzzLoad \ + internal/extract:FuzzStripMarkup \ + internal/extract:FuzzZipText fuzz: ## run every fuzz target for FUZZTIME each (default 20s); a crasher is saved under testdata/fuzz @for t in $(FUZZ_TARGETS); do \ diff --git a/docs/design.md b/docs/design.md index 411eeba..11e859e 100644 --- a/docs/design.md +++ b/docs/design.md @@ -233,7 +233,8 @@ quantifiers, and inline flags `(?i)` `(?-i)`; no backreferences or lookaround. data and the pattern. `type` is always case-insensitive. A regex can override `case` locally with `(?i)` or `(?-i)`. Folding decomposes letters (Unicode NFD), drops combining marks, and maps the letters that do not -decompose (ł ø đ ħ ß æ œ and their capitals) to ASCII. +decompose (ł ø đ ħ ß æ œ and their capitals, ẞ included) to ASCII. Invalid +UTF-8 is replaced by U+FFFD before folding. ### 5.4 Evaluation @@ -348,7 +349,7 @@ appear in the config. Failures (unreadable, tool missing, timeout) are never cached. - The cache is discarded whole when the extractor fingerprint changes: a tool installed, removed or replaced, or krino's extraction code changing - (`extract.Version`). + (`extract.Version`), or its text normalisation changing (`norm.Version`). - Planning a run (`-n` included) reads the cache and writes it back holding only files still in the directory, under the directory's lock, via a temporary file renamed into place; the directory is 0700, the file 0600. diff --git a/internal/config/fuzz_test.go b/internal/config/fuzz_test.go new file mode 100644 index 0000000..32926d6 --- /dev/null +++ b/internal/config/fuzz_test.go @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package config + +import ( + "fmt" + "strconv" + "testing" + "time" +) + +// FuzzParseSize: a size either fails to parse or is a non-negative number +// of bytes that reads back the same written as plain digits - no overflow +// ever wraps a huge size negative. +func FuzzParseSize(f *testing.F) { + for _, s := range []string{"50M", "0", "1T", "9223372036854775807", "8388608T", "8388607T", "-1", "1.5G", "", "G", "007K"} { + f.Add(s) + } + f.Fuzz(func(t *testing.T, s string) { + v, err := ParseSize(s) + if err != nil { + return + } + if v < 0 { + t.Fatalf("ParseSize(%q) = %d, negative", s, v) + } + if back, err := ParseSize(strconv.FormatInt(v, 10)); err != nil || back != v { + t.Fatalf("ParseSize(%q) = %d, which reads back as %d, %v", s, v, back, err) + } + }) +} + +// FuzzParseDuration: a duration either fails to parse or is a non-negative +// whole number of seconds that reads back the same written in seconds. +func FuzzParseDuration(f *testing.F) { + for _, s := range []string{"30d", "0s", "2m", "1w", "15250284452w", "9223372036s", "d", "", "-1d", "1.5h"} { + f.Add(s) + } + f.Fuzz(func(t *testing.T, s string) { + d, err := ParseDuration(s) + if err != nil { + return + } + if d < 0 || d%time.Second != 0 { + t.Fatalf("ParseDuration(%q) = %v", s, d) + } + if back, err := ParseDuration(fmt.Sprintf("%ds", d/time.Second)); err != nil || back != d { + t.Fatalf("ParseDuration(%q) = %v, which reads back as %v, %v", s, d, back, err) + } + }) +} diff --git a/internal/config/skel.go b/internal/config/skel.go index 65d9f95..ca914e3 100644 --- a/internal/config/skel.go +++ b/internal/config/skel.go @@ -10,6 +10,7 @@ import ( "io/fs" "os" "path/filepath" + "unicode/utf8" "krino/internal/sexp" "krino/internal/xdg" @@ -66,6 +67,11 @@ func NewDir(mainFile, name, path string) (string, error) { if err != nil { return "", err } + // The path is written into the new file, and config files are UTF-8 + // text (the reader refuses anything else). + if !utf8.ValidString(abs) { + return "", fmt.Errorf("%q is not valid UTF-8; krino's config is UTF-8 text, so rename the directory first", abs) + } if fi, err := os.Stat(abs); err != nil || !fi.IsDir() { return "", fmt.Errorf("%s is not a directory", abs) } diff --git a/internal/config/skel_test.go b/internal/config/skel_test.go index 476af65..3815336 100644 --- a/internal/config/skel_test.go +++ b/internal/config/skel_test.go @@ -5,6 +5,7 @@ package config import ( "bytes" "errors" + "fmt" "io/fs" "os" "path/filepath" @@ -242,7 +243,7 @@ func TestNewDirErrors(t *testing.T) { home := t.TempDir() t.Setenv("HOME", home) writeFiles(t, home, map[string]string{ - "krino.conf": `(include "a")`, "dirs/a.conf": `(path "~")`, "x/.keep": "", "f": "", + "krino.conf": `(include "a")`, "dirs/a.conf": `(path "~")`, "x/.keep": "", "f": "", "lat\xe9n/.keep": "", }) main := filepath.Join(home, "krino.conf") tests := []struct{ name, path, want string }{ @@ -251,6 +252,7 @@ func TestNewDirErrors(t *testing.T) { {"b", "~/f", filepath.Join(home, "f") + " is not a directory"}, {"b", "~/missing", filepath.Join(home, "missing") + " is not a directory"}, {"a", "~/x", `"a" is already included`}, + {"b", "~/lat\xe9n", fmt.Sprintf("%q is not valid UTF-8; krino's config is UTF-8 text, so rename the directory first", filepath.Join(home, "lat\xe9n"))}, } for _, tt := range tests { if _, err := NewDir(main, tt.name, tt.path); err == nil || err.Error() != tt.want { diff --git a/internal/engine/match.go b/internal/engine/match.go index 1a8077a..b922829 100644 --- a/internal/engine/match.go +++ b/internal/engine/match.go @@ -17,6 +17,7 @@ import ( "krino/internal/cond" "krino/internal/config" "krino/internal/kwcache" + "krino/internal/norm" "krino/internal/plan" "krino/internal/scan" "krino/internal/xdg" @@ -296,6 +297,13 @@ func (e *Engine) Explain(ctx context.Context, path string) (*Explanation, error) return &Explanation{Dir: d, File: sf, Skip: skip, Excludes: excludes, Excluded: excluded, Rules: rules}, nil } +// cacheFingerprint identifies what a cached keyword answer depends on +// besides the file: the extractor (its version and tools) and the +// normalisation version. A cache written under any other is discarded. +func (e *Engine) cacheFingerprint() string { + return fmt.Sprintf("%s norm%d", e.Extract.Fingerprint(), norm.Version) +} + // cacheFile is d's keyword cache file. func (e *Engine) cacheFile(d *Dir) string { return filepath.Join(e.CacheDir, d.Name+".cache") @@ -308,7 +316,7 @@ func (e *Engine) openCache(run *matchRun) []string { if e.CacheDir == "" || len(run.d.ContentKeywords) == 0 { return nil } - c, err := kwcache.Load(e.cacheFile(run.d), e.Extract.Fingerprint()) + c, err := kwcache.Load(e.cacheFile(run.d), e.cacheFingerprint()) run.cache = c if err != nil { return []string{"cache: " + err.Error() + " (starting a new one)"} diff --git a/internal/extract/fuzz_test.go b/internal/extract/fuzz_test.go new file mode 100644 index 0000000..5f707d2 --- /dev/null +++ b/internal/extract/fuzz_test.go @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package extract + +import ( + "archive/zip" + "bytes" + "context" + "os" + "path/filepath" + "testing" +) + +// FuzzStripMarkup: decoding and stripping any bytes as HTML never panics. +func FuzzStripMarkup(f *testing.F) { + for _, s := range []string{"

a & b

", "y", "