From 4c6fadfab5434317357ad0272ace7927d2945942 Mon Sep 17 00:00:00 2001 From: Lukasz Kasprzak Date: Thu, 17 Sep 2026 14:07:37 +0200 Subject: max-read bounds what a file becomes, not only what is read max-read gates on file size before reading, then the text was read whole and decoded: a file that is not valid UTF-8 decodes one byte per code point and doubles, and normalising and folding copy that again per set of options, with GOMAXPROCS files in flight. Twelve 40 MB files reached 3.3 GB - enough to put a laptop into the OOM killer, with no attacker involved, just a few big .log or .csv files. Two bounds. The decoded text is cut to max-read at a rune boundary, so the ceiling means what a reader takes it to mean. And extraction of files at or above 4 MiB is rationed to two at a time, since holding several large texts at once is what multiplies the ceiling; smaller files, which is nearly all of them, are untouched. twelve 40 MB files: peak RSS 3294 MB -> 728 MB, wall 27s -> 46s two thousand small files: 0.05s both ways The wall-clock cost falls entirely on large files needing extraction, and buys a program that finishes instead of being killed. --- internal/engine/facts.go | 26 ++++++++++++++++++++++++++ internal/extract/extract.go | 6 +++--- internal/extract/plain.go | 35 ++++++++++++++++++++++++++++++----- 3 files changed, 59 insertions(+), 8 deletions(-) (limited to 'internal') diff --git a/internal/engine/facts.go b/internal/engine/facts.go index 94ac3f4..6540fdd 100644 --- a/internal/engine/facts.go +++ b/internal/engine/facts.go @@ -37,12 +37,28 @@ type matchRun struct { files []scan.File cache *kwcache.Cache // nil: no keyword cache + // bigText limits how many large files have their text in memory at + // once. Extracting is otherwise GOMAXPROCS-wide, and a large file's + // text costs several times its own size: it is read whole, a file that + // is not valid UTF-8 doubles as it decodes, and each distinct set of + // case/fold options makes another normalised copy. Sixteen 49 MB text + // files reached 2.7 GB. Small files are unaffected, which is nearly + // every file. + bigText chan struct{} + mu sync.Mutex dupOnce map[string]*sync.Once dupIdx map[string]*dup.Index warn []string } +// bigTextSize is the file size above which extraction is rationed, and +// bigTextAtOnce is how many such files may be in flight together. +const ( + bigTextSize = 4 << 20 // 4 MiB + bigTextAtOnce = 2 +) + // newMatchRun builds a matchRun over files, the set a (duplicate ...) test // with no directories of its own checks against. func newMatchRun(e *Engine, d *Dir, ctx context.Context, now time.Time, files []scan.File) *matchRun { @@ -54,6 +70,7 @@ func newMatchRun(e *Engine, d *Dir, ctx context.Context, now time.Time, files [] files: files, dupOnce: make(map[string]*sync.Once), dupIdx: make(map[string]*dup.Index), + bigText: make(chan struct{}, bigTextAtOnce), } } @@ -225,6 +242,15 @@ func (f *facts) ContentContains(opt cond.Options, keywords []string) (int, error // plus the asking test's (opt, keywords), from it. func (f *facts) extract(opt cond.Options, keywords []string) { f.contentDone = true + if f.file.Size >= bigTextSize && f.run.bigText != nil { + // Wait for a slot rather than hold several large files' text at + // once. Cancellation is still noticed: Text takes the same ctx. + select { + case f.run.bigText <- struct{}{}: + defer func() { <-f.run.bigText }() + case <-f.run.ctx.Done(): + } + } text, err := f.run.e.Extract.Text(f.run.ctx, f.file.Path, f.file.Size, f.run.d.Settings.MaxRead) switch { case errors.Is(err, extract.ErrPartial): diff --git a/internal/extract/extract.go b/internal/extract/extract.go index f8f98df..8e6c6c5 100644 --- a/internal/extract/extract.go +++ b/internal/extract/extract.go @@ -178,14 +178,14 @@ func (e *Extractor) Text(ctx context.Context, path string, size, maxRead int64) case toolExt[ext] != "": return e.legacyText(ctx, path, ext, maxRead) case markupExt[ext]: - raw, err := readDecoded(path) + raw, err := readDecoded(path, maxRead) if err != nil { return "", err } return stripMarkup(raw), nil case plainExt[ext]: - return readDecoded(path) + return readDecoded(path, maxRead) default: - return sniffText(path) + return sniffText(path, maxRead) } } diff --git a/internal/extract/plain.go b/internal/extract/plain.go index 956c3e6..8fec8c3 100644 --- a/internal/extract/plain.go +++ b/internal/extract/plain.go @@ -21,12 +21,29 @@ const sniffSize = 8192 // leading UTF-8 BOM is stripped and the rest used as is; a UTF-16 LE or BE // BOM is decoded with unicode/utf16; otherwise valid UTF-8 is used as is, // and any other invalid UTF-8 is decoded one byte per Latin-1 code point. -func readDecoded(path string) (string, error) { +func readDecoded(path string, maxRead int64) (string, error) { data, err := os.ReadFile(path) if err != nil { return "", err } - return decode(data), nil + // max-read bounds the bytes read; it must bound what they become as + // well. A file that is not valid UTF-8 decodes one byte per code point + // and so doubles, and everything downstream - normalising, folding - + // copies that again, per set of options, with several files in flight. + return bounded(decode(data), maxRead), nil +} + +// truncateAtRune is the largest cut at or below n that does not split a +// rune, so a bounded text is still valid UTF-8. +func truncateAtRune(s string, n int64) int { + i := int(n) + if i >= len(s) { + return len(s) + } + for i > 0 && !utf8.RuneStart(s[i]) { + i-- + } + return i } // sniffText decides whether an unknown extension is text, reading at most @@ -48,7 +65,7 @@ func readDecoded(path string) (string, error) { // unsupported; sample itself, used below to build the returned text, is // left untouched — the rest of the file (read after the check) supplies // the bytes trimming set aside. -func sniffText(path string) (string, error) { +func sniffText(path string, maxRead int64) (string, error) { f, err := os.Open(path) if err != nil { return "", err @@ -79,12 +96,20 @@ func sniffText(path string) (string, error) { data := append(sample, rest...) if utf16BOM { - return decode(data), nil + return bounded(decode(data), maxRead), nil } if !utf8.Valid(data) || bytes.Contains(data, []byte{0}) { return "", ErrUnsupported } - return decode(data), nil + return bounded(decode(data), maxRead), nil +} + +// bounded cuts text to maxRead at a rune boundary; 0 means unlimited. +func bounded(text string, maxRead int64) string { + if maxRead <= 0 || int64(len(text)) <= maxRead { + return text + } + return text[:truncateAtRune(text, maxRead)] } // trimIncompleteTrailingRune drops an incomplete UTF-8 sequence left -- cgit v1.3