// SPDX-License-Identifier: GPL-3.0-or-later package extract import ( "bytes" "context" "errors" "fmt" "os/exec" "path/filepath" "time" ) // maxToolOutput caps the bytes kept from an external tool's stdout; more // than this returns ErrTooLarge. A package var, not a const, so a test can // lower it without generating gigabytes of fake output. B1: the directory's // max-read may cap a single extraction further still — see budget. var maxToolOutput int64 = 64 << 20 // budget returns the smaller of fixed (the package's own default ceiling — // maxToolOutput or zipBudget) and maxRead, the directory's configured // max-read; maxRead 0 means unlimited, so fixed alone applies. B1: a // single extraction's output must never exceed the ceiling the user set, // even when that ceiling is below the fixed default. func budget(fixed, maxRead int64) int64 { if maxRead > 0 && maxRead < fixed { return maxRead } return fixed } // maxStderr caps the bytes kept from an external tool's stderr — enough // for a diagnostic first line. Unlike stdout, crossing this never kills // the tool: stderr noise is not grounds to abort an otherwise-working // extraction, only grounds to stop remembering more of it. const maxStderr = 4 << 10 // maxErrLine caps how much of stderr's first line is folded into the // error text run() returns, so a flooding tool cannot make that message // itself unbounded. const maxErrLine = 200 // boundedWriter keeps at most limit bytes written to it and silently // discards the rest, always reporting success to the writer — an // io.Writer that returns an error would abort the copy goroutine // exec.Cmd runs for Stdout/Stderr, which is not what should happen here: // the tool must keep being drained (or be killed outright, via // onOverflow) rather than have its pipe start backing up. If onOverflow // is set, it fires exactly once, the moment the total ever written first // exceeds limit; run() uses it on stdout, and only stdout, to cancel the // command immediately rather than let an over-producing tool run until // e.Timeout. Write is only ever called by the single copy goroutine // exec.Cmd runs per stream, so no locking is needed; run() only reads a // boundedWriter's fields after cmd.Run() has returned, which happens // strictly after that goroutine has finished (Wait's documented // synchronisation), giving the read a safe happens-before. type boundedWriter struct { limit int64 onOverflow func() buf bytes.Buffer total int64 overflowed bool } func (w *boundedWriter) Write(p []byte) (int, error) { w.total += int64(len(p)) if w.total > w.limit { if !w.overflowed { w.overflowed = true if w.onOverflow != nil { w.onOverflow() } } return len(p), nil } w.buf.Write(p) return len(p), nil } // run executes tool (looked up in e.tools, its absolute path) with args, // under a timeout of e.Timeout, and returns its stdout as text, its // stdout capped at maxOut bytes (the caller passes budget(maxToolOutput, // maxRead), B1). No shell is involved: exec.CommandContext runs the // tool's path directly with args passed separately, so nothing in a // hostile filename or argument is ever interpreted. The environment is // inherited unchanged. // // os/exec is left to own all the copying — cmd.Stdout and cmd.Stderr are // bounded writers, and cmd.Run does the reading — so that cmd.WaitDelay's // hang protection actually applies: WaitDelay bounds how long Wait spends // on I/O after the process itself has exited (or after ctx is done), // forcibly closing the pipes once that grace period elapses. An earlier // version of this function read stdout itself, ahead of Wait, which // starved WaitDelay of the thing it bounds: once that manual read // stopped (at EOF, or at the output cap), Wait was left blocked on // whatever was still holding the pipe open — a grandchild the tool // backgrounded and left running, or the tool itself blocked writing to a // pipe nobody was draining once the cap was hit — with nothing left to // force it closed. In both shapes the call could run for the full // e.Timeout (or longer) instead of returning promptly. // // Stdout is capped at maxOut bytes (the caller's budget(maxToolOutput, // maxRead), B1): crossing it cancels the command immediately, via the // boundedWriter's onOverflow, and run reports ErrTooLarge. Stderr is // capped at maxStderr bytes and never // cancels anything; only its first line, truncated to maxErrLine bytes, // ever reaches an error message, so a tool flooding stderr costs bounded // memory and produces a bounded error. // // Killing the command — by the caller's ctx being cancelled, by // e.Timeout expiring, or by the stdout cap being crossed — can leave // cmd.Run reporting exec.ErrWaitDelay even though the process's own exit // status was clean: SIGKILL forces the pipes closed without giving the // child a chance to flush or exit on its own. That alone is not a // failure (see the ErrWaitDelay case below); only a genuinely non-zero // exit is treated as one. // // A non-zero exit returns " failed: " (or // " failed: " when stderr was empty), the line capped to // maxErrLine bytes. A caller-cancelled ctx returns promptly, with an // error wrapping ctx.Err(); e.Timeout expiring on its own returns // " timed out after ". func (e *Extractor) run(ctx context.Context, maxOut int64, tool string, args ...string) (string, error) { path := e.tools[tool] runCtx, cancel := context.WithTimeout(ctx, e.Timeout) defer cancel() cmd := exec.CommandContext(runCtx, path, args...) cmd.WaitDelay = time.Second stdout := &boundedWriter{limit: maxOut, onOverflow: cancel} stderr := &boundedWriter{limit: maxStderr} cmd.Stdout = stdout cmd.Stderr = stderr err := cmd.Run() switch { case stdout.overflowed: // Checked first: killing the tool for overflow also cancels // runCtx, so without this ordering the case below would report // the cancellation as a timeout instead of what it actually was. return "", ErrTooLarge case ctx.Err() != nil: // The caller's own context, not the internal e.Timeout deadline // derived from it — checked before runCtx's, since runCtx // inherits the caller's cancellation too and would otherwise be // indistinguishable from it below. return "", fmt.Errorf("%s: %w", tool, ctx.Err()) case runCtx.Err() == context.DeadlineExceeded: return "", fmt.Errorf("%s timed out after %s", tool, e.Timeout) } if err != nil { if errors.Is(err, exec.ErrWaitDelay) && cmd.ProcessState != nil && cmd.ProcessState.ExitCode() == 0 { return stdout.buf.String(), nil } if line := truncate(firstLine(stderr.buf.Bytes()), maxErrLine); line != "" { return "", fmt.Errorf("%s failed: %s", tool, line) } return "", fmt.Errorf("%s failed: %s", tool, err) } return stdout.buf.String(), nil } // firstLine returns the first non-empty line of b, trimmed of its // trailing newline, or "" if b holds nothing but blank lines. func firstLine(b []byte) string { for _, line := range bytes.Split(b, []byte("\n")) { if len(bytes.TrimSpace(line)) > 0 { return string(bytes.TrimRight(line, "\r")) } } return "" } // truncate returns s cut to at most n bytes, so text built from // untrusted tool output has a hard, predictable bound on its length // regardless of what the tool wrote. It may cut a multi-byte UTF-8 // sequence in two; a trailing partial rune in a diagnostic error message // is an acceptable cost for a byte bound that never slips. func truncate(s string, n int) string { if len(s) > n { return s[:n] } return s } // pdfText extracts text from a PDF with pdftotext, its output capped per // budget(maxToolOutput, maxRead) (B1). func (e *Extractor) pdfText(ctx context.Context, path string, maxRead int64) (string, error) { if e.tools["pdftotext"] == "" { return "", &ToolMissingError{Tool: "pdftotext"} } abs, err := filepath.Abs(path) if err != nil { return "", err } return e.run(ctx, budget(maxToolOutput, maxRead), "pdftotext", "-q", "-enc", "UTF-8", abs, "-") } // legacyText extracts text from a legacy binary Office format (doc xls // ppt) with the external tool toolExt names, its output capped per // budget(maxToolOutput, maxRead) (B1). .doc prefers antiword, falling // back to catdoc if antiword is absent or fails. func (e *Extractor) legacyText(ctx context.Context, path, ext string, maxRead int64) (string, error) { abs, err := filepath.Abs(path) if err != nil { return "", err } out := budget(maxToolOutput, maxRead) switch ext { case "doc": haveAntiword := e.tools["antiword"] != "" haveCatdoc := e.tools["catdoc"] != "" if !haveAntiword && !haveCatdoc { return "", &ToolMissingError{Tool: "antiword or catdoc"} } if haveAntiword { text, err := e.run(ctx, out, "antiword", "-m", "UTF-8.txt", abs) if err == nil { return text, nil } if !haveCatdoc { return "", err } } return e.run(ctx, out, "catdoc", "-d", "utf-8", abs) case "xls": if e.tools["xls2csv"] == "" { return "", &ToolMissingError{Tool: "xls2csv"} } return e.run(ctx, out, "xls2csv", "-d", "utf-8", abs) case "ppt": if e.tools["catppt"] == "" { return "", &ToolMissingError{Tool: "catppt"} } return e.run(ctx, out, "catppt", "-d", "utf-8", abs) default: return "", errors.New("extract: unreachable: legacyText called with unknown extension " + ext) } }