krino: matching — scan, ignore, conditions, extraction, duplicates, explain, dry run

1 files changed, 239 insertions, 0 deletions

diff --git a/internal/extract/tools_test.go b/internal/extract/tools_test.go
new file mode 100644
index 0000000..4a03f1b
--- /dev/null
+++ b/internal/extract/tools_test.go
@@ -0,0 +1,239 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package extract
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+// fakeTool writes an executable shell script named name into dir.
+func fakeTool(t *testing.T, dir, name, body string) {
+	t.Helper()
+	script := "#!/bin/sh\n" + body + "\n"
+	if err := os.WriteFile(filepath.Join(dir, name), []byte(script), 0o755); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestPdfViaFakeTool(t *testing.T) {
+	bin := t.TempDir()
+	argsFile := filepath.Join(t.TempDir(), "args")
+	fakeTool(t, bin, "pdftotext", `printf '%s\n' "$@" > "`+argsFile+`"; echo "acme ltd invoice"`)
+	e := newWithPath(bin)
+	p := file(t, "-leading-dash.pdf", []byte("%PDF-1.4"))
+	got, err := text(t, e, p, 0)
+	if err != nil || strings.TrimSpace(got) != "acme ltd invoice" {
+		t.Fatalf("got %q, %v", got, err)
+	}
+	args, _ := os.ReadFile(argsFile)
+	want := "-q\n-enc\nUTF-8\n" + p + "\n-\n"
+	if string(args) != want {
+		t.Fatalf("pdftotext args:\n%q\nwant\n%q", args, want)
+	}
+	if !filepath.IsAbs(strings.Split(string(args), "\n")[3]) {
+		t.Fatal("path argument is not absolute")
+	}
+}
+
+func TestLegacyFormats(t *testing.T) {
+	bin := t.TempDir()
+	fakeTool(t, bin, "catdoc", `echo "from catdoc"`)
+	fakeTool(t, bin, "xls2csv", `echo "from xls2csv"`)
+	fakeTool(t, bin, "catppt", `echo "from catppt"`)
+	e := newWithPath(bin) // no antiword: .doc falls back to catdoc
+	for ext, want := range map[string]string{"doc": "from catdoc", "xls": "from xls2csv", "ppt": "from catppt"} {
+		got, err := text(t, e, file(t, "f."+ext, []byte("x")), 0)
+		if err != nil || strings.TrimSpace(got) != want {
+			t.Errorf("%s: got %q, %v", ext, got, err)
+		}
+	}
+	fakeTool(t, bin, "antiword", `echo "from antiword"`)
+	e = newWithPath(bin)
+	if got, _ := text(t, e, file(t, "g.doc", []byte("x")), 0); strings.TrimSpace(got) != "from antiword" {
+		t.Errorf("antiword not preferred: %q", got)
+	}
+	fakeTool(t, bin, "antiword", `echo "antiword broke" >&2; exit 1`)
+	e = newWithPath(bin)
+	if got, _ := text(t, e, file(t, "h.doc", []byte("x")), 0); strings.TrimSpace(got) != "from catdoc" {
+		t.Errorf("no fallback to catdoc after antiword failed: %q", got)
+	}
+}
+
+func TestToolErrors(t *testing.T) {
+	e := newWithPath(t.TempDir())
+	var tm *ToolMissingError
+	if _, err := text(t, e, file(t, "a.pdf", []byte("x")), 0); !errors.As(err, &tm) || tm.Error() != "needs pdftotext, not installed" {
+		t.Errorf("missing pdftotext: %v", err)
+	}
+	if _, err := text(t, e, file(t, "a.doc", []byte("x")), 0); err == nil || err.Error() != "needs antiword or catdoc, not installed" {
+		t.Errorf("missing doc tools: %v", err)
+	}
+	bin := t.TempDir()
+	fakeTool(t, bin, "pdftotext", `echo "Syntax Error: broken xref" >&2; exit 3`)
+	e = newWithPath(bin)
+	if _, err := text(t, e, file(t, "b.pdf", []byte("x")), 0); err == nil || err.Error() != "pdftotext failed: Syntax Error: broken xref" {
+		t.Errorf("failing tool: %v", err)
+	}
+	fakeTool(t, bin, "pdftotext", `sleep 5`)
+	e = newWithPath(bin)
+	e.Timeout = 200 * time.Millisecond
+	start := time.Now()
+	_, err := text(t, e, file(t, "c.pdf", []byte("x")), 0)
+	if err == nil || !strings.Contains(err.Error(), "pdftotext timed out after 200ms") {
+		t.Errorf("slow tool: %v", err)
+	}
+	if time.Since(start) > 3*time.Second {
+		t.Errorf("timeout took %v", time.Since(start))
+	}
+}
+
+// minimalPDF returns a valid one-page PDF showing text in Helvetica.
+func minimalPDF(text string) []byte {
+	var b bytes.Buffer
+	var offsets []int
+	obj := func(s string) { offsets = append(offsets, b.Len()); b.WriteString(s) }
+	b.WriteString("%PDF-1.4\n")
+	obj("1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n")
+	obj("2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj\n")
+	obj("3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >> endobj\n")
+	stream := "BT /F1 24 Tf 72 700 Td (" + text + ") Tj ET"
+	obj(fmt.Sprintf("4 0 obj << /Length %d >> stream\n%s\nendstream endobj\n", len(stream), stream))
+	obj("5 0 obj << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> endobj\n")
+	xref := b.Len()
+	fmt.Fprintf(&b, "xref\n0 %d\n0000000000 65535 f \n", len(offsets)+1)
+	for _, o := range offsets {
+		fmt.Fprintf(&b, "%010d 00000 n \n", o)
+	}
+	fmt.Fprintf(&b, "trailer << /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n", len(offsets)+1, xref)
+	return b.Bytes()
+}
+
+func TestRealPdftotext(t *testing.T) {
+	if _, err := exec.LookPath("pdftotext"); err != nil {
+		t.Skip("pdftotext not installed")
+	}
+	e := New()
+	got, err := text(t, e, file(t, "hello.pdf", minimalPDF("Hello acme ltd")), 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(got, "Hello acme ltd") {
+		t.Fatalf("pdftotext gave %q", got)
+	}
+}
+
+// TestOrphanChildDoesNotHangRun: a tool that exits itself but leaves a
+// backgrounded grandchild holding its stdout pipe open must not make
+// run() wait for that grandchild. Only os/exec itself, copying into
+// cmd.Stdout and bounding the post-exit wait via WaitDelay, can end this
+// promptly; run() manually draining a StdoutPipe before calling Wait
+// starves WaitDelay of the thing it bounds, since by the time Wait runs
+// there is nothing left for it to forcibly cut off.
+func TestOrphanChildDoesNotHangRun(t *testing.T) {
+	bin := t.TempDir()
+	fakeTool(t, bin, "pdftotext", "sleep 5 & echo x")
+	e := newWithPath(bin)
+	e.Timeout = 30 * time.Second
+
+	start := time.Now()
+	got, err := e.run(context.Background(), maxToolOutput, "pdftotext")
+	if err != nil {
+		t.Fatalf("orphan: %v", err)
+	}
+	if strings.TrimSpace(got) != "x" {
+		t.Errorf("orphan: got %q, want %q", got, "x")
+	}
+	if d := time.Since(start); d > 3*time.Second {
+		t.Errorf("orphan: took %v, want well under e.Timeout (30s) and under 3s", d)
+	}
+}
+
+// TestStdoutOverflowKillsToolPromptly: a tool that keeps writing past
+// maxToolOutput must be killed the moment the cap is crossed, not left
+// running until e.Timeout expires — once run() stops draining its pipe,
+// an unkilled tool blocks on its own write() and never exits on its own.
+func TestStdoutOverflowKillsToolPromptly(t *testing.T) {
+	old := maxToolOutput
+	maxToolOutput = 1 << 20
+	defer func() { maxToolOutput = old }()
+
+	bin := t.TempDir()
+	fakeTool(t, bin, "pdftotext", "yes aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
+	e := newWithPath(bin)
+	e.Timeout = 30 * time.Second
+
+	start := time.Now()
+	_, err := e.run(context.Background(), maxToolOutput, "pdftotext")
+	if !errors.Is(err, ErrTooLarge) {
+		t.Fatalf("overflow: got %v, want ErrTooLarge", err)
+	}
+	if d := time.Since(start); d > 5*time.Second {
+		t.Errorf("overflow: took %v, want under 5s (well under e.Timeout=30s)", d)
+	}
+}
+
+// TestCallerCancelReturnsPromptly: cancelling the ctx passed to run()
+// (distinct from e.Timeout's own internal deadline, which is untouched
+// here) must stop the tool and return quickly, with an error wrapping
+// the caller's own context.Canceled — not silently absorbed into a
+// generic "<tool> failed: ..." string, and not held open until
+// e.Timeout.
+func TestCallerCancelReturnsPromptly(t *testing.T) {
+	bin := t.TempDir()
+	fakeTool(t, bin, "pdftotext", "sleep 10")
+	e := newWithPath(bin)
+	e.Timeout = 30 * time.Second
+
+	ctx, cancel := context.WithCancel(context.Background())
+	time.AfterFunc(100*time.Millisecond, cancel)
+
+	start := time.Now()
+	_, err := e.run(ctx, maxToolOutput, "pdftotext")
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("caller cancel: got %v, want an error wrapping context.Canceled", err)
+	}
+	if d := time.Since(start); d > 3*time.Second {
+		t.Errorf("caller cancel: took %v, want under 3s", d)
+	}
+}
+
+// TestStderrFloodBounded: a tool that floods stderr must not blow up the
+// size of the error message run() produces — only a bounded prefix of
+// its first line may ever reach the returned error text, and capturing
+// it at all must not cost unbounded memory.
+func TestStderrFloodBounded(t *testing.T) {
+	bin := t.TempDir()
+	fakeTool(t, bin, "pdftotext", `head -c 10000000 /dev/zero | tr '\0' x >&2; exit 1`)
+	e := newWithPath(bin)
+
+	_, err := e.run(context.Background(), maxToolOutput, "pdftotext")
+	if err == nil {
+		t.Fatal("stderr flood: want an error")
+	}
+	if max := len("pdftotext failed: ") + 200; len(err.Error()) > max {
+		t.Errorf("stderr flood: message is %d bytes, want <=%d", len(err.Error()), max)
+	}
+}
+
+// TestMaxReadCapsToolOutput: B1. A directory's max-read, when smaller than
+// the fixed maxToolOutput default, caps a single tool's output on its
+// own — proven here with maxToolOutput left at its default, so only
+// budget(maxToolOutput, maxRead) picking the smaller maxRead explains the
+// result.
+func TestMaxReadCapsToolOutput(t *testing.T) {
+	bin := t.TempDir()
+	fakeTool(t, bin, "pdftotext", `yes aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa | head -c 200000`)
+	e := newWithPath(bin)
+	if _, err := text(t, e, file(t, "small.pdf", []byte("%PDF-1.4")), 1024); !errors.Is(err, ErrTooLarge) {
+		t.Fatalf("got %v, want ErrTooLarge (max-read 1024 should have capped a 200000-byte tool output)", err)
+	}
+}