// SPDX-License-Identifier: GPL-3.0-or-later package extract import ( "bytes" "context" "errors" "fmt" "os" "os/exec" "path/filepath" "strings" "testing" "time" ) // fakeTool writes an executable shell script named name into dir. func fakeTool(t *testing.T, dir, name, body string) { t.Helper() script := "#!/bin/sh\n" + body + "\n" if err := os.WriteFile(filepath.Join(dir, name), []byte(script), 0o755); err != nil { t.Fatal(err) } } func TestPdfViaFakeTool(t *testing.T) { bin := t.TempDir() argsFile := filepath.Join(t.TempDir(), "args") fakeTool(t, bin, "pdftotext", `printf '%s\n' "$@" > "`+argsFile+`"; echo "acme ltd invoice"`) e := newWithPath(bin) p := file(t, "-leading-dash.pdf", []byte("%PDF-1.4")) got, err := text(t, e, p, 0) if err != nil || strings.TrimSpace(got) != "acme ltd invoice" { t.Fatalf("got %q, %v", got, err) } args, _ := os.ReadFile(argsFile) want := "-q\n-enc\nUTF-8\n" + p + "\n-\n" if string(args) != want { t.Fatalf("pdftotext args:\n%q\nwant\n%q", args, want) } if !filepath.IsAbs(strings.Split(string(args), "\n")[3]) { t.Fatal("path argument is not absolute") } } func TestLegacyFormats(t *testing.T) { bin := t.TempDir() fakeTool(t, bin, "catdoc", `echo "from catdoc"`) fakeTool(t, bin, "xls2csv", `echo "from xls2csv"`) fakeTool(t, bin, "catppt", `echo "from catppt"`) e := newWithPath(bin) // no antiword: .doc falls back to catdoc for ext, want := range map[string]string{"doc": "from catdoc", "xls": "from xls2csv", "ppt": "from catppt"} { got, err := text(t, e, file(t, "f."+ext, []byte("x")), 0) if err != nil || strings.TrimSpace(got) != want { t.Errorf("%s: got %q, %v", ext, got, err) } } fakeTool(t, bin, "antiword", `echo "from antiword"`) e = newWithPath(bin) if got, _ := text(t, e, file(t, "g.doc", []byte("x")), 0); strings.TrimSpace(got) != "from antiword" { t.Errorf("antiword not preferred: %q", got) } fakeTool(t, bin, "antiword", `echo "antiword broke" >&2; exit 1`) e = newWithPath(bin) if got, _ := text(t, e, file(t, "h.doc", []byte("x")), 0); strings.TrimSpace(got) != "from catdoc" { t.Errorf("no fallback to catdoc after antiword failed: %q", got) } } func TestToolErrors(t *testing.T) { e := newWithPath(t.TempDir()) var tm *ToolMissingError if _, err := text(t, e, file(t, "a.pdf", []byte("x")), 0); !errors.As(err, &tm) || tm.Error() != "needs pdftotext, not installed" { t.Errorf("missing pdftotext: %v", err) } if _, err := text(t, e, file(t, "a.doc", []byte("x")), 0); err == nil || err.Error() != "needs antiword or catdoc, not installed" { t.Errorf("missing doc tools: %v", err) } bin := t.TempDir() fakeTool(t, bin, "pdftotext", `echo "Syntax Error: broken xref" >&2; exit 3`) e = newWithPath(bin) if _, err := text(t, e, file(t, "b.pdf", []byte("x")), 0); err == nil || err.Error() != "pdftotext failed: Syntax Error: broken xref" { t.Errorf("failing tool: %v", err) } fakeTool(t, bin, "pdftotext", `sleep 5`) e = newWithPath(bin) e.Timeout = 200 * time.Millisecond start := time.Now() _, err := text(t, e, file(t, "c.pdf", []byte("x")), 0) if err == nil || !strings.Contains(err.Error(), "pdftotext timed out after 200ms") { t.Errorf("slow tool: %v", err) } if time.Since(start) > 3*time.Second { t.Errorf("timeout took %v", time.Since(start)) } } // minimalPDF returns a valid one-page PDF showing text in Helvetica. func minimalPDF(text string) []byte { var b bytes.Buffer var offsets []int obj := func(s string) { offsets = append(offsets, b.Len()); b.WriteString(s) } b.WriteString("%PDF-1.4\n") obj("1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n") obj("2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj\n") obj("3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >> endobj\n") stream := "BT /F1 24 Tf 72 700 Td (" + text + ") Tj ET" obj(fmt.Sprintf("4 0 obj << /Length %d >> stream\n%s\nendstream endobj\n", len(stream), stream)) obj("5 0 obj << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> endobj\n") xref := b.Len() fmt.Fprintf(&b, "xref\n0 %d\n0000000000 65535 f \n", len(offsets)+1) for _, o := range offsets { fmt.Fprintf(&b, "%010d 00000 n \n", o) } fmt.Fprintf(&b, "trailer << /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n", len(offsets)+1, xref) return b.Bytes() } func TestRealPdftotext(t *testing.T) { if _, err := exec.LookPath("pdftotext"); err != nil { t.Skip("pdftotext not installed") } e := New() got, err := text(t, e, file(t, "hello.pdf", minimalPDF("Hello acme ltd")), 0) if err != nil { t.Fatal(err) } if !strings.Contains(got, "Hello acme ltd") { t.Fatalf("pdftotext gave %q", got) } } // TestOrphanChildDoesNotHangRun: a tool that exits itself but leaves a // backgrounded grandchild holding its stdout pipe open must not make // run() wait for that grandchild. Only os/exec itself, copying into // cmd.Stdout and bounding the post-exit wait via WaitDelay, can end this // promptly; run() manually draining a StdoutPipe before calling Wait // starves WaitDelay of the thing it bounds, since by the time Wait runs // there is nothing left for it to forcibly cut off. func TestOrphanChildDoesNotHangRun(t *testing.T) { bin := t.TempDir() fakeTool(t, bin, "pdftotext", "sleep 5 & echo x") e := newWithPath(bin) e.Timeout = 30 * time.Second start := time.Now() got, err := e.run(context.Background(), maxToolOutput, "pdftotext") if err != nil { t.Fatalf("orphan: %v", err) } if strings.TrimSpace(got) != "x" { t.Errorf("orphan: got %q, want %q", got, "x") } if d := time.Since(start); d > 3*time.Second { t.Errorf("orphan: took %v, want well under e.Timeout (30s) and under 3s", d) } } // TestStdoutOverflowKillsToolPromptly: a tool that keeps writing past // maxToolOutput must be killed the moment the cap is crossed, not left // running until e.Timeout expires — once run() stops draining its pipe, // an unkilled tool blocks on its own write() and never exits on its own. func TestStdoutOverflowKillsToolPromptly(t *testing.T) { old := maxToolOutput maxToolOutput = 1 << 20 defer func() { maxToolOutput = old }() bin := t.TempDir() fakeTool(t, bin, "pdftotext", "yes aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") e := newWithPath(bin) e.Timeout = 30 * time.Second start := time.Now() _, err := e.run(context.Background(), maxToolOutput, "pdftotext") if !errors.Is(err, ErrTooLarge) { t.Fatalf("overflow: got %v, want ErrTooLarge", err) } if d := time.Since(start); d > 5*time.Second { t.Errorf("overflow: took %v, want under 5s (well under e.Timeout=30s)", d) } } // TestCallerCancelReturnsPromptly: cancelling the ctx passed to run() // (distinct from e.Timeout's own internal deadline, which is untouched // here) must stop the tool and return quickly, with an error wrapping // the caller's own context.Canceled — not silently absorbed into a // generic " failed: ..." string, and not held open until // e.Timeout. func TestCallerCancelReturnsPromptly(t *testing.T) { bin := t.TempDir() fakeTool(t, bin, "pdftotext", "sleep 10") e := newWithPath(bin) e.Timeout = 30 * time.Second ctx, cancel := context.WithCancel(context.Background()) time.AfterFunc(100*time.Millisecond, cancel) start := time.Now() _, err := e.run(ctx, maxToolOutput, "pdftotext") if !errors.Is(err, context.Canceled) { t.Fatalf("caller cancel: got %v, want an error wrapping context.Canceled", err) } if d := time.Since(start); d > 3*time.Second { t.Errorf("caller cancel: took %v, want under 3s", d) } } // TestStderrFloodBounded: a tool that floods stderr must not blow up the // size of the error message run() produces — only a bounded prefix of // its first line may ever reach the returned error text, and capturing // it at all must not cost unbounded memory. func TestStderrFloodBounded(t *testing.T) { bin := t.TempDir() fakeTool(t, bin, "pdftotext", `head -c 10000000 /dev/zero | tr '\0' x >&2; exit 1`) e := newWithPath(bin) _, err := e.run(context.Background(), maxToolOutput, "pdftotext") if err == nil { t.Fatal("stderr flood: want an error") } if max := len("pdftotext failed: ") + 200; len(err.Error()) > max { t.Errorf("stderr flood: message is %d bytes, want <=%d", len(err.Error()), max) } } // TestMaxReadCapsToolOutput: B1. A directory's max-read, when smaller than // the fixed maxToolOutput default, caps a single tool's output on its // own — proven here with maxToolOutput left at its default, so only // budget(maxToolOutput, maxRead) picking the smaller maxRead explains the // result. func TestMaxReadCapsToolOutput(t *testing.T) { bin := t.TempDir() fakeTool(t, bin, "pdftotext", `yes aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa | head -c 200000`) e := newWithPath(bin) if _, err := text(t, e, file(t, "small.pdf", []byte("%PDF-1.4")), 1024); !errors.Is(err, ErrTooLarge) { t.Fatalf("got %v, want ErrTooLarge (max-read 1024 should have capped a 200000-byte tool output)", err) } } // TestFingerprintFollowsTools: the fingerprint changes when a tool is // installed or replaced, and not otherwise. func TestFingerprintFollowsTools(t *testing.T) { bin := t.TempDir() none := newWithPath(bin).Fingerprint() if none != newWithPath(bin).Fingerprint() { t.Fatal("fingerprint not stable") } tool := filepath.Join(bin, "pdftotext") if err := os.WriteFile(tool, []byte("#!/bin/sh\n"), 0o755); err != nil { t.Fatal(err) } installed := newWithPath(bin).Fingerprint() if installed == none { t.Error("installing pdftotext did not change the fingerprint") } if err := os.WriteFile(tool, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { t.Fatal(err) } if newWithPath(bin).Fingerprint() == installed { t.Error("replacing pdftotext did not change the fingerprint") } } // TestToolLookupSkipsRelativePathEntries: a relative PATH entry would make // a tool's path relative to the working directory - a bin/pdftotext left by // an unpacked download - so it is ignored (review planapply F7). func TestToolLookupSkipsRelativePathEntries(t *testing.T) { wd := t.TempDir() if err := os.Mkdir(filepath.Join(wd, "bin"), 0o755); err != nil { t.Fatal(err) } fakeTool(t, filepath.Join(wd, "bin"), "pdftotext", "echo injected") t.Chdir(wd) if e := newWithPath("bin"); e.tools["pdftotext"] != "" { t.Errorf("tool found through a relative PATH entry: %q", e.tools["pdftotext"]) } }