// SPDX-License-Identifier: GPL-3.0-or-later package extract import ( "bytes" "context" "errors" "os" "path/filepath" "strings" "syscall" "testing" ) // file writes data to name in a temp dir and returns the path. func file(t *testing.T, name string, data []byte) string { t.Helper() p := filepath.Join(t.TempDir(), name) if err := os.WriteFile(p, data, 0o644); err != nil { t.Fatal(err) } return p } func text(t *testing.T, e *Extractor, p string, maxRead int64) (string, error) { t.Helper() fi, err := os.Stat(p) if err != nil { t.Fatal(err) } return e.Text(context.Background(), p, fi.Size(), maxRead) } func TestPlainEncodings(t *testing.T) { e := newWithPath("") utf16le := []byte{0xFF, 0xFE, 'A', 0, 'c', 0, 'm', 0, 'e', 0} tests := []struct { name string data []byte want string }{ {"a.txt", []byte("Faktura acme ltd\n"), "Faktura acme ltd\n"}, {"bom.txt", []byte("\xEF\xBB\xBFhello"), "hello"}, {"latin1.txt", []byte("Gr\xfc\xdfe"), "Grüße"}, {"u16.txt", utf16le, "Acme"}, {"notes.md", []byte("# Title\nbody"), "# Title\nbody"}, {"empty.txt", nil, ""}, {"README", []byte("no extension but text"), "no extension but text"}, } for _, tt := range tests { got, err := text(t, e, file(t, tt.name, tt.data), 0) if err != nil || got != tt.want { t.Errorf("%s: got %q, %v; want %q", tt.name, got, err, tt.want) } } } func TestMarkup(t *testing.T) { e := newWithPath("") tests := []struct { name string src string want []string // at least one of these must be a substring wantAny bool // if true, want is an alternative set: any one suffices bad []string // none of these may be a substring }{ { name: "tags, script/style dropped, entities decoded", src: "" + "

Faktura VAT & co

acme ltd

", want: []string{"Faktura VAT & co", "acme ltd"}, bad: []string{"color:red", "var x", "

"}, }, { name: "a lone < followed by a digit or space is literal text", src: "

price < 500 zl, done

", want: []string{"price < 500 zl, done"}, wantAny: true, }, { name: "an HTML comment is skipped, not its neighbours", src: "

", want: []string{"a b", "ab"}, wantAny: true, bad: []string{"hidden"}, }, } for _, tt := range tests { got, err := text(t, e, file(t, "page.html", []byte(tt.src)), 0) if err != nil { t.Fatalf("%s: %v", tt.name, err) } if tt.wantAny { ok := false for _, w := range tt.want { if strings.Contains(got, w) { ok = true break } } if !ok { t.Errorf("%s: markup text %q has none of %q", tt.name, got, tt.want) } } else { for _, want := range tt.want { if !strings.Contains(got, want) { t.Errorf("%s: markup text %q lacks %q", tt.name, got, want) } } } for _, bad := range tt.bad { if strings.Contains(got, bad) { t.Errorf("%s: markup text %q still contains %q", tt.name, got, bad) } } } } func TestUnsupportedAndTooLarge(t *testing.T) { e := newWithPath("") png := []byte("\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR") if _, err := text(t, e, file(t, "image.png", png), 0); !errors.Is(err, ErrUnsupported) { t.Errorf("png: %v, want ErrUnsupported", err) } if _, err := text(t, e, file(t, "blob.bin", []byte{1, 2, 0, 3}), 0); !errors.Is(err, ErrUnsupported) { t.Errorf("binary: %v, want ErrUnsupported", err) } if _, err := text(t, e, file(t, "big.txt", []byte(strings.Repeat("x", 100))), 50); !errors.Is(err, ErrTooLarge) { t.Errorf("big: %v, want ErrTooLarge", err) } } func TestToolMissingError(t *testing.T) { err := &ToolMissingError{Tool: "pdftotext"} if err.Error() != "needs pdftotext, not installed" { t.Fatalf("got %q", err.Error()) } } func TestToolsListedInOrder(t *testing.T) { var names []string for _, tl := range newWithPath("").Tools() { names = append(names, tl.Name) if tl.Path != "" { t.Errorf("%s found with an empty PATH", tl.Name) } } if strings.Join(names, " ") != "pdftotext antiword catdoc xls2csv catppt" { t.Fatalf("tools = %v", names) } } // TestSniffWholeFileMustBeValid: the first 8 KiB sniffs as plain ASCII text, // but the file goes on to hold an invalid UTF-8 byte and a NUL past that // sample — sniffText must reject the whole file, not just decode what the // sample alone promised (it must not fall back to Latin-1 the way a known // text extension would). Such a file - a self-extracting installer, say - // has no text in krino's sense. func TestSniffWholeFileMustBeValid(t *testing.T) { e := newWithPath("") data := append([]byte(strings.Repeat("x", 8192)), 0xFF, 0x00) if _, err := text(t, e, file(t, "blob.data", data), 0); !errors.Is(err, ErrUnsupported) { t.Errorf("got %v, want ErrUnsupported", err) } } // TestSniffLargeBinaryRejected: an unrecognised-extension file whose very // first byte is invalid UTF-8 is rejected from the sample alone; this only // checks the outcome (ErrUnsupported), not that the rest of the megabyte // went unread — that efficiency claim isn't something a black-box test can // time reliably. func TestSniffLargeBinaryRejected(t *testing.T) { e := newWithPath("") data := make([]byte, 1<<20) // 1 MiB, far past the 8 KiB sniff window data[0] = 0xFF // invalid UTF-8 lead byte, visible in the sample if _, err := text(t, e, file(t, "huge.blob", data), 0); !errors.Is(err, ErrUnsupported) { t.Errorf("got %v, want ErrUnsupported", err) } } // TestToolLookupSkipsNonRegular: a FIFO named like a tool, executable bits // and all, must never be picked up — only a regular file counts. func TestToolLookupSkipsNonRegular(t *testing.T) { dir := t.TempDir() fifo := filepath.Join(dir, "pdftotext") if err := syscall.Mkfifo(fifo, 0o755); err != nil { t.Skipf("mkfifo not available: %v", err) } for _, tl := range newWithPath(dir).Tools() { if tl.Name == "pdftotext" && tl.Path != "" { t.Errorf("pdftotext resolved to a non-regular file: %s", tl.Path) } } } // TestUnterminatedTagKeepsRemainder: an unterminated ordinary tag (no // closing '>') must not discard the rest of the file - only the malformed // tag markup itself is unrecoverable; whatever follows it is still real // content and must still reach the extracted text. func TestUnterminatedTagKeepsRemainder(t *testing.T) { e := newWithPath("") src := "

before

limit { t.Errorf("decoding %d bytes allocates %d bytes, over %d", len(data), got, limit) } }