1 files changed, 229 insertions, 0 deletions
diff --git a/internal/extract/plain_test.go b/internal/extract/plain_test.go
new file mode 100644
index 0000000..045ea26
--- /dev/null
+++ b/internal/extract/plain_test.go
@@ -0,0 +1,229 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package extract
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"testing"
+)
+
+// file writes data to name in a temp dir and returns the path.
+func file(t *testing.T, name string, data []byte) string {
+	t.Helper()
+	p := filepath.Join(t.TempDir(), name)
+	if err := os.WriteFile(p, data, 0o644); err != nil {
+		t.Fatal(err)
+	}
+	return p
+}
+
+func text(t *testing.T, e *Extractor, p string, maxRead int64) (string, error) {
+	t.Helper()
+	fi, err := os.Stat(p)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return e.Text(context.Background(), p, fi.Size(), maxRead)
+}
+
+func TestPlainEncodings(t *testing.T) {
+	e := newWithPath("")
+	utf16le := []byte{0xFF, 0xFE, 'A', 0, 'c', 0, 'm', 0, 'e', 0}
+	tests := []struct {
+		name string
+		data []byte
+		want string
+	}{
+		{"a.txt", []byte("Faktura acme ltd\n"), "Faktura acme ltd\n"},
+		{"bom.txt", []byte("\xEF\xBB\xBFhello"), "hello"},
+		{"latin1.txt", []byte("Gr\xfc\xdfe"), "Grüße"},
+		{"u16.txt", utf16le, "Acme"},
+		{"notes.md", []byte("# Title\nbody"), "# Title\nbody"},
+		{"empty.txt", nil, ""},
+		{"README", []byte("no extension but text"), "no extension but text"},
+	}
+	for _, tt := range tests {
+		got, err := text(t, e, file(t, tt.name, tt.data), 0)
+		if err != nil || got != tt.want {
+			t.Errorf("%s: got %q, %v; want %q", tt.name, got, err, tt.want)
+		}
+	}
+}
+
+func TestMarkup(t *testing.T) {
+	e := newWithPath("")
+	tests := []struct {
+		name    string
+		src     string
+		want    []string // at least one of these must be a substring
+		wantAny bool     // if true, want is an alternative set: any one suffices
+		bad     []string // none of these may be a substring
+	}{
+		{
+			name: "tags, script/style dropped, entities decoded",
+			src: "<html><head><style>p{color:red}</style><script>var x='acme'</script></head>" +
+				"<body><p>Faktura&nbsp;VAT &amp; co</p><p>acme&#32;ltd</p></body></html>",
+			want: []string{"Faktura VAT & co", "acme ltd"},
+			bad:  []string{"color:red", "var x", "<p>"},
+		},
+		{
+			name:    "a lone < followed by a digit or space is literal text",
+			src:     "<p>price < 500 zl, done</p>",
+			want:    []string{"price < 500 zl, done"},
+			wantAny: true,
+		},
+		{
+			name:    "an HTML comment is skipped, not its neighbours",
+			src:     "<p>a<!-- hidden -->b</p>",
+			want:    []string{"a b", "ab"},
+			wantAny: true,
+			bad:     []string{"hidden"},
+		},
+	}
+	for _, tt := range tests {
+		got, err := text(t, e, file(t, "page.html", []byte(tt.src)), 0)
+		if err != nil {
+			t.Fatalf("%s: %v", tt.name, err)
+		}
+		if tt.wantAny {
+			ok := false
+			for _, w := range tt.want {
+				if strings.Contains(got, w) {
+					ok = true
+					break
+				}
+			}
+			if !ok {
+				t.Errorf("%s: markup text %q has none of %q", tt.name, got, tt.want)
+			}
+		} else {
+			for _, want := range tt.want {
+				if !strings.Contains(got, want) {
+					t.Errorf("%s: markup text %q lacks %q", tt.name, got, want)
+				}
+			}
+		}
+		for _, bad := range tt.bad {
+			if strings.Contains(got, bad) {
+				t.Errorf("%s: markup text %q still contains %q", tt.name, got, bad)
+			}
+		}
+	}
+}
+
+func TestUnsupportedAndTooLarge(t *testing.T) {
+	e := newWithPath("")
+	png := []byte("\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR")
+	if _, err := text(t, e, file(t, "image.png", png), 0); !errors.Is(err, ErrUnsupported) {
+		t.Errorf("png: %v, want ErrUnsupported", err)
+	}
+	if _, err := text(t, e, file(t, "blob.bin", []byte{1, 2, 0, 3}), 0); !errors.Is(err, ErrUnsupported) {
+		t.Errorf("binary: %v, want ErrUnsupported", err)
+	}
+	if _, err := text(t, e, file(t, "big.txt", []byte(strings.Repeat("x", 100))), 50); !errors.Is(err, ErrTooLarge) {
+		t.Errorf("big: %v, want ErrTooLarge", err)
+	}
+}
+
+func TestToolMissingError(t *testing.T) {
+	err := &ToolMissingError{Tool: "pdftotext"}
+	if err.Error() != "needs pdftotext, not installed" {
+		t.Fatalf("got %q", err.Error())
+	}
+}
+
+func TestToolsListedInOrder(t *testing.T) {
+	var names []string
+	for _, tl := range newWithPath("").Tools() {
+		names = append(names, tl.Name)
+		if tl.Path != "" {
+			t.Errorf("%s found with an empty PATH", tl.Name)
+		}
+	}
+	if strings.Join(names, " ") != "pdftotext antiword catdoc xls2csv catppt" {
+		t.Fatalf("tools = %v", names)
+	}
+}
+
+// TestSniffWholeFileMustBeValid: the first 8 KiB sniffs as plain ASCII text,
+// but the file goes on to hold an invalid UTF-8 byte and a NUL past that
+// sample — sniffText must reject the whole file, not just decode what the
+// sample alone promised (it must not fall back to Latin-1 the way a known
+// text extension would).
+func TestSniffWholeFileMustBeValid(t *testing.T) {
+	e := newWithPath("")
+	data := append([]byte(strings.Repeat("x", 8192)), 0xFF, 0x00)
+	if _, err := text(t, e, file(t, "blob.data", data), 0); !errors.Is(err, ErrUnsupported) {
+		t.Errorf("got %v, want ErrUnsupported", err)
+	}
+}
+
+// TestSniffLargeBinaryRejected: an unrecognised-extension file whose very
+// first byte is invalid UTF-8 is rejected from the sample alone; this only
+// checks the outcome (ErrUnsupported), not that the rest of the megabyte
+// went unread — that efficiency claim isn't something a black-box test can
+// time reliably.
+func TestSniffLargeBinaryRejected(t *testing.T) {
+	e := newWithPath("")
+	data := make([]byte, 1<<20) // 1 MiB, far past the 8 KiB sniff window
+	data[0] = 0xFF              // invalid UTF-8 lead byte, visible in the sample
+	if _, err := text(t, e, file(t, "huge.blob", data), 0); !errors.Is(err, ErrUnsupported) {
+		t.Errorf("got %v, want ErrUnsupported", err)
+	}
+}
+
+// TestToolLookupSkipsNonRegular: a FIFO named like a tool, executable bits
+// and all, must never be picked up — only a regular file counts.
+func TestToolLookupSkipsNonRegular(t *testing.T) {
+	dir := t.TempDir()
+	fifo := filepath.Join(dir, "pdftotext")
+	if err := syscall.Mkfifo(fifo, 0o755); err != nil {
+		t.Skipf("mkfifo not available: %v", err)
+	}
+	for _, tl := range newWithPath(dir).Tools() {
+		if tl.Name == "pdftotext" && tl.Path != "" {
+			t.Errorf("pdftotext resolved to a non-regular file: %s", tl.Path)
+		}
+	}
+}
+
+// TestUnterminatedTagKeepsRemainder: D1. An unterminated ordinary tag (no
+// closing '>') must not discard the rest of the file - only the malformed
+// tag markup itself is unrecoverable; whatever follows it is still real
+// content and must still reach the extracted text.
+func TestUnterminatedTagKeepsRemainder(t *testing.T) {
+	e := newWithPath("")
+	src := "<p>before</p><p unterminated text after"
+	got, err := text(t, e, file(t, "broken.html", []byte(src)), 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(got, "before") {
+		t.Errorf("text before the unterminated tag missing: %q", got)
+	}
+	if !strings.Contains(got, "unterminated text after") {
+		t.Errorf("text after the unterminated tag was discarded: %q", got)
+	}
+}
+
+// TestSniffRuneStraddlingSampleBoundary: D2. A multi-byte rune ("ż", two
+// UTF-8 bytes) placed exactly so its lead byte is the sniff sample's last
+// byte and its continuation byte falls just past it must not make an
+// otherwise valid UTF-8 file sniff as unsupported.
+func TestSniffRuneStraddlingSampleBoundary(t *testing.T) {
+	e := newWithPath("")
+	prefix := strings.Repeat("a", sniffSize-1)
+	data := []byte(prefix + "ż" + "bcd")
+	got, err := text(t, e, file(t, "straddle.blob", data), 0)
+	if err != nil {
+		t.Fatalf("valid UTF-8 with a rune straddling the sniff boundary: %v", err)
+	}
+	if want := "żbcd"; !strings.HasSuffix(got, want) {
+		t.Errorf("got tail %q, want it to end in %q", got[len(got)-8:], want)
+	}
+}