1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
|
// SPDX-License-Identifier: GPL-3.0-or-later
package extract
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"syscall"
"testing"
)
// file writes data to name in a temp dir and returns the path.
func file(t *testing.T, name string, data []byte) string {
t.Helper()
p := filepath.Join(t.TempDir(), name)
if err := os.WriteFile(p, data, 0o644); err != nil {
t.Fatal(err)
}
return p
}
func text(t *testing.T, e *Extractor, p string, maxRead int64) (string, error) {
t.Helper()
fi, err := os.Stat(p)
if err != nil {
t.Fatal(err)
}
return e.Text(context.Background(), p, fi.Size(), maxRead)
}
func TestPlainEncodings(t *testing.T) {
e := newWithPath("")
utf16le := []byte{0xFF, 0xFE, 'A', 0, 'c', 0, 'm', 0, 'e', 0}
tests := []struct {
name string
data []byte
want string
}{
{"a.txt", []byte("Faktura acme ltd\n"), "Faktura acme ltd\n"},
{"bom.txt", []byte("\xEF\xBB\xBFhello"), "hello"},
{"latin1.txt", []byte("Gr\xfc\xdfe"), "Grüße"},
{"u16.txt", utf16le, "Acme"},
{"notes.md", []byte("# Title\nbody"), "# Title\nbody"},
{"empty.txt", nil, ""},
{"README", []byte("no extension but text"), "no extension but text"},
}
for _, tt := range tests {
got, err := text(t, e, file(t, tt.name, tt.data), 0)
if err != nil || got != tt.want {
t.Errorf("%s: got %q, %v; want %q", tt.name, got, err, tt.want)
}
}
}
func TestMarkup(t *testing.T) {
e := newWithPath("")
tests := []struct {
name string
src string
want []string // at least one of these must be a substring
wantAny bool // if true, want is an alternative set: any one suffices
bad []string // none of these may be a substring
}{
{
name: "tags, script/style dropped, entities decoded",
src: "<html><head><style>p{color:red}</style><script>var x='acme'</script></head>" +
"<body><p>Faktura VAT & co</p><p>acme ltd</p></body></html>",
want: []string{"Faktura VAT & co", "acme ltd"},
bad: []string{"color:red", "var x", "<p>"},
},
{
name: "a lone < followed by a digit or space is literal text",
src: "<p>price < 500 zl, done</p>",
want: []string{"price < 500 zl, done"},
wantAny: true,
},
{
name: "an HTML comment is skipped, not its neighbours",
src: "<p>a<!-- hidden -->b</p>",
want: []string{"a b", "ab"},
wantAny: true,
bad: []string{"hidden"},
},
}
for _, tt := range tests {
got, err := text(t, e, file(t, "page.html", []byte(tt.src)), 0)
if err != nil {
t.Fatalf("%s: %v", tt.name, err)
}
if tt.wantAny {
ok := false
for _, w := range tt.want {
if strings.Contains(got, w) {
ok = true
break
}
}
if !ok {
t.Errorf("%s: markup text %q has none of %q", tt.name, got, tt.want)
}
} else {
for _, want := range tt.want {
if !strings.Contains(got, want) {
t.Errorf("%s: markup text %q lacks %q", tt.name, got, want)
}
}
}
for _, bad := range tt.bad {
if strings.Contains(got, bad) {
t.Errorf("%s: markup text %q still contains %q", tt.name, got, bad)
}
}
}
}
func TestUnsupportedAndTooLarge(t *testing.T) {
e := newWithPath("")
png := []byte("\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR")
if _, err := text(t, e, file(t, "image.png", png), 0); !errors.Is(err, ErrUnsupported) {
t.Errorf("png: %v, want ErrUnsupported", err)
}
if _, err := text(t, e, file(t, "blob.bin", []byte{1, 2, 0, 3}), 0); !errors.Is(err, ErrUnsupported) {
t.Errorf("binary: %v, want ErrUnsupported", err)
}
if _, err := text(t, e, file(t, "big.txt", []byte(strings.Repeat("x", 100))), 50); !errors.Is(err, ErrTooLarge) {
t.Errorf("big: %v, want ErrTooLarge", err)
}
}
func TestToolMissingError(t *testing.T) {
err := &ToolMissingError{Tool: "pdftotext"}
if err.Error() != "needs pdftotext, not installed" {
t.Fatalf("got %q", err.Error())
}
}
func TestToolsListedInOrder(t *testing.T) {
var names []string
for _, tl := range newWithPath("").Tools() {
names = append(names, tl.Name)
if tl.Path != "" {
t.Errorf("%s found with an empty PATH", tl.Name)
}
}
if strings.Join(names, " ") != "pdftotext antiword catdoc xls2csv catppt" {
t.Fatalf("tools = %v", names)
}
}
// TestSniffWholeFileMustBeValid: the first 8 KiB sniffs as plain ASCII text,
// but the file goes on to hold an invalid UTF-8 byte and a NUL past that
// sample — sniffText must reject the whole file, not just decode what the
// sample alone promised (it must not fall back to Latin-1 the way a known
// text extension would). It is not "no text in this format" either: the
// text it began with could hold a keyword, so it is ErrMixed, a read
// failure, and a content exclude fails closed on it (plan 10 re-check R4).
func TestSniffWholeFileMustBeValid(t *testing.T) {
e := newWithPath("")
data := append([]byte(strings.Repeat("x", 8192)), 0xFF, 0x00)
_, err := text(t, e, file(t, "blob.data", data), 0)
if !errors.Is(err, ErrMixed) || errors.Is(err, ErrUnsupported) {
t.Errorf("got %v, want ErrMixed and not ErrUnsupported", err)
}
}
// TestSniffLargeBinaryRejected: an unrecognised-extension file whose very
// first byte is invalid UTF-8 is rejected from the sample alone; this only
// checks the outcome (ErrUnsupported), not that the rest of the megabyte
// went unread — that efficiency claim isn't something a black-box test can
// time reliably.
func TestSniffLargeBinaryRejected(t *testing.T) {
e := newWithPath("")
data := make([]byte, 1<<20) // 1 MiB, far past the 8 KiB sniff window
data[0] = 0xFF // invalid UTF-8 lead byte, visible in the sample
if _, err := text(t, e, file(t, "huge.blob", data), 0); !errors.Is(err, ErrUnsupported) {
t.Errorf("got %v, want ErrUnsupported", err)
}
}
// TestToolLookupSkipsNonRegular: a FIFO named like a tool, executable bits
// and all, must never be picked up — only a regular file counts.
func TestToolLookupSkipsNonRegular(t *testing.T) {
dir := t.TempDir()
fifo := filepath.Join(dir, "pdftotext")
if err := syscall.Mkfifo(fifo, 0o755); err != nil {
t.Skipf("mkfifo not available: %v", err)
}
for _, tl := range newWithPath(dir).Tools() {
if tl.Name == "pdftotext" && tl.Path != "" {
t.Errorf("pdftotext resolved to a non-regular file: %s", tl.Path)
}
}
}
// TestUnterminatedTagKeepsRemainder: D1. An unterminated ordinary tag (no
// closing '>') must not discard the rest of the file - only the malformed
// tag markup itself is unrecoverable; whatever follows it is still real
// content and must still reach the extracted text.
func TestUnterminatedTagKeepsRemainder(t *testing.T) {
e := newWithPath("")
src := "<p>before</p><p unterminated text after"
got, err := text(t, e, file(t, "broken.html", []byte(src)), 0)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(got, "before") {
t.Errorf("text before the unterminated tag missing: %q", got)
}
if !strings.Contains(got, "unterminated text after") {
t.Errorf("text after the unterminated tag was discarded: %q", got)
}
}
// TestSniffRuneStraddlingSampleBoundary: D2. A multi-byte rune ("ż", two
// UTF-8 bytes) placed exactly so its lead byte is the sniff sample's last
// byte and its continuation byte falls just past it must not make an
// otherwise valid UTF-8 file sniff as unsupported.
func TestSniffRuneStraddlingSampleBoundary(t *testing.T) {
e := newWithPath("")
prefix := strings.Repeat("a", sniffSize-1)
data := []byte(prefix + "ż" + "bcd")
got, err := text(t, e, file(t, "straddle.blob", data), 0)
if err != nil {
t.Fatalf("valid UTF-8 with a rune straddling the sniff boundary: %v", err)
}
if want := "żbcd"; !strings.HasSuffix(got, want) {
t.Errorf("got tail %q, want it to end in %q", got[len(got)-8:], want)
}
}
|
