1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
|
// SPDX-License-Identifier: GPL-3.0-or-later
// Package extract gets text out of files so rules can test their content.
// Text is returned raw; callers normalise it with norm.Text.
package extract
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"time"
)
// Version is the version of the text this package extracts. Bump it
// whenever a change could make any format's text differ, so every keyword
// cache built from the old text is discarded (Fingerprint).
const Version = 2
var (
// ErrUnsupported is returned when the format carries no text krino
// knows how to extract.
ErrUnsupported = errors.New("no text in this format")
// ErrTooLarge is returned when the file is larger than the configured
// max-read; nothing is read in that case.
ErrTooLarge = errors.New("larger than max-read")
)
// ToolMissingError is returned when a format needs an external tool that
// was not found on this system.
type ToolMissingError struct{ Tool string }
func (e *ToolMissingError) Error() string {
return "needs " + e.Tool + ", not installed"
}
// Tool is one external extractor and the absolute path it was found at, or
// "" if it was not found.
type Tool struct{ Name, Path string }
// toolNames lists the external tools Extractor looks up, in the order
// Tools() reports them.
var toolNames = []string{"pdftotext", "antiword", "catdoc", "xls2csv", "catppt"}
// markupExt is the set of markup extensions: tags stripped, entities
// decoded.
var markupExt = map[string]bool{
"html": true, "htm": true, "xhtml": true, "xml": true, "svg": true,
}
// plainExt is the set of extensions read as plain text without further
// inspection.
var plainExt = map[string]bool{
"txt": true, "md": true, "log": true, "csv": true, "tsv": true,
"json": true, "yaml": true, "yml": true, "toml": true, "ini": true,
"conf": true, "cfg": true, "rtf": true, "tex": true, "go": true,
"c": true, "h": true, "cpp": true, "hpp": true, "py": true, "sh": true,
"js": true, "ts": true, "rs": true, "java": true, "rb": true,
"pl": true, "lua": true, "css": true, "sql": true,
}
// zipExt is the set of Office/OpenDocument/ebook formats: a zip container
// plus XML inside it (Task 5).
var zipExt = map[string]bool{
"docx": true, "xlsx": true, "pptx": true,
"odt": true, "ods": true, "odp": true,
"epub": true,
}
// toolExt maps an extension to the external tool it needs (Task 6). pdf is
// handled separately, since it has its own fixed command line.
var toolExt = map[string]string{
"doc": "antiword", // falls back to catdoc
"xls": "xls2csv",
"ppt": "catppt",
}
// Extractor gets text out of files, using the external tools it found at
// construction.
type Extractor struct {
tools map[string]string // name -> absolute path, only tools found
Timeout time.Duration // per external tool run
}
// New builds an Extractor, looking up each external tool once in the
// process's $PATH, with a 30 s per-tool Timeout.
func New() *Extractor {
return newWithPath(os.Getenv("PATH"))
}
// newWithPath builds an Extractor looking up tools in the given PATH-style
// list instead of the process environment, so tests control what is found.
// It does not use exec.LookPath, which reads the process's own PATH; it
// walks path itself.
func newWithPath(path string) *Extractor {
dirs := filepath.SplitList(path)
tools := make(map[string]string, len(toolNames))
for _, name := range toolNames {
for _, dir := range dirs {
// A relative entry would find a tool relative to the working
// directory - a bin/pdftotext an unpacked download left behind
// (review planapply F7) - so only absolute entries count.
if dir == "" || !filepath.IsAbs(dir) {
continue
}
p := filepath.Join(dir, name)
info, err := os.Stat(p)
if err != nil || !info.Mode().IsRegular() {
continue // no such file, or a directory/FIFO/socket/device
}
if info.Mode()&0o111 == 0 {
continue // not executable
}
tools[name] = p
break
}
}
return &Extractor{tools: tools, Timeout: 30 * time.Second}
}
// Fingerprint identifies what text this Extractor would produce: Version,
// and each external tool found with its path, size and modification time.
// A keyword cache built under another fingerprint is discarded, so
// installing, removing or upgrading a tool invalidates it.
func (e *Extractor) Fingerprint() string {
var b strings.Builder
fmt.Fprintf(&b, "v%d", Version)
for _, name := range toolNames {
p := e.tools[name]
if p == "" {
continue
}
fmt.Fprintf(&b, " %s=%s", name, p)
if fi, err := os.Stat(p); err == nil {
fmt.Fprintf(&b, ":%d:%d", fi.Size(), fi.ModTime().UnixNano())
}
}
return b.String()
}
// Tools reports every external tool Extractor knows about, in a fixed
// order, with the path it was found at or "" if it was not found.
func (e *Extractor) Tools() []Tool {
out := make([]Tool, len(toolNames))
for i, name := range toolNames {
out[i] = Tool{Name: name, Path: e.tools[name]}
}
return out
}
// Text returns path's raw text content: it does not normalise (callers use
// norm.Text). size is the file's size, as already known to the caller;
// maxRead is the configured ceiling, 0 meaning unlimited. Dispatch is on
// the lower-cased extension; a format extension with no reader yet
// implemented returns ErrUnsupported.
func (e *Extractor) Text(ctx context.Context, path string, size, maxRead int64) (string, error) {
if maxRead > 0 && size > maxRead {
return "", ErrTooLarge
}
ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(path), "."))
switch {
case ext == "pdf":
return e.pdfText(ctx, path, maxRead)
case zipExt[ext]:
return zipText(ctx, path, ext, maxRead)
case toolExt[ext] != "":
return e.legacyText(ctx, path, ext, maxRead)
case markupExt[ext]:
raw, err := readDecoded(path)
if err != nil {
return "", err
}
return stripMarkup(raw), nil
case plainExt[ext]:
return readDecoded(path)
default:
return sniffText(path)
}
}
|
