1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
|
// SPDX-License-Identifier: GPL-3.0-or-later
// Package extract gets text out of files so rules can test their content.
// Text is returned raw; callers normalise it with norm.Text.
package extract
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"time"
)
var (
// ErrUnsupported is returned when the format carries no text krino
// knows how to extract.
ErrUnsupported = errors.New("no text in this format")
// ErrTooLarge is returned when the file is larger than the configured
// max-read; nothing is read in that case.
ErrTooLarge = errors.New("larger than max-read")
)
// ToolMissingError is returned when a format needs an external tool that
// was not found on this system.
type ToolMissingError struct{ Tool string }
func (e *ToolMissingError) Error() string {
return "needs " + e.Tool + ", not installed"
}
// Tool is one external extractor and the absolute path it was found at, or
// "" if it was not found.
type Tool struct{ Name, Path string }
// toolNames lists the external tools Extractor looks up, in the order
// Tools() reports them.
var toolNames = []string{"pdftotext", "antiword", "catdoc", "xls2csv", "catppt"}
// markupExt is the set of markup extensions: tags stripped, entities
// decoded.
var markupExt = map[string]bool{
"html": true, "htm": true, "xhtml": true, "xml": true, "svg": true,
}
// plainExt is the set of extensions read as plain text without further
// inspection.
var plainExt = map[string]bool{
"txt": true, "md": true, "log": true, "csv": true, "tsv": true,
"json": true, "yaml": true, "yml": true, "toml": true, "ini": true,
"conf": true, "cfg": true, "rtf": true, "tex": true, "go": true,
"c": true, "h": true, "cpp": true, "hpp": true, "py": true, "sh": true,
"js": true, "ts": true, "rs": true, "java": true, "rb": true,
"pl": true, "lua": true, "css": true, "sql": true,
}
// zipExt is the set of Office/OpenDocument/ebook formats: a zip container
// plus XML inside it (Task 5).
var zipExt = map[string]bool{
"docx": true, "xlsx": true, "pptx": true,
"odt": true, "ods": true, "odp": true,
"epub": true,
}
// toolExt maps an extension to the external tool it needs (Task 6). pdf is
// handled separately, since it has its own fixed command line.
var toolExt = map[string]string{
"doc": "antiword", // falls back to catdoc
"xls": "xls2csv",
"ppt": "catppt",
}
// Extractor gets text out of files, using the external tools it found at
// construction.
type Extractor struct {
tools map[string]string // name -> absolute path, only tools found
Timeout time.Duration // per external tool run
}
// New builds an Extractor, looking up each external tool once in the
// process's $PATH, with a 30 s per-tool Timeout.
func New() *Extractor {
return newWithPath(os.Getenv("PATH"))
}
// newWithPath builds an Extractor looking up tools in the given PATH-style
// list instead of the process environment, so tests control what is found.
// It does not use exec.LookPath, which reads the process's own PATH; it
// walks path itself.
func newWithPath(path string) *Extractor {
dirs := filepath.SplitList(path)
tools := make(map[string]string, len(toolNames))
for _, name := range toolNames {
for _, dir := range dirs {
if dir == "" {
continue
}
p := filepath.Join(dir, name)
info, err := os.Stat(p)
if err != nil || !info.Mode().IsRegular() {
continue // no such file, or a directory/FIFO/socket/device
}
if info.Mode()&0o111 == 0 {
continue // not executable
}
tools[name] = p
break
}
}
return &Extractor{tools: tools, Timeout: 30 * time.Second}
}
// Tools reports every external tool Extractor knows about, in a fixed
// order, with the path it was found at or "" if it was not found.
func (e *Extractor) Tools() []Tool {
out := make([]Tool, len(toolNames))
for i, name := range toolNames {
out[i] = Tool{Name: name, Path: e.tools[name]}
}
return out
}
// Text returns path's raw text content: it does not normalise (callers use
// norm.Text). size is the file's size, as already known to the caller;
// maxRead is the configured ceiling, 0 meaning unlimited. Dispatch is on
// the lower-cased extension; a format extension with no reader yet
// implemented returns ErrUnsupported.
func (e *Extractor) Text(ctx context.Context, path string, size, maxRead int64) (string, error) {
if maxRead > 0 && size > maxRead {
return "", ErrTooLarge
}
ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(path), "."))
switch {
case ext == "pdf":
return e.pdfText(ctx, path, maxRead)
case zipExt[ext]:
return zipText(ctx, path, ext, maxRead)
case toolExt[ext] != "":
return e.legacyText(ctx, path, ext, maxRead)
case markupExt[ext]:
raw, err := readDecoded(path)
if err != nil {
return "", err
}
return stripMarkup(raw), nil
case plainExt[ext]:
return readDecoded(path)
default:
return sniffText(path)
}
}
|
