From 3b36a48b7ce5a53a9366f3b31f94311f178e2553 Mon Sep 17 00:00:00 2001 From: Lukasz Kasprzak Date: Sat, 12 Sep 2026 01:22:12 +0200 Subject: krino: matching — scan, ignore, conditions, extraction, duplicates, explain, dry run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/extract/extract.go | 154 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 internal/extract/extract.go (limited to 'internal/extract/extract.go') diff --git a/internal/extract/extract.go b/internal/extract/extract.go new file mode 100644 index 0000000..aa3ae93 --- /dev/null +++ b/internal/extract/extract.go @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +// Package extract gets text out of files so rules can test their content. +// Text is returned raw; callers normalise it with norm.Text. +package extract + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "time" +) + +var ( + // ErrUnsupported is returned when the format carries no text krino + // knows how to extract. + ErrUnsupported = errors.New("no text in this format") + // ErrTooLarge is returned when the file is larger than the configured + // max-read; nothing is read in that case. + ErrTooLarge = errors.New("larger than max-read") +) + +// ToolMissingError is returned when a format needs an external tool that +// was not found on this system. +type ToolMissingError struct{ Tool string } + +func (e *ToolMissingError) Error() string { + return "needs " + e.Tool + ", not installed" +} + +// Tool is one external extractor and the absolute path it was found at, or +// "" if it was not found. +type Tool struct{ Name, Path string } + +// toolNames lists the external tools Extractor looks up, in the order +// Tools() reports them. +var toolNames = []string{"pdftotext", "antiword", "catdoc", "xls2csv", "catppt"} + +// markupExt is the set of markup extensions: tags stripped, entities +// decoded. +var markupExt = map[string]bool{ + "html": true, "htm": true, "xhtml": true, "xml": true, "svg": true, +} + +// plainExt is the set of extensions read as plain text without further +// inspection. +var plainExt = map[string]bool{ + "txt": true, "md": true, "log": true, "csv": true, "tsv": true, + "json": true, "yaml": true, "yml": true, "toml": true, "ini": true, + "conf": true, "cfg": true, "rtf": true, "tex": true, "go": true, + "c": true, "h": true, "cpp": true, "hpp": true, "py": true, "sh": true, + "js": true, "ts": true, "rs": true, "java": true, "rb": true, + "pl": true, "lua": true, "css": true, "sql": true, +} + +// zipExt is the set of Office/OpenDocument/ebook formats: a zip container +// plus XML inside it (Task 5). +var zipExt = map[string]bool{ + "docx": true, "xlsx": true, "pptx": true, + "odt": true, "ods": true, "odp": true, + "epub": true, +} + +// toolExt maps an extension to the external tool it needs (Task 6). pdf is +// handled separately, since it has its own fixed command line. +var toolExt = map[string]string{ + "doc": "antiword", // falls back to catdoc + "xls": "xls2csv", + "ppt": "catppt", +} + +// Extractor gets text out of files, using the external tools it found at +// construction. +type Extractor struct { + tools map[string]string // name -> absolute path, only tools found + Timeout time.Duration // per external tool run +} + +// New builds an Extractor, looking up each external tool once in the +// process's $PATH, with a 30 s per-tool Timeout. +func New() *Extractor { + return newWithPath(os.Getenv("PATH")) +} + +// newWithPath builds an Extractor looking up tools in the given PATH-style +// list instead of the process environment, so tests control what is found. +// It does not use exec.LookPath, which reads the process's own PATH; it +// walks path itself. +func newWithPath(path string) *Extractor { + dirs := filepath.SplitList(path) + tools := make(map[string]string, len(toolNames)) + for _, name := range toolNames { + for _, dir := range dirs { + if dir == "" { + continue + } + p := filepath.Join(dir, name) + info, err := os.Stat(p) + if err != nil || !info.Mode().IsRegular() { + continue // no such file, or a directory/FIFO/socket/device + } + if info.Mode()&0o111 == 0 { + continue // not executable + } + tools[name] = p + break + } + } + return &Extractor{tools: tools, Timeout: 30 * time.Second} +} + +// Tools reports every external tool Extractor knows about, in a fixed +// order, with the path it was found at or "" if it was not found. +func (e *Extractor) Tools() []Tool { + out := make([]Tool, len(toolNames)) + for i, name := range toolNames { + out[i] = Tool{Name: name, Path: e.tools[name]} + } + return out +} + +// Text returns path's raw text content: it does not normalise (callers use +// norm.Text). size is the file's size, as already known to the caller; +// maxRead is the configured ceiling, 0 meaning unlimited. Dispatch is on +// the lower-cased extension; a format extension with no reader yet +// implemented returns ErrUnsupported. +func (e *Extractor) Text(ctx context.Context, path string, size, maxRead int64) (string, error) { + if maxRead > 0 && size > maxRead { + return "", ErrTooLarge + } + + ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(path), ".")) + + switch { + case ext == "pdf": + return e.pdfText(ctx, path, maxRead) + case zipExt[ext]: + return zipText(ctx, path, ext, maxRead) + case toolExt[ext] != "": + return e.legacyText(ctx, path, ext, maxRead) + case markupExt[ext]: + raw, err := readDecoded(path) + if err != nil { + return "", err + } + return stripMarkup(raw), nil + case plainExt[ext]: + return readDecoded(path) + default: + return sniffText(path) + } +} -- cgit v1.3