krino: matching — scan, ignore, conditions, extraction, duplicates, explain, dry run

1 files changed, 154 insertions, 0 deletions

diff --git a/internal/extract/extract.go b/internal/extract/extract.go
new file mode 100644
index 0000000..aa3ae93
--- /dev/null
+++ b/internal/extract/extract.go
@@ -0,0 +1,154 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+// Package extract gets text out of files so rules can test their content.
+// Text is returned raw; callers normalise it with norm.Text.
+package extract
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+var (
+	// ErrUnsupported is returned when the format carries no text krino
+	// knows how to extract.
+	ErrUnsupported = errors.New("no text in this format")
+	// ErrTooLarge is returned when the file is larger than the configured
+	// max-read; nothing is read in that case.
+	ErrTooLarge = errors.New("larger than max-read")
+)
+
+// ToolMissingError is returned when a format needs an external tool that
+// was not found on this system.
+type ToolMissingError struct{ Tool string }
+
+func (e *ToolMissingError) Error() string {
+	return "needs " + e.Tool + ", not installed"
+}
+
+// Tool is one external extractor and the absolute path it was found at, or
+// "" if it was not found.
+type Tool struct{ Name, Path string }
+
+// toolNames lists the external tools Extractor looks up, in the order
+// Tools() reports them.
+var toolNames = []string{"pdftotext", "antiword", "catdoc", "xls2csv", "catppt"}
+
+// markupExt is the set of markup extensions: tags stripped, entities
+// decoded.
+var markupExt = map[string]bool{
+	"html": true, "htm": true, "xhtml": true, "xml": true, "svg": true,
+}
+
+// plainExt is the set of extensions read as plain text without further
+// inspection.
+var plainExt = map[string]bool{
+	"txt": true, "md": true, "log": true, "csv": true, "tsv": true,
+	"json": true, "yaml": true, "yml": true, "toml": true, "ini": true,
+	"conf": true, "cfg": true, "rtf": true, "tex": true, "go": true,
+	"c": true, "h": true, "cpp": true, "hpp": true, "py": true, "sh": true,
+	"js": true, "ts": true, "rs": true, "java": true, "rb": true,
+	"pl": true, "lua": true, "css": true, "sql": true,
+}
+
+// zipExt is the set of Office/OpenDocument/ebook formats: a zip container
+// plus XML inside it (Task 5).
+var zipExt = map[string]bool{
+	"docx": true, "xlsx": true, "pptx": true,
+	"odt": true, "ods": true, "odp": true,
+	"epub": true,
+}
+
+// toolExt maps an extension to the external tool it needs (Task 6). pdf is
+// handled separately, since it has its own fixed command line.
+var toolExt = map[string]string{
+	"doc": "antiword", // falls back to catdoc
+	"xls": "xls2csv",
+	"ppt": "catppt",
+}
+
+// Extractor gets text out of files, using the external tools it found at
+// construction.
+type Extractor struct {
+	tools   map[string]string // name -> absolute path, only tools found
+	Timeout time.Duration     // per external tool run
+}
+
+// New builds an Extractor, looking up each external tool once in the
+// process's $PATH, with a 30 s per-tool Timeout.
+func New() *Extractor {
+	return newWithPath(os.Getenv("PATH"))
+}
+
+// newWithPath builds an Extractor looking up tools in the given PATH-style
+// list instead of the process environment, so tests control what is found.
+// It does not use exec.LookPath, which reads the process's own PATH; it
+// walks path itself.
+func newWithPath(path string) *Extractor {
+	dirs := filepath.SplitList(path)
+	tools := make(map[string]string, len(toolNames))
+	for _, name := range toolNames {
+		for _, dir := range dirs {
+			if dir == "" {
+				continue
+			}
+			p := filepath.Join(dir, name)
+			info, err := os.Stat(p)
+			if err != nil || !info.Mode().IsRegular() {
+				continue // no such file, or a directory/FIFO/socket/device
+			}
+			if info.Mode()&0o111 == 0 {
+				continue // not executable
+			}
+			tools[name] = p
+			break
+		}
+	}
+	return &Extractor{tools: tools, Timeout: 30 * time.Second}
+}
+
+// Tools reports every external tool Extractor knows about, in a fixed
+// order, with the path it was found at or "" if it was not found.
+func (e *Extractor) Tools() []Tool {
+	out := make([]Tool, len(toolNames))
+	for i, name := range toolNames {
+		out[i] = Tool{Name: name, Path: e.tools[name]}
+	}
+	return out
+}
+
+// Text returns path's raw text content: it does not normalise (callers use
+// norm.Text). size is the file's size, as already known to the caller;
+// maxRead is the configured ceiling, 0 meaning unlimited. Dispatch is on
+// the lower-cased extension; a format extension with no reader yet
+// implemented returns ErrUnsupported.
+func (e *Extractor) Text(ctx context.Context, path string, size, maxRead int64) (string, error) {
+	if maxRead > 0 && size > maxRead {
+		return "", ErrTooLarge
+	}
+
+	ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(path), "."))
+
+	switch {
+	case ext == "pdf":
+		return e.pdfText(ctx, path, maxRead)
+	case zipExt[ext]:
+		return zipText(ctx, path, ext, maxRead)
+	case toolExt[ext] != "":
+		return e.legacyText(ctx, path, ext, maxRead)
+	case markupExt[ext]:
+		raw, err := readDecoded(path)
+		if err != nil {
+			return "", err
+		}
+		return stripMarkup(raw), nil
+	case plainExt[ext]:
+		return readDecoded(path)
+	default:
+		return sniffText(path)
+	}
+}