krino: matching — scan, ignore, conditions, extraction, duplicates, explain, dry run

1 files changed, 224 insertions, 0 deletions

diff --git a/internal/extract/zipxml.go b/internal/extract/zipxml.go
new file mode 100644
index 0000000..1b3ba82
--- /dev/null
+++ b/internal/extract/zipxml.go
@@ -0,0 +1,224 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package extract
+
+import (
+	"archive/zip"
+	"context"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"path"
+	"strings"
+)
+
+// zipBudget caps the total uncompressed bytes read from one archive's
+// matched entries, guarding against a zip bomb; a test lowers it. B1: the
+// directory's max-read may cap a single archive further still — see
+// budget (tools.go).
+var zipBudget int64 = 64 << 20
+
+// zipPatterns maps a zip-based format's extension to the path.Match
+// patterns (tried in matchEntry) of the entries that carry its text.
+// Invoices often carry the tax number only in a header or footer, which is
+// why docx's header*/footer* entries are included; ODF keeps headers and
+// footers in styles.xml, not content.xml, so both are read.
+var zipPatterns = map[string][]string{
+	"docx": {"word/document.xml", "word/header*.xml", "word/footer*.xml", "word/footnotes.xml"},
+	"xlsx": {"xl/sharedStrings.xml", "xl/worksheets/sheet*.xml"},
+	"pptx": {"ppt/slides/slide*.xml"},
+	"odt":  {"content.xml", "styles.xml"},
+	"ods":  {"content.xml", "styles.xml"},
+	"odp":  {"content.xml", "styles.xml"},
+	"epub": {"*.xhtml", "*.html", "*.htm"},
+}
+
+// zipText extracts text from a zip-based document (docx xlsx pptx odt ods
+// odp epub): it opens the archive and, for each entry matching the
+// format's zipPatterns in archive order, decodes its XML character data
+// into the result with xmlText, separating entries with a newline. A
+// corrupt archive returns the zip package's own error, not
+// ErrUnsupported: the file claims a format it does not have, which the
+// user should see. Reading stops with ErrTooLarge immediately once the
+// entries read from the archive exceed budget(zipBudget, maxRead)
+// uncompressed bytes in total (B1: maxRead, the directory's configured
+// ceiling, may cap this lower than the fixed zipBudget). ctx is checked
+// between entries so a cancelled extraction stops
+// promptly.
+//
+// Any other per-entry failure — the entry won't open, or its XML is
+// malformed — is lenient rather than fatal: whatever text that entry had
+// already yielded (xmlText writes as it walks, so a syntax error partway
+// through still leaves the text read up to that point) is kept, and the
+// archive keeps going to its remaining entries, since one bad part (a
+// corrupt header, say) should not blank out a document's otherwise
+// readable body. The first such error is remembered, wrapped as "<entry
+// name>: <err>", and returned only if no matched entry ever wrote any
+// character data at all — an error report is more useful than silent
+// empty text when nothing could be read. "Wrote any character data" is
+// tracked per entry (via the builder's length just before and after that
+// entry's own xmlText call, not the whole archive's final length): the
+// newline zipText adds to separate a successful entry from the next one
+// would otherwise make an entry that parsed cleanly but held no text of
+// its own (an empty element, say) look like it had produced something,
+// which could then mask a later entry's genuine failure.
+func zipText(ctx context.Context, path, ext string, maxRead int64) (string, error) {
+	zr, err := zip.OpenReader(path)
+	if err != nil {
+		return "", err
+	}
+	defer zr.Close()
+
+	patterns := zipPatterns[ext]
+	var b strings.Builder
+	remaining := budget(zipBudget, maxRead)
+	var firstErr error
+	wroteText := false
+	for _, f := range zr.File {
+		if err := ctx.Err(); err != nil {
+			return "", err
+		}
+		if !matchEntry(patterns, f.Name) {
+			continue
+		}
+		before := b.Len()
+		err := readZipEntry(f, &remaining, &b)
+		// Measured before the separator below is written, so a
+		// separator alone (an entry that parsed but held no character
+		// data) never counts as "wrote text" — only xmlText's own
+		// writes do, whether or not this entry went on to error.
+		if b.Len() > before {
+			wroteText = true
+		}
+		if err != nil {
+			if errors.Is(err, ErrTooLarge) {
+				return "", ErrTooLarge
+			}
+			if firstErr == nil {
+				firstErr = fmt.Errorf("%s: %w", f.Name, err)
+			}
+			continue
+		}
+		b.WriteByte('\n')
+	}
+	if !wroteText && firstErr != nil {
+		return "", firstErr
+	}
+	return b.String(), nil
+}
+
+// matchEntry reports whether name is one of the entries a format reads:
+// each pattern is tried first against the full entry name — which is what
+// the docx/xlsx/pptx/odt directory-qualified patterns need — and, failing
+// that, against name's base name. The base-name fallback is what lets
+// epub's bare "*.xhtml"/"*.html"/"*.htm" find chapters nested at any depth
+// inside the archive; applied to every format, it also means a nested
+// part sharing a matched base name is picked up deliberately, not by
+// accident — e.g. an ODF embedded object's own "Object 1/content.xml"
+// matches odt/ods/odp's bare "content.xml" pattern alongside the
+// document's own content.xml, because an embedded chart's or formula's
+// text is text the document shows its reader.
+func matchEntry(patterns []string, name string) bool {
+	base := path.Base(name)
+	for _, p := range patterns {
+		if ok, _ := path.Match(p, name); ok {
+			return true
+		}
+		if ok, _ := path.Match(p, base); ok {
+			return true
+		}
+	}
+	return false
+}
+
+// readZipEntry opens one matched zip entry, decodes its text into b
+// through a budgetedReader sharing remaining across the whole archive, and
+// reports ErrTooLarge if that budget was exceeded — checked on the reader
+// itself after xmlText returns, since the XML decoder may not pass the
+// reader's own error through unchanged (a truncated entry can look like a
+// cleanly finished document).
+func readZipEntry(f *zip.File, remaining *int64, b *strings.Builder) error {
+	rc, err := f.Open()
+	if err != nil {
+		return err
+	}
+	defer rc.Close()
+
+	br := &budgetedReader{r: rc, remaining: remaining}
+	err = xmlText(br, b)
+	if br.exceeded {
+		return ErrTooLarge
+	}
+	return err
+}
+
+// budgetedReader wraps a zip entry's reader, decrementing remaining — a
+// counter shared across every entry read from one archive — as bytes are
+// read. Once remaining is exhausted it stops reading and reports io.EOF
+// instead, recording that in exceeded so the caller can tell a genuine
+// end of document from a budget cutoff.
+type budgetedReader struct {
+	r         io.Reader
+	remaining *int64
+	exceeded  bool
+}
+
+func (br *budgetedReader) Read(p []byte) (int, error) {
+	if *br.remaining <= 0 {
+		br.exceeded = true
+		return 0, io.EOF
+	}
+	if int64(len(p)) > *br.remaining {
+		p = p[:*br.remaining]
+	}
+	n, err := br.r.Read(p)
+	*br.remaining -= int64(n)
+	return n, err
+}
+
+// xmlText appends the character data of one XML entry to b, with the
+// separators described above. inSharedCell tracks xlsx <c t="s">.
+func xmlText(r io.Reader, b *strings.Builder) error {
+	dec := xml.NewDecoder(r)
+	dec.Strict = false
+	dec.Entity = xml.HTMLEntity
+	sharedCell, inV := false, false
+	for {
+		tok, err := dec.Token()
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+		switch t := tok.(type) {
+		case xml.StartElement:
+			switch t.Name.Local {
+			case "s", "tab", "br", "line-break", "cr":
+				b.WriteByte(' ')
+			case "c":
+				sharedCell = false
+				for _, a := range t.Attr {
+					if a.Name.Local == "t" && a.Value == "s" {
+						sharedCell = true
+					}
+				}
+			case "v":
+				inV = true
+			}
+		case xml.EndElement:
+			switch t.Name.Local {
+			case "p", "h", "tc", "tr", "td", "th", "li", "si", "c", "row", "div", "title":
+				b.WriteByte('\n')
+			case "v":
+				inV = false
+			}
+		case xml.CharData:
+			if inV && sharedCell {
+				continue
+			}
+			b.Write(t)
+		}
+	}
+}