krino: matching — scan, ignore, conditions, extraction, duplicates, explain, dry run

1 files changed, 282 insertions, 0 deletions

diff --git a/internal/extract/plain.go b/internal/extract/plain.go
new file mode 100644
index 0000000..d245a41
--- /dev/null
+++ b/internal/extract/plain.go
@@ -0,0 +1,282 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package extract
+
+import (
+	"bytes"
+	"encoding/binary"
+	"html"
+	"io"
+	"os"
+	"strings"
+	"unicode/utf16"
+	"unicode/utf8"
+)
+
+// sniffSize is how much of an unknown-extension file is inspected to guess
+// whether it is text (design.md §6).
+const sniffSize = 8192
+
+// readDecoded reads path whole and decodes it per the encoding rules: a
+// leading UTF-8 BOM is stripped and the rest used as is; a UTF-16 LE or BE
+// BOM is decoded with unicode/utf16; otherwise valid UTF-8 is used as is,
+// and any other invalid UTF-8 is decoded one byte per Latin-1 code point.
+func readDecoded(path string) (string, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return "", err
+	}
+	return decode(data), nil
+}
+
+// sniffText decides whether an unknown extension is text, reading at most
+// sniffSize bytes before deciding: a file whose first sniffSize bytes are
+// not a UTF-16 BOM and not valid-UTF8-with-no-NUL is rejected as
+// ErrUnsupported without reading any further (so a large binary file is
+// never read in full just to be rejected). Only once that sample passes is
+// the rest of the file read; a UTF-16 BOM is trusted as plain text from
+// the sample alone (UTF-16 text is full of NUL bytes by design), but a
+// sample that merely looks like UTF-8 must hold for the WHOLE file — no
+// NUL byte anywhere, and no invalid UTF-8 anywhere past the sample — or
+// the file is ErrUnsupported after all; the Latin-1 fallback in decode
+// never applies to a sniffed file, only to a file whose extension already
+// names it as text. D2: when the file continues past the sample (n ==
+// sniffSize), the validity check is run against a trimmed copy with any
+// incomplete trailing rune removed, so a multi-byte rune that happens to
+// straddle byte sniffSize does not make an otherwise-valid file sniff as
+// unsupported; sample itself, used below to build the returned text, is
+// left untouched — the rest of the file (read after the check) supplies
+// the bytes trimming set aside.
+func sniffText(path string) (string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	sample := make([]byte, sniffSize)
+	n, err := io.ReadFull(f, sample)
+	if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
+		return "", err
+	}
+	sample = sample[:n]
+
+	check := sample
+	if n == sniffSize {
+		check = trimIncompleteTrailingRune(sample)
+	}
+
+	utf16BOM := hasUTF16BOM(sample)
+	if !utf16BOM && !(utf8.Valid(check) && !bytes.Contains(check, []byte{0})) {
+		return "", ErrUnsupported
+	}
+
+	rest, err := io.ReadAll(f)
+	if err != nil {
+		return "", err
+	}
+	data := append(sample, rest...)
+
+	if utf16BOM {
+		return decode(data), nil
+	}
+	if !utf8.Valid(data) || bytes.Contains(data, []byte{0}) {
+		return "", ErrUnsupported
+	}
+	return decode(data), nil
+}
+
+// trimIncompleteTrailingRune drops an incomplete UTF-8 sequence left
+// dangling at the very end of b — D2's fix for a rune cut off exactly at
+// the sniff sample's boundary. It looks back at most utf8.UTFMax-1 bytes
+// for the start of the trailing rune; if the bytes from there to the end
+// are not a complete encoding (utf8.FullRune), that partial rune is cut,
+// since more bytes to finish it may simply not have been read yet. A
+// sample already ending cleanly (the common case, and every all-ASCII
+// sample) is returned unchanged.
+func trimIncompleteTrailingRune(b []byte) []byte {
+	end := len(b)
+	start := end - 1
+	for start >= 0 && end-start < utf8.UTFMax && !utf8.RuneStart(b[start]) {
+		start--
+	}
+	if start < 0 || utf8.FullRune(b[start:end]) {
+		return b
+	}
+	return b[:start]
+}
+
+// hasUTF16BOM reports whether b begins with a UTF-16 little- or big-endian
+// byte-order mark.
+func hasUTF16BOM(b []byte) bool {
+	return len(b) >= 2 && ((b[0] == 0xFF && b[1] == 0xFE) || (b[0] == 0xFE && b[1] == 0xFF))
+}
+
+// decode applies the encoding rules to a whole file's bytes.
+func decode(data []byte) string {
+	switch {
+	case len(data) >= 2 && data[0] == 0xFF && data[1] == 0xFE:
+		return decodeUTF16(data[2:], binary.LittleEndian)
+	case len(data) >= 2 && data[0] == 0xFE && data[1] == 0xFF:
+		return decodeUTF16(data[2:], binary.BigEndian)
+	case len(data) >= 3 && data[0] == 0xEF && data[1] == 0xBB && data[2] == 0xBF:
+		return string(data[3:])
+	case utf8.Valid(data):
+		return string(data)
+	default:
+		return decodeLatin1(data)
+	}
+}
+
+// decodeUTF16 decodes b (already past the BOM) as UTF-16 in the given byte
+// order; a trailing odd byte with no pair is dropped.
+func decodeUTF16(b []byte, order binary.ByteOrder) string {
+	n := len(b) / 2
+	units := make([]uint16, n)
+	for i := 0; i < n; i++ {
+		units[i] = order.Uint16(b[i*2 : i*2+2])
+	}
+	return string(utf16.Decode(units))
+}
+
+// decodeLatin1 decodes b as Latin-1: each byte is its own Unicode code
+// point.
+func decodeLatin1(b []byte) string {
+	r := make([]rune, len(b))
+	for i, c := range b {
+		r[i] = rune(c)
+	}
+	return string(r)
+}
+
+// stripMarkup turns decoded HTML/XML/SVG text into plain text: a small
+// scanner replaces every <...> tag with a space, dropping the contents of
+// <script> and <style> along with their tags and skipping <!-- ... -->
+// comments outright, then entities are decoded with html.UnescapeString.
+// html.UnescapeString turns &nbsp; into U+00A0 (a non-breaking space, not
+// a plain space); since the source markup used it as ordinary inter-word
+// spacing, it is folded to a regular space here too.
+//
+// A '<' only starts a tag when followed by a letter, '/', '!' or '?' —
+// HTML's own rule for what can open a tag, close tag, comment/doctype, or
+// processing instruction. Anything else (a digit, a space, end of string)
+// is literal text, so "a < b" is not mistaken for markup.
+func stripMarkup(s string) string {
+	var b strings.Builder
+	b.Grow(len(s))
+	i, n := 0, len(s)
+	for i < n {
+		if s[i] != '<' || !startsTag(s, i) {
+			b.WriteByte(s[i])
+			i++
+			continue
+		}
+
+		if strings.HasPrefix(s[i:], "<!--") {
+			end := n
+			if k := strings.Index(s[i+4:], "-->"); k != -1 {
+				end = i + 4 + k + len("-->")
+			}
+			b.WriteByte(' ')
+			i = end
+			continue
+		}
+
+		j := i + 1
+		closing := false
+		if j < n && s[j] == '/' {
+			closing = true
+			j++
+		}
+		nameStart := j
+		for j < n && isTagNameByte(s[j]) {
+			j++
+		}
+		name := strings.ToLower(s[nameStart:j])
+
+		gt := strings.IndexByte(s[j:], '>')
+		if gt == -1 {
+			// D1: an unterminated tag (no closing '>') can no longer be
+			// parsed as markup, but that is no reason to discard the rest
+			// of the file - copy it through as literal text instead of
+			// simply stopping the scan.
+			b.WriteString(s[i:])
+			break
+		}
+		end := j + gt + 1
+
+		if !closing && (name == "script" || name == "style") {
+			if close := indexCloseTag(s[end:], name); close != -1 {
+				end += close
+				if gt2 := strings.IndexByte(s[end:], '>'); gt2 != -1 {
+					end += gt2 + 1
+				} else {
+					end = n
+				}
+			} else {
+				end = n
+			}
+		}
+
+		b.WriteByte(' ')
+		i = end
+	}
+
+	return strings.ReplaceAll(html.UnescapeString(b.String()), string(nbsp), " ")
+}
+
+// startsTag reports whether s[i] == '<' begins a tag-like construct: the
+// next character is a letter, '/', '!' or '?'. A '<' at the very end of s,
+// or followed by anything else (digit, space, punctuation), is literal
+// text instead.
+func startsTag(s string, i int) bool {
+	if i+1 >= len(s) {
+		return false
+	}
+	c := s[i+1]
+	return c == '/' || c == '!' || c == '?' ||
+		(c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
+}
+
+// nbsp is U+00A0, NO-BREAK SPACE: what html.UnescapeString decodes &nbsp;
+// to, folded to a regular space since the source markup used it as one.
+const nbsp = rune(0xA0)
+
+// isTagNameByte reports whether c can appear in an HTML/XML tag name.
+func isTagNameByte(c byte) bool {
+	return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '-' || c == ':' || c == '_'
+}
+
+// indexCloseTag returns the index in s of the first ASCII case-insensitive
+// occurrence of "</name", or -1.
+func indexCloseTag(s, name string) int {
+	target := "</" + name
+	tn := len(target)
+	for i := 0; i+tn <= len(s); i++ {
+		if asciiEqualFold(s[i:i+tn], target) {
+			return i
+		}
+	}
+	return -1
+}
+
+// asciiEqualFold reports whether a and b are equal, ASCII letters compared
+// without regard to case.
+func asciiEqualFold(a, b string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := 0; i < len(a); i++ {
+		ca, cb := a[i], b[i]
+		if 'A' <= ca && ca <= 'Z' {
+			ca += 'a' - 'A'
+		}
+		if 'A' <= cb && cb <= 'Z' {
+			cb += 'a' - 'A'
+		}
+		if ca != cb {
+			return false
+		}
+	}
+	return true
+}