path: root/internal/extract/plain.go

// SPDX-License-Identifier: GPL-3.0-or-later

package extract

import (
	"bytes"
	"encoding/binary"
	"html"
	"io"
	"os"
	"strings"
	"unicode/utf16"
	"unicode/utf8"
)

// sniffSize is how much of an unknown-extension file is inspected to guess
// whether it is text (design.md §6).
const sniffSize = 8192

// readDecoded reads path whole and decodes it per the encoding rules: a
// leading UTF-8 BOM is stripped and the rest used as is; a UTF-16 LE or BE
// BOM is decoded with unicode/utf16; otherwise valid UTF-8 is used as is,
// and any other invalid UTF-8 is decoded one byte per Latin-1 code point.
func readDecoded(path string) (string, error) {
	data, err := os.ReadFile(path)
	if err != nil {
		return "", err
	}
	return decode(data), nil
}

// sniffText decides whether an unknown extension is text, reading at most
// sniffSize bytes before deciding: a file whose first sniffSize bytes are
// not a UTF-16 BOM and not valid-UTF8-with-no-NUL is rejected as
// ErrUnsupported without reading any further (so a large binary file is
// never read in full just to be rejected). Only once that sample passes is
// the rest of the file read; a UTF-16 BOM is trusted as plain text from
// the sample alone (UTF-16 text is full of NUL bytes by design), but a
// sample that merely looks like UTF-8 must hold for the WHOLE file — no
// NUL byte anywhere, and no invalid UTF-8 anywhere past the sample — or
// the file is ErrMixed, unreadable; the Latin-1 fallback in decode
// never applies to a sniffed file, only to a file whose extension already
// names it as text. D2: when the file continues past the sample (n ==
// sniffSize), the validity check is run against a trimmed copy with any
// incomplete trailing rune removed, so a multi-byte rune that happens to
// straddle byte sniffSize does not make an otherwise-valid file sniff as
// unsupported; sample itself, used below to build the returned text, is
// left untouched — the rest of the file (read after the check) supplies
// the bytes trimming set aside.
func sniffText(path string) (string, error) {
	f, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer f.Close()

	sample := make([]byte, sniffSize)
	n, err := io.ReadFull(f, sample)
	if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
		return "", err
	}
	sample = sample[:n]

	check := sample
	if n == sniffSize {
		check = trimIncompleteTrailingRune(sample)
	}

	utf16BOM := hasUTF16BOM(sample)
	if !utf16BOM && !(utf8.Valid(check) && !bytes.Contains(check, []byte{0})) {
		return "", ErrUnsupported
	}

	rest, err := io.ReadAll(f)
	if err != nil {
		return "", err
	}
	data := append(sample, rest...)

	if utf16BOM {
		return decode(data), nil
	}
	if !utf8.Valid(data) || bytes.Contains(data, []byte{0}) {
		return "", ErrMixed
	}
	return decode(data), nil
}

// trimIncompleteTrailingRune drops an incomplete UTF-8 sequence left
// dangling at the very end of b — D2's fix for a rune cut off exactly at
// the sniff sample's boundary. It looks back at most utf8.UTFMax-1 bytes
// for the start of the trailing rune; if the bytes from there to the end
// are not a complete encoding (utf8.FullRune), that partial rune is cut,
// since more bytes to finish it may simply not have been read yet. A
// sample already ending cleanly (the common case, and every all-ASCII
// sample) is returned unchanged.
func trimIncompleteTrailingRune(b []byte) []byte {
	end := len(b)
	start := end - 1
	for start >= 0 && end-start < utf8.UTFMax && !utf8.RuneStart(b[start]) {
		start--
	}
	if start < 0 || utf8.FullRune(b[start:end]) {
		return b
	}
	return b[:start]
}

// hasUTF16BOM reports whether b begins with a UTF-16 little- or big-endian
// byte-order mark.
func hasUTF16BOM(b []byte) bool {
	return len(b) >= 2 && ((b[0] == 0xFF && b[1] == 0xFE) || (b[0] == 0xFE && b[1] == 0xFF))
}

// decode applies the encoding rules to a whole file's bytes.
func decode(data []byte) string {
	switch {
	case len(data) >= 2 && data[0] == 0xFF && data[1] == 0xFE:
		return decodeUTF16(data[2:], binary.LittleEndian)
	case len(data) >= 2 && data[0] == 0xFE && data[1] == 0xFF:
		return decodeUTF16(data[2:], binary.BigEndian)
	case len(data) >= 3 && data[0] == 0xEF && data[1] == 0xBB && data[2] == 0xBF:
		return string(data[3:])
	case utf8.Valid(data):
		return string(data)
	default:
		return decodeLatin1(data)
	}
}

// decodeUTF16 decodes b (already past the BOM) as UTF-16 in the given byte
// order; a trailing odd byte with no pair is dropped.
func decodeUTF16(b []byte, order binary.ByteOrder) string {
	n := len(b) / 2
	units := make([]uint16, n)
	for i := 0; i < n; i++ {
		units[i] = order.Uint16(b[i*2 : i*2+2])
	}
	return string(utf16.Decode(units))
}

// decodeLatin1 decodes b as Latin-1: each byte is its own Unicode code
// point.
func decodeLatin1(b []byte) string {
	r := make([]rune, len(b))
	for i, c := range b {
		r[i] = rune(c)
	}
	return string(r)
}

// stripMarkup turns decoded HTML/XML/SVG text into plain text: a small
// scanner replaces every <...> tag with a space, dropping the contents of
// <script> and <style> along with their tags and skipping <!-- ... -->
// comments outright, then entities are decoded with html.UnescapeString.
// html.UnescapeString turns &nbsp; into U+00A0 (a non-breaking space, not
// a plain space); since the source markup used it as ordinary inter-word
// spacing, it is folded to a regular space here too.
//
// A '<' only starts a tag when followed by a letter, '/', '!' or '?' —
// HTML's own rule for what can open a tag, close tag, comment/doctype, or
// processing instruction. Anything else (a digit, a space, end of string)
// is literal text, so "a < b" is not mistaken for markup.
func stripMarkup(s string) string {
	var b strings.Builder
	b.Grow(len(s))
	i, n := 0, len(s)
	for i < n {
		if s[i] != '<' || !startsTag(s, i) {
			b.WriteByte(s[i])
			i++
			continue
		}

		if strings.HasPrefix(s[i:], "<!--") {
			end := n
			if k := strings.Index(s[i+4:], "-->"); k != -1 {
				end = i + 4 + k + len("-->")
			}
			b.WriteByte(' ')
			i = end
			continue
		}

		j := i + 1
		closing := false
		if j < n && s[j] == '/' {
			closing = true
			j++
		}
		nameStart := j
		for j < n && isTagNameByte(s[j]) {
			j++
		}
		name := strings.ToLower(s[nameStart:j])

		gt := strings.IndexByte(s[j:], '>')
		if gt == -1 {
			// D1: an unterminated tag (no closing '>') can no longer be
			// parsed as markup, but that is no reason to discard the rest
			// of the file - copy it through as literal text instead of
			// simply stopping the scan.
			b.WriteString(s[i:])
			break
		}
		end := j + gt + 1

		if !closing && (name == "script" || name == "style") {
			if close := indexCloseTag(s[end:], name); close != -1 {
				end += close
				if gt2 := strings.IndexByte(s[end:], '>'); gt2 != -1 {
					end += gt2 + 1
				} else {
					end = n
				}
			} else {
				end = n
			}
		}

		b.WriteByte(' ')
		i = end
	}

	return strings.ReplaceAll(html.UnescapeString(b.String()), string(nbsp), " ")
}

// startsTag reports whether s[i] == '<' begins a tag-like construct: the
// next character is a letter, '/', '!' or '?'. A '<' at the very end of s,
// or followed by anything else (digit, space, punctuation), is literal
// text instead.
func startsTag(s string, i int) bool {
	if i+1 >= len(s) {
		return false
	}
	c := s[i+1]
	return c == '/' || c == '!' || c == '?' ||
		(c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
}

// nbsp is U+00A0, NO-BREAK SPACE: what html.UnescapeString decodes &nbsp;
// to, folded to a regular space since the source markup used it as one.
const nbsp = rune(0xA0)

// isTagNameByte reports whether c can appear in an HTML/XML tag name.
func isTagNameByte(c byte) bool {
	return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '-' || c == ':' || c == '_'
}

// indexCloseTag returns the index in s of the first ASCII case-insensitive
// occurrence of "</name", or -1.
func indexCloseTag(s, name string) int {
	target := "</" + name
	tn := len(target)
	for i := 0; i+tn <= len(s); i++ {
		if asciiEqualFold(s[i:i+tn], target) {
			return i
		}
	}
	return -1
}

// asciiEqualFold reports whether a and b are equal, ASCII letters compared
// without regard to case.
func asciiEqualFold(a, b string) bool {
	if len(a) != len(b) {
		return false
	}
	for i := 0; i < len(a); i++ {
		ca, cb := a[i], b[i]
		if 'A' <= ca && ca <= 'Z' {
			ca += 'a' - 'A'
		}
		if 'A' <= cb && cb <= 'Z' {
			cb += 'a' - 'A'
		}
		if ca != cb {
			return false
		}
	}
	return true
}