diff options
Diffstat (limited to 'internal/extract/plain.go')
| -rw-r--r-- | internal/extract/plain.go | 37 |
1 files changed, 18 insertions, 19 deletions
diff --git a/internal/extract/plain.go b/internal/extract/plain.go index beffae1..956c3e6 100644 --- a/internal/extract/plain.go +++ b/internal/extract/plain.go @@ -39,13 +39,12 @@ func readDecoded(path string) (string, error) { // sample that merely looks like UTF-8 must hold for the WHOLE file — no // NUL byte anywhere, and no invalid UTF-8 anywhere past the sample — or // the file is ErrUnsupported after all (a self-extracting installer has no -// text in krino's sense, decided after the plan 10 re-check); the -// Latin-1 fallback in decode -// never applies to a sniffed file, only to a file whose extension already -// names it as text. D2: when the file continues past the sample (n == -// sniffSize), the validity check is run against a trimmed copy with any -// incomplete trailing rune removed, so a multi-byte rune that happens to -// straddle byte sniffSize does not make an otherwise-valid file sniff as +// text in krino's sense); the Latin-1 fallback in decode never applies to +// a sniffed file, only to a file whose extension already names it as +// text. When the file continues past the sample (n == sniffSize), the +// validity check is run against a trimmed copy with any incomplete +// trailing rune removed, so a multi-byte rune that happens to straddle +// byte sniffSize does not make an otherwise-valid file sniff as // unsupported; sample itself, used below to build the returned text, is // left untouched — the rest of the file (read after the check) supplies // the bytes trimming set aside. @@ -89,13 +88,13 @@ func sniffText(path string) (string, error) { } // trimIncompleteTrailingRune drops an incomplete UTF-8 sequence left -// dangling at the very end of b — D2's fix for a rune cut off exactly at -// the sniff sample's boundary. It looks back at most utf8.UTFMax-1 bytes -// for the start of the trailing rune; if the bytes from there to the end -// are not a complete encoding (utf8.FullRune), that partial rune is cut, -// since more bytes to finish it may simply not have been read yet. A -// sample already ending cleanly (the common case, and every all-ASCII -// sample) is returned unchanged. +// dangling at the very end of b, so a rune cut off exactly at the sniff +// sample's boundary is not mistaken for invalid UTF-8. It looks back at +// most utf8.UTFMax-1 bytes for the start of the trailing rune; if the +// bytes from there to the end are not a complete encoding (utf8.FullRune), +// that partial rune is cut, since more bytes to finish it may simply not +// have been read yet. A sample already ending cleanly (the common case, +// and every all-ASCII sample) is returned unchanged. func trimIncompleteTrailingRune(b []byte) []byte { end := len(b) start := end - 1 @@ -143,7 +142,7 @@ func decodeUTF16(b []byte, order binary.ByteOrder) string { // decodeLatin1 decodes b as Latin-1: each byte is its own Unicode code // point. Built directly as UTF-8 (at most two bytes per input byte), not -// through a []rune of four bytes per input byte (triage 28d). +// through a []rune of four bytes per input byte. func decodeLatin1(b []byte) string { var s strings.Builder s.Grow(len(b) * 2) @@ -200,10 +199,10 @@ func stripMarkup(s string) string { gt := strings.IndexByte(s[j:], '>') if gt == -1 { - // D1: an unterminated tag (no closing '>') can no longer be - // parsed as markup, but that is no reason to discard the rest - // of the file - copy it through as literal text instead of - // simply stopping the scan. + // An unterminated tag (no closing '>') can no longer be + // parsed as markup, but that is no reason to discard the + // rest of the file - copy it through as literal text + // instead of simply stopping the scan. b.WriteString(s[i:]) break } |