diff options
| -rw-r--r-- | CHANGELOG.md | 7 | ||||
| -rw-r--r-- | docs/design.md | 8 | ||||
| -rw-r--r-- | internal/engine/exclude_test.go | 14 | ||||
| -rw-r--r-- | internal/extract/extract.go | 5 | ||||
| -rw-r--r-- | internal/extract/plain.go | 6 | ||||
| -rw-r--r-- | internal/extract/plain_test.go | 10 | ||||
| -rw-r--r-- | man/krino.conf.5 | 8 |
7 files changed, 31 insertions, 27 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index aa02136..beedcdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ ## Unreleased +## 0.0.8 — unreleased + +- A file of no known extension that starts as text and turns binary further + on (a self-extracting installer) has no content, like an image: it no + longer matches a content test, is not set aside by a content exclude, and + raises no warning. + ## 0.0.7 — 2026-09-14 Security and correctness hardening, from a threat model (spec §15.1), an diff --git a/docs/design.md b/docs/design.md index 6b94fc8..4fa5b4a 100644 --- a/docs/design.md +++ b/docs/design.md @@ -204,9 +204,9 @@ cannot read the file (over `max-read`, a tool missing, failing or timing out), the exclude holds, and the file is set aside as "(content unreadable)" with the warning. A file whose format has no text at all (an image, an archive) is not unreadable: it contains no keyword, so its -content tests are simply false and raise no warning. A file of no known -extension that starts as text and holds binary data further on is -unreadable. An exclude protects files, so a file krino +content tests are simply false and raise no warning. So is a file of no +known extension that starts as text and holds binary data further on, such +as a self-extracting installer. An exclude protects files, so a file krino could not check is left alone. Mistakes in krino.conf's excludes are reported even while no directory is included. @@ -324,7 +324,7 @@ Nothing is deleted, and undo restores them. | Format | Method | |---|---| -| text | known text extensions, or detected from the first 8 KiB: valid UTF-8, or UTF-16 with a BOM, and no NUL bytes. A file detected as text that holds a NUL or invalid UTF-8 further on cannot be read (§4.6, §5.4) | +| text | known text extensions; any other file whose first 8 KiB start with a UTF-16 BOM, or are valid UTF-8 with no NUL byte and so is the rest of it (one that turns binary further on, like a self-extracting installer, has no content) | | pdf | `pdftotext -q -enc UTF-8 FILE -`, 30 s timeout | | docx xlsx pptx odt ods odp epub | zip plus streaming XML, Go standard library | | html xml | tags removed, entities decoded | diff --git a/internal/engine/exclude_test.go b/internal/engine/exclude_test.go index 4e9f6e7..e5231f8 100644 --- a/internal/engine/exclude_test.go +++ b/internal/engine/exclude_test.go @@ -303,10 +303,12 @@ func TestNoTextFormatIsNoMatch(t *testing.T) { } } -// TestTextTurningBinaryFailsClosed: a file with no known extension whose -// first 8 KiB read as text but which holds a NUL further on is unreadable, -// not "no text": a content exclude still sets it aside (plan 10 re-check R4). -func TestTextTurningBinaryFailsClosed(t *testing.T) { +// TestTextTurningBinaryIsNoText: a file with no known extension whose first +// 8 KiB read as text but which holds a NUL further on - a self-extracting +// installer - counts as having no text, like an image: a content exclude +// does not set it aside and there is no warning (Łukasz, after the plan 10 +// re-check: "ignore such ones"). +func TestTextTurningBinaryIsNoText(t *testing.T) { mixed := "confidential " + strings.Repeat("x", 9000) + "\x00tail" h, _ := excludeTree(t, map[string]string{"mixed": mixed}) main := writeConfig(t, h, `(include "dl")`, map[string]string{"dl": ` @@ -323,8 +325,8 @@ func TestTextTurningBinaryFailsClosed(t *testing.T) { t.Fatal(err) } for _, fm := range r.Matched { - if fm.File.Rel == "mixed" && (!strings.HasSuffix(fm.Excluded, "(content unreadable)") || len(fm.Rules) != 0) { - t.Errorf("mixed: Excluded %q, rules %d; want set aside as unreadable", fm.Excluded, len(fm.Rules)) + if fm.File.Rel == "mixed" && (fm.Excluded != "" || len(fm.Rules) != 1 || len(fm.Warnings) != 0) { + t.Errorf("mixed: Excluded %q, rules %d, warnings %v; want the rule and no warning", fm.Excluded, len(fm.Rules), fm.Warnings) } } } diff --git a/internal/extract/extract.go b/internal/extract/extract.go index 376e09b..7e3adec 100644 --- a/internal/extract/extract.go +++ b/internal/extract/extract.go @@ -23,11 +23,6 @@ var ( // ErrUnsupported is returned when the format carries no text krino // knows how to extract. ErrUnsupported = errors.New("no text in this format") - // ErrMixed is returned for a file of no known extension whose first - // 8 KiB read as text but which holds a NUL or invalid UTF-8 further on. - // Unlike ErrUnsupported it is a read failure: the text it began with - // could hold a keyword (plan 10 re-check R4). - ErrMixed = errors.New("text at the start, binary data further on") // ErrTooLarge is returned when the file is larger than the configured // max-read; nothing is read in that case. ErrTooLarge = errors.New("larger than max-read") diff --git a/internal/extract/plain.go b/internal/extract/plain.go index f7470be..3f04c7e 100644 --- a/internal/extract/plain.go +++ b/internal/extract/plain.go @@ -38,7 +38,9 @@ func readDecoded(path string) (string, error) { // the sample alone (UTF-16 text is full of NUL bytes by design), but a // sample that merely looks like UTF-8 must hold for the WHOLE file — no // NUL byte anywhere, and no invalid UTF-8 anywhere past the sample — or -// the file is ErrMixed, unreadable; the Latin-1 fallback in decode +// the file is ErrUnsupported after all (a self-extracting installer has no +// text in krino's sense, Łukasz's decision after the plan 10 re-check); the +// Latin-1 fallback in decode // never applies to a sniffed file, only to a file whose extension already // names it as text. D2: when the file continues past the sample (n == // sniffSize), the validity check is run against a trimmed copy with any @@ -81,7 +83,7 @@ func sniffText(path string) (string, error) { return decode(data), nil } if !utf8.Valid(data) || bytes.Contains(data, []byte{0}) { - return "", ErrMixed + return "", ErrUnsupported } return decode(data), nil } diff --git a/internal/extract/plain_test.go b/internal/extract/plain_test.go index f49caf8..5752bf8 100644 --- a/internal/extract/plain_test.go +++ b/internal/extract/plain_test.go @@ -154,15 +154,13 @@ func TestToolsListedInOrder(t *testing.T) { // but the file goes on to hold an invalid UTF-8 byte and a NUL past that // sample — sniffText must reject the whole file, not just decode what the // sample alone promised (it must not fall back to Latin-1 the way a known -// text extension would). It is not "no text in this format" either: the -// text it began with could hold a keyword, so it is ErrMixed, a read -// failure, and a content exclude fails closed on it (plan 10 re-check R4). +// text extension would). Such a file - a self-extracting installer, say - +// has no text in krino's sense (Łukasz, after the plan 10 re-check). func TestSniffWholeFileMustBeValid(t *testing.T) { e := newWithPath("") data := append([]byte(strings.Repeat("x", 8192)), 0xFF, 0x00) - _, err := text(t, e, file(t, "blob.data", data), 0) - if !errors.Is(err, ErrMixed) || errors.Is(err, ErrUnsupported) { - t.Errorf("got %v, want ErrMixed and not ErrUnsupported", err) + if _, err := text(t, e, file(t, "blob.data", data), 0); !errors.Is(err, ErrUnsupported) { + t.Errorf("got %v, want ErrUnsupported", err) } } diff --git a/man/krino.conf.5 b/man/krino.conf.5 index 01d0f3d..28d7034 100644 --- a/man/krino.conf.5 +++ b/man/krino.conf.5 @@ -281,8 +281,8 @@ the exclude holds and the file is set aside as with a warning. A file whose format has no text at all, such as an image or an archive, is not unreadable: its content tests are false, with no warning. -A file of no known extension that starts as text and holds binary data -further on is unreadable. +So is a file of no known extension that starts as text and holds binary +data further on, such as a self-extracting installer. .Sh RULES .Bd -literal -offset indent (rule NAME ITEM...) @@ -498,8 +498,8 @@ Groups overlap; a file can belong to several. .It Sy text Known text extensions, or detected from the first 8 KiB: valid UTF-8, or UTF-16 with a BOM, and no NUL bytes. -A file detected as text that holds a NUL or invalid UTF-8 further on cannot -be read. +A file that holds a NUL or invalid UTF-8 further on, such as a +self-extracting installer, has no content. .It Sy pdf .Ic pdftotext .Pq 30 second timeout . |