diff options
| -rw-r--r-- | CHANGELOG.md | 3 | ||||
| -rw-r--r-- | docs/design.md | 5 | ||||
| -rw-r--r-- | internal/engine/engine.go | 6 | ||||
| -rw-r--r-- | internal/engine/exclude_test.go | 7 | ||||
| -rw-r--r-- | internal/extract/plain.go | 12 | ||||
| -rw-r--r-- | internal/extract/plain_test.go | 16 |
6 files changed, 42 insertions, 7 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 39ab7fb..37a9f50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ could change its answer; `explain` shows `?`. A document read only in part answers the keywords found in what was read, and leaves the others unknown. +- krino.conf's excludes are checked with the defaults' `case` and `fold` + even while no directory is included; Latin-1 text is decoded in a third + of the memory. - `-v` lists the rule destinations inside a directory that were not scanned, so files there are no longer invisible. - Plan and table widths count terminal columns: a name in a wide script diff --git a/docs/design.md b/docs/design.md index c9ebe50..ef52d7e 100644 --- a/docs/design.md +++ b/docs/design.md @@ -279,7 +279,10 @@ by size. Only groups of two or more are hashed: first the first and last 64 KiB, then SHA-256 of the whole file. Empty files are never duplicates. Two names for the same file — the same device and inode, as a hardlink creates — are never duplicates of each other; they may still both be -duplicates of a separate identical file. +duplicates of a separate identical file. The original is elected once for +all identical content, so a scanned file whose hardlink under a DIR is +elected is that original, and not a duplicate, even when the DIR also +holds a separate copy. The **original** in each group is, in order of preference: a file under one of the given DIRs, then the oldest by mtime, then the shortest name, then diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 7a83ccd..00a249d 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -93,8 +93,12 @@ func Load(mainFile string, names ...string) (*Engine, []*config.Diag) { // With no directory to compile them for, krino.conf's excludes are // still checked, so a mistake is reported before one is included // (review cli F10). + // Compiled with the defaults' case and fold, as a directory without + // settings of its own would compile them (triage 28g). + def := cfg.Main.Defaults.Over(config.Builtin()) + opt := cond.Options{IgnoreCase: def.Case == config.CaseIgnore, Fold: def.Fold} for _, x := range cfg.Main.Excludes { - _, cerrs := cond.Compile(cfg.Main.File, x.When, cond.Options{IgnoreCase: true, Fold: true}) + _, cerrs := cond.Compile(cfg.Main.File, x.When, opt) errs = append(errs, cerrs...) } } diff --git a/internal/engine/exclude_test.go b/internal/engine/exclude_test.go index cd1cc4b..8308a82 100644 --- a/internal/engine/exclude_test.go +++ b/internal/engine/exclude_test.go @@ -261,6 +261,13 @@ func TestLoadChecksMainExcludesWithoutDirectories(t *testing.T) { if _, errs := Load(main); len(errs) == 0 { t.Error("a broken krino.conf exclude was not reported") } + // Checked with the defaults' case and fold, as a directory would compile + // them (triage 28g): a keyword of a lone combining mark is empty only + // when folded. + main = writeConfig(t, h, "(include)\n(defaults (fold no))\n(exclude (content \"\u0301\"))\n", nil) + if _, errs := Load(main); len(errs) != 0 { + t.Errorf("checked with fold on, though the defaults say no: %v", errs) + } } // TestNoTextFormatIsNoMatch: a file whose format has no text cannot contain diff --git a/internal/extract/plain.go b/internal/extract/plain.go index 3f04c7e..ce9dedc 100644 --- a/internal/extract/plain.go +++ b/internal/extract/plain.go @@ -142,13 +142,15 @@ func decodeUTF16(b []byte, order binary.ByteOrder) string { } // decodeLatin1 decodes b as Latin-1: each byte is its own Unicode code -// point. +// point. Built directly as UTF-8 (at most two bytes per input byte), not +// through a []rune of four bytes per input byte (triage 28d). func decodeLatin1(b []byte) string { - r := make([]rune, len(b)) - for i, c := range b { - r[i] = rune(c) + var s strings.Builder + s.Grow(len(b) * 2) + for _, c := range b { + s.WriteRune(rune(c)) } - return string(r) + return s.String() } // stripMarkup turns decoded HTML/XML/SVG text into plain text: a small diff --git a/internal/extract/plain_test.go b/internal/extract/plain_test.go index 5752bf8..c8e1008 100644 --- a/internal/extract/plain_test.go +++ b/internal/extract/plain_test.go @@ -3,6 +3,7 @@ package extract import ( + "bytes" "context" "errors" "os" @@ -228,3 +229,18 @@ func TestSniffRuneStraddlingSampleBoundary(t *testing.T) { t.Errorf("got tail %q, want it to end in %q", got[len(got)-8:], want) } } + +// TestLatin1DecodingMemory: decoding Latin-1 allocates about what the UTF-8 +// result needs (at most two bytes per input byte), not a four-byte rune per +// input byte on top of it (triage 28d). +func TestLatin1DecodingMemory(t *testing.T) { + data := bytes.Repeat([]byte("Gr\xfc\xdfe "), 16000) + r := testing.Benchmark(func(b *testing.B) { + for range b.N { + decodeLatin1(data) + } + }) + if got, limit := r.AllocedBytesPerOp(), int64(3*len(data)); got > limit { + t.Errorf("decoding %d bytes allocates %d bytes, over %d", len(data), got, limit) + } +} |