main excludes checked with the defaults' case and fold; Latin-1 decoding memory; hardlink election documented

6 files changed, 42 insertions, 7 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 39ab7fb..37a9f50 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,9 @@
   could change its answer; `explain` shows `?`. A document read only in part
   answers the keywords found in what was read, and leaves the others
   unknown.
+- krino.conf's excludes are checked with the defaults' `case` and `fold`
+  even while no directory is included; Latin-1 text is decoded in a third
+  of the memory.
 - `-v` lists the rule destinations inside a directory that were not
   scanned, so files there are no longer invisible.
 - Plan and table widths count terminal columns: a name in a wide script
diff --git a/docs/design.md b/docs/design.md
index c9ebe50..ef52d7e 100644
--- a/docs/design.md
+++ b/docs/design.md
@@ -279,7 +279,10 @@ by size. Only groups of two or more are hashed: first the first and last
 64 KiB, then SHA-256 of the whole file. Empty files are never duplicates.
 Two names for the same file — the same device and inode, as a hardlink
 creates — are never duplicates of each other; they may still both be
-duplicates of a separate identical file.
+duplicates of a separate identical file. The original is elected once for
+all identical content, so a scanned file whose hardlink under a DIR is
+elected is that original, and not a duplicate, even when the DIR also
+holds a separate copy.
 
 The **original** in each group is, in order of preference: a file under one
 of the given DIRs, then the oldest by mtime, then the shortest name, then
diff --git a/internal/engine/engine.go b/internal/engine/engine.go
index 7a83ccd..00a249d 100644
--- a/internal/engine/engine.go
+++ b/internal/engine/engine.go
@@ -93,8 +93,12 @@ func Load(mainFile string, names ...string) (*Engine, []*config.Diag) {
 		// With no directory to compile them for, krino.conf's excludes are
 		// still checked, so a mistake is reported before one is included
 		// (review cli F10).
+		// Compiled with the defaults' case and fold, as a directory without
+		// settings of its own would compile them (triage 28g).
+		def := cfg.Main.Defaults.Over(config.Builtin())
+		opt := cond.Options{IgnoreCase: def.Case == config.CaseIgnore, Fold: def.Fold}
 		for _, x := range cfg.Main.Excludes {
-			_, cerrs := cond.Compile(cfg.Main.File, x.When, cond.Options{IgnoreCase: true, Fold: true})
+			_, cerrs := cond.Compile(cfg.Main.File, x.When, opt)
 			errs = append(errs, cerrs...)
 		}
 	}
diff --git a/internal/engine/exclude_test.go b/internal/engine/exclude_test.go
index cd1cc4b..8308a82 100644
--- a/internal/engine/exclude_test.go
+++ b/internal/engine/exclude_test.go
@@ -261,6 +261,13 @@ func TestLoadChecksMainExcludesWithoutDirectories(t *testing.T) {
 	if _, errs := Load(main); len(errs) == 0 {
 		t.Error("a broken krino.conf exclude was not reported")
 	}
+	// Checked with the defaults' case and fold, as a directory would compile
+	// them (triage 28g): a keyword of a lone combining mark is empty only
+	// when folded.
+	main = writeConfig(t, h, "(include)\n(defaults (fold no))\n(exclude (content \"\u0301\"))\n", nil)
+	if _, errs := Load(main); len(errs) != 0 {
+		t.Errorf("checked with fold on, though the defaults say no: %v", errs)
+	}
 }
 
 // TestNoTextFormatIsNoMatch: a file whose format has no text cannot contain
diff --git a/internal/extract/plain.go b/internal/extract/plain.go
index 3f04c7e..ce9dedc 100644
--- a/internal/extract/plain.go
+++ b/internal/extract/plain.go
@@ -142,13 +142,15 @@ func decodeUTF16(b []byte, order binary.ByteOrder) string {
 }
 
 // decodeLatin1 decodes b as Latin-1: each byte is its own Unicode code
-// point.
+// point. Built directly as UTF-8 (at most two bytes per input byte), not
+// through a []rune of four bytes per input byte (triage 28d).
 func decodeLatin1(b []byte) string {
-	r := make([]rune, len(b))
-	for i, c := range b {
-		r[i] = rune(c)
+	var s strings.Builder
+	s.Grow(len(b) * 2)
+	for _, c := range b {
+		s.WriteRune(rune(c))
 	}
-	return string(r)
+	return s.String()
 }
 
 // stripMarkup turns decoded HTML/XML/SVG text into plain text: a small
diff --git a/internal/extract/plain_test.go b/internal/extract/plain_test.go
index 5752bf8..c8e1008 100644
--- a/internal/extract/plain_test.go
+++ b/internal/extract/plain_test.go
@@ -3,6 +3,7 @@
 package extract
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"os"
@@ -228,3 +229,18 @@ func TestSniffRuneStraddlingSampleBoundary(t *testing.T) {
 		t.Errorf("got tail %q, want it to end in %q", got[len(got)-8:], want)
 	}
 }
+
+// TestLatin1DecodingMemory: decoding Latin-1 allocates about what the UTF-8
+// result needs (at most two bytes per input byte), not a four-byte rune per
+// input byte on top of it (triage 28d).
+func TestLatin1DecodingMemory(t *testing.T) {
+	data := bytes.Repeat([]byte("Gr\xfc\xdfe "), 16000)
+	r := testing.Benchmark(func(b *testing.B) {
+		for range b.N {
+			decodeLatin1(data)
+		}
+	})
+	if got, limit := r.AllocedBytesPerOp(), int64(3*len(data)); got > limit {
+		t.Errorf("decoding %d bytes allocates %d bytes, over %d", len(data), got, limit)
+	}
+}