aboutsummaryrefslogtreecommitdiff
path: root/internal/extract
diff options
context:
space:
mode:
Diffstat (limited to 'internal/extract')
-rw-r--r--internal/extract/plain.go12
-rw-r--r--internal/extract/plain_test.go16
2 files changed, 23 insertions, 5 deletions
diff --git a/internal/extract/plain.go b/internal/extract/plain.go
index 3f04c7e..ce9dedc 100644
--- a/internal/extract/plain.go
+++ b/internal/extract/plain.go
@@ -142,13 +142,15 @@ func decodeUTF16(b []byte, order binary.ByteOrder) string {
}
// decodeLatin1 decodes b as Latin-1: each byte is its own Unicode code
-// point.
+// point. Built directly as UTF-8 (at most two bytes per input byte), not
+// through a []rune of four bytes per input byte (triage 28d).
func decodeLatin1(b []byte) string {
- r := make([]rune, len(b))
- for i, c := range b {
- r[i] = rune(c)
+ var s strings.Builder
+ s.Grow(len(b) * 2)
+ for _, c := range b {
+ s.WriteRune(rune(c))
}
- return string(r)
+ return s.String()
}
// stripMarkup turns decoded HTML/XML/SVG text into plain text: a small
diff --git a/internal/extract/plain_test.go b/internal/extract/plain_test.go
index 5752bf8..c8e1008 100644
--- a/internal/extract/plain_test.go
+++ b/internal/extract/plain_test.go
@@ -3,6 +3,7 @@
package extract
import (
+ "bytes"
"context"
"errors"
"os"
@@ -228,3 +229,18 @@ func TestSniffRuneStraddlingSampleBoundary(t *testing.T) {
t.Errorf("got tail %q, want it to end in %q", got[len(got)-8:], want)
}
}
+
+// TestLatin1DecodingMemory: decoding Latin-1 allocates about what the UTF-8
+// result needs (at most two bytes per input byte), not a four-byte rune per
+// input byte on top of it (triage 28d).
+func TestLatin1DecodingMemory(t *testing.T) {
+ data := bytes.Repeat([]byte("Gr\xfc\xdfe "), 16000)
+ r := testing.Benchmark(func(b *testing.B) {
+ for range b.N {
+ decodeLatin1(data)
+ }
+ })
+ if got, limit := r.AllocedBytesPerOp(), int64(3*len(data)); got > limit {
+ t.Errorf("decoding %d bytes allocates %d bytes, over %d", len(data), got, limit)
+ }
+}