aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLukasz Kasprzak <lukas@labunix.xyz>2026-08-19 07:55:53 +0200
committerLukasz Kasprzak <lukas@labunix.xyz>2026-08-19 07:55:53 +0200
commit7228d0634a960c38da96b80378ec617768dfedf7 (patch)
tree8b2f67c185eb87d28d63ca231082596014667bb2
parenteff4b89cff4e1b8fcceb23c54cb62ec636ce62fe (diff)
downloadcolitur-7228d0634a960c38da96b80378ec617768dfedf7.tar.gz
colitur-7228d0634a960c38da96b80378ec617768dfedf7.zip
fix(render): make fold_ics total on arbitrary octet strings
fold_ics's UTF-8 backoff loop could back `cut` all the way down to `pos` on 74+ consecutive continuation bytes (0x80-0xBF), producing a zero-length chunk and recursing on the identical position forever -- not producible by valid UTF-8, whose longest continuation run is 3, but the kernel's own totality requirement covers arbitrary octet strings, not only valid ones. When backoff finds no boundary inside the window, cut hard at the limit instead, so forward progress is unconditional. test_fold_never_splits_utf8 previously asserted only that unfolding reproduced the original bytes, a property folding preserves at any cut position and therefore blind to a boundary violation. It now also asserts the named property directly: no continuation chunk may start with a UTF-8 continuation byte. A new regression test feeds fold_ics 100 consecutive continuation bytes and asserts it terminates with every line at or under 75 octets.
-rw-r--r--lib/render/escape.ml8
-rw-r--r--test/test_escape.ml42
2 files changed, 42 insertions, 8 deletions
diff --git a/lib/render/escape.ml b/lib/render/escape.ml
index 53fdd46..f506c44 100644
--- a/lib/render/escape.ml
+++ b/lib/render/escape.ml
@@ -77,6 +77,14 @@ let fold_ics line =
else begin
let cut = ref (pos + limit) in
while !cut > pos && is_cont line.[!cut] do decr cut done;
+ (* Valid UTF-8's longest continuation run is 3, so backoff finds a
+ boundary within a few bytes. But this function must stay TOTAL on
+ ARBITRARY octet strings, not only valid UTF-8: 74+ consecutive
+ continuation bytes back `cut` all the way down to `pos`, which would
+ yield a zero-length chunk and recurse on the identical position
+ forever. When backoff finds no boundary inside the window, cut hard
+ at the limit instead -- forward progress is then unconditional. *)
+ if !cut = pos then cut := pos + limit;
if not first then Buffer.add_char b ' ';
Buffer.add_string b (String.sub line pos (!cut - pos));
Buffer.add_string b "\r\n";
diff --git a/test/test_escape.ml b/test/test_escape.ml
index 1587388..35a9b9d 100644
--- a/test/test_escape.ml
+++ b/test/test_escape.ml
@@ -70,16 +70,41 @@ let test_fold_long_line () =
let test_fold_never_splits_utf8 () =
let long = "SUMMARY:" ^ String.concat "" (List.init 40 (fun _ -> "\xc4\x99\xc5\x9b\xc4\x87")) in
let out = E.fold_ics long in
- let stripped =
- String.concat ""
- (List.filter_map
- (fun l ->
- let l = if l <> "" && l.[String.length l - 1] = '\r' then String.sub l 0 (String.length l - 1) else l in
- if l = "" then None else if l.[0] = ' ' then Some (String.sub l 1 (String.length l - 1)) else Some l)
- (String.split_on_char '\n' out))
+ let chunks =
+ List.filter_map
+ (fun l ->
+ let l = if l <> "" && l.[String.length l - 1] = '\r' then String.sub l 0 (String.length l - 1) else l in
+ if l = "" then None else if l.[0] = ' ' then Some (String.sub l 1 (String.length l - 1)) else Some l)
+ (String.split_on_char '\n' out)
in
+ (* The named property: no chunk may START with a UTF-8 continuation byte
+ (0x80-0xBF) -- that would mean the previous fold cut mid-character.
+ Unfolding losslessly (below) cannot detect this on its own: concatenation
+ is insensitive to where the cuts fell, so a fold at ANY position still
+ round-trips. *)
+ List.iter
+ (fun c ->
+ if String.length c > 0 then
+ Alcotest.(check bool) "chunk does not start mid-UTF-8" true (Char.code c.[0] land 0xC0 <> 0x80))
+ chunks;
+ let stripped = String.concat "" chunks in
check "unfolds to the original" long stripped
+(* B1 regression: 100 consecutive UTF-8 continuation bytes (0x80-0xBF) is not
+ producible by valid UTF-8 (whose longest continuation run is 3), but
+ fold_ics must stay TOTAL on arbitrary octet strings. A backoff loop with no
+ hard-cut fallback backs `cut` all the way down to `pos`, yielding a
+ zero-length chunk and recursing on the identical position forever. *)
+let test_fold_pathological_input_terminates () =
+ let pathological = String.make 100 '\x80' in
+ let out = E.fold_ics pathological in
+ let lines = String.split_on_char '\n' out in
+ List.iter
+ (fun l ->
+ let l = if l <> "" && l.[String.length l - 1] = '\r' then String.sub l 0 (String.length l - 1) else l in
+ if String.length l > 75 then Alcotest.failf "line of %d octets exceeds 75" (String.length l))
+ (List.filter (fun l -> l <> "") lines)
+
let suite =
( "Escape",
[ Alcotest.test_case "latex" `Quick test_latex;
@@ -90,4 +115,5 @@ let suite =
Alcotest.test_case "flavour names" `Quick test_flavour_names;
Alcotest.test_case "fold: short unchanged" `Quick test_fold_short_line_unchanged;
Alcotest.test_case "fold: long line" `Quick test_fold_long_line;
- Alcotest.test_case "fold: never splits utf8" `Quick test_fold_never_splits_utf8 ] )
+ Alcotest.test_case "fold: never splits utf8" `Quick test_fold_never_splits_utf8;
+ Alcotest.test_case "fold: pathological input terminates" `Quick test_fold_pathological_input_terminates ] )