From eff4b89cff4e1b8fcceb23c54cb62ec636ce62fe Mon Sep 17 00:00:00 2001 From: Lukasz Kasprzak Date: Wed, 19 Aug 2026 07:47:30 +0200 Subject: feat(render): per-flavour escaping and RFC 5545 line folding Six flavours: latex, groff, html, xml, ics, none. Markdown, AsciiDoc and plain text map to none deliberately -- their metacharacters are context-dependent and escaping them aggressively produces worse output than not escaping. An unrecognised extension returns None rather than falling back to none: guessing the flavour wrong produces malformed output that looks fine until it does not. Folding backs off to a non-continuation byte, so a fold never splits a UTF-8 sequence -- the failure mode that would corrupt Polish and Latin names in a published feed. --- test/test_escape.ml | 93 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 test/test_escape.ml (limited to 'test/test_escape.ml') diff --git a/test/test_escape.ml b/test/test_escape.ml new file mode 100644 index 0000000..1587388 --- /dev/null +++ b/test/test_escape.ml @@ -0,0 +1,93 @@ +module E = Colitur_render.Escape + +let check = Alcotest.(check string) + +let test_latex () = + check "ampersand" "\\&" (E.apply E.Latex "&"); + check "percent" "\\%" (E.apply E.Latex "%"); + check "underscore" "\\_" (E.apply E.Latex "_"); + check "braces" "\\{\\}" (E.apply E.Latex "{}"); + check "backslash first" "\\textbackslash{}" (E.apply E.Latex "\\"); + (* A real feast name that would otherwise break a .tex build. *) + check "real name" "Ss.mi Nominis Iesu \\& Mari\xc3\xa6" + (E.apply E.Latex "Ss.mi Nominis Iesu & Mari\xc3\xa6") + +let test_groff () = + check "backslash" "\\e" (E.apply E.Groff "\\"); + (* RG-irrelevant but groff-critical: a leading dot starts a request. *) + check "leading dot" "\\&.Ss" (E.apply E.Groff ".Ss"); + check "leading quote" "\\&'tis" (E.apply E.Groff "'tis"); + check "interior dot untouched" "Ss.mi" (E.apply E.Groff "Ss.mi") + +let test_html_xml () = + check "amp first" "&lt;" (E.apply E.Html "<"); + check "angles" "<b>" (E.apply E.Html ""); + check "quote" """ (E.apply E.Html "\""); + check "xml same" "<b>" (E.apply E.Xml "") + +let test_ics () = + check "comma" "\\," (E.apply E.Ics ","); + check "semicolon" "\\;" (E.apply E.Ics ";"); + check "backslash" "\\\\" (E.apply E.Ics "\\"); + check "newline" "\\n" (E.apply E.Ics "\n") + +let test_none_is_identity () = + check "none" "& < > \\ % {}" (E.apply E.None_ "& < > \\ % {}") + +let test_flavour_names () = + List.iter + (fun f -> Alcotest.(check bool) "roundtrip" true (E.of_string (E.to_string f) = Some f)) + [ E.Latex; E.Groff; E.Html; E.Xml; E.Ics; E.None_ ]; + Alcotest.(check bool) "tex" true (E.of_extension ".tex" = Some E.Latex); + Alcotest.(check bool) "ms" true (E.of_extension ".ms" = Some E.Groff); + Alcotest.(check bool) "mom" true (E.of_extension ".mom" = Some E.Groff); + Alcotest.(check bool) "html" true (E.of_extension ".html" = Some E.Html); + Alcotest.(check bool) "md is none" true (E.of_extension ".md" = Some E.None_); + Alcotest.(check bool) "adoc is none" true (E.of_extension ".adoc" = Some E.None_); + Alcotest.(check bool) "txt is none" true (E.of_extension ".txt" = Some E.None_); + (* Spec section 5: an unknown extension is an ERROR, never a silent fallback. *) + Alcotest.(check bool) "unknown is None" true (E.of_extension ".wat" = None) + +let test_fold_short_line_unchanged () = + check "short" "SUMMARY:Feast\r\n" (E.fold_ics "SUMMARY:Feast") + +let test_fold_long_line () = + let long = "DESCRIPTION:" ^ String.make 200 'x' in + let out = E.fold_ics long in + let lines = String.split_on_char '\n' out in + List.iter + (fun l -> + let l = if l <> "" && l.[String.length l - 1] = '\r' then String.sub l 0 (String.length l - 1) else l in + if String.length l > 75 then Alcotest.failf "line of %d octets exceeds 75" (String.length l)) + (List.filter (fun l -> l <> "") lines); + (* Continuation lines begin with exactly one space (RFC 5545 section 3.1). *) + match lines with + | _ :: second :: _ -> Alcotest.(check bool) "continuation starts with space" true (second.[0] = ' ') + | _ -> Alcotest.fail "expected the line to fold" + +(* RFC 5545 section 3.1 folds on OCTET boundaries; splitting mid-UTF-8 corrupts + Polish and Latin names, which is the whole reason this is not a template job. *) +let test_fold_never_splits_utf8 () = + let long = "SUMMARY:" ^ String.concat "" (List.init 40 (fun _ -> "\xc4\x99\xc5\x9b\xc4\x87")) in + let out = E.fold_ics long in + let stripped = + String.concat "" + (List.filter_map + (fun l -> + let l = if l <> "" && l.[String.length l - 1] = '\r' then String.sub l 0 (String.length l - 1) else l in + if l = "" then None else if l.[0] = ' ' then Some (String.sub l 1 (String.length l - 1)) else Some l) + (String.split_on_char '\n' out)) + in + check "unfolds to the original" long stripped + +let suite = + ( "Escape", + [ Alcotest.test_case "latex" `Quick test_latex; + Alcotest.test_case "groff" `Quick test_groff; + Alcotest.test_case "html/xml" `Quick test_html_xml; + Alcotest.test_case "ics" `Quick test_ics; + Alcotest.test_case "none is identity" `Quick test_none_is_identity; + Alcotest.test_case "flavour names" `Quick test_flavour_names; + Alcotest.test_case "fold: short unchanged" `Quick test_fold_short_line_unchanged; + Alcotest.test_case "fold: long line" `Quick test_fold_long_line; + Alcotest.test_case "fold: never splits utf8" `Quick test_fold_never_splits_utf8 ] ) -- cgit v1.3 From 7228d0634a960c38da96b80378ec617768dfedf7 Mon Sep 17 00:00:00 2001 From: Lukasz Kasprzak Date: Wed, 19 Aug 2026 07:55:53 +0200 Subject: fix(render): make fold_ics total on arbitrary octet strings fold_ics's UTF-8 backoff loop could back `cut` all the way down to `pos` on 74+ consecutive continuation bytes (0x80-0xBF), producing a zero-length chunk and recursing on the identical position forever -- not producible by valid UTF-8, whose longest continuation run is 3, but the kernel's own totality requirement covers arbitrary octet strings, not only valid ones. When backoff finds no boundary inside the window, cut hard at the limit instead, so forward progress is unconditional. test_fold_never_splits_utf8 previously asserted only that unfolding reproduced the original bytes, a property folding preserves at any cut position and therefore blind to a boundary violation. It now also asserts the named property directly: no continuation chunk may start with a UTF-8 continuation byte. A new regression test feeds fold_ics 100 consecutive continuation bytes and asserts it terminates with every line at or under 75 octets. --- lib/render/escape.ml | 8 ++++++++ test/test_escape.ml | 42 ++++++++++++++++++++++++++++++++++-------- 2 files changed, 42 insertions(+), 8 deletions(-) (limited to 'test/test_escape.ml') diff --git a/lib/render/escape.ml b/lib/render/escape.ml index 53fdd46..f506c44 100644 --- a/lib/render/escape.ml +++ b/lib/render/escape.ml @@ -77,6 +77,14 @@ let fold_ics line = else begin let cut = ref (pos + limit) in while !cut > pos && is_cont line.[!cut] do decr cut done; + (* Valid UTF-8's longest continuation run is 3, so backoff finds a + boundary within a few bytes. But this function must stay TOTAL on + ARBITRARY octet strings, not only valid UTF-8: 74+ consecutive + continuation bytes back `cut` all the way down to `pos`, which would + yield a zero-length chunk and recurse on the identical position + forever. When backoff finds no boundary inside the window, cut hard + at the limit instead -- forward progress is then unconditional. *) + if !cut = pos then cut := pos + limit; if not first then Buffer.add_char b ' '; Buffer.add_string b (String.sub line pos (!cut - pos)); Buffer.add_string b "\r\n"; diff --git a/test/test_escape.ml b/test/test_escape.ml index 1587388..35a9b9d 100644 --- a/test/test_escape.ml +++ b/test/test_escape.ml @@ -70,16 +70,41 @@ let test_fold_long_line () = let test_fold_never_splits_utf8 () = let long = "SUMMARY:" ^ String.concat "" (List.init 40 (fun _ -> "\xc4\x99\xc5\x9b\xc4\x87")) in let out = E.fold_ics long in - let stripped = - String.concat "" - (List.filter_map - (fun l -> - let l = if l <> "" && l.[String.length l - 1] = '\r' then String.sub l 0 (String.length l - 1) else l in - if l = "" then None else if l.[0] = ' ' then Some (String.sub l 1 (String.length l - 1)) else Some l) - (String.split_on_char '\n' out)) + let chunks = + List.filter_map + (fun l -> + let l = if l <> "" && l.[String.length l - 1] = '\r' then String.sub l 0 (String.length l - 1) else l in + if l = "" then None else if l.[0] = ' ' then Some (String.sub l 1 (String.length l - 1)) else Some l) + (String.split_on_char '\n' out) in + (* The named property: no chunk may START with a UTF-8 continuation byte + (0x80-0xBF) -- that would mean the previous fold cut mid-character. + Unfolding losslessly (below) cannot detect this on its own: concatenation + is insensitive to where the cuts fell, so a fold at ANY position still + round-trips. *) + List.iter + (fun c -> + if String.length c > 0 then + Alcotest.(check bool) "chunk does not start mid-UTF-8" true (Char.code c.[0] land 0xC0 <> 0x80)) + chunks; + let stripped = String.concat "" chunks in check "unfolds to the original" long stripped +(* B1 regression: 100 consecutive UTF-8 continuation bytes (0x80-0xBF) is not + producible by valid UTF-8 (whose longest continuation run is 3), but + fold_ics must stay TOTAL on arbitrary octet strings. A backoff loop with no + hard-cut fallback backs `cut` all the way down to `pos`, yielding a + zero-length chunk and recursing on the identical position forever. *) +let test_fold_pathological_input_terminates () = + let pathological = String.make 100 '\x80' in + let out = E.fold_ics pathological in + let lines = String.split_on_char '\n' out in + List.iter + (fun l -> + let l = if l <> "" && l.[String.length l - 1] = '\r' then String.sub l 0 (String.length l - 1) else l in + if String.length l > 75 then Alcotest.failf "line of %d octets exceeds 75" (String.length l)) + (List.filter (fun l -> l <> "") lines) + let suite = ( "Escape", [ Alcotest.test_case "latex" `Quick test_latex; @@ -90,4 +115,5 @@ let suite = Alcotest.test_case "flavour names" `Quick test_flavour_names; Alcotest.test_case "fold: short unchanged" `Quick test_fold_short_line_unchanged; Alcotest.test_case "fold: long line" `Quick test_fold_long_line; - Alcotest.test_case "fold: never splits utf8" `Quick test_fold_never_splits_utf8 ] ) + Alcotest.test_case "fold: never splits utf8" `Quick test_fold_never_splits_utf8; + Alcotest.test_case "fold: pathological input terminates" `Quick test_fold_pathological_input_terminates ] ) -- cgit v1.3