From eff4b89cff4e1b8fcceb23c54cb62ec636ce62fe Mon Sep 17 00:00:00 2001 From: Lukasz Kasprzak Date: Wed, 19 Aug 2026 07:47:30 +0200 Subject: feat(render): per-flavour escaping and RFC 5545 line folding Six flavours: latex, groff, html, xml, ics, none. Markdown, AsciiDoc and plain text map to none deliberately -- their metacharacters are context-dependent and escaping them aggressively produces worse output than not escaping. An unrecognised extension returns None rather than falling back to none: guessing the flavour wrong produces malformed output that looks fine until it does not. Folding backs off to a non-continuation byte, so a fold never splits a UTF-8 sequence -- the failure mode that would corrupt Polish and Latin names in a published feed. --- lib/render/escape.ml | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 lib/render/escape.ml (limited to 'lib/render/escape.ml') diff --git a/lib/render/escape.ml b/lib/render/escape.ml new file mode 100644 index 0000000..53fdd46 --- /dev/null +++ b/lib/render/escape.ml @@ -0,0 +1,87 @@ +type flavour = Latex | Groff | Html | Xml | Ics | None_ + +let all = [ Latex; Groff; Html; Xml; Ics; None_ ] + +let to_string = function + | Latex -> "latex" | Groff -> "groff" | Html -> "html" + | Xml -> "xml" | Ics -> "ics" | None_ -> "none" + +let of_string = function + | "latex" -> Some Latex | "groff" -> Some Groff | "html" -> Some Html + | "xml" -> Some Xml | "ics" -> Some Ics | "none" -> Some None_ + | _ -> None + +let of_extension = function + | ".tex" -> Some Latex + | ".ms" | ".mom" | ".me" -> Some Groff + | ".html" | ".htm" -> Some Html + | ".xml" -> Some Xml + | ".ics" -> Some Ics + | ".md" | ".adoc" | ".txt" -> Some None_ + | _ -> None + +(* Replace each character with its expansion, in ONE pass. A sequence of + String.concat replacements would double-escape: "&" -> "\\&" then the + backslash rule would rewrite the backslash it just introduced. *) +let expand f s = + let b = Buffer.create (String.length s + 16) in + String.iter (fun c -> Buffer.add_string b (f c)) s; + Buffer.contents b + +let latex = function + | '\\' -> "\\textbackslash{}" + | '{' -> "\\{" | '}' -> "\\}" + | '$' -> "\\$" | '&' -> "\\&" | '#' -> "\\#" + | '_' -> "\\_" | '%' -> "\\%" + | '^' -> "\\textasciicircum{}" + | '~' -> "\\textasciitilde{}" + | c -> String.make 1 c + +let html = function + | '&' -> "&" | '<' -> "<" | '>' -> ">" + | '"' -> """ | '\'' -> "'" + | c -> String.make 1 c + +let ics = function + | '\\' -> "\\\\" | ';' -> "\\;" | ',' -> "\\," + | '\n' -> "\\n" | '\r' -> "" + | c -> String.make 1 c + +(* groff: a backslash starts an escape, and a '.' or '\'' in COLUMN ONE starts a + request. \& is the zero-width non-printing character that defuses it. *) +let groff s = + let escaped = expand (function '\\' -> "\\e" | c -> String.make 1 c) s in + if String.length escaped > 0 && (escaped.[0] = '.' || escaped.[0] = '\'') then "\\&" ^ escaped + else escaped + +let apply flavour s = + match flavour with + | Latex -> expand latex s + | Groff -> groff s + | Html | Xml -> expand html s + | Ics -> expand ics s + | None_ -> s + +(* RFC 5545 section 3.1. A continuation byte is 0x80-0xBF; backing off to a + non-continuation byte keeps every fold on a character boundary. *) +let fold_ics line = + let n = String.length line in + let b = Buffer.create (n + (n / 70) + 8) in + let is_cont c = Char.code c land 0xC0 = 0x80 in + let rec go pos first = + let limit = if first then 75 else 74 (* the leading space costs one octet *) in + if n - pos <= limit then ( + if not first then Buffer.add_char b ' '; + Buffer.add_string b (String.sub line pos (n - pos)); + Buffer.add_string b "\r\n") + else begin + let cut = ref (pos + limit) in + while !cut > pos && is_cont line.[!cut] do decr cut done; + if not first then Buffer.add_char b ' '; + Buffer.add_string b (String.sub line pos (!cut - pos)); + Buffer.add_string b "\r\n"; + go !cut false + end + in + go 0 true; + Buffer.contents b -- cgit v1.3 From 7228d0634a960c38da96b80378ec617768dfedf7 Mon Sep 17 00:00:00 2001 From: Lukasz Kasprzak Date: Wed, 19 Aug 2026 07:55:53 +0200 Subject: fix(render): make fold_ics total on arbitrary octet strings fold_ics's UTF-8 backoff loop could back `cut` all the way down to `pos` on 74+ consecutive continuation bytes (0x80-0xBF), producing a zero-length chunk and recursing on the identical position forever -- not producible by valid UTF-8, whose longest continuation run is 3, but the kernel's own totality requirement covers arbitrary octet strings, not only valid ones. When backoff finds no boundary inside the window, cut hard at the limit instead, so forward progress is unconditional. test_fold_never_splits_utf8 previously asserted only that unfolding reproduced the original bytes, a property folding preserves at any cut position and therefore blind to a boundary violation. It now also asserts the named property directly: no continuation chunk may start with a UTF-8 continuation byte. A new regression test feeds fold_ics 100 consecutive continuation bytes and asserts it terminates with every line at or under 75 octets. --- lib/render/escape.ml | 8 ++++++++ test/test_escape.ml | 42 ++++++++++++++++++++++++++++++++++-------- 2 files changed, 42 insertions(+), 8 deletions(-) (limited to 'lib/render/escape.ml') diff --git a/lib/render/escape.ml b/lib/render/escape.ml index 53fdd46..f506c44 100644 --- a/lib/render/escape.ml +++ b/lib/render/escape.ml @@ -77,6 +77,14 @@ let fold_ics line = else begin let cut = ref (pos + limit) in while !cut > pos && is_cont line.[!cut] do decr cut done; + (* Valid UTF-8's longest continuation run is 3, so backoff finds a + boundary within a few bytes. But this function must stay TOTAL on + ARBITRARY octet strings, not only valid UTF-8: 74+ consecutive + continuation bytes back `cut` all the way down to `pos`, which would + yield a zero-length chunk and recurse on the identical position + forever. When backoff finds no boundary inside the window, cut hard + at the limit instead -- forward progress is then unconditional. *) + if !cut = pos then cut := pos + limit; if not first then Buffer.add_char b ' '; Buffer.add_string b (String.sub line pos (!cut - pos)); Buffer.add_string b "\r\n"; diff --git a/test/test_escape.ml b/test/test_escape.ml index 1587388..35a9b9d 100644 --- a/test/test_escape.ml +++ b/test/test_escape.ml @@ -70,16 +70,41 @@ let test_fold_long_line () = let test_fold_never_splits_utf8 () = let long = "SUMMARY:" ^ String.concat "" (List.init 40 (fun _ -> "\xc4\x99\xc5\x9b\xc4\x87")) in let out = E.fold_ics long in - let stripped = - String.concat "" - (List.filter_map - (fun l -> - let l = if l <> "" && l.[String.length l - 1] = '\r' then String.sub l 0 (String.length l - 1) else l in - if l = "" then None else if l.[0] = ' ' then Some (String.sub l 1 (String.length l - 1)) else Some l) - (String.split_on_char '\n' out)) + let chunks = + List.filter_map + (fun l -> + let l = if l <> "" && l.[String.length l - 1] = '\r' then String.sub l 0 (String.length l - 1) else l in + if l = "" then None else if l.[0] = ' ' then Some (String.sub l 1 (String.length l - 1)) else Some l) + (String.split_on_char '\n' out) in + (* The named property: no chunk may START with a UTF-8 continuation byte + (0x80-0xBF) -- that would mean the previous fold cut mid-character. + Unfolding losslessly (below) cannot detect this on its own: concatenation + is insensitive to where the cuts fell, so a fold at ANY position still + round-trips. *) + List.iter + (fun c -> + if String.length c > 0 then + Alcotest.(check bool) "chunk does not start mid-UTF-8" true (Char.code c.[0] land 0xC0 <> 0x80)) + chunks; + let stripped = String.concat "" chunks in check "unfolds to the original" long stripped +(* B1 regression: 100 consecutive UTF-8 continuation bytes (0x80-0xBF) is not + producible by valid UTF-8 (whose longest continuation run is 3), but + fold_ics must stay TOTAL on arbitrary octet strings. A backoff loop with no + hard-cut fallback backs `cut` all the way down to `pos`, yielding a + zero-length chunk and recursing on the identical position forever. *) +let test_fold_pathological_input_terminates () = + let pathological = String.make 100 '\x80' in + let out = E.fold_ics pathological in + let lines = String.split_on_char '\n' out in + List.iter + (fun l -> + let l = if l <> "" && l.[String.length l - 1] = '\r' then String.sub l 0 (String.length l - 1) else l in + if String.length l > 75 then Alcotest.failf "line of %d octets exceeds 75" (String.length l)) + (List.filter (fun l -> l <> "") lines) + let suite = ( "Escape", [ Alcotest.test_case "latex" `Quick test_latex; @@ -90,4 +115,5 @@ let suite = Alcotest.test_case "flavour names" `Quick test_flavour_names; Alcotest.test_case "fold: short unchanged" `Quick test_fold_short_line_unchanged; Alcotest.test_case "fold: long line" `Quick test_fold_long_line; - Alcotest.test_case "fold: never splits utf8" `Quick test_fold_never_splits_utf8 ] ) + Alcotest.test_case "fold: never splits utf8" `Quick test_fold_never_splits_utf8; + Alcotest.test_case "fold: pathological input terminates" `Quick test_fold_pathological_input_terminates ] ) -- cgit v1.3