From 9cc763c169a6ce4ebbdbed506c5eb2dad1f9f3c1 Mon Sep 17 00:00:00 2001 From: Lukasz Kasprzak Date: Wed, 26 Aug 2026 13:30:02 +0200 Subject: fix(citation): recognise English-canonical OF books, verse sub-letters 35% of OF citation fields (259/730 on colitur readings --rite of 2026) printed unconverted -- 1 John renders on 2 January but not 3 January. Two independent causes, both in the citation/siglum path, neither in the OF data itself: 1. Book.table only ever surveyed the three EF citation-bearing files, so 345 references citing a book no EF file happens to use (Job, Ruth, Judges, 1/2 Samuel, 1/2 Chronicles, 1/2 Maccabees, Baruch, Ecclesiastes, Habakkuk, Haggai, Nahum, Zechariah, Zephaniah, Deuteronomy, Amos, Micah, Lamentations, Ezra, Joshua, 2/3 John, Jude, Philemon, plus "Isaiah"/"Jeremiah"/"Ezekiel"/"Malachi"/"Mat"/ "The Acts"/"Tobit"/"Song of Solomon" spelling variants of books EF already knows) failed as "unknown book". Added as a new, separate of_lectionary_table rather than folded into the EF-surveyed table: none of these 25 new books is attested in the EF's own scans, and lang/la.ini's own header refuses to fabricate an uncited Latin title, so they are resolvable (parse + render, falling back to their own English spelling) but deliberately excluded from Book.all -- test_lang_coverage.ml's la.ini-completeness promise is preserved exactly for the ids it already covered, not silently weakened. 2. Parse's grammar could not read a verse number carrying a lectionary sub-verse letter ("11a", "1bcde") at all -- the dominant remaining failure shape once (1) was fixed. verse_range now carries a verse_num { n; suffix } on each boundary, PRESERVED through rendering rather than dropped (dropping would silently lose real precision the source text carries). Chapter numbers are untouched (nothing in the data ever attaches a letter to one). A third, subtler bug surfaced by (1): registering "jude"/"philemon"/ "2 John"/"3 John" exposed Parse's existing "leading comma-number is a chapter" heuristic misreading a single-chapter book's bare verse list ("Jude 17,20b-25") as chapter 17 -- a wrong PARSE, worse than the previous safe "unknown book" failure. Book.is_single_chapter now tells Parse to skip that heuristic for the four one-chapter books and default to chapter 1. Residual, honestly enumerated rather than forced to zero: 41 distinct references (of 1540) are hyphenated ranges crossing a chapter boundary ("2:29-3:6") -- a Parse.t shape verse_range/part do not represent, a type restructuring deliberately not attempted this task. Pinned exactly by the new test_citation_coverage_of.ml, both directions (a new failure or one of these 41 starting to convert both go red), and disclosed in data/of/lectionary.sexp's own regenerated provenance header (tools/ bootstrap_lectionary_of.ml now runs the same parser at generation time and names the count and the set). Verified EF-unaffected: git diff v1.0.0..HEAD -- lib/kernel/ lib/rites/rite_ef/ data/ef/ is empty, and `colitur day`/`readings` output for 2027 is byte-identical against the pre-fix binary. --- lib/citation/parse.ml | 105 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 76 insertions(+), 29 deletions(-) (limited to 'lib/citation/parse.ml') diff --git a/lib/citation/parse.ml b/lib/citation/parse.ml index 2f2316e..2c73600 100644 --- a/lib/citation/parse.ml +++ b/lib/citation/parse.ml @@ -1,6 +1,19 @@ (* SPDX-License-Identifier: AGPL-3.0-or-later *) -type verse_range = { first : int; last : int option } +(* Fix wave I2 (final-review.md, 2026-08-25-colitur-of-phases-3-5): a VERSE + number (never a chapter number -- see [num_opt]'s own citation below) may + carry a trailing lowercase-letter sub-verse marker, standard lectionary + notation for "part of this verse" ("11a" = the first clause of verse 11) + and sometimes several run together ("1bcde" = parts b through e of verse + 1). [suffix] is [""] for the overwhelming majority of verse numbers -- + every EF citation, checked: none carries one at all (grep across + data/ef/{lectionary,sanctoral,commons}.sexp finds zero) -- so this is + purely additive for EF and does not change how any existing citation + parses or renders. Carried through and RENDERED, not stripped: dropping + it would silently lose real precision a reader can see in the source + text, trading "wrong format, complete" for "right format, incomplete". *) +type verse_num = { n : int; suffix : string } +type verse_range = { first : verse_num; last : verse_num option } type part = { chapter : int; verses : verse_range list } type t = { book : Book.id; parts : part list } @@ -75,15 +88,38 @@ let int_opt s = if not ok then None else match int_of_string_opt s with Some n when n > 0 -> Some n | _ -> None +(* [num_opt] is [int_opt] widened to accept a trailing sub-verse letter run + ([verse_num]'s own citation above) -- used ONLY for verse numbers inside + [parse_range] below. Chapter numbers ([parse_part]'s own [int_opt c] + calls) are deliberately left on plain [int_opt]: nothing in the data ever + attaches a sub-verse letter to a CHAPTER, and keeping chapter-parsing on + the original, narrower function means this change cannot loosen what a + chapter number is allowed to look like. *) +let split_num_suffix s = + let n = String.length s in + let i = ref 0 in + while !i < n && s.[!i] >= '0' && s.[!i] <= '9' do incr i done; + if !i = 0 then None + else + let digits = String.sub s 0 !i and suffix = String.sub s !i (n - !i) in + if String.for_all (fun c -> c >= 'a' && c <= 'z') suffix then Some (digits, suffix) else None + +let num_opt s : verse_num option = + match split_num_suffix (String.trim s) with + | None -> None + | Some (digits, suffix) -> ( match int_opt digits with Some n -> Some { n; suffix } | None -> None) + (* "20-32" -> {first=20; last=Some 32}; "21" -> {first=21; last=None} *) let parse_range s = match split_on '-' s with - | [ a ] -> ( match int_opt a with Some f -> Some { first = f; last = None } | None -> None) + | [ a ] -> ( match num_opt a with Some f -> Some { first = f; last = None } | None -> None) | [ a; b ] -> ( - match (int_opt a, int_opt b) with + match (num_opt a, num_opt b) with (* A descending range ("1:20-10") is always a transcription error; - accepting it would render back out as a citation nobody can follow. *) - | Some f, Some l when l >= f -> Some { first = f; last = Some l } + accepting it would render back out as a citation nobody can follow. + Compared on the NUMBER only -- "11a-11b" is a real, ascending + sub-verse range even though nothing here orders letters. *) + | Some f, Some l when l.n >= f.n -> Some { first = f; last = Some l } | _ -> None) | _ -> None @@ -97,35 +133,45 @@ let parse_ranges s = pieces (Some []) (* One ";"-separated part. [inherited] is the chapter of the previous part, - used when this one names none (rule 1 in the grammar table). *) -let parse_part ~inherited s = + used when this one names none (rule 1 in the grammar table). + [single_chapter] ({!Book.is_single_chapter}, fix wave I2) skips rule 3 + entirely: a single-chapter book's citations are bare verses with no + chapter at all, so a leading comma-piece that looks like a number is + always a VERSE, never a chapter introduction -- see book.ml's own + citation for the real, wrongly-parsed example this was found on. *) +let parse_part ~single_chapter ~inherited s = match split_on ':' s with | [ c; v ] -> ( (* explicit "chapter:verses" *) match (int_opt c, parse_ranges v) with | Some ch, Some vs -> Some { chapter = ch; verses = vs } | _ -> None) - | [ only ] -> ( - (* Either "chapter, verses" (rule 3) or bare verses inheriting a - chapter. Distinguish on whether the FIRST comma-piece is a lone - number followed by more pieces -- a leading number followed by - at least one further piece is a chapter introduction ("15, 1-46"); - a lone piece on its own, or a part with no more pieces to follow, - can only be verses inheriting the previous chapter. *) - let pieces = split_on ',' only in - match (pieces, inherited) with - | first :: (_ :: _ as rest), _ when int_opt first <> None && String.contains only ',' -> ( - (* "15, 1-46" -> chapter 15. This never fires for a part that - already named its chapter via ":" -- those match the [c; v] - branch above and never reach here. *) - match (int_opt first, parse_ranges (String.concat "," rest)) with - | Some ch, Some vs -> Some { chapter = ch; verses = vs } - | _ -> None) - | _, Some ch -> ( - match parse_ranges only with - | Some vs -> Some { chapter = ch; verses = vs } - | None -> None) - | _, None -> None) + | [ only ] -> + if single_chapter then + match parse_ranges only with + | Some vs -> Some { chapter = Option.value inherited ~default:1; verses = vs } + | None -> None + else ( + (* Either "chapter, verses" (rule 3) or bare verses inheriting a + chapter. Distinguish on whether the FIRST comma-piece is a lone + number followed by more pieces -- a leading number followed by + at least one further piece is a chapter introduction ("15, 1-46"); + a lone piece on its own, or a part with no more pieces to follow, + can only be verses inheriting the previous chapter. *) + let pieces = split_on ',' only in + match (pieces, inherited) with + | first :: (_ :: _ as rest), _ when int_opt first <> None && String.contains only ',' -> ( + (* "15, 1-46" -> chapter 15. This never fires for a part that + already named its chapter via ":" -- those match the [c; v] + branch above and never reach here. *) + match (int_opt first, parse_ranges (String.concat "," rest)) with + | Some ch, Some vs -> Some { chapter = ch; verses = vs } + | _ -> None) + | _, Some ch -> ( + match parse_ranges only with + | Some vs -> Some { chapter = ch; verses = vs } + | None -> None) + | _, None -> None) | _ -> None let parse s = @@ -144,10 +190,11 @@ let parse s = (* A trailing ";" leaves an empty piece: drop it rather than failing. Only if nothing remains is it an error. *) let pieces = List.filter (fun p -> p <> "") (split_on ';' tail) in + let single_chapter = Book.is_single_chapter book in let rec go inherited = function | [] -> Ok [] | p :: rest -> ( - match parse_part ~inherited p with + match parse_part ~single_chapter ~inherited p with | None -> Error ("cannot read reference: " ^ p) | Some part -> ( match go (Some part.chapter) rest with -- cgit v1.3