From 9cc763c169a6ce4ebbdbed506c5eb2dad1f9f3c1 Mon Sep 17 00:00:00 2001 From: Lukasz Kasprzak Date: Wed, 26 Aug 2026 13:30:02 +0200 Subject: fix(citation): recognise English-canonical OF books, verse sub-letters 35% of OF citation fields (259/730 on colitur readings --rite of 2026) printed unconverted -- 1 John renders on 2 January but not 3 January. Two independent causes, both in the citation/siglum path, neither in the OF data itself: 1. Book.table only ever surveyed the three EF citation-bearing files, so 345 references citing a book no EF file happens to use (Job, Ruth, Judges, 1/2 Samuel, 1/2 Chronicles, 1/2 Maccabees, Baruch, Ecclesiastes, Habakkuk, Haggai, Nahum, Zechariah, Zephaniah, Deuteronomy, Amos, Micah, Lamentations, Ezra, Joshua, 2/3 John, Jude, Philemon, plus "Isaiah"/"Jeremiah"/"Ezekiel"/"Malachi"/"Mat"/ "The Acts"/"Tobit"/"Song of Solomon" spelling variants of books EF already knows) failed as "unknown book". Added as a new, separate of_lectionary_table rather than folded into the EF-surveyed table: none of these 25 new books is attested in the EF's own scans, and lang/la.ini's own header refuses to fabricate an uncited Latin title, so they are resolvable (parse + render, falling back to their own English spelling) but deliberately excluded from Book.all -- test_lang_coverage.ml's la.ini-completeness promise is preserved exactly for the ids it already covered, not silently weakened. 2. Parse's grammar could not read a verse number carrying a lectionary sub-verse letter ("11a", "1bcde") at all -- the dominant remaining failure shape once (1) was fixed. verse_range now carries a verse_num { n; suffix } on each boundary, PRESERVED through rendering rather than dropped (dropping would silently lose real precision the source text carries). Chapter numbers are untouched (nothing in the data ever attaches a letter to one). A third, subtler bug surfaced by (1): registering "jude"/"philemon"/ "2 John"/"3 John" exposed Parse's existing "leading comma-number is a chapter" heuristic misreading a single-chapter book's bare verse list ("Jude 17,20b-25") as chapter 17 -- a wrong PARSE, worse than the previous safe "unknown book" failure. Book.is_single_chapter now tells Parse to skip that heuristic for the four one-chapter books and default to chapter 1. Residual, honestly enumerated rather than forced to zero: 41 distinct references (of 1540) are hyphenated ranges crossing a chapter boundary ("2:29-3:6") -- a Parse.t shape verse_range/part do not represent, a type restructuring deliberately not attempted this task. Pinned exactly by the new test_citation_coverage_of.ml, both directions (a new failure or one of these 41 starting to convert both go red), and disclosed in data/of/lectionary.sexp's own regenerated provenance header (tools/ bootstrap_lectionary_of.ml now runs the same parser at generation time and names the count and the set). Verified EF-unaffected: git diff v1.0.0..HEAD -- lib/kernel/ lib/rites/rite_ef/ data/ef/ is empty, and `colitur day`/`readings` output for 2027 is byte-identical against the pre-fix binary. --- test/test_citation.ml | 9 ++- test/test_citation_coverage_of.ml | 140 ++++++++++++++++++++++++++++++++++++++ test/test_colitur.ml | 1 + 3 files changed, 147 insertions(+), 3 deletions(-) create mode 100644 test/test_citation_coverage_of.ml (limited to 'test') diff --git a/test/test_citation.ml b/test/test_citation.ml index dae4610..3cbc8c6 100644 --- a/test/test_citation.ml +++ b/test/test_citation.ml @@ -267,10 +267,11 @@ module P = Colitur_citation.Parse (* Render a parse back to a debug string so a test can assert shape compactly: "book|chapter:v-v,v-v|chapter:v". *) let show (t : P.t) = + let num (v : P.verse_num) = string_of_int v.P.n ^ v.P.suffix in let range (r : P.verse_range) = match r.P.last with - | None -> string_of_int r.P.first - | Some l -> Printf.sprintf "%d-%d" r.P.first l + | None -> num r.P.first + | Some l -> Printf.sprintf "%s-%s" (num r.P.first) (num l) in let part (p : P.part) = Printf.sprintf "%d:%s" p.P.chapter @@ -396,7 +397,9 @@ let test_subst_unknown_placeholder_alone () = the assertion isolates exactly the numeral and nothing else. *) let roman_of n = let luke = match B.of_token "Luke" with Some id -> id | None -> assert false in - let t : P.t = { P.book = luke; parts = [ { P.chapter = n; verses = [ { P.first = 1; last = None } ] } ] } in + let t : P.t = + { P.book = luke; parts = [ { P.chapter = n; verses = [ { P.first = { P.n = 1; suffix = "" }; last = None } ] } ] } + in let style = R.style_of_fields [ ("book_sep", ""); ("chapter_verse", "{chapter_roman}") ] in R.render style ~names:(fun _ _ -> "") t diff --git a/test/test_citation_coverage_of.ml b/test/test_citation_coverage_of.ml new file mode 100644 index 0000000..3444706 --- /dev/null +++ b/test/test_citation_coverage_of.ml @@ -0,0 +1,140 @@ +(* SPDX-License-Identifier: AGPL-3.0-or-later *) + +(* Fix wave I2 (final-review.md, 2026-08-25-colitur-of-phases-3-5): the OF + counterpart of test_citation_coverage.ml. That file's own header states + the rule this one follows: "every citation the engine can emit must + parse". EF reaches zero unconverted citations; OF does not, and this + file is the honest record of exactly where it falls short, mirroring the + review's own explicit instruction -- "enumerate them rather than + lowering the bar to fit". + + MEASURED, not estimated: before this fix wave, 259 of 730 citation + fields in `colitur readings --rite of 2026` (35%) printed unconverted -- + 345 distinct book spellings unregistered ({!Colitur_citation.Book}), + plus verse sub-letter markers ("11a") and a handful of single-chapter- + book misreads ({!Colitur_citation.Book.is_single_chapter}) the parser + could not read at all. All of that is now fixed at the source (book.ml's + own [of_lectionary_table] and parse.ml's own [verse_num]/[single_chapter] + additions), closing every genuinely fixable shape. What remains, walking + the FULL data/of/lectionary.sexp (not merely what one civil year happens + to observe -- a stricter check than EF's own year-walk, and the same one + tools/bootstrap_lectionary_of.ml's own generated header now runs and + discloses): 41 distinct references, out of 1540 total, EVERY ONE a + hyphenated range crossing a chapter boundary ("2:29-3:6", "1 John") -- + a {!Colitur_citation.Parse.t} shape [part]/[verse_range] do not + represent (a [part] is one chapter and a list of within-chapter ranges; + representing a cross-chapter range would need restructuring that type + and {!Colitur_citation.Render}'s own consumption of it), deliberately + not built this task. + + This file pins that residual EXACTLY -- not merely "at least these fail" + -- so it fails loudly in BOTH directions: a NEW unconverted reference + appearing (a regression) and one of these 41 starting to convert (this + list going stale, e.g. because a future task builds cross-chapter + support) are both caught, the same discipline every allow-list in this + project's suite already follows (`expected_rows`, `expected_bands`, the + litcal/missalemeum comparators). *) + +let lectionary_path = "../data/of/lectionary.sexp" + +let lectionary () = + match Colitur_kernel.Lectionary.load lectionary_path with + | Ok l -> l + | Error e -> Alcotest.failf "%s: %s" lectionary_path e + +(* The exact, pinned residual -- copied from data/of/lectionary.sexp's own + generated "Citations that do not parse" listing, itself produced by the + SAME {!Colitur_citation.Parse.parse} call this test makes, so the two + can never silently drift apart without one of them visibly failing to + regenerate/re-test clean. *) +let known_cross_chapter_residual = + [ "1 Corinthians 12:31-13:13"; "1 John 1:5-2:2"; "1 John 2:29-3:6"; "1 John 3:22-4:6"; + "1 John 4:19-5:4"; "2 Corinthians 3:15-4:1,3-6"; "2 Samuel 18:9-10,14b,24-25a,30-19:3"; + "Colossians 1:24-2:3"; "Ecclesiastes 11:9-12:8"; "Ephesians 4:32-5:8"; "Exodus 11:10-12:14"; + "Exodus 14:21-15:1"; "Ezekiel 2:8-3:4"; "Galatians 4:22-24,26-27,31-5:1"; "Genesis 1:1-2:2"; + "Genesis 1:20-2:4a"; "Habakkuk 1:12-2:4"; "Hebrews 7:25-8:6"; "Isaiah 52:13-53:12"; + "Isaiah 8:23b-9:3"; "John 15:26-16:4a"; "John 18:1-19:42"; "Jonah 1:1-2:2,11"; "Luke 7:36-8:3"; + "Malachi 1:14b-2:2b,8-10"; "Mark 2:23-3:6"; "Mark 8:34-9:1"; "Matthew 10:34-11:1"; + "Matthew 18:21-19:1"; "Matthew 9:35-10:1,5a,6-8"; "Matthew 9:36-10:8"; + "Numbers 13:1-2,25-14:1,26a-29a,34-35"; "Philippians 3:17-4:1"; "Revelation 20:1-4,11-21:2"; + "Sirach 27:30-28:7"; "The Acts 12:24-13:5a"; "The Acts 17:15,22-18:1"; "The Acts 7:51-8:1a"; + "Wisdom 11:22-12:2"; "Wisdom 2:23-3:9"; "Wisdom 7:22b-8:1" ] + +(* Every failure this walk finds really is a chapter-crossing hyphen range + (a "-" whose right side itself contains a ":") -- checked mechanically, + not merely asserted, so a future genuinely-different failure shape + cannot hide inside this test by coincidentally also being in the pinned + list above. No [Str]/regex (frozen deps): plain character scanning. *) +let looks_chapter_crossing reference = + let n = String.length reference in + let rec scan i saw_dash = + if i >= n then false + else if reference.[i] = '-' then scan (i + 1) true + else if reference.[i] = ':' && saw_dash then true + else scan (i + 1) saw_dash + in + scan 0 false + +let test_of_citations_convert_or_are_the_known_cross_chapter_residual () = + let total = ref 0 in + let bad = ref [] in + List.iter + (fun (_slug, cits) -> + List.iter + (fun (c : Colitur_kernel.Citation.t) -> + incr total; + match Colitur_citation.Parse.parse c.Colitur_kernel.Citation.reference with + | Ok _ -> () + | Error _ -> + if not (List.mem c.Colitur_kernel.Citation.reference !bad) then + bad := c.Colitur_kernel.Citation.reference :: !bad) + cits) + (Colitur_kernel.Lectionary.entries (lectionary ())); + let bad = List.sort compare !bad in + Alcotest.(check int) "1540 citation fields in data/of/lectionary.sexp today -- if this changes, the \ + pinned residual below may need updating too, not just this count" + 1540 !total; + Alcotest.(check (list string)) "the unconverted set is EXACTLY the known cross-chapter residual, no \ + more and no fewer" + (List.sort compare known_cross_chapter_residual) bad; + List.iter + (fun r -> + Alcotest.(check bool) (Printf.sprintf "%s: really is a chapter-crossing range, not some other \ + unrelated failure shape" r) + true (looks_chapter_crossing r)) + bad + +(* Everything OUTSIDE the pinned residual round-trips through Sigla, the + same structural (not textual) comparison test_citation_coverage.ml's own + EF version makes, and for the identical reason its own header gives. *) +let test_round_trip_outside_the_residual () = + let sg = + Colitur_citation.Sigla.make ~style:Colitur_citation.Render.default_style + ~tradition:Colitur_citation.Book.vulgate + ~names:(fun id _form -> Colitur_citation.Book.default_spelling id) + in + let bad = ref [] in + List.iter + (fun (_slug, cits) -> + List.iter + (fun (c : Colitur_kernel.Citation.t) -> + let r = c.Colitur_kernel.Citation.reference in + if not (List.mem r known_cross_chapter_residual) then + match Colitur_citation.Parse.parse r with + | Error e -> bad := Printf.sprintf "%s: unexpectedly does not parse (%s)" r e :: !bad + | Ok first -> ( + let rendered = Colitur_citation.Sigla.format sg r in + match Colitur_citation.Parse.parse rendered with + | Error e -> bad := Printf.sprintf "%s: re-parse failed: %s" r e :: !bad + | Ok again -> if again <> first then bad := Printf.sprintf "%s: structure changed: %s" r rendered :: !bad)) + cits) + (Colitur_kernel.Lectionary.entries (lectionary ())); + Alcotest.(check (list string)) "every OF citation outside the pinned residual round-trips through Sigla" + [] (List.sort compare !bad) + +let suite = + ( "citation-coverage-of", + [ Alcotest.test_case "every OF citation converts, or is the pinned cross-chapter residual" `Slow + test_of_citations_convert_or_are_the_known_cross_chapter_residual; + Alcotest.test_case "every OF citation outside the residual round-trips" `Slow + test_round_trip_outside_the_residual ] ) diff --git a/test/test_colitur.ml b/test/test_colitur.ml index 6c00ee4..a47576b 100644 --- a/test/test_colitur.ml +++ b/test/test_colitur.ml @@ -6,6 +6,7 @@ let () = Test_lang.suite; Test_lang_coverage.suite; Test_citation_coverage.suite; + Test_citation_coverage_of.suite; Test_citation.suite; ("parse", Test_citation.parse_suite); Test_citation.render_suite; -- cgit v1.3