diff options
Diffstat (limited to 'tools/extract_lms_ordo.ml')
| -rw-r--r-- | tools/extract_lms_ordo.ml | 729 |
1 files changed, 729 insertions, 0 deletions
diff --git a/tools/extract_lms_ordo.ml b/tools/extract_lms_ordo.ml new file mode 100644 index 0000000..582fb6e --- /dev/null +++ b/tools/extract_lms_ordo.ml @@ -0,0 +1,729 @@ +(* Task 6 (2026-08-21-colitur-celebrant-rubrics-phase1): turns pdftotext's + -layout dump of the Latin Mass Society's printed Ordo into + test/fixtures/lms-ordo-2024-2025.sexp, one row per UNIVERSAL-calendar + day (England & Wales diocesan variants are recorded as PRESENT via + [has_diocesan_variant] but their own text is discarded -- see the + fixture's own provenance header, and task-6-report.md's "how much of + the Ordo's year is comparable" accounting, for why). + + NOT a general Ordo parser: hand-rolled against ONE PDF's own column + layout (Author "Peter Day-Milne", Producer "LibreOffice 7.4", pdftotext + 1.6), verified line-by-line against the source before being trusted -- + see task-6-report.md for the full characterisation record (the + opposite-prediction probe technique CLAUDE.md's "burned twice" note + demands). Frozen deps forbid Str/regex; every match below is + String.sub/index/split_on_char, the same discipline tools/ + bootstrap_sanctoral.ml's own hand-rolled INI reader already uses for + someone else's format. + + THE KEY STRUCTURAL FACT this parser leans on, established during + characterisation: every day's UNIVERSAL entry (the General Roman + Calendar's own text) appears FIRST in reading order, before any + diocesan variant block (which is introduced by a line ending in ':', + e.g. "Westminster, Clifton, Plymouth:" or "In all Dioceses of ENGLAND + and WALES ... follows:"). So the FIRST "Gl"/"Cr" token pair and the + FIRST "Mass of ..."/roman-numeral-BVM line found within a day's block + are ALWAYS the universal entry's own, even if the colon-boundary + detection below is imperfect -- diocesan text can only ever appear + AFTER, never before, so it cannot introduce a false FIRST match. This + is why [scan_block] restricts its search to lines before the first + colon-terminated line (belt), while the ordering argument above is the + suspenders. + + Usage: + pdftotext -layout docs/research/ordo/lms-ordo-2024-2025.pdf /tmp/lms.txt + dune exec tools/extract_lms_ordo.exe -- /tmp/lms.txt \ + docs/research/ordo/lms-ordo-2024-2025.pdf \ + test/fixtures/lms-ordo-2024-2025.sexp *) + +open Sexplib0.Sexp_conv +module Date = Colitur_kernel.Date + +let die fmt = Printf.ksprintf (fun s -> prerr_endline ("extract_lms_ordo: " ^ s); exit 1) fmt + +(* Mirrored, not shared, by test/test_lms_ordo.ml -- tools/ and test/ have + no common .mli either could hang a shared type from, the same reasoning + test_oracle.ml's own header gives for duplicating [sha256_of_file]. *) +type row = { + date : string; (** ISO-8601 *) + weekday : string; + (** As printed ("Sat", "Sun", ...) -- cross-checked below against + [date]'s own computed weekday at extraction time (fatal on + mismatch: the whole point of carrying both is to catch a date- + resolution bug here, not paper over one). *) + title : string; (** the day's own title line: feast name, class, colour, votive codes *) + formulary_override : string option; + (** the literal "Mass of ..." text, when the universal block names + one; [None] = the day says its own Mass. *) + bvm_numeral : string option; + (** "I".."V" when [formulary_override] is one of the five anchored + "<N> Mass of BVM" lines (whole-line match, never a substring -- + see the module header on why "V Mass of BVM" must never be + matched as a substring of "IV Mass of BVM"). *) + gloria : bool option; (** [None] only if genuinely not found -- reported, not silently dropped *) + creed : bool option; + praef : string option; + (** RAW trailing text from the FIRST "Pr of ..." / "Common Pr" match + in the universal block's own reading order -- CAPTURED, NOT + CLASSIFIED here, the same discipline + tools/extract_fiuv_ordo.ml's own [row.praef] documents for a + different publisher's format. See the provenance header's own + "Pr of" characterisation bullet for what this column can and + cannot show: this Ordo prints OPTION LISTS ("Pr of X or Pr of Y + or Common Pr"), some of whose named options ("Martyrs", "All + Saints and Patron Saints", "the Dedication of a Church", "the + Most Holy Sacrament", "St John the Baptist", "the Angels") are + NOT among the fourteen the 1962 Missale Romanum's own RG 484-497 + enumerate -- ad libitum extras this publisher includes alongside + a genuine RG 482 answer, never the sole option on any row found + in this corpus. Classification into {!Colitur_kernel.Preface.t} + happens test-side (test/test_lms_ordo.ml), by MEMBERSHIP in the + parsed option set, not string equality -- see that file's own + [classify_praef] citation for the full account. *) + has_diocesan_variant : bool; + (** whether a diocese-specific block followed the universal one + anywhere in this day's raw text (informational only -- see the + module header on why correctness of the other fields does not + depend on this being exact). *) +} +[@@deriving sexp] + +(* --- tiny hand-rolled helpers, no Str/regex ------------------------------ *) + +let split_ws s = + String.split_on_char ' ' (String.map (fun c -> if c = '\t' then ' ' else c) s) + |> List.filter (fun t -> t <> "") + +let is_all_digits s = s <> "" && String.for_all (fun c -> c >= '0' && c <= '9') s + +let has_prefix ~prefix s = + String.length s >= String.length prefix && String.sub s 0 (String.length prefix) = prefix + +let ends_with_colon s = s <> "" && s.[String.length s - 1] = ':' + +let months = + [| "January"; "February"; "March"; "April"; "May"; "June"; "July"; "August"; "September"; "October"; + "November"; "December" |] + +let month_index name = + let rec go i = if i >= Array.length months then None else if months.(i) = name then Some (i + 1) else go (i + 1) in + go 0 + +let weekday_of_abbrev = function + | "Sun" -> Some Date.Sun + | "Mon" -> Some Date.Mon + | "Tue" -> Some Date.Tue + | "Wed" -> Some Date.Wed + | "Thu" -> Some Date.Thu + | "Fri" -> Some Date.Fri + | "Sat" -> Some Date.Sat + | _ -> None + +(* Left-column codes (Abbreviations, "Left column": "+EW"/"+"/"Pl"/"Ind") -- + printed in a dedicated margin column that pdftotext -layout reproduces + as literal leading tokens on an otherwise-ordinary content line. Strip + ALL that appear (there can be two, "Pl Ind" together -- the 31 Dec/ + 1 Jan examples in the source). *) +let strip_left_column tokens = + let is_marker = function "Pl" | "Ind" | "+EW" | "+" -> true | _ -> false in + let rec go = function t :: rest when is_marker t -> go rest | ts -> ts in + go tokens + +let bvm_numeral_strings = [ ("I", "I Mass of BVM"); ("II", "II Mass of BVM"); ("III", "III Mass of BVM"); + ("IV", "IV Mass of BVM"); ("V", "V Mass of BVM") ] + +(* Anchored WHOLE-LINE match against all five, never a substring test -- + this is the exact discipline the coordinator addendum records: "V Mass + of BVM" is a substring of "IV Mass of BVM", so any check here MUST + compare the full line, not search for the numeral's own string inside + it. *) +let bvm_numeral_of_line line = List.assoc_opt line (List.map (fun (n, s) -> (s, n)) bvm_numeral_strings) + +(* --- the day-block scanner ------------------------------------------------ *) + +(* [lines]: this day's own raw continuation lines, marker-stripped and + trimmed, title line NOT included (the title never carries Gl/Cr/"Mass + of" itself in this source). Returns (formulary_override, bvm_numeral, + gloria, creed, praef, has_diocesan_variant), each derived from the + portion BEFORE the first colon-terminated (diocesan) line -- see the + module header for why this is belt-and-suspenders, not load-bearing on + its own. *) +let scan_block lines = + let rec universal_prefix = function + | [] -> [] + | l :: _ when ends_with_colon l -> [] + | l :: rest -> l :: universal_prefix rest + in + let has_diocesan = List.exists ends_with_colon lines in + let universal = universal_prefix lines in + let formulary_override, bvm_numeral = + let rec find = function + | [] -> (None, None) + | l :: rest -> ( + match bvm_numeral_of_line l with + | Some n -> (Some l, Some n) + | None -> if has_prefix ~prefix:"Mass of " l then (Some l, None) else find rest) + in + find universal + in + let all_tokens = List.concat_map split_ws universal in + let rec find_flag word = function + | prev :: (cur :: _ as rest) -> if cur = word then Some (prev <> "No") else find_flag word rest + | [ cur ] -> if cur = word then Some true else None + | [] -> None + in + (* [find_flag] needs a sentinel predecessor for a match at position 0 + (never observed in this source -- Gl/Cr always follow at least "No" + or a preceding word on the same "Gl Cr Pr of ..." line -- but handled + rather than assumed: a bare leading "Gl"/"Cr" with no "No" before it + reads as [true], the same as any other non-"No" predecessor would). *) + let gloria = find_flag "Gl" ("" :: all_tokens) in + let creed = find_flag "Cr" ("" :: all_tokens) in + (* [praef]: LINE-SCOPED (not [all_tokens]'s flattened stream), because + the captured text runs to the END of whichever physical line carries + it -- flattening first would glue an unrelated following line's own + prose onto it. "Pr" always abbreviates "Preface" in this source's own + body text (checked directly: its only OTHER appearance is the front- + matter Abbreviations glossary entry "Pr Preface", outside the Ordo + body this tool scans), so a whole-token ("Pr","of") pair, or a bare + ("Common","Pr") pair when no proper/de-Tempore option is offered, is + the sole anchor needed -- no reject-list, unlike the BVM roman + numeral's own substring trap, because nothing else in this corpus's + body ever produces either two-token sequence. *) + let find_praef_in_line line = + let toks = Array.of_list (split_ws line) in + let n = Array.length toks in + let rec go i = + if i + 1 >= n then None + else if (toks.(i) = "Pr" && toks.(i + 1) = "of") || (toks.(i) = "Common" && toks.(i + 1) = "Pr") then + Some (String.concat " " (Array.to_list (Array.sub toks i (n - i)))) + else go (i + 1) + in + go 0 + in + let praef = + let rec find = function + | [] -> None + | l :: rest -> ( match find_praef_in_line l with Some s -> Some s | None -> find rest) + in + find universal + in + (formulary_override, bvm_numeral, gloria, creed, praef, has_diocesan) + +(* --- the top-level line scan --------------------------------------------- *) + +type block = { b_weekday : string; b_day : int; b_month : int; b_year : int; b_title : string; b_lines : string list } + +(* Generalised (Witnesses task, 2026-08-22) from a single hardcoded + sentinel ["End";"of";"November";"2024"], which assumed every edition + opens its Ordo body with an "End of November <year>" stub section. That + assumption FAILED characterisation against the two newly-acquired + editions: the 2023-2024 PDF has NO such stub -- Advent Sunday 2023 fell + on 3 December, so only 2 tail days of the old liturgical year remained + (1-2 December), and the compiler folded them directly into the + "December 2023" header rather than giving them their own mini-section. + So a start marker anchored to "End of November" cannot generalise. + + What DOES generalise, checked directly against all three editions' + pdftotext -layout dumps: exactly one line in the whole document begins + with the bare, all-uppercase token "ORDO" -- either alone ("ORDO", + 2024-2025/2025-2026, introducing an "End of <Month> <Year>" stub next) + or fused with the edition's own year range ("ORDO 2023-2024", + introducing the first REAL month header directly, no stub). This is + NEVER the Table of Contents' own entry, which prints title-case "Ordo + 20XX-20XX" followed by dot leaders and a page number -- a different + string ("Ordo", not "ORDO"), confirmed by grepping all three raw dumps + for both patterns before relying on either. *) +let is_ordo_marker tokens = match tokens with "ORDO" :: _ -> true | _ -> false + +(* The "End of <Month> <Year>" stub heading, when an edition has one -- + parsed directly (never hardcoded to November/a specific year) so the + edition's own initial (month, year) state comes from the text itself, + not an assumption baked into the tool. *) +let parse_end_of_month tokens = + match tokens with + | [ "End"; "of"; m; y ] when is_all_digits y -> ( match month_index m with Some mi -> Some (mi, int_of_string y) | None -> None) + | _ -> None + +let is_stop_tail_line tokens = + match tokens with tok :: _ -> String.length tok = 4 && is_all_digits tok | [] -> false + +let is_month_header tokens = + match tokens with + | m :: rest when month_index m <> None -> List.for_all is_all_digits rest + | _ -> false + +let read_lines path = + let ic = open_in path in + let rec loop acc = match input_line ic with l -> loop (l :: acc) | exception End_of_file -> List.rev acc in + let ls = loop [] in + close_in ic; + ls + +let () = + if Array.length Sys.argv <> 4 then + die "usage: extract_lms_ordo <pdftotext-layout.txt> <source.pdf> <dest.sexp>"; + let txt_path = Sys.argv.(1) in + let pdf_path = Sys.argv.(2) in + let dest = Sys.argv.(3) in + let all_lines = read_lines txt_path in + let arr = Array.of_list all_lines in + let n = Array.length arr in + let trimmed i = String.trim arr.(i) in + let ordo_marker_idx = + let rec go i = if i >= n then die "no line begins with the bare \"ORDO\" body marker (see the tool's own comment on [is_ordo_marker] -- this edition's format may have drifted)" + else if is_ordo_marker (split_ws (trimmed i)) then i else go (i + 1) + in + go 0 + in + (* The first non-blank line after the marker is either an "End of <Month> + <Year>" stub (consumed here as a pure sentinel, exactly as the old + hardcoded design did -- [start_idx] becomes ITS OWN index so the main + loop below begins right after it) or a real "<Month> <Year>" header + with an explicit year token (an edition with no stub, e.g. 2023-2024 -- + [start_idx] becomes the ORDO marker's own index instead, so the main + loop re-encounters this header line on its very first iteration and + [is_month_header]'s own existing per-line handling sets [cur_month]/ + [cur_year] from it normally). Either way [init_month]/[init_year] come + directly from parsed text, never a hardcoded constant -- a missing + explicit year on this FIRST header is a genuine format break, so it + dies loudly rather than silently seeding a wrong year. *) + let rec first_nonblank i = + if i >= n then die "no content found after the ORDO marker at line %d" ordo_marker_idx + else if trimmed i = "" then first_nonblank (i + 1) + else i + in + let content_idx = first_nonblank (ordo_marker_idx + 1) in + let content_toks = split_ws (trimmed content_idx) in + let start_idx, init_month, init_year = + match parse_end_of_month content_toks with + | Some (mi, y) -> (content_idx, mi, y) + | None -> ( + match content_toks with + | m :: (y :: _ as rest) when month_index m <> None && List.for_all is_all_digits rest -> + (ordo_marker_idx, (match month_index m with Some x -> x | None -> assert false), int_of_string y) + | _ -> + die "first content after the ORDO marker (%S) is neither an \"End of <Month> <Year>\" stub nor a \"<Month> <Year>\" header carrying an explicit year" + (String.concat " " content_toks)) + in + let stop_idx = + let rec go i = + if i >= n then n else if is_stop_tail_line (split_ws (trimmed i)) then i else go (i + 1) + in + go (start_idx + 1) + in + Printf.printf "extract_lms_ordo: Ordo body lines %d..%d (of %d total), initial month/year %d/%d\n" start_idx stop_idx n + init_month init_year; + (* --- the scan: month/year tracker + block accumulator ------------------ *) + let cur_month = ref init_month and cur_year = ref init_year in + let blocks = ref [] in + let cur = ref None in + let flush () = match !cur with Some b -> blocks := b :: !blocks; cur := None | None -> () in + for i = start_idx + 1 to stop_idx - 1 do + let raw = trimmed i in + if raw = "" then () + else if String.length raw >= 1 && (let rec contains s sub = + let ls = String.length s and lu = String.length sub in + if lu = 0 then true else if ls < lu then false + else if String.sub s 0 lu = sub then true + else contains (String.sub s 1 (ls - 1)) sub + in + contains raw "Latin Mass Society Ordo") + then () (* page footer *) + else + let toks = split_ws raw in + if is_month_header toks then begin + match toks with + | m :: rest -> ( + match month_index m with + | Some mi -> + if mi <> !cur_month then begin + if mi < !cur_month then incr cur_year; + cur_month := mi + end; + (* an explicit 4-digit year token, when present, overrides + the transition inference outright -- belt and + suspenders, same reasoning as [scan_block] above *) + List.iter (fun t -> if String.length t = 4 && is_all_digits t then cur_year := int_of_string t) rest + | None -> ()) + | [] -> () + end + else + match toks with + | wd :: dn :: _ when weekday_of_abbrev wd <> None && is_all_digits dn && int_of_string dn >= 1 + && int_of_string dn <= 31 -> + flush (); + let title = String.concat " " (List.tl (List.tl toks)) in + cur := Some { b_weekday = wd; b_day = int_of_string dn; b_month = !cur_month; b_year = !cur_year; + b_title = title; b_lines = [] } + | _ -> ( + match !cur with + | None -> () + | Some b -> + let stripped = strip_left_column toks in + if stripped = [] then () + else cur := Some { b with b_lines = b.b_lines @ [ String.concat " " stripped ] }) + done; + flush (); + let blocks = List.rev !blocks in + Printf.printf "extract_lms_ordo: %d day-blocks parsed\n" (List.length blocks); + let rows = + List.map + (fun b -> + let date = + match Date.make ~year:b.b_year ~month:b.b_month ~day:b.b_day with + | Ok d -> d + | Error e -> die "%s %d/%d/%d: %s" b.b_weekday b.b_year b.b_month b.b_day e + in + (match weekday_of_abbrev b.b_weekday with + | Some w when w = Date.weekday date -> () + | Some _ -> + die "%s: printed weekday %S does not match computed weekday %S -- date resolution bug" + (Date.to_iso8601 date) b.b_weekday (Date.weekday_to_string (Date.weekday date)) + | None -> die "%s: unrecognised weekday abbreviation %S" (Date.to_iso8601 date) b.b_weekday); + let formulary_override, bvm_numeral, gloria, creed, praef, has_diocesan_variant = scan_block b.b_lines in + { date = Date.to_iso8601 date; weekday = b.b_weekday; title = b.b_title; formulary_override; bvm_numeral; + gloria; creed; praef; has_diocesan_variant }) + blocks + in + let missing_gloria = List.filter (fun r -> r.gloria = None) rows in + let missing_creed = List.filter (fun r -> r.creed = None) rows in + let missing_praef = List.filter (fun r -> r.praef = None) rows in + if missing_gloria <> [] then + Printf.printf "extract_lms_ordo: WARNING %d rows with no Gloria found: %s\n" (List.length missing_gloria) + (String.concat ", " (List.map (fun r -> r.date) missing_gloria)); + if missing_creed <> [] then + Printf.printf "extract_lms_ordo: WARNING %d rows with no Creed found: %s\n" (List.length missing_creed) + (String.concat ", " (List.map (fun r -> r.date) missing_creed)); + if missing_praef <> [] then + Printf.printf "extract_lms_ordo: WARNING %d rows with no Pr-of/Common-Pr found: %s\n" (List.length missing_praef) + (String.concat ", " (List.map (fun r -> r.date) missing_praef)); + let with_variant = List.length (List.filter (fun r -> r.has_diocesan_variant) rows) in + let with_override = List.length (List.filter (fun r -> r.formulary_override <> None) rows) in + let with_bvm = List.length (List.filter (fun r -> r.bvm_numeral <> None) rows) in + Printf.printf "extract_lms_ordo: %d rows; %d with a diocesan variant present; %d with a formulary override; \ + %d with a BVM roman numeral; %d with no Pr-of/Common-Pr text\n" + (List.length rows) with_variant with_override with_bvm (List.length missing_praef); + (* --- provenance header + write ----------------------------------------- *) + let sha256_of_file path = + let cmd = Printf.sprintf "sha256sum %s" (Filename.quote path) in + let ic = Unix.open_process_in cmd in + let line = try input_line ic with End_of_file -> die "sha256sum produced no output for %s" path in + (match Unix.close_process_in ic with Unix.WEXITED 0 -> () | _ -> die "sha256sum failed for %s" path); + match String.index_opt line ' ' with Some i -> String.sub line 0 i | None -> die "unexpected sha256sum output: %S" line + in + let today () = + let tm = Unix.gmtime (Unix.time ()) in + Printf.sprintf "%04d-%02d-%02d" (tm.Unix.tm_year + 1900) (tm.Unix.tm_mon + 1) tm.Unix.tm_mday + in + let pdf_sha = sha256_of_file pdf_path in + (* --- per-edition provenance + characterisation, Witnesses task --------- + Generalised 2026-08-22 from Task 6's single hand-written header (which + named "lms-ordo-2024-2025.sexp" literally throughout). Facts 1/3/4 + below are STRUCTURAL -- true of the publisher's whole known corpus, + re-checked directly against the two new editions' own pdftotext dumps + before being asserted here again, not merely copied forward. Fact 2 + (the Creed opposite-prediction probes) is inherently edition-specific + (the actual calendar dates differ year to year, and one edition's own + calendar even breaks the standard pairing -- see the 2025-2026 note + below), so it is looked up per [dest] and, critically, RE-VERIFIED + against this run's own [rows] before being printed: a probe that + fails is a [die], not a silently-wrong "confirmed" claim in a fixture + header nobody will re-check by eye. *) + let base = Filename.basename dest in + let pdf_base = Filename.basename pdf_path in + let edition_label, isbn, copyright_year, pdf_meta, url, url_note, window_open_desc = + match base with + | "lms-ordo-2023-2024.sexp" -> + ( "2023-2024", + "978-1-7392096-2-9", + "2023", + {|Author "Peter Day-Milne", Producer "LibreOffice 6.4", CreationDate 2023-11-01, 140 pages|}, + "https://lms.org.uk/sites/default/files/u5374/ordo_2023-2024_2.2_final.pdf", + "identified 2026-08-22 via web search matching the PDF's own edition year to the publisher's own hosting path; NOT independently re-downloaded and byte-compared in this session -- recorded honestly rather than presented as verified", + {|1 December 2023 (this edition carries NO "End of November" stub section at all -- Advent Sunday 2023 fell on 3 December, so only 2 tail days of the prior liturgical year remained, and the compiler folded them directly into the "December 2023" header instead of giving them their own mini-section; see CHARACTERISATION below)|} + ) + | "lms-ordo-2025-2026.sexp" -> + ( "2025-2026", + "978-1-7392096-5-0", + "2025", + {|Author "Peter Day-Milne", Producer "LibreOffice 25.2.6.2 (X86_64) / LibreOffice Community", CreationDate 2025-10-23, 146 pages|}, + "(not found -- see the note on this line)", + "NOT found by URL-pattern web search this session (unlike the 2023-2024 and 2024-2025 editions, no exact download link matching this PDF's own filename/version turned up under lms.org.uk/sites/default/files/u5374/; several near-miss candidates were checked and rejected -- none matched this PDF's own metadata) -- left honestly blank rather than guessed; the file was supplied locally to this session, not fetched by this tool", + {|28 November 2025 ("End of November 2025", the same shape as the 2024-2025 edition -- Advent Sunday 2025 fell on 30 November, so only 3 tail days remained; note this excludes 27 November 2025, which the PRECEDING edition's own window (2024-2025, through 2025-12-31) already covers -- a one-day-short overlap between consecutive editions' own windows, not a gap or a bug in either extraction|} + ) + | "lms-ordo-2024-2025.sexp" -> + (* Preface-witnesses task (2026-08-23): brought into this same + auto-generated, probe-verified header template so all three + editions carry the identical characterisation discipline -- + Task 6's own original HAND-WRITTEN header (never regenerated by + the Witnesses task, which only added table entries for the two + NEW editions) is superseded by this run; every fact below is + re-derived from this PDF's own text/metadata, not copied + blind from that older header (though it agrees with it in + every particular checked). *) + ( "2024-2025", + "9781739209636", + "2024", + {|Author "Peter Day-Milne", Producer "LibreOffice 7.4", CreationDate 2024-11-06, 140 pages|}, + "https://lms.org.uk/sites/default/files/u5374/ordo_2.21_2024-2025.pdf", + "identified via web search matching the PDF's own version number and filename to the publisher's own hosting path (product page: https://lms.org.uk/product/latin-mass-society-ordo-2024-25); NOT independently re-downloaded and byte-compared in this session -- recorded honestly rather than presented as verified", + {|27 November 2024 ("End of November 2024", the same shape as the 2025-2026 edition -- Advent Sunday 2024 fell on 1 December, so 4 tail days remained (27-30 November))|} + ) + | _ -> die "no provenance/characterisation entry for %s -- add one to extract_lms_ordo.ml's own per-edition table before generating this fixture (the tool refuses to emit an under-characterised header)" base + in + let creed_of_date rows d = List.find_map (fun r -> if String.equal r.date d then r.creed else None) rows in + let probes = + match base with + | "lms-ordo-2023-2024.sexp" -> + [ ("2023-12-03", true, "Advent Sunday, RG 475(a)"); + ("2023-12-04", false, "St Peter Chrysologus, III class, ordinary Advent day, RG 476(b)/(d)"); + ("2023-12-26", true, "St Stephen, II class, inside the Nativity octave, RG 475(d)"); + ("2024-08-15", true, "the Assumption, I class, RG 475(b)"); + ("2024-08-16", false, "St Joachim, II class, no Lord/BVM/apostle clause applies, RG 476(b)") ] + | "lms-ordo-2025-2026.sexp" -> + [ ("2025-11-30", true, "Advent Sunday, RG 475(a)"); + ("2025-12-01", false, "ordinary Advent feria, RG 476(b)/(d)"); + ("2025-12-26", true, "St Stephen, II class, inside the Nativity octave, RG 475(d)"); + ( "2026-08-10", false, + "St Laurence, II class martyr, no Lord/BVM/apostle clause applies, RG 476(b) -- St Joachim (16 August) is IMPEDED by a Sunday in this edition's own calendar (12th Sunday after Pentecost), so this edition needed a DIFFERENT sharp non-octave II-class pairing than the 2023-2024/2024-2025 editions used; a real, edition-specific finding, not an extraction quirk" + ); + ("2026-08-15", true, "the Assumption, I class, RG 475(b)") ] + | "lms-ordo-2024-2025.sexp" -> + [ ("2024-12-01", true, "Advent Sunday, RG 475(a)"); + ("2024-12-02", false, "ordinary Advent feria, RG 476(b)/(d)"); + ("2024-12-26", true, "St Stephen, II class, inside the Nativity octave, RG 475(d)"); + ("2025-08-15", true, "the Assumption, I class, RG 475(b)"); + ("2025-08-16", false, "St Joachim, II class, no Lord/BVM/apostle clause applies, RG 476(b)") ] + | _ -> die "no probe list for %s" base + in + List.iter + (fun (d, expected, desc) -> + match creed_of_date rows d with + | Some got when Bool.equal got expected -> () + | Some got -> die "characterisation probe FAILED: %s (%s) -- expected creed=%b, extracted creed=%b" d desc expected got + | None -> die "characterisation probe FAILED: %s (%s) -- no row extracted for this date at all" d desc) + probes; + Printf.printf "extract_lms_ordo: all %d characterisation probes confirmed for %s\n" (List.length probes) edition_label; + (* Preface-witnesses task (2026-08-23): a SECOND opposite-prediction + probe set, over [praef] rather than [creed] -- the same discipline, + applied to the new column this task adds, per the task brief's own + "probe dates where the rule predicts OPPOSITE answers before + adjudicating anything" instruction. Three universal, unambiguous + fixed-date feasts per edition, each landing on a DIFFERENT named + preface (RG 484 Nativity, RG 497 Apostles, RG 499 Requiem) -- checked + directly against this run's own [rows] before this header is + written, a hard [die] on failure, not a printed claim. All Souls' + own date is edition-specific (2 November, TRANSFERRED to 3 November + in the one edition -- 2024-2025 -- where 2 November falls on a + Sunday; verified directly against each PDF's own text, not assumed + from RG 96 alone). *) + let praef_of_date rows d = List.find_map (fun r -> if String.equal r.date d then r.praef else None) rows in + let praef_probes = + match base with + | "lms-ordo-2023-2024.sexp" -> + [ ("2023-12-25", "Pr of the Nativity", "the Nativity of Our Lord, I class, RG 484(a)"); + ("2024-06-29", "Pr of the Apostles", "SS Peter & Paul, I class, RG 497"); + ("2024-11-02", "Pr of the Dead", "All Souls' Day, I class, RG 499") ] + | "lms-ordo-2024-2025.sexp" -> + [ ("2024-12-25", "Pr of the Nativity", "the Nativity of Our Lord, I class, RG 484(a)"); + ("2025-06-29", "Pr of the Apostles", "SS Peter & Paul, I class, RG 497"); + ( "2025-11-03", "Pr of the Dead", + "All Souls' Day, I class, RG 499 -- TRANSFERRED from 2 November, a Sunday in this edition's own calendar" ) + ] + | "lms-ordo-2025-2026.sexp" -> + [ ("2025-12-25", "Pr of the Nativity", "the Nativity of Our Lord, I class, RG 484(a)"); + ("2026-06-29", "Pr of the Apostles", "SS Peter & Paul, I class, RG 497"); + ("2026-11-02", "Pr of the Dead", "All Souls' Day, I class, RG 499") ] + | _ -> die "no praef probe list for %s" base + in + List.iter + (fun (d, expected_prefix, desc) -> + match praef_of_date rows d with + | Some got when has_prefix ~prefix:expected_prefix got -> () + | Some got -> + die "praef characterisation probe FAILED: %s (%s) -- expected prefix %S, extracted %S" d desc + expected_prefix got + | None -> die "praef characterisation probe FAILED: %s (%s) -- no praef extracted for this date at all" d desc) + praef_probes; + Printf.printf "extract_lms_ordo: all %d praef characterisation probes confirmed for %s\n" (List.length praef_probes) + edition_label; + let praef_probe_lines = + String.concat "\n" + (List.map (fun (d, expected_prefix, desc) -> Printf.sprintf "; %s (%s) -> %S" d desc expected_prefix) praef_probes) + in + let missing_praef_count = List.length missing_praef in + let probe_lines = + String.concat "\n" + (List.map + (fun (d, expected, desc) -> Printf.sprintf "; %s (%s) -> %s" d desc (if expected then "TRUE" else "FALSE")) + probes) + in + let header = + Printf.sprintf + {|; %s -- Witnesses task (2026-08-22-colitur-celebrant-rubrics-phase1), +; extending Task 6's single LMS fixture (2024-2025) to three editions of +; the Latin Mass Society's own printed Ordo for England & Wales -- a +; lineage independent of Divinum Officium/missalemeum/lectio (layers 3-4) +; and of the electronic LT.txt transcription (layers 1-2/5): a physically +; printed, professionally-compiled liturgical calendar with its own ISBN. +; +; Generator: tools/extract_lms_ordo.ml -- do not hand-edit; re-run against +; a fresh pdftotext dump and commit the diff instead. +; +; Source: "The Ordo %s", compiled by Peter Day-Milne, The Latin +; Mass Society, 9 Mallow Street, London EC1Y 8RQ (title page and +; colophon, verified directly against the PDF's own extracted text -- +; ISBN %s, Copyright (c) The Latin Mass Society, UK %s). +; PDF metadata: %s. +; URL (%s): +; %s +; The local PDF (docs/research/ordo/%s) is gitignored +; (CLAUDE.md: "docs/ is gitignored"), so this header, not git, is this +; fixture's only record of where it came from. +; SHA-256 of docs/research/ordo/%s: %s +; Extracted (UTC): %s +; Exact commands: +; pdftotext -layout docs/research/ordo/%s /tmp/lms.txt +; dune exec tools/extract_lms_ordo.exe -- /tmp/lms.txt \ +; docs/research/ordo/%s %s +; +; COVERAGE: %d day-rows, %s through %s (starting %s, +; through the day before a malformed trailing entry the extractor +; deliberately stops before -- see CHARACTERISATION below). +; +; CHARACTERISATION FINDINGS (Step 1, non-negotiable per the task brief -- +; the extraordinaryform.org Ordo silently omitted St Lawrence's vigil on +; EVERY date it covered, which nearly produced a false corroboration +; during the RG 33 work; every LMS edition, including this one, is probed +; with the same discipline before a single divergence is adjudicated): +; +; 1. Gl/Cr are printed PER MASS-OPTION, not per day (confirmed again for +; THIS edition: it carries many more Gl/Cr pairs than day-rows, because +; every diocesan variant repeats its own "Gl ... Cr ... Pr of ..." +; line). This extractor resolves the day's OWN office by taking the +; FIRST Gl/Cr pair in reading order, which the source's own layout +; guarantees is the universal entry's (diocesan variants are always +; introduced by a colon-terminated diocese-list line that comes AFTER +; the universal block, never before -- checked structurally, not +; assumed, in every sampled block during development, this edition +; included). +; +; 2. OPPOSITE-PREDICTION PROBES against Rite_ef.Rubrics_ef.creed (RG +; 475-476), RE-CHECKED BY THIS TOOL RUN (not merely copied from a prior +; edition's findings) before a single row of this fixture is trusted -- +; the technique CLAUDE.md records as the one that catches a source +; silently omitting the very thing it is meant to witness. This run's +; own extracted [rows] were checked against every one of the following +; before this header was even written; a failure here is a hard [die], +; not a printed claim: +%s +; All %d predictions confirmed, both directions (TRUE and FALSE each +; independently witnessed) -- this edition is discriminating, not a +; constant, on the Creed. +; +; 3. The BVM-Saturday roman numeral (I-V, "Missae de sancta Maria in +; sabbato", RG 309(a)) is anchored by WHOLE-LINE match against the five +; literal strings below, NEVER by substring search: "V Mass of BVM" is +; a substring of "IV Mass of BVM", so a naive `contains` check +; misreads every fourth-Mass day as the fifth (the coordinator's own +; misreading, recorded and retracted in commit 27b07b4 before Task 6 +; existed; the same anchoring logic runs unchanged for every edition). +; +; 4. SCOPE EXCLUDED, quantified, not silently dropped: +; - Diocesan variants: present on %d of %d days (%s%%) -- their own +; text is read only far enough to set [has_diocesan_variant], +; never compared, because colitur computes the UNIVERSAL General +; Roman Calendar only, with no diocesan overlay loaded. +; - The single malformed trailing entry ("...The OCTAVE DAY of the +; NATIVITY..." with a bare 4-digit token where a weekday+day-number +; pair belongs) is excluded outright: the extractor's own stop +; marker (a line whose first token is a bare 4-digit token) halts +; the scan before it. CONFIRMED IDENTICAL, word for word, across +; ALL FOUR known editions of this publisher's Ordo (2023-2024, +; 2024-2025, 2025-2026, and the 2024-2025 fixture's own prior +; header) -- always the literal stray year token "2025", regardless +; of which edition or which real trailing year it should read: a +; stale copy-paste leftover the compiler's own master document +; carries forward unedited release to release (even the 2024-2025 +; PDF's own tail page footer nearby still reads "Latin Mass Society +; Ordo 2021-2022"), not something specific to this edition. +; - Front matter, the Abbreviations/Rubrical-Primer/Breviary sections, +; and the "When may I say...?"/Appendix tail are never scanned at +; all (the extractor's own start/stop markers bound it to the Ordo +; proper). +; +; 5. THE "Pr of" (PREFACE) COLUMN (Preface-witnesses task, 2026-08-23) -- +; [row.praef], the RAW text from the FIRST "Pr of ..." / "Common Pr" +; match in the universal block's own reading order (the same +; FIRST-in-reading-order discipline finding 1 above already +; establishes for Gl/Cr). CAPTURED, NOT CLASSIFIED by this tool -- +; classification into {!Colitur_kernel.Preface.t} and the comparison +; against {!Rite_ef.Rubrics_ef.preface} both happen test-side +; (test/test_lms_ordo.ml's own [classify_praef]). +; +; CHARACTERISATION FINDING, non-negotiable per the same discipline as +; finding 2: this Ordo prints OPTION LISTS ("Pr of X or Pr of Y or +; Common Pr"), not always a single value -- and several of the named +; options ("Martyrs", "All Saints and Patron Saints" [almost always +; diocesan -- a Patronal feast's own preface -- but checked directly to +; occur ONCE in THIS edition's own UNIVERSAL block too, All Saints' Day +; itself, 1 November, paired with "or Common Pr"/"or Pr of the +; Trinity" depending on whether 1 November falls on a Sunday that +; edition, exactly matching {!Rite_ef.Rubrics_ef.preface}'s own RG +; 494(b) Sunday-gated Trinity fallback either way -- see that +; function's own citation], "the Dedication of a Church", "the Most Holy +; Sacrament" [Corpus Christi's own universal entry], "St John the +; Baptist", "the Angels") are genuinely NOT among the fourteen the +; 1962 Missale Romanum's own RG 484-497 enumerate (docs/research/LT.txt, +; lines 3936-4020, checked directly: no "de Martyribus"/"de Angelis"/ +; "de Dedicatione"/"de Ss.mo Sacramento"/"de S. Ioanne Baptista" clause +; exists anywhere in 482-499). Every one of these extras, in every +; instance found in this edition's UNIVERSAL block, is printed +; ALONGSIDE a genuine RG 482 answer (most often "or Common Pr"; on 24 +; March-shaped days, "or Pr of Lent", the live RG 486(b) de-Tempore +; answer) -- never as the row's OWN AND ONLY option. This is why the +; test-side comparison checks colitur's single computed answer is a +; MEMBER of this row's own parsed option set (dropping any unmapped +; extra name from that set first), not string equality against +; whichever option happens to print first -- the same "capture, don't +; force a match invented at extraction time" discipline +; tools/extract_fiuv_ordo.ml's own [row.praef] doc comment already +; states for a different publisher's format. +; +; OPPOSITE-PREDICTION PROBES against Rite_ef.Rubrics_ef.preface (RG +; 482-499), the SAME discipline as finding 2, applied to this new +; column -- RE-CHECKED BY THIS TOOL RUN before this header was +; written; a failed probe is a hard [die]: +%s +; All %d predictions confirmed -- this column, too, is discriminating, +; not a constant, across three different RG-cited outcomes (Nativity/ +; Apostles/Requiem). +; +; COVERAGE: %d of %d rows have no "Pr of"/"Common Pr" text found at +; all -- checked directly against this window's own Good Friday +; (Easter-2): its block carries NO Gl/Cr/Pr line whatsoever in this +; source (the 1955-restored Holy Week's own liturgical action has no +; Mass that day at all -- the SAME structural fact +; {!test_creed_coverage}'s own [window_good_fridays] already asserts +; for Gl/Cr), so [praef] shares that single expected gap rather than +; having a coverage gap of its own. +|} + dest edition_label isbn copyright_year pdf_meta url_note url pdf_base pdf_base pdf_sha (today ()) pdf_base + pdf_base dest + (List.length rows) + (match rows with r :: _ -> r.date | [] -> "?") + (match List.rev rows with r :: _ -> r.date | [] -> "?") + window_open_desc probe_lines (List.length probes) with_variant (List.length rows) + (Printf.sprintf "%.1f" (100.0 *. float_of_int with_variant /. float_of_int (List.length rows))) + praef_probe_lines (List.length praef_probes) missing_praef_count (List.length rows) + in + let body = Sexplib.Sexp.to_string_hum ~indent:2 (sexp_of_list sexp_of_row rows) in + let oc = open_out dest in + Fun.protect + ~finally:(fun () -> close_out_noerr oc) + (fun () -> + output_string oc header; + output_string oc body; + output_string oc "\n"); + Printf.printf "extract_lms_ordo: wrote %d rows to %s\n" (List.length rows) dest |