diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/extract_lms_ordo.ml | 251 |
1 files changed, 189 insertions, 62 deletions
diff --git a/tools/extract_lms_ordo.ml b/tools/extract_lms_ordo.ml index b7c97fd..8680a91 100644 --- a/tools/extract_lms_ordo.ml +++ b/tools/extract_lms_ordo.ml @@ -167,7 +167,36 @@ let scan_block lines = type block = { b_weekday : string; b_day : int; b_month : int; b_year : int; b_title : string; b_lines : string list } -let start_sentinel = [ "End"; "of"; "November"; "2024" ] +(* Generalised (Witnesses task, 2026-08-22) from a single hardcoded + sentinel ["End";"of";"November";"2024"], which assumed every edition + opens its Ordo body with an "End of November <year>" stub section. That + assumption FAILED characterisation against the two newly-acquired + editions: the 2023-2024 PDF has NO such stub -- Advent Sunday 2023 fell + on 3 December, so only 2 tail days of the old liturgical year remained + (1-2 December), and the compiler folded them directly into the + "December 2023" header rather than giving them their own mini-section. + So a start marker anchored to "End of November" cannot generalise. + + What DOES generalise, checked directly against all three editions' + pdftotext -layout dumps: exactly one line in the whole document begins + with the bare, all-uppercase token "ORDO" -- either alone ("ORDO", + 2024-2025/2025-2026, introducing an "End of <Month> <Year>" stub next) + or fused with the edition's own year range ("ORDO 2023-2024", + introducing the first REAL month header directly, no stub). This is + NEVER the Table of Contents' own entry, which prints title-case "Ordo + 20XX-20XX" followed by dot leaders and a page number -- a different + string ("Ordo", not "ORDO"), confirmed by grepping all three raw dumps + for both patterns before relying on either. *) +let is_ordo_marker tokens = match tokens with "ORDO" :: _ -> true | _ -> false + +(* The "End of <Month> <Year>" stub heading, when an edition has one -- + parsed directly (never hardcoded to November/a specific year) so the + edition's own initial (month, year) state comes from the text itself, + not an assumption baked into the tool. *) +let parse_end_of_month tokens = + match tokens with + | [ "End"; "of"; m; y ] when is_all_digits y -> ( match month_index m with Some mi -> Some (mi, int_of_string y) | None -> None) + | _ -> None let is_stop_tail_line tokens = match tokens with tok :: _ -> String.length tok = 4 && is_all_digits tok | [] -> false @@ -194,21 +223,52 @@ let () = let arr = Array.of_list all_lines in let n = Array.length arr in let trimmed i = String.trim arr.(i) in - let start_idx = - let rec go i = if i >= n then die "start sentinel %S not found" (String.concat " " start_sentinel) - else if split_ws (trimmed i) = start_sentinel then i else go (i + 1) + let ordo_marker_idx = + let rec go i = if i >= n then die "no line begins with the bare \"ORDO\" body marker (see the tool's own comment on [is_ordo_marker] -- this edition's format may have drifted)" + else if is_ordo_marker (split_ws (trimmed i)) then i else go (i + 1) in go 0 in + (* The first non-blank line after the marker is either an "End of <Month> + <Year>" stub (consumed here as a pure sentinel, exactly as the old + hardcoded design did -- [start_idx] becomes ITS OWN index so the main + loop below begins right after it) or a real "<Month> <Year>" header + with an explicit year token (an edition with no stub, e.g. 2023-2024 -- + [start_idx] becomes the ORDO marker's own index instead, so the main + loop re-encounters this header line on its very first iteration and + [is_month_header]'s own existing per-line handling sets [cur_month]/ + [cur_year] from it normally). Either way [init_month]/[init_year] come + directly from parsed text, never a hardcoded constant -- a missing + explicit year on this FIRST header is a genuine format break, so it + dies loudly rather than silently seeding a wrong year. *) + let rec first_nonblank i = + if i >= n then die "no content found after the ORDO marker at line %d" ordo_marker_idx + else if trimmed i = "" then first_nonblank (i + 1) + else i + in + let content_idx = first_nonblank (ordo_marker_idx + 1) in + let content_toks = split_ws (trimmed content_idx) in + let start_idx, init_month, init_year = + match parse_end_of_month content_toks with + | Some (mi, y) -> (content_idx, mi, y) + | None -> ( + match content_toks with + | m :: (y :: _ as rest) when month_index m <> None && List.for_all is_all_digits rest -> + (ordo_marker_idx, (match month_index m with Some x -> x | None -> assert false), int_of_string y) + | _ -> + die "first content after the ORDO marker (%S) is neither an \"End of <Month> <Year>\" stub nor a \"<Month> <Year>\" header carrying an explicit year" + (String.concat " " content_toks)) + in let stop_idx = let rec go i = if i >= n then n else if is_stop_tail_line (split_ws (trimmed i)) then i else go (i + 1) in go (start_idx + 1) in - Printf.printf "extract_lms_ordo: Ordo body lines %d..%d (of %d total)\n" start_idx stop_idx n; + Printf.printf "extract_lms_ordo: Ordo body lines %d..%d (of %d total), initial month/year %d/%d\n" start_idx stop_idx n + init_month init_year; (* --- the scan: month/year tracker + block accumulator ------------------ *) - let cur_month = ref 11 and cur_year = ref 2024 in + let cur_month = ref init_month and cur_year = ref init_year in let blocks = ref [] in let cur = ref None in let flush () = match !cur with Some b -> blocks := b :: !blocks; cur := None | None -> () in @@ -306,11 +366,81 @@ let () = Printf.sprintf "%04d-%02d-%02d" (tm.Unix.tm_year + 1900) (tm.Unix.tm_mon + 1) tm.Unix.tm_mday in let pdf_sha = sha256_of_file pdf_path in + (* --- per-edition provenance + characterisation, Witnesses task --------- + Generalised 2026-08-22 from Task 6's single hand-written header (which + named "lms-ordo-2024-2025.sexp" literally throughout). Facts 1/3/4 + below are STRUCTURAL -- true of the publisher's whole known corpus, + re-checked directly against the two new editions' own pdftotext dumps + before being asserted here again, not merely copied forward. Fact 2 + (the Creed opposite-prediction probes) is inherently edition-specific + (the actual calendar dates differ year to year, and one edition's own + calendar even breaks the standard pairing -- see the 2025-2026 note + below), so it is looked up per [dest] and, critically, RE-VERIFIED + against this run's own [rows] before being printed: a probe that + fails is a [die], not a silently-wrong "confirmed" claim in a fixture + header nobody will re-check by eye. *) + let base = Filename.basename dest in + let pdf_base = Filename.basename pdf_path in + let edition_label, isbn, copyright_year, pdf_meta, url, url_note, window_open_desc = + match base with + | "lms-ordo-2023-2024.sexp" -> + ( "2023-2024", + "978-1-7392096-2-9", + "2023", + {|Author "Peter Day-Milne", Producer "LibreOffice 6.4", CreationDate 2023-11-01, 140 pages|}, + "https://lms.org.uk/sites/default/files/u5374/ordo_2023-2024_2.2_final.pdf", + "identified 2026-08-22 via web search matching the PDF's own edition year to the publisher's own hosting path; NOT independently re-downloaded and byte-compared in this session -- recorded honestly rather than presented as verified", + {|1 December 2023 (this edition carries NO "End of November" stub section at all -- Advent Sunday 2023 fell on 3 December, so only 2 tail days of the prior liturgical year remained, and the compiler folded them directly into the "December 2023" header instead of giving them their own mini-section; see CHARACTERISATION below)|} + ) + | "lms-ordo-2025-2026.sexp" -> + ( "2025-2026", + "978-1-7392096-5-0", + "2025", + {|Author "Peter Day-Milne", Producer "LibreOffice 25.2.6.2 (X86_64) / LibreOffice Community", CreationDate 2025-10-23, 146 pages|}, + "(not found -- see the note on this line)", + "NOT found by URL-pattern web search this session (unlike the 2023-2024 and 2024-2025 editions, no exact download link matching this PDF's own filename/version turned up under lms.org.uk/sites/default/files/u5374/; several near-miss candidates were checked and rejected -- none matched this PDF's own metadata) -- left honestly blank rather than guessed; the file was supplied locally to this session, not fetched by this tool", + {|28 November 2025 ("End of November 2025", the same shape as the 2024-2025 edition -- Advent Sunday 2025 fell on 30 November, so only 3 tail days remained; note this excludes 27 November 2025, which the PRECEDING edition's own window (2024-2025, through 2025-12-31) already covers -- a one-day-short overlap between consecutive editions' own windows, not a gap or a bug in either extraction|} + ) + | _ -> die "no provenance/characterisation entry for %s -- add one to extract_lms_ordo.ml's own per-edition table before generating this fixture (the tool refuses to emit an under-characterised header)" base + in + let creed_of_date rows d = List.find_map (fun r -> if String.equal r.date d then r.creed else None) rows in + let probes = + match base with + | "lms-ordo-2023-2024.sexp" -> + [ ("2023-12-03", true, "Advent Sunday, RG 475(a)"); + ("2023-12-04", false, "St Peter Chrysologus, III class, ordinary Advent day, RG 476(b)/(d)"); + ("2023-12-26", true, "St Stephen, II class, inside the Nativity octave, RG 475(d)"); + ("2024-08-15", true, "the Assumption, I class, RG 475(b)"); + ("2024-08-16", false, "St Joachim, II class, no Lord/BVM/apostle clause applies, RG 476(b)") ] + | "lms-ordo-2025-2026.sexp" -> + [ ("2025-11-30", true, "Advent Sunday, RG 475(a)"); + ("2025-12-01", false, "ordinary Advent feria, RG 476(b)/(d)"); + ("2025-12-26", true, "St Stephen, II class, inside the Nativity octave, RG 475(d)"); + ( "2026-08-10", false, + "St Laurence, II class martyr, no Lord/BVM/apostle clause applies, RG 476(b) -- St Joachim (16 August) is IMPEDED by a Sunday in this edition's own calendar (12th Sunday after Pentecost), so this edition needed a DIFFERENT sharp non-octave II-class pairing than the 2023-2024/2024-2025 editions used; a real, edition-specific finding, not an extraction quirk" + ); + ("2026-08-15", true, "the Assumption, I class, RG 475(b)") ] + | _ -> die "no probe list for %s" base + in + List.iter + (fun (d, expected, desc) -> + match creed_of_date rows d with + | Some got when Bool.equal got expected -> () + | Some got -> die "characterisation probe FAILED: %s (%s) -- expected creed=%b, extracted creed=%b" d desc expected got + | None -> die "characterisation probe FAILED: %s (%s) -- no row extracted for this date at all" d desc) + probes; + Printf.printf "extract_lms_ordo: all %d characterisation probes confirmed for %s\n" (List.length probes) edition_label; + let probe_lines = + String.concat "\n" + (List.map + (fun (d, expected, desc) -> Printf.sprintf "; %s (%s) -> %s" d desc (if expected then "TRUE" else "FALSE")) + probes) + in let header = Printf.sprintf - {|; test/fixtures/lms-ordo-2024-2025.sexp -- Task 6 (2026-08-21-colitur- -; celebrant-rubrics-phase1)'s sixth validation layer: the Latin Mass -; Society's own printed Ordo, an England & Wales diocesan Ordo -- a + {|; %s -- Witnesses task (2026-08-22-colitur-celebrant-rubrics-phase1), +; extending Task 6's single LMS fixture (2024-2025) to three editions of +; the Latin Mass Society's own printed Ordo for England & Wales -- a ; lineage independent of Divinum Officium/missalemeum/lectio (layers 3-4) ; and of the electronic LT.txt transcription (layers 1-2/5): a physically ; printed, professionally-compiled liturgical calendar with its own ISBN. @@ -318,97 +448,94 @@ let () = ; Generator: tools/extract_lms_ordo.ml -- do not hand-edit; re-run against ; a fresh pdftotext dump and commit the diff instead. ; -; Source: "The Ordo 2024-2025", compiled by Peter Day-Milne, The Latin +; Source: "The Ordo %s", compiled by Peter Day-Milne, The Latin ; Mass Society, 9 Mallow Street, London EC1Y 8RQ (title page and ; colophon, verified directly against the PDF's own extracted text -- -; ISBN 9781739209636, Copyright (c) The Latin Mass Society, UK 2024). -; PDF metadata: Author "Peter Day-Milne", Producer "LibreOffice 7.4", -; CreationDate 2024-11-06, 140 pages. -; URL (identified 2026-08-22 via web search matching the PDF's own -; version number and filename to the publisher's own hosting path; -; NOT independently re-downloaded and byte-compared in this session -- -; recorded honestly rather than presented as verified): -; https://lms.org.uk/sites/default/files/u5374/ordo_2.21_2024-2025.pdf -; (product page: https://lms.org.uk/product/latin-mass-society-ordo-2024-25) -; The local PDF (docs/research/ordo/lms-ordo-2024-2025.pdf) is gitignored +; ISBN %s, Copyright (c) The Latin Mass Society, UK %s). +; PDF metadata: %s. +; URL (%s): +; %s +; The local PDF (docs/research/ordo/%s) is gitignored ; (CLAUDE.md: "docs/ is gitignored"), so this header, not git, is this ; fixture's only record of where it came from. -; SHA-256 of docs/research/ordo/lms-ordo-2024-2025.pdf: %s +; SHA-256 of docs/research/ordo/%s: %s ; Extracted (UTC): %s ; Exact commands: -; pdftotext -layout docs/research/ordo/lms-ordo-2024-2025.pdf /tmp/lms.txt +; pdftotext -layout docs/research/ordo/%s /tmp/lms.txt ; dune exec tools/extract_lms_ordo.exe -- /tmp/lms.txt \ -; docs/research/ordo/lms-ordo-2024-2025.pdf test/fixtures/lms-ordo-2024-2025.sexp +; docs/research/ordo/%s %s ; -; COVERAGE: %d day-rows, %s through %s (2024-11-27, "End of November 2024", +; COVERAGE: %d day-rows, %s through %s (starting %s, ; through the day before a malformed trailing entry the extractor ; deliberately stops before -- see CHARACTERISATION below). ; ; CHARACTERISATION FINDINGS (Step 1, non-negotiable per the task brief -- ; the extraordinaryform.org Ordo silently omitted St Lawrence's vigil on ; EVERY date it covered, which nearly produced a false corroboration -; during the RG 33 work; this Ordo was probed with the same discipline -; before a single divergence was adjudicated): +; during the RG 33 work; every LMS edition, including this one, is probed +; with the same discipline before a single divergence is adjudicated): ; -; 1. Gl/Cr are printed PER MASS-OPTION, not per day (confirmed: this PDF -; alone -- not the multi-year corpus the task brief's own grep count -; was measured against -- carries many more Gl/Cr pairs than day-rows, -; because every diocesan variant repeats its own "Gl ... Cr ... Pr of -; ..." line). This extractor resolves the day's OWN office by taking -; the FIRST Gl/Cr pair in reading order, which the source's own layout +; 1. Gl/Cr are printed PER MASS-OPTION, not per day (confirmed again for +; THIS edition: it carries many more Gl/Cr pairs than day-rows, because +; every diocesan variant repeats its own "Gl ... Cr ... Pr of ..." +; line). This extractor resolves the day's OWN office by taking the +; FIRST Gl/Cr pair in reading order, which the source's own layout ; guarantees is the universal entry's (diocesan variants are always ; introduced by a colon-terminated diocese-list line that comes AFTER ; the universal block, never before -- checked structurally, not -; assumed, in every sampled block during development). +; assumed, in every sampled block during development, this edition +; included). ; ; 2. OPPOSITE-PREDICTION PROBES against Rite_ef.Rubrics_ef.creed (RG -; 475-476), checked BEFORE trusting a single row of this fixture -- +; 475-476), RE-CHECKED BY THIS TOOL RUN (not merely copied from a prior +; edition's findings) before a single row of this fixture is trusted -- ; the technique CLAUDE.md records as the one that catches a source -; silently omitting the very thing it is meant to witness: -; 2024-12-01 (Advent Sunday, RG 475(a)) -- Ordo "No Gl Cr" -> TRUE -; 2024-12-02 (ordinary Advent feria) -- Ordo "Gl No Cr" -> FALSE -; 2024-12-26 (St Stephen, II class, but inside the Nativity octave, -; RG 475(d)) -- Ordo "Gl Cr" -> TRUE -; 2025-08-15 (the Assumption, I class, RG 475(b)) -; -- Ordo "Gl Cr" -> TRUE -; 2025-08-16 (St Joachim, II class, no Lord/BVM/apostle clause -; applies, RG 476(b)) -- Ordo "Gl No Cr" -> FALSE -; All five predictions confirmed, both directions (TRUE and FALSE each -; independently witnessed, including the Stephen/Joachim pair, which -; are BOTH II-class non-Sunday saints and differ ONLY on the octave -; question) -- this source is discriminating, not a constant, on the -; Creed. See task-6-report.md for the full record. +; silently omitting the very thing it is meant to witness. This run's +; own extracted [rows] were checked against every one of the following +; before this header was even written; a failure here is a hard [die], +; not a printed claim: +%s +; All %d predictions confirmed, both directions (TRUE and FALSE each +; independently witnessed) -- this edition is discriminating, not a +; constant, on the Creed. ; ; 3. The BVM-Saturday roman numeral (I-V, "Missae de sancta Maria in ; sabbato", RG 309(a)) is anchored by WHOLE-LINE match against the five ; literal strings below, NEVER by substring search: "V Mass of BVM" is ; a substring of "IV Mass of BVM", so a naive `contains` check ; misreads every fourth-Mass day as the fifth (the coordinator's own -; misreading, recorded and retracted in commit 27b07b4 before this -; task existed). +; misreading, recorded and retracted in commit 27b07b4 before Task 6 +; existed; the same anchoring logic runs unchanged for every edition). ; ; 4. SCOPE EXCLUDED, quantified, not silently dropped: ; - Diocesan variants: present on %d of %d days (%s%%) -- their own ; text is read only far enough to set [has_diocesan_variant], ; never compared, because colitur computes the UNIVERSAL General ; Roman Calendar only, with no diocesan overlay loaded. -; - The single malformed trailing entry ("2025 1 The OCTAVE DAY of -; the NATIVITY..." printed with "Thu" wrapped onto the following -; line and the year token "2025" where "2026" belongs -- a literal -; duplicate, word for word, of the fixture's own already-extracted -; 2025-01-01 row) is excluded outright: the extractor's own stop -; marker (a line whose first token is a bare 4-digit year) halts -; the scan before it, so this fixture's last row is 2025-12-31. +; - The single malformed trailing entry ("...The OCTAVE DAY of the +; NATIVITY..." with a bare 4-digit token where a weekday+day-number +; pair belongs) is excluded outright: the extractor's own stop +; marker (a line whose first token is a bare 4-digit token) halts +; the scan before it. CONFIRMED IDENTICAL, word for word, across +; ALL FOUR known editions of this publisher's Ordo (2023-2024, +; 2024-2025, 2025-2026, and the 2024-2025 fixture's own prior +; header) -- always the literal stray year token "2025", regardless +; of which edition or which real trailing year it should read: a +; stale copy-paste leftover the compiler's own master document +; carries forward unedited release to release (even the 2024-2025 +; PDF's own tail page footer nearby still reads "Latin Mass Society +; Ordo 2021-2022"), not something specific to this edition. ; - Front matter, the Abbreviations/Rubrical-Primer/Breviary sections, ; and the "When may I say...?"/Appendix tail are never scanned at -; all (the extractor's own start/stop sentinels bound it to the -; Ordo proper, "End of November 2024" through the stop marker -; above). +; all (the extractor's own start/stop markers bound it to the Ordo +; proper). |} - pdf_sha (today ()) (List.length rows) + dest edition_label isbn copyright_year pdf_meta url_note url pdf_base pdf_base pdf_sha (today ()) pdf_base + pdf_base dest + (List.length rows) (match rows with r :: _ -> r.date | [] -> "?") (match List.rev rows with r :: _ -> r.date | [] -> "?") - with_variant (List.length rows) + window_open_desc probe_lines (List.length probes) with_variant (List.length rows) (Printf.sprintf "%.1f" (100.0 *. float_of_int with_variant /. float_of_int (List.length rows))) in let body = Sexplib.Sexp.to_string_hum ~indent:2 (sexp_of_list sexp_of_row rows) in |