diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/bootstrap_lectionary.ml | 70 | ||||
| -rw-r--r-- | tools/dune | 27 | ||||
| -rw-r--r-- | tools/extract_efdotorg_ordo.py | 336 | ||||
| -rw-r--r-- | tools/extract_fiuv_ordo.ml | 855 | ||||
| -rw-r--r-- | tools/extract_lms_ordo.ml | 729 |
5 files changed, 2017 insertions, 0 deletions
diff --git a/tools/bootstrap_lectionary.ml b/tools/bootstrap_lectionary.ml index 7df290b..1e9b8ed 100644 --- a/tools/bootstrap_lectionary.ml +++ b/tools/bootstrap_lectionary.ml @@ -637,6 +637,75 @@ let trinity_week_entries = lectionary has been bitten by before. *) [ "monday"; "tuesday"; "wednesday"; "friday"; "saturday" ] +(* RG 299's own escape clause, second instance (celebrant-rubrics-phase1, + Bug 2, 2026-08-22). The Proprium de Tempore carries an explicit + marginal rubric immediately after the Ascension's own Mass propers, + before "Dominica post Ascensionem" begins (scan1.txt:20749-20754, both + scans corroborate): "Diebus a feria VI post Ascensionem usque ad + feriam VI ante vigiliam Pentecostes inclusive, dicitur Missa de festo + Ascensionis, cum Gloria et praefatione de Ascensione, sine Credo et + Communicantes proprio" -- on the days from the Friday after the + Ascension until the Friday before the Vigil of Pentecost INCLUSIVE, + the MASS OF THE FEAST OF THE ASCENSION is said (with the Gloria and + the Ascension preface, without the Creed and the proper + Communicantes). Corroborated independently by RG 490(b) (LT.txt:3981- + 3985), which assigns the Ascension preface "tamquam de Tempore" to + every Mass in that same window lacking a proper preface of its own -- + consistent only if the Mass repeated there is the Ascension's own, + not a numbered Sunday's (which would need no preface override at + all). + + Witnessed directly, not merely inferred from the rubric: the LMS Ordo + prints "Mass of the Ascension" literally, across all three fixture + windows this project has built (2023-2024: 1 day; 2024-2025: 3 days; + 2025-2026: 2 days -- data/ef/expected-divergences-lms.sexp's own L2, + closed by this entry), on EVERY affected day in BOTH the remainder of + Ascension's own week and the following week, not only the first -- + ruling out the alternative reading that the second week instead + resumes "Dominica post Ascensionem"'s own Mass. The FIUV Ordo's own + title convention for these days (2025-2026 fixture, 2026-05-21/22, + "Feria V/VI Dom. post Ascensionem, De ea") could look like the + opposite at a glance -- it is not: FIUV's title names which WEEK a + feria belongs to (a bookkeeping label, the same role colitur's own + week-numbered slug plays), while its own [praef] field for those + exact two days independently reads "Ascensionis", confirming the Mass + actually said is still the Ascension's, exactly as the marginal + rubric and RG 490(b) both require. + + The window's two Saturdays (Easter+41, Easter+48) are DELIBERATELY + NOT given entries here. Easter+48 is the Vigil of Pentecost, a named + day with its own Mass ({!Rite_ef.Temporal_ef.named}), outside the + rubric's own "usque ad feriam VI" upper bound. Easter+41 is an + ordinary IV-class Saturday, so {!Rite_ef.Lectionary_ef.readings}'s + own RG 78/RG 309(a) votive-Mass-of-Our-Lady-on-Saturday branch ALWAYS + intercepts it before step 2 is ever reached (checked directly: + [is_bvm_saturday_office] fires unconditionally on any unoccupied + Class4 Saturday, which every ordinary Paschaltide Saturday is) -- an + entry for either Saturday slug would be dead data that + [assert_reachable] cannot catch (both slugs it would name ARE real, + reachable Temporal_ef offices; the shadowing happens one level up, in + [readings]' own step ordering, not in slug reachability). + + Six ferias, matching the Ascension's own citation already given above + (Act. 1, 1-11 / Marc. 16, 14-20, scan1.txt:20701-20730, + [ef-ascension]'s own entry): the one remaining non-Saturday day of + Ascension's own week (Easter+40) and the five weekdays of the + following week (Easter+43..47), bounded above by that week's own + Friday -- the Saturday after it is the Vigil of Pentecost, named, + never reaching this table. *) +let ascension_week_citations = pair ~first:"Acts 1:1-11" ~gospel:"Mark 16:14-20" + +let ascension_week_entries = + List.map + (fun slug -> (slug_or_die slug, ascension_week_citations)) + [ "ef-easter-6-friday"; + "ef-easter-7-monday"; + "ef-easter-7-tuesday"; + "ef-easter-7-wednesday"; + "ef-easter-7-thursday"; + "ef-easter-7-friday" + ] + let epiphanytide_opening_entries = List.concat_map (fun (family, wds, cs) -> @@ -789,6 +858,7 @@ let () = translated @ vigil_entries @ holy_week_entries @ passion_tuesday_entry @ ember_saturday_corrections @ nativity_octave_entries @ holy_name_entries @ epiphanytide_opening_entries @ trinity_week_entries @ movable_feast_entries + @ ascension_week_entries in assert_reachable entries; let lect = @@ -23,6 +23,20 @@ (name bootstrap_lectionary) (libraries colitur_kernel rite_ef unix sexplib)) +; Task 6 (2026-08-21-colitur-celebrant-rubrics-phase1): turns pdftotext's +; -layout dump of the Latin Mass Society Ordo into test/fixtures/ +; lms-ordo-2024-2025.sexp. Run via `dune exec tools/extract_lms_ordo.exe -- +; <pdftotext-layout.txt> <source.pdf> <dest.sexp>`. Same `unix`-for- +; `sha256sum` usage as the two bootstrap executables above; `colitur_kernel` +; only, no `rite_ef` -- this tool validates dates through `Date.make` but +; reads no EF-specific vocabulary. +(executable + (name extract_lms_ordo) + (modules extract_lms_ordo) + (libraries colitur_kernel unix sexplib) + (preprocess + (pps ppx_sexp_conv))) + ; check_citations.py's own self-test (test_check_citations.py). Python, not ; OCaml, so it cannot be a `(test ...)` stanza -- an alias rule invoking it ; directly is dune's own documented shape for a non-OCaml check. Wired into @@ -41,3 +55,16 @@ (deps check_citations.py test_check_citations.py) (action (run python3 test_check_citations.py))) + +; Witnesses task (2026-08-22-colitur-celebrant-rubrics-phase1): turns +; pdftotext's -layout dump of the FIUV universal Ordo into +; test/fixtures/fiuv-ordo-2025-2026.sexp. Run via `dune exec +; tools/extract_fiuv_ordo.exe -- <pdftotext-layout.txt> <source.pdf> +; <dest.sexp>`. Same `unix`-for-`sha256sum` usage as extract_lms_ordo.ml; +; `colitur_kernel` only, no `rite_ef` -- same reasoning, dates only. +(executable + (name extract_fiuv_ordo) + (modules extract_fiuv_ordo) + (libraries colitur_kernel unix sexplib) + (preprocess + (pps ppx_sexp_conv))) diff --git a/tools/extract_efdotorg_ordo.py b/tools/extract_efdotorg_ordo.py new file mode 100644 index 0000000..05ef014 --- /dev/null +++ b/tools/extract_efdotorg_ordo.py @@ -0,0 +1,336 @@ +#!/usr/bin/env python3 +# tools/extract_efdotorg_ordo.py -- efdotorg-preface task +# (2026-08-24-colitur-celebrant-rubrics-phase1). Turns the ALREADY-PARSED +# extraordinaryform.org corpus (docs/research/ordo/ordo-2024-2027.json, +# 1091 days, produced by docs/research/ordo/parse_ordo.py and audited for +# fidelity in .superpowers/ordo-fidelity-report.md -- 1091/1091 against an +# independently-written parser) into three sexp fixtures, +# test/fixtures/efdotorg-ordo-{2024-2025,2025-2026,2026-2027}.sexp, one row +# per calendar day, mirroring how tools/extract_fiuv_ordo.ml and +# tools/extract_lms_ordo.ml shape their own fixtures. +# +# PYTHON, NOT OCAML -- same reasoning tools/extract_missalemeum_oracle.py's +# own header already gives: there is no JSON library in this project's +# frozen deps (dune alcotest qcheck qcheck-alcotest sexplib ppx_sexp_conv), +# and hand-writing one is out of scope ("no hand-written JSON parser" is +# about colitur's OWN sexp format, not a licence to reimplement JSON). The +# JSON itself is already the parsed artefact ("do not rebuild the parser" -- +# this script does not re-read the PDFs at all, only the JSON parse_ordo.py +# already produced from them). +# +# Usage: +# python3 tools/extract_efdotorg_ordo.py \ +# docs/research/ordo/ordo-2024-2027.json test/fixtures +# +# Writes the three fixtures named above into the given output directory. +# +# --------------------------------------------------------------------------- +# CHARACTERISATION (re-derived independently against the live JSON below, +# not copied from .superpowers/ordo-fidelity-report.md's own prose summary +# -- that report's own filter was RE-RUN here and its counts reproduced +# exactly, which is the independent confirmation, not a citation of trust): +# +# 1. THE MENU TRAP (element [0] is not reliably the day's own office). +# Re-derived over the whole 1091-day JSON: walk each day's Mass-option +# list; skip a leading entry whose `mass` starts with "Requiem:" (a +# votive Mass for the Dead, listed ahead of the real office on an +# unimpeded low-class day -- 99 instances, confirmed never the SOLE +# option and never followed by a second Requiem entry); then, if the +# NEXT entry's `mass` starts with "and " (a preliminary rite -- Blessing +# of Ashes/Palms, Candlemas, Holy Saturday's daytime office -- joined to +# its real companion Mass by that literal prefix), treat that next entry +# as the office instead (12 instances). 99 + 12 = 111, exactly the +# figure ordo-fidelity-report.md's own independent audit already found +# -- reproduced, not merely trusted. +# +# 2. THE "CHRIST THE KING" CLASSIFIER GAP -- a NEW finding, not in the +# prior fidelity audit. Of the 1091 office-selected rows, exactly 6 have +# `preface: null`: 3 Good Friday (genuinely no preface -- the +# 1955-restored Holy Week's Good Friday has no Mass at all, RG 28) and 3 +# Christ the King (25/26/31 October across the three editions). The +# latter is a real extraction-classifier gap, not a source blank: +# docs/research/ordo/parse_ordo.py's own hardcoded `PREFACES` set (its +# module-level constant) does not include "Christ the King" as a +# recognised preface name, so its own `opt()` function's `pref = f[0] if +# f and f[0] in PREFACES else None` falls through and files the literal +# string "Christ the King" into `comms` instead of `preface` -- checked +# directly against all three instances, `comms == ['Christ the King']` +# on every one, no other None-preface row has this shape. RECOVERED +# here (reading `comms` when `preface` is null and `comms` is exactly +# `['Christ the King']`), not silently left blank, and not a parser +# rewrite -- parse_ordo.py itself and its JSON output are untouched; +# this is a documented, characterised repair applied only inside this +# comparison-layer generator, reading a field the JSON already carries. +# +# 3. "ADVENT" IS NOT A PREFACE, AND NEITHER IS ITS MAPPING A CONSTANT. +# RG 484-499 name no Advent preface among the fourteen; this source's +# own "Advent" label is an informal convention (plausibly Novus-Ordo- +# influenced) covering BOTH of two genuinely different 1962 outcomes: +# - RG 494(b), "in dominicis Adventus... praefatio de Ss.ma +# Trinitate" -- every ADVENT SUNDAY takes the TRINITY preface as +# its own de-Tempore grant (lib/rites/rite_ef/rubrics_ef.ml's own +# [preface] function already implements this, cross-checked against +# the FIUV and LMS Ordos independently -- see that function's own +# header). Confirmed live in this JSON: "Advent" is the raw label on +# every Advent Sunday in the corpus too (11 of the 12 possible +# Advent-Sunday instances across three editions; the twelfth, +# 2024-12-08, is impeded by the Immaculate Conception, a fixed +# I-class BVM feast, and correctly carries "BVM" instead). +# - RG 498 (the residual Common) governs every Advent FERIA (a +# non-Sunday day with no proper of its own). +# So "Advent" is NOT simply "colitur's Common" (an earlier, declined +# version of this task's own reasoning) -- it is Trinity on a Sunday, +# Common otherwise, decided by the civil weekday alone (never by +# parsing this source's own text further, and never by trusting +# colitur's own answer circularly: the civil weekday is public, +# independently computable information). Encoded in the comparator +# (test/test_efdotorg_ordo.ml), not baked into this generator, because +# it needs the day's own weekday, which this fixture's `date` field +# already carries losslessly. +# +# 4. THE KNOWN VIGIL-OF-ST-LAWRENCE DEFECT (docs/research/ordo/ +# PROVENANCE-ordo-corpus.md, .superpowers/ordo-class-report.md) is +# RECONFIRMED present at the same two dates, unchanged: 2025-08-09 and +# 2027-08-09 both list "St. Romanus, Martyr" (cls 4) as the day's own +# office, with no Vigil of St Lawrence entry anywhere in either day's +# option list (2026-08-09 is a Sunday, impeding the vigil on every +# source, so it is not part of this population). NOT re-derived from +# scratch -- already fully adjudicated in the class/colour layers +# named above; carried forward here as a known fact to characterise +# against, not a new investigation. +# +# 5. TWO ALREADY-DOCUMENTED SOURCE-PDF DEFECTS (ordo-fidelity-report.md) +# do not need a fresh finding here, only a check of whether they touch +# the preface field at all -- they do not, checked directly: +# - 2027-03-03 is missing from the JSON outright (a source PDF +# row-drop); 2027-03-02 carries 03-03's own Wednesday content under +# 03-02's date instead. Both dates' own `preface` value happens to +# read "Lent" regardless of which day's content is attached (both +# are ordinary Lenten ferias in the same week), so the mislabelling +# does not propagate into a preface divergence -- checked, not +# assumed. 2027-03-03 (the missing day) is still emitted as its own +# "no data" row below (mass/cls/colour/praef all absent), the same +# shape Good Friday's genuine blank already uses, so the fixture +# keeps one row per calendar day. +# - 2026-12-13 (Gaudete Sunday) carries the wrong COLOUR/commemoration +# in the source PDF (a known, isolated editorial error, already +# allow-listed for class/colour purposes elsewhere) -- its own +# `preface` value is untouched ("Advent", exactly like every other +# Advent Sunday), confirmed directly against this JSON. +# +# 6. RULE A -- "SATURDAY OF OUR LADY" IS UNCONDITIONALLY THE OFFICE. A +# second, NEW office-selection finding, beyond the two patterns in +# finding 1 above: 38 days across the whole corpus list an entry whose +# `mass` is exactly "Saturday of Our Lady" (RG 78's own votive BVM +# Saturday Office) ALONGSIDE another named saint's own optional Mass +# (structurally identical to the Requiem pattern -- a low-class +# "priest's menu" option printed ahead of, or beside, the real office). +# Checked EXHAUSTIVELY, not sampled: on all 38, colitur's own real +# computed office is this BVM Office (`dune exec bin/main.exe -- +# rubrics <year>`, cross-checked against `day`'s own "Officium sanctae +# Mariae in sabbato" line) -- including the ONE case where a further +# listed entry outranks every OTHER candidate on the page by class +# number (18 January 2025's own "Chair of St. Peter (for Church +# Unity)", cls 2): that entry is not even part of the 1962 General +# Roman Calendar at all (the single Roman Chair-of-Peter feast was +# consolidated to 22 February by the 1960 Rubricae Generales; colitur's +# own sanctoral data has no 18 January entry for it, and neither the +# LMS nor the FIUV Ordo -- both already-validated witnesses -- ever +# name it), so its presence does not defeat the rule. UNCONDITIONAL, +# not merely "no lower-class competitor": empirically the simpler, +# equally correct reading of the whole 38-instance population. +# +# (USA)/(Can)-TAGGED REGIONAL INDULTS were checked and found NEVER to be +# the JSON's own element [0] anywhere in the whole 1091-day corpus (0 +# instances) -- so, unlike ordo-fidelity-report.md's own +# class/colour-comparison method, THIS generator needs no indult- +# stripping step: Rules 1/6 above never reach far enough into an +# option list for an indult's presence to matter. Checked, not assumed. +# +# 7. THE RESIDUAL -- 11 rows, all 3 editions combined, 7 distinct root +# causes, every one individually adjudicated in +# data/ef/expected-divergences-efdotorg.sexp (not attempted here, a +# THIRD, generic "absorption" rule was deliberately NOT built: it would +# have resolved 2 of these 11 rows (Our Lady of Mt Carmel, 16 July) but +# ALSO wrongly mis-fired on a genuinely different shape (Ss Philip & +# James/Rogation Day, 11 May, where an equal-CLASS "Rogation Day" +# option's own [comms] names the real feast for the OPPOSITE reason -- +# an optional votive substitution, not a rank-based demotion -- and the +# JSON alone cannot distinguish the two shapes without already knowing +# colitur's own Commemoration_only/Feast status distinction, which +# would defeat the point of an independent witness). Manufacturing a +# rule that fixes 2 cases while breaking a 3rd is a worse trade than +# leaving all of them to individual adjudication. +# +# --------------------------------------------------------------------------- +import datetime +import json +import subprocess +import sys + +EDITIONS = [ + ("2024-2025", "2024-12-01", "2025-11-29"), + ("2025-2026", "2025-11-30", "2026-11-28"), + ("2026-2027", "2026-11-29", "2027-11-27"), +] +# Boundaries re-derived, not copied: both PDF-to-PDF seams are exact (no +# overlap, no gap) and 364 + 364 + 363 = 1091, the JSON's own total day +# count -- checked below in main(), not merely asserted here. + + +def sha256_of(path): + out = subprocess.run(["sha256sum", path], capture_output=True, text=True, check=True) + return out.stdout.split()[0] + + +def pick_office(opts): + """The menu-trap filter -- see this module's own header, findings 1 and 6.""" + i = 0 + if opts[0]["mass"].startswith("Requiem:"): + i = 1 + if i < len(opts) - 1 and opts[i + 1]["mass"].startswith("and "): + i += 1 + # Rule A (finding 6): "Saturday of Our Lady", whenever listed at all, IS + # the office -- checked exhaustively over all 38 instances across the + # whole 1091-day corpus (not a sample): colitur's own RG 78 BVM-Saturday + # Office wins on every single one, including the one case where a + # competing entry outranks every OTHER real candidate on the page (a + # Chair-of-St-Peter, 18 January, entry that is not even part of the + # 1962 calendar at all -- see finding 6). No guard/condition needed; + # unconditional preference is the empirically safe rule. + bvm_saturday = next((o for o in opts if o["mass"] == "Saturday of Our Lady"), None) + if bvm_saturday is not None: + return bvm_saturday + return opts[i] + + +def sexp_string(s): + out = [] + for ch in s: + b = ch.encode("utf-8") + if ch == '"': + out.append('\\"') + elif ch == "\\": + out.append("\\\\") + elif len(b) == 1 and 32 <= b[0] < 127: + out.append(ch) + else: + for byte in b: + out.append("\\%03d" % byte) + return '"' + "".join(out) + '"' + + +def sexp_opt_string(s): + return "(" + sexp_string(s) + ")" if s is not None else "()" + + +def sexp_opt_int(n): + return f"({n})" if n is not None else "()" + + +def daterange(lo, hi): + d = datetime.date.fromisoformat(lo) + end = datetime.date.fromisoformat(hi) + while d <= end: + yield d.isoformat() + d += datetime.timedelta(days=1) + + +def build_rows(data, lo, hi): + rows = [] + for date in daterange(lo, hi): + opts = data.get(date) + if opts is None: + # 2027-03-03 only -- see this module's own header, finding 5. + rows.append((date, None, None, None, None)) + continue + office = pick_office(opts) + praef = office["preface"] + if praef is None and office.get("comms") == ["Christ the King"]: + praef = "Christ the King" # RECOVERED -- see finding 2 above. + rows.append((date, office["mass"], office["cls"], office["colour"], praef)) + return rows + + +def write_fixture(out_path, label, lo, hi, rows, json_path, json_sha): + n_missing = sum(1 for r in rows if r[1] is None) + n_none_praef = sum(1 for r in rows if r[1] is not None and r[4] is None) + with open(out_path, "w") as f: + f.write(f"""; test/fixtures/efdotorg-ordo-{label}.sexp -- efdotorg-preface task +; (2026-08-24-colitur-celebrant-rubrics-phase1): the THIRD independent +; preface witness (RG 482-499), extraordinaryform.org's {label} Ordo, +; extracted from the already-parsed, fidelity-audited corpus +; docs/research/ordo/ordo-2024-2027.json (SHA-256 {json_sha}) rather than +; re-parsed from the PDF -- see tools/extract_efdotorg_ordo.py's own +; header for the full characterisation this fixture rests on (the menu- +; trap office filter, the Christ-the-King classifier-gap recovery, why +; "Advent" is not baked into a single mapping here, the known Vigil-of- +; St-Lawrence gap, and the two source-PDF defects that turn out not to +; touch this field). +; +; Generator: tools/extract_efdotorg_ordo.py -- do not hand-edit; re-run +; against the same JSON and commit the diff instead. +; python3 tools/extract_efdotorg_ordo.py {json_path} test/fixtures +; +; Source PDF (gitignored, CLAUDE.md: "docs/ is gitignored", see +; docs/research/ordo/SHA256SUMS for its own hash): {label}Ordo.pdf, +; extraordinaryform.org. +; +; COVERAGE: {len(rows)} day-rows, {lo}..{hi} (this edition's own civil-date +; span, re-derived from the JSON's own seams -- see the generator's own +; EDITIONS comment). {n_missing} row(s) are a genuine SOURCE GAP (2027-03-03 +; only, a PDF row-drop, ordo-fidelity-report.md) -- mass/cls/colour/praef +; all absent, the same shape a structurally blank day (Good Friday) uses. +; {n_none_praef} further row(s) have mass/cls/colour but no praef at all +; (Good Friday -- RG 28, no Mass in the 1955-restored Holy Week). +; +; Each row's `praef` is RAW TEXT from the day's own OFFICE (after the +; menu-trap filter, with the Christ-the-King recovery already applied) -- +; classified against {{!Colitur_kernel.Preface.t}} by +; test/test_efdotorg_ordo.ml, not here. +""") + f.write("(\n") + for date, mass, cls, colour, praef in rows: + f.write( + f" ((date {date}) (mass {sexp_opt_string(mass)}) (cls {sexp_opt_int(cls)}) " + f"(colour {sexp_opt_string(colour)}) (praef {sexp_opt_string(praef)}))\n" + ) + f.write(")\n") + print(f"wrote {len(rows)} rows to {out_path}", file=sys.stderr) + print(f"SHA-256 of {out_path}: {sha256_of(out_path)}", file=sys.stderr) + + +def main(): + if len(sys.argv) != 3: + print(f"usage: {sys.argv[0]} <ordo-2024-2027.json> <out-dir>", file=sys.stderr) + sys.exit(2) + json_path, out_dir = sys.argv[1], sys.argv[2] + with open(json_path) as f: + data = json.load(f) + json_sha = sha256_of(json_path) + + # The three windows span 1092 CALENDAR days (one more than the JSON's + # own 1091 keys, because 2027-03-03 is a genuine source gap -- see + # finding 5 above); of those, exactly 1091 must be present as JSON + # keys and exactly 1 (2027-03-03 alone) absent, or the boundaries are + # wrong. + all_dates = [d for _, lo, hi in EDITIONS for d in daterange(lo, hi)] + present = [d for d in all_dates if d in data] + missing = [d for d in all_dates if d not in data] + if len(all_dates) != 1092 or len(present) != 1091 or missing != ["2027-03-03"]: + print( + f"ERROR: edition boundaries cover {len(all_dates)} calendar days, " + f"{len(present)} present in the JSON, missing={missing} " + "(expected 1092/1091/['2027-03-03'])", + file=sys.stderr, + ) + sys.exit(1) + + for label, lo, hi in EDITIONS: + rows = build_rows(data, lo, hi) + out_path = f"{out_dir}/efdotorg-ordo-{label}.sexp" + write_fixture(out_path, label, lo, hi, rows, json_path, json_sha) + + +if __name__ == "__main__": + main() diff --git a/tools/extract_fiuv_ordo.ml b/tools/extract_fiuv_ordo.ml new file mode 100644 index 0000000..26f5b16 --- /dev/null +++ b/tools/extract_fiuv_ordo.ml @@ -0,0 +1,855 @@ +(* Witnesses task (2026-08-22-colitur-celebrant-rubrics-phase1): turns + pdftotext's -layout dump of the FIUV (Foederatio Internationalis Una + Voce) universal Ordo into test/fixtures/fiuv-ordo-2025-2026.sexp, one + row per day, 2025-11-27..2026-12-31. + + A DIFFERENT format from tools/extract_lms_ordo.ml's own LMS PDFs in + every way that matters (see docs/research/ordo/PROVENANCE-ordo-corpus.md): + UNIVERSAL (no diocesan variants to exclude at all -- one block per day, + not "the first of several"), IN LATIN, using the rubrics' own + vocabulary ("Missa pr., Gloria, sine Credo, praef. comm."), and BOTH + directions of Gloria/Credo are stated explicitly in the source's own + words ("Gloria"/"sine Gloria", "Credo"/"sine Credo") rather than by a + printed flag letter. Hand-rolled, no Str/regex (frozen deps) -- the + same String.sub/index/split_on_char discipline extract_lms_ordo.ml + already uses for a different publisher's layout. + + THE KEY STRUCTURAL FACTS this parser leans on, established during + characterisation (see the fixture's own provenance header for the + full record): + + 1. A day-start line's own FIRST token is always "<1-2 digit day>." + (e.g. "27.", "3."), immediately followed by a colour code (not + anchored on below -- colour is captured as raw prose, never parsed + into a closed set, because a few real days use a COMPOUND colour + ("Viol. in Off., Alb. in Missa." on Holy Thursday) or drop the + token to a blank cell entirely (13 December 2026, a rendering + quirk) -- neither is needed for anything this task compares). + + 2. Month headers/running-footers name a month EITHER in English with a + trailing ROMAN-NUMERAL year ("November MMXXV", the book's own first + two months only) OR bare in Latin with no year at all + ("Januarius", "Februarius", ... every month after). The English + Nov/Dec spellings happen to be IDENTICAL to their Latin + equivalents, so one 12-entry table covers both vocabularies. + CHARACTERISATION FINDING: the SECOND header ("December MMXV") is a + publisher-side TYPO -- MMXV is 2015, ten years off, contradicted by + every neighbouring date and by month-order inference alone. This + parser therefore reads a Roman-numeral year ONLY off the very + FIRST header encountered (the anchor) and tracks every subsequent + month purely by wraparound inference (increment the year exactly + once, at the Dec->Jan transition) -- the typo is never read at all, + not merely tolerated. + + 3. Every real Mass rubric line starts with the literal token "Missa" + (401 of ~400 real day-blocks, confirmed structurally). A SECOND, + alternate Mass option, when the day offers one, is introduced by + the CAPITALISED two-token sequence "Vel Missa" -- never bare lower- + case "vel" alone, which is the ordinary Latin conjunction "or" and + appears constantly inside ordinary prose (e.g. "praef. comm. vel de + Martyribus", part of ONE preface's own text, not a second Mass + option) -- so only "Vel Missa", both tokens, anchors a real + alternate-option boundary. This parser reads Gloria/Credo/praef. + from the FIRST Mass option only, bounded above by whichever comes + first: "Vel Missa" or "VESPERAE"/"VESPERA" -- the identical + "day's own PRIMARY office, not a menu entry" discipline + extract_lms_ordo.ml already uses for the LMS diocesan-variant + colon boundary. + + 4. SUBSTRING TRAP, found and defended against: "Gloria" is also the + first word of "Gloria Patri" (the psalm doxology, "Glory be to the + Father"), a DIFFERENT liturgical unit that has nothing to do with + whether the Mass's own Gloria in excelsis is said -- and it can + appear INSIDE the very Mass clause being scanned (Good Friday: + "Missa pr., (omittuntur ps. Iudica me et Gloria Patri), Gloria, + sine Credo, praef. comm." -- the real, standalone "Gloria," follows + immediately after). [find_word] below rejects any "Gloria" hit + whose very next token is "Patri", the same whole-token discipline + extract_lms_ordo.ml's own header describes for "V Mass of BVM" + being a substring of "IV Mass of BVM". + + 5. A WHOLE-SECOND-OFFICE TRAP, found and defended against + (celebrant-rubrics-phase1 Phase 2 correction, 2026-08-22, closing + data/ef/expected-divergences-fiuv.sexp's own F3): six real days + print BOTH a strict-1962 reading (a privileged Lenten/Passiontide + feria, saint reduced to a commemoration, "Off. feriale") AND, after + a literal separator token, a SECOND, complete alternative office + (the same saint kept outright under the 2020 decree Cum + Sanctissima, "Off. ordinarium", with its OWN "Ad Mat. ... Te Deum" + lessons). The separator is an en-dash, "VEL" (Latin "or"), another + en-dash, glued by pdftotext into ONE token with no internal spaces + -- "\xe2\x80\x93VEL\xe2\x80\x93" ("VEL" wrapped in en-dashes) -- + confirmed by direct byte inspection of the raw dump, NOT the same + shape as [extract_missa_fields]'s own "Vel"+"Missa" two-token + boundary above (a narrower "choice of Mass formulary within the + SAME office" marker, e.g. a Common vs a Proper Mass for one saint). + [extract_missa_fields]'s own Gloria/Credo/praef span happened to + stay correctly bounded to the PRIMARY office on all six real + instances even before this fix, because a Vespers line ("VESPER..." + -- already one of its own boundary tokens) always intervenes before + the separator in this corpus; [extract_te_deum] had NO such + protection at all -- it scans the WHOLE block's own text for "Ad + Mat." unbounded, so on these six days (whose PRIMARY office states + no "Ad Mat." of its own at all -- an ordinary privileged feria's + Matins needs no special note) it read the SECOND office's own "Ad + Mat. ... Te Deum." instead, manufacturing a same-day contradiction + that F3 first recorded as "colitur's own possible gap". [split_at_alt_office] + below cuts every block's own token array at this separator BEFORE + any field extraction runs, so every extractor now reads the PRIMARY + office only; the alternative is captured, not discarded, in its own + [alt_te_deum] field (the row type's own comment has the full + account of why only this one field, not a parallel alt_gloria/ + alt_credo/alt_class -- those three were never wrong, so widening + them now would be undirected scope creep, not a fix). + + Usage: + pdftotext -layout docs/research/ordo/fiuv-ordo-2025-2026.pdf /tmp/fiuv.txt + dune exec tools/extract_fiuv_ordo.exe -- /tmp/fiuv.txt \ + docs/research/ordo/fiuv-ordo-2025-2026.pdf \ + test/fixtures/fiuv-ordo-2025-2026.sexp *) + +open Sexplib0.Sexp_conv +module Date = Colitur_kernel.Date + +let die fmt = Printf.ksprintf (fun s -> prerr_endline ("extract_fiuv_ordo: " ^ s); exit 1) fmt + +(* Mirrored, not shared, by test/test_fiuv_ordo.ml -- tools/ and test/ + have no common .mli either could hang a shared type from, the same + reasoning extract_lms_ordo.ml's own header already gives. *) +type row = { + date : string; (** ISO-8601 *) + class_ : string option; (** raw, e.g. "III cl.", "III cl. (Priv.)", "I cl." -- [None] only if genuinely unparseable, read off the PRIMARY office alone *) + title : string; (** the day-start line's own text before the class marker, PRIMARY office alone *) + te_deum : bool option; (** [None] only if neither "Te Deum" nor "non dicitur Te Deum"/"sine Te Deum" is found WITHIN THE PRIMARY OFFICE (see [split_at_alt_office]) *) + gloria : bool option; (** PRIMARY office *) + credo : bool option; (** PRIMARY office *) + praef : string option; (** raw trailing text after "praef." within the primary Mass clause -- CAPTURED, NOT VALIDATED *) + alt_te_deum : bool option; + (** the SECOND, Cum-Sanctissima-alternative office's own Te Deum + status, when this day prints one (see [split_at_alt_office]'s + own citation) -- [None] both when the day has no such + alternative at all (394 of 400 real days) AND when it has one + but that alternative's own text doesn't state Te Deum either + (never observed live: on all six real instances the alternative + states it unnegated). Captured rather than discarded because a + FUTURE Cum-Sanctissima overlay (CLAUDE.md's own binding + decision 2 -- an addition, never core) would have exactly this + shape to compare against; not compared by anything in this + project yet. Deliberately NOT joined by an [alt_gloria]/ + [alt_credo]/[alt_class] -- see [split_at_alt_office]'s own + citation for why only this one field was ever wrong. *) +} +[@@deriving sexp] + +(* The literal separator between a day's PRIMARY (strict 1962) office and + a printed SECOND, Cum-Sanctissima-2020 alternative -- see the module + header's own point 5 for the full account of the bug this closes + (data/ef/expected-divergences-fiuv.sexp's own F3) and why it is a + DIFFERENT token from [extract_missa_fields]'s own "Vel"+"Missa" + two-token boundary. Exactly 6 occurrences in the whole corpus, + confirmed by direct grep of the raw pdftotext dump -- all 6 on the + exact dates F3 named (2026-03-06/07/09/12/21/24). *) +let alt_office_marker = "\xe2\x80\x93VEL\xe2\x80\x93" + +let split_at_alt_office tokens = + let n = Array.length tokens in + let rec go i = if i >= n then None else if tokens.(i) = alt_office_marker then Some i else go (i + 1) in + match go 0 with + | None -> (tokens, None) + | Some i -> (Array.sub tokens 0 i, Some (Array.sub tokens (i + 1) (n - i - 1))) + +(* --- tiny hand-rolled helpers, no Str/regex ------------------------------ *) + +let split_ws s = + String.split_on_char ' ' (String.map (fun c -> if c = '\t' then ' ' else c) s) + |> List.filter (fun t -> t <> "") + +let is_all_digits s = s <> "" && String.for_all (fun c -> c >= '0' && c <= '9') s + +let has_prefix ~prefix s = + String.length s >= String.length prefix && String.sub s 0 (String.length prefix) = prefix + +(* Strips ASCII trailing punctuation only (.,();) -- tokens in this source + never carry other trailing marks that matter to comparisons below. *) +let rec strip_trailing_punct s = + let n = String.length s in + if n = 0 then s + else + match s.[n - 1] with + | '.' | ',' | ')' | ';' | ':' -> strip_trailing_punct (String.sub s 0 (n - 1)) + | _ -> s + +let contains s ~sub = + let ls = String.length s and lu = String.length sub in + if lu = 0 then true + else + let rec go i = if i + lu > ls then false else if String.sub s i lu = sub then true else go (i + 1) in + go 0 + +let index_of s ~sub = + let ls = String.length s and lu = String.length sub in + if lu = 0 then Some 0 + else + let rec go i = if i + lu > ls then None else if String.sub s i lu = sub then Some i else go (i + 1) in + go 0 + +let months = + [| "Januarius"; "Februarius"; "Martius"; "Aprilis"; "Maius"; "Junius"; "Julius"; "Augustus"; "September"; + "October"; "November"; "December" |] + +let month_index name = + let rec go i = if i >= Array.length months then None else if months.(i) = name then Some (i + 1) else go (i + 1) in + go 0 + +(* Roman numerals, subtractive form -- only ever called once, on the + FIRST month header's own year token (the anchor); see the module + header on why every LATER header's own year, when present at all, is + deliberately never read. *) +let roman_value = function + | 'I' -> Some 1 | 'V' -> Some 5 | 'X' -> Some 10 | 'L' -> Some 50 | 'C' -> Some 100 | 'D' -> Some 500 + | 'M' -> Some 1000 | _ -> None + +let parse_roman s = + let n = String.length s in + let rec go i acc = + if i >= n then Some acc + else + match roman_value s.[i] with + | None -> None + | Some v -> ( + let next = if i + 1 < n then roman_value s.[i + 1] else None in + match next with + | Some nv when nv > v -> go (i + 2) (acc + nv - v) + | _ -> go (i + 1) (acc + v)) + in + if n = 0 then None else go 0 0 + +(* --- day-block scanning --------------------------------------------------- *) + +type block = { b_day : int; b_month : int; b_year : int; b_lines : string list } + +let is_day_start_line toks = + match toks with + | t :: _ :: _ when String.length t >= 2 && t.[String.length t - 1] = '.' -> + let core = String.sub t 0 (String.length t - 1) in + if is_all_digits core then + let d = int_of_string core in + if d >= 1 && d <= 31 then Some d else None + else None + | _ -> None + +(* A month running-header/footer line: first token is a recognised month + name (English Nov/Dec spelled identically to Latin), optionally + followed by a Roman-numeral year and/or a page number -- ALWAYS pure + noise for the day-block scanner (never a continuation line's own + content), but the FIRST time a given month index is seen it also + drives the [cur_month]/[cur_year] state transition. *) +let is_month_marker_line toks = match toks with m :: _ -> month_index m <> None | [] -> false + +(* Left-margin indulgence-notation codes ("Ind." = an indulgence is + available, "Plen." = a plenary one, "DFP" unexplained but structurally + identical -- all three print in the SAME left-margin column pdftotext + -layout preserves, glued onto the following content line as literal + leading tokens, the identical trap extract_lms_ordo.ml's own + [strip_left_column] already documents for the LMS PDFs' "Pl"/"Ind" + column. Found by tracing a real parse failure (8 December 2025's own + class marker split across two physical lines by "DFP" sitting between + "I" and "cl."), not assumed in advance: catalogued via a frequency + scan of every short capitalised token opening a continuation line, and + deliberately does NOT include "RM" (11 occurrences, ALSO short and + capitalised) -- checked directly, "RM" is a real citation abbreviation + ("Rubricae Missalis", e.g. "vide RM 440"), not a margin code, so + stripping it would corrupt real text for no parsing benefit. *) +let strip_left_column tokens = + let is_marker t = t = "Ind." || t = "Plen." || t = "DFP" in + let rec go = function t :: rest when is_marker t -> go rest | ts -> ts in + go tokens + +(* The malformed trailing entry's own distinctive two-token prefix, + "1st Jan" -- NOT a generic "first token is a bare 4-digit number" + check (extract_lms_ordo.ml's own approach): that generic shape false- + positived here, on real body content deep in October ("...20 augusti + / 1885, 26 augusti 1886..." -- an indulgence-decree date citation + whose line-wrap happens to put a 4-digit year token first on its own + physical line). This source's OWN tail artefact is reliably + distinguished only by its literal, unique "1st Jan" opening -- found + by tracing the false stop, not assumed in advance. *) +let is_stop_tail_line toks = match toks with "1st" :: "Jan" :: _ -> true | _ -> false + +let read_lines path = + let ic = open_in path in + let rec loop acc = match input_line ic with l -> loop (l :: acc) | exception End_of_file -> List.rev acc in + let ls = loop [] in + close_in ic; + ls + +(* --- per-block field extraction ------------------------------------------- *) + +(* First WHOLE-TOKEN hit of [word] in [tokens] starting at-or-after + [from], REJECTING a hit whose immediately-following token (stripped) + is [reject_next] -- the "Gloria Patri" trap, see the module header. + Returns the hit's own index and whether the token immediately BEFORE + it (stripped) is "sine" (a negation). *) +let find_word tokens ~from ~word ~reject_next = + let n = Array.length tokens in + let rec go i = + if i >= n then None + else if strip_trailing_punct tokens.(i) = word then + let next_is_rejected = i + 1 < n && strip_trailing_punct tokens.(i + 1) = reject_next in + if next_is_rejected then go (i + 1) + else + let negated = i > 0 && strip_trailing_punct tokens.(i - 1) = "sine" in + Some (i, negated) + else go (i + 1) + in + go from + +(* [class_]: the roman-numeral token immediately followed by a token + starting "cl" (covers "cl.", "cl.,"), whole-token matched -- never a + substring search (the same discipline the LMS extractor's own BVM + numeral anchor uses, for the identical reason: "I" is a substring of + "II"/"III"/"IV"). Also detects the trailing "(Priv.)" two tokens later. *) +let extract_class tokens = + let n = Array.length tokens in + let is_class_numeral t = t = "I" || t = "II" || t = "III" || t = "IV" in + let rec go i = + if i + 1 >= n then None + else if is_class_numeral tokens.(i) && has_prefix ~prefix:"cl" tokens.(i + 1) then + let base = tokens.(i) ^ " cl." in + if i + 2 < n && strip_trailing_punct tokens.(i + 2) = "(Priv" then Some (base ^ " (Priv.)") else Some base + else go (i + 1) + in + go 0 + +let extract_title tokens = + let n = Array.length tokens in + let is_class_numeral t = t = "I" || t = "II" || t = "III" || t = "IV" in + let rec find i = if i + 1 >= n then n else if is_class_numeral tokens.(i) && has_prefix ~prefix:"cl" tokens.(i + 1) then i else find (i + 1) in + let stop = find 0 in + String.concat " " (Array.to_list (Array.sub tokens 0 stop)) + +(* Te Deum: bounded to the "Ad Mat." .. "Ad Laudes" span specifically + (both casings occur: "Ad MAT."/"Ad Mat."/"ad Mat.", "Ad LAUDES"/ + "Ad Laudes") -- NOT the whole block, because a day can separately + mention "Te Deum" in an unrelated INDULGENCE note (31 December: "Hodie + ad solemnem recitationem hymni Te Deum, indulgentia plenaria lucrari + potest" -- about gaining an indulgence for the New Year's Eve Te Deum + of thanksgiving, not about whether it is sung at that day's own + Matins) that a whole-block search would wrongly read as a positive hit. *) +let extract_te_deum full_text = + let find_ci needles start = + List.fold_left + (fun acc needle -> + match acc with + | Some _ -> acc + | None -> ( + match index_of (String.sub full_text start (String.length full_text - start)) ~sub:needle with + | Some i -> Some (start + i) + | None -> None)) + None needles + in + match find_ci [ "Ad Mat."; "Ad MAT."; "ad Mat." ] 0 with + | None -> None + | Some mat_start -> ( + let laudes_start = + match find_ci [ "Ad Laudes"; "Ad LAUDES"; "ad Laudes" ] mat_start with + | Some i -> i + | None -> String.length full_text + in + let span = String.sub full_text mat_start (laudes_start - mat_start) in + (* TWO negative phrasings the source actually uses, found live + (celebrant-rubrics-phase1 Phase 2, 2026-08-22): "non dicitur Te + Deum" (e.g. ordinary Time-after-Pentecost ferias, 27/28 November) + AND, separately, "sine Te Deum" (e.g. every Sunday, every Ember + day, every privileged Lenten/Passiontide feria carrying a + commemoration -- Advent I, 30 November: "...3a de homilia (cum + suo R), sine Te Deum."). The ORIGINAL version of this function + checked only the first phrasing, so "sine Te Deum" fell through + to the bare "Te Deum" substring test and was wrongly read as a + POSITIVE hit -- confirmed by grepping the raw pdftotext dump + directly against a first, uncorrected run's own output: EVERY + date this bug affected showed "sine Te Deum" in the source and + [Some true] in the fixture, a 100% correlation, not a handful of + coincidences. This is the SAME "does the source negate the + hymn's own name with a DIFFERENT word than the one this parser + already checks for" shape [find_word]'s own "Gloria Patri" + substring trap already documents for Gloria -- this trap simply + went unnoticed until Phase 2 actually compared the extracted + values against colitur's own output and against the raw text by + hand, rather than only checking coverage counts. *) + if contains span ~sub:"non dicitur Te Deum" then Some false + else if contains span ~sub:"sine Te Deum" then Some false + else if contains span ~sub:"Te Deum" then Some true + else None) + +(* Gloria/Credo/praef.: bounded to the FIRST Mass option only -- from the + token "Missa" to whichever comes first: "Vel" immediately followed by + "Missa" (a real second option), or a token starting "VESPER" (Vespers + info -- a PREFIX match, not one or two hardcoded exact spellings, + found necessary by tracing a real over-capture: this source uses at + least four surface forms depending on grammatical case and whether + pdftotext renders the Æ ligature as one glyph or splits it -- + "VESPERÆ", "VESPERAS", "VESPERA" (Ad VESPERA, rare) and "VESPERÆ" + with the ligature reproduced as a single non-ASCII codepoint an exact- + string match against the two ASCII spellings alone could never catch, + which let a whole day's own trailing Vespers/Compline prose leak into + [praef] uncaught until checked against the task brief's own worked + example day, 19 September), or the end of the block. *) +let extract_missa_fields tokens = + let n = Array.length tokens in + (* [Missa] (singular, 549 of the corpus's own occurrences of any "Miss-" + word) anchors 398 of the 400 real days; the two genuine exceptions + are Good Friday and Holy Saturday, which have no Mass at all in the + 1955-restored Holy Week (matching extract_lms_ordo.ml's own + identically-shaped finding for the LMS corpus). CHRISTMAS DAY is a + THIRD, found by tracing a real false-None result: its own rubric + reads "Hodie celebrantur tres Missae pr., Gloria, Credo..." ("today + three Masses are celebrated..."), using the PLURAL "Missae"/"Missæ" + (both spellings occur across the fixture's two Christmas Days) -- + [tokens.(i) = "Missa"] alone never matches it, silently returning + [None] for Gloria/Credo/praef on the single most doctrinally + unambiguous day in the whole calendar. Widened to accept "Missae"/ + "Missæ" too, guarded against the ONE real collision risk: "Hodie + prohibentur omnes Missae defunctorum..." (a Requiem-Mass-prohibition + notice, common at the end of many day-blocks, unrelated to the + day's own Mass) also contains "Missae" -- rejected here by checking + the immediately FOLLOWING token is not "defunctorum". *) + let missa_idx = + let rec go i = + if i >= n then None + else if (tokens.(i) = "Missa" || tokens.(i) = "Missae" || tokens.(i) = "Missæ") + && not (i + 1 < n && strip_trailing_punct tokens.(i + 1) = "defunctorum") + then Some i + else go (i + 1) + in + go 0 + in + match missa_idx with + | None -> (None, None, None) + | Some m -> + let end_idx = + let rec go i = + if i >= n then n + else if tokens.(i) = "Vel" && i + 1 < n && tokens.(i + 1) = "Missa" then i + (* A THIRD, narrower alternate-Mass marker, found while auditing + for the "-VEL-" trap above (celebrant-rubrics-phase1 Phase 2 + correction, 2026-08-22): "Vel (sec. decretum <<Cum sanct.>>) + Missa ..." offers an alternate MASS FORMULARY under the 2020 + decree Cum Sanctissima WITHIN the same single printed office + (a narrower thing than [split_at_alt_office]'s own full + second-office split above -- there is no second "Ad Mat." set + of lessons here, just a second Mass choice). Checked against + the raw dump directly: 20 occurrences, ALL 20 of the + capitalised "Vel" + open-paren shape, zero false positives + (the ordinary lower-case "vel" conjunction, e.g. "praef. + comm. vel de Martyribus", never starts with an open-paren + token, so this check never fires on it). Every one of the 20 + real days already restates Gloria/Credo explicitly BEFORE + reaching this marker, so this was NEVER a live divergence + (confirmed by a direct diff of every field this fixture + compares, before and after this hardening: zero rows + changed) -- added defensively, so a FUTURE year whose + primary Mass clause happens not to restate Gloria/Credo + before this marker cannot silently read the Cum-Sanctissima + alternative's own value instead, the identical failure shape + as the "-VEL-" bug above, just not yet observed live here. *) + else if tokens.(i) = "Vel" && i + 1 < n && has_prefix ~prefix:"(" tokens.(i + 1) then i + else if has_prefix ~prefix:"VESPER" tokens.(i) then i + else go (i + 1) + in + go (m + 1) + in + let span = Array.sub tokens m (end_idx - m) in + let gloria = + match find_word span ~from:0 ~word:"Gloria" ~reject_next:"Patri" with + | Some (_, negated) -> Some (not negated) + | None -> None + in + let credo = + match find_word span ~from:0 ~word:"Credo" ~reject_next:"__never__" with + | Some (_, negated) -> Some (not negated) + | None -> None + in + let praef = + let sn = Array.length span in + let rec go i = if i >= sn then None else if strip_trailing_punct span.(i) = "praef" then Some i else go (i + 1) in + match go 0 with + | None -> None + | Some i -> + let rest = Array.to_list (Array.sub span (i + 1) (sn - i - 1)) in + if rest = [] then None else Some (String.concat " " rest) + in + (gloria, credo, praef) + +(* --- top level -------------------------------------------------------------- *) + +let () = + if Array.length Sys.argv <> 4 then + die "usage: extract_fiuv_ordo <pdftotext-layout.txt> <source.pdf> <dest.sexp>"; + let txt_path = Sys.argv.(1) in + let pdf_path = Sys.argv.(2) in + let dest = Sys.argv.(3) in + let all_lines = read_lines txt_path in + let arr = Array.of_list all_lines in + let n = Array.length arr in + let trimmed i = String.trim arr.(i) in + (* Anchor: the FIRST month-marker line carrying a parseable Roman year + -- established during characterisation to be line 47, "November + MMXXV". Search generically rather than hardcoding the line number, + so a re-extraction against a re-flowed pdftotext dump still finds it. *) + let anchor_idx, anchor_month, anchor_year = + let rec go i = + if i >= n then die "no month header with a parseable Roman-numeral year found (expected the book's own first header, e.g. \"November MMXXV\")" + else + let toks = split_ws (trimmed i) in + match toks with + | m :: y :: _ -> ( + match (month_index m, parse_roman y) with + | Some mi, Some yr when yr > 1000 -> (i, mi, yr) + | _ -> go (i + 1)) + | _ -> go (i + 1) + in + go 0 + in + let stop_idx = + let rec go i = if i >= n then n else if is_stop_tail_line (split_ws (trimmed i)) then i else go (i + 1) in + go (anchor_idx + 1) + in + Printf.printf "extract_fiuv_ordo: Ordo body lines %d..%d (of %d total), anchor %d/%d\n" anchor_idx stop_idx n + anchor_month anchor_year; + let cur_month = ref anchor_month and cur_year = ref anchor_year in + let blocks = ref [] in + let cur = ref None in + let flush () = match !cur with Some b -> blocks := b :: !blocks; cur := None | None -> () in + for i = anchor_idx + 1 to stop_idx - 1 do + let raw = trimmed i in + if raw = "" then () + else + let toks = split_ws raw in + if is_month_marker_line toks then ( + match toks with + | m :: _ -> ( + match month_index m with + | Some mi -> + if mi <> !cur_month then begin + if mi < !cur_month then incr cur_year; + cur_month := mi + end + | None -> ()) + | [] -> ()) + else + match is_day_start_line toks with + | Some day -> + flush (); + cur := Some { b_day = day; b_month = !cur_month; b_year = !cur_year; b_lines = toks } + | None -> ( + match !cur with + | None -> () + | Some b -> cur := Some { b with b_lines = b.b_lines @ strip_left_column toks }) + done; + flush (); + let blocks = List.rev !blocks in + Printf.printf "extract_fiuv_ordo: %d day-blocks parsed\n" (List.length blocks); + let rows = + List.map + (fun b -> + let date = + match Date.make ~year:b.b_year ~month:b.b_month ~day:b.b_day with + | Ok d -> d + | Error e -> die "%d/%d/%d: %s" b.b_year b.b_month b.b_day e + in + let tokens = Array.of_list b.b_lines in + (* [b_lines] tokens 0 is the day's own leading "<n>." marker -- + dropped before field extraction, kept for nothing (the day + number is already known from [b_day]). *) + let content = if Array.length tokens > 0 then Array.sub tokens 1 (Array.length tokens - 1) else tokens in + (* Cut at the "–VEL–" alternative-office boundary, if this day has + one, BEFORE any field extraction -- see [split_at_alt_office]'s + own citation. Every field below now reads the PRIMARY office + only; [alt_te_deum] alone also reads the second, when present. *) + let primary, alt = split_at_alt_office content in + let class_ = extract_class primary in + let title = extract_title primary in + let full_text = String.concat " " (Array.to_list primary) in + let te_deum = extract_te_deum full_text in + let alt_te_deum = + match alt with None -> None | Some alt_tokens -> extract_te_deum (String.concat " " (Array.to_list alt_tokens)) + in + let gloria, credo, praef = extract_missa_fields primary in + { date = Date.to_iso8601 date; class_; title; te_deum; gloria; credo; praef; alt_te_deum }) + blocks + in + let missing_class = List.filter (fun r -> r.class_ = None) rows in + let missing_gloria = List.filter (fun r -> r.gloria = None) rows in + let missing_credo = List.filter (fun r -> r.credo = None) rows in + let missing_te_deum = List.filter (fun r -> r.te_deum = None) rows in + if missing_class <> [] then + Printf.printf "extract_fiuv_ordo: WARNING %d rows with no class found: %s\n" (List.length missing_class) + (String.concat ", " (List.map (fun r -> r.date) missing_class)); + Printf.printf + "extract_fiuv_ordo: %d rows; %d with no Gloria found; %d with no Credo found; %d with no Te Deum found\n" + (List.length rows) (List.length missing_gloria) (List.length missing_credo) (List.length missing_te_deum); + (* CHARACTERISATION, Step C of the task brief, non-negotiable: probed + BEFORE trusting a single row of this fixture, and RE-VERIFIED here + against this run's own [rows] -- a failed probe is a hard [die], the + same self-verifying discipline extract_lms_ordo.ml already uses, + never a printed claim resting on a one-off manual check that could + silently rot on a re-extraction. Every date/expectation pair below + was confirmed directly against the raw pdftotext dump before being + encoded here. *) + let creed_of_date d = List.find_map (fun r -> if String.equal r.date d then r.credo else None) rows in + let probes = + [ ("2025-11-30", true, "Advent Sunday, I class, RG 475(a)"); + ("2025-12-01", false, "ordinary Advent feria, III class, RG 476(b)/(d)"); + ("2025-12-26", true, "St Stephen, II class, inside the Nativity octave, RG 475(d)"); + ("2026-08-10", false, "St Laurence, II class martyr, no Lord/BVM/apostle clause applies, RG 476(b)"); + ("2026-08-15", true, "the Assumption, I class, RG 475(b)") ] + in + List.iter + (fun (d, expected, desc) -> + match creed_of_date d with + | Some got when Bool.equal got expected -> () + | Some got -> die "characterisation probe FAILED: %s (%s) -- expected creed=%b, extracted creed=%b" d desc expected got + | None -> die "characterisation probe FAILED: %s (%s) -- no row extracted for this date at all" d desc) + probes; + Printf.printf "extract_fiuv_ordo: all %d characterisation probes confirmed\n" (List.length probes); + (* Second characterisation probe, same discipline: the "–VEL–" + alternative-office split (module header point 5) must fire on + EXACTLY these six dates, no more, no fewer -- a hard [die], not a + printed claim, so a future re-extraction against a re-flowed dump + silently gaining or losing an instance fails loudly rather than + quietly changing F3's own population. *) + let alt_office_dates = List.filter_map (fun r -> if r.alt_te_deum <> None then Some r.date else None) rows in + let expected_alt_office_dates = + [ "2026-03-06"; "2026-03-07"; "2026-03-09"; "2026-03-12"; "2026-03-21"; "2026-03-24" ] + in + if List.sort String.compare alt_office_dates <> expected_alt_office_dates then + die "alt-office-split characterisation probe FAILED: expected exactly %s, got %s" + (String.concat ", " expected_alt_office_dates) + (String.concat ", " (List.sort String.compare alt_office_dates)); + Printf.printf "extract_fiuv_ordo: alt-office-split characterisation probe confirmed (%d dates)\n" + (List.length expected_alt_office_dates); + let probe_lines = + String.concat "\n" + (List.map + (fun (d, expected, desc) -> Printf.sprintf "; %s (%s) -> %s" d desc (if expected then "TRUE" else "FALSE")) + probes) + in + let sha256_of_file path = + let cmd = Printf.sprintf "sha256sum %s" (Filename.quote path) in + let ic = Unix.open_process_in cmd in + let line = try input_line ic with End_of_file -> die "sha256sum produced no output for %s" path in + (match Unix.close_process_in ic with Unix.WEXITED 0 -> () | _ -> die "sha256sum failed for %s" path); + match String.index_opt line ' ' with Some i -> String.sub line 0 i | None -> die "unexpected sha256sum output: %S" line + in + let today () = + let tm = Unix.gmtime (Unix.time ()) in + Printf.sprintf "%04d-%02d-%02d" (tm.Unix.tm_year + 1900) (tm.Unix.tm_mon + 1) tm.Unix.tm_mday + in + let pdf_sha = sha256_of_file pdf_path in + let pdf_base = Filename.basename pdf_path in + let gloria_true = List.length (List.filter (fun r -> r.gloria = Some true) rows) in + let gloria_false = List.length (List.filter (fun r -> r.gloria = Some false) rows) in + let credo_true = List.length (List.filter (fun r -> r.credo = Some true) rows) in + let credo_false = List.length (List.filter (fun r -> r.credo = Some false) rows) in + let te_deum_true = List.length (List.filter (fun r -> r.te_deum = Some true) rows) in + let te_deum_false = List.length (List.filter (fun r -> r.te_deum = Some false) rows) in + let alt_te_deum_true = List.length (List.filter (fun r -> r.alt_te_deum = Some true) rows) in + let alt_te_deum_false = List.length (List.filter (fun r -> r.alt_te_deum = Some false) rows) in + let alt_te_deum_none = List.length (List.filter (fun r -> r.alt_te_deum = None) rows) in + let header = + Printf.sprintf + {|; %s -- Witnesses task (2026-08-22-colitur- +; celebrant-rubrics-phase1)'s SEVENTH validation layer, and the first +; UNIVERSAL (non-diocesan), SECOND-COMPILER witness: the FIUV (Foederatio +; Internationalis Una Voce) Ordo, compiled independently of the Latin Mass +; Society's own three editions (test/test_lms_ordo.ml) -- see +; docs/research/ordo/PROVENANCE-ordo-corpus.md for the full four-reasons +; account of why this source is the more valuable of the two newly- +; acquired ones. +; +; Generator: tools/extract_fiuv_ordo.ml -- do not hand-edit; re-run +; against a fresh pdftotext dump and commit the diff instead. +; +; Source: "Ordo Divini Officii recitandi sacrique peragendi secundum +; antiquam Ritus Romani formam pro anno Domini 2026", Foederatio +; Internationalis Una Voce, compiled by Joseph Shaw (Praefatio signed +; "Joseph Shaw, President", Feast of the Nativity of Our Lady [8 Sept] +; 2025) -- title page and Praefatio, verified directly against the PDF's +; own extracted text. 2025 is stated as the Federation's own 60th +; anniversary year. +; PDF metadata: Creator "TeX", Producer "pdfTeX-1.40.22", CreationDate +; 2025-11-08, 128 pages. +; URL: identified via web search as +; https://lms.org.uk/sites/default/files/u5374/fiuv_ordo_2025-2026_1.1_a5_format.pdf +; (hosted by the Latin Mass Society's own site alongside its own three +; editions -- Joseph Shaw is both FIUV President and LMS Chairman, which +; plausibly also explains why this PDF's own trailing malformed entry +; (see COVERAGE below) is structurally identical to the LMS PDFs' own, +; despite the two being compiled by different people); NOT independently +; re-downloaded and byte-compared in this session -- recorded honestly +; rather than presented as verified, the same discipline the LMS +; fixtures' own headers use. +; The local PDF (docs/research/ordo/%s) is gitignored (CLAUDE.md: "docs/ +; is gitignored"), so this header, not git, is this fixture's only record +; of where it came from. +; SHA-256 of docs/research/ordo/%s: %s +; Extracted (UTC): %s +; Exact commands: +; pdftotext -layout docs/research/ordo/%s /tmp/fiuv.txt +; dune exec tools/extract_fiuv_ordo.exe -- /tmp/fiuv.txt \ +; docs/research/ordo/%s %s +; +; COVERAGE: %d day-rows, %s through %s. Starts at the book's own first +; real content line ("November MMXXV", the FIRST month header found with +; a parseable Roman-numeral year -- used as this parser's sole date +; anchor, see the tool's own header on why no LATER header's year is ever +; read, the second one being a publisher typo, "December MMXV" for 2025). +; Stops before a malformed trailing entry, structurally the SAME artefact +; found in all three LMS editions (a stray, mislabelled duplicate of the +; fixture's own already-extracted 1 January row, headed "1st Jan / 2025." +; -- see extract_lms_ordo.ml's own COVERAGE section for the fuller +; account of this shared publisher-side quirk). +; +; CHARACTERISATION FINDINGS (non-negotiable per the task brief -- the +; extraordinaryform.org Ordo silently omitted St Lawrence's vigil on +; EVERY date it covered, which nearly produced a false corroboration +; during the RG 33 work; this Ordo was probed with the same discipline +; before a single divergence was adjudicated): +; +; 1. UNIVERSAL, not diocesan: unlike the three LMS editions (which had to +; exclude diocesan variants from every comparison, quantified in each +; of their own fixture headers), this Ordo carries ONE block per day, +; always. There is no [has_diocesan_variant] field in this fixture's +; own [row] at all -- there is nothing to exclude. +; +; 2. OPPOSITE-PREDICTION PROBES against Rite_ef.Rubrics_ef.creed (RG +; 475-476), RE-CHECKED BY THIS TOOL RUN against its own extracted +; [rows] before this header was even written -- a failed probe is a +; hard [die], never a printed claim: +%s +; All %d predictions confirmed, both directions -- this source is +; discriminating, not a constant, on the Creed. +; +; 3. THE SUBSTRING TRAP this parser defends against, found live in this +; corpus: "Gloria" is also the first word of "Gloria Patri" (the psalm +; doxology), which can appear INSIDE the very Mass clause being +; scanned (Good Friday, 3 April 2026: "Missa pr., (omittuntur ps. +; Iudica me et Gloria Patri), Gloria, sine Credo, praef. comm." -- the +; real, standalone Gloria mention follows immediately after). See +; [find_word]'s own citation. +; +; 4. RAW SUBSTRING COUNTS measured against this session's own pdftotext +; dump, for comparison against the task brief's own figures (328 +; Gloria / 132 sine Gloria / 119 Credo / 260 sine Credo, 262 Te Deum): +; "sine Gloria" 132 (EXACT match), "sine Credo" 260 (EXACT match), raw +; "Te Deum" substring 262 (EXACT match, includes negated occurrences +; as substring hits -- 42 of them are "non dicitur Te Deum"). Raw +; "Gloria"/"Credo" substring totals (494/411) do NOT match the brief's +; implied positive totals (328/119) even after subtracting the 25 +; "Gloria Patri" occurrences (469, still 9 over) -- not chased further +; here; this extractor's own token-scoped, Missa-clause-bounded counts +; below are what this fixture actually asserts, not a forced match to +; the brief's approximate figures. +; +; 5. THE ALTERNATIVE-OFFICE TRAP (celebrant-rubrics-phase1 Phase 2 +; correction, 2026-08-22, closing F3): six real days print BOTH a +; strict-1962 reading and a complete SECOND office (the same saint +; kept outright under the 2020 decree Cum Sanctissima), separated by +; a literal "-VEL-" token. [extract_te_deum] used to scan the WHOLE +; block unbounded, so on these six days -- whose PRIMARY office states +; no "Ad Mat." of its own -- it silently read the SECOND office's own +; Te Deum instead, manufacturing a same-day contradiction with colitur +; that F3 first recorded as "colitur's own possible gap". Every field +; now reads the PRIMARY office only (module header point 5 has the +; full account); the alternative is captured, not discarded, in its +; own [alt_te_deum] field. Confirmed to fire on EXACTLY six dates by a +; hard characterisation probe (this run's own output, above), not +; merely asserted here. +; +; MEASURED DISTRIBUTION (this extraction, not the brief's figures): Gloria +; true=%d false=%d unresolved=%d; Credo true=%d false=%d unresolved=%d; +; Te Deum true=%d false=%d unresolved=%d; alt Te Deum (the six "-VEL-" +; Cum Sanctissima alternatives, point 5 above) true=%d false=%d none=%d +; (a day contributes to "none" here either because it has no "-VEL-" +; alternative at all -- 394 of 400 -- or, for a day that does, because +; that alternative's own text doesn't state Te Deum either; not +; distinguished by this one count). +; +; FIX HISTORY (fixed text, HISTORICAL counts from the run that found each +; bug, not re-verified live by every later run -- this section exists so +; regenerating this fixture from a fresh pdftotext dump, which rewrites +; this whole header from the template, does not silently erase either +; finding; earlier this header carried the first entry below as a +; hand-appended note that a regeneration between then and now had +; already dropped once): +; +; 1. RE-EXTRACTED (celebrant-rubrics-phase1 Phase 2, 2026-08-22): the FIRST +; extraction's own Te Deum parser recognised only ONE of the source's two +; negative phrasings ("non dicitur Te Deum") -- "sine Te Deum" (how the +; source actually negates a SUNDAY's own Te Deum, among others) fell +; through to a bare "Te Deum" substring match and was wrongly read +; [true]. Found by hand, comparing the raw pdftotext dump against this +; fixture's own values directly, not by any coverage check (both counted +; the same 127 unresolved rows before and after). Fixed in +; tools/extract_fiuv_ordo.ml's own [extract_te_deum] (see its own +; citation); this fixture re-extracted from the SAME pdftotext dump +; (SHA-256 of the source PDF unchanged, above). 24 of 400 rows changed, +; every one Te Deum true->false, every one independently confirmed +; against the raw text: true 231->207, false 42->66, unresolved 127 +; unchanged. See lib/rites/rite_ef/rubrics_ef.ml's own [te_deum] header +; (237(a)'s comment) for the full account of what this corrected, and +; the .superpowers task report for the complete before/after date list. +; +; 2. RE-EXTRACTED (celebrant-rubrics-phase1 Phase 2 correction, 2026-08-22): +; module header point 5 and CHARACTERISATION point 5 above have the +; full account of the "-VEL-" alternative-office bug this closes +; (data/ef/expected-divergences-fiuv.sexp's own F3). This fixture +; re-extracted from the SAME pdftotext dump (SHA-256 of the source PDF +; unchanged, above). 6 of 400 rows changed, every one Te Deum +; true->unresolved (2026-03-06/07/09/12/21/24), every one independently +; confirmed against the raw text: Te Deum true 207->201 (this run's own +; unresolved count went 127->133, consistent with those same 6 rows). +; Gloria/Credo/class_/title/praef are UNCHANGED on all 400 rows -- +; those fields were already correctly bounded to the primary office +; before this fix (a Vespers-line token always intervened before the +; "-VEL-" separator on all six real instances), confirmed by a direct +; diff of the fixture before and after, not merely by argument. +; +; 3. RE-EXTRACTED (celebrant-rubrics-phase1 same correction, defensive +; hardening, 2026-08-22): auditing for the SAME trap elsewhere (the +; task brief's own item 5) found a THIRD, narrower alternate-Mass +; marker, "Vel (sec. decretum <<Cum sanct.>>) Missa ...", offering a +; Cum-Sanctissima Mass FORMULARY choice within a single office +; (unlike point 5 above, no second "Ad Mat." set of lessons -- a +; narrower thing than a full alternate office). 20 occurrences in the +; whole corpus, confirmed by direct grep; on all 20, Gloria/Credo were +; ALREADY stated before reaching this marker, so this was NEVER a +; live divergence for those two fields (confirmed by this same diff: +; 0 Gloria/Credo/class_/title/alt_te_deum rows changed). [praef] +; alone changed on 20 of 400 rows (its own "rest of the primary +; clause" capture no longer runs past this marker into the +; alternative's own text) -- harmless in itself ([praef] is +; documented CAPTURED, NOT VALIDATED, compared by nothing), but added +; as a defensive boundary in [extract_missa_fields] so a FUTURE year +; whose primary clause does not restate Gloria/Credo before this +; marker cannot silently read the alternative's own value instead -- +; the identical failure shape as point 2 above, closed before it was +; ever observed live rather than after. +|} + dest pdf_base pdf_base pdf_sha (today ()) pdf_base pdf_base dest (List.length rows) + (match rows with r :: _ -> r.date | [] -> "?") + (match List.rev rows with r :: _ -> r.date | [] -> "?") + probe_lines (List.length probes) gloria_true gloria_false (List.length missing_gloria) credo_true credo_false + (List.length missing_credo) te_deum_true te_deum_false (List.length missing_te_deum) alt_te_deum_true + alt_te_deum_false alt_te_deum_none + in + let body = Sexplib.Sexp.to_string_hum ~indent:2 (sexp_of_list sexp_of_row rows) in + let oc = open_out dest in + Fun.protect + ~finally:(fun () -> close_out_noerr oc) + (fun () -> + output_string oc header; + output_string oc body; + output_string oc "\n"); + Printf.printf "extract_fiuv_ordo: wrote %d rows to %s\n" (List.length rows) dest diff --git a/tools/extract_lms_ordo.ml b/tools/extract_lms_ordo.ml new file mode 100644 index 0000000..582fb6e --- /dev/null +++ b/tools/extract_lms_ordo.ml @@ -0,0 +1,729 @@ +(* Task 6 (2026-08-21-colitur-celebrant-rubrics-phase1): turns pdftotext's + -layout dump of the Latin Mass Society's printed Ordo into + test/fixtures/lms-ordo-2024-2025.sexp, one row per UNIVERSAL-calendar + day (England & Wales diocesan variants are recorded as PRESENT via + [has_diocesan_variant] but their own text is discarded -- see the + fixture's own provenance header, and task-6-report.md's "how much of + the Ordo's year is comparable" accounting, for why). + + NOT a general Ordo parser: hand-rolled against ONE PDF's own column + layout (Author "Peter Day-Milne", Producer "LibreOffice 7.4", pdftotext + 1.6), verified line-by-line against the source before being trusted -- + see task-6-report.md for the full characterisation record (the + opposite-prediction probe technique CLAUDE.md's "burned twice" note + demands). Frozen deps forbid Str/regex; every match below is + String.sub/index/split_on_char, the same discipline tools/ + bootstrap_sanctoral.ml's own hand-rolled INI reader already uses for + someone else's format. + + THE KEY STRUCTURAL FACT this parser leans on, established during + characterisation: every day's UNIVERSAL entry (the General Roman + Calendar's own text) appears FIRST in reading order, before any + diocesan variant block (which is introduced by a line ending in ':', + e.g. "Westminster, Clifton, Plymouth:" or "In all Dioceses of ENGLAND + and WALES ... follows:"). So the FIRST "Gl"/"Cr" token pair and the + FIRST "Mass of ..."/roman-numeral-BVM line found within a day's block + are ALWAYS the universal entry's own, even if the colon-boundary + detection below is imperfect -- diocesan text can only ever appear + AFTER, never before, so it cannot introduce a false FIRST match. This + is why [scan_block] restricts its search to lines before the first + colon-terminated line (belt), while the ordering argument above is the + suspenders. + + Usage: + pdftotext -layout docs/research/ordo/lms-ordo-2024-2025.pdf /tmp/lms.txt + dune exec tools/extract_lms_ordo.exe -- /tmp/lms.txt \ + docs/research/ordo/lms-ordo-2024-2025.pdf \ + test/fixtures/lms-ordo-2024-2025.sexp *) + +open Sexplib0.Sexp_conv +module Date = Colitur_kernel.Date + +let die fmt = Printf.ksprintf (fun s -> prerr_endline ("extract_lms_ordo: " ^ s); exit 1) fmt + +(* Mirrored, not shared, by test/test_lms_ordo.ml -- tools/ and test/ have + no common .mli either could hang a shared type from, the same reasoning + test_oracle.ml's own header gives for duplicating [sha256_of_file]. *) +type row = { + date : string; (** ISO-8601 *) + weekday : string; + (** As printed ("Sat", "Sun", ...) -- cross-checked below against + [date]'s own computed weekday at extraction time (fatal on + mismatch: the whole point of carrying both is to catch a date- + resolution bug here, not paper over one). *) + title : string; (** the day's own title line: feast name, class, colour, votive codes *) + formulary_override : string option; + (** the literal "Mass of ..." text, when the universal block names + one; [None] = the day says its own Mass. *) + bvm_numeral : string option; + (** "I".."V" when [formulary_override] is one of the five anchored + "<N> Mass of BVM" lines (whole-line match, never a substring -- + see the module header on why "V Mass of BVM" must never be + matched as a substring of "IV Mass of BVM"). *) + gloria : bool option; (** [None] only if genuinely not found -- reported, not silently dropped *) + creed : bool option; + praef : string option; + (** RAW trailing text from the FIRST "Pr of ..." / "Common Pr" match + in the universal block's own reading order -- CAPTURED, NOT + CLASSIFIED here, the same discipline + tools/extract_fiuv_ordo.ml's own [row.praef] documents for a + different publisher's format. See the provenance header's own + "Pr of" characterisation bullet for what this column can and + cannot show: this Ordo prints OPTION LISTS ("Pr of X or Pr of Y + or Common Pr"), some of whose named options ("Martyrs", "All + Saints and Patron Saints", "the Dedication of a Church", "the + Most Holy Sacrament", "St John the Baptist", "the Angels") are + NOT among the fourteen the 1962 Missale Romanum's own RG 484-497 + enumerate -- ad libitum extras this publisher includes alongside + a genuine RG 482 answer, never the sole option on any row found + in this corpus. Classification into {!Colitur_kernel.Preface.t} + happens test-side (test/test_lms_ordo.ml), by MEMBERSHIP in the + parsed option set, not string equality -- see that file's own + [classify_praef] citation for the full account. *) + has_diocesan_variant : bool; + (** whether a diocese-specific block followed the universal one + anywhere in this day's raw text (informational only -- see the + module header on why correctness of the other fields does not + depend on this being exact). *) +} +[@@deriving sexp] + +(* --- tiny hand-rolled helpers, no Str/regex ------------------------------ *) + +let split_ws s = + String.split_on_char ' ' (String.map (fun c -> if c = '\t' then ' ' else c) s) + |> List.filter (fun t -> t <> "") + +let is_all_digits s = s <> "" && String.for_all (fun c -> c >= '0' && c <= '9') s + +let has_prefix ~prefix s = + String.length s >= String.length prefix && String.sub s 0 (String.length prefix) = prefix + +let ends_with_colon s = s <> "" && s.[String.length s - 1] = ':' + +let months = + [| "January"; "February"; "March"; "April"; "May"; "June"; "July"; "August"; "September"; "October"; + "November"; "December" |] + +let month_index name = + let rec go i = if i >= Array.length months then None else if months.(i) = name then Some (i + 1) else go (i + 1) in + go 0 + +let weekday_of_abbrev = function + | "Sun" -> Some Date.Sun + | "Mon" -> Some Date.Mon + | "Tue" -> Some Date.Tue + | "Wed" -> Some Date.Wed + | "Thu" -> Some Date.Thu + | "Fri" -> Some Date.Fri + | "Sat" -> Some Date.Sat + | _ -> None + +(* Left-column codes (Abbreviations, "Left column": "+EW"/"+"/"Pl"/"Ind") -- + printed in a dedicated margin column that pdftotext -layout reproduces + as literal leading tokens on an otherwise-ordinary content line. Strip + ALL that appear (there can be two, "Pl Ind" together -- the 31 Dec/ + 1 Jan examples in the source). *) +let strip_left_column tokens = + let is_marker = function "Pl" | "Ind" | "+EW" | "+" -> true | _ -> false in + let rec go = function t :: rest when is_marker t -> go rest | ts -> ts in + go tokens + +let bvm_numeral_strings = [ ("I", "I Mass of BVM"); ("II", "II Mass of BVM"); ("III", "III Mass of BVM"); + ("IV", "IV Mass of BVM"); ("V", "V Mass of BVM") ] + +(* Anchored WHOLE-LINE match against all five, never a substring test -- + this is the exact discipline the coordinator addendum records: "V Mass + of BVM" is a substring of "IV Mass of BVM", so any check here MUST + compare the full line, not search for the numeral's own string inside + it. *) +let bvm_numeral_of_line line = List.assoc_opt line (List.map (fun (n, s) -> (s, n)) bvm_numeral_strings) + +(* --- the day-block scanner ------------------------------------------------ *) + +(* [lines]: this day's own raw continuation lines, marker-stripped and + trimmed, title line NOT included (the title never carries Gl/Cr/"Mass + of" itself in this source). Returns (formulary_override, bvm_numeral, + gloria, creed, praef, has_diocesan_variant), each derived from the + portion BEFORE the first colon-terminated (diocesan) line -- see the + module header for why this is belt-and-suspenders, not load-bearing on + its own. *) +let scan_block lines = + let rec universal_prefix = function + | [] -> [] + | l :: _ when ends_with_colon l -> [] + | l :: rest -> l :: universal_prefix rest + in + let has_diocesan = List.exists ends_with_colon lines in + let universal = universal_prefix lines in + let formulary_override, bvm_numeral = + let rec find = function + | [] -> (None, None) + | l :: rest -> ( + match bvm_numeral_of_line l with + | Some n -> (Some l, Some n) + | None -> if has_prefix ~prefix:"Mass of " l then (Some l, None) else find rest) + in + find universal + in + let all_tokens = List.concat_map split_ws universal in + let rec find_flag word = function + | prev :: (cur :: _ as rest) -> if cur = word then Some (prev <> "No") else find_flag word rest + | [ cur ] -> if cur = word then Some true else None + | [] -> None + in + (* [find_flag] needs a sentinel predecessor for a match at position 0 + (never observed in this source -- Gl/Cr always follow at least "No" + or a preceding word on the same "Gl Cr Pr of ..." line -- but handled + rather than assumed: a bare leading "Gl"/"Cr" with no "No" before it + reads as [true], the same as any other non-"No" predecessor would). *) + let gloria = find_flag "Gl" ("" :: all_tokens) in + let creed = find_flag "Cr" ("" :: all_tokens) in + (* [praef]: LINE-SCOPED (not [all_tokens]'s flattened stream), because + the captured text runs to the END of whichever physical line carries + it -- flattening first would glue an unrelated following line's own + prose onto it. "Pr" always abbreviates "Preface" in this source's own + body text (checked directly: its only OTHER appearance is the front- + matter Abbreviations glossary entry "Pr Preface", outside the Ordo + body this tool scans), so a whole-token ("Pr","of") pair, or a bare + ("Common","Pr") pair when no proper/de-Tempore option is offered, is + the sole anchor needed -- no reject-list, unlike the BVM roman + numeral's own substring trap, because nothing else in this corpus's + body ever produces either two-token sequence. *) + let find_praef_in_line line = + let toks = Array.of_list (split_ws line) in + let n = Array.length toks in + let rec go i = + if i + 1 >= n then None + else if (toks.(i) = "Pr" && toks.(i + 1) = "of") || (toks.(i) = "Common" && toks.(i + 1) = "Pr") then + Some (String.concat " " (Array.to_list (Array.sub toks i (n - i)))) + else go (i + 1) + in + go 0 + in + let praef = + let rec find = function + | [] -> None + | l :: rest -> ( match find_praef_in_line l with Some s -> Some s | None -> find rest) + in + find universal + in + (formulary_override, bvm_numeral, gloria, creed, praef, has_diocesan) + +(* --- the top-level line scan --------------------------------------------- *) + +type block = { b_weekday : string; b_day : int; b_month : int; b_year : int; b_title : string; b_lines : string list } + +(* Generalised (Witnesses task, 2026-08-22) from a single hardcoded + sentinel ["End";"of";"November";"2024"], which assumed every edition + opens its Ordo body with an "End of November <year>" stub section. That + assumption FAILED characterisation against the two newly-acquired + editions: the 2023-2024 PDF has NO such stub -- Advent Sunday 2023 fell + on 3 December, so only 2 tail days of the old liturgical year remained + (1-2 December), and the compiler folded them directly into the + "December 2023" header rather than giving them their own mini-section. + So a start marker anchored to "End of November" cannot generalise. + + What DOES generalise, checked directly against all three editions' + pdftotext -layout dumps: exactly one line in the whole document begins + with the bare, all-uppercase token "ORDO" -- either alone ("ORDO", + 2024-2025/2025-2026, introducing an "End of <Month> <Year>" stub next) + or fused with the edition's own year range ("ORDO 2023-2024", + introducing the first REAL month header directly, no stub). This is + NEVER the Table of Contents' own entry, which prints title-case "Ordo + 20XX-20XX" followed by dot leaders and a page number -- a different + string ("Ordo", not "ORDO"), confirmed by grepping all three raw dumps + for both patterns before relying on either. *) +let is_ordo_marker tokens = match tokens with "ORDO" :: _ -> true | _ -> false + +(* The "End of <Month> <Year>" stub heading, when an edition has one -- + parsed directly (never hardcoded to November/a specific year) so the + edition's own initial (month, year) state comes from the text itself, + not an assumption baked into the tool. *) +let parse_end_of_month tokens = + match tokens with + | [ "End"; "of"; m; y ] when is_all_digits y -> ( match month_index m with Some mi -> Some (mi, int_of_string y) | None -> None) + | _ -> None + +let is_stop_tail_line tokens = + match tokens with tok :: _ -> String.length tok = 4 && is_all_digits tok | [] -> false + +let is_month_header tokens = + match tokens with + | m :: rest when month_index m <> None -> List.for_all is_all_digits rest + | _ -> false + +let read_lines path = + let ic = open_in path in + let rec loop acc = match input_line ic with l -> loop (l :: acc) | exception End_of_file -> List.rev acc in + let ls = loop [] in + close_in ic; + ls + +let () = + if Array.length Sys.argv <> 4 then + die "usage: extract_lms_ordo <pdftotext-layout.txt> <source.pdf> <dest.sexp>"; + let txt_path = Sys.argv.(1) in + let pdf_path = Sys.argv.(2) in + let dest = Sys.argv.(3) in + let all_lines = read_lines txt_path in + let arr = Array.of_list all_lines in + let n = Array.length arr in + let trimmed i = String.trim arr.(i) in + let ordo_marker_idx = + let rec go i = if i >= n then die "no line begins with the bare \"ORDO\" body marker (see the tool's own comment on [is_ordo_marker] -- this edition's format may have drifted)" + else if is_ordo_marker (split_ws (trimmed i)) then i else go (i + 1) + in + go 0 + in + (* The first non-blank line after the marker is either an "End of <Month> + <Year>" stub (consumed here as a pure sentinel, exactly as the old + hardcoded design did -- [start_idx] becomes ITS OWN index so the main + loop below begins right after it) or a real "<Month> <Year>" header + with an explicit year token (an edition with no stub, e.g. 2023-2024 -- + [start_idx] becomes the ORDO marker's own index instead, so the main + loop re-encounters this header line on its very first iteration and + [is_month_header]'s own existing per-line handling sets [cur_month]/ + [cur_year] from it normally). Either way [init_month]/[init_year] come + directly from parsed text, never a hardcoded constant -- a missing + explicit year on this FIRST header is a genuine format break, so it + dies loudly rather than silently seeding a wrong year. *) + let rec first_nonblank i = + if i >= n then die "no content found after the ORDO marker at line %d" ordo_marker_idx + else if trimmed i = "" then first_nonblank (i + 1) + else i + in + let content_idx = first_nonblank (ordo_marker_idx + 1) in + let content_toks = split_ws (trimmed content_idx) in + let start_idx, init_month, init_year = + match parse_end_of_month content_toks with + | Some (mi, y) -> (content_idx, mi, y) + | None -> ( + match content_toks with + | m :: (y :: _ as rest) when month_index m <> None && List.for_all is_all_digits rest -> + (ordo_marker_idx, (match month_index m with Some x -> x | None -> assert false), int_of_string y) + | _ -> + die "first content after the ORDO marker (%S) is neither an \"End of <Month> <Year>\" stub nor a \"<Month> <Year>\" header carrying an explicit year" + (String.concat " " content_toks)) + in + let stop_idx = + let rec go i = + if i >= n then n else if is_stop_tail_line (split_ws (trimmed i)) then i else go (i + 1) + in + go (start_idx + 1) + in + Printf.printf "extract_lms_ordo: Ordo body lines %d..%d (of %d total), initial month/year %d/%d\n" start_idx stop_idx n + init_month init_year; + (* --- the scan: month/year tracker + block accumulator ------------------ *) + let cur_month = ref init_month and cur_year = ref init_year in + let blocks = ref [] in + let cur = ref None in + let flush () = match !cur with Some b -> blocks := b :: !blocks; cur := None | None -> () in + for i = start_idx + 1 to stop_idx - 1 do + let raw = trimmed i in + if raw = "" then () + else if String.length raw >= 1 && (let rec contains s sub = + let ls = String.length s and lu = String.length sub in + if lu = 0 then true else if ls < lu then false + else if String.sub s 0 lu = sub then true + else contains (String.sub s 1 (ls - 1)) sub + in + contains raw "Latin Mass Society Ordo") + then () (* page footer *) + else + let toks = split_ws raw in + if is_month_header toks then begin + match toks with + | m :: rest -> ( + match month_index m with + | Some mi -> + if mi <> !cur_month then begin + if mi < !cur_month then incr cur_year; + cur_month := mi + end; + (* an explicit 4-digit year token, when present, overrides + the transition inference outright -- belt and + suspenders, same reasoning as [scan_block] above *) + List.iter (fun t -> if String.length t = 4 && is_all_digits t then cur_year := int_of_string t) rest + | None -> ()) + | [] -> () + end + else + match toks with + | wd :: dn :: _ when weekday_of_abbrev wd <> None && is_all_digits dn && int_of_string dn >= 1 + && int_of_string dn <= 31 -> + flush (); + let title = String.concat " " (List.tl (List.tl toks)) in + cur := Some { b_weekday = wd; b_day = int_of_string dn; b_month = !cur_month; b_year = !cur_year; + b_title = title; b_lines = [] } + | _ -> ( + match !cur with + | None -> () + | Some b -> + let stripped = strip_left_column toks in + if stripped = [] then () + else cur := Some { b with b_lines = b.b_lines @ [ String.concat " " stripped ] }) + done; + flush (); + let blocks = List.rev !blocks in + Printf.printf "extract_lms_ordo: %d day-blocks parsed\n" (List.length blocks); + let rows = + List.map + (fun b -> + let date = + match Date.make ~year:b.b_year ~month:b.b_month ~day:b.b_day with + | Ok d -> d + | Error e -> die "%s %d/%d/%d: %s" b.b_weekday b.b_year b.b_month b.b_day e + in + (match weekday_of_abbrev b.b_weekday with + | Some w when w = Date.weekday date -> () + | Some _ -> + die "%s: printed weekday %S does not match computed weekday %S -- date resolution bug" + (Date.to_iso8601 date) b.b_weekday (Date.weekday_to_string (Date.weekday date)) + | None -> die "%s: unrecognised weekday abbreviation %S" (Date.to_iso8601 date) b.b_weekday); + let formulary_override, bvm_numeral, gloria, creed, praef, has_diocesan_variant = scan_block b.b_lines in + { date = Date.to_iso8601 date; weekday = b.b_weekday; title = b.b_title; formulary_override; bvm_numeral; + gloria; creed; praef; has_diocesan_variant }) + blocks + in + let missing_gloria = List.filter (fun r -> r.gloria = None) rows in + let missing_creed = List.filter (fun r -> r.creed = None) rows in + let missing_praef = List.filter (fun r -> r.praef = None) rows in + if missing_gloria <> [] then + Printf.printf "extract_lms_ordo: WARNING %d rows with no Gloria found: %s\n" (List.length missing_gloria) + (String.concat ", " (List.map (fun r -> r.date) missing_gloria)); + if missing_creed <> [] then + Printf.printf "extract_lms_ordo: WARNING %d rows with no Creed found: %s\n" (List.length missing_creed) + (String.concat ", " (List.map (fun r -> r.date) missing_creed)); + if missing_praef <> [] then + Printf.printf "extract_lms_ordo: WARNING %d rows with no Pr-of/Common-Pr found: %s\n" (List.length missing_praef) + (String.concat ", " (List.map (fun r -> r.date) missing_praef)); + let with_variant = List.length (List.filter (fun r -> r.has_diocesan_variant) rows) in + let with_override = List.length (List.filter (fun r -> r.formulary_override <> None) rows) in + let with_bvm = List.length (List.filter (fun r -> r.bvm_numeral <> None) rows) in + Printf.printf "extract_lms_ordo: %d rows; %d with a diocesan variant present; %d with a formulary override; \ + %d with a BVM roman numeral; %d with no Pr-of/Common-Pr text\n" + (List.length rows) with_variant with_override with_bvm (List.length missing_praef); + (* --- provenance header + write ----------------------------------------- *) + let sha256_of_file path = + let cmd = Printf.sprintf "sha256sum %s" (Filename.quote path) in + let ic = Unix.open_process_in cmd in + let line = try input_line ic with End_of_file -> die "sha256sum produced no output for %s" path in + (match Unix.close_process_in ic with Unix.WEXITED 0 -> () | _ -> die "sha256sum failed for %s" path); + match String.index_opt line ' ' with Some i -> String.sub line 0 i | None -> die "unexpected sha256sum output: %S" line + in + let today () = + let tm = Unix.gmtime (Unix.time ()) in + Printf.sprintf "%04d-%02d-%02d" (tm.Unix.tm_year + 1900) (tm.Unix.tm_mon + 1) tm.Unix.tm_mday + in + let pdf_sha = sha256_of_file pdf_path in + (* --- per-edition provenance + characterisation, Witnesses task --------- + Generalised 2026-08-22 from Task 6's single hand-written header (which + named "lms-ordo-2024-2025.sexp" literally throughout). Facts 1/3/4 + below are STRUCTURAL -- true of the publisher's whole known corpus, + re-checked directly against the two new editions' own pdftotext dumps + before being asserted here again, not merely copied forward. Fact 2 + (the Creed opposite-prediction probes) is inherently edition-specific + (the actual calendar dates differ year to year, and one edition's own + calendar even breaks the standard pairing -- see the 2025-2026 note + below), so it is looked up per [dest] and, critically, RE-VERIFIED + against this run's own [rows] before being printed: a probe that + fails is a [die], not a silently-wrong "confirmed" claim in a fixture + header nobody will re-check by eye. *) + let base = Filename.basename dest in + let pdf_base = Filename.basename pdf_path in + let edition_label, isbn, copyright_year, pdf_meta, url, url_note, window_open_desc = + match base with + | "lms-ordo-2023-2024.sexp" -> + ( "2023-2024", + "978-1-7392096-2-9", + "2023", + {|Author "Peter Day-Milne", Producer "LibreOffice 6.4", CreationDate 2023-11-01, 140 pages|}, + "https://lms.org.uk/sites/default/files/u5374/ordo_2023-2024_2.2_final.pdf", + "identified 2026-08-22 via web search matching the PDF's own edition year to the publisher's own hosting path; NOT independently re-downloaded and byte-compared in this session -- recorded honestly rather than presented as verified", + {|1 December 2023 (this edition carries NO "End of November" stub section at all -- Advent Sunday 2023 fell on 3 December, so only 2 tail days of the prior liturgical year remained, and the compiler folded them directly into the "December 2023" header instead of giving them their own mini-section; see CHARACTERISATION below)|} + ) + | "lms-ordo-2025-2026.sexp" -> + ( "2025-2026", + "978-1-7392096-5-0", + "2025", + {|Author "Peter Day-Milne", Producer "LibreOffice 25.2.6.2 (X86_64) / LibreOffice Community", CreationDate 2025-10-23, 146 pages|}, + "(not found -- see the note on this line)", + "NOT found by URL-pattern web search this session (unlike the 2023-2024 and 2024-2025 editions, no exact download link matching this PDF's own filename/version turned up under lms.org.uk/sites/default/files/u5374/; several near-miss candidates were checked and rejected -- none matched this PDF's own metadata) -- left honestly blank rather than guessed; the file was supplied locally to this session, not fetched by this tool", + {|28 November 2025 ("End of November 2025", the same shape as the 2024-2025 edition -- Advent Sunday 2025 fell on 30 November, so only 3 tail days remained; note this excludes 27 November 2025, which the PRECEDING edition's own window (2024-2025, through 2025-12-31) already covers -- a one-day-short overlap between consecutive editions' own windows, not a gap or a bug in either extraction|} + ) + | "lms-ordo-2024-2025.sexp" -> + (* Preface-witnesses task (2026-08-23): brought into this same + auto-generated, probe-verified header template so all three + editions carry the identical characterisation discipline -- + Task 6's own original HAND-WRITTEN header (never regenerated by + the Witnesses task, which only added table entries for the two + NEW editions) is superseded by this run; every fact below is + re-derived from this PDF's own text/metadata, not copied + blind from that older header (though it agrees with it in + every particular checked). *) + ( "2024-2025", + "9781739209636", + "2024", + {|Author "Peter Day-Milne", Producer "LibreOffice 7.4", CreationDate 2024-11-06, 140 pages|}, + "https://lms.org.uk/sites/default/files/u5374/ordo_2.21_2024-2025.pdf", + "identified via web search matching the PDF's own version number and filename to the publisher's own hosting path (product page: https://lms.org.uk/product/latin-mass-society-ordo-2024-25); NOT independently re-downloaded and byte-compared in this session -- recorded honestly rather than presented as verified", + {|27 November 2024 ("End of November 2024", the same shape as the 2025-2026 edition -- Advent Sunday 2024 fell on 1 December, so 4 tail days remained (27-30 November))|} + ) + | _ -> die "no provenance/characterisation entry for %s -- add one to extract_lms_ordo.ml's own per-edition table before generating this fixture (the tool refuses to emit an under-characterised header)" base + in + let creed_of_date rows d = List.find_map (fun r -> if String.equal r.date d then r.creed else None) rows in + let probes = + match base with + | "lms-ordo-2023-2024.sexp" -> + [ ("2023-12-03", true, "Advent Sunday, RG 475(a)"); + ("2023-12-04", false, "St Peter Chrysologus, III class, ordinary Advent day, RG 476(b)/(d)"); + ("2023-12-26", true, "St Stephen, II class, inside the Nativity octave, RG 475(d)"); + ("2024-08-15", true, "the Assumption, I class, RG 475(b)"); + ("2024-08-16", false, "St Joachim, II class, no Lord/BVM/apostle clause applies, RG 476(b)") ] + | "lms-ordo-2025-2026.sexp" -> + [ ("2025-11-30", true, "Advent Sunday, RG 475(a)"); + ("2025-12-01", false, "ordinary Advent feria, RG 476(b)/(d)"); + ("2025-12-26", true, "St Stephen, II class, inside the Nativity octave, RG 475(d)"); + ( "2026-08-10", false, + "St Laurence, II class martyr, no Lord/BVM/apostle clause applies, RG 476(b) -- St Joachim (16 August) is IMPEDED by a Sunday in this edition's own calendar (12th Sunday after Pentecost), so this edition needed a DIFFERENT sharp non-octave II-class pairing than the 2023-2024/2024-2025 editions used; a real, edition-specific finding, not an extraction quirk" + ); + ("2026-08-15", true, "the Assumption, I class, RG 475(b)") ] + | "lms-ordo-2024-2025.sexp" -> + [ ("2024-12-01", true, "Advent Sunday, RG 475(a)"); + ("2024-12-02", false, "ordinary Advent feria, RG 476(b)/(d)"); + ("2024-12-26", true, "St Stephen, II class, inside the Nativity octave, RG 475(d)"); + ("2025-08-15", true, "the Assumption, I class, RG 475(b)"); + ("2025-08-16", false, "St Joachim, II class, no Lord/BVM/apostle clause applies, RG 476(b)") ] + | _ -> die "no probe list for %s" base + in + List.iter + (fun (d, expected, desc) -> + match creed_of_date rows d with + | Some got when Bool.equal got expected -> () + | Some got -> die "characterisation probe FAILED: %s (%s) -- expected creed=%b, extracted creed=%b" d desc expected got + | None -> die "characterisation probe FAILED: %s (%s) -- no row extracted for this date at all" d desc) + probes; + Printf.printf "extract_lms_ordo: all %d characterisation probes confirmed for %s\n" (List.length probes) edition_label; + (* Preface-witnesses task (2026-08-23): a SECOND opposite-prediction + probe set, over [praef] rather than [creed] -- the same discipline, + applied to the new column this task adds, per the task brief's own + "probe dates where the rule predicts OPPOSITE answers before + adjudicating anything" instruction. Three universal, unambiguous + fixed-date feasts per edition, each landing on a DIFFERENT named + preface (RG 484 Nativity, RG 497 Apostles, RG 499 Requiem) -- checked + directly against this run's own [rows] before this header is + written, a hard [die] on failure, not a printed claim. All Souls' + own date is edition-specific (2 November, TRANSFERRED to 3 November + in the one edition -- 2024-2025 -- where 2 November falls on a + Sunday; verified directly against each PDF's own text, not assumed + from RG 96 alone). *) + let praef_of_date rows d = List.find_map (fun r -> if String.equal r.date d then r.praef else None) rows in + let praef_probes = + match base with + | "lms-ordo-2023-2024.sexp" -> + [ ("2023-12-25", "Pr of the Nativity", "the Nativity of Our Lord, I class, RG 484(a)"); + ("2024-06-29", "Pr of the Apostles", "SS Peter & Paul, I class, RG 497"); + ("2024-11-02", "Pr of the Dead", "All Souls' Day, I class, RG 499") ] + | "lms-ordo-2024-2025.sexp" -> + [ ("2024-12-25", "Pr of the Nativity", "the Nativity of Our Lord, I class, RG 484(a)"); + ("2025-06-29", "Pr of the Apostles", "SS Peter & Paul, I class, RG 497"); + ( "2025-11-03", "Pr of the Dead", + "All Souls' Day, I class, RG 499 -- TRANSFERRED from 2 November, a Sunday in this edition's own calendar" ) + ] + | "lms-ordo-2025-2026.sexp" -> + [ ("2025-12-25", "Pr of the Nativity", "the Nativity of Our Lord, I class, RG 484(a)"); + ("2026-06-29", "Pr of the Apostles", "SS Peter & Paul, I class, RG 497"); + ("2026-11-02", "Pr of the Dead", "All Souls' Day, I class, RG 499") ] + | _ -> die "no praef probe list for %s" base + in + List.iter + (fun (d, expected_prefix, desc) -> + match praef_of_date rows d with + | Some got when has_prefix ~prefix:expected_prefix got -> () + | Some got -> + die "praef characterisation probe FAILED: %s (%s) -- expected prefix %S, extracted %S" d desc + expected_prefix got + | None -> die "praef characterisation probe FAILED: %s (%s) -- no praef extracted for this date at all" d desc) + praef_probes; + Printf.printf "extract_lms_ordo: all %d praef characterisation probes confirmed for %s\n" (List.length praef_probes) + edition_label; + let praef_probe_lines = + String.concat "\n" + (List.map (fun (d, expected_prefix, desc) -> Printf.sprintf "; %s (%s) -> %S" d desc expected_prefix) praef_probes) + in + let missing_praef_count = List.length missing_praef in + let probe_lines = + String.concat "\n" + (List.map + (fun (d, expected, desc) -> Printf.sprintf "; %s (%s) -> %s" d desc (if expected then "TRUE" else "FALSE")) + probes) + in + let header = + Printf.sprintf + {|; %s -- Witnesses task (2026-08-22-colitur-celebrant-rubrics-phase1), +; extending Task 6's single LMS fixture (2024-2025) to three editions of +; the Latin Mass Society's own printed Ordo for England & Wales -- a +; lineage independent of Divinum Officium/missalemeum/lectio (layers 3-4) +; and of the electronic LT.txt transcription (layers 1-2/5): a physically +; printed, professionally-compiled liturgical calendar with its own ISBN. +; +; Generator: tools/extract_lms_ordo.ml -- do not hand-edit; re-run against +; a fresh pdftotext dump and commit the diff instead. +; +; Source: "The Ordo %s", compiled by Peter Day-Milne, The Latin +; Mass Society, 9 Mallow Street, London EC1Y 8RQ (title page and +; colophon, verified directly against the PDF's own extracted text -- +; ISBN %s, Copyright (c) The Latin Mass Society, UK %s). +; PDF metadata: %s. +; URL (%s): +; %s +; The local PDF (docs/research/ordo/%s) is gitignored +; (CLAUDE.md: "docs/ is gitignored"), so this header, not git, is this +; fixture's only record of where it came from. +; SHA-256 of docs/research/ordo/%s: %s +; Extracted (UTC): %s +; Exact commands: +; pdftotext -layout docs/research/ordo/%s /tmp/lms.txt +; dune exec tools/extract_lms_ordo.exe -- /tmp/lms.txt \ +; docs/research/ordo/%s %s +; +; COVERAGE: %d day-rows, %s through %s (starting %s, +; through the day before a malformed trailing entry the extractor +; deliberately stops before -- see CHARACTERISATION below). +; +; CHARACTERISATION FINDINGS (Step 1, non-negotiable per the task brief -- +; the extraordinaryform.org Ordo silently omitted St Lawrence's vigil on +; EVERY date it covered, which nearly produced a false corroboration +; during the RG 33 work; every LMS edition, including this one, is probed +; with the same discipline before a single divergence is adjudicated): +; +; 1. Gl/Cr are printed PER MASS-OPTION, not per day (confirmed again for +; THIS edition: it carries many more Gl/Cr pairs than day-rows, because +; every diocesan variant repeats its own "Gl ... Cr ... Pr of ..." +; line). This extractor resolves the day's OWN office by taking the +; FIRST Gl/Cr pair in reading order, which the source's own layout +; guarantees is the universal entry's (diocesan variants are always +; introduced by a colon-terminated diocese-list line that comes AFTER +; the universal block, never before -- checked structurally, not +; assumed, in every sampled block during development, this edition +; included). +; +; 2. OPPOSITE-PREDICTION PROBES against Rite_ef.Rubrics_ef.creed (RG +; 475-476), RE-CHECKED BY THIS TOOL RUN (not merely copied from a prior +; edition's findings) before a single row of this fixture is trusted -- +; the technique CLAUDE.md records as the one that catches a source +; silently omitting the very thing it is meant to witness. This run's +; own extracted [rows] were checked against every one of the following +; before this header was even written; a failure here is a hard [die], +; not a printed claim: +%s +; All %d predictions confirmed, both directions (TRUE and FALSE each +; independently witnessed) -- this edition is discriminating, not a +; constant, on the Creed. +; +; 3. The BVM-Saturday roman numeral (I-V, "Missae de sancta Maria in +; sabbato", RG 309(a)) is anchored by WHOLE-LINE match against the five +; literal strings below, NEVER by substring search: "V Mass of BVM" is +; a substring of "IV Mass of BVM", so a naive `contains` check +; misreads every fourth-Mass day as the fifth (the coordinator's own +; misreading, recorded and retracted in commit 27b07b4 before Task 6 +; existed; the same anchoring logic runs unchanged for every edition). +; +; 4. SCOPE EXCLUDED, quantified, not silently dropped: +; - Diocesan variants: present on %d of %d days (%s%%) -- their own +; text is read only far enough to set [has_diocesan_variant], +; never compared, because colitur computes the UNIVERSAL General +; Roman Calendar only, with no diocesan overlay loaded. +; - The single malformed trailing entry ("...The OCTAVE DAY of the +; NATIVITY..." with a bare 4-digit token where a weekday+day-number +; pair belongs) is excluded outright: the extractor's own stop +; marker (a line whose first token is a bare 4-digit token) halts +; the scan before it. CONFIRMED IDENTICAL, word for word, across +; ALL FOUR known editions of this publisher's Ordo (2023-2024, +; 2024-2025, 2025-2026, and the 2024-2025 fixture's own prior +; header) -- always the literal stray year token "2025", regardless +; of which edition or which real trailing year it should read: a +; stale copy-paste leftover the compiler's own master document +; carries forward unedited release to release (even the 2024-2025 +; PDF's own tail page footer nearby still reads "Latin Mass Society +; Ordo 2021-2022"), not something specific to this edition. +; - Front matter, the Abbreviations/Rubrical-Primer/Breviary sections, +; and the "When may I say...?"/Appendix tail are never scanned at +; all (the extractor's own start/stop markers bound it to the Ordo +; proper). +; +; 5. THE "Pr of" (PREFACE) COLUMN (Preface-witnesses task, 2026-08-23) -- +; [row.praef], the RAW text from the FIRST "Pr of ..." / "Common Pr" +; match in the universal block's own reading order (the same +; FIRST-in-reading-order discipline finding 1 above already +; establishes for Gl/Cr). CAPTURED, NOT CLASSIFIED by this tool -- +; classification into {!Colitur_kernel.Preface.t} and the comparison +; against {!Rite_ef.Rubrics_ef.preface} both happen test-side +; (test/test_lms_ordo.ml's own [classify_praef]). +; +; CHARACTERISATION FINDING, non-negotiable per the same discipline as +; finding 2: this Ordo prints OPTION LISTS ("Pr of X or Pr of Y or +; Common Pr"), not always a single value -- and several of the named +; options ("Martyrs", "All Saints and Patron Saints" [almost always +; diocesan -- a Patronal feast's own preface -- but checked directly to +; occur ONCE in THIS edition's own UNIVERSAL block too, All Saints' Day +; itself, 1 November, paired with "or Common Pr"/"or Pr of the +; Trinity" depending on whether 1 November falls on a Sunday that +; edition, exactly matching {!Rite_ef.Rubrics_ef.preface}'s own RG +; 494(b) Sunday-gated Trinity fallback either way -- see that +; function's own citation], "the Dedication of a Church", "the Most Holy +; Sacrament" [Corpus Christi's own universal entry], "St John the +; Baptist", "the Angels") are genuinely NOT among the fourteen the +; 1962 Missale Romanum's own RG 484-497 enumerate (docs/research/LT.txt, +; lines 3936-4020, checked directly: no "de Martyribus"/"de Angelis"/ +; "de Dedicatione"/"de Ss.mo Sacramento"/"de S. Ioanne Baptista" clause +; exists anywhere in 482-499). Every one of these extras, in every +; instance found in this edition's UNIVERSAL block, is printed +; ALONGSIDE a genuine RG 482 answer (most often "or Common Pr"; on 24 +; March-shaped days, "or Pr of Lent", the live RG 486(b) de-Tempore +; answer) -- never as the row's OWN AND ONLY option. This is why the +; test-side comparison checks colitur's single computed answer is a +; MEMBER of this row's own parsed option set (dropping any unmapped +; extra name from that set first), not string equality against +; whichever option happens to print first -- the same "capture, don't +; force a match invented at extraction time" discipline +; tools/extract_fiuv_ordo.ml's own [row.praef] doc comment already +; states for a different publisher's format. +; +; OPPOSITE-PREDICTION PROBES against Rite_ef.Rubrics_ef.preface (RG +; 482-499), the SAME discipline as finding 2, applied to this new +; column -- RE-CHECKED BY THIS TOOL RUN before this header was +; written; a failed probe is a hard [die]: +%s +; All %d predictions confirmed -- this column, too, is discriminating, +; not a constant, across three different RG-cited outcomes (Nativity/ +; Apostles/Requiem). +; +; COVERAGE: %d of %d rows have no "Pr of"/"Common Pr" text found at +; all -- checked directly against this window's own Good Friday +; (Easter-2): its block carries NO Gl/Cr/Pr line whatsoever in this +; source (the 1955-restored Holy Week's own liturgical action has no +; Mass that day at all -- the SAME structural fact +; {!test_creed_coverage}'s own [window_good_fridays] already asserts +; for Gl/Cr), so [praef] shares that single expected gap rather than +; having a coverage gap of its own. +|} + dest edition_label isbn copyright_year pdf_meta url_note url pdf_base pdf_base pdf_sha (today ()) pdf_base + pdf_base dest + (List.length rows) + (match rows with r :: _ -> r.date | [] -> "?") + (match List.rev rows with r :: _ -> r.date | [] -> "?") + window_open_desc probe_lines (List.length probes) with_variant (List.length rows) + (Printf.sprintf "%.1f" (100.0 *. float_of_int with_variant /. float_of_int (List.length rows))) + praef_probe_lines (List.length praef_probes) missing_praef_count (List.length rows) + in + let body = Sexplib.Sexp.to_string_hum ~indent:2 (sexp_of_list sexp_of_row rows) in + let oc = open_out dest in + Fun.protect + ~finally:(fun () -> close_out_noerr oc) + (fun () -> + output_string oc header; + output_string oc body; + output_string oc "\n"); + Printf.printf "extract_lms_ordo: wrote %d rows to %s\n" (List.length rows) dest |