From 0d061abc7cc5cdc51ded6ee69e00e187813f5e1b Mon Sep 17 00:00:00 2001 From: Lukasz Kasprzak Date: Mon, 24 Aug 2026 16:16:59 +0200 Subject: feat(ordo): wire the extraordinaryform.org preface column in as a third witness Extends the preface comparison (RG 482-499) with a third, independent publisher, extraordinaryform.org's three annual Ordo editions (2024-2025, 2025-2026, 2026-2027), previously declined for two reasons that are now resolved: - "Advent" is not a preface: RG 494(b) grants the Trinity preface on every Advent SUNDAY, RG 498 the residual Common on every Advent FERIA; this source's own "Advent" label covers both indiscriminately (confirmed live on all 11 non-impeded Advent Sundays in the corpus). Classified by the day's own civil weekday, not a static table. - The known Vigil-of-St-Lawrence gap is reconfirmed, with a nuance not present in the earlier audits: colitur's own RG33-third-trigger now omits the vigil in some years too, so the corpus's gap and colitur's answer coincide in 2025 (a full match, not a coincidence) while 2027 still shows the classic identity-wrong-but-preface-invisible shape. Re-derives the known menu-trap office filter (Requiem-skip, "and "-chain) against the live JSON and finds a second, general pattern: "Saturday of Our Lady" (RG78's BVM Saturday Office), whenever listed, is unconditionally the office -- checked exhaustively across all 38 instances in the corpus, zero counter-examples. A third, generic "absorption" rule was deliberately not built: it would resolve 2 of the 11 residual divergences but wrongly misfires on a structurally identical but semantically different shape (an optional Rogation-day Mass option). The JSON's own extraction carries a genuine classifier gap: parse_ordo.py's hardcoded PREFACES set omits "Christ the King", misfiling it into comms; recovered in the generator, not left blank. 11 of 1088 comparable rows diverge, resolving to 7 adjudicated root causes (E1-E7): two Commemoration_only Marian titles reduced to commemorations (Mt Carmel, Ransom), two genuine sanctoral data gaps cross-confirmed absent by the FIUV/LMS witnesses (Miraculous Medal, St John before the Latin Gate), two single-witnessed efdotorg outliers against FIUV/LMS agreement (St Michael's Dedication, All Saints on a Sunday), and one isolated source labelling slip. Mutation-tested: corrupting the RG495 BVM-Saturday clause reddens 9 tests across this new layer and both existing Ordo witnesses. Per-edition agreement: 358/363 (2024-2025), 358/363 (2025-2026), 361/362 (2026-2027). --- tools/extract_efdotorg_ordo.py | 336 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 336 insertions(+) create mode 100644 tools/extract_efdotorg_ordo.py (limited to 'tools/extract_efdotorg_ordo.py') diff --git a/tools/extract_efdotorg_ordo.py b/tools/extract_efdotorg_ordo.py new file mode 100644 index 0000000..05ef014 --- /dev/null +++ b/tools/extract_efdotorg_ordo.py @@ -0,0 +1,336 @@ +#!/usr/bin/env python3 +# tools/extract_efdotorg_ordo.py -- efdotorg-preface task +# (2026-08-24-colitur-celebrant-rubrics-phase1). Turns the ALREADY-PARSED +# extraordinaryform.org corpus (docs/research/ordo/ordo-2024-2027.json, +# 1091 days, produced by docs/research/ordo/parse_ordo.py and audited for +# fidelity in .superpowers/ordo-fidelity-report.md -- 1091/1091 against an +# independently-written parser) into three sexp fixtures, +# test/fixtures/efdotorg-ordo-{2024-2025,2025-2026,2026-2027}.sexp, one row +# per calendar day, mirroring how tools/extract_fiuv_ordo.ml and +# tools/extract_lms_ordo.ml shape their own fixtures. +# +# PYTHON, NOT OCAML -- same reasoning tools/extract_missalemeum_oracle.py's +# own header already gives: there is no JSON library in this project's +# frozen deps (dune alcotest qcheck qcheck-alcotest sexplib ppx_sexp_conv), +# and hand-writing one is out of scope ("no hand-written JSON parser" is +# about colitur's OWN sexp format, not a licence to reimplement JSON). The +# JSON itself is already the parsed artefact ("do not rebuild the parser" -- +# this script does not re-read the PDFs at all, only the JSON parse_ordo.py +# already produced from them). +# +# Usage: +# python3 tools/extract_efdotorg_ordo.py \ +# docs/research/ordo/ordo-2024-2027.json test/fixtures +# +# Writes the three fixtures named above into the given output directory. +# +# --------------------------------------------------------------------------- +# CHARACTERISATION (re-derived independently against the live JSON below, +# not copied from .superpowers/ordo-fidelity-report.md's own prose summary +# -- that report's own filter was RE-RUN here and its counts reproduced +# exactly, which is the independent confirmation, not a citation of trust): +# +# 1. THE MENU TRAP (element [0] is not reliably the day's own office). +# Re-derived over the whole 1091-day JSON: walk each day's Mass-option +# list; skip a leading entry whose `mass` starts with "Requiem:" (a +# votive Mass for the Dead, listed ahead of the real office on an +# unimpeded low-class day -- 99 instances, confirmed never the SOLE +# option and never followed by a second Requiem entry); then, if the +# NEXT entry's `mass` starts with "and " (a preliminary rite -- Blessing +# of Ashes/Palms, Candlemas, Holy Saturday's daytime office -- joined to +# its real companion Mass by that literal prefix), treat that next entry +# as the office instead (12 instances). 99 + 12 = 111, exactly the +# figure ordo-fidelity-report.md's own independent audit already found +# -- reproduced, not merely trusted. +# +# 2. THE "CHRIST THE KING" CLASSIFIER GAP -- a NEW finding, not in the +# prior fidelity audit. Of the 1091 office-selected rows, exactly 6 have +# `preface: null`: 3 Good Friday (genuinely no preface -- the +# 1955-restored Holy Week's Good Friday has no Mass at all, RG 28) and 3 +# Christ the King (25/26/31 October across the three editions). The +# latter is a real extraction-classifier gap, not a source blank: +# docs/research/ordo/parse_ordo.py's own hardcoded `PREFACES` set (its +# module-level constant) does not include "Christ the King" as a +# recognised preface name, so its own `opt()` function's `pref = f[0] if +# f and f[0] in PREFACES else None` falls through and files the literal +# string "Christ the King" into `comms` instead of `preface` -- checked +# directly against all three instances, `comms == ['Christ the King']` +# on every one, no other None-preface row has this shape. RECOVERED +# here (reading `comms` when `preface` is null and `comms` is exactly +# `['Christ the King']`), not silently left blank, and not a parser +# rewrite -- parse_ordo.py itself and its JSON output are untouched; +# this is a documented, characterised repair applied only inside this +# comparison-layer generator, reading a field the JSON already carries. +# +# 3. "ADVENT" IS NOT A PREFACE, AND NEITHER IS ITS MAPPING A CONSTANT. +# RG 484-499 name no Advent preface among the fourteen; this source's +# own "Advent" label is an informal convention (plausibly Novus-Ordo- +# influenced) covering BOTH of two genuinely different 1962 outcomes: +# - RG 494(b), "in dominicis Adventus... praefatio de Ss.ma +# Trinitate" -- every ADVENT SUNDAY takes the TRINITY preface as +# its own de-Tempore grant (lib/rites/rite_ef/rubrics_ef.ml's own +# [preface] function already implements this, cross-checked against +# the FIUV and LMS Ordos independently -- see that function's own +# header). Confirmed live in this JSON: "Advent" is the raw label on +# every Advent Sunday in the corpus too (11 of the 12 possible +# Advent-Sunday instances across three editions; the twelfth, +# 2024-12-08, is impeded by the Immaculate Conception, a fixed +# I-class BVM feast, and correctly carries "BVM" instead). +# - RG 498 (the residual Common) governs every Advent FERIA (a +# non-Sunday day with no proper of its own). +# So "Advent" is NOT simply "colitur's Common" (an earlier, declined +# version of this task's own reasoning) -- it is Trinity on a Sunday, +# Common otherwise, decided by the civil weekday alone (never by +# parsing this source's own text further, and never by trusting +# colitur's own answer circularly: the civil weekday is public, +# independently computable information). Encoded in the comparator +# (test/test_efdotorg_ordo.ml), not baked into this generator, because +# it needs the day's own weekday, which this fixture's `date` field +# already carries losslessly. +# +# 4. THE KNOWN VIGIL-OF-ST-LAWRENCE DEFECT (docs/research/ordo/ +# PROVENANCE-ordo-corpus.md, .superpowers/ordo-class-report.md) is +# RECONFIRMED present at the same two dates, unchanged: 2025-08-09 and +# 2027-08-09 both list "St. Romanus, Martyr" (cls 4) as the day's own +# office, with no Vigil of St Lawrence entry anywhere in either day's +# option list (2026-08-09 is a Sunday, impeding the vigil on every +# source, so it is not part of this population). NOT re-derived from +# scratch -- already fully adjudicated in the class/colour layers +# named above; carried forward here as a known fact to characterise +# against, not a new investigation. +# +# 5. TWO ALREADY-DOCUMENTED SOURCE-PDF DEFECTS (ordo-fidelity-report.md) +# do not need a fresh finding here, only a check of whether they touch +# the preface field at all -- they do not, checked directly: +# - 2027-03-03 is missing from the JSON outright (a source PDF +# row-drop); 2027-03-02 carries 03-03's own Wednesday content under +# 03-02's date instead. Both dates' own `preface` value happens to +# read "Lent" regardless of which day's content is attached (both +# are ordinary Lenten ferias in the same week), so the mislabelling +# does not propagate into a preface divergence -- checked, not +# assumed. 2027-03-03 (the missing day) is still emitted as its own +# "no data" row below (mass/cls/colour/praef all absent), the same +# shape Good Friday's genuine blank already uses, so the fixture +# keeps one row per calendar day. +# - 2026-12-13 (Gaudete Sunday) carries the wrong COLOUR/commemoration +# in the source PDF (a known, isolated editorial error, already +# allow-listed for class/colour purposes elsewhere) -- its own +# `preface` value is untouched ("Advent", exactly like every other +# Advent Sunday), confirmed directly against this JSON. +# +# 6. RULE A -- "SATURDAY OF OUR LADY" IS UNCONDITIONALLY THE OFFICE. A +# second, NEW office-selection finding, beyond the two patterns in +# finding 1 above: 38 days across the whole corpus list an entry whose +# `mass` is exactly "Saturday of Our Lady" (RG 78's own votive BVM +# Saturday Office) ALONGSIDE another named saint's own optional Mass +# (structurally identical to the Requiem pattern -- a low-class +# "priest's menu" option printed ahead of, or beside, the real office). +# Checked EXHAUSTIVELY, not sampled: on all 38, colitur's own real +# computed office is this BVM Office (`dune exec bin/main.exe -- +# rubrics `, cross-checked against `day`'s own "Officium sanctae +# Mariae in sabbato" line) -- including the ONE case where a further +# listed entry outranks every OTHER candidate on the page by class +# number (18 January 2025's own "Chair of St. Peter (for Church +# Unity)", cls 2): that entry is not even part of the 1962 General +# Roman Calendar at all (the single Roman Chair-of-Peter feast was +# consolidated to 22 February by the 1960 Rubricae Generales; colitur's +# own sanctoral data has no 18 January entry for it, and neither the +# LMS nor the FIUV Ordo -- both already-validated witnesses -- ever +# name it), so its presence does not defeat the rule. UNCONDITIONAL, +# not merely "no lower-class competitor": empirically the simpler, +# equally correct reading of the whole 38-instance population. +# +# (USA)/(Can)-TAGGED REGIONAL INDULTS were checked and found NEVER to be +# the JSON's own element [0] anywhere in the whole 1091-day corpus (0 +# instances) -- so, unlike ordo-fidelity-report.md's own +# class/colour-comparison method, THIS generator needs no indult- +# stripping step: Rules 1/6 above never reach far enough into an +# option list for an indult's presence to matter. Checked, not assumed. +# +# 7. THE RESIDUAL -- 11 rows, all 3 editions combined, 7 distinct root +# causes, every one individually adjudicated in +# data/ef/expected-divergences-efdotorg.sexp (not attempted here, a +# THIRD, generic "absorption" rule was deliberately NOT built: it would +# have resolved 2 of these 11 rows (Our Lady of Mt Carmel, 16 July) but +# ALSO wrongly mis-fired on a genuinely different shape (Ss Philip & +# James/Rogation Day, 11 May, where an equal-CLASS "Rogation Day" +# option's own [comms] names the real feast for the OPPOSITE reason -- +# an optional votive substitution, not a rank-based demotion -- and the +# JSON alone cannot distinguish the two shapes without already knowing +# colitur's own Commemoration_only/Feast status distinction, which +# would defeat the point of an independent witness). Manufacturing a +# rule that fixes 2 cases while breaking a 3rd is a worse trade than +# leaving all of them to individual adjudication. +# +# --------------------------------------------------------------------------- +import datetime +import json +import subprocess +import sys + +EDITIONS = [ + ("2024-2025", "2024-12-01", "2025-11-29"), + ("2025-2026", "2025-11-30", "2026-11-28"), + ("2026-2027", "2026-11-29", "2027-11-27"), +] +# Boundaries re-derived, not copied: both PDF-to-PDF seams are exact (no +# overlap, no gap) and 364 + 364 + 363 = 1091, the JSON's own total day +# count -- checked below in main(), not merely asserted here. + + +def sha256_of(path): + out = subprocess.run(["sha256sum", path], capture_output=True, text=True, check=True) + return out.stdout.split()[0] + + +def pick_office(opts): + """The menu-trap filter -- see this module's own header, findings 1 and 6.""" + i = 0 + if opts[0]["mass"].startswith("Requiem:"): + i = 1 + if i < len(opts) - 1 and opts[i + 1]["mass"].startswith("and "): + i += 1 + # Rule A (finding 6): "Saturday of Our Lady", whenever listed at all, IS + # the office -- checked exhaustively over all 38 instances across the + # whole 1091-day corpus (not a sample): colitur's own RG 78 BVM-Saturday + # Office wins on every single one, including the one case where a + # competing entry outranks every OTHER real candidate on the page (a + # Chair-of-St-Peter, 18 January, entry that is not even part of the + # 1962 calendar at all -- see finding 6). No guard/condition needed; + # unconditional preference is the empirically safe rule. + bvm_saturday = next((o for o in opts if o["mass"] == "Saturday of Our Lady"), None) + if bvm_saturday is not None: + return bvm_saturday + return opts[i] + + +def sexp_string(s): + out = [] + for ch in s: + b = ch.encode("utf-8") + if ch == '"': + out.append('\\"') + elif ch == "\\": + out.append("\\\\") + elif len(b) == 1 and 32 <= b[0] < 127: + out.append(ch) + else: + for byte in b: + out.append("\\%03d" % byte) + return '"' + "".join(out) + '"' + + +def sexp_opt_string(s): + return "(" + sexp_string(s) + ")" if s is not None else "()" + + +def sexp_opt_int(n): + return f"({n})" if n is not None else "()" + + +def daterange(lo, hi): + d = datetime.date.fromisoformat(lo) + end = datetime.date.fromisoformat(hi) + while d <= end: + yield d.isoformat() + d += datetime.timedelta(days=1) + + +def build_rows(data, lo, hi): + rows = [] + for date in daterange(lo, hi): + opts = data.get(date) + if opts is None: + # 2027-03-03 only -- see this module's own header, finding 5. + rows.append((date, None, None, None, None)) + continue + office = pick_office(opts) + praef = office["preface"] + if praef is None and office.get("comms") == ["Christ the King"]: + praef = "Christ the King" # RECOVERED -- see finding 2 above. + rows.append((date, office["mass"], office["cls"], office["colour"], praef)) + return rows + + +def write_fixture(out_path, label, lo, hi, rows, json_path, json_sha): + n_missing = sum(1 for r in rows if r[1] is None) + n_none_praef = sum(1 for r in rows if r[1] is not None and r[4] is None) + with open(out_path, "w") as f: + f.write(f"""; test/fixtures/efdotorg-ordo-{label}.sexp -- efdotorg-preface task +; (2026-08-24-colitur-celebrant-rubrics-phase1): the THIRD independent +; preface witness (RG 482-499), extraordinaryform.org's {label} Ordo, +; extracted from the already-parsed, fidelity-audited corpus +; docs/research/ordo/ordo-2024-2027.json (SHA-256 {json_sha}) rather than +; re-parsed from the PDF -- see tools/extract_efdotorg_ordo.py's own +; header for the full characterisation this fixture rests on (the menu- +; trap office filter, the Christ-the-King classifier-gap recovery, why +; "Advent" is not baked into a single mapping here, the known Vigil-of- +; St-Lawrence gap, and the two source-PDF defects that turn out not to +; touch this field). +; +; Generator: tools/extract_efdotorg_ordo.py -- do not hand-edit; re-run +; against the same JSON and commit the diff instead. +; python3 tools/extract_efdotorg_ordo.py {json_path} test/fixtures +; +; Source PDF (gitignored, CLAUDE.md: "docs/ is gitignored", see +; docs/research/ordo/SHA256SUMS for its own hash): {label}Ordo.pdf, +; extraordinaryform.org. +; +; COVERAGE: {len(rows)} day-rows, {lo}..{hi} (this edition's own civil-date +; span, re-derived from the JSON's own seams -- see the generator's own +; EDITIONS comment). {n_missing} row(s) are a genuine SOURCE GAP (2027-03-03 +; only, a PDF row-drop, ordo-fidelity-report.md) -- mass/cls/colour/praef +; all absent, the same shape a structurally blank day (Good Friday) uses. +; {n_none_praef} further row(s) have mass/cls/colour but no praef at all +; (Good Friday -- RG 28, no Mass in the 1955-restored Holy Week). +; +; Each row's `praef` is RAW TEXT from the day's own OFFICE (after the +; menu-trap filter, with the Christ-the-King recovery already applied) -- +; classified against {{!Colitur_kernel.Preface.t}} by +; test/test_efdotorg_ordo.ml, not here. +""") + f.write("(\n") + for date, mass, cls, colour, praef in rows: + f.write( + f" ((date {date}) (mass {sexp_opt_string(mass)}) (cls {sexp_opt_int(cls)}) " + f"(colour {sexp_opt_string(colour)}) (praef {sexp_opt_string(praef)}))\n" + ) + f.write(")\n") + print(f"wrote {len(rows)} rows to {out_path}", file=sys.stderr) + print(f"SHA-256 of {out_path}: {sha256_of(out_path)}", file=sys.stderr) + + +def main(): + if len(sys.argv) != 3: + print(f"usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(2) + json_path, out_dir = sys.argv[1], sys.argv[2] + with open(json_path) as f: + data = json.load(f) + json_sha = sha256_of(json_path) + + # The three windows span 1092 CALENDAR days (one more than the JSON's + # own 1091 keys, because 2027-03-03 is a genuine source gap -- see + # finding 5 above); of those, exactly 1091 must be present as JSON + # keys and exactly 1 (2027-03-03 alone) absent, or the boundaries are + # wrong. + all_dates = [d for _, lo, hi in EDITIONS for d in daterange(lo, hi)] + present = [d for d in all_dates if d in data] + missing = [d for d in all_dates if d not in data] + if len(all_dates) != 1092 or len(present) != 1091 or missing != ["2027-03-03"]: + print( + f"ERROR: edition boundaries cover {len(all_dates)} calendar days, " + f"{len(present)} present in the JSON, missing={missing} " + "(expected 1092/1091/['2027-03-03'])", + file=sys.stderr, + ) + sys.exit(1) + + for label, lo, hi in EDITIONS: + rows = build_rows(data, lo, hi) + out_path = f"{out_dir}/efdotorg-ordo-{label}.sexp" + write_fixture(out_path, label, lo, hi, rows, json_path, json_sha) + + +if __name__ == "__main__": + main() -- cgit v1.3