summaryrefslogtreecommitdiff
path: root/tools
diff options
context:
space:
mode:
authorLukasz Kasprzak <lukas@labunix.xyz>2026-08-24 16:29:23 +0200
committerLukasz Kasprzak <lukas@labunix.xyz>2026-08-24 16:29:23 +0200
commit8a5da8756cf6b57aa36c01b71ce893f7967d3a29 (patch)
tree10873a692400fa25ce54c67ac9fc34d46af86272 /tools
parent4f87cbde4ee79ac96aa7ebfed105b3a5ec02be43 (diff)
parent73b15551804bb63ee0081005e2869d36afb54be2 (diff)
downloadcolitur-8a5da8756cf6b57aa36c01b71ce893f7967d3a29.tar.gz
colitur-8a5da8756cf6b57aa36c01b71ce893f7967d3a29.zip
merge: celebrant Mass rubrics, and four rubrical corrections
colitur now prints what a real ordo prints for the Mass: which formulary is said and how it was reached, the Gloria, the Creed, the preface, and the commemorations with their Low-Mass/sung distinction. colitur rubrics joins day and readings. Four defects were found and fixed on the way, each by an external witness rather than by inspection: the Creed said at Requiem Masses (RG 476(f)), the missing bissextile shift of St Matthias and St Gabriel (2 041 leap years), Rogation Monday and Tuesday coloured violet in Paschaltide (RG 119(b)), and the ferias after the Ascension resuming the wrong Sunday's Mass rather than the Ascension's. Validation gained a sixth layer and then some: the preface is checked against three independent publishers over seven witness-years (FIUV, three LMS editions, three extraordinaryform.org editions), none of which shares the Divinum Officium -> missalemeum -> lectio lineage the older layers all descend from.
Diffstat (limited to 'tools')
-rw-r--r--tools/bootstrap_lectionary.ml70
-rw-r--r--tools/dune27
-rw-r--r--tools/extract_efdotorg_ordo.py336
-rw-r--r--tools/extract_fiuv_ordo.ml855
-rw-r--r--tools/extract_lms_ordo.ml729
5 files changed, 2017 insertions, 0 deletions
diff --git a/tools/bootstrap_lectionary.ml b/tools/bootstrap_lectionary.ml
index 7df290b..1e9b8ed 100644
--- a/tools/bootstrap_lectionary.ml
+++ b/tools/bootstrap_lectionary.ml
@@ -637,6 +637,75 @@ let trinity_week_entries =
lectionary has been bitten by before. *)
[ "monday"; "tuesday"; "wednesday"; "friday"; "saturday" ]
+(* RG 299's own escape clause, second instance (celebrant-rubrics-phase1,
+ Bug 2, 2026-08-22). The Proprium de Tempore carries an explicit
+ marginal rubric immediately after the Ascension's own Mass propers,
+ before "Dominica post Ascensionem" begins (scan1.txt:20749-20754, both
+ scans corroborate): "Diebus a feria VI post Ascensionem usque ad
+ feriam VI ante vigiliam Pentecostes inclusive, dicitur Missa de festo
+ Ascensionis, cum Gloria et praefatione de Ascensione, sine Credo et
+ Communicantes proprio" -- on the days from the Friday after the
+ Ascension until the Friday before the Vigil of Pentecost INCLUSIVE,
+ the MASS OF THE FEAST OF THE ASCENSION is said (with the Gloria and
+ the Ascension preface, without the Creed and the proper
+ Communicantes). Corroborated independently by RG 490(b) (LT.txt:3981-
+ 3985), which assigns the Ascension preface "tamquam de Tempore" to
+ every Mass in that same window lacking a proper preface of its own --
+ consistent only if the Mass repeated there is the Ascension's own,
+ not a numbered Sunday's (which would need no preface override at
+ all).
+
+ Witnessed directly, not merely inferred from the rubric: the LMS Ordo
+ prints "Mass of the Ascension" literally, across all three fixture
+ windows this project has built (2023-2024: 1 day; 2024-2025: 3 days;
+ 2025-2026: 2 days -- data/ef/expected-divergences-lms.sexp's own L2,
+ closed by this entry), on EVERY affected day in BOTH the remainder of
+ Ascension's own week and the following week, not only the first --
+ ruling out the alternative reading that the second week instead
+ resumes "Dominica post Ascensionem"'s own Mass. The FIUV Ordo's own
+ title convention for these days (2025-2026 fixture, 2026-05-21/22,
+ "Feria V/VI Dom. post Ascensionem, De ea") could look like the
+ opposite at a glance -- it is not: FIUV's title names which WEEK a
+ feria belongs to (a bookkeeping label, the same role colitur's own
+ week-numbered slug plays), while its own [praef] field for those
+ exact two days independently reads "Ascensionis", confirming the Mass
+ actually said is still the Ascension's, exactly as the marginal
+ rubric and RG 490(b) both require.
+
+ The window's two Saturdays (Easter+41, Easter+48) are DELIBERATELY
+ NOT given entries here. Easter+48 is the Vigil of Pentecost, a named
+ day with its own Mass ({!Rite_ef.Temporal_ef.named}), outside the
+ rubric's own "usque ad feriam VI" upper bound. Easter+41 is an
+ ordinary IV-class Saturday, so {!Rite_ef.Lectionary_ef.readings}'s
+ own RG 78/RG 309(a) votive-Mass-of-Our-Lady-on-Saturday branch ALWAYS
+ intercepts it before step 2 is ever reached (checked directly:
+ [is_bvm_saturday_office] fires unconditionally on any unoccupied
+ Class4 Saturday, which every ordinary Paschaltide Saturday is) -- an
+ entry for either Saturday slug would be dead data that
+ [assert_reachable] cannot catch (both slugs it would name ARE real,
+ reachable Temporal_ef offices; the shadowing happens one level up, in
+ [readings]' own step ordering, not in slug reachability).
+
+ Six ferias, matching the Ascension's own citation already given above
+ (Act. 1, 1-11 / Marc. 16, 14-20, scan1.txt:20701-20730,
+ [ef-ascension]'s own entry): the one remaining non-Saturday day of
+ Ascension's own week (Easter+40) and the five weekdays of the
+ following week (Easter+43..47), bounded above by that week's own
+ Friday -- the Saturday after it is the Vigil of Pentecost, named,
+ never reaching this table. *)
+let ascension_week_citations = pair ~first:"Acts 1:1-11" ~gospel:"Mark 16:14-20"
+
+let ascension_week_entries =
+ List.map
+ (fun slug -> (slug_or_die slug, ascension_week_citations))
+ [ "ef-easter-6-friday";
+ "ef-easter-7-monday";
+ "ef-easter-7-tuesday";
+ "ef-easter-7-wednesday";
+ "ef-easter-7-thursday";
+ "ef-easter-7-friday"
+ ]
+
let epiphanytide_opening_entries =
List.concat_map
(fun (family, wds, cs) ->
@@ -789,6 +858,7 @@ let () =
translated @ vigil_entries @ holy_week_entries @ passion_tuesday_entry
@ ember_saturday_corrections @ nativity_octave_entries @ holy_name_entries
@ epiphanytide_opening_entries @ trinity_week_entries @ movable_feast_entries
+ @ ascension_week_entries
in
assert_reachable entries;
let lect =
diff --git a/tools/dune b/tools/dune
index 0dd0937..9a4dea5 100644
--- a/tools/dune
+++ b/tools/dune
@@ -23,6 +23,20 @@
(name bootstrap_lectionary)
(libraries colitur_kernel rite_ef unix sexplib))
+; Task 6 (2026-08-21-colitur-celebrant-rubrics-phase1): turns pdftotext's
+; -layout dump of the Latin Mass Society Ordo into test/fixtures/
+; lms-ordo-2024-2025.sexp. Run via `dune exec tools/extract_lms_ordo.exe --
+; <pdftotext-layout.txt> <source.pdf> <dest.sexp>`. Same `unix`-for-
+; `sha256sum` usage as the two bootstrap executables above; `colitur_kernel`
+; only, no `rite_ef` -- this tool validates dates through `Date.make` but
+; reads no EF-specific vocabulary.
+(executable
+ (name extract_lms_ordo)
+ (modules extract_lms_ordo)
+ (libraries colitur_kernel unix sexplib)
+ (preprocess
+ (pps ppx_sexp_conv)))
+
; check_citations.py's own self-test (test_check_citations.py). Python, not
; OCaml, so it cannot be a `(test ...)` stanza -- an alias rule invoking it
; directly is dune's own documented shape for a non-OCaml check. Wired into
@@ -41,3 +55,16 @@
(deps check_citations.py test_check_citations.py)
(action
(run python3 test_check_citations.py)))
+
+; Witnesses task (2026-08-22-colitur-celebrant-rubrics-phase1): turns
+; pdftotext's -layout dump of the FIUV universal Ordo into
+; test/fixtures/fiuv-ordo-2025-2026.sexp. Run via `dune exec
+; tools/extract_fiuv_ordo.exe -- <pdftotext-layout.txt> <source.pdf>
+; <dest.sexp>`. Same `unix`-for-`sha256sum` usage as extract_lms_ordo.ml;
+; `colitur_kernel` only, no `rite_ef` -- same reasoning, dates only.
+(executable
+ (name extract_fiuv_ordo)
+ (modules extract_fiuv_ordo)
+ (libraries colitur_kernel unix sexplib)
+ (preprocess
+ (pps ppx_sexp_conv)))
diff --git a/tools/extract_efdotorg_ordo.py b/tools/extract_efdotorg_ordo.py
new file mode 100644
index 0000000..05ef014
--- /dev/null
+++ b/tools/extract_efdotorg_ordo.py
@@ -0,0 +1,336 @@
+#!/usr/bin/env python3
+# tools/extract_efdotorg_ordo.py -- efdotorg-preface task
+# (2026-08-24-colitur-celebrant-rubrics-phase1). Turns the ALREADY-PARSED
+# extraordinaryform.org corpus (docs/research/ordo/ordo-2024-2027.json,
+# 1091 days, produced by docs/research/ordo/parse_ordo.py and audited for
+# fidelity in .superpowers/ordo-fidelity-report.md -- 1091/1091 against an
+# independently-written parser) into three sexp fixtures,
+# test/fixtures/efdotorg-ordo-{2024-2025,2025-2026,2026-2027}.sexp, one row
+# per calendar day, mirroring how tools/extract_fiuv_ordo.ml and
+# tools/extract_lms_ordo.ml shape their own fixtures.
+#
+# PYTHON, NOT OCAML -- same reasoning tools/extract_missalemeum_oracle.py's
+# own header already gives: there is no JSON library in this project's
+# frozen deps (dune alcotest qcheck qcheck-alcotest sexplib ppx_sexp_conv),
+# and hand-writing one is out of scope ("no hand-written JSON parser" is
+# about colitur's OWN sexp format, not a licence to reimplement JSON). The
+# JSON itself is already the parsed artefact ("do not rebuild the parser" --
+# this script does not re-read the PDFs at all, only the JSON parse_ordo.py
+# already produced from them).
+#
+# Usage:
+# python3 tools/extract_efdotorg_ordo.py \
+# docs/research/ordo/ordo-2024-2027.json test/fixtures
+#
+# Writes the three fixtures named above into the given output directory.
+#
+# ---------------------------------------------------------------------------
+# CHARACTERISATION (re-derived independently against the live JSON below,
+# not copied from .superpowers/ordo-fidelity-report.md's own prose summary
+# -- that report's own filter was RE-RUN here and its counts reproduced
+# exactly, which is the independent confirmation, not a citation of trust):
+#
+# 1. THE MENU TRAP (element [0] is not reliably the day's own office).
+# Re-derived over the whole 1091-day JSON: walk each day's Mass-option
+# list; skip a leading entry whose `mass` starts with "Requiem:" (a
+# votive Mass for the Dead, listed ahead of the real office on an
+# unimpeded low-class day -- 99 instances, confirmed never the SOLE
+# option and never followed by a second Requiem entry); then, if the
+# NEXT entry's `mass` starts with "and " (a preliminary rite -- Blessing
+# of Ashes/Palms, Candlemas, Holy Saturday's daytime office -- joined to
+# its real companion Mass by that literal prefix), treat that next entry
+# as the office instead (12 instances). 99 + 12 = 111, exactly the
+# figure ordo-fidelity-report.md's own independent audit already found
+# -- reproduced, not merely trusted.
+#
+# 2. THE "CHRIST THE KING" CLASSIFIER GAP -- a NEW finding, not in the
+# prior fidelity audit. Of the 1091 office-selected rows, exactly 6 have
+# `preface: null`: 3 Good Friday (genuinely no preface -- the
+# 1955-restored Holy Week's Good Friday has no Mass at all, RG 28) and 3
+# Christ the King (25/26/31 October across the three editions). The
+# latter is a real extraction-classifier gap, not a source blank:
+# docs/research/ordo/parse_ordo.py's own hardcoded `PREFACES` set (its
+# module-level constant) does not include "Christ the King" as a
+# recognised preface name, so its own `opt()` function's `pref = f[0] if
+# f and f[0] in PREFACES else None` falls through and files the literal
+# string "Christ the King" into `comms` instead of `preface` -- checked
+# directly against all three instances, `comms == ['Christ the King']`
+# on every one, no other None-preface row has this shape. RECOVERED
+# here (reading `comms` when `preface` is null and `comms` is exactly
+# `['Christ the King']`), not silently left blank, and not a parser
+# rewrite -- parse_ordo.py itself and its JSON output are untouched;
+# this is a documented, characterised repair applied only inside this
+# comparison-layer generator, reading a field the JSON already carries.
+#
+# 3. "ADVENT" IS NOT A PREFACE, AND NEITHER IS ITS MAPPING A CONSTANT.
+# RG 484-499 name no Advent preface among the fourteen; this source's
+# own "Advent" label is an informal convention (plausibly Novus-Ordo-
+# influenced) covering BOTH of two genuinely different 1962 outcomes:
+# - RG 494(b), "in dominicis Adventus... praefatio de Ss.ma
+# Trinitate" -- every ADVENT SUNDAY takes the TRINITY preface as
+# its own de-Tempore grant (lib/rites/rite_ef/rubrics_ef.ml's own
+# [preface] function already implements this, cross-checked against
+# the FIUV and LMS Ordos independently -- see that function's own
+# header). Confirmed live in this JSON: "Advent" is the raw label on
+# every Advent Sunday in the corpus too (11 of the 12 possible
+# Advent-Sunday instances across three editions; the twelfth,
+# 2024-12-08, is impeded by the Immaculate Conception, a fixed
+# I-class BVM feast, and correctly carries "BVM" instead).
+# - RG 498 (the residual Common) governs every Advent FERIA (a
+# non-Sunday day with no proper of its own).
+# So "Advent" is NOT simply "colitur's Common" (an earlier, declined
+# version of this task's own reasoning) -- it is Trinity on a Sunday,
+# Common otherwise, decided by the civil weekday alone (never by
+# parsing this source's own text further, and never by trusting
+# colitur's own answer circularly: the civil weekday is public,
+# independently computable information). Encoded in the comparator
+# (test/test_efdotorg_ordo.ml), not baked into this generator, because
+# it needs the day's own weekday, which this fixture's `date` field
+# already carries losslessly.
+#
+# 4. THE KNOWN VIGIL-OF-ST-LAWRENCE DEFECT (docs/research/ordo/
+# PROVENANCE-ordo-corpus.md, .superpowers/ordo-class-report.md) is
+# RECONFIRMED present at the same two dates, unchanged: 2025-08-09 and
+# 2027-08-09 both list "St. Romanus, Martyr" (cls 4) as the day's own
+# office, with no Vigil of St Lawrence entry anywhere in either day's
+# option list (2026-08-09 is a Sunday, impeding the vigil on every
+# source, so it is not part of this population). NOT re-derived from
+# scratch -- already fully adjudicated in the class/colour layers
+# named above; carried forward here as a known fact to characterise
+# against, not a new investigation.
+#
+# 5. TWO ALREADY-DOCUMENTED SOURCE-PDF DEFECTS (ordo-fidelity-report.md)
+# do not need a fresh finding here, only a check of whether they touch
+# the preface field at all -- they do not, checked directly:
+# - 2027-03-03 is missing from the JSON outright (a source PDF
+# row-drop); 2027-03-02 carries 03-03's own Wednesday content under
+# 03-02's date instead. Both dates' own `preface` value happens to
+# read "Lent" regardless of which day's content is attached (both
+# are ordinary Lenten ferias in the same week), so the mislabelling
+# does not propagate into a preface divergence -- checked, not
+# assumed. 2027-03-03 (the missing day) is still emitted as its own
+# "no data" row below (mass/cls/colour/praef all absent), the same
+# shape Good Friday's genuine blank already uses, so the fixture
+# keeps one row per calendar day.
+# - 2026-12-13 (Gaudete Sunday) carries the wrong COLOUR/commemoration
+# in the source PDF (a known, isolated editorial error, already
+# allow-listed for class/colour purposes elsewhere) -- its own
+# `preface` value is untouched ("Advent", exactly like every other
+# Advent Sunday), confirmed directly against this JSON.
+#
+# 6. RULE A -- "SATURDAY OF OUR LADY" IS UNCONDITIONALLY THE OFFICE. A
+# second, NEW office-selection finding, beyond the two patterns in
+# finding 1 above: 38 days across the whole corpus list an entry whose
+# `mass` is exactly "Saturday of Our Lady" (RG 78's own votive BVM
+# Saturday Office) ALONGSIDE another named saint's own optional Mass
+# (structurally identical to the Requiem pattern -- a low-class
+# "priest's menu" option printed ahead of, or beside, the real office).
+# Checked EXHAUSTIVELY, not sampled: on all 38, colitur's own real
+# computed office is this BVM Office (`dune exec bin/main.exe --
+# rubrics <year>`, cross-checked against `day`'s own "Officium sanctae
+# Mariae in sabbato" line) -- including the ONE case where a further
+# listed entry outranks every OTHER candidate on the page by class
+# number (18 January 2025's own "Chair of St. Peter (for Church
+# Unity)", cls 2): that entry is not even part of the 1962 General
+# Roman Calendar at all (the single Roman Chair-of-Peter feast was
+# consolidated to 22 February by the 1960 Rubricae Generales; colitur's
+# own sanctoral data has no 18 January entry for it, and neither the
+# LMS nor the FIUV Ordo -- both already-validated witnesses -- ever
+# name it), so its presence does not defeat the rule. UNCONDITIONAL,
+# not merely "no lower-class competitor": empirically the simpler,
+# equally correct reading of the whole 38-instance population.
+#
+# (USA)/(Can)-TAGGED REGIONAL INDULTS were checked and found NEVER to be
+# the JSON's own element [0] anywhere in the whole 1091-day corpus (0
+# instances) -- so, unlike ordo-fidelity-report.md's own
+# class/colour-comparison method, THIS generator needs no indult-
+# stripping step: Rules 1/6 above never reach far enough into an
+# option list for an indult's presence to matter. Checked, not assumed.
+#
+# 7. THE RESIDUAL -- 11 rows, all 3 editions combined, 7 distinct root
+# causes, every one individually adjudicated in
+# data/ef/expected-divergences-efdotorg.sexp (not attempted here, a
+# THIRD, generic "absorption" rule was deliberately NOT built: it would
+# have resolved 2 of these 11 rows (Our Lady of Mt Carmel, 16 July) but
+# ALSO wrongly mis-fired on a genuinely different shape (Ss Philip &
+# James/Rogation Day, 11 May, where an equal-CLASS "Rogation Day"
+# option's own [comms] names the real feast for the OPPOSITE reason --
+# an optional votive substitution, not a rank-based demotion -- and the
+# JSON alone cannot distinguish the two shapes without already knowing
+# colitur's own Commemoration_only/Feast status distinction, which
+# would defeat the point of an independent witness). Manufacturing a
+# rule that fixes 2 cases while breaking a 3rd is a worse trade than
+# leaving all of them to individual adjudication.
+#
+# ---------------------------------------------------------------------------
+import datetime
+import json
+import subprocess
+import sys
+
+EDITIONS = [
+ ("2024-2025", "2024-12-01", "2025-11-29"),
+ ("2025-2026", "2025-11-30", "2026-11-28"),
+ ("2026-2027", "2026-11-29", "2027-11-27"),
+]
+# Boundaries re-derived, not copied: both PDF-to-PDF seams are exact (no
+# overlap, no gap) and 364 + 364 + 363 = 1091, the JSON's own total day
+# count -- checked below in main(), not merely asserted here.
+
+
+def sha256_of(path):
+ out = subprocess.run(["sha256sum", path], capture_output=True, text=True, check=True)
+ return out.stdout.split()[0]
+
+
+def pick_office(opts):
+ """The menu-trap filter -- see this module's own header, findings 1 and 6."""
+ i = 0
+ if opts[0]["mass"].startswith("Requiem:"):
+ i = 1
+ if i < len(opts) - 1 and opts[i + 1]["mass"].startswith("and "):
+ i += 1
+ # Rule A (finding 6): "Saturday of Our Lady", whenever listed at all, IS
+ # the office -- checked exhaustively over all 38 instances across the
+ # whole 1091-day corpus (not a sample): colitur's own RG 78 BVM-Saturday
+ # Office wins on every single one, including the one case where a
+ # competing entry outranks every OTHER real candidate on the page (a
+ # Chair-of-St-Peter, 18 January, entry that is not even part of the
+ # 1962 calendar at all -- see finding 6). No guard/condition needed;
+ # unconditional preference is the empirically safe rule.
+ bvm_saturday = next((o for o in opts if o["mass"] == "Saturday of Our Lady"), None)
+ if bvm_saturday is not None:
+ return bvm_saturday
+ return opts[i]
+
+
+def sexp_string(s):
+ out = []
+ for ch in s:
+ b = ch.encode("utf-8")
+ if ch == '"':
+ out.append('\\"')
+ elif ch == "\\":
+ out.append("\\\\")
+ elif len(b) == 1 and 32 <= b[0] < 127:
+ out.append(ch)
+ else:
+ for byte in b:
+ out.append("\\%03d" % byte)
+ return '"' + "".join(out) + '"'
+
+
+def sexp_opt_string(s):
+ return "(" + sexp_string(s) + ")" if s is not None else "()"
+
+
+def sexp_opt_int(n):
+ return f"({n})" if n is not None else "()"
+
+
+def daterange(lo, hi):
+ d = datetime.date.fromisoformat(lo)
+ end = datetime.date.fromisoformat(hi)
+ while d <= end:
+ yield d.isoformat()
+ d += datetime.timedelta(days=1)
+
+
+def build_rows(data, lo, hi):
+ rows = []
+ for date in daterange(lo, hi):
+ opts = data.get(date)
+ if opts is None:
+ # 2027-03-03 only -- see this module's own header, finding 5.
+ rows.append((date, None, None, None, None))
+ continue
+ office = pick_office(opts)
+ praef = office["preface"]
+ if praef is None and office.get("comms") == ["Christ the King"]:
+ praef = "Christ the King" # RECOVERED -- see finding 2 above.
+ rows.append((date, office["mass"], office["cls"], office["colour"], praef))
+ return rows
+
+
+def write_fixture(out_path, label, lo, hi, rows, json_path, json_sha):
+ n_missing = sum(1 for r in rows if r[1] is None)
+ n_none_praef = sum(1 for r in rows if r[1] is not None and r[4] is None)
+ with open(out_path, "w") as f:
+ f.write(f"""; test/fixtures/efdotorg-ordo-{label}.sexp -- efdotorg-preface task
+; (2026-08-24-colitur-celebrant-rubrics-phase1): the THIRD independent
+; preface witness (RG 482-499), extraordinaryform.org's {label} Ordo,
+; extracted from the already-parsed, fidelity-audited corpus
+; docs/research/ordo/ordo-2024-2027.json (SHA-256 {json_sha}) rather than
+; re-parsed from the PDF -- see tools/extract_efdotorg_ordo.py's own
+; header for the full characterisation this fixture rests on (the menu-
+; trap office filter, the Christ-the-King classifier-gap recovery, why
+; "Advent" is not baked into a single mapping here, the known Vigil-of-
+; St-Lawrence gap, and the two source-PDF defects that turn out not to
+; touch this field).
+;
+; Generator: tools/extract_efdotorg_ordo.py -- do not hand-edit; re-run
+; against the same JSON and commit the diff instead.
+; python3 tools/extract_efdotorg_ordo.py {json_path} test/fixtures
+;
+; Source PDF (gitignored, CLAUDE.md: "docs/ is gitignored", see
+; docs/research/ordo/SHA256SUMS for its own hash): {label}Ordo.pdf,
+; extraordinaryform.org.
+;
+; COVERAGE: {len(rows)} day-rows, {lo}..{hi} (this edition's own civil-date
+; span, re-derived from the JSON's own seams -- see the generator's own
+; EDITIONS comment). {n_missing} row(s) are a genuine SOURCE GAP (2027-03-03
+; only, a PDF row-drop, ordo-fidelity-report.md) -- mass/cls/colour/praef
+; all absent, the same shape a structurally blank day (Good Friday) uses.
+; {n_none_praef} further row(s) have mass/cls/colour but no praef at all
+; (Good Friday -- RG 28, no Mass in the 1955-restored Holy Week).
+;
+; Each row's `praef` is RAW TEXT from the day's own OFFICE (after the
+; menu-trap filter, with the Christ-the-King recovery already applied) --
+; classified against {{!Colitur_kernel.Preface.t}} by
+; test/test_efdotorg_ordo.ml, not here.
+""")
+ f.write("(\n")
+ for date, mass, cls, colour, praef in rows:
+ f.write(
+ f" ((date {date}) (mass {sexp_opt_string(mass)}) (cls {sexp_opt_int(cls)}) "
+ f"(colour {sexp_opt_string(colour)}) (praef {sexp_opt_string(praef)}))\n"
+ )
+ f.write(")\n")
+ print(f"wrote {len(rows)} rows to {out_path}", file=sys.stderr)
+ print(f"SHA-256 of {out_path}: {sha256_of(out_path)}", file=sys.stderr)
+
+
+def main():
+ if len(sys.argv) != 3:
+ print(f"usage: {sys.argv[0]} <ordo-2024-2027.json> <out-dir>", file=sys.stderr)
+ sys.exit(2)
+ json_path, out_dir = sys.argv[1], sys.argv[2]
+ with open(json_path) as f:
+ data = json.load(f)
+ json_sha = sha256_of(json_path)
+
+ # The three windows span 1092 CALENDAR days (one more than the JSON's
+ # own 1091 keys, because 2027-03-03 is a genuine source gap -- see
+ # finding 5 above); of those, exactly 1091 must be present as JSON
+ # keys and exactly 1 (2027-03-03 alone) absent, or the boundaries are
+ # wrong.
+ all_dates = [d for _, lo, hi in EDITIONS for d in daterange(lo, hi)]
+ present = [d for d in all_dates if d in data]
+ missing = [d for d in all_dates if d not in data]
+ if len(all_dates) != 1092 or len(present) != 1091 or missing != ["2027-03-03"]:
+ print(
+ f"ERROR: edition boundaries cover {len(all_dates)} calendar days, "
+ f"{len(present)} present in the JSON, missing={missing} "
+ "(expected 1092/1091/['2027-03-03'])",
+ file=sys.stderr,
+ )
+ sys.exit(1)
+
+ for label, lo, hi in EDITIONS:
+ rows = build_rows(data, lo, hi)
+ out_path = f"{out_dir}/efdotorg-ordo-{label}.sexp"
+ write_fixture(out_path, label, lo, hi, rows, json_path, json_sha)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/extract_fiuv_ordo.ml b/tools/extract_fiuv_ordo.ml
new file mode 100644
index 0000000..26f5b16
--- /dev/null
+++ b/tools/extract_fiuv_ordo.ml
@@ -0,0 +1,855 @@
+(* Witnesses task (2026-08-22-colitur-celebrant-rubrics-phase1): turns
+ pdftotext's -layout dump of the FIUV (Foederatio Internationalis Una
+ Voce) universal Ordo into test/fixtures/fiuv-ordo-2025-2026.sexp, one
+ row per day, 2025-11-27..2026-12-31.
+
+ A DIFFERENT format from tools/extract_lms_ordo.ml's own LMS PDFs in
+ every way that matters (see docs/research/ordo/PROVENANCE-ordo-corpus.md):
+ UNIVERSAL (no diocesan variants to exclude at all -- one block per day,
+ not "the first of several"), IN LATIN, using the rubrics' own
+ vocabulary ("Missa pr., Gloria, sine Credo, praef. comm."), and BOTH
+ directions of Gloria/Credo are stated explicitly in the source's own
+ words ("Gloria"/"sine Gloria", "Credo"/"sine Credo") rather than by a
+ printed flag letter. Hand-rolled, no Str/regex (frozen deps) -- the
+ same String.sub/index/split_on_char discipline extract_lms_ordo.ml
+ already uses for a different publisher's layout.
+
+ THE KEY STRUCTURAL FACTS this parser leans on, established during
+ characterisation (see the fixture's own provenance header for the
+ full record):
+
+ 1. A day-start line's own FIRST token is always "<1-2 digit day>."
+ (e.g. "27.", "3."), immediately followed by a colour code (not
+ anchored on below -- colour is captured as raw prose, never parsed
+ into a closed set, because a few real days use a COMPOUND colour
+ ("Viol. in Off., Alb. in Missa." on Holy Thursday) or drop the
+ token to a blank cell entirely (13 December 2026, a rendering
+ quirk) -- neither is needed for anything this task compares).
+
+ 2. Month headers/running-footers name a month EITHER in English with a
+ trailing ROMAN-NUMERAL year ("November MMXXV", the book's own first
+ two months only) OR bare in Latin with no year at all
+ ("Januarius", "Februarius", ... every month after). The English
+ Nov/Dec spellings happen to be IDENTICAL to their Latin
+ equivalents, so one 12-entry table covers both vocabularies.
+ CHARACTERISATION FINDING: the SECOND header ("December MMXV") is a
+ publisher-side TYPO -- MMXV is 2015, ten years off, contradicted by
+ every neighbouring date and by month-order inference alone. This
+ parser therefore reads a Roman-numeral year ONLY off the very
+ FIRST header encountered (the anchor) and tracks every subsequent
+ month purely by wraparound inference (increment the year exactly
+ once, at the Dec->Jan transition) -- the typo is never read at all,
+ not merely tolerated.
+
+ 3. Every real Mass rubric line starts with the literal token "Missa"
+ (401 of ~400 real day-blocks, confirmed structurally). A SECOND,
+ alternate Mass option, when the day offers one, is introduced by
+ the CAPITALISED two-token sequence "Vel Missa" -- never bare lower-
+ case "vel" alone, which is the ordinary Latin conjunction "or" and
+ appears constantly inside ordinary prose (e.g. "praef. comm. vel de
+ Martyribus", part of ONE preface's own text, not a second Mass
+ option) -- so only "Vel Missa", both tokens, anchors a real
+ alternate-option boundary. This parser reads Gloria/Credo/praef.
+ from the FIRST Mass option only, bounded above by whichever comes
+ first: "Vel Missa" or "VESPERAE"/"VESPERA" -- the identical
+ "day's own PRIMARY office, not a menu entry" discipline
+ extract_lms_ordo.ml already uses for the LMS diocesan-variant
+ colon boundary.
+
+ 4. SUBSTRING TRAP, found and defended against: "Gloria" is also the
+ first word of "Gloria Patri" (the psalm doxology, "Glory be to the
+ Father"), a DIFFERENT liturgical unit that has nothing to do with
+ whether the Mass's own Gloria in excelsis is said -- and it can
+ appear INSIDE the very Mass clause being scanned (Good Friday:
+ "Missa pr., (omittuntur ps. Iudica me et Gloria Patri), Gloria,
+ sine Credo, praef. comm." -- the real, standalone "Gloria," follows
+ immediately after). [find_word] below rejects any "Gloria" hit
+ whose very next token is "Patri", the same whole-token discipline
+ extract_lms_ordo.ml's own header describes for "V Mass of BVM"
+ being a substring of "IV Mass of BVM".
+
+ 5. A WHOLE-SECOND-OFFICE TRAP, found and defended against
+ (celebrant-rubrics-phase1 Phase 2 correction, 2026-08-22, closing
+ data/ef/expected-divergences-fiuv.sexp's own F3): six real days
+ print BOTH a strict-1962 reading (a privileged Lenten/Passiontide
+ feria, saint reduced to a commemoration, "Off. feriale") AND, after
+ a literal separator token, a SECOND, complete alternative office
+ (the same saint kept outright under the 2020 decree Cum
+ Sanctissima, "Off. ordinarium", with its OWN "Ad Mat. ... Te Deum"
+ lessons). The separator is an en-dash, "VEL" (Latin "or"), another
+ en-dash, glued by pdftotext into ONE token with no internal spaces
+ -- "\xe2\x80\x93VEL\xe2\x80\x93" ("VEL" wrapped in en-dashes) --
+ confirmed by direct byte inspection of the raw dump, NOT the same
+ shape as [extract_missa_fields]'s own "Vel"+"Missa" two-token
+ boundary above (a narrower "choice of Mass formulary within the
+ SAME office" marker, e.g. a Common vs a Proper Mass for one saint).
+ [extract_missa_fields]'s own Gloria/Credo/praef span happened to
+ stay correctly bounded to the PRIMARY office on all six real
+ instances even before this fix, because a Vespers line ("VESPER..."
+ -- already one of its own boundary tokens) always intervenes before
+ the separator in this corpus; [extract_te_deum] had NO such
+ protection at all -- it scans the WHOLE block's own text for "Ad
+ Mat." unbounded, so on these six days (whose PRIMARY office states
+ no "Ad Mat." of its own at all -- an ordinary privileged feria's
+ Matins needs no special note) it read the SECOND office's own "Ad
+ Mat. ... Te Deum." instead, manufacturing a same-day contradiction
+ that F3 first recorded as "colitur's own possible gap". [split_at_alt_office]
+ below cuts every block's own token array at this separator BEFORE
+ any field extraction runs, so every extractor now reads the PRIMARY
+ office only; the alternative is captured, not discarded, in its own
+ [alt_te_deum] field (the row type's own comment has the full
+ account of why only this one field, not a parallel alt_gloria/
+ alt_credo/alt_class -- those three were never wrong, so widening
+ them now would be undirected scope creep, not a fix).
+
+ Usage:
+ pdftotext -layout docs/research/ordo/fiuv-ordo-2025-2026.pdf /tmp/fiuv.txt
+ dune exec tools/extract_fiuv_ordo.exe -- /tmp/fiuv.txt \
+ docs/research/ordo/fiuv-ordo-2025-2026.pdf \
+ test/fixtures/fiuv-ordo-2025-2026.sexp *)
+
+open Sexplib0.Sexp_conv
+module Date = Colitur_kernel.Date
+
+let die fmt = Printf.ksprintf (fun s -> prerr_endline ("extract_fiuv_ordo: " ^ s); exit 1) fmt
+
+(* Mirrored, not shared, by test/test_fiuv_ordo.ml -- tools/ and test/
+ have no common .mli either could hang a shared type from, the same
+ reasoning extract_lms_ordo.ml's own header already gives. *)
+type row = {
+ date : string; (** ISO-8601 *)
+ class_ : string option; (** raw, e.g. "III cl.", "III cl. (Priv.)", "I cl." -- [None] only if genuinely unparseable, read off the PRIMARY office alone *)
+ title : string; (** the day-start line's own text before the class marker, PRIMARY office alone *)
+ te_deum : bool option; (** [None] only if neither "Te Deum" nor "non dicitur Te Deum"/"sine Te Deum" is found WITHIN THE PRIMARY OFFICE (see [split_at_alt_office]) *)
+ gloria : bool option; (** PRIMARY office *)
+ credo : bool option; (** PRIMARY office *)
+ praef : string option; (** raw trailing text after "praef." within the primary Mass clause -- CAPTURED, NOT VALIDATED *)
+ alt_te_deum : bool option;
+ (** the SECOND, Cum-Sanctissima-alternative office's own Te Deum
+ status, when this day prints one (see [split_at_alt_office]'s
+ own citation) -- [None] both when the day has no such
+ alternative at all (394 of 400 real days) AND when it has one
+ but that alternative's own text doesn't state Te Deum either
+ (never observed live: on all six real instances the alternative
+ states it unnegated). Captured rather than discarded because a
+ FUTURE Cum-Sanctissima overlay (CLAUDE.md's own binding
+ decision 2 -- an addition, never core) would have exactly this
+ shape to compare against; not compared by anything in this
+ project yet. Deliberately NOT joined by an [alt_gloria]/
+ [alt_credo]/[alt_class] -- see [split_at_alt_office]'s own
+ citation for why only this one field was ever wrong. *)
+}
+[@@deriving sexp]
+
+(* The literal separator between a day's PRIMARY (strict 1962) office and
+ a printed SECOND, Cum-Sanctissima-2020 alternative -- see the module
+ header's own point 5 for the full account of the bug this closes
+ (data/ef/expected-divergences-fiuv.sexp's own F3) and why it is a
+ DIFFERENT token from [extract_missa_fields]'s own "Vel"+"Missa"
+ two-token boundary. Exactly 6 occurrences in the whole corpus,
+ confirmed by direct grep of the raw pdftotext dump -- all 6 on the
+ exact dates F3 named (2026-03-06/07/09/12/21/24). *)
+let alt_office_marker = "\xe2\x80\x93VEL\xe2\x80\x93"
+
+let split_at_alt_office tokens =
+ let n = Array.length tokens in
+ let rec go i = if i >= n then None else if tokens.(i) = alt_office_marker then Some i else go (i + 1) in
+ match go 0 with
+ | None -> (tokens, None)
+ | Some i -> (Array.sub tokens 0 i, Some (Array.sub tokens (i + 1) (n - i - 1)))
+
+(* --- tiny hand-rolled helpers, no Str/regex ------------------------------ *)
+
+let split_ws s =
+ String.split_on_char ' ' (String.map (fun c -> if c = '\t' then ' ' else c) s)
+ |> List.filter (fun t -> t <> "")
+
+let is_all_digits s = s <> "" && String.for_all (fun c -> c >= '0' && c <= '9') s
+
+let has_prefix ~prefix s =
+ String.length s >= String.length prefix && String.sub s 0 (String.length prefix) = prefix
+
+(* Strips ASCII trailing punctuation only (.,();) -- tokens in this source
+ never carry other trailing marks that matter to comparisons below. *)
+let rec strip_trailing_punct s =
+ let n = String.length s in
+ if n = 0 then s
+ else
+ match s.[n - 1] with
+ | '.' | ',' | ')' | ';' | ':' -> strip_trailing_punct (String.sub s 0 (n - 1))
+ | _ -> s
+
+let contains s ~sub =
+ let ls = String.length s and lu = String.length sub in
+ if lu = 0 then true
+ else
+ let rec go i = if i + lu > ls then false else if String.sub s i lu = sub then true else go (i + 1) in
+ go 0
+
+let index_of s ~sub =
+ let ls = String.length s and lu = String.length sub in
+ if lu = 0 then Some 0
+ else
+ let rec go i = if i + lu > ls then None else if String.sub s i lu = sub then Some i else go (i + 1) in
+ go 0
+
+let months =
+ [| "Januarius"; "Februarius"; "Martius"; "Aprilis"; "Maius"; "Junius"; "Julius"; "Augustus"; "September";
+ "October"; "November"; "December" |]
+
+let month_index name =
+ let rec go i = if i >= Array.length months then None else if months.(i) = name then Some (i + 1) else go (i + 1) in
+ go 0
+
+(* Roman numerals, subtractive form -- only ever called once, on the
+ FIRST month header's own year token (the anchor); see the module
+ header on why every LATER header's own year, when present at all, is
+ deliberately never read. *)
+let roman_value = function
+ | 'I' -> Some 1 | 'V' -> Some 5 | 'X' -> Some 10 | 'L' -> Some 50 | 'C' -> Some 100 | 'D' -> Some 500
+ | 'M' -> Some 1000 | _ -> None
+
+let parse_roman s =
+ let n = String.length s in
+ let rec go i acc =
+ if i >= n then Some acc
+ else
+ match roman_value s.[i] with
+ | None -> None
+ | Some v -> (
+ let next = if i + 1 < n then roman_value s.[i + 1] else None in
+ match next with
+ | Some nv when nv > v -> go (i + 2) (acc + nv - v)
+ | _ -> go (i + 1) (acc + v))
+ in
+ if n = 0 then None else go 0 0
+
+(* --- day-block scanning --------------------------------------------------- *)
+
+type block = { b_day : int; b_month : int; b_year : int; b_lines : string list }
+
+let is_day_start_line toks =
+ match toks with
+ | t :: _ :: _ when String.length t >= 2 && t.[String.length t - 1] = '.' ->
+ let core = String.sub t 0 (String.length t - 1) in
+ if is_all_digits core then
+ let d = int_of_string core in
+ if d >= 1 && d <= 31 then Some d else None
+ else None
+ | _ -> None
+
+(* A month running-header/footer line: first token is a recognised month
+ name (English Nov/Dec spelled identically to Latin), optionally
+ followed by a Roman-numeral year and/or a page number -- ALWAYS pure
+ noise for the day-block scanner (never a continuation line's own
+ content), but the FIRST time a given month index is seen it also
+ drives the [cur_month]/[cur_year] state transition. *)
+let is_month_marker_line toks = match toks with m :: _ -> month_index m <> None | [] -> false
+
+(* Left-margin indulgence-notation codes ("Ind." = an indulgence is
+ available, "Plen." = a plenary one, "DFP" unexplained but structurally
+ identical -- all three print in the SAME left-margin column pdftotext
+ -layout preserves, glued onto the following content line as literal
+ leading tokens, the identical trap extract_lms_ordo.ml's own
+ [strip_left_column] already documents for the LMS PDFs' "Pl"/"Ind"
+ column. Found by tracing a real parse failure (8 December 2025's own
+ class marker split across two physical lines by "DFP" sitting between
+ "I" and "cl."), not assumed in advance: catalogued via a frequency
+ scan of every short capitalised token opening a continuation line, and
+ deliberately does NOT include "RM" (11 occurrences, ALSO short and
+ capitalised) -- checked directly, "RM" is a real citation abbreviation
+ ("Rubricae Missalis", e.g. "vide RM 440"), not a margin code, so
+ stripping it would corrupt real text for no parsing benefit. *)
+let strip_left_column tokens =
+ let is_marker t = t = "Ind." || t = "Plen." || t = "DFP" in
+ let rec go = function t :: rest when is_marker t -> go rest | ts -> ts in
+ go tokens
+
+(* The malformed trailing entry's own distinctive two-token prefix,
+ "1st Jan" -- NOT a generic "first token is a bare 4-digit number"
+ check (extract_lms_ordo.ml's own approach): that generic shape false-
+ positived here, on real body content deep in October ("...20 augusti
+ / 1885, 26 augusti 1886..." -- an indulgence-decree date citation
+ whose line-wrap happens to put a 4-digit year token first on its own
+ physical line). This source's OWN tail artefact is reliably
+ distinguished only by its literal, unique "1st Jan" opening -- found
+ by tracing the false stop, not assumed in advance. *)
+let is_stop_tail_line toks = match toks with "1st" :: "Jan" :: _ -> true | _ -> false
+
+let read_lines path =
+ let ic = open_in path in
+ let rec loop acc = match input_line ic with l -> loop (l :: acc) | exception End_of_file -> List.rev acc in
+ let ls = loop [] in
+ close_in ic;
+ ls
+
+(* --- per-block field extraction ------------------------------------------- *)
+
+(* First WHOLE-TOKEN hit of [word] in [tokens] starting at-or-after
+ [from], REJECTING a hit whose immediately-following token (stripped)
+ is [reject_next] -- the "Gloria Patri" trap, see the module header.
+ Returns the hit's own index and whether the token immediately BEFORE
+ it (stripped) is "sine" (a negation). *)
+let find_word tokens ~from ~word ~reject_next =
+ let n = Array.length tokens in
+ let rec go i =
+ if i >= n then None
+ else if strip_trailing_punct tokens.(i) = word then
+ let next_is_rejected = i + 1 < n && strip_trailing_punct tokens.(i + 1) = reject_next in
+ if next_is_rejected then go (i + 1)
+ else
+ let negated = i > 0 && strip_trailing_punct tokens.(i - 1) = "sine" in
+ Some (i, negated)
+ else go (i + 1)
+ in
+ go from
+
+(* [class_]: the roman-numeral token immediately followed by a token
+ starting "cl" (covers "cl.", "cl.,"), whole-token matched -- never a
+ substring search (the same discipline the LMS extractor's own BVM
+ numeral anchor uses, for the identical reason: "I" is a substring of
+ "II"/"III"/"IV"). Also detects the trailing "(Priv.)" two tokens later. *)
+let extract_class tokens =
+ let n = Array.length tokens in
+ let is_class_numeral t = t = "I" || t = "II" || t = "III" || t = "IV" in
+ let rec go i =
+ if i + 1 >= n then None
+ else if is_class_numeral tokens.(i) && has_prefix ~prefix:"cl" tokens.(i + 1) then
+ let base = tokens.(i) ^ " cl." in
+ if i + 2 < n && strip_trailing_punct tokens.(i + 2) = "(Priv" then Some (base ^ " (Priv.)") else Some base
+ else go (i + 1)
+ in
+ go 0
+
+let extract_title tokens =
+ let n = Array.length tokens in
+ let is_class_numeral t = t = "I" || t = "II" || t = "III" || t = "IV" in
+ let rec find i = if i + 1 >= n then n else if is_class_numeral tokens.(i) && has_prefix ~prefix:"cl" tokens.(i + 1) then i else find (i + 1) in
+ let stop = find 0 in
+ String.concat " " (Array.to_list (Array.sub tokens 0 stop))
+
+(* Te Deum: bounded to the "Ad Mat." .. "Ad Laudes" span specifically
+ (both casings occur: "Ad MAT."/"Ad Mat."/"ad Mat.", "Ad LAUDES"/
+ "Ad Laudes") -- NOT the whole block, because a day can separately
+ mention "Te Deum" in an unrelated INDULGENCE note (31 December: "Hodie
+ ad solemnem recitationem hymni Te Deum, indulgentia plenaria lucrari
+ potest" -- about gaining an indulgence for the New Year's Eve Te Deum
+ of thanksgiving, not about whether it is sung at that day's own
+ Matins) that a whole-block search would wrongly read as a positive hit. *)
+let extract_te_deum full_text =
+ let find_ci needles start =
+ List.fold_left
+ (fun acc needle ->
+ match acc with
+ | Some _ -> acc
+ | None -> (
+ match index_of (String.sub full_text start (String.length full_text - start)) ~sub:needle with
+ | Some i -> Some (start + i)
+ | None -> None))
+ None needles
+ in
+ match find_ci [ "Ad Mat."; "Ad MAT."; "ad Mat." ] 0 with
+ | None -> None
+ | Some mat_start -> (
+ let laudes_start =
+ match find_ci [ "Ad Laudes"; "Ad LAUDES"; "ad Laudes" ] mat_start with
+ | Some i -> i
+ | None -> String.length full_text
+ in
+ let span = String.sub full_text mat_start (laudes_start - mat_start) in
+ (* TWO negative phrasings the source actually uses, found live
+ (celebrant-rubrics-phase1 Phase 2, 2026-08-22): "non dicitur Te
+ Deum" (e.g. ordinary Time-after-Pentecost ferias, 27/28 November)
+ AND, separately, "sine Te Deum" (e.g. every Sunday, every Ember
+ day, every privileged Lenten/Passiontide feria carrying a
+ commemoration -- Advent I, 30 November: "...3a de homilia (cum
+ suo R), sine Te Deum."). The ORIGINAL version of this function
+ checked only the first phrasing, so "sine Te Deum" fell through
+ to the bare "Te Deum" substring test and was wrongly read as a
+ POSITIVE hit -- confirmed by grepping the raw pdftotext dump
+ directly against a first, uncorrected run's own output: EVERY
+ date this bug affected showed "sine Te Deum" in the source and
+ [Some true] in the fixture, a 100% correlation, not a handful of
+ coincidences. This is the SAME "does the source negate the
+ hymn's own name with a DIFFERENT word than the one this parser
+ already checks for" shape [find_word]'s own "Gloria Patri"
+ substring trap already documents for Gloria -- this trap simply
+ went unnoticed until Phase 2 actually compared the extracted
+ values against colitur's own output and against the raw text by
+ hand, rather than only checking coverage counts. *)
+ if contains span ~sub:"non dicitur Te Deum" then Some false
+ else if contains span ~sub:"sine Te Deum" then Some false
+ else if contains span ~sub:"Te Deum" then Some true
+ else None)
+
+(* Gloria/Credo/praef.: bounded to the FIRST Mass option only -- from the
+ token "Missa" to whichever comes first: "Vel" immediately followed by
+ "Missa" (a real second option), or a token starting "VESPER" (Vespers
+ info -- a PREFIX match, not one or two hardcoded exact spellings,
+ found necessary by tracing a real over-capture: this source uses at
+ least four surface forms depending on grammatical case and whether
+ pdftotext renders the Æ ligature as one glyph or splits it --
+ "VESPERÆ", "VESPERAS", "VESPERA" (Ad VESPERA, rare) and "VESPERÆ"
+ with the ligature reproduced as a single non-ASCII codepoint an exact-
+ string match against the two ASCII spellings alone could never catch,
+ which let a whole day's own trailing Vespers/Compline prose leak into
+ [praef] uncaught until checked against the task brief's own worked
+ example day, 19 September), or the end of the block. *)
+let extract_missa_fields tokens =
+ let n = Array.length tokens in
+ (* [Missa] (singular, 549 of the corpus's own occurrences of any "Miss-"
+ word) anchors 398 of the 400 real days; the two genuine exceptions
+ are Good Friday and Holy Saturday, which have no Mass at all in the
+ 1955-restored Holy Week (matching extract_lms_ordo.ml's own
+ identically-shaped finding for the LMS corpus). CHRISTMAS DAY is a
+ THIRD, found by tracing a real false-None result: its own rubric
+ reads "Hodie celebrantur tres Missae pr., Gloria, Credo..." ("today
+ three Masses are celebrated..."), using the PLURAL "Missae"/"Missæ"
+ (both spellings occur across the fixture's two Christmas Days) --
+ [tokens.(i) = "Missa"] alone never matches it, silently returning
+ [None] for Gloria/Credo/praef on the single most doctrinally
+ unambiguous day in the whole calendar. Widened to accept "Missae"/
+ "Missæ" too, guarded against the ONE real collision risk: "Hodie
+ prohibentur omnes Missae defunctorum..." (a Requiem-Mass-prohibition
+ notice, common at the end of many day-blocks, unrelated to the
+ day's own Mass) also contains "Missae" -- rejected here by checking
+ the immediately FOLLOWING token is not "defunctorum". *)
+ let missa_idx =
+ let rec go i =
+ if i >= n then None
+ else if (tokens.(i) = "Missa" || tokens.(i) = "Missae" || tokens.(i) = "Missæ")
+ && not (i + 1 < n && strip_trailing_punct tokens.(i + 1) = "defunctorum")
+ then Some i
+ else go (i + 1)
+ in
+ go 0
+ in
+ match missa_idx with
+ | None -> (None, None, None)
+ | Some m ->
+ let end_idx =
+ let rec go i =
+ if i >= n then n
+ else if tokens.(i) = "Vel" && i + 1 < n && tokens.(i + 1) = "Missa" then i
+ (* A THIRD, narrower alternate-Mass marker, found while auditing
+ for the "-VEL-" trap above (celebrant-rubrics-phase1 Phase 2
+ correction, 2026-08-22): "Vel (sec. decretum <<Cum sanct.>>)
+ Missa ..." offers an alternate MASS FORMULARY under the 2020
+ decree Cum Sanctissima WITHIN the same single printed office
+ (a narrower thing than [split_at_alt_office]'s own full
+ second-office split above -- there is no second "Ad Mat." set
+ of lessons here, just a second Mass choice). Checked against
+ the raw dump directly: 20 occurrences, ALL 20 of the
+ capitalised "Vel" + open-paren shape, zero false positives
+ (the ordinary lower-case "vel" conjunction, e.g. "praef.
+ comm. vel de Martyribus", never starts with an open-paren
+ token, so this check never fires on it). Every one of the 20
+ real days already restates Gloria/Credo explicitly BEFORE
+ reaching this marker, so this was NEVER a live divergence
+ (confirmed by a direct diff of every field this fixture
+ compares, before and after this hardening: zero rows
+ changed) -- added defensively, so a FUTURE year whose
+ primary Mass clause happens not to restate Gloria/Credo
+ before this marker cannot silently read the Cum-Sanctissima
+ alternative's own value instead, the identical failure shape
+ as the "-VEL-" bug above, just not yet observed live here. *)
+ else if tokens.(i) = "Vel" && i + 1 < n && has_prefix ~prefix:"(" tokens.(i + 1) then i
+ else if has_prefix ~prefix:"VESPER" tokens.(i) then i
+ else go (i + 1)
+ in
+ go (m + 1)
+ in
+ let span = Array.sub tokens m (end_idx - m) in
+ let gloria =
+ match find_word span ~from:0 ~word:"Gloria" ~reject_next:"Patri" with
+ | Some (_, negated) -> Some (not negated)
+ | None -> None
+ in
+ let credo =
+ match find_word span ~from:0 ~word:"Credo" ~reject_next:"__never__" with
+ | Some (_, negated) -> Some (not negated)
+ | None -> None
+ in
+ let praef =
+ let sn = Array.length span in
+ let rec go i = if i >= sn then None else if strip_trailing_punct span.(i) = "praef" then Some i else go (i + 1) in
+ match go 0 with
+ | None -> None
+ | Some i ->
+ let rest = Array.to_list (Array.sub span (i + 1) (sn - i - 1)) in
+ if rest = [] then None else Some (String.concat " " rest)
+ in
+ (gloria, credo, praef)
+
+(* --- top level -------------------------------------------------------------- *)
+
+let () =
+ if Array.length Sys.argv <> 4 then
+ die "usage: extract_fiuv_ordo <pdftotext-layout.txt> <source.pdf> <dest.sexp>";
+ let txt_path = Sys.argv.(1) in
+ let pdf_path = Sys.argv.(2) in
+ let dest = Sys.argv.(3) in
+ let all_lines = read_lines txt_path in
+ let arr = Array.of_list all_lines in
+ let n = Array.length arr in
+ let trimmed i = String.trim arr.(i) in
+ (* Anchor: the FIRST month-marker line carrying a parseable Roman year
+ -- established during characterisation to be line 47, "November
+ MMXXV". Search generically rather than hardcoding the line number,
+ so a re-extraction against a re-flowed pdftotext dump still finds it. *)
+ let anchor_idx, anchor_month, anchor_year =
+ let rec go i =
+ if i >= n then die "no month header with a parseable Roman-numeral year found (expected the book's own first header, e.g. \"November MMXXV\")"
+ else
+ let toks = split_ws (trimmed i) in
+ match toks with
+ | m :: y :: _ -> (
+ match (month_index m, parse_roman y) with
+ | Some mi, Some yr when yr > 1000 -> (i, mi, yr)
+ | _ -> go (i + 1))
+ | _ -> go (i + 1)
+ in
+ go 0
+ in
+ let stop_idx =
+ let rec go i = if i >= n then n else if is_stop_tail_line (split_ws (trimmed i)) then i else go (i + 1) in
+ go (anchor_idx + 1)
+ in
+ Printf.printf "extract_fiuv_ordo: Ordo body lines %d..%d (of %d total), anchor %d/%d\n" anchor_idx stop_idx n
+ anchor_month anchor_year;
+ let cur_month = ref anchor_month and cur_year = ref anchor_year in
+ let blocks = ref [] in
+ let cur = ref None in
+ let flush () = match !cur with Some b -> blocks := b :: !blocks; cur := None | None -> () in
+ for i = anchor_idx + 1 to stop_idx - 1 do
+ let raw = trimmed i in
+ if raw = "" then ()
+ else
+ let toks = split_ws raw in
+ if is_month_marker_line toks then (
+ match toks with
+ | m :: _ -> (
+ match month_index m with
+ | Some mi ->
+ if mi <> !cur_month then begin
+ if mi < !cur_month then incr cur_year;
+ cur_month := mi
+ end
+ | None -> ())
+ | [] -> ())
+ else
+ match is_day_start_line toks with
+ | Some day ->
+ flush ();
+ cur := Some { b_day = day; b_month = !cur_month; b_year = !cur_year; b_lines = toks }
+ | None -> (
+ match !cur with
+ | None -> ()
+ | Some b -> cur := Some { b with b_lines = b.b_lines @ strip_left_column toks })
+ done;
+ flush ();
+ let blocks = List.rev !blocks in
+ Printf.printf "extract_fiuv_ordo: %d day-blocks parsed\n" (List.length blocks);
+ let rows =
+ List.map
+ (fun b ->
+ let date =
+ match Date.make ~year:b.b_year ~month:b.b_month ~day:b.b_day with
+ | Ok d -> d
+ | Error e -> die "%d/%d/%d: %s" b.b_year b.b_month b.b_day e
+ in
+ let tokens = Array.of_list b.b_lines in
+ (* [b_lines] tokens 0 is the day's own leading "<n>." marker --
+ dropped before field extraction, kept for nothing (the day
+ number is already known from [b_day]). *)
+ let content = if Array.length tokens > 0 then Array.sub tokens 1 (Array.length tokens - 1) else tokens in
+ (* Cut at the "–VEL–" alternative-office boundary, if this day has
+ one, BEFORE any field extraction -- see [split_at_alt_office]'s
+ own citation. Every field below now reads the PRIMARY office
+ only; [alt_te_deum] alone also reads the second, when present. *)
+ let primary, alt = split_at_alt_office content in
+ let class_ = extract_class primary in
+ let title = extract_title primary in
+ let full_text = String.concat " " (Array.to_list primary) in
+ let te_deum = extract_te_deum full_text in
+ let alt_te_deum =
+ match alt with None -> None | Some alt_tokens -> extract_te_deum (String.concat " " (Array.to_list alt_tokens))
+ in
+ let gloria, credo, praef = extract_missa_fields primary in
+ { date = Date.to_iso8601 date; class_; title; te_deum; gloria; credo; praef; alt_te_deum })
+ blocks
+ in
+ let missing_class = List.filter (fun r -> r.class_ = None) rows in
+ let missing_gloria = List.filter (fun r -> r.gloria = None) rows in
+ let missing_credo = List.filter (fun r -> r.credo = None) rows in
+ let missing_te_deum = List.filter (fun r -> r.te_deum = None) rows in
+ if missing_class <> [] then
+ Printf.printf "extract_fiuv_ordo: WARNING %d rows with no class found: %s\n" (List.length missing_class)
+ (String.concat ", " (List.map (fun r -> r.date) missing_class));
+ Printf.printf
+ "extract_fiuv_ordo: %d rows; %d with no Gloria found; %d with no Credo found; %d with no Te Deum found\n"
+ (List.length rows) (List.length missing_gloria) (List.length missing_credo) (List.length missing_te_deum);
+ (* CHARACTERISATION, Step C of the task brief, non-negotiable: probed
+ BEFORE trusting a single row of this fixture, and RE-VERIFIED here
+ against this run's own [rows] -- a failed probe is a hard [die], the
+ same self-verifying discipline extract_lms_ordo.ml already uses,
+ never a printed claim resting on a one-off manual check that could
+ silently rot on a re-extraction. Every date/expectation pair below
+ was confirmed directly against the raw pdftotext dump before being
+ encoded here. *)
+ let creed_of_date d = List.find_map (fun r -> if String.equal r.date d then r.credo else None) rows in
+ let probes =
+ [ ("2025-11-30", true, "Advent Sunday, I class, RG 475(a)");
+ ("2025-12-01", false, "ordinary Advent feria, III class, RG 476(b)/(d)");
+ ("2025-12-26", true, "St Stephen, II class, inside the Nativity octave, RG 475(d)");
+ ("2026-08-10", false, "St Laurence, II class martyr, no Lord/BVM/apostle clause applies, RG 476(b)");
+ ("2026-08-15", true, "the Assumption, I class, RG 475(b)") ]
+ in
+ List.iter
+ (fun (d, expected, desc) ->
+ match creed_of_date d with
+ | Some got when Bool.equal got expected -> ()
+ | Some got -> die "characterisation probe FAILED: %s (%s) -- expected creed=%b, extracted creed=%b" d desc expected got
+ | None -> die "characterisation probe FAILED: %s (%s) -- no row extracted for this date at all" d desc)
+ probes;
+ Printf.printf "extract_fiuv_ordo: all %d characterisation probes confirmed\n" (List.length probes);
+ (* Second characterisation probe, same discipline: the "–VEL–"
+ alternative-office split (module header point 5) must fire on
+ EXACTLY these six dates, no more, no fewer -- a hard [die], not a
+ printed claim, so a future re-extraction against a re-flowed dump
+ silently gaining or losing an instance fails loudly rather than
+ quietly changing F3's own population. *)
+ let alt_office_dates = List.filter_map (fun r -> if r.alt_te_deum <> None then Some r.date else None) rows in
+ let expected_alt_office_dates =
+ [ "2026-03-06"; "2026-03-07"; "2026-03-09"; "2026-03-12"; "2026-03-21"; "2026-03-24" ]
+ in
+ if List.sort String.compare alt_office_dates <> expected_alt_office_dates then
+ die "alt-office-split characterisation probe FAILED: expected exactly %s, got %s"
+ (String.concat ", " expected_alt_office_dates)
+ (String.concat ", " (List.sort String.compare alt_office_dates));
+ Printf.printf "extract_fiuv_ordo: alt-office-split characterisation probe confirmed (%d dates)\n"
+ (List.length expected_alt_office_dates);
+ let probe_lines =
+ String.concat "\n"
+ (List.map
+ (fun (d, expected, desc) -> Printf.sprintf "; %s (%s) -> %s" d desc (if expected then "TRUE" else "FALSE"))
+ probes)
+ in
+ let sha256_of_file path =
+ let cmd = Printf.sprintf "sha256sum %s" (Filename.quote path) in
+ let ic = Unix.open_process_in cmd in
+ let line = try input_line ic with End_of_file -> die "sha256sum produced no output for %s" path in
+ (match Unix.close_process_in ic with Unix.WEXITED 0 -> () | _ -> die "sha256sum failed for %s" path);
+ match String.index_opt line ' ' with Some i -> String.sub line 0 i | None -> die "unexpected sha256sum output: %S" line
+ in
+ let today () =
+ let tm = Unix.gmtime (Unix.time ()) in
+ Printf.sprintf "%04d-%02d-%02d" (tm.Unix.tm_year + 1900) (tm.Unix.tm_mon + 1) tm.Unix.tm_mday
+ in
+ let pdf_sha = sha256_of_file pdf_path in
+ let pdf_base = Filename.basename pdf_path in
+ let gloria_true = List.length (List.filter (fun r -> r.gloria = Some true) rows) in
+ let gloria_false = List.length (List.filter (fun r -> r.gloria = Some false) rows) in
+ let credo_true = List.length (List.filter (fun r -> r.credo = Some true) rows) in
+ let credo_false = List.length (List.filter (fun r -> r.credo = Some false) rows) in
+ let te_deum_true = List.length (List.filter (fun r -> r.te_deum = Some true) rows) in
+ let te_deum_false = List.length (List.filter (fun r -> r.te_deum = Some false) rows) in
+ let alt_te_deum_true = List.length (List.filter (fun r -> r.alt_te_deum = Some true) rows) in
+ let alt_te_deum_false = List.length (List.filter (fun r -> r.alt_te_deum = Some false) rows) in
+ let alt_te_deum_none = List.length (List.filter (fun r -> r.alt_te_deum = None) rows) in
+ let header =
+ Printf.sprintf
+ {|; %s -- Witnesses task (2026-08-22-colitur-
+; celebrant-rubrics-phase1)'s SEVENTH validation layer, and the first
+; UNIVERSAL (non-diocesan), SECOND-COMPILER witness: the FIUV (Foederatio
+; Internationalis Una Voce) Ordo, compiled independently of the Latin Mass
+; Society's own three editions (test/test_lms_ordo.ml) -- see
+; docs/research/ordo/PROVENANCE-ordo-corpus.md for the full four-reasons
+; account of why this source is the more valuable of the two newly-
+; acquired ones.
+;
+; Generator: tools/extract_fiuv_ordo.ml -- do not hand-edit; re-run
+; against a fresh pdftotext dump and commit the diff instead.
+;
+; Source: "Ordo Divini Officii recitandi sacrique peragendi secundum
+; antiquam Ritus Romani formam pro anno Domini 2026", Foederatio
+; Internationalis Una Voce, compiled by Joseph Shaw (Praefatio signed
+; "Joseph Shaw, President", Feast of the Nativity of Our Lady [8 Sept]
+; 2025) -- title page and Praefatio, verified directly against the PDF's
+; own extracted text. 2025 is stated as the Federation's own 60th
+; anniversary year.
+; PDF metadata: Creator "TeX", Producer "pdfTeX-1.40.22", CreationDate
+; 2025-11-08, 128 pages.
+; URL: identified via web search as
+; https://lms.org.uk/sites/default/files/u5374/fiuv_ordo_2025-2026_1.1_a5_format.pdf
+; (hosted by the Latin Mass Society's own site alongside its own three
+; editions -- Joseph Shaw is both FIUV President and LMS Chairman, which
+; plausibly also explains why this PDF's own trailing malformed entry
+; (see COVERAGE below) is structurally identical to the LMS PDFs' own,
+; despite the two being compiled by different people); NOT independently
+; re-downloaded and byte-compared in this session -- recorded honestly
+; rather than presented as verified, the same discipline the LMS
+; fixtures' own headers use.
+; The local PDF (docs/research/ordo/%s) is gitignored (CLAUDE.md: "docs/
+; is gitignored"), so this header, not git, is this fixture's only record
+; of where it came from.
+; SHA-256 of docs/research/ordo/%s: %s
+; Extracted (UTC): %s
+; Exact commands:
+; pdftotext -layout docs/research/ordo/%s /tmp/fiuv.txt
+; dune exec tools/extract_fiuv_ordo.exe -- /tmp/fiuv.txt \
+; docs/research/ordo/%s %s
+;
+; COVERAGE: %d day-rows, %s through %s. Starts at the book's own first
+; real content line ("November MMXXV", the FIRST month header found with
+; a parseable Roman-numeral year -- used as this parser's sole date
+; anchor, see the tool's own header on why no LATER header's year is ever
+; read, the second one being a publisher typo, "December MMXV" for 2025).
+; Stops before a malformed trailing entry, structurally the SAME artefact
+; found in all three LMS editions (a stray, mislabelled duplicate of the
+; fixture's own already-extracted 1 January row, headed "1st Jan / 2025."
+; -- see extract_lms_ordo.ml's own COVERAGE section for the fuller
+; account of this shared publisher-side quirk).
+;
+; CHARACTERISATION FINDINGS (non-negotiable per the task brief -- the
+; extraordinaryform.org Ordo silently omitted St Lawrence's vigil on
+; EVERY date it covered, which nearly produced a false corroboration
+; during the RG 33 work; this Ordo was probed with the same discipline
+; before a single divergence was adjudicated):
+;
+; 1. UNIVERSAL, not diocesan: unlike the three LMS editions (which had to
+; exclude diocesan variants from every comparison, quantified in each
+; of their own fixture headers), this Ordo carries ONE block per day,
+; always. There is no [has_diocesan_variant] field in this fixture's
+; own [row] at all -- there is nothing to exclude.
+;
+; 2. OPPOSITE-PREDICTION PROBES against Rite_ef.Rubrics_ef.creed (RG
+; 475-476), RE-CHECKED BY THIS TOOL RUN against its own extracted
+; [rows] before this header was even written -- a failed probe is a
+; hard [die], never a printed claim:
+%s
+; All %d predictions confirmed, both directions -- this source is
+; discriminating, not a constant, on the Creed.
+;
+; 3. THE SUBSTRING TRAP this parser defends against, found live in this
+; corpus: "Gloria" is also the first word of "Gloria Patri" (the psalm
+; doxology), which can appear INSIDE the very Mass clause being
+; scanned (Good Friday, 3 April 2026: "Missa pr., (omittuntur ps.
+; Iudica me et Gloria Patri), Gloria, sine Credo, praef. comm." -- the
+; real, standalone Gloria mention follows immediately after). See
+; [find_word]'s own citation.
+;
+; 4. RAW SUBSTRING COUNTS measured against this session's own pdftotext
+; dump, for comparison against the task brief's own figures (328
+; Gloria / 132 sine Gloria / 119 Credo / 260 sine Credo, 262 Te Deum):
+; "sine Gloria" 132 (EXACT match), "sine Credo" 260 (EXACT match), raw
+; "Te Deum" substring 262 (EXACT match, includes negated occurrences
+; as substring hits -- 42 of them are "non dicitur Te Deum"). Raw
+; "Gloria"/"Credo" substring totals (494/411) do NOT match the brief's
+; implied positive totals (328/119) even after subtracting the 25
+; "Gloria Patri" occurrences (469, still 9 over) -- not chased further
+; here; this extractor's own token-scoped, Missa-clause-bounded counts
+; below are what this fixture actually asserts, not a forced match to
+; the brief's approximate figures.
+;
+; 5. THE ALTERNATIVE-OFFICE TRAP (celebrant-rubrics-phase1 Phase 2
+; correction, 2026-08-22, closing F3): six real days print BOTH a
+; strict-1962 reading and a complete SECOND office (the same saint
+; kept outright under the 2020 decree Cum Sanctissima), separated by
+; a literal "-VEL-" token. [extract_te_deum] used to scan the WHOLE
+; block unbounded, so on these six days -- whose PRIMARY office states
+; no "Ad Mat." of its own -- it silently read the SECOND office's own
+; Te Deum instead, manufacturing a same-day contradiction with colitur
+; that F3 first recorded as "colitur's own possible gap". Every field
+; now reads the PRIMARY office only (module header point 5 has the
+; full account); the alternative is captured, not discarded, in its
+; own [alt_te_deum] field. Confirmed to fire on EXACTLY six dates by a
+; hard characterisation probe (this run's own output, above), not
+; merely asserted here.
+;
+; MEASURED DISTRIBUTION (this extraction, not the brief's figures): Gloria
+; true=%d false=%d unresolved=%d; Credo true=%d false=%d unresolved=%d;
+; Te Deum true=%d false=%d unresolved=%d; alt Te Deum (the six "-VEL-"
+; Cum Sanctissima alternatives, point 5 above) true=%d false=%d none=%d
+; (a day contributes to "none" here either because it has no "-VEL-"
+; alternative at all -- 394 of 400 -- or, for a day that does, because
+; that alternative's own text doesn't state Te Deum either; not
+; distinguished by this one count).
+;
+; FIX HISTORY (fixed text, HISTORICAL counts from the run that found each
+; bug, not re-verified live by every later run -- this section exists so
+; regenerating this fixture from a fresh pdftotext dump, which rewrites
+; this whole header from the template, does not silently erase either
+; finding; earlier this header carried the first entry below as a
+; hand-appended note that a regeneration between then and now had
+; already dropped once):
+;
+; 1. RE-EXTRACTED (celebrant-rubrics-phase1 Phase 2, 2026-08-22): the FIRST
+; extraction's own Te Deum parser recognised only ONE of the source's two
+; negative phrasings ("non dicitur Te Deum") -- "sine Te Deum" (how the
+; source actually negates a SUNDAY's own Te Deum, among others) fell
+; through to a bare "Te Deum" substring match and was wrongly read
+; [true]. Found by hand, comparing the raw pdftotext dump against this
+; fixture's own values directly, not by any coverage check (both counted
+; the same 127 unresolved rows before and after). Fixed in
+; tools/extract_fiuv_ordo.ml's own [extract_te_deum] (see its own
+; citation); this fixture re-extracted from the SAME pdftotext dump
+; (SHA-256 of the source PDF unchanged, above). 24 of 400 rows changed,
+; every one Te Deum true->false, every one independently confirmed
+; against the raw text: true 231->207, false 42->66, unresolved 127
+; unchanged. See lib/rites/rite_ef/rubrics_ef.ml's own [te_deum] header
+; (237(a)'s comment) for the full account of what this corrected, and
+; the .superpowers task report for the complete before/after date list.
+;
+; 2. RE-EXTRACTED (celebrant-rubrics-phase1 Phase 2 correction, 2026-08-22):
+; module header point 5 and CHARACTERISATION point 5 above have the
+; full account of the "-VEL-" alternative-office bug this closes
+; (data/ef/expected-divergences-fiuv.sexp's own F3). This fixture
+; re-extracted from the SAME pdftotext dump (SHA-256 of the source PDF
+; unchanged, above). 6 of 400 rows changed, every one Te Deum
+; true->unresolved (2026-03-06/07/09/12/21/24), every one independently
+; confirmed against the raw text: Te Deum true 207->201 (this run's own
+; unresolved count went 127->133, consistent with those same 6 rows).
+; Gloria/Credo/class_/title/praef are UNCHANGED on all 400 rows --
+; those fields were already correctly bounded to the primary office
+; before this fix (a Vespers-line token always intervened before the
+; "-VEL-" separator on all six real instances), confirmed by a direct
+; diff of the fixture before and after, not merely by argument.
+;
+; 3. RE-EXTRACTED (celebrant-rubrics-phase1 same correction, defensive
+; hardening, 2026-08-22): auditing for the SAME trap elsewhere (the
+; task brief's own item 5) found a THIRD, narrower alternate-Mass
+; marker, "Vel (sec. decretum <<Cum sanct.>>) Missa ...", offering a
+; Cum-Sanctissima Mass FORMULARY choice within a single office
+; (unlike point 5 above, no second "Ad Mat." set of lessons -- a
+; narrower thing than a full alternate office). 20 occurrences in the
+; whole corpus, confirmed by direct grep; on all 20, Gloria/Credo were
+; ALREADY stated before reaching this marker, so this was NEVER a
+; live divergence for those two fields (confirmed by this same diff:
+; 0 Gloria/Credo/class_/title/alt_te_deum rows changed). [praef]
+; alone changed on 20 of 400 rows (its own "rest of the primary
+; clause" capture no longer runs past this marker into the
+; alternative's own text) -- harmless in itself ([praef] is
+; documented CAPTURED, NOT VALIDATED, compared by nothing), but added
+; as a defensive boundary in [extract_missa_fields] so a FUTURE year
+; whose primary clause does not restate Gloria/Credo before this
+; marker cannot silently read the alternative's own value instead --
+; the identical failure shape as point 2 above, closed before it was
+; ever observed live rather than after.
+|}
+ dest pdf_base pdf_base pdf_sha (today ()) pdf_base pdf_base dest (List.length rows)
+ (match rows with r :: _ -> r.date | [] -> "?")
+ (match List.rev rows with r :: _ -> r.date | [] -> "?")
+ probe_lines (List.length probes) gloria_true gloria_false (List.length missing_gloria) credo_true credo_false
+ (List.length missing_credo) te_deum_true te_deum_false (List.length missing_te_deum) alt_te_deum_true
+ alt_te_deum_false alt_te_deum_none
+ in
+ let body = Sexplib.Sexp.to_string_hum ~indent:2 (sexp_of_list sexp_of_row rows) in
+ let oc = open_out dest in
+ Fun.protect
+ ~finally:(fun () -> close_out_noerr oc)
+ (fun () ->
+ output_string oc header;
+ output_string oc body;
+ output_string oc "\n");
+ Printf.printf "extract_fiuv_ordo: wrote %d rows to %s\n" (List.length rows) dest
diff --git a/tools/extract_lms_ordo.ml b/tools/extract_lms_ordo.ml
new file mode 100644
index 0000000..582fb6e
--- /dev/null
+++ b/tools/extract_lms_ordo.ml
@@ -0,0 +1,729 @@
+(* Task 6 (2026-08-21-colitur-celebrant-rubrics-phase1): turns pdftotext's
+ -layout dump of the Latin Mass Society's printed Ordo into
+ test/fixtures/lms-ordo-2024-2025.sexp, one row per UNIVERSAL-calendar
+ day (England & Wales diocesan variants are recorded as PRESENT via
+ [has_diocesan_variant] but their own text is discarded -- see the
+ fixture's own provenance header, and task-6-report.md's "how much of
+ the Ordo's year is comparable" accounting, for why).
+
+ NOT a general Ordo parser: hand-rolled against ONE PDF's own column
+ layout (Author "Peter Day-Milne", Producer "LibreOffice 7.4", pdftotext
+ 1.6), verified line-by-line against the source before being trusted --
+ see task-6-report.md for the full characterisation record (the
+ opposite-prediction probe technique CLAUDE.md's "burned twice" note
+ demands). Frozen deps forbid Str/regex; every match below is
+ String.sub/index/split_on_char, the same discipline tools/
+ bootstrap_sanctoral.ml's own hand-rolled INI reader already uses for
+ someone else's format.
+
+ THE KEY STRUCTURAL FACT this parser leans on, established during
+ characterisation: every day's UNIVERSAL entry (the General Roman
+ Calendar's own text) appears FIRST in reading order, before any
+ diocesan variant block (which is introduced by a line ending in ':',
+ e.g. "Westminster, Clifton, Plymouth:" or "In all Dioceses of ENGLAND
+ and WALES ... follows:"). So the FIRST "Gl"/"Cr" token pair and the
+ FIRST "Mass of ..."/roman-numeral-BVM line found within a day's block
+ are ALWAYS the universal entry's own, even if the colon-boundary
+ detection below is imperfect -- diocesan text can only ever appear
+ AFTER, never before, so it cannot introduce a false FIRST match. This
+ is why [scan_block] restricts its search to lines before the first
+ colon-terminated line (belt), while the ordering argument above is the
+ suspenders.
+
+ Usage:
+ pdftotext -layout docs/research/ordo/lms-ordo-2024-2025.pdf /tmp/lms.txt
+ dune exec tools/extract_lms_ordo.exe -- /tmp/lms.txt \
+ docs/research/ordo/lms-ordo-2024-2025.pdf \
+ test/fixtures/lms-ordo-2024-2025.sexp *)
+
+open Sexplib0.Sexp_conv
+module Date = Colitur_kernel.Date
+
+let die fmt = Printf.ksprintf (fun s -> prerr_endline ("extract_lms_ordo: " ^ s); exit 1) fmt
+
+(* Mirrored, not shared, by test/test_lms_ordo.ml -- tools/ and test/ have
+ no common .mli either could hang a shared type from, the same reasoning
+ test_oracle.ml's own header gives for duplicating [sha256_of_file]. *)
+type row = {
+ date : string; (** ISO-8601 *)
+ weekday : string;
+ (** As printed ("Sat", "Sun", ...) -- cross-checked below against
+ [date]'s own computed weekday at extraction time (fatal on
+ mismatch: the whole point of carrying both is to catch a date-
+ resolution bug here, not paper over one). *)
+ title : string; (** the day's own title line: feast name, class, colour, votive codes *)
+ formulary_override : string option;
+ (** the literal "Mass of ..." text, when the universal block names
+ one; [None] = the day says its own Mass. *)
+ bvm_numeral : string option;
+ (** "I".."V" when [formulary_override] is one of the five anchored
+ "<N> Mass of BVM" lines (whole-line match, never a substring --
+ see the module header on why "V Mass of BVM" must never be
+ matched as a substring of "IV Mass of BVM"). *)
+ gloria : bool option; (** [None] only if genuinely not found -- reported, not silently dropped *)
+ creed : bool option;
+ praef : string option;
+ (** RAW trailing text from the FIRST "Pr of ..." / "Common Pr" match
+ in the universal block's own reading order -- CAPTURED, NOT
+ CLASSIFIED here, the same discipline
+ tools/extract_fiuv_ordo.ml's own [row.praef] documents for a
+ different publisher's format. See the provenance header's own
+ "Pr of" characterisation bullet for what this column can and
+ cannot show: this Ordo prints OPTION LISTS ("Pr of X or Pr of Y
+ or Common Pr"), some of whose named options ("Martyrs", "All
+ Saints and Patron Saints", "the Dedication of a Church", "the
+ Most Holy Sacrament", "St John the Baptist", "the Angels") are
+ NOT among the fourteen the 1962 Missale Romanum's own RG 484-497
+ enumerate -- ad libitum extras this publisher includes alongside
+ a genuine RG 482 answer, never the sole option on any row found
+ in this corpus. Classification into {!Colitur_kernel.Preface.t}
+ happens test-side (test/test_lms_ordo.ml), by MEMBERSHIP in the
+ parsed option set, not string equality -- see that file's own
+ [classify_praef] citation for the full account. *)
+ has_diocesan_variant : bool;
+ (** whether a diocese-specific block followed the universal one
+ anywhere in this day's raw text (informational only -- see the
+ module header on why correctness of the other fields does not
+ depend on this being exact). *)
+}
+[@@deriving sexp]
+
+(* --- tiny hand-rolled helpers, no Str/regex ------------------------------ *)
+
+let split_ws s =
+ String.split_on_char ' ' (String.map (fun c -> if c = '\t' then ' ' else c) s)
+ |> List.filter (fun t -> t <> "")
+
+let is_all_digits s = s <> "" && String.for_all (fun c -> c >= '0' && c <= '9') s
+
+let has_prefix ~prefix s =
+ String.length s >= String.length prefix && String.sub s 0 (String.length prefix) = prefix
+
+let ends_with_colon s = s <> "" && s.[String.length s - 1] = ':'
+
+let months =
+ [| "January"; "February"; "March"; "April"; "May"; "June"; "July"; "August"; "September"; "October";
+ "November"; "December" |]
+
+let month_index name =
+ let rec go i = if i >= Array.length months then None else if months.(i) = name then Some (i + 1) else go (i + 1) in
+ go 0
+
+let weekday_of_abbrev = function
+ | "Sun" -> Some Date.Sun
+ | "Mon" -> Some Date.Mon
+ | "Tue" -> Some Date.Tue
+ | "Wed" -> Some Date.Wed
+ | "Thu" -> Some Date.Thu
+ | "Fri" -> Some Date.Fri
+ | "Sat" -> Some Date.Sat
+ | _ -> None
+
+(* Left-column codes (Abbreviations, "Left column": "+EW"/"+"/"Pl"/"Ind") --
+ printed in a dedicated margin column that pdftotext -layout reproduces
+ as literal leading tokens on an otherwise-ordinary content line. Strip
+ ALL that appear (there can be two, "Pl Ind" together -- the 31 Dec/
+ 1 Jan examples in the source). *)
+let strip_left_column tokens =
+ let is_marker = function "Pl" | "Ind" | "+EW" | "+" -> true | _ -> false in
+ let rec go = function t :: rest when is_marker t -> go rest | ts -> ts in
+ go tokens
+
+let bvm_numeral_strings = [ ("I", "I Mass of BVM"); ("II", "II Mass of BVM"); ("III", "III Mass of BVM");
+ ("IV", "IV Mass of BVM"); ("V", "V Mass of BVM") ]
+
+(* Anchored WHOLE-LINE match against all five, never a substring test --
+ this is the exact discipline the coordinator addendum records: "V Mass
+ of BVM" is a substring of "IV Mass of BVM", so any check here MUST
+ compare the full line, not search for the numeral's own string inside
+ it. *)
+let bvm_numeral_of_line line = List.assoc_opt line (List.map (fun (n, s) -> (s, n)) bvm_numeral_strings)
+
+(* --- the day-block scanner ------------------------------------------------ *)
+
+(* [lines]: this day's own raw continuation lines, marker-stripped and
+ trimmed, title line NOT included (the title never carries Gl/Cr/"Mass
+ of" itself in this source). Returns (formulary_override, bvm_numeral,
+ gloria, creed, praef, has_diocesan_variant), each derived from the
+ portion BEFORE the first colon-terminated (diocesan) line -- see the
+ module header for why this is belt-and-suspenders, not load-bearing on
+ its own. *)
+let scan_block lines =
+ let rec universal_prefix = function
+ | [] -> []
+ | l :: _ when ends_with_colon l -> []
+ | l :: rest -> l :: universal_prefix rest
+ in
+ let has_diocesan = List.exists ends_with_colon lines in
+ let universal = universal_prefix lines in
+ let formulary_override, bvm_numeral =
+ let rec find = function
+ | [] -> (None, None)
+ | l :: rest -> (
+ match bvm_numeral_of_line l with
+ | Some n -> (Some l, Some n)
+ | None -> if has_prefix ~prefix:"Mass of " l then (Some l, None) else find rest)
+ in
+ find universal
+ in
+ let all_tokens = List.concat_map split_ws universal in
+ let rec find_flag word = function
+ | prev :: (cur :: _ as rest) -> if cur = word then Some (prev <> "No") else find_flag word rest
+ | [ cur ] -> if cur = word then Some true else None
+ | [] -> None
+ in
+ (* [find_flag] needs a sentinel predecessor for a match at position 0
+ (never observed in this source -- Gl/Cr always follow at least "No"
+ or a preceding word on the same "Gl Cr Pr of ..." line -- but handled
+ rather than assumed: a bare leading "Gl"/"Cr" with no "No" before it
+ reads as [true], the same as any other non-"No" predecessor would). *)
+ let gloria = find_flag "Gl" ("" :: all_tokens) in
+ let creed = find_flag "Cr" ("" :: all_tokens) in
+ (* [praef]: LINE-SCOPED (not [all_tokens]'s flattened stream), because
+ the captured text runs to the END of whichever physical line carries
+ it -- flattening first would glue an unrelated following line's own
+ prose onto it. "Pr" always abbreviates "Preface" in this source's own
+ body text (checked directly: its only OTHER appearance is the front-
+ matter Abbreviations glossary entry "Pr Preface", outside the Ordo
+ body this tool scans), so a whole-token ("Pr","of") pair, or a bare
+ ("Common","Pr") pair when no proper/de-Tempore option is offered, is
+ the sole anchor needed -- no reject-list, unlike the BVM roman
+ numeral's own substring trap, because nothing else in this corpus's
+ body ever produces either two-token sequence. *)
+ let find_praef_in_line line =
+ let toks = Array.of_list (split_ws line) in
+ let n = Array.length toks in
+ let rec go i =
+ if i + 1 >= n then None
+ else if (toks.(i) = "Pr" && toks.(i + 1) = "of") || (toks.(i) = "Common" && toks.(i + 1) = "Pr") then
+ Some (String.concat " " (Array.to_list (Array.sub toks i (n - i))))
+ else go (i + 1)
+ in
+ go 0
+ in
+ let praef =
+ let rec find = function
+ | [] -> None
+ | l :: rest -> ( match find_praef_in_line l with Some s -> Some s | None -> find rest)
+ in
+ find universal
+ in
+ (formulary_override, bvm_numeral, gloria, creed, praef, has_diocesan)
+
+(* --- the top-level line scan --------------------------------------------- *)
+
+type block = { b_weekday : string; b_day : int; b_month : int; b_year : int; b_title : string; b_lines : string list }
+
+(* Generalised (Witnesses task, 2026-08-22) from a single hardcoded
+ sentinel ["End";"of";"November";"2024"], which assumed every edition
+ opens its Ordo body with an "End of November <year>" stub section. That
+ assumption FAILED characterisation against the two newly-acquired
+ editions: the 2023-2024 PDF has NO such stub -- Advent Sunday 2023 fell
+ on 3 December, so only 2 tail days of the old liturgical year remained
+ (1-2 December), and the compiler folded them directly into the
+ "December 2023" header rather than giving them their own mini-section.
+ So a start marker anchored to "End of November" cannot generalise.
+
+ What DOES generalise, checked directly against all three editions'
+ pdftotext -layout dumps: exactly one line in the whole document begins
+ with the bare, all-uppercase token "ORDO" -- either alone ("ORDO",
+ 2024-2025/2025-2026, introducing an "End of <Month> <Year>" stub next)
+ or fused with the edition's own year range ("ORDO 2023-2024",
+ introducing the first REAL month header directly, no stub). This is
+ NEVER the Table of Contents' own entry, which prints title-case "Ordo
+ 20XX-20XX" followed by dot leaders and a page number -- a different
+ string ("Ordo", not "ORDO"), confirmed by grepping all three raw dumps
+ for both patterns before relying on either. *)
+let is_ordo_marker tokens = match tokens with "ORDO" :: _ -> true | _ -> false
+
+(* The "End of <Month> <Year>" stub heading, when an edition has one --
+ parsed directly (never hardcoded to November/a specific year) so the
+ edition's own initial (month, year) state comes from the text itself,
+ not an assumption baked into the tool. *)
+let parse_end_of_month tokens =
+ match tokens with
+ | [ "End"; "of"; m; y ] when is_all_digits y -> ( match month_index m with Some mi -> Some (mi, int_of_string y) | None -> None)
+ | _ -> None
+
+let is_stop_tail_line tokens =
+ match tokens with tok :: _ -> String.length tok = 4 && is_all_digits tok | [] -> false
+
+let is_month_header tokens =
+ match tokens with
+ | m :: rest when month_index m <> None -> List.for_all is_all_digits rest
+ | _ -> false
+
+let read_lines path =
+ let ic = open_in path in
+ let rec loop acc = match input_line ic with l -> loop (l :: acc) | exception End_of_file -> List.rev acc in
+ let ls = loop [] in
+ close_in ic;
+ ls
+
+let () =
+ if Array.length Sys.argv <> 4 then
+ die "usage: extract_lms_ordo <pdftotext-layout.txt> <source.pdf> <dest.sexp>";
+ let txt_path = Sys.argv.(1) in
+ let pdf_path = Sys.argv.(2) in
+ let dest = Sys.argv.(3) in
+ let all_lines = read_lines txt_path in
+ let arr = Array.of_list all_lines in
+ let n = Array.length arr in
+ let trimmed i = String.trim arr.(i) in
+ let ordo_marker_idx =
+ let rec go i = if i >= n then die "no line begins with the bare \"ORDO\" body marker (see the tool's own comment on [is_ordo_marker] -- this edition's format may have drifted)"
+ else if is_ordo_marker (split_ws (trimmed i)) then i else go (i + 1)
+ in
+ go 0
+ in
+ (* The first non-blank line after the marker is either an "End of <Month>
+ <Year>" stub (consumed here as a pure sentinel, exactly as the old
+ hardcoded design did -- [start_idx] becomes ITS OWN index so the main
+ loop below begins right after it) or a real "<Month> <Year>" header
+ with an explicit year token (an edition with no stub, e.g. 2023-2024 --
+ [start_idx] becomes the ORDO marker's own index instead, so the main
+ loop re-encounters this header line on its very first iteration and
+ [is_month_header]'s own existing per-line handling sets [cur_month]/
+ [cur_year] from it normally). Either way [init_month]/[init_year] come
+ directly from parsed text, never a hardcoded constant -- a missing
+ explicit year on this FIRST header is a genuine format break, so it
+ dies loudly rather than silently seeding a wrong year. *)
+ let rec first_nonblank i =
+ if i >= n then die "no content found after the ORDO marker at line %d" ordo_marker_idx
+ else if trimmed i = "" then first_nonblank (i + 1)
+ else i
+ in
+ let content_idx = first_nonblank (ordo_marker_idx + 1) in
+ let content_toks = split_ws (trimmed content_idx) in
+ let start_idx, init_month, init_year =
+ match parse_end_of_month content_toks with
+ | Some (mi, y) -> (content_idx, mi, y)
+ | None -> (
+ match content_toks with
+ | m :: (y :: _ as rest) when month_index m <> None && List.for_all is_all_digits rest ->
+ (ordo_marker_idx, (match month_index m with Some x -> x | None -> assert false), int_of_string y)
+ | _ ->
+ die "first content after the ORDO marker (%S) is neither an \"End of <Month> <Year>\" stub nor a \"<Month> <Year>\" header carrying an explicit year"
+ (String.concat " " content_toks))
+ in
+ let stop_idx =
+ let rec go i =
+ if i >= n then n else if is_stop_tail_line (split_ws (trimmed i)) then i else go (i + 1)
+ in
+ go (start_idx + 1)
+ in
+ Printf.printf "extract_lms_ordo: Ordo body lines %d..%d (of %d total), initial month/year %d/%d\n" start_idx stop_idx n
+ init_month init_year;
+ (* --- the scan: month/year tracker + block accumulator ------------------ *)
+ let cur_month = ref init_month and cur_year = ref init_year in
+ let blocks = ref [] in
+ let cur = ref None in
+ let flush () = match !cur with Some b -> blocks := b :: !blocks; cur := None | None -> () in
+ for i = start_idx + 1 to stop_idx - 1 do
+ let raw = trimmed i in
+ if raw = "" then ()
+ else if String.length raw >= 1 && (let rec contains s sub =
+ let ls = String.length s and lu = String.length sub in
+ if lu = 0 then true else if ls < lu then false
+ else if String.sub s 0 lu = sub then true
+ else contains (String.sub s 1 (ls - 1)) sub
+ in
+ contains raw "Latin Mass Society Ordo")
+ then () (* page footer *)
+ else
+ let toks = split_ws raw in
+ if is_month_header toks then begin
+ match toks with
+ | m :: rest -> (
+ match month_index m with
+ | Some mi ->
+ if mi <> !cur_month then begin
+ if mi < !cur_month then incr cur_year;
+ cur_month := mi
+ end;
+ (* an explicit 4-digit year token, when present, overrides
+ the transition inference outright -- belt and
+ suspenders, same reasoning as [scan_block] above *)
+ List.iter (fun t -> if String.length t = 4 && is_all_digits t then cur_year := int_of_string t) rest
+ | None -> ())
+ | [] -> ()
+ end
+ else
+ match toks with
+ | wd :: dn :: _ when weekday_of_abbrev wd <> None && is_all_digits dn && int_of_string dn >= 1
+ && int_of_string dn <= 31 ->
+ flush ();
+ let title = String.concat " " (List.tl (List.tl toks)) in
+ cur := Some { b_weekday = wd; b_day = int_of_string dn; b_month = !cur_month; b_year = !cur_year;
+ b_title = title; b_lines = [] }
+ | _ -> (
+ match !cur with
+ | None -> ()
+ | Some b ->
+ let stripped = strip_left_column toks in
+ if stripped = [] then ()
+ else cur := Some { b with b_lines = b.b_lines @ [ String.concat " " stripped ] })
+ done;
+ flush ();
+ let blocks = List.rev !blocks in
+ Printf.printf "extract_lms_ordo: %d day-blocks parsed\n" (List.length blocks);
+ let rows =
+ List.map
+ (fun b ->
+ let date =
+ match Date.make ~year:b.b_year ~month:b.b_month ~day:b.b_day with
+ | Ok d -> d
+ | Error e -> die "%s %d/%d/%d: %s" b.b_weekday b.b_year b.b_month b.b_day e
+ in
+ (match weekday_of_abbrev b.b_weekday with
+ | Some w when w = Date.weekday date -> ()
+ | Some _ ->
+ die "%s: printed weekday %S does not match computed weekday %S -- date resolution bug"
+ (Date.to_iso8601 date) b.b_weekday (Date.weekday_to_string (Date.weekday date))
+ | None -> die "%s: unrecognised weekday abbreviation %S" (Date.to_iso8601 date) b.b_weekday);
+ let formulary_override, bvm_numeral, gloria, creed, praef, has_diocesan_variant = scan_block b.b_lines in
+ { date = Date.to_iso8601 date; weekday = b.b_weekday; title = b.b_title; formulary_override; bvm_numeral;
+ gloria; creed; praef; has_diocesan_variant })
+ blocks
+ in
+ let missing_gloria = List.filter (fun r -> r.gloria = None) rows in
+ let missing_creed = List.filter (fun r -> r.creed = None) rows in
+ let missing_praef = List.filter (fun r -> r.praef = None) rows in
+ if missing_gloria <> [] then
+ Printf.printf "extract_lms_ordo: WARNING %d rows with no Gloria found: %s\n" (List.length missing_gloria)
+ (String.concat ", " (List.map (fun r -> r.date) missing_gloria));
+ if missing_creed <> [] then
+ Printf.printf "extract_lms_ordo: WARNING %d rows with no Creed found: %s\n" (List.length missing_creed)
+ (String.concat ", " (List.map (fun r -> r.date) missing_creed));
+ if missing_praef <> [] then
+ Printf.printf "extract_lms_ordo: WARNING %d rows with no Pr-of/Common-Pr found: %s\n" (List.length missing_praef)
+ (String.concat ", " (List.map (fun r -> r.date) missing_praef));
+ let with_variant = List.length (List.filter (fun r -> r.has_diocesan_variant) rows) in
+ let with_override = List.length (List.filter (fun r -> r.formulary_override <> None) rows) in
+ let with_bvm = List.length (List.filter (fun r -> r.bvm_numeral <> None) rows) in
+ Printf.printf "extract_lms_ordo: %d rows; %d with a diocesan variant present; %d with a formulary override; \
+ %d with a BVM roman numeral; %d with no Pr-of/Common-Pr text\n"
+ (List.length rows) with_variant with_override with_bvm (List.length missing_praef);
+ (* --- provenance header + write ----------------------------------------- *)
+ let sha256_of_file path =
+ let cmd = Printf.sprintf "sha256sum %s" (Filename.quote path) in
+ let ic = Unix.open_process_in cmd in
+ let line = try input_line ic with End_of_file -> die "sha256sum produced no output for %s" path in
+ (match Unix.close_process_in ic with Unix.WEXITED 0 -> () | _ -> die "sha256sum failed for %s" path);
+ match String.index_opt line ' ' with Some i -> String.sub line 0 i | None -> die "unexpected sha256sum output: %S" line
+ in
+ let today () =
+ let tm = Unix.gmtime (Unix.time ()) in
+ Printf.sprintf "%04d-%02d-%02d" (tm.Unix.tm_year + 1900) (tm.Unix.tm_mon + 1) tm.Unix.tm_mday
+ in
+ let pdf_sha = sha256_of_file pdf_path in
+ (* --- per-edition provenance + characterisation, Witnesses task ---------
+ Generalised 2026-08-22 from Task 6's single hand-written header (which
+ named "lms-ordo-2024-2025.sexp" literally throughout). Facts 1/3/4
+ below are STRUCTURAL -- true of the publisher's whole known corpus,
+ re-checked directly against the two new editions' own pdftotext dumps
+ before being asserted here again, not merely copied forward. Fact 2
+ (the Creed opposite-prediction probes) is inherently edition-specific
+ (the actual calendar dates differ year to year, and one edition's own
+ calendar even breaks the standard pairing -- see the 2025-2026 note
+ below), so it is looked up per [dest] and, critically, RE-VERIFIED
+ against this run's own [rows] before being printed: a probe that
+ fails is a [die], not a silently-wrong "confirmed" claim in a fixture
+ header nobody will re-check by eye. *)
+ let base = Filename.basename dest in
+ let pdf_base = Filename.basename pdf_path in
+ let edition_label, isbn, copyright_year, pdf_meta, url, url_note, window_open_desc =
+ match base with
+ | "lms-ordo-2023-2024.sexp" ->
+ ( "2023-2024",
+ "978-1-7392096-2-9",
+ "2023",
+ {|Author "Peter Day-Milne", Producer "LibreOffice 6.4", CreationDate 2023-11-01, 140 pages|},
+ "https://lms.org.uk/sites/default/files/u5374/ordo_2023-2024_2.2_final.pdf",
+ "identified 2026-08-22 via web search matching the PDF's own edition year to the publisher's own hosting path; NOT independently re-downloaded and byte-compared in this session -- recorded honestly rather than presented as verified",
+ {|1 December 2023 (this edition carries NO "End of November" stub section at all -- Advent Sunday 2023 fell on 3 December, so only 2 tail days of the prior liturgical year remained, and the compiler folded them directly into the "December 2023" header instead of giving them their own mini-section; see CHARACTERISATION below)|}
+ )
+ | "lms-ordo-2025-2026.sexp" ->
+ ( "2025-2026",
+ "978-1-7392096-5-0",
+ "2025",
+ {|Author "Peter Day-Milne", Producer "LibreOffice 25.2.6.2 (X86_64) / LibreOffice Community", CreationDate 2025-10-23, 146 pages|},
+ "(not found -- see the note on this line)",
+ "NOT found by URL-pattern web search this session (unlike the 2023-2024 and 2024-2025 editions, no exact download link matching this PDF's own filename/version turned up under lms.org.uk/sites/default/files/u5374/; several near-miss candidates were checked and rejected -- none matched this PDF's own metadata) -- left honestly blank rather than guessed; the file was supplied locally to this session, not fetched by this tool",
+ {|28 November 2025 ("End of November 2025", the same shape as the 2024-2025 edition -- Advent Sunday 2025 fell on 30 November, so only 3 tail days remained; note this excludes 27 November 2025, which the PRECEDING edition's own window (2024-2025, through 2025-12-31) already covers -- a one-day-short overlap between consecutive editions' own windows, not a gap or a bug in either extraction|}
+ )
+ | "lms-ordo-2024-2025.sexp" ->
+ (* Preface-witnesses task (2026-08-23): brought into this same
+ auto-generated, probe-verified header template so all three
+ editions carry the identical characterisation discipline --
+ Task 6's own original HAND-WRITTEN header (never regenerated by
+ the Witnesses task, which only added table entries for the two
+ NEW editions) is superseded by this run; every fact below is
+ re-derived from this PDF's own text/metadata, not copied
+ blind from that older header (though it agrees with it in
+ every particular checked). *)
+ ( "2024-2025",
+ "9781739209636",
+ "2024",
+ {|Author "Peter Day-Milne", Producer "LibreOffice 7.4", CreationDate 2024-11-06, 140 pages|},
+ "https://lms.org.uk/sites/default/files/u5374/ordo_2.21_2024-2025.pdf",
+ "identified via web search matching the PDF's own version number and filename to the publisher's own hosting path (product page: https://lms.org.uk/product/latin-mass-society-ordo-2024-25); NOT independently re-downloaded and byte-compared in this session -- recorded honestly rather than presented as verified",
+ {|27 November 2024 ("End of November 2024", the same shape as the 2025-2026 edition -- Advent Sunday 2024 fell on 1 December, so 4 tail days remained (27-30 November))|}
+ )
+ | _ -> die "no provenance/characterisation entry for %s -- add one to extract_lms_ordo.ml's own per-edition table before generating this fixture (the tool refuses to emit an under-characterised header)" base
+ in
+ let creed_of_date rows d = List.find_map (fun r -> if String.equal r.date d then r.creed else None) rows in
+ let probes =
+ match base with
+ | "lms-ordo-2023-2024.sexp" ->
+ [ ("2023-12-03", true, "Advent Sunday, RG 475(a)");
+ ("2023-12-04", false, "St Peter Chrysologus, III class, ordinary Advent day, RG 476(b)/(d)");
+ ("2023-12-26", true, "St Stephen, II class, inside the Nativity octave, RG 475(d)");
+ ("2024-08-15", true, "the Assumption, I class, RG 475(b)");
+ ("2024-08-16", false, "St Joachim, II class, no Lord/BVM/apostle clause applies, RG 476(b)") ]
+ | "lms-ordo-2025-2026.sexp" ->
+ [ ("2025-11-30", true, "Advent Sunday, RG 475(a)");
+ ("2025-12-01", false, "ordinary Advent feria, RG 476(b)/(d)");
+ ("2025-12-26", true, "St Stephen, II class, inside the Nativity octave, RG 475(d)");
+ ( "2026-08-10", false,
+ "St Laurence, II class martyr, no Lord/BVM/apostle clause applies, RG 476(b) -- St Joachim (16 August) is IMPEDED by a Sunday in this edition's own calendar (12th Sunday after Pentecost), so this edition needed a DIFFERENT sharp non-octave II-class pairing than the 2023-2024/2024-2025 editions used; a real, edition-specific finding, not an extraction quirk"
+ );
+ ("2026-08-15", true, "the Assumption, I class, RG 475(b)") ]
+ | "lms-ordo-2024-2025.sexp" ->
+ [ ("2024-12-01", true, "Advent Sunday, RG 475(a)");
+ ("2024-12-02", false, "ordinary Advent feria, RG 476(b)/(d)");
+ ("2024-12-26", true, "St Stephen, II class, inside the Nativity octave, RG 475(d)");
+ ("2025-08-15", true, "the Assumption, I class, RG 475(b)");
+ ("2025-08-16", false, "St Joachim, II class, no Lord/BVM/apostle clause applies, RG 476(b)") ]
+ | _ -> die "no probe list for %s" base
+ in
+ List.iter
+ (fun (d, expected, desc) ->
+ match creed_of_date rows d with
+ | Some got when Bool.equal got expected -> ()
+ | Some got -> die "characterisation probe FAILED: %s (%s) -- expected creed=%b, extracted creed=%b" d desc expected got
+ | None -> die "characterisation probe FAILED: %s (%s) -- no row extracted for this date at all" d desc)
+ probes;
+ Printf.printf "extract_lms_ordo: all %d characterisation probes confirmed for %s\n" (List.length probes) edition_label;
+ (* Preface-witnesses task (2026-08-23): a SECOND opposite-prediction
+ probe set, over [praef] rather than [creed] -- the same discipline,
+ applied to the new column this task adds, per the task brief's own
+ "probe dates where the rule predicts OPPOSITE answers before
+ adjudicating anything" instruction. Three universal, unambiguous
+ fixed-date feasts per edition, each landing on a DIFFERENT named
+ preface (RG 484 Nativity, RG 497 Apostles, RG 499 Requiem) -- checked
+ directly against this run's own [rows] before this header is
+ written, a hard [die] on failure, not a printed claim. All Souls'
+ own date is edition-specific (2 November, TRANSFERRED to 3 November
+ in the one edition -- 2024-2025 -- where 2 November falls on a
+ Sunday; verified directly against each PDF's own text, not assumed
+ from RG 96 alone). *)
+ let praef_of_date rows d = List.find_map (fun r -> if String.equal r.date d then r.praef else None) rows in
+ let praef_probes =
+ match base with
+ | "lms-ordo-2023-2024.sexp" ->
+ [ ("2023-12-25", "Pr of the Nativity", "the Nativity of Our Lord, I class, RG 484(a)");
+ ("2024-06-29", "Pr of the Apostles", "SS Peter & Paul, I class, RG 497");
+ ("2024-11-02", "Pr of the Dead", "All Souls' Day, I class, RG 499") ]
+ | "lms-ordo-2024-2025.sexp" ->
+ [ ("2024-12-25", "Pr of the Nativity", "the Nativity of Our Lord, I class, RG 484(a)");
+ ("2025-06-29", "Pr of the Apostles", "SS Peter & Paul, I class, RG 497");
+ ( "2025-11-03", "Pr of the Dead",
+ "All Souls' Day, I class, RG 499 -- TRANSFERRED from 2 November, a Sunday in this edition's own calendar" )
+ ]
+ | "lms-ordo-2025-2026.sexp" ->
+ [ ("2025-12-25", "Pr of the Nativity", "the Nativity of Our Lord, I class, RG 484(a)");
+ ("2026-06-29", "Pr of the Apostles", "SS Peter & Paul, I class, RG 497");
+ ("2026-11-02", "Pr of the Dead", "All Souls' Day, I class, RG 499") ]
+ | _ -> die "no praef probe list for %s" base
+ in
+ List.iter
+ (fun (d, expected_prefix, desc) ->
+ match praef_of_date rows d with
+ | Some got when has_prefix ~prefix:expected_prefix got -> ()
+ | Some got ->
+ die "praef characterisation probe FAILED: %s (%s) -- expected prefix %S, extracted %S" d desc
+ expected_prefix got
+ | None -> die "praef characterisation probe FAILED: %s (%s) -- no praef extracted for this date at all" d desc)
+ praef_probes;
+ Printf.printf "extract_lms_ordo: all %d praef characterisation probes confirmed for %s\n" (List.length praef_probes)
+ edition_label;
+ let praef_probe_lines =
+ String.concat "\n"
+ (List.map (fun (d, expected_prefix, desc) -> Printf.sprintf "; %s (%s) -> %S" d desc expected_prefix) praef_probes)
+ in
+ let missing_praef_count = List.length missing_praef in
+ let probe_lines =
+ String.concat "\n"
+ (List.map
+ (fun (d, expected, desc) -> Printf.sprintf "; %s (%s) -> %s" d desc (if expected then "TRUE" else "FALSE"))
+ probes)
+ in
+ let header =
+ Printf.sprintf
+ {|; %s -- Witnesses task (2026-08-22-colitur-celebrant-rubrics-phase1),
+; extending Task 6's single LMS fixture (2024-2025) to three editions of
+; the Latin Mass Society's own printed Ordo for England & Wales -- a
+; lineage independent of Divinum Officium/missalemeum/lectio (layers 3-4)
+; and of the electronic LT.txt transcription (layers 1-2/5): a physically
+; printed, professionally-compiled liturgical calendar with its own ISBN.
+;
+; Generator: tools/extract_lms_ordo.ml -- do not hand-edit; re-run against
+; a fresh pdftotext dump and commit the diff instead.
+;
+; Source: "The Ordo %s", compiled by Peter Day-Milne, The Latin
+; Mass Society, 9 Mallow Street, London EC1Y 8RQ (title page and
+; colophon, verified directly against the PDF's own extracted text --
+; ISBN %s, Copyright (c) The Latin Mass Society, UK %s).
+; PDF metadata: %s.
+; URL (%s):
+; %s
+; The local PDF (docs/research/ordo/%s) is gitignored
+; (CLAUDE.md: "docs/ is gitignored"), so this header, not git, is this
+; fixture's only record of where it came from.
+; SHA-256 of docs/research/ordo/%s: %s
+; Extracted (UTC): %s
+; Exact commands:
+; pdftotext -layout docs/research/ordo/%s /tmp/lms.txt
+; dune exec tools/extract_lms_ordo.exe -- /tmp/lms.txt \
+; docs/research/ordo/%s %s
+;
+; COVERAGE: %d day-rows, %s through %s (starting %s,
+; through the day before a malformed trailing entry the extractor
+; deliberately stops before -- see CHARACTERISATION below).
+;
+; CHARACTERISATION FINDINGS (Step 1, non-negotiable per the task brief --
+; the extraordinaryform.org Ordo silently omitted St Lawrence's vigil on
+; EVERY date it covered, which nearly produced a false corroboration
+; during the RG 33 work; every LMS edition, including this one, is probed
+; with the same discipline before a single divergence is adjudicated):
+;
+; 1. Gl/Cr are printed PER MASS-OPTION, not per day (confirmed again for
+; THIS edition: it carries many more Gl/Cr pairs than day-rows, because
+; every diocesan variant repeats its own "Gl ... Cr ... Pr of ..."
+; line). This extractor resolves the day's OWN office by taking the
+; FIRST Gl/Cr pair in reading order, which the source's own layout
+; guarantees is the universal entry's (diocesan variants are always
+; introduced by a colon-terminated diocese-list line that comes AFTER
+; the universal block, never before -- checked structurally, not
+; assumed, in every sampled block during development, this edition
+; included).
+;
+; 2. OPPOSITE-PREDICTION PROBES against Rite_ef.Rubrics_ef.creed (RG
+; 475-476), RE-CHECKED BY THIS TOOL RUN (not merely copied from a prior
+; edition's findings) before a single row of this fixture is trusted --
+; the technique CLAUDE.md records as the one that catches a source
+; silently omitting the very thing it is meant to witness. This run's
+; own extracted [rows] were checked against every one of the following
+; before this header was even written; a failure here is a hard [die],
+; not a printed claim:
+%s
+; All %d predictions confirmed, both directions (TRUE and FALSE each
+; independently witnessed) -- this edition is discriminating, not a
+; constant, on the Creed.
+;
+; 3. The BVM-Saturday roman numeral (I-V, "Missae de sancta Maria in
+; sabbato", RG 309(a)) is anchored by WHOLE-LINE match against the five
+; literal strings below, NEVER by substring search: "V Mass of BVM" is
+; a substring of "IV Mass of BVM", so a naive `contains` check
+; misreads every fourth-Mass day as the fifth (the coordinator's own
+; misreading, recorded and retracted in commit 27b07b4 before Task 6
+; existed; the same anchoring logic runs unchanged for every edition).
+;
+; 4. SCOPE EXCLUDED, quantified, not silently dropped:
+; - Diocesan variants: present on %d of %d days (%s%%) -- their own
+; text is read only far enough to set [has_diocesan_variant],
+; never compared, because colitur computes the UNIVERSAL General
+; Roman Calendar only, with no diocesan overlay loaded.
+; - The single malformed trailing entry ("...The OCTAVE DAY of the
+; NATIVITY..." with a bare 4-digit token where a weekday+day-number
+; pair belongs) is excluded outright: the extractor's own stop
+; marker (a line whose first token is a bare 4-digit token) halts
+; the scan before it. CONFIRMED IDENTICAL, word for word, across
+; ALL FOUR known editions of this publisher's Ordo (2023-2024,
+; 2024-2025, 2025-2026, and the 2024-2025 fixture's own prior
+; header) -- always the literal stray year token "2025", regardless
+; of which edition or which real trailing year it should read: a
+; stale copy-paste leftover the compiler's own master document
+; carries forward unedited release to release (even the 2024-2025
+; PDF's own tail page footer nearby still reads "Latin Mass Society
+; Ordo 2021-2022"), not something specific to this edition.
+; - Front matter, the Abbreviations/Rubrical-Primer/Breviary sections,
+; and the "When may I say...?"/Appendix tail are never scanned at
+; all (the extractor's own start/stop markers bound it to the Ordo
+; proper).
+;
+; 5. THE "Pr of" (PREFACE) COLUMN (Preface-witnesses task, 2026-08-23) --
+; [row.praef], the RAW text from the FIRST "Pr of ..." / "Common Pr"
+; match in the universal block's own reading order (the same
+; FIRST-in-reading-order discipline finding 1 above already
+; establishes for Gl/Cr). CAPTURED, NOT CLASSIFIED by this tool --
+; classification into {!Colitur_kernel.Preface.t} and the comparison
+; against {!Rite_ef.Rubrics_ef.preface} both happen test-side
+; (test/test_lms_ordo.ml's own [classify_praef]).
+;
+; CHARACTERISATION FINDING, non-negotiable per the same discipline as
+; finding 2: this Ordo prints OPTION LISTS ("Pr of X or Pr of Y or
+; Common Pr"), not always a single value -- and several of the named
+; options ("Martyrs", "All Saints and Patron Saints" [almost always
+; diocesan -- a Patronal feast's own preface -- but checked directly to
+; occur ONCE in THIS edition's own UNIVERSAL block too, All Saints' Day
+; itself, 1 November, paired with "or Common Pr"/"or Pr of the
+; Trinity" depending on whether 1 November falls on a Sunday that
+; edition, exactly matching {!Rite_ef.Rubrics_ef.preface}'s own RG
+; 494(b) Sunday-gated Trinity fallback either way -- see that
+; function's own citation], "the Dedication of a Church", "the Most Holy
+; Sacrament" [Corpus Christi's own universal entry], "St John the
+; Baptist", "the Angels") are genuinely NOT among the fourteen the
+; 1962 Missale Romanum's own RG 484-497 enumerate (docs/research/LT.txt,
+; lines 3936-4020, checked directly: no "de Martyribus"/"de Angelis"/
+; "de Dedicatione"/"de Ss.mo Sacramento"/"de S. Ioanne Baptista" clause
+; exists anywhere in 482-499). Every one of these extras, in every
+; instance found in this edition's UNIVERSAL block, is printed
+; ALONGSIDE a genuine RG 482 answer (most often "or Common Pr"; on 24
+; March-shaped days, "or Pr of Lent", the live RG 486(b) de-Tempore
+; answer) -- never as the row's OWN AND ONLY option. This is why the
+; test-side comparison checks colitur's single computed answer is a
+; MEMBER of this row's own parsed option set (dropping any unmapped
+; extra name from that set first), not string equality against
+; whichever option happens to print first -- the same "capture, don't
+; force a match invented at extraction time" discipline
+; tools/extract_fiuv_ordo.ml's own [row.praef] doc comment already
+; states for a different publisher's format.
+;
+; OPPOSITE-PREDICTION PROBES against Rite_ef.Rubrics_ef.preface (RG
+; 482-499), the SAME discipline as finding 2, applied to this new
+; column -- RE-CHECKED BY THIS TOOL RUN before this header was
+; written; a failed probe is a hard [die]:
+%s
+; All %d predictions confirmed -- this column, too, is discriminating,
+; not a constant, across three different RG-cited outcomes (Nativity/
+; Apostles/Requiem).
+;
+; COVERAGE: %d of %d rows have no "Pr of"/"Common Pr" text found at
+; all -- checked directly against this window's own Good Friday
+; (Easter-2): its block carries NO Gl/Cr/Pr line whatsoever in this
+; source (the 1955-restored Holy Week's own liturgical action has no
+; Mass that day at all -- the SAME structural fact
+; {!test_creed_coverage}'s own [window_good_fridays] already asserts
+; for Gl/Cr), so [praef] shares that single expected gap rather than
+; having a coverage gap of its own.
+|}
+ dest edition_label isbn copyright_year pdf_meta url_note url pdf_base pdf_base pdf_sha (today ()) pdf_base
+ pdf_base dest
+ (List.length rows)
+ (match rows with r :: _ -> r.date | [] -> "?")
+ (match List.rev rows with r :: _ -> r.date | [] -> "?")
+ window_open_desc probe_lines (List.length probes) with_variant (List.length rows)
+ (Printf.sprintf "%.1f" (100.0 *. float_of_int with_variant /. float_of_int (List.length rows)))
+ praef_probe_lines (List.length praef_probes) missing_praef_count (List.length rows)
+ in
+ let body = Sexplib.Sexp.to_string_hum ~indent:2 (sexp_of_list sexp_of_row rows) in
+ let oc = open_out dest in
+ Fun.protect
+ ~finally:(fun () -> close_out_noerr oc)
+ (fun () ->
+ output_string oc header;
+ output_string oc body;
+ output_string oc "\n");
+ Printf.printf "extract_lms_ordo: wrote %d rows to %s\n" (List.length rows) dest