From 22824ef50abfeb497f04e73f88d0f7ab2e164eaf Mon Sep 17 00:00:00 2001 From: Lukasz Kasprzak Date: Wed, 19 Aug 2026 14:35:36 +0200 Subject: fix(lang): correct two wrong citation lines, add check-citations Two of la.ini's LT.txt: citations pointed at the wrong line -- the Latin itself was right, only the pinned line was wrong: - advent cited LT.txt:8631 ("Tempus Nativitatis"); the real "Tempus Adventus" heading is at 8609. - ef-christ-the-king and [rank]'s own citation both pointed near "Dominica ultima Octobris" (12459) when the text they actually quote, "D.NI NOSTRI JESU CHRISTI REGIS" and "I classis", sits two and three lines further down, at 12461 and 12462. ef-christmas-sunday-0 was marked PATTERN but LT.txt:8644 is the identical string verbatim -- relabelled as a direct citation, not constructed. Added tools/check_citations.py and `make check-citations`: for every LT.txt: citation outside a PATTERN block, confirms a +-2-line window around line n actually contains the Latin text the citation claims, rather than trusting each of the 38 citations by hand. Follows check-schema/check-templates' own precedent -- docs/ is gitignored, so the target prints SKIPPED loudly and exits 0 when docs/research/LT.txt is absent, never a silent pass. The checker's own teeth are proven three ways: replayed against the pre-fix file it independently re-derives both corrections above; a fresh mutation (redirecting one citation to an unrelated line) is caught and reverted; the fixed file passes clean, 147 citations checked, 0 wrong. --- tools/check_citations.py | 232 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 232 insertions(+) create mode 100755 tools/check_citations.py (limited to 'tools') diff --git a/tools/check_citations.py b/tools/check_citations.py new file mode 100755 index 0000000..750f0a4 --- /dev/null +++ b/tools/check_citations.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +"""check_citations.py -- verify every "LT.txt:" citation in lang/la.ini +actually resolves to the Latin text it claims, in docs/research/LT.txt. + +Run via `make check-citations`. Exits 2 with a report if any citation is +wrong; exits 0 (silently, bar a summary line) if every citation checked out; +exits 0 with a loud "SKIPPED" line if docs/research/LT.txt is not present +(it is gitignored -- see below). + +WHAT THIS CHECKS, PRECISELY (a heuristic, not a proof) +------------------------------------------------------- +lang/la.ini's own comments cite a Missal heading in one of two shapes: + + 1. A LEADING comment block, then a group of entries it covers, e.g. + "; Ash Wednesday and the three days after it -- LT.txt:8686-8689." + followed by four `key = value` lines. + 2. A TRAILING comment immediately under the ONE entry it explains, e.g. + "advent = Tempus Adventus" then "; LT.txt:8609." on the next line, + with no blank line -- [season]'s own style. + +For each individual cited line number (after expanding "A-B" ranges and +comma lists), this script builds a POOL of candidate Latin phrases: the +entry/entries the citation is attached to (the single preceding entry for +the trailing shape, the group of following entries for the leading shape), +PLUS every double-quoted Latin phrase appearing anywhere in that comment +block (comments routinely quote an ALTERNATIVE heading being discussed, not +only the chosen entry's own value -- see e.g. the [season] block's +time-after-epiphany caveat). Each pool item keeps its OWN distinctive-word +set (>=4 letters, not on the small stopword list below, j/i and +ae/oe/diacritics normalised) -- items are not flattened into one bag. A +citation PASSES if a window of LT.txt[n-2 .. n+2] (+-2 lines, since a +heading can wrap) contains ALL of at least one single pool item's words -- +not merely ANY word from ANY item. That distinction matters: a flattened +any-word-overlap check let a citation bundling two claims onto one line +number ("D.NI NOSTRI JESU CHRISTI REGIS / I classis", cited at LT.txt:12459) +pass on the strength of the first half alone (found two lines away, at the +edge of tolerance) even though the second half ("I classis") was three +lines away and never actually checked -- one of the two real citation bugs +this script exists to catch. Requiring one item's FULL word-set closes +that gap. + +This is deliberately a LOOSE, word-overlap check, not an exact-phrase +match: la.ini spells abbreviations out in full (Sanctissimi, not Ss.mi) and +normalises j->i, and requiring a byte-exact substring would either force +every citation's prose to repeat the raw OCR text verbatim (defeating the +point of writing readable comments) or produce false failures having +nothing to do with a wrong line number. The trade-off is disclosed, not +hidden: this catches a citation pointing at UNRELATED content (the two real +bugs this script exists because of: LT.txt:8631 cited for "Tempus +Adventus" is actually "Tempus Nativitatis"; LT.txt:12459 cited for +"D.NI NOSTRI JESU CHRISTI REGIS / I classis" is actually just "Dominica +ultima Octobris") -- it does not, and cannot, prove a citation is the BEST +possible line, only that it is not obviously wrong. + +Only citations OUTSIDE a "PATTERN" block are checked: a PATTERN entry makes +no claim that its own line is a direct heading, so a "LT.txt:N" mentioned +in its comment (e.g. citing the GRAMMAR another day's heading attests, not +this day's own heading) is not a provenance claim for THIS entry and would +otherwise produce a meaningless failure. +""" +import re +import sys +import unicodedata +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +LA_INI = ROOT / "lang" / "la.ini" +LT_TXT = ROOT / "docs" / "research" / "LT.txt" + +STOPWORDS = { + "in", "de", "et", "ad", "post", "ante", "cum", "per", "seu", "infra", + "vel", "si", "haec", "hoc", "hic", "qui", "quae", "quod", "quia", + "tempus", "dominica", "dominicam", "dominicae", "feria", "feriae", + "sabbato", "sabbatum", "die", "diebus", "eodem", "anno", "eius", + "sancti", "sancta", "sanctae", "sancto", "sanctorum", "sanctus", + "domini", "dominus", "octava", "octavam", "octavas", + "missae", "missa", "proprium", "gregorianus", "cantus", "pdf", + "forma", "longior", "brevior", "vide", "etiam", "dom", "prosper", + "sacro", "actio", "electronica", "formam", "novissimae", "variationes", + "copyright", "archivum", "liturgicum", "missale", "romanum", "index", + "www", "http", "https", "htm", "html", "com", "romanum", "text", +} + + +def normalize_word(w: str) -> str: + w = w.lower() + w = unicodedata.normalize("NFKD", w) + w = "".join(c for c in w if not unicodedata.combining(c)) + w = w.replace("æ", "ae").replace("œ", "oe") + w = re.sub(r"[^a-z]", "", w) + w = w.replace("j", "i") + return w + + +def distinctive_words(text: str) -> set: + out = set() + for tok in re.split(r"\s+", text): + w = normalize_word(tok) + if len(w) >= 4 and w not in STOPWORDS: + out.add(w) + return out + + +def expand_citation_spec(spec: str): + """'8618,8620,8622' -> [8618,8620,8622]; '8691-8717' -> [8691..8717].""" + nums = [] + for tok in spec.split(","): + tok = tok.strip() + m = re.fullmatch(r"(\d{2,6})-(\d{2,6})", tok) + if m: + a, b = int(m.group(1)), int(m.group(2)) + if a <= b and (b - a) <= 200: + nums.extend(range(a, b + 1)) + continue + m = re.fullmatch(r"(\d{2,6})", tok) + if m: + nums.append(int(m.group(1))) + return nums + + +CITATION_RE = re.compile(r"LT\.txt:\s*((?:\d{2,6}(?:-\d{2,6})?)(?:\s*,\s*\d{2,6}(?:-\d{2,6})?)*)") +QUOTE_RE = re.compile(r'"([^"]{3,})"') + + +def parse_blocks(la_ini_text: str): + """Split la.ini into blocks on blank lines and [section] headers. Each + block is a list of (kind, content) where kind is 'comment' or 'entry', + content is the stripped comment text or (key, value).""" + blocks = [] + cur = [] + for raw in la_ini_text.split("\n"): + line = raw.rstrip("\n") + stripped = line.strip() + if stripped == "" or stripped.startswith("["): + if cur: + blocks.append(cur) + cur = [] + continue + if stripped.startswith(";"): + cur.append(("comment", stripped[1:].strip())) + elif "=" in stripped: + k, _, v = stripped.partition("=") + cur.append(("entry", (k.strip(), v.strip()))) + # anything else (shouldn't occur) is ignored + if cur: + blocks.append(cur) + return blocks + + +def check(la_ini_text: str, lt_lines: list): + findings = [] + checked = 0 + for block in parse_blocks(la_ini_text): + block_comment = "\n".join(c for k, c in block if k == "comment") + if "PATTERN" in block_comment: + continue + entries = [c for k, c in block if k == "entry"] + if not entries: + continue + quotes = QUOTE_RE.findall(block_comment) + entries_before = [] + for k, c in block: + if k == "comment": + for m in CITATION_RE.finditer(c): + nums = expand_citation_spec(m.group(1)) + if entries_before: + pool_entries = [entries_before[-1]] + else: + # leading citation: pool = every entry in the block + # (entries after this comment, i.e. all of them, + # since none has been seen yet) + pool_entries = entries + # Pool items are kept SEPARATE (not flattened into one + # bag of words): a citation passes only if the window + # fully covers -- ALL the distinctive words of -- at + # least one single pool item (one quoted phrase, or one + # entry's own value). A flattened "any word from any + # pool item" bag is too permissive: it let a citation + # bundling two claims onto one line number ("D.NI + # NOSTRI JESU CHRISTI REGIS / I classis") pass on the + # strength of the FIRST half alone, even though the + # second half ("I classis") was not actually nearby -- + # exactly the shape of one of the two real citation + # bugs this script was written to catch. Verified by + # replay against the pre-fix file (see the task report). + pool_items = [distinctive_words(q) for q in quotes] + for _, v in pool_entries: + pool_items.append(distinctive_words(v)) + pool_items = [p for p in pool_items if p] + for n in nums: + checked += 1 + lo, hi = max(1, n - 2), min(len(lt_lines), n + 2) + window_text = " ".join(lt_lines[lo - 1 : hi]) + window_words = distinctive_words(window_text) + if not any(item <= window_words for item in pool_items): + entry_desc = ", ".join(f"{k}={v}" for k, v in pool_entries) + findings.append( + { + "line": n, + "entries": entry_desc, + "actual": lt_lines[n - 1].strip() if 1 <= n <= len(lt_lines) else "(out of range)", + "window": window_text.strip()[:160], + } + ) + else: + entries_before.append(c) + return checked, findings + + +def main(): + if not LT_TXT.exists(): + print( + "SKIPPED: docs/research/LT.txt is absent (docs/ is gitignored -- " + "present locally only). Citations are NOT verified this run." + ) + return 0 + la_ini_text = LA_INI.read_text(encoding="utf-8") + lt_lines = LT_TXT.read_text(encoding="utf-8", errors="replace").split("\n") + checked, findings = check(la_ini_text, lt_lines) + if findings: + print(f"check-citations: {len(findings)} of {checked} citations look wrong:\n") + for f in findings: + print(f" LT.txt:{f['line']} cited for [{f['entries']}]") + print(f" actual line {f['line']}: {f['actual']!r}") + print(f" window (+-2): {f['window']!r}\n") + return 2 + print(f"check-citations: {checked} LT.txt citations checked, 0 look wrong.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) -- cgit v1.3