From 7f263a0ec9a91d1a036cfd22ed38354d06500b1d Mon Sep 17 00:00:00 2001 From: Lukasz Kasprzak Date: Wed, 19 Aug 2026 15:01:18 +0200 Subject: fix(tools): harden check_citations.py against its own self-poisoning bug Reproduced the defect: reintroducing the exact historical citation bug (pointing class-1's citation back at LT.txt:12459, the value a prior fix round corrected away from) made the tool report "147 citations checked, 0 look wrong". The mechanism was that the corrective comment documenting the old bug quotes the wrong historical value, and the checker pooled every quoted phrase from the whole surrounding comment block, so citing the wrong line matched the comment explaining why it was wrong. Four changes: 1. The word pool for a citation is now scoped to the entry(ies) it is attached to only -- never to quoted text elsewhere in the comment. This is the direct fix for the self-poisoning bug. 2. A citation whose pool has fewer than two distinctive words (Latin liturgical headings are short and stopword-heavy) cannot discriminate the right line from a wrong nearby one. Such a citation is now reported CANNOT VERIFY and fails the target, instead of silently passing. 3. The blanket +-2-line tolerance is gone. A bare "LT.txt:N" is checked at line N only; a heading that genuinely wraps must say so explicitly as "LT.txt:N-M". The allowance moves into the data, where it is visible. 4. The tool gets its own test suite, tools/test_check_citations.py, with a synthetic fixture covering: a correct citation, off-by-one and off-by-three mismatches, an explicit wrap range, a degenerate pool, a PATTERN-marked entry with no citation, and a dedicated regression test for the self-poisoning case itself. Wired into `dune test` via a new (rule (alias runtest) ...) in tools/dune (a plain (test ...) stanza cannot run a Python script), so it runs with the rest of the suite, not only as a `make` target. Added a --file/--lt-file override to check_citations.py so the tool (and its own tests) can point at a fixture without touching the real lang/la.ini or docs/research/LT.txt. Confirmed the "SKIPPED, exit 0" behaviour for a missing docs/research/LT.txt is unchanged. tools/__pycache__/ (a stray artefact of this script, previously untracked and ungitignored) is now in .gitignore. Measured against the current lang/la.ini (another task is still landing its sanctoral entries on this branch): 15 of 275 citations now look wrong and 42 more cannot be verified, both far above the 0 the unhardened tool reported. Not fixed here -- the data pass is separate, once the sanctoral entries land. --- tools/check_citations.py | 318 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 229 insertions(+), 89 deletions(-) (limited to 'tools/check_citations.py') diff --git a/tools/check_citations.py b/tools/check_citations.py index 750f0a4..752977e 100755 --- a/tools/check_citations.py +++ b/tools/check_citations.py @@ -2,10 +2,55 @@ """check_citations.py -- verify every "LT.txt:" citation in lang/la.ini actually resolves to the Latin text it claims, in docs/research/LT.txt. -Run via `make check-citations`. Exits 2 with a report if any citation is -wrong; exits 0 (silently, bar a summary line) if every citation checked out; -exits 0 with a loud "SKIPPED" line if docs/research/LT.txt is not present -(it is gitignored -- see below). +Run via `make check-citations` (or directly: `python3 tools/check_citations.py +[--file LA_INI] [--lt-file LT_TXT]`). Exits 2 with a report if any citation +looks WRONG or CANNOT BE VERIFIED; exits 0 (with a summary line) only if +every citation checked out cleanly; exits 0 with a loud "SKIPPED" line if +docs/research/LT.txt is not present (it is gitignored -- see below). + +THIS SCRIPT WAS ITSELF FOUND TO BE SELF-POISONING (2026-08-19) AND HARDENED +------------------------------------------------------------------------ +An earlier version pooled distinctive words from the whole COMMENT BLOCK +around a citation, including any double-quoted phrase the comment happened +to mention -- and comments routinely quote a WRONG historical value while +explaining a past fix (e.g. "CORRECTED: previously cited LT.txt:12459, +which is 'Dominica ultima Octobris', not this heading"). That quote landed +in the pool, so re-introducing the exact bug being documented -- citing +LT.txt:12459 again -- matched the very quote correcting it, and the script +reported "0 look wrong". A verification tool whose own documentation of a +fix defeats the check for that fix is worse than no tool: it manufactures +false confidence. Proven with a reproduction: reintroducing that one wrong +citation into a real copy of lang/la.ini produced zero findings on the +pre-hardening script. Four changes closed this, in order of how directly +each one addresses the reproduction: + + 1. THE POOL IS SCOPED TO THE CITATION'S OWN ENTRY, never to the + surrounding comment's quoted text. A citation is verified against the + name it claims, not against anything quoted nearby -- see + `distinctive_words` / the per-citation `pool_items` construction + below. This alone closes the reproduced bug (see `test_check_citations.py`'s + own `test_self_poisoning_quote_does_not_pass`). + 2. A POOL TOO THIN TO VERIFY FAILS CLOSED. Latin liturgical headings are + short and heavily stopword-laden ("Tempus Adventus", "I classis", + "albus"): after stripping stopwords, MANY single-entry pools collapse + to one word or none -- a one-word "match" proves nothing (it is as + likely to hit an unrelated nearby heading as the right one). Such a + citation is reported CANNOT VERIFY, not PASS, and it fails the target + exactly like a genuine mismatch: absence of evidence is not evidence + of correctness, and this script must not report it as one. + 3. NO MORE BLANKET +-2-LINE TOLERANCE. A citation is checked at the EXACT + line it names. A heading that genuinely spans more than one physical + line in the source must say so explicitly, "LT.txt:8609-8610" -- the + allowance moves into the data, where a reader (and a future citation + added nearby) can see it, instead of silently forgiving ANY citation + within two lines of the truth. A bare "LT.txt:N" is checked at line N + only. + 4. THIS SCRIPT NOW HAS ITS OWN TEST SUITE (test_check_citations.py, + wired into `dune test` via tools/dune's own runtest rule) -- the + single most important change. The reproduced bug survived as long as + it did specifically BECAUSE nothing exercised this script's own + logic against a known-wrong citation. The self-poisoning case above + is now a permanent regression test. WHAT THIS CHECKS, PRECISELY (a heuristic, not a proof) ------------------------------------------------------- @@ -13,44 +58,51 @@ lang/la.ini's own comments cite a Missal heading in one of two shapes: 1. A LEADING comment block, then a group of entries it covers, e.g. "; Ash Wednesday and the three days after it -- LT.txt:8686-8689." - followed by four `key = value` lines. + followed by four `key = value` lines. The pool for every citation + found in such a comment is EVERY entry in the group (there is no + positional correspondence encoded in the data between a particular + cited line and a particular entry in the list). 2. A TRAILING comment immediately under the ONE entry it explains, e.g. "advent = Tempus Adventus" then "; LT.txt:8609." on the next line, - with no blank line -- [season]'s own style. - -For each individual cited line number (after expanding "A-B" ranges and -comma lists), this script builds a POOL of candidate Latin phrases: the -entry/entries the citation is attached to (the single preceding entry for -the trailing shape, the group of following entries for the leading shape), -PLUS every double-quoted Latin phrase appearing anywhere in that comment -block (comments routinely quote an ALTERNATIVE heading being discussed, not -only the chosen entry's own value -- see e.g. the [season] block's -time-after-epiphany caveat). Each pool item keeps its OWN distinctive-word -set (>=4 letters, not on the small stopword list below, j/i and -ae/oe/diacritics normalised) -- items are not flattened into one bag. A -citation PASSES if a window of LT.txt[n-2 .. n+2] (+-2 lines, since a -heading can wrap) contains ALL of at least one single pool item's words -- -not merely ANY word from ANY item. That distinction matters: a flattened -any-word-overlap check let a citation bundling two claims onto one line -number ("D.NI NOSTRI JESU CHRISTI REGIS / I classis", cited at LT.txt:12459) -pass on the strength of the first half alone (found two lines away, at the -edge of tolerance) even though the second half ("I classis") was three -lines away and never actually checked -- one of the two real citation bugs -this script exists to catch. Requiring one item's FULL word-set closes -that gap. - -This is deliberately a LOOSE, word-overlap check, not an exact-phrase -match: la.ini spells abbreviations out in full (Sanctissimi, not Ss.mi) and -normalises j->i, and requiring a byte-exact substring would either force -every citation's prose to repeat the raw OCR text verbatim (defeating the -point of writing readable comments) or produce false failures having -nothing to do with a wrong line number. The trade-off is disclosed, not -hidden: this catches a citation pointing at UNRELATED content (the two real -bugs this script exists because of: LT.txt:8631 cited for "Tempus -Adventus" is actually "Tempus Nativitatis"; LT.txt:12459 cited for -"D.NI NOSTRI JESU CHRISTI REGIS / I classis" is actually just "Dominica -ultima Octobris") -- it does not, and cannot, prove a citation is the BEST -possible line, only that it is not obviously wrong. + with no blank line -- [season]'s own style. The pool is that one entry + alone. + +A citation is either a bare line number ("LT.txt:8609") or an explicit +range ("LT.txt:8609-8610") for a heading that genuinely wraps across +physical lines in the source; a comma-separated list ("LT.txt:8618,8620, +8622") is several independent citations, each checked on its own. For a +bare number the window is that one line; for a range it is the union of +every line in the range (inclusive). There is no other tolerance. + +For each citation, the POOL is the set of candidate Latin phrases it could +be defending: the entry (trailing shape) or every entry in the group +(leading shape) -- nothing pulled from quoted prose elsewhere in the +comment (see the self-poisoning account above). Each pool item keeps its +OWN distinctive-word set (>=4 letters, not on the small stopword list +below, j/i and ae/oe/diacritics normalised) -- items are never flattened +into one shared bag, for the same reason quotes were removed: a citation +bundling two claims onto one line number must not pass on the strength of +an unrelated pool item's words. + +An item with fewer than two distinctive words is DEGENERATE -- it cannot +discriminate the right line from a wrong nearby one, so it is excluded from +matching. If every item in a citation's pool is degenerate, the citation is +reported CANNOT VERIFY (counted and failed, never silently skipped or +silently passed). Otherwise the citation PASSES if the cited window's text +contains ALL of at least one non-degenerate pool item's words, and FAILS +otherwise. + +This is deliberately a LOOSE, word-overlap check within the (now exact) +window, not a byte-exact phrase match: la.ini spells abbreviations out in +full (Sanctissimi, not Ss.mi) and normalises j->i, and requiring a +byte-exact substring would either force every citation's prose to repeat +the raw OCR text verbatim (defeating the point of writing readable +comments) or produce false failures having nothing to do with a wrong line +number. What it proves is narrower than a byte-exact match, and is +disclosed as such: PASS means "the claimed name's distinctive words are +present, in full, at the exact line(s) cited" -- not that the citation is +the best possible line, only that it is not obviously wrong and is not +resting on a coincidence-prone single word. Only citations OUTSIDE a "PATTERN" block are checked: a PATTERN entry makes no claim that its own line is a direct heading, so a "LT.txt:N" mentioned @@ -58,14 +110,15 @@ in its comment (e.g. citing the GRAMMAR another day's heading attests, not this day's own heading) is not a provenance claim for THIS entry and would otherwise produce a meaningless failure. """ +import argparse import re import sys import unicodedata from pathlib import Path ROOT = Path(__file__).resolve().parent.parent -LA_INI = ROOT / "lang" / "la.ini" -LT_TXT = ROOT / "docs" / "research" / "LT.txt" +DEFAULT_LA_INI = ROOT / "lang" / "la.ini" +DEFAULT_LT_TXT = ROOT / "docs" / "research" / "LT.txt" STOPWORDS = { "in", "de", "et", "ad", "post", "ante", "cum", "per", "seu", "infra", @@ -81,6 +134,11 @@ STOPWORDS = { "www", "http", "https", "htm", "html", "com", "romanum", "text", } +# A pool item with fewer than this many distinctive words cannot +# discriminate the right line from a wrong nearby one -- see the module +# docstring's item 2. +MIN_DISTINCTIVE_WORDS = 2 + def normalize_word(w: str) -> str: w = w.lower() @@ -101,25 +159,42 @@ def distinctive_words(text: str) -> set: return out -def expand_citation_spec(spec: str): - """'8618,8620,8622' -> [8618,8620,8622]; '8691-8717' -> [8691..8717].""" - nums = [] +class CitationRef: + """One citation token: 'label' is what the data actually wrote + ("8609" or "8609-8610"); 'lines' is the fully-expanded, sorted list of + line numbers the label names -- a single-element list for a bare + number, the whole inclusive range for an explicit wrap.""" + + __slots__ = ("label", "lines") + + def __init__(self, label, lines): + self.label = label + self.lines = lines + + +def parse_citation_spec(spec: str): + """'8618,8620,8622' -> three exact-line CitationRefs; '8609-8610' -> one + CitationRef spanning both lines (an explicit wrapped heading). Each + comma-separated token is independent; a malformed token (b < a, or a + span so wide it is almost certainly a typo, capped at 200 lines) is + silently dropped rather than crashing on bad data, matching this + script's existing tolerance elsewhere for data it does not own.""" + refs = [] for tok in spec.split(","): tok = tok.strip() m = re.fullmatch(r"(\d{2,6})-(\d{2,6})", tok) if m: a, b = int(m.group(1)), int(m.group(2)) if a <= b and (b - a) <= 200: - nums.extend(range(a, b + 1)) + refs.append(CitationRef(tok, list(range(a, b + 1)))) continue m = re.fullmatch(r"(\d{2,6})", tok) if m: - nums.append(int(m.group(1))) - return nums + refs.append(CitationRef(tok, [int(m.group(1))])) + return refs CITATION_RE = re.compile(r"LT\.txt:\s*((?:\d{2,6}(?:-\d{2,6})?)(?:\s*,\s*\d{2,6}(?:-\d{2,6})?)*)") -QUOTE_RE = re.compile(r'"([^"]{3,})"') def parse_blocks(la_ini_text: str): @@ -147,9 +222,23 @@ def parse_blocks(la_ini_text: str): return blocks +def window_text_for(lt_lines, lines): + lo, hi = lines[0], lines[-1] + lo_c, hi_c = max(1, lo), min(len(lt_lines), hi) + if lo_c > hi_c: + return "" + return " ".join(lt_lines[lo_c - 1 : hi_c]) + + def check(la_ini_text: str, lt_lines: list): - findings = [] + """Returns a dict: checked (int), passed (int), findings (list -- wrong + citations), unverifiable (list -- degenerate-pool citations). Both + findings and unverifiable are things a human must look at; only + 'passed' citations required no human attention.""" checked = 0 + passed = 0 + findings = [] + unverifiable = [] for block in parse_blocks(la_ini_text): block_comment = "\n".join(c for k, c in block if k == "comment") if "PATTERN" in block_comment: @@ -157,75 +246,126 @@ def check(la_ini_text: str, lt_lines: list): entries = [c for k, c in block if k == "entry"] if not entries: continue - quotes = QUOTE_RE.findall(block_comment) entries_before = [] for k, c in block: if k == "comment": for m in CITATION_RE.finditer(c): - nums = expand_citation_spec(m.group(1)) if entries_before: pool_entries = [entries_before[-1]] else: # leading citation: pool = every entry in the block - # (entries after this comment, i.e. all of them, - # since none has been seen yet) + # (no positional correspondence is encoded between + # a specific cited line and a specific entry). pool_entries = entries + entry_desc = ", ".join(f"{k}={v}" for k, v in pool_entries) # Pool items are kept SEPARATE (not flattened into one # bag of words): a citation passes only if the window # fully covers -- ALL the distinctive words of -- at - # least one single pool item (one quoted phrase, or one - # entry's own value). A flattened "any word from any - # pool item" bag is too permissive: it let a citation - # bundling two claims onto one line number ("D.NI - # NOSTRI JESU CHRISTI REGIS / I classis") pass on the - # strength of the FIRST half alone, even though the - # second half ("I classis") was not actually nearby -- - # exactly the shape of one of the two real citation - # bugs this script was written to catch. Verified by - # replay against the pre-fix file (see the task report). - pool_items = [distinctive_words(q) for q in quotes] - for _, v in pool_entries: - pool_items.append(distinctive_words(v)) - pool_items = [p for p in pool_items if p] - for n in nums: + # least one single pool item. See the module docstring + # for why (the self-poisoning bug and the two-claims- + # on-one-line-number bug this discipline catches). + pool_items = [distinctive_words(v) for _, v in pool_entries] + strong_items = [p for p in pool_items if len(p) >= MIN_DISTINCTIVE_WORDS] + best_len = max((len(p) for p in pool_items), default=0) + for ref in parse_citation_spec(m.group(1)): checked += 1 - lo, hi = max(1, n - 2), min(len(lt_lines), n + 2) - window_text = " ".join(lt_lines[lo - 1 : hi]) + window_text = window_text_for(lt_lines, ref.lines) window_words = distinctive_words(window_text) - if not any(item <= window_words for item in pool_items): - entry_desc = ", ".join(f"{k}={v}" for k, v in pool_entries) + if not strong_items: + unverifiable.append( + { + "label": ref.label, + "entries": entry_desc, + "best_len": best_len, + "pool_words": sorted(set().union(*pool_items)) if pool_items else [], + } + ) + continue + if any(item <= window_words for item in strong_items): + passed += 1 + else: + n = ref.lines[0] findings.append( { - "line": n, + "label": ref.label, "entries": entry_desc, - "actual": lt_lines[n - 1].strip() if 1 <= n <= len(lt_lines) else "(out of range)", + "actual": ( + lt_lines[n - 1].strip() + if 1 <= n <= len(lt_lines) + else "(out of range)" + ), "window": window_text.strip()[:160], } ) else: entries_before.append(c) - return checked, findings + return { + "checked": checked, + "passed": passed, + "findings": findings, + "unverifiable": unverifiable, + } + + +def build_arg_parser(): + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + p.add_argument( + "--file", + dest="la_ini", + type=Path, + default=DEFAULT_LA_INI, + help=f"the la.ini-shaped file to check (default: {DEFAULT_LA_INI})", + ) + p.add_argument( + "--lt-file", + dest="lt_txt", + type=Path, + default=DEFAULT_LT_TXT, + help=f"the LT.txt transcription to check against (default: {DEFAULT_LT_TXT})", + ) + return p -def main(): - if not LT_TXT.exists(): +def main(argv=None): + args = build_arg_parser().parse_args(argv) + if not args.lt_txt.exists(): print( - "SKIPPED: docs/research/LT.txt is absent (docs/ is gitignored -- " + f"SKIPPED: {args.lt_txt} is absent (docs/ is gitignored -- " "present locally only). Citations are NOT verified this run." ) return 0 - la_ini_text = LA_INI.read_text(encoding="utf-8") - lt_lines = LT_TXT.read_text(encoding="utf-8", errors="replace").split("\n") - checked, findings = check(la_ini_text, lt_lines) + la_ini_text = args.la_ini.read_text(encoding="utf-8") + lt_lines = args.lt_txt.read_text(encoding="utf-8", errors="replace").split("\n") + result = check(la_ini_text, lt_lines) + checked = result["checked"] + findings = result["findings"] + unverifiable = result["unverifiable"] + + if not findings and not unverifiable: + print(f"check-citations: {checked} LT.txt citations checked, 0 look wrong.") + return 0 + + print( + f"check-citations: {len(findings)} of {checked} citations look wrong, " + f"{len(unverifiable)} CANNOT VERIFY (pool too thin -- see below):\n" + ) if findings: - print(f"check-citations: {len(findings)} of {checked} citations look wrong:\n") + print("WRONG:\n") for f in findings: - print(f" LT.txt:{f['line']} cited for [{f['entries']}]") - print(f" actual line {f['line']}: {f['actual']!r}") - print(f" window (+-2): {f['window']!r}\n") - return 2 - print(f"check-citations: {checked} LT.txt citations checked, 0 look wrong.") - return 0 + print(f" LT.txt:{f['label']} cited for [{f['entries']}]") + print(f" actual: {f['actual']!r}") + print(f" window: {f['window']!r}\n") + if unverifiable: + print("CANNOT VERIFY (a human must adjudicate these by hand):\n") + for u in unverifiable: + words = ", ".join(u["pool_words"]) if u["pool_words"] else "(none)" + print(f" LT.txt:{u['label']} cited for [{u['entries']}]") + print( + f" pool too thin to verify: best candidate has " + f"{u['best_len']} distinctive word(s) (need >= {MIN_DISTINCTIVE_WORDS}); " + f"pool words: {words}\n" + ) + return 2 if __name__ == "__main__": -- cgit v1.3