diff options
Diffstat (limited to 'tools/check_citations.py')
| -rwxr-xr-x | tools/check_citations.py | 314 |
1 files changed, 227 insertions, 87 deletions
diff --git a/tools/check_citations.py b/tools/check_citations.py index 750f0a4..752977e 100755 --- a/tools/check_citations.py +++ b/tools/check_citations.py @@ -2,10 +2,55 @@ """check_citations.py -- verify every "LT.txt:<n>" citation in lang/la.ini actually resolves to the Latin text it claims, in docs/research/LT.txt. -Run via `make check-citations`. Exits 2 with a report if any citation is -wrong; exits 0 (silently, bar a summary line) if every citation checked out; -exits 0 with a loud "SKIPPED" line if docs/research/LT.txt is not present -(it is gitignored -- see below). +Run via `make check-citations` (or directly: `python3 tools/check_citations.py +[--file LA_INI] [--lt-file LT_TXT]`). Exits 2 with a report if any citation +looks WRONG or CANNOT BE VERIFIED; exits 0 (with a summary line) only if +every citation checked out cleanly; exits 0 with a loud "SKIPPED" line if +docs/research/LT.txt is not present (it is gitignored -- see below). + +THIS SCRIPT WAS ITSELF FOUND TO BE SELF-POISONING (2026-08-19) AND HARDENED +------------------------------------------------------------------------ +An earlier version pooled distinctive words from the whole COMMENT BLOCK +around a citation, including any double-quoted phrase the comment happened +to mention -- and comments routinely quote a WRONG historical value while +explaining a past fix (e.g. "CORRECTED: previously cited LT.txt:12459, +which is 'Dominica ultima Octobris', not this heading"). That quote landed +in the pool, so re-introducing the exact bug being documented -- citing +LT.txt:12459 again -- matched the very quote correcting it, and the script +reported "0 look wrong". A verification tool whose own documentation of a +fix defeats the check for that fix is worse than no tool: it manufactures +false confidence. Proven with a reproduction: reintroducing that one wrong +citation into a real copy of lang/la.ini produced zero findings on the +pre-hardening script. Four changes closed this, in order of how directly +each one addresses the reproduction: + + 1. THE POOL IS SCOPED TO THE CITATION'S OWN ENTRY, never to the + surrounding comment's quoted text. A citation is verified against the + name it claims, not against anything quoted nearby -- see + `distinctive_words` / the per-citation `pool_items` construction + below. This alone closes the reproduced bug (see `test_check_citations.py`'s + own `test_self_poisoning_quote_does_not_pass`). + 2. A POOL TOO THIN TO VERIFY FAILS CLOSED. Latin liturgical headings are + short and heavily stopword-laden ("Tempus Adventus", "I classis", + "albus"): after stripping stopwords, MANY single-entry pools collapse + to one word or none -- a one-word "match" proves nothing (it is as + likely to hit an unrelated nearby heading as the right one). Such a + citation is reported CANNOT VERIFY, not PASS, and it fails the target + exactly like a genuine mismatch: absence of evidence is not evidence + of correctness, and this script must not report it as one. + 3. NO MORE BLANKET +-2-LINE TOLERANCE. A citation is checked at the EXACT + line it names. A heading that genuinely spans more than one physical + line in the source must say so explicitly, "LT.txt:8609-8610" -- the + allowance moves into the data, where a reader (and a future citation + added nearby) can see it, instead of silently forgiving ANY citation + within two lines of the truth. A bare "LT.txt:N" is checked at line N + only. + 4. THIS SCRIPT NOW HAS ITS OWN TEST SUITE (test_check_citations.py, + wired into `dune test` via tools/dune's own runtest rule) -- the + single most important change. The reproduced bug survived as long as + it did specifically BECAUSE nothing exercised this script's own + logic against a known-wrong citation. The self-poisoning case above + is now a permanent regression test. WHAT THIS CHECKS, PRECISELY (a heuristic, not a proof) ------------------------------------------------------- @@ -13,44 +58,51 @@ lang/la.ini's own comments cite a Missal heading in one of two shapes: 1. A LEADING comment block, then a group of entries it covers, e.g. "; Ash Wednesday and the three days after it -- LT.txt:8686-8689." - followed by four `key = value` lines. + followed by four `key = value` lines. The pool for every citation + found in such a comment is EVERY entry in the group (there is no + positional correspondence encoded in the data between a particular + cited line and a particular entry in the list). 2. A TRAILING comment immediately under the ONE entry it explains, e.g. "advent = Tempus Adventus" then "; LT.txt:8609." on the next line, - with no blank line -- [season]'s own style. + with no blank line -- [season]'s own style. The pool is that one entry + alone. + +A citation is either a bare line number ("LT.txt:8609") or an explicit +range ("LT.txt:8609-8610") for a heading that genuinely wraps across +physical lines in the source; a comma-separated list ("LT.txt:8618,8620, +8622") is several independent citations, each checked on its own. For a +bare number the window is that one line; for a range it is the union of +every line in the range (inclusive). There is no other tolerance. -For each individual cited line number (after expanding "A-B" ranges and -comma lists), this script builds a POOL of candidate Latin phrases: the -entry/entries the citation is attached to (the single preceding entry for -the trailing shape, the group of following entries for the leading shape), -PLUS every double-quoted Latin phrase appearing anywhere in that comment -block (comments routinely quote an ALTERNATIVE heading being discussed, not -only the chosen entry's own value -- see e.g. the [season] block's -time-after-epiphany caveat). Each pool item keeps its OWN distinctive-word -set (>=4 letters, not on the small stopword list below, j/i and -ae/oe/diacritics normalised) -- items are not flattened into one bag. A -citation PASSES if a window of LT.txt[n-2 .. n+2] (+-2 lines, since a -heading can wrap) contains ALL of at least one single pool item's words -- -not merely ANY word from ANY item. That distinction matters: a flattened -any-word-overlap check let a citation bundling two claims onto one line -number ("D.NI NOSTRI JESU CHRISTI REGIS / I classis", cited at LT.txt:12459) -pass on the strength of the first half alone (found two lines away, at the -edge of tolerance) even though the second half ("I classis") was three -lines away and never actually checked -- one of the two real citation bugs -this script exists to catch. Requiring one item's FULL word-set closes -that gap. +For each citation, the POOL is the set of candidate Latin phrases it could +be defending: the entry (trailing shape) or every entry in the group +(leading shape) -- nothing pulled from quoted prose elsewhere in the +comment (see the self-poisoning account above). Each pool item keeps its +OWN distinctive-word set (>=4 letters, not on the small stopword list +below, j/i and ae/oe/diacritics normalised) -- items are never flattened +into one shared bag, for the same reason quotes were removed: a citation +bundling two claims onto one line number must not pass on the strength of +an unrelated pool item's words. -This is deliberately a LOOSE, word-overlap check, not an exact-phrase -match: la.ini spells abbreviations out in full (Sanctissimi, not Ss.mi) and -normalises j->i, and requiring a byte-exact substring would either force -every citation's prose to repeat the raw OCR text verbatim (defeating the -point of writing readable comments) or produce false failures having -nothing to do with a wrong line number. The trade-off is disclosed, not -hidden: this catches a citation pointing at UNRELATED content (the two real -bugs this script exists because of: LT.txt:8631 cited for "Tempus -Adventus" is actually "Tempus Nativitatis"; LT.txt:12459 cited for -"D.NI NOSTRI JESU CHRISTI REGIS / I classis" is actually just "Dominica -ultima Octobris") -- it does not, and cannot, prove a citation is the BEST -possible line, only that it is not obviously wrong. +An item with fewer than two distinctive words is DEGENERATE -- it cannot +discriminate the right line from a wrong nearby one, so it is excluded from +matching. If every item in a citation's pool is degenerate, the citation is +reported CANNOT VERIFY (counted and failed, never silently skipped or +silently passed). Otherwise the citation PASSES if the cited window's text +contains ALL of at least one non-degenerate pool item's words, and FAILS +otherwise. + +This is deliberately a LOOSE, word-overlap check within the (now exact) +window, not a byte-exact phrase match: la.ini spells abbreviations out in +full (Sanctissimi, not Ss.mi) and normalises j->i, and requiring a +byte-exact substring would either force every citation's prose to repeat +the raw OCR text verbatim (defeating the point of writing readable +comments) or produce false failures having nothing to do with a wrong line +number. What it proves is narrower than a byte-exact match, and is +disclosed as such: PASS means "the claimed name's distinctive words are +present, in full, at the exact line(s) cited" -- not that the citation is +the best possible line, only that it is not obviously wrong and is not +resting on a coincidence-prone single word. Only citations OUTSIDE a "PATTERN" block are checked: a PATTERN entry makes no claim that its own line is a direct heading, so a "LT.txt:N" mentioned @@ -58,14 +110,15 @@ in its comment (e.g. citing the GRAMMAR another day's heading attests, not this day's own heading) is not a provenance claim for THIS entry and would otherwise produce a meaningless failure. """ +import argparse import re import sys import unicodedata from pathlib import Path ROOT = Path(__file__).resolve().parent.parent -LA_INI = ROOT / "lang" / "la.ini" -LT_TXT = ROOT / "docs" / "research" / "LT.txt" +DEFAULT_LA_INI = ROOT / "lang" / "la.ini" +DEFAULT_LT_TXT = ROOT / "docs" / "research" / "LT.txt" STOPWORDS = { "in", "de", "et", "ad", "post", "ante", "cum", "per", "seu", "infra", @@ -81,6 +134,11 @@ STOPWORDS = { "www", "http", "https", "htm", "html", "com", "romanum", "text", } +# A pool item with fewer than this many distinctive words cannot +# discriminate the right line from a wrong nearby one -- see the module +# docstring's item 2. +MIN_DISTINCTIVE_WORDS = 2 + def normalize_word(w: str) -> str: w = w.lower() @@ -101,25 +159,42 @@ def distinctive_words(text: str) -> set: return out -def expand_citation_spec(spec: str): - """'8618,8620,8622' -> [8618,8620,8622]; '8691-8717' -> [8691..8717].""" - nums = [] +class CitationRef: + """One citation token: 'label' is what the data actually wrote + ("8609" or "8609-8610"); 'lines' is the fully-expanded, sorted list of + line numbers the label names -- a single-element list for a bare + number, the whole inclusive range for an explicit wrap.""" + + __slots__ = ("label", "lines") + + def __init__(self, label, lines): + self.label = label + self.lines = lines + + +def parse_citation_spec(spec: str): + """'8618,8620,8622' -> three exact-line CitationRefs; '8609-8610' -> one + CitationRef spanning both lines (an explicit wrapped heading). Each + comma-separated token is independent; a malformed token (b < a, or a + span so wide it is almost certainly a typo, capped at 200 lines) is + silently dropped rather than crashing on bad data, matching this + script's existing tolerance elsewhere for data it does not own.""" + refs = [] for tok in spec.split(","): tok = tok.strip() m = re.fullmatch(r"(\d{2,6})-(\d{2,6})", tok) if m: a, b = int(m.group(1)), int(m.group(2)) if a <= b and (b - a) <= 200: - nums.extend(range(a, b + 1)) + refs.append(CitationRef(tok, list(range(a, b + 1)))) continue m = re.fullmatch(r"(\d{2,6})", tok) if m: - nums.append(int(m.group(1))) - return nums + refs.append(CitationRef(tok, [int(m.group(1))])) + return refs CITATION_RE = re.compile(r"LT\.txt:\s*((?:\d{2,6}(?:-\d{2,6})?)(?:\s*,\s*\d{2,6}(?:-\d{2,6})?)*)") -QUOTE_RE = re.compile(r'"([^"]{3,})"') def parse_blocks(la_ini_text: str): @@ -147,9 +222,23 @@ def parse_blocks(la_ini_text: str): return blocks +def window_text_for(lt_lines, lines): + lo, hi = lines[0], lines[-1] + lo_c, hi_c = max(1, lo), min(len(lt_lines), hi) + if lo_c > hi_c: + return "" + return " ".join(lt_lines[lo_c - 1 : hi_c]) + + def check(la_ini_text: str, lt_lines: list): - findings = [] + """Returns a dict: checked (int), passed (int), findings (list -- wrong + citations), unverifiable (list -- degenerate-pool citations). Both + findings and unverifiable are things a human must look at; only + 'passed' citations required no human attention.""" checked = 0 + passed = 0 + findings = [] + unverifiable = [] for block in parse_blocks(la_ini_text): block_comment = "\n".join(c for k, c in block if k == "comment") if "PATTERN" in block_comment: @@ -157,75 +246,126 @@ def check(la_ini_text: str, lt_lines: list): entries = [c for k, c in block if k == "entry"] if not entries: continue - quotes = QUOTE_RE.findall(block_comment) entries_before = [] for k, c in block: if k == "comment": for m in CITATION_RE.finditer(c): - nums = expand_citation_spec(m.group(1)) if entries_before: pool_entries = [entries_before[-1]] else: # leading citation: pool = every entry in the block - # (entries after this comment, i.e. all of them, - # since none has been seen yet) + # (no positional correspondence is encoded between + # a specific cited line and a specific entry). pool_entries = entries + entry_desc = ", ".join(f"{k}={v}" for k, v in pool_entries) # Pool items are kept SEPARATE (not flattened into one # bag of words): a citation passes only if the window # fully covers -- ALL the distinctive words of -- at - # least one single pool item (one quoted phrase, or one - # entry's own value). A flattened "any word from any - # pool item" bag is too permissive: it let a citation - # bundling two claims onto one line number ("D.NI - # NOSTRI JESU CHRISTI REGIS / I classis") pass on the - # strength of the FIRST half alone, even though the - # second half ("I classis") was not actually nearby -- - # exactly the shape of one of the two real citation - # bugs this script was written to catch. Verified by - # replay against the pre-fix file (see the task report). - pool_items = [distinctive_words(q) for q in quotes] - for _, v in pool_entries: - pool_items.append(distinctive_words(v)) - pool_items = [p for p in pool_items if p] - for n in nums: + # least one single pool item. See the module docstring + # for why (the self-poisoning bug and the two-claims- + # on-one-line-number bug this discipline catches). + pool_items = [distinctive_words(v) for _, v in pool_entries] + strong_items = [p for p in pool_items if len(p) >= MIN_DISTINCTIVE_WORDS] + best_len = max((len(p) for p in pool_items), default=0) + for ref in parse_citation_spec(m.group(1)): checked += 1 - lo, hi = max(1, n - 2), min(len(lt_lines), n + 2) - window_text = " ".join(lt_lines[lo - 1 : hi]) + window_text = window_text_for(lt_lines, ref.lines) window_words = distinctive_words(window_text) - if not any(item <= window_words for item in pool_items): - entry_desc = ", ".join(f"{k}={v}" for k, v in pool_entries) + if not strong_items: + unverifiable.append( + { + "label": ref.label, + "entries": entry_desc, + "best_len": best_len, + "pool_words": sorted(set().union(*pool_items)) if pool_items else [], + } + ) + continue + if any(item <= window_words for item in strong_items): + passed += 1 + else: + n = ref.lines[0] findings.append( { - "line": n, + "label": ref.label, "entries": entry_desc, - "actual": lt_lines[n - 1].strip() if 1 <= n <= len(lt_lines) else "(out of range)", + "actual": ( + lt_lines[n - 1].strip() + if 1 <= n <= len(lt_lines) + else "(out of range)" + ), "window": window_text.strip()[:160], } ) else: entries_before.append(c) - return checked, findings + return { + "checked": checked, + "passed": passed, + "findings": findings, + "unverifiable": unverifiable, + } -def main(): - if not LT_TXT.exists(): +def build_arg_parser(): + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + p.add_argument( + "--file", + dest="la_ini", + type=Path, + default=DEFAULT_LA_INI, + help=f"the la.ini-shaped file to check (default: {DEFAULT_LA_INI})", + ) + p.add_argument( + "--lt-file", + dest="lt_txt", + type=Path, + default=DEFAULT_LT_TXT, + help=f"the LT.txt transcription to check against (default: {DEFAULT_LT_TXT})", + ) + return p + + +def main(argv=None): + args = build_arg_parser().parse_args(argv) + if not args.lt_txt.exists(): print( - "SKIPPED: docs/research/LT.txt is absent (docs/ is gitignored -- " + f"SKIPPED: {args.lt_txt} is absent (docs/ is gitignored -- " "present locally only). Citations are NOT verified this run." ) return 0 - la_ini_text = LA_INI.read_text(encoding="utf-8") - lt_lines = LT_TXT.read_text(encoding="utf-8", errors="replace").split("\n") - checked, findings = check(la_ini_text, lt_lines) + la_ini_text = args.la_ini.read_text(encoding="utf-8") + lt_lines = args.lt_txt.read_text(encoding="utf-8", errors="replace").split("\n") + result = check(la_ini_text, lt_lines) + checked = result["checked"] + findings = result["findings"] + unverifiable = result["unverifiable"] + + if not findings and not unverifiable: + print(f"check-citations: {checked} LT.txt citations checked, 0 look wrong.") + return 0 + + print( + f"check-citations: {len(findings)} of {checked} citations look wrong, " + f"{len(unverifiable)} CANNOT VERIFY (pool too thin -- see below):\n" + ) if findings: - print(f"check-citations: {len(findings)} of {checked} citations look wrong:\n") + print("WRONG:\n") for f in findings: - print(f" LT.txt:{f['line']} cited for [{f['entries']}]") - print(f" actual line {f['line']}: {f['actual']!r}") - print(f" window (+-2): {f['window']!r}\n") - return 2 - print(f"check-citations: {checked} LT.txt citations checked, 0 look wrong.") - return 0 + print(f" LT.txt:{f['label']} cited for [{f['entries']}]") + print(f" actual: {f['actual']!r}") + print(f" window: {f['window']!r}\n") + if unverifiable: + print("CANNOT VERIFY (a human must adjudicate these by hand):\n") + for u in unverifiable: + words = ", ".join(u["pool_words"]) if u["pool_words"] else "(none)" + print(f" LT.txt:{u['label']} cited for [{u['entries']}]") + print( + f" pool too thin to verify: best candidate has " + f"{u['best_len']} distinctive word(s) (need >= {MIN_DISTINCTIVE_WORDS}); " + f"pool words: {words}\n" + ) + return 2 if __name__ == "__main__": |