From 390535ff00d61df63a15de6fba23761a2c57cead Mon Sep 17 00:00:00 2001
From: Lukasz Kasprzak <lukas@labunix.xyz>
Date: Wed, 29 Jul 2026 11:12:24 +0200
Subject: bible(wuj): rebuild from source, repair to exact Vulgate parity

The Wujek corpus was missing text and mis-numbering what it had: 453
corpus-check warnings, 8 absent chapters, 84 duplicate rows, and 357 chapters
holding fewer verses than the Vulgate.

Most of it was never a sourcing problem. The original harvest of
biblia.info.pl assumed one verse is one <p>, but many psalms print the
superscription in the anchored paragraph and the psalm's opening line in an
unanchored one behind a drop cap -- so 44 psalms lost their first line, and
Acts 6:5 and others went the same way. Re-harvested with the anchor treated as
a verse START rather than a whole verse (scripts/scrape-wujek.py), which also
had to cope with four anchor shapes across books, hidden page markers opening
paragraphs mid-verse, chapter ids that are simply wrong (Mark labels 87
anchors "15:*" across chapters 14-16), and Psalms heading its divisions
"Psalm CXVII" where every other book says "Rozdzial N".

Structure was then repaired against the Clementine Vulgate, by hand where a
rule would have guessed:

  * The Acts 5 interleaved three streams -- Acts 4:27-37 duplicated, the real
    5:1-37, and 5:38-39 mislabelled 28-29. Verified against the Latin, the
    duplicates dropped, the two renumbered. Acts 4:27 kept the site's own
    wording in place of a 1962 reading, removing a seam.
  * The site heads psalms by Hebrew division; its "Psalm 114" is two psalms.
    Split into Vulgate 113:9-26 and 114:1-9, confirmed at all four boundaries.
  * A corrupt anchor for 1 Chronicles 9:11 pushed 34 paragraphs of genealogy
    into chapter 10. Counts corroborate: 10 + 34 = 44 verses, remainder 14,
    both exactly the Vulgate's.
  * 16 chapters looked short at the end; only two verses were truly absent.
    The rest were merges, split at anchors read off the Latin -- including one
    across a chapter boundary (Colossians 4:1 sat inside 3:25) and four
    numbering offsets where a mid-chapter merge shifted everything after it.

wuj now carries exactly the Vulgate's verse set: 35810 rows, every one of the
1334 chapters matching, no duplicates, no gaps. corpus-check: 0 warnings.

The ~300 verses that could not come from biblia.info.pl are recorded in
NOTICE; they are not public domain.
---
 scripts/scrape-wujek.py | 177 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 177 insertions(+)
 create mode 100644 scripts/scrape-wujek.py

(limited to 'scripts/scrape-wujek.py')

diff --git a/scripts/scrape-wujek.py b/scripts/scrape-wujek.py
new file mode 100644
index 0000000..4940fdd
--- /dev/null
+++ b/scripts/scrape-wujek.py
@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+"""Regenerate the bulk of the Wujek (wuj) corpus from biblia.info.pl.
+
+Why this exists: the original harvest silently lost text, because a verse is
+not one <p>. Many psalms are printed as
+
+    <p><span id="104:1"><sup>1</sup></span> Alleluja. <sup>[108]</sup></p>
+    <p><big>W</big>yznawajcie Panu, i wzywajcie imienia jego, ...</p>
+
+The second paragraph carries the rest of the verse and has NO anchor, so
+taking one paragraph per anchor kept "Alleluja." and dropped the psalm's
+opening line -- 44 psalms lost their first line that way, plus scattered
+verses elsewhere (Acts 6:5 among them). A verse runs from its anchor to the
+NEXT anchor, across however many paragraphs.
+
+Four further traps, each of which silently lost or invented verses:
+
+  * The anchor's shape differs per book: Psalms <sup>1</sup>, Sirach <sup></sup>
+    (a drop cap replaces the numeral), Genesis no <sup> at all, and some carry
+    "2&#160;". Constraining it to a digit dropped whole books.
+  * Hidden page markers <span id="1003" style="display:none;"> open paragraphs
+    mid-verse. Treated as verse starts they invent ~1000 verses; ignored at the
+    structural level they swallow the real verse (this is what hid Acts 6:5).
+  * The anchor's CHAPTER part is unreliable -- Mark labels 87 anchors "15:*"
+    spanning chapters 14-16. The <h3> headings are correct, so headings win.
+  * Psalms head their divisions "Psalm CXVII", every other book "Rozdzial N".
+    Missing that put Psalm 118's 176 verses inside Psalm 117.
+
+Tags are stripped WITHOUT inserting whitespace, or the <big>W</big> drop cap
+yields "W yznawajcie".
+
+This regenerates most of the corpus. It does NOT reproduce it exactly: the
+committed wuj.tsv has since had chapter/verse structure repaired against the
+Vulgate by hand (Acts 5, Psalms 113/114, 1 Chronicles 9/10 and a number of
+merged verses), and carries passages this site does not hold at all. See
+NOTICE. So --write is a starting point for a fresh harvest, not a way to
+rebuild the shipped file.
+
+Requires network; downloads are cached under scripts/.wujek-cache/ so a re-run
+costs nothing. From the repo root:
+
+    python3 scripts/scrape-wujek.py                 # report only
+    python3 scripts/scrape-wujek.py --out FILE      # write elsewhere to diff
+"""
+import html
+import os
+import re
+import sys
+import time
+import urllib.request
+
+BASE = "https://www.biblia.info.pl/bibliawujka/"
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+CACHE = os.path.join(ROOT, "scripts", ".wujek-cache")
+CORPUS = os.path.join(ROOT, "internal", "bible", "corpora_optional", "wuj.tsv")
+
+FOOTNOTE = re.compile(r'<sup[^>]*class="reference"[^>]*>.*?</sup>', re.S)
+ANCHOR = re.compile(r'<span id="(\d+):(\d+)"\s*>.*?</span>', re.S)
+PAGEMARK = re.compile(r'<span id="\d+"\s+style="display:none;?"[^>]*>.*?</span>', re.S)
+HEAD = re.compile(r"<h3[^>]*>(?:\s*<span[^>]*>)?\s*(?:Rozdzia[lł]|Psalm)\s+([IVXLC]+)\b", re.I)
+SPANLED = re.compile(r"^\s*<span\b[^>]*>(.*?)</span>", re.S)
+SUPNUM = re.compile(r"<sup[^>]*>\s*(\d+)", re.S)
+BLOCK = re.compile(r"(<h3[^>]*>.*?</h3>|<p\b[^>]*>.*?</p>)", re.S)
+TAG = re.compile(r"<[^>]+>")
+
+
+def fetch(slug):
+    os.makedirs(CACHE, exist_ok=True)
+    path = os.path.join(CACHE, slug)
+    if os.path.exists(path) and os.path.getsize(path) > 2000:
+        return open(path, encoding="utf-8", errors="replace").read()
+    req = urllib.request.Request(BASE + slug, headers={"User-Agent": "Mozilla/5.0"})
+    with urllib.request.urlopen(req, timeout=40) as r:
+        data = r.read().decode("utf-8", "replace")
+    open(path, "w", encoding="utf-8").write(data)
+    time.sleep(1.0)                        # be polite to a small volunteer site
+    return data
+
+
+def text_of(fragment):
+    s = PAGEMARK.sub("", fragment)
+    s = FOOTNOTE.sub("", s)
+    s = ANCHOR.sub("", s)
+    s = TAG.sub("", s)                     # no space inserted: keeps "Wyznawajcie"
+    return re.sub(r"\s+", " ", html.unescape(s)).strip()
+
+
+def roman(s):
+    vals = {"I": 1, "V": 5, "X": 10, "L": 50, "C": 100}
+    n = 0
+    for i, c in enumerate(s.upper()):
+        v = vals.get(c, 0)
+        n += -v if i + 1 < len(s) and vals.get(s[i + 1].upper(), 0) > v else v
+    return n
+
+
+def parse(page):
+    """-> [(chapter, verse, text)] for one book page."""
+    out, cur, ch, last = [], None, None, None
+    for blk in BLOCK.findall(page):
+        h = HEAD.match(blk.strip())
+        if h:
+            ch, last = roman(h.group(1)), None
+            continue
+        blk = PAGEMARK.sub("", blk)        # before the structural test, not after
+        led = SPANLED.match(blk[blk.find(">") + 1:])
+        m = ANCHOR.search(blk)
+        c = v = None
+        if led is not None:
+            if m and m.group(1).isdigit():
+                v = int(m.group(2))
+                c = ch if ch is not None else int(m.group(1))
+            elif "<sup" in led.group(1):
+                sm = SUPNUM.search(led.group(1))
+                c = ch
+                v = int(sm.group(1)) if sm else (1 if last is None else last + 1)
+        if c is None:                       # continuation, or text before ch. 1
+            if cur is not None:
+                extra = text_of(blk)
+                if extra:
+                    cur[2] = (cur[2] + " " + extra).strip()
+            continue
+        if last is not None and v <= 2 and last >= 5 and c == ch:
+            c = ch = ch + 1                 # a heading we still failed to see
+        if cur:
+            out.append(cur)
+        body = blk[m.end():] if (m and m.group(1).isdigit()) else blk
+        cur = [c, v, text_of(body)]
+        ch, last = c, v
+    if cur:
+        out.append(cur)
+    return out
+
+
+def schema():
+    """(num, name, abbrev) per book, from the corpus this regenerates."""
+    seen, out = set(), []
+    for line in open(CORPUS, encoding="utf-8"):
+        f = line.rstrip("\n").split("\t")
+        if len(f) == 6 and f[0] not in seen:
+            seen.add(f[0])
+            out.append((int(f[2]), f[0], f[1]))
+    return sorted(out)
+
+
+def main():
+    idx = fetch("index.html")
+    slugs, seen = [], set()
+    for u, _ in re.findall(r'href=[\'"]([A-Za-z0-9]+\.html)[\'"][^>]*>([^<]{1,40})<', idx):
+        if u != "index.html" and u not in seen:
+            seen.add(u)
+            slugs.append(u)
+    sch = schema()
+    if len(slugs) != len(sch):
+        sys.exit("site lists %d books, corpus has %d" % (len(slugs), len(sch)))
+
+    rows = []
+    for slug, (num, name, abbr) in zip(slugs, sch):
+        vs = parse(fetch(slug))
+        for ch, v, t in vs:
+            if t:
+                rows.append("%s\t%s\t%d\t%d\t%d\t%s" % (name, abbr, num, ch, v, t))
+        print("  %-17s %-10s %5d verses" % (name, slug, len(vs)), file=sys.stderr)
+
+    print("\n%d rows" % len(rows), file=sys.stderr)
+    args = sys.argv[1:]
+    if "--out" not in args:
+        print("not written; pass --out FILE to save (see the module docstring "
+              "before overwriting the corpus)", file=sys.stderr)
+        return
+    dest = args[args.index("--out") + 1]
+    with open(dest, "w", encoding="utf-8") as f:
+        f.write("\n".join(rows) + "\n")
+    print("wrote %s" % dest, file=sys.stderr)
+
+
+main()
-- 
cgit v1.3