1 files changed, 177 insertions, 0 deletions
diff --git a/scripts/scrape-wujek.py b/scripts/scrape-wujek.py
new file mode 100644
index 0000000..4940fdd
--- /dev/null
+++ b/scripts/scrape-wujek.py
@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+"""Regenerate the bulk of the Wujek (wuj) corpus from biblia.info.pl.
+
+Why this exists: the original harvest silently lost text, because a verse is
+not one <p>. Many psalms are printed as
+
+    <p><span id="104:1"><sup>1</sup></span> Alleluja. <sup>[108]</sup></p>
+    <p><big>W</big>yznawajcie Panu, i wzywajcie imienia jego, ...</p>
+
+The second paragraph carries the rest of the verse and has NO anchor, so
+taking one paragraph per anchor kept "Alleluja." and dropped the psalm's
+opening line -- 44 psalms lost their first line that way, plus scattered
+verses elsewhere (Acts 6:5 among them). A verse runs from its anchor to the
+NEXT anchor, across however many paragraphs.
+
+Four further traps, each of which silently lost or invented verses:
+
+  * The anchor's shape differs per book: Psalms <sup>1</sup>, Sirach <sup></sup>
+    (a drop cap replaces the numeral), Genesis no <sup> at all, and some carry
+    "2&#160;". Constraining it to a digit dropped whole books.
+  * Hidden page markers <span id="1003" style="display:none;"> open paragraphs
+    mid-verse. Treated as verse starts they invent ~1000 verses; ignored at the
+    structural level they swallow the real verse (this is what hid Acts 6:5).
+  * The anchor's CHAPTER part is unreliable -- Mark labels 87 anchors "15:*"
+    spanning chapters 14-16. The <h3> headings are correct, so headings win.
+  * Psalms head their divisions "Psalm CXVII", every other book "Rozdzial N".
+    Missing that put Psalm 118's 176 verses inside Psalm 117.
+
+Tags are stripped WITHOUT inserting whitespace, or the <big>W</big> drop cap
+yields "W yznawajcie".
+
+This regenerates most of the corpus. It does NOT reproduce it exactly: the
+committed wuj.tsv has since had chapter/verse structure repaired against the
+Vulgate by hand (Acts 5, Psalms 113/114, 1 Chronicles 9/10 and a number of
+merged verses), and carries passages this site does not hold at all. See
+NOTICE. So --write is a starting point for a fresh harvest, not a way to
+rebuild the shipped file.
+
+Requires network; downloads are cached under scripts/.wujek-cache/ so a re-run
+costs nothing. From the repo root:
+
+    python3 scripts/scrape-wujek.py                 # report only
+    python3 scripts/scrape-wujek.py --out FILE      # write elsewhere to diff
+"""
+import html
+import os
+import re
+import sys
+import time
+import urllib.request
+
+BASE = "https://www.biblia.info.pl/bibliawujka/"
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+CACHE = os.path.join(ROOT, "scripts", ".wujek-cache")
+CORPUS = os.path.join(ROOT, "internal", "bible", "corpora_optional", "wuj.tsv")
+
+FOOTNOTE = re.compile(r'<sup[^>]*class="reference"[^>]*>.*?</sup>', re.S)
+ANCHOR = re.compile(r'<span id="(\d+):(\d+)"\s*>.*?</span>', re.S)
+PAGEMARK = re.compile(r'<span id="\d+"\s+style="display:none;?"[^>]*>.*?</span>', re.S)
+HEAD = re.compile(r"<h3[^>]*>(?:\s*<span[^>]*>)?\s*(?:Rozdzia[lł]|Psalm)\s+([IVXLC]+)\b", re.I)
+SPANLED = re.compile(r"^\s*<span\b[^>]*>(.*?)</span>", re.S)
+SUPNUM = re.compile(r"<sup[^>]*>\s*(\d+)", re.S)
+BLOCK = re.compile(r"(<h3[^>]*>.*?</h3>|<p\b[^>]*>.*?</p>)", re.S)
+TAG = re.compile(r"<[^>]+>")
+
+
+def fetch(slug):
+    os.makedirs(CACHE, exist_ok=True)
+    path = os.path.join(CACHE, slug)
+    if os.path.exists(path) and os.path.getsize(path) > 2000:
+        return open(path, encoding="utf-8", errors="replace").read()
+    req = urllib.request.Request(BASE + slug, headers={"User-Agent": "Mozilla/5.0"})
+    with urllib.request.urlopen(req, timeout=40) as r:
+        data = r.read().decode("utf-8", "replace")
+    open(path, "w", encoding="utf-8").write(data)
+    time.sleep(1.0)                        # be polite to a small volunteer site
+    return data
+
+
+def text_of(fragment):
+    s = PAGEMARK.sub("", fragment)
+    s = FOOTNOTE.sub("", s)
+    s = ANCHOR.sub("", s)
+    s = TAG.sub("", s)                     # no space inserted: keeps "Wyznawajcie"
+    return re.sub(r"\s+", " ", html.unescape(s)).strip()
+
+
+def roman(s):
+    vals = {"I": 1, "V": 5, "X": 10, "L": 50, "C": 100}
+    n = 0
+    for i, c in enumerate(s.upper()):
+        v = vals.get(c, 0)
+        n += -v if i + 1 < len(s) and vals.get(s[i + 1].upper(), 0) > v else v
+    return n
+
+
+def parse(page):
+    """-> [(chapter, verse, text)] for one book page."""
+    out, cur, ch, last = [], None, None, None
+    for blk in BLOCK.findall(page):
+        h = HEAD.match(blk.strip())
+        if h:
+            ch, last = roman(h.group(1)), None
+            continue
+        blk = PAGEMARK.sub("", blk)        # before the structural test, not after
+        led = SPANLED.match(blk[blk.find(">") + 1:])
+        m = ANCHOR.search(blk)
+        c = v = None
+        if led is not None:
+            if m and m.group(1).isdigit():
+                v = int(m.group(2))
+                c = ch if ch is not None else int(m.group(1))
+            elif "<sup" in led.group(1):
+                sm = SUPNUM.search(led.group(1))
+                c = ch
+                v = int(sm.group(1)) if sm else (1 if last is None else last + 1)
+        if c is None:                       # continuation, or text before ch. 1
+            if cur is not None:
+                extra = text_of(blk)
+                if extra:
+                    cur[2] = (cur[2] + " " + extra).strip()
+            continue
+        if last is not None and v <= 2 and last >= 5 and c == ch:
+            c = ch = ch + 1                 # a heading we still failed to see
+        if cur:
+            out.append(cur)
+        body = blk[m.end():] if (m and m.group(1).isdigit()) else blk
+        cur = [c, v, text_of(body)]
+        ch, last = c, v
+    if cur:
+        out.append(cur)
+    return out
+
+
+def schema():
+    """(num, name, abbrev) per book, from the corpus this regenerates."""
+    seen, out = set(), []
+    for line in open(CORPUS, encoding="utf-8"):
+        f = line.rstrip("\n").split("\t")
+        if len(f) == 6 and f[0] not in seen:
+            seen.add(f[0])
+            out.append((int(f[2]), f[0], f[1]))
+    return sorted(out)
+
+
+def main():
+    idx = fetch("index.html")
+    slugs, seen = [], set()
+    for u, _ in re.findall(r'href=[\'"]([A-Za-z0-9]+\.html)[\'"][^>]*>([^<]{1,40})<', idx):
+        if u != "index.html" and u not in seen:
+            seen.add(u)
+            slugs.append(u)
+    sch = schema()
+    if len(slugs) != len(sch):
+        sys.exit("site lists %d books, corpus has %d" % (len(slugs), len(sch)))
+
+    rows = []
+    for slug, (num, name, abbr) in zip(slugs, sch):
+        vs = parse(fetch(slug))
+        for ch, v, t in vs:
+            if t:
+                rows.append("%s\t%s\t%d\t%d\t%d\t%s" % (name, abbr, num, ch, v, t))
+        print("  %-17s %-10s %5d verses" % (name, slug, len(vs)), file=sys.stderr)
+
+    print("\n%d rows" % len(rows), file=sys.stderr)
+    args = sys.argv[1:]
+    if "--out" not in args:
+        print("not written; pass --out FILE to save (see the module docstring "
+              "before overwriting the corpus)", file=sys.stderr)
+        return
+    dest = args[args.index("--out") + 1]
+    with open(dest, "w", encoding="utf-8") as f:
+        f.write("\n".join(rows) + "\n")
+    print("wrote %s" % dest, file=sys.stderr)
+
+
+main()