#!/usr/bin/env python3
"""Regenerate the bulk of the Wujek (wuj) corpus from biblia.info.pl.

Why this exists: the original harvest silently lost text, because a verse is
not one <p>. Many psalms are printed as

    <p><span id="104:1"><sup>1</sup></span> Alleluja. <sup>[108]</sup></p>
    <p><big>W</big>yznawajcie Panu, i wzywajcie imienia jego, ...</p>

The second paragraph carries the rest of the verse and has NO anchor, so
taking one paragraph per anchor kept "Alleluja." and dropped the psalm's
opening line -- 44 psalms lost their first line that way, plus scattered
verses elsewhere (Acts 6:5 among them). A verse runs from its anchor to the
NEXT anchor, across however many paragraphs.

Four further traps, each of which silently lost or invented verses:

  * The anchor's shape differs per book: Psalms <sup>1</sup>, Sirach <sup></sup>
    (a drop cap replaces the numeral), Genesis no <sup> at all, and some carry
    "2&#160;". Constraining it to a digit dropped whole books.
  * Hidden page markers <span id="1003" style="display:none;"> open paragraphs
    mid-verse. Treated as verse starts they invent ~1000 verses; ignored at the
    structural level they swallow the real verse (this is what hid Acts 6:5).
  * The anchor's CHAPTER part is unreliable -- Mark labels 87 anchors "15:*"
    spanning chapters 14-16. The <h3> headings are correct, so headings win.
  * Psalms head their divisions "Psalm CXVII", every other book "Rozdzial N".
    Missing that put Psalm 118's 176 verses inside Psalm 117.

Tags are stripped WITHOUT inserting whitespace, or the <big>W</big> drop cap
yields "W yznawajcie".

This regenerates most of the corpus. It does NOT reproduce it exactly: the
committed wuj.tsv has since had chapter/verse structure repaired against the
Vulgate by hand (Acts 5, Psalms 113/114, 1 Chronicles 9/10 and a number of
merged verses), and carries passages this site does not hold at all. See
NOTICE. So --write is a starting point for a fresh harvest, not a way to
rebuild the shipped file.

Requires network; downloads are cached under scripts/.wujek-cache/ so a re-run
costs nothing. From the repo root:

    python3 scripts/scrape-wujek.py                 # report only
    python3 scripts/scrape-wujek.py --out FILE      # write elsewhere to diff
"""
import html
import os
import re
import sys
import time
import urllib.request

BASE = "https://www.biblia.info.pl/bibliawujka/"
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
CACHE = os.path.join(ROOT, "scripts", ".wujek-cache")
CORPUS = os.path.join(ROOT, "internal", "bible", "corpora_optional", "wuj.tsv")

FOOTNOTE = re.compile(r'<sup[^>]*class="reference"[^>]*>.*?</sup>', re.S)
ANCHOR = re.compile(r'<span id="(\d+):(\d+)"\s*>.*?</span>', re.S)
PAGEMARK = re.compile(r'<span id="\d+"\s+style="display:none;?"[^>]*>.*?</span>', re.S)
HEAD = re.compile(r"<h3[^>]*>(?:\s*<span[^>]*>)?\s*(?:Rozdzia[lł]|Psalm)\s+([IVXLC]+)\b", re.I)
SPANLED = re.compile(r"^\s*<span\b[^>]*>(.*?)</span>", re.S)
SUPNUM = re.compile(r"<sup[^>]*>\s*(\d+)", re.S)
BLOCK = re.compile(r"(<h3[^>]*>.*?</h3>|<p\b[^>]*>.*?</p>)", re.S)
TAG = re.compile(r"<[^>]+>")


def fetch(slug):
    os.makedirs(CACHE, exist_ok=True)
    path = os.path.join(CACHE, slug)
    if os.path.exists(path) and os.path.getsize(path) > 2000:
        return open(path, encoding="utf-8", errors="replace").read()
    req = urllib.request.Request(BASE + slug, headers={"User-Agent": "Mozilla/5.0"})
    with urllib.request.urlopen(req, timeout=40) as r:
        data = r.read().decode("utf-8", "replace")
    open(path, "w", encoding="utf-8").write(data)
    time.sleep(1.0)                        # be polite to a small volunteer site
    return data


def text_of(fragment):
    s = PAGEMARK.sub("", fragment)
    s = FOOTNOTE.sub("", s)
    s = ANCHOR.sub("", s)
    s = TAG.sub("", s)                     # no space inserted: keeps "Wyznawajcie"
    return re.sub(r"\s+", " ", html.unescape(s)).strip()


def roman(s):
    vals = {"I": 1, "V": 5, "X": 10, "L": 50, "C": 100}
    n = 0
    for i, c in enumerate(s.upper()):
        v = vals.get(c, 0)
        n += -v if i + 1 < len(s) and vals.get(s[i + 1].upper(), 0) > v else v
    return n


def parse(page):
    """-> [(chapter, verse, text)] for one book page."""
    out, cur, ch, last = [], None, None, None
    for blk in BLOCK.findall(page):
        h = HEAD.match(blk.strip())
        if h:
            ch, last = roman(h.group(1)), None
            continue
        blk = PAGEMARK.sub("", blk)        # before the structural test, not after
        led = SPANLED.match(blk[blk.find(">") + 1:])
        m = ANCHOR.search(blk)
        c = v = None
        if led is not None:
            if m and m.group(1).isdigit():
                v = int(m.group(2))
                c = ch if ch is not None else int(m.group(1))
            elif "<sup" in led.group(1):
                sm = SUPNUM.search(led.group(1))
                c = ch
                v = int(sm.group(1)) if sm else (1 if last is None else last + 1)
        if c is None:                       # continuation, or text before ch. 1
            if cur is not None:
                extra = text_of(blk)
                if extra:
                    cur[2] = (cur[2] + " " + extra).strip()
            continue
        if last is not None and v <= 2 and last >= 5 and c == ch:
            c = ch = ch + 1                 # a heading we still failed to see
        if cur:
            out.append(cur)
        body = blk[m.end():] if (m and m.group(1).isdigit()) else blk
        cur = [c, v, text_of(body)]
        ch, last = c, v
    if cur:
        out.append(cur)
    return out


def schema():
    """(num, name, abbrev) per book, from the corpus this regenerates."""
    seen, out = set(), []
    for line in open(CORPUS, encoding="utf-8"):
        f = line.rstrip("\n").split("\t")
        if len(f) == 6 and f[0] not in seen:
            seen.add(f[0])
            out.append((int(f[2]), f[0], f[1]))
    return sorted(out)


def main():
    idx = fetch("index.html")
    slugs, seen = [], set()
    for u, _ in re.findall(r'href=[\'"]([A-Za-z0-9]+\.html)[\'"][^>]*>([^<]{1,40})<', idx):
        if u != "index.html" and u not in seen:
            seen.add(u)
            slugs.append(u)
    sch = schema()
    if len(slugs) != len(sch):
        sys.exit("site lists %d books, corpus has %d" % (len(slugs), len(sch)))

    rows = []
    for slug, (num, name, abbr) in zip(slugs, sch):
        vs = parse(fetch(slug))
        for ch, v, t in vs:
            if t:
                rows.append("%s\t%s\t%d\t%d\t%d\t%s" % (name, abbr, num, ch, v, t))
        print("  %-17s %-10s %5d verses" % (name, slug, len(vs)), file=sys.stderr)

    print("\n%d rows" % len(rows), file=sys.stderr)
    args = sys.argv[1:]
    if "--out" not in args:
        print("not written; pass --out FILE to save (see the module docstring "
              "before overwriting the corpus)", file=sys.stderr)
        return
    dest = args[args.index("--out") + 1]
    with open(dest, "w", encoding="utf-8") as f:
        f.write("\n".join(rows) + "\n")
    print("wrote %s" % dest, file=sys.stderr)


main()