1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
|
#!/usr/bin/env python3
"""Regenerate the bulk of the Wujek (wuj) corpus from biblia.info.pl.
Why this exists: the original harvest silently lost text, because a verse is
not one <p>. Many psalms are printed as
<p><span id="104:1"><sup>1</sup></span> Alleluja. <sup>[108]</sup></p>
<p><big>W</big>yznawajcie Panu, i wzywajcie imienia jego, ...</p>
The second paragraph carries the rest of the verse and has NO anchor, so
taking one paragraph per anchor kept "Alleluja." and dropped the psalm's
opening line -- 44 psalms lost their first line that way, plus scattered
verses elsewhere (Acts 6:5 among them). A verse runs from its anchor to the
NEXT anchor, across however many paragraphs.
Four further traps, each of which silently lost or invented verses:
* The anchor's shape differs per book: Psalms <sup>1</sup>, Sirach <sup></sup>
(a drop cap replaces the numeral), Genesis no <sup> at all, and some carry
"2 ". Constraining it to a digit dropped whole books.
* Hidden page markers <span id="1003" style="display:none;"> open paragraphs
mid-verse. Treated as verse starts they invent ~1000 verses; ignored at the
structural level they swallow the real verse (this is what hid Acts 6:5).
* The anchor's CHAPTER part is unreliable -- Mark labels 87 anchors "15:*"
spanning chapters 14-16. The <h3> headings are correct, so headings win.
* Psalms head their divisions "Psalm CXVII", every other book "Rozdzial N".
Missing that put Psalm 118's 176 verses inside Psalm 117.
Tags are stripped WITHOUT inserting whitespace, or the <big>W</big> drop cap
yields "W yznawajcie".
This regenerates most of the corpus. It does NOT reproduce it exactly: the
committed wuj.tsv has since had chapter/verse structure repaired against the
Vulgate by hand (Acts 5, Psalms 113/114, 1 Chronicles 9/10 and a number of
merged verses), and carries passages this site does not hold at all. See
NOTICE. So --write is a starting point for a fresh harvest, not a way to
rebuild the shipped file.
Requires network; downloads are cached under scripts/.wujek-cache/ so a re-run
costs nothing. From the repo root:
python3 scripts/scrape-wujek.py # report only
python3 scripts/scrape-wujek.py --out FILE # write elsewhere to diff
"""
import html
import os
import re
import sys
import time
import urllib.request
BASE = "https://www.biblia.info.pl/bibliawujka/"
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
CACHE = os.path.join(ROOT, "scripts", ".wujek-cache")
CORPUS = os.path.join(ROOT, "internal", "bible", "corpora_optional", "wuj.tsv")
FOOTNOTE = re.compile(r'<sup[^>]*class="reference"[^>]*>.*?</sup>', re.S)
ANCHOR = re.compile(r'<span id="(\d+):(\d+)"\s*>.*?</span>', re.S)
PAGEMARK = re.compile(r'<span id="\d+"\s+style="display:none;?"[^>]*>.*?</span>', re.S)
HEAD = re.compile(r"<h3[^>]*>(?:\s*<span[^>]*>)?\s*(?:Rozdzia[lł]|Psalm)\s+([IVXLC]+)\b", re.I)
SPANLED = re.compile(r"^\s*<span\b[^>]*>(.*?)</span>", re.S)
SUPNUM = re.compile(r"<sup[^>]*>\s*(\d+)", re.S)
BLOCK = re.compile(r"(<h3[^>]*>.*?</h3>|<p\b[^>]*>.*?</p>)", re.S)
TAG = re.compile(r"<[^>]+>")
def fetch(slug):
os.makedirs(CACHE, exist_ok=True)
path = os.path.join(CACHE, slug)
if os.path.exists(path) and os.path.getsize(path) > 2000:
return open(path, encoding="utf-8", errors="replace").read()
req = urllib.request.Request(BASE + slug, headers={"User-Agent": "Mozilla/5.0"})
with urllib.request.urlopen(req, timeout=40) as r:
data = r.read().decode("utf-8", "replace")
open(path, "w", encoding="utf-8").write(data)
time.sleep(1.0) # be polite to a small volunteer site
return data
def text_of(fragment):
s = PAGEMARK.sub("", fragment)
s = FOOTNOTE.sub("", s)
s = ANCHOR.sub("", s)
s = TAG.sub("", s) # no space inserted: keeps "Wyznawajcie"
return re.sub(r"\s+", " ", html.unescape(s)).strip()
def roman(s):
vals = {"I": 1, "V": 5, "X": 10, "L": 50, "C": 100}
n = 0
for i, c in enumerate(s.upper()):
v = vals.get(c, 0)
n += -v if i + 1 < len(s) and vals.get(s[i + 1].upper(), 0) > v else v
return n
def parse(page):
"""-> [(chapter, verse, text)] for one book page."""
out, cur, ch, last = [], None, None, None
for blk in BLOCK.findall(page):
h = HEAD.match(blk.strip())
if h:
ch, last = roman(h.group(1)), None
continue
blk = PAGEMARK.sub("", blk) # before the structural test, not after
led = SPANLED.match(blk[blk.find(">") + 1:])
m = ANCHOR.search(blk)
c = v = None
if led is not None:
if m and m.group(1).isdigit():
v = int(m.group(2))
c = ch if ch is not None else int(m.group(1))
elif "<sup" in led.group(1):
sm = SUPNUM.search(led.group(1))
c = ch
v = int(sm.group(1)) if sm else (1 if last is None else last + 1)
if c is None: # continuation, or text before ch. 1
if cur is not None:
extra = text_of(blk)
if extra:
cur[2] = (cur[2] + " " + extra).strip()
continue
if last is not None and v <= 2 and last >= 5 and c == ch:
c = ch = ch + 1 # a heading we still failed to see
if cur:
out.append(cur)
body = blk[m.end():] if (m and m.group(1).isdigit()) else blk
cur = [c, v, text_of(body)]
ch, last = c, v
if cur:
out.append(cur)
return out
def schema():
"""(num, name, abbrev) per book, from the corpus this regenerates."""
seen, out = set(), []
for line in open(CORPUS, encoding="utf-8"):
f = line.rstrip("\n").split("\t")
if len(f) == 6 and f[0] not in seen:
seen.add(f[0])
out.append((int(f[2]), f[0], f[1]))
return sorted(out)
def main():
idx = fetch("index.html")
slugs, seen = [], set()
for u, _ in re.findall(r'href=[\'"]([A-Za-z0-9]+\.html)[\'"][^>]*>([^<]{1,40})<', idx):
if u != "index.html" and u not in seen:
seen.add(u)
slugs.append(u)
sch = schema()
if len(slugs) != len(sch):
sys.exit("site lists %d books, corpus has %d" % (len(slugs), len(sch)))
rows = []
for slug, (num, name, abbr) in zip(slugs, sch):
vs = parse(fetch(slug))
for ch, v, t in vs:
if t:
rows.append("%s\t%s\t%d\t%d\t%d\t%s" % (name, abbr, num, ch, v, t))
print(" %-17s %-10s %5d verses" % (name, slug, len(vs)), file=sys.stderr)
print("\n%d rows" % len(rows), file=sys.stderr)
args = sys.argv[1:]
if "--out" not in args:
print("not written; pass --out FILE to save (see the module docstring "
"before overwriting the corpus)", file=sys.stderr)
return
dest = args[args.index("--out") + 1]
with open(dest, "w", encoding="utf-8") as f:
f.write("\n".join(rows) + "\n")
print("wrote %s" % dest, file=sys.stderr)
main()
|
