blob: e9d6f63a4b736e6c090e796957cb4af105e4b411 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
|
#!/bin/sh
# snapshot-sources.sh -- archive the raw upstream data the generator scripts
# fetch, so lectio's liturgical data can be regenerated and re-audited OFFLINE
# even if the third-party sites change or disappear (the 25-year-longevity aim).
#
# Idempotent/resumable: an existing non-empty target file is skipped, so a
# re-run only fills gaps. Every fetch is appended to sources/manifest.tsv
# (path <TAB> url <TAB> sha256 <TAB> bytes). Polite: a real UA + a short delay.
#
# Coverage is deliberately BOUNDED (see sources/README.md); widen the missalemeum
# span or add providers by editing the ranges below and re-running.
#
# Usage: sh scripts/snapshot-sources.sh [provider ...]
# providers: cr litcal getbible missalemeum (default: all)
set -eu
ROOT=$(cd "$(dirname "$0")/.." && pwd)
SNAP="$ROOT/sources"
MAN="$SNAP/manifest.tsv"
UA='lectio-source-archive (offline liturgical calendar; contact repo owner)'
# catholic-resources.org's WAF answers 465 to unusual UAs, so it gets a plain
# browser UA; every other provider accepts the descriptive archive UA above.
BROWSER_UA='Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0'
CUR_UA="$UA"
SLEEP=0.15
mkdir -p "$SNAP"
[ -f "$MAN" ] || printf 'path\turl\tsha256\tbytes\n' > "$MAN"
# get URL -> relative path under sources/ (skip if already archived non-empty).
get() {
url=$1; rel=$2; dst="$SNAP/$rel"
if [ -s "$dst" ]; then return 0; fi
mkdir -p "$(dirname "$dst")"
code=$(curl -sSL -A "$CUR_UA" --max-time 30 -o "$dst.tmp" -w '%{http_code}' "$url" 2>/dev/null || echo 000)
if [ "$code" = 200 ] && [ -s "$dst.tmp" ]; then
mv "$dst.tmp" "$dst"
printf '%s\t%s\t%s\t%s\n' "$rel" "$url" "$(sha256sum "$dst" | cut -d' ' -f1)" "$(wc -c < "$dst")" >> "$MAN"
printf ' + %s\n' "$rel"
else
rm -f "$dst.tmp"
printf ' ! %s (HTTP %s)\n' "$rel" "$code" >&2
fi
sleep "$SLEEP"
}
snap_cr() {
echo "catholic-resources.org (OF lectionary):"
CUR_UA="$BROWSER_UA"
base=https://www.catholic-resources.org/Lectionary
# Sunday cycles + seasons (1998 USL) and weekday/sanctoral tables (2002 USL).
for p in OrdinaryA OrdinaryB OrdinaryC Advent Christmas Lent Easter Solemnities; do
get "$base/1998USL-$p.htm" "catholic-resources/1998USL-$p.htm"
done
for p in Sanctoral Weekdays-OT-I Weekdays-OT-II Weekdays-AdventChristmas Weekdays-Lent Weekdays-Easter; do
get "$base/2002USL-$p.htm" "catholic-resources/2002USL-$p.htm"
done
}
snap_litcal() {
echo "litcal (OF calendar oracle, whole-year JSON):"
for y in 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035; do
get "https://litcal.johnromanodorazio.com/api/dev/calendar/$y?locale=en" "litcal/calendar-$y-en.json"
done
}
snap_getbible() {
echo "get.bible (Douay-Rheims deuterocanon source):"
# The books + chapter counts gen-deutero.py backfills (getbible douayrheims
# book numbers). Tobit 69, Judith 70, Wisdom 73, Sirach 74, Baruch 75,
# 1/2 Maccabees 80/81; plus Daniel 13-14 / Esther additions live in-book.
for spec in "69:14" "70:16" "73:19" "74:51" "75:6" "80:16" "81:15"; do
nr=${spec%:*}; last=${spec#*:}
ch=1
while [ "$ch" -le "$last" ]; do
get "https://api.getbible.net/v2/douayrheims/$nr/$ch.json" "getbible/douayrheims-$nr-$ch.json"
ch=$((ch + 1))
done
done
}
snap_missalemeum() {
# EF calendar + readings + Polish saint names. Per-date only; BOUNDED to a
# representative span (widen MM_FROM/MM_TO to reproduce the full harvest).
MM_FROM=${MM_FROM:-2026-01-01}
MM_TO=${MM_TO:-2027-12-31}
echo "missalemeum (EF propers, $MM_FROM .. $MM_TO, en+pl):"
d=$MM_FROM
end=$(date -I -d "$MM_TO +1 day")
while [ "$d" != "$end" ]; do
for lang in en pl; do
get "https://www.missalemeum.com/$lang/api/v5/proper/$d" "missalemeum/$lang/$d.json"
done
d=$(date -I -d "$d +1 day")
done
}
which=${*:-cr litcal getbible missalemeum}
for p in $which; do
case "$p" in
cr) snap_cr ;;
litcal) snap_litcal ;;
getbible) snap_getbible ;;
missalemeum) snap_missalemeum ;;
*) echo "unknown provider: $p" >&2 ;;
esac
done
echo "done. manifest: $MAN"
|
