diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/snapshot-sources.sh | 108 |
1 files changed, 108 insertions, 0 deletions
diff --git a/scripts/snapshot-sources.sh b/scripts/snapshot-sources.sh new file mode 100755 index 0000000..e9d6f63 --- /dev/null +++ b/scripts/snapshot-sources.sh @@ -0,0 +1,108 @@ +#!/bin/sh +# snapshot-sources.sh -- archive the raw upstream data the generator scripts +# fetch, so lectio's liturgical data can be regenerated and re-audited OFFLINE +# even if the third-party sites change or disappear (the 25-year-longevity aim). +# +# Idempotent/resumable: an existing non-empty target file is skipped, so a +# re-run only fills gaps. Every fetch is appended to sources/manifest.tsv +# (path <TAB> url <TAB> sha256 <TAB> bytes). Polite: a real UA + a short delay. +# +# Coverage is deliberately BOUNDED (see sources/README.md); widen the missalemeum +# span or add providers by editing the ranges below and re-running. +# +# Usage: sh scripts/snapshot-sources.sh [provider ...] +# providers: cr litcal getbible missalemeum (default: all) +set -eu + +ROOT=$(cd "$(dirname "$0")/.." && pwd) +SNAP="$ROOT/sources" +MAN="$SNAP/manifest.tsv" +UA='lectio-source-archive (offline liturgical calendar; contact repo owner)' +# catholic-resources.org's WAF answers 465 to unusual UAs, so it gets a plain +# browser UA; every other provider accepts the descriptive archive UA above. +BROWSER_UA='Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0' +CUR_UA="$UA" +SLEEP=0.15 + +mkdir -p "$SNAP" +[ -f "$MAN" ] || printf 'path\turl\tsha256\tbytes\n' > "$MAN" + +# get URL -> relative path under sources/ (skip if already archived non-empty). +get() { + url=$1; rel=$2; dst="$SNAP/$rel" + if [ -s "$dst" ]; then return 0; fi + mkdir -p "$(dirname "$dst")" + code=$(curl -sSL -A "$CUR_UA" --max-time 30 -o "$dst.tmp" -w '%{http_code}' "$url" 2>/dev/null || echo 000) + if [ "$code" = 200 ] && [ -s "$dst.tmp" ]; then + mv "$dst.tmp" "$dst" + printf '%s\t%s\t%s\t%s\n' "$rel" "$url" "$(sha256sum "$dst" | cut -d' ' -f1)" "$(wc -c < "$dst")" >> "$MAN" + printf ' + %s\n' "$rel" + else + rm -f "$dst.tmp" + printf ' ! %s (HTTP %s)\n' "$rel" "$code" >&2 + fi + sleep "$SLEEP" +} + +snap_cr() { + echo "catholic-resources.org (OF lectionary):" + CUR_UA="$BROWSER_UA" + base=https://www.catholic-resources.org/Lectionary + # Sunday cycles + seasons (1998 USL) and weekday/sanctoral tables (2002 USL). + for p in OrdinaryA OrdinaryB OrdinaryC Advent Christmas Lent Easter Solemnities; do + get "$base/1998USL-$p.htm" "catholic-resources/1998USL-$p.htm" + done + for p in Sanctoral Weekdays-OT-I Weekdays-OT-II Weekdays-AdventChristmas Weekdays-Lent Weekdays-Easter; do + get "$base/2002USL-$p.htm" "catholic-resources/2002USL-$p.htm" + done +} + +snap_litcal() { + echo "litcal (OF calendar oracle, whole-year JSON):" + for y in 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035; do + get "https://litcal.johnromanodorazio.com/api/dev/calendar/$y?locale=en" "litcal/calendar-$y-en.json" + done +} + +snap_getbible() { + echo "get.bible (Douay-Rheims deuterocanon source):" + # The books + chapter counts gen-deutero.py backfills (getbible douayrheims + # book numbers). Tobit 69, Judith 70, Wisdom 73, Sirach 74, Baruch 75, + # 1/2 Maccabees 80/81; plus Daniel 13-14 / Esther additions live in-book. + for spec in "69:14" "70:16" "73:19" "74:51" "75:6" "80:16" "81:15"; do + nr=${spec%:*}; last=${spec#*:} + ch=1 + while [ "$ch" -le "$last" ]; do + get "https://api.getbible.net/v2/douayrheims/$nr/$ch.json" "getbible/douayrheims-$nr-$ch.json" + ch=$((ch + 1)) + done + done +} + +snap_missalemeum() { + # EF calendar + readings + Polish saint names. Per-date only; BOUNDED to a + # representative span (widen MM_FROM/MM_TO to reproduce the full harvest). + MM_FROM=${MM_FROM:-2026-01-01} + MM_TO=${MM_TO:-2027-12-31} + echo "missalemeum (EF propers, $MM_FROM .. $MM_TO, en+pl):" + d=$MM_FROM + end=$(date -I -d "$MM_TO +1 day") + while [ "$d" != "$end" ]; do + for lang in en pl; do + get "https://www.missalemeum.com/$lang/api/v5/proper/$d" "missalemeum/$lang/$d.json" + done + d=$(date -I -d "$d +1 day") + done +} + +which=${*:-cr litcal getbible missalemeum} +for p in $which; do + case "$p" in + cr) snap_cr ;; + litcal) snap_litcal ;; + getbible) snap_getbible ;; + missalemeum) snap_missalemeum ;; + *) echo "unknown provider: $p" >&2 ;; + esac +done +echo "done. manifest: $MAN" |