aboutsummaryrefslogtreecommitdiff
path: root/internal/liturgy/store.go
diff options
context:
space:
mode:
authorLukasz Kasprzak <lukas@labunix.xyz>2026-07-23 21:01:05 +0200
committerLukasz Kasprzak <lukas@labunix.xyz>2026-07-23 21:01:05 +0200
commitffa4676a34851d667acbf8a460321ff93310b7ba (patch)
tree69b686dcb235cd3473d9ad28f2c69748358a81f7 /internal/liturgy/store.go
parentf4e15c0c6e0f2e7a0a0d0654a63824834c9e4166 (diff)
downloadlectio-ffa4676a34851d667acbf8a460321ff93310b7ba.tar.gz
lectio-ffa4676a34851d667acbf8a460321ff93310b7ba.zip
cli: flag-driven interface (short+long); -u maximal blip-proof harvest
Replace the today/date/show/compare/update subcommands with a single flag-driven interface like the Python predecessor: lectio [DATE] -a/-b/-c/-r/-w/-R/-o/-l/-g/-u, short and long names bound to the same variable, DATE (YYYY-MM-DD) extracted from any position in the args. Fix Harvest's "-u" to walk to the true maximal horizon without a transient network error being mistaken for it: a fetch failure now retries a few times with backoff and, if it still fails, is reported as an error (after saving partial progress via writeSigla), while a Parse failure (the real unpublished-horizon placeholder) still stops cleanly with a nil error.
Diffstat (limited to 'internal/liturgy/store.go')
-rw-r--r--internal/liturgy/store.go55
1 files changed, 45 insertions, 10 deletions
diff --git a/internal/liturgy/store.go b/internal/liturgy/store.go
index 6762585..cc176e8 100644
--- a/internal/liturgy/store.go
+++ b/internal/liturgy/store.go
@@ -9,6 +9,15 @@ import (
"time"
)
+// harvestRetries is how many times Harvest attempts a single date's fetch
+// before treating it as a genuine network failure rather than transient
+// hiccup; harvestRetryDelay is the backoff slept between attempts. Both are
+// package vars so tests can shrink the delay instead of waiting on it.
+var (
+ harvestRetries = 3
+ harvestRetryDelay = 2 * time.Second
+)
+
// siglaRow is one line of the sigla TSV: a date's section label and the
// scripture citation extracted from its heading.
type siglaRow struct {
@@ -89,17 +98,23 @@ func writeSigla(path string, byDate map[string][]siglaRow) error {
// Harvest walks dates forward from fromDate, fetching and parsing each day's
// page and recording every section's citation to the sigla TSV, for up to
-// maxDays days (0 = walk until the unpublished horizon). It stops the first
-// time a date fails to fetch or parse -- niedziela.pl's "Przykro nam"
-// placeholder (or any other parse failure) marks the horizon the site
-// hasn't published past yet, not an error to report.
+// maxDays days (0 = walk until the unpublished horizon). It stops cleanly
+// (nil error) the first time a date fails to PARSE -- niedziela.pl's
+// "Przykro nam" placeholder (or any other parse failure) marks the horizon
+// the site hasn't published past yet, not an error to report. A date that
+// fails to FETCH, by contrast, is a transient network problem, not the
+// horizon: Harvest retries it a few times (harvestRetries, backing off
+// harvestRetryDelay between attempts) and, if it still fails, stops and
+// returns an error -- but only after saving whatever was harvested up to
+// that point, so the caller never loses progress to a blip.
//
// Re-harvesting a date replaces its rows in the TSV rather than duplicating
// them, so running Harvest again over an already-harvested range is safe.
// It also warms the HTML/JSON cache for every date it successfully harvests.
//
// It returns how many days were harvested and the furthest (most recent)
-// date reached.
+// date reached, alongside any fetch error (nil on a clean parse-horizon
+// stop or on reaching maxDays).
func Harvest(fromDate string, maxDays int) (added int, furthest string, err error) {
start, err := time.Parse("2006-01-02", fromDate)
if err != nil {
@@ -114,16 +129,32 @@ func Harvest(fromDate string, maxDays int) (added int, furthest string, err erro
dir := cacheDir()
day := start
+ var harvestErr error
for i := 0; maxDays == 0 || i < maxDays; i++ {
dateStr := day.Format("2006-01-02")
- page, ferr := fetch(dateStr)
+ var page string
+ var ferr error
+ for attempt := 1; attempt <= harvestRetries; attempt++ {
+ page, ferr = fetch(dateStr)
+ if ferr == nil {
+ break
+ }
+ if attempt < harvestRetries {
+ time.Sleep(harvestRetryDelay)
+ }
+ }
if ferr != nil {
- break // unreachable site or network error: stop, not a hard failure
+ // A genuine network/transport error, not the horizon: don't
+ // silently stop as if the site simply hadn't published this
+ // date yet. Record it and stop walking, but writeSigla below
+ // still runs so progress made so far isn't lost.
+ harvestErr = fmt.Errorf("harvest interrupted at %s: %w", dateStr, ferr)
+ break
}
secs, perr := Parse(page)
if perr != nil {
- break // unpublished horizon (or unparsable page): stop walking
+ break // unpublished horizon (or unparsable page): stop walking, cleanly
}
var rows []siglaRow
@@ -148,8 +179,12 @@ func Harvest(fromDate string, maxDays int) (added int, furthest string, err erro
day = day.AddDate(0, 0, 1)
}
- if err := writeSigla(path, byDate); err != nil {
- return added, furthest, err
+ werr := writeSigla(path, byDate)
+ if harvestErr != nil {
+ return added, furthest, harvestErr
+ }
+ if werr != nil {
+ return added, furthest, werr
}
return added, furthest, nil
}