diff options
Diffstat (limited to 'internal/liturgy/store.go')
| -rw-r--r-- | internal/liturgy/store.go | 55 |
1 files changed, 45 insertions, 10 deletions
diff --git a/internal/liturgy/store.go b/internal/liturgy/store.go index 6762585..cc176e8 100644 --- a/internal/liturgy/store.go +++ b/internal/liturgy/store.go @@ -9,6 +9,15 @@ import ( "time" ) +// harvestRetries is how many times Harvest attempts a single date's fetch +// before treating it as a genuine network failure rather than transient +// hiccup; harvestRetryDelay is the backoff slept between attempts. Both are +// package vars so tests can shrink the delay instead of waiting on it. +var ( + harvestRetries = 3 + harvestRetryDelay = 2 * time.Second +) + // siglaRow is one line of the sigla TSV: a date's section label and the // scripture citation extracted from its heading. type siglaRow struct { @@ -89,17 +98,23 @@ func writeSigla(path string, byDate map[string][]siglaRow) error { // Harvest walks dates forward from fromDate, fetching and parsing each day's // page and recording every section's citation to the sigla TSV, for up to -// maxDays days (0 = walk until the unpublished horizon). It stops the first -// time a date fails to fetch or parse -- niedziela.pl's "Przykro nam" -// placeholder (or any other parse failure) marks the horizon the site -// hasn't published past yet, not an error to report. +// maxDays days (0 = walk until the unpublished horizon). It stops cleanly +// (nil error) the first time a date fails to PARSE -- niedziela.pl's +// "Przykro nam" placeholder (or any other parse failure) marks the horizon +// the site hasn't published past yet, not an error to report. A date that +// fails to FETCH, by contrast, is a transient network problem, not the +// horizon: Harvest retries it a few times (harvestRetries, backing off +// harvestRetryDelay between attempts) and, if it still fails, stops and +// returns an error -- but only after saving whatever was harvested up to +// that point, so the caller never loses progress to a blip. // // Re-harvesting a date replaces its rows in the TSV rather than duplicating // them, so running Harvest again over an already-harvested range is safe. // It also warms the HTML/JSON cache for every date it successfully harvests. // // It returns how many days were harvested and the furthest (most recent) -// date reached. +// date reached, alongside any fetch error (nil on a clean parse-horizon +// stop or on reaching maxDays). func Harvest(fromDate string, maxDays int) (added int, furthest string, err error) { start, err := time.Parse("2006-01-02", fromDate) if err != nil { @@ -114,16 +129,32 @@ func Harvest(fromDate string, maxDays int) (added int, furthest string, err erro dir := cacheDir() day := start + var harvestErr error for i := 0; maxDays == 0 || i < maxDays; i++ { dateStr := day.Format("2006-01-02") - page, ferr := fetch(dateStr) + var page string + var ferr error + for attempt := 1; attempt <= harvestRetries; attempt++ { + page, ferr = fetch(dateStr) + if ferr == nil { + break + } + if attempt < harvestRetries { + time.Sleep(harvestRetryDelay) + } + } if ferr != nil { - break // unreachable site or network error: stop, not a hard failure + // A genuine network/transport error, not the horizon: don't + // silently stop as if the site simply hadn't published this + // date yet. Record it and stop walking, but writeSigla below + // still runs so progress made so far isn't lost. + harvestErr = fmt.Errorf("harvest interrupted at %s: %w", dateStr, ferr) + break } secs, perr := Parse(page) if perr != nil { - break // unpublished horizon (or unparsable page): stop walking + break // unpublished horizon (or unparsable page): stop walking, cleanly } var rows []siglaRow @@ -148,8 +179,12 @@ func Harvest(fromDate string, maxDays int) (added int, furthest string, err erro day = day.AddDate(0, 0, 1) } - if err := writeSigla(path, byDate); err != nil { - return added, furthest, err + werr := writeSigla(path, byDate) + if harvestErr != nil { + return added, furthest, harvestErr + } + if werr != nil { + return added, furthest, werr } return added, furthest, nil } |