diff options
| author | Lukasz Kasprzak <lukas@labunix.xyz> | 2026-07-23 21:01:05 +0200 |
|---|---|---|
| committer | Lukasz Kasprzak <lukas@labunix.xyz> | 2026-07-23 21:01:05 +0200 |
| commit | ffa4676a34851d667acbf8a460321ff93310b7ba (patch) | |
| tree | 69b686dcb235cd3473d9ad28f2c69748358a81f7 /internal/liturgy | |
| parent | f4e15c0c6e0f2e7a0a0d0654a63824834c9e4166 (diff) | |
| download | lectio-ffa4676a34851d667acbf8a460321ff93310b7ba.tar.gz lectio-ffa4676a34851d667acbf8a460321ff93310b7ba.zip | |
cli: flag-driven interface (short+long); -u maximal blip-proof harvest
Replace the today/date/show/compare/update subcommands with a single
flag-driven interface like the Python predecessor: lectio [DATE]
-a/-b/-c/-r/-w/-R/-o/-l/-g/-u, short and long names bound to the same
variable, DATE (YYYY-MM-DD) extracted from any position in the args.
Fix Harvest's "-u" to walk to the true maximal horizon without a
transient network error being mistaken for it: a fetch failure now
retries a few times with backoff and, if it still fails, is reported
as an error (after saving partial progress via writeSigla), while a
Parse failure (the real unpublished-horizon placeholder) still stops
cleanly with a nil error.
Diffstat (limited to 'internal/liturgy')
| -rw-r--r-- | internal/liturgy/store.go | 55 | ||||
| -rw-r--r-- | internal/liturgy/store_test.go | 51 |
2 files changed, 96 insertions, 10 deletions
diff --git a/internal/liturgy/store.go b/internal/liturgy/store.go index 6762585..cc176e8 100644 --- a/internal/liturgy/store.go +++ b/internal/liturgy/store.go @@ -9,6 +9,15 @@ import ( "time" ) +// harvestRetries is how many times Harvest attempts a single date's fetch +// before treating it as a genuine network failure rather than transient +// hiccup; harvestRetryDelay is the backoff slept between attempts. Both are +// package vars so tests can shrink the delay instead of waiting on it. +var ( + harvestRetries = 3 + harvestRetryDelay = 2 * time.Second +) + // siglaRow is one line of the sigla TSV: a date's section label and the // scripture citation extracted from its heading. type siglaRow struct { @@ -89,17 +98,23 @@ func writeSigla(path string, byDate map[string][]siglaRow) error { // Harvest walks dates forward from fromDate, fetching and parsing each day's // page and recording every section's citation to the sigla TSV, for up to -// maxDays days (0 = walk until the unpublished horizon). It stops the first -// time a date fails to fetch or parse -- niedziela.pl's "Przykro nam" -// placeholder (or any other parse failure) marks the horizon the site -// hasn't published past yet, not an error to report. +// maxDays days (0 = walk until the unpublished horizon). It stops cleanly +// (nil error) the first time a date fails to PARSE -- niedziela.pl's +// "Przykro nam" placeholder (or any other parse failure) marks the horizon +// the site hasn't published past yet, not an error to report. A date that +// fails to FETCH, by contrast, is a transient network problem, not the +// horizon: Harvest retries it a few times (harvestRetries, backing off +// harvestRetryDelay between attempts) and, if it still fails, stops and +// returns an error -- but only after saving whatever was harvested up to +// that point, so the caller never loses progress to a blip. // // Re-harvesting a date replaces its rows in the TSV rather than duplicating // them, so running Harvest again over an already-harvested range is safe. // It also warms the HTML/JSON cache for every date it successfully harvests. // // It returns how many days were harvested and the furthest (most recent) -// date reached. +// date reached, alongside any fetch error (nil on a clean parse-horizon +// stop or on reaching maxDays). func Harvest(fromDate string, maxDays int) (added int, furthest string, err error) { start, err := time.Parse("2006-01-02", fromDate) if err != nil { @@ -114,16 +129,32 @@ func Harvest(fromDate string, maxDays int) (added int, furthest string, err erro dir := cacheDir() day := start + var harvestErr error for i := 0; maxDays == 0 || i < maxDays; i++ { dateStr := day.Format("2006-01-02") - page, ferr := fetch(dateStr) + var page string + var ferr error + for attempt := 1; attempt <= harvestRetries; attempt++ { + page, ferr = fetch(dateStr) + if ferr == nil { + break + } + if attempt < harvestRetries { + time.Sleep(harvestRetryDelay) + } + } if ferr != nil { - break // unreachable site or network error: stop, not a hard failure + // A genuine network/transport error, not the horizon: don't + // silently stop as if the site simply hadn't published this + // date yet. Record it and stop walking, but writeSigla below + // still runs so progress made so far isn't lost. + harvestErr = fmt.Errorf("harvest interrupted at %s: %w", dateStr, ferr) + break } secs, perr := Parse(page) if perr != nil { - break // unpublished horizon (or unparsable page): stop walking + break // unpublished horizon (or unparsable page): stop walking, cleanly } var rows []siglaRow @@ -148,8 +179,12 @@ func Harvest(fromDate string, maxDays int) (added int, furthest string, err erro day = day.AddDate(0, 0, 1) } - if err := writeSigla(path, byDate); err != nil { - return added, furthest, err + werr := writeSigla(path, byDate) + if harvestErr != nil { + return added, furthest, harvestErr + } + if werr != nil { + return added, furthest, werr } return added, furthest, nil } diff --git a/internal/liturgy/store_test.go b/internal/liturgy/store_test.go index 08fd1a1..1bf8343 100644 --- a/internal/liturgy/store_test.go +++ b/internal/liturgy/store_test.go @@ -6,6 +6,7 @@ import ( "os" "strings" "testing" + "time" ) func TestHarvestAndOffline(t *testing.T) { @@ -47,3 +48,53 @@ func TestHarvestAndOffline(t *testing.T) { t.Error("offline first czytanie PartID missing") } } + +// TestHarvestFetchErrorSavesPartialProgress exercises the transient-error +// path: a genuine network/transport failure on a date must NOT be mistaken +// for the unpublished horizon (that is Parse's job, on a "Przykro nam" page +// -- see TestHarvestAndOffline above, which stays nil-error). It must +// instead surface as a returned error, after writeSigla has still saved +// whatever was harvested before the failing date. +func TestHarvestFetchErrorSavesPartialProgress(t *testing.T) { + html, _ := os.ReadFile("testdata/2026-07-22.html") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "2026-07-22") { + w.Write(html) + return + } + // Simulate a network/transport error (not a "Przykro nam" horizon + // page) by hijacking the connection and closing it without a + // response, so the client sees a read/EOF error. + hj, ok := w.(http.Hijacker) + if !ok { + t.Fatal("test server ResponseWriter does not support hijacking") + } + conn, _, err := hj.Hijack() + if err != nil { + t.Fatal(err) + } + conn.Close() + })) + defer srv.Close() + t.Setenv("XDG_CACHE_HOME", t.TempDir()) + t.Setenv("XDG_DATA_HOME", t.TempDir()) + baseURL = srv.URL + "/liturgia/%s/Ewangelia" + + origRetries, origDelay := harvestRetries, harvestRetryDelay + harvestRetries, harvestRetryDelay = 2, time.Millisecond + defer func() { harvestRetries, harvestRetryDelay = origRetries, origDelay }() + + added, furthest, err := Harvest("2026-07-22", 0) + if err == nil { + t.Fatal("harvest: want error on a fetch failure, got nil") + } + if added != 1 || furthest != "2026-07-22" { + t.Errorf("harvest: added=%d furthest=%q, want added=1 furthest=2026-07-22 (only the one date fetched before the network error)", added, furthest) + } + + // Progress made before the failing date must still be on disk. + secs, offErr := LoadOffline("2026-07-22") + if offErr != nil || len(secs) == 0 { + t.Errorf("harvest: partial progress not saved: offErr=%v secs=%v", offErr, secs) + } +} |
