summaryrefslogtreecommitdiff
path: root/internal/liturgy
diff options
context:
space:
mode:
authorLukasz Kasprzak <lukas@labunix.xyz>2026-07-23 21:01:05 +0200
committerLukasz Kasprzak <lukas@labunix.xyz>2026-07-23 21:01:05 +0200
commitffa4676a34851d667acbf8a460321ff93310b7ba (patch)
tree69b686dcb235cd3473d9ad28f2c69748358a81f7 /internal/liturgy
parentf4e15c0c6e0f2e7a0a0d0654a63824834c9e4166 (diff)
downloadlectio-ffa4676a34851d667acbf8a460321ff93310b7ba.tar.gz
lectio-ffa4676a34851d667acbf8a460321ff93310b7ba.zip
cli: flag-driven interface (short+long); -u maximal blip-proof harvest
Replace the today/date/show/compare/update subcommands with a single flag-driven interface like the Python predecessor: lectio [DATE] -a/-b/-c/-r/-w/-R/-o/-l/-g/-u, short and long names bound to the same variable, DATE (YYYY-MM-DD) extracted from any position in the args. Fix Harvest's "-u" to walk to the true maximal horizon without a transient network error being mistaken for it: a fetch failure now retries a few times with backoff and, if it still fails, is reported as an error (after saving partial progress via writeSigla), while a Parse failure (the real unpublished-horizon placeholder) still stops cleanly with a nil error.
Diffstat (limited to 'internal/liturgy')
-rw-r--r--internal/liturgy/store.go55
-rw-r--r--internal/liturgy/store_test.go51
2 files changed, 96 insertions, 10 deletions
diff --git a/internal/liturgy/store.go b/internal/liturgy/store.go
index 6762585..cc176e8 100644
--- a/internal/liturgy/store.go
+++ b/internal/liturgy/store.go
@@ -9,6 +9,15 @@ import (
"time"
)
+// harvestRetries is how many times Harvest attempts a single date's fetch
+// before treating it as a genuine network failure rather than transient
+// hiccup; harvestRetryDelay is the backoff slept between attempts. Both are
+// package vars so tests can shrink the delay instead of waiting on it.
+var (
+ harvestRetries = 3
+ harvestRetryDelay = 2 * time.Second
+)
+
// siglaRow is one line of the sigla TSV: a date's section label and the
// scripture citation extracted from its heading.
type siglaRow struct {
@@ -89,17 +98,23 @@ func writeSigla(path string, byDate map[string][]siglaRow) error {
// Harvest walks dates forward from fromDate, fetching and parsing each day's
// page and recording every section's citation to the sigla TSV, for up to
-// maxDays days (0 = walk until the unpublished horizon). It stops the first
-// time a date fails to fetch or parse -- niedziela.pl's "Przykro nam"
-// placeholder (or any other parse failure) marks the horizon the site
-// hasn't published past yet, not an error to report.
+// maxDays days (0 = walk until the unpublished horizon). It stops cleanly
+// (nil error) the first time a date fails to PARSE -- niedziela.pl's
+// "Przykro nam" placeholder (or any other parse failure) marks the horizon
+// the site hasn't published past yet, not an error to report. A date that
+// fails to FETCH, by contrast, is a transient network problem, not the
+// horizon: Harvest retries it a few times (harvestRetries, backing off
+// harvestRetryDelay between attempts) and, if it still fails, stops and
+// returns an error -- but only after saving whatever was harvested up to
+// that point, so the caller never loses progress to a blip.
//
// Re-harvesting a date replaces its rows in the TSV rather than duplicating
// them, so running Harvest again over an already-harvested range is safe.
// It also warms the HTML/JSON cache for every date it successfully harvests.
//
// It returns how many days were harvested and the furthest (most recent)
-// date reached.
+// date reached, alongside any fetch error (nil on a clean parse-horizon
+// stop or on reaching maxDays).
func Harvest(fromDate string, maxDays int) (added int, furthest string, err error) {
start, err := time.Parse("2006-01-02", fromDate)
if err != nil {
@@ -114,16 +129,32 @@ func Harvest(fromDate string, maxDays int) (added int, furthest string, err erro
dir := cacheDir()
day := start
+ var harvestErr error
for i := 0; maxDays == 0 || i < maxDays; i++ {
dateStr := day.Format("2006-01-02")
- page, ferr := fetch(dateStr)
+ var page string
+ var ferr error
+ for attempt := 1; attempt <= harvestRetries; attempt++ {
+ page, ferr = fetch(dateStr)
+ if ferr == nil {
+ break
+ }
+ if attempt < harvestRetries {
+ time.Sleep(harvestRetryDelay)
+ }
+ }
if ferr != nil {
- break // unreachable site or network error: stop, not a hard failure
+ // A genuine network/transport error, not the horizon: don't
+ // silently stop as if the site simply hadn't published this
+ // date yet. Record it and stop walking, but writeSigla below
+ // still runs so progress made so far isn't lost.
+ harvestErr = fmt.Errorf("harvest interrupted at %s: %w", dateStr, ferr)
+ break
}
secs, perr := Parse(page)
if perr != nil {
- break // unpublished horizon (or unparsable page): stop walking
+ break // unpublished horizon (or unparsable page): stop walking, cleanly
}
var rows []siglaRow
@@ -148,8 +179,12 @@ func Harvest(fromDate string, maxDays int) (added int, furthest string, err erro
day = day.AddDate(0, 0, 1)
}
- if err := writeSigla(path, byDate); err != nil {
- return added, furthest, err
+ werr := writeSigla(path, byDate)
+ if harvestErr != nil {
+ return added, furthest, harvestErr
+ }
+ if werr != nil {
+ return added, furthest, werr
}
return added, furthest, nil
}
diff --git a/internal/liturgy/store_test.go b/internal/liturgy/store_test.go
index 08fd1a1..1bf8343 100644
--- a/internal/liturgy/store_test.go
+++ b/internal/liturgy/store_test.go
@@ -6,6 +6,7 @@ import (
"os"
"strings"
"testing"
+ "time"
)
func TestHarvestAndOffline(t *testing.T) {
@@ -47,3 +48,53 @@ func TestHarvestAndOffline(t *testing.T) {
t.Error("offline first czytanie PartID missing")
}
}
+
+// TestHarvestFetchErrorSavesPartialProgress exercises the transient-error
+// path: a genuine network/transport failure on a date must NOT be mistaken
+// for the unpublished horizon (that is Parse's job, on a "Przykro nam" page
+// -- see TestHarvestAndOffline above, which stays nil-error). It must
+// instead surface as a returned error, after writeSigla has still saved
+// whatever was harvested before the failing date.
+func TestHarvestFetchErrorSavesPartialProgress(t *testing.T) {
+ html, _ := os.ReadFile("testdata/2026-07-22.html")
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.Contains(r.URL.Path, "2026-07-22") {
+ w.Write(html)
+ return
+ }
+ // Simulate a network/transport error (not a "Przykro nam" horizon
+ // page) by hijacking the connection and closing it without a
+ // response, so the client sees a read/EOF error.
+ hj, ok := w.(http.Hijacker)
+ if !ok {
+ t.Fatal("test server ResponseWriter does not support hijacking")
+ }
+ conn, _, err := hj.Hijack()
+ if err != nil {
+ t.Fatal(err)
+ }
+ conn.Close()
+ }))
+ defer srv.Close()
+ t.Setenv("XDG_CACHE_HOME", t.TempDir())
+ t.Setenv("XDG_DATA_HOME", t.TempDir())
+ baseURL = srv.URL + "/liturgia/%s/Ewangelia"
+
+ origRetries, origDelay := harvestRetries, harvestRetryDelay
+ harvestRetries, harvestRetryDelay = 2, time.Millisecond
+ defer func() { harvestRetries, harvestRetryDelay = origRetries, origDelay }()
+
+ added, furthest, err := Harvest("2026-07-22", 0)
+ if err == nil {
+ t.Fatal("harvest: want error on a fetch failure, got nil")
+ }
+ if added != 1 || furthest != "2026-07-22" {
+ t.Errorf("harvest: added=%d furthest=%q, want added=1 furthest=2026-07-22 (only the one date fetched before the network error)", added, furthest)
+ }
+
+ // Progress made before the failing date must still be on disk.
+ secs, offErr := LoadOffline("2026-07-22")
+ if offErr != nil || len(secs) == 0 {
+ t.Errorf("harvest: partial progress not saved: offErr=%v secs=%v", offErr, secs)
+ }
+}