From 806503cc3da3be227855910b222de8b72be549bc Mon Sep 17 00:00:00 2001 From: Lukasz Kasprzak Date: Wed, 29 Jul 2026 12:09:57 +0200 Subject: bible(corpus-check): warn on a verse present but stubbed vs the Vulgate The count-based checks (gap, duplicate, missing chapter) only see a verse's presence, so a verse that IS there but truncated -- a lost opening line, a merge artifact, the exact class that lost 44 psalm openings and Acts 6:5 from the old Wujek scrape -- passes silently. Add a warning for it, compared CROSS-CORPUS to the aligned Vulgate verse (never within the chapter). That is deliberate: a genuinely terse verse ("Non occides", "Jesus wept") is short in the Vulgate too, so it is not flagged; only a corpus stub against a substantial Vulgate verse fires (corpus <12 chars, <1/4 of a Vulgate verse >=40). Rune-counted (script-agnostic) and skipped on the Psalms when the corpus renumbers them (psalm_system != vulgate), where per-verse alignment with the Vulgate would not hold. Warning only, never an error. Zero false positives across vul/wuj/drb/grb; a `stub` testdata fixture (Genesis 1:1 = "x") exercises a real hit. --- internal/bible/testdata/corpora/stub.ini | 4 ++++ internal/bible/testdata/corpora/stub.tsv | 2 ++ internal/bible/validate.go | 38 +++++++++++++++++++++++++++++--- internal/bible/validate_test.go | 19 +++++++++++++++- 4 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 internal/bible/testdata/corpora/stub.ini create mode 100644 internal/bible/testdata/corpora/stub.tsv diff --git a/internal/bible/testdata/corpora/stub.ini b/internal/bible/testdata/corpora/stub.ini new file mode 100644 index 0000000..3ba9b6e --- /dev/null +++ b/internal/bible/testdata/corpora/stub.ini @@ -0,0 +1,4 @@ +; test fixture: a present-but-truncated verse (Genesis 1:1 is a stub vs vul) +lang = en +name = Stub Test Corpus +psalm_system = vulgate diff --git a/internal/bible/testdata/corpora/stub.tsv b/internal/bible/testdata/corpora/stub.tsv new file mode 100644 index 0000000..3dd4551 --- /dev/null +++ b/internal/bible/testdata/corpora/stub.tsv @@ -0,0 +1,2 @@ +Genesis Gn 1 1 1 x +Genesis Gn 1 1 2 And the earth was void and empty, and darkness covered the deep. diff --git a/internal/bible/validate.go b/internal/bible/validate.go index 27ec519..9c99436 100644 --- a/internal/bible/validate.go +++ b/internal/bible/validate.go @@ -3,6 +3,8 @@ package bible import ( "fmt" "sort" + "strings" + "unicode/utf8" ) // CorpusReport is the outcome of validating one corpus's sidecar + text (see @@ -34,7 +36,10 @@ var validPsalmSystems = map[string]bool{"vulgate": true, "hebrew": true, "drb": // chapter, or a repeated verse number within a chapter, is a warning only -- // real source texts legitimately do this (e.g. the LXX's lettered doublet // verses in 3 Kingdoms, or a scanned translation's occasional merged verse), -// and it must never turn a corpus that otherwise loads fine into a failure. +// and it must never turn a corpus that otherwise loads fine into a failure. A +// verse that IS present but whose text is a stub next to the aligned Vulgate +// verse (a likely truncation or lost line -- the one defect the count-based +// checks cannot see) is likewise a warning. func CheckCorpus(code string) CorpusReport { r := CorpusReport{Code: code} @@ -62,6 +67,7 @@ func CheckCorpus(code string) CorpusReport { ref := load("vul") for book, chaps := range c.books { + refBook := ref.books[book] // nil if vul lacks the book (e.g. an EF-only name) for ch, verses := range chaps { seen := map[int]bool{} maxV := 0 @@ -77,9 +83,35 @@ func CheckCorpus(code string) CorpusReport { if maxV > len(seen) { r.Warnings = append(r.Warnings, fmt.Sprintf("%s %d: verse gap (have %d verse(s), highest numbered %d)", book, ch, len(seen), maxV)) } + + // Suspiciously-short verse: present, but a stub next to the SAME verse + // in the reference Vulgate -- a likely truncation or lost line (an + // opening dropped in a scrape, a merge artifact) that the + // presence-only checks cannot see, since the verse IS there. The + // comparison is cross-corpus, not within the chapter, precisely so a + // genuinely terse verse ("Non occides", "Jesus wept") is NOT flagged: + // the Vulgate's own copy is short too, so the guard below fails. It + // fires only when the corpus verse is a stub AND the aligned Vulgate + // verse is substantial. Skipped on the Psalms when the corpus + // renumbers them (psalm_system != vulgate), where the per-verse + // alignment with the Vulgate does not hold. + if refBook != nil && !(book == "Psalms" && m.PsalmSystem != "vulgate") { + if refCh, ok := refBook[ch]; ok { + refLen := make(map[int]int, len(refCh)) + for _, rv := range refCh { + refLen[rv.Verse] = utf8.RuneCountInString(strings.TrimSpace(rv.Text)) + } + for _, v := range verses { + n := utf8.RuneCountInString(strings.TrimSpace(v.Text)) + if rl := refLen[v.Verse]; rl >= 40 && n < 12 && n*4 < rl { + r.Warnings = append(r.Warnings, fmt.Sprintf("%s %d:%d: text suspiciously short (%d chars vs vul %d)", book, ch, v.Verse, n, rl)) + } + } + } + } } - if refChaps, ok := ref.books[book]; ok { - for ch := range refChaps { + if refBook != nil { + for ch := range refBook { if _, have := chaps[ch]; !have { r.Warnings = append(r.Warnings, fmt.Sprintf("%s: missing chapter %d (present in vul)", book, ch)) } diff --git a/internal/bible/validate_test.go b/internal/bible/validate_test.go index 9cfec26..a3ed622 100644 --- a/internal/bible/validate_test.go +++ b/internal/bible/validate_test.go @@ -1,6 +1,9 @@ package bible -import "testing" +import ( + "strings" + "testing" +) func TestCheckCorpus(t *testing.T) { SetUserCorporaDir("testdata/corpora") @@ -21,4 +24,18 @@ func TestCheckCorpus(t *testing.T) { if r := CheckCorpus("badsystem"); r.OK() { t.Fatal("bad psalm_system not caught") } + // A verse present but a stub next to the Vulgate's (Genesis 1:1 = "x") is a + // warning, never an error -- the count-based checks can't see it. + if r := CheckCorpus("stub"); !hasWarning(r.Warnings, "suspiciously short") { + t.Fatalf("stub verse not warned; warnings: %v", r.Warnings) + } +} + +func hasWarning(ws []string, substr string) bool { + for _, w := range ws { + if strings.Contains(w, substr) { + return true + } + } + return false } -- cgit v1.3