diff options
Diffstat (limited to 'test/test_validate.ml')
| -rw-r--r-- | test/test_validate.ml | 72 |
1 files changed, 71 insertions, 1 deletions
diff --git a/test/test_validate.ml b/test/test_validate.ml index 4b9c3c0..ee7f288 100644 --- a/test/test_validate.ml +++ b/test/test_validate.ml @@ -90,6 +90,74 @@ let prop_invariants = (QCheck.int_range 1583 9998) (fun y -> run y = []) +(* ---- final fix wave, item 6: the exhaustive sweep, committed ---- + + The property above samples 200 of 8 416 years (2.4% of the domain) on a + RANDOM seed -- QCheck.Test.make with no ~seed argument draws a fresh one + from the environment/OS entropy each run, and two consecutive runs of + this suite were observed using different seeds (see the task report for + the transcript). CLAUDE.md's standing claim that Validate is "clean + across all 8 416 years -- exhaustive, not sampled" was true whenever it + was last actually re-run in full, but no committed artifact pinned it, + and a year-specific regression (one bad year among 8 416) would show up + in this suite only intermittently -- roughly 200/8416 of the time per + run, i.e. most runs would NOT catch it. + + This is that committed artifact: every year 1583..9999, not a sample. + Tagged `Slow (matching this file's own naming for the check it performs + -- see [suite] below), but Alcotest's speed-level filtering is deliberately + NOT used to keep it out of the default `dune test`: that filtering (the + `-q`/`--quick-tests` flag, or dune wiring the runtest action to pass it) + is ALL-OR-NOTHING per speed level, and this codebase already tags SIX + OTHER cases `Slow -- the two pre-existing exhaustive Computus checks + (test_computus.ml, both genuinely fast, sub-second) AND, found while + implementing this item, EVERY QCheck property in the whole suite + (test_date.ml x3, test_overlay.ml, test_temporal_ef.ml, and + [prop_invariants] immediately above, since QCheck_alcotest.to_alcotest + defaults ~speed_level to `Slow when not given explicitly, which none of + this codebase's call sites do). Wiring `-q` at the dune level was tried + and reverted: it made the default `dune test` report 251 tests instead + of (the then-current) 260, silently excluding [prop_invariants] itself + -- the "confidence-to-9999" mechanism CLAUDE.md documents as this + project's central property-testing story -- along with five other + properties, none of which this task asked to remove from the fast path. + That is a far bigger, unintended regression than the one line this item + asks to add. + + Instead, this test gates its OWN expensive body on an environment + variable, [COLITUR_EXHAUSTIVE_SWEEP], and calls {!Alcotest.skip} (marked + SKIPPED, not silently passed, when unset) so `dune test`'s default run + stays at its normal speed and reports the skip honestly rather than a + vacuous green. To run the real sweep (~35-45s, see the report for the + measured figure): + + COLITUR_EXHAUSTIVE_SWEEP=1 dune test --force + + or invoke the built executable directly with the same variable set. *) +let colitur_exhaustive_sweep_env = "COLITUR_EXHAUSTIVE_SWEEP" + +(* 9999 is a documented, non-regression truncation, not a fresh finding: + [test_year_9999_does_not_raise] above already pins that [run 9999] + reports exactly a "seasons" failure (the domain's own ceiling truncates + the scan mid-Christmastide) and nothing else -- reused here rather than + calling [check_year] on 9999, which would fail this sweep on a shape + that is not a regression. *) +let test_exhaustive_domain_sweep () = + if Sys.getenv_opt colitur_exhaustive_sweep_env = None then Alcotest.skip () + else begin + for y = 1583 to 9998 do + check_year y + done; + let fs = run 9999 in + Alcotest.(check bool) "9999: no coverage failures (temporal stayed total through the clamp)" true + (not (List.exists (fun f -> f.Val.check = "coverage") fs)); + Alcotest.(check bool) "9999: seasons check flags the truncated final year as incomplete" true + (List.exists (fun f -> f.Val.check = "seasons") fs); + Alcotest.(check (list string)) "9999: nothing OTHER than the documented seasons truncation fired" + [ "seasons" ] + (List.sort_uniq compare (List.map (fun f -> f.Val.check) fs)) + end + (* ---- negative-path fixture (Task 14 review, finding 1) ---- Everything above only exercises the CLEAN path against real EF data: an @@ -628,5 +696,7 @@ let suite = Alcotest.test_case "admission fires" `Quick test_admission_fires; Alcotest.test_case "observed fires" `Quick test_observed_fires; Alcotest.test_case "resolution checks clean on a well-behaved layer" `Quick - test_resolution_checks_clean_on_a_well_behaved_layer ] + test_resolution_checks_clean_on_a_well_behaved_layer; + Alcotest.test_case "exhaustive domain sweep (1583..9999), committed not sampled" `Slow + test_exhaustive_domain_sweep ] @ List.map QCheck_alcotest.to_alcotest [ prop_invariants ] ) |