ci: fail the build when documentation falls behind

Adding a flag and forgetting the man page was the kind of thing only a reader
would catch, and the README had already drifted: it documented seven of fifteen
flags, claimed three cross-compilation targets where the Makefile builds six,
and its example output predated the humidity column.

Flag definitions move into defineFlags(), so the test enumerates the same set
run() does rather than a hand-copied list that could drift in its own right.
usage() gains a writer so its output can be captured. Tests then assert that
every flag reaches -h, the README and the man page; that every key written into
the generated config is documented; and that every column name is explained.

The man page check normalises roff's \- hyphen escape first -- without that,
every multi-word flag looks undocumented when it is not.

1 files changed, 122 insertions, 0 deletions

diff --git a/cmd/prognosis/docs_test.go b/cmd/prognosis/docs_test.go
new file mode 100644
index 0000000..6aa97c0
--- /dev/null
+++ b/cmd/prognosis/docs_test.go
@@ -0,0 +1,122 @@
+package main
+
+import (
+	"bytes"
+	"flag"
+	"io"
+	"os"
+	"regexp"
+	"strings"
+	"testing"
+
+	"github.com/lukaszkasprzak/prognosis/internal/config"
+)
+
+func readRepoFile(t *testing.T, rel string) string {
+	t.Helper()
+	// Tests run in the package directory; the docs live at the repo root.
+	b, err := os.ReadFile("../../" + rel)
+	if err != nil {
+		t.Fatalf("cannot read %s: %v", rel, err)
+	}
+	// roff escapes a literal hyphen as \- , so "-no-warnings" is written
+	// "\-no\-warnings". Undo that before searching, or every multi-word flag
+	// looks undocumented when it is not.
+	return strings.ReplaceAll(string(b), `\-`, "-")
+}
+
+// Every flag must appear in -h, in the README and in the man page.
+//
+// The flag set comes from defineFlags, the same function run() uses, so this
+// cannot be satisfied by a stale hand-written list: adding a flag and
+// forgetting to document it fails the build.
+func TestEveryFlagIsDocumented(t *testing.T) {
+	fs := flag.NewFlagSet("prognosis", flag.ContinueOnError)
+	fs.SetOutput(io.Discard)
+	defineFlags(fs)
+
+	var help bytes.Buffer
+	usageTo(&help)
+
+	docs := map[string]string{
+		"-h":              help.String(),
+		"README.md":       readRepoFile(t, "README.md"),
+		"man/prognosis.1": readRepoFile(t, "man/prognosis.1"),
+	}
+
+	fs.VisitAll(func(f *flag.Flag) {
+		for where, text := range docs {
+			if !strings.Contains(text, "-"+f.Name) {
+				t.Errorf("flag -%s is not documented in %s", f.Name, where)
+			}
+		}
+	})
+}
+
+// The reverse: -h must not advertise a flag that does not exist, which would
+// send someone chasing a typo.
+func TestHelpAdvertisesNoPhantomFlags(t *testing.T) {
+	fs := flag.NewFlagSet("prognosis", flag.ContinueOnError)
+	fs.SetOutput(io.Discard)
+	defineFlags(fs)
+
+	real := map[string]bool{}
+	fs.VisitAll(func(f *flag.Flag) { real[f.Name] = true })
+
+	var help bytes.Buffer
+	usageTo(&help)
+	for _, m := range regexp.MustCompile(`(?m)^  -([a-z-]+)`).FindAllStringSubmatch(help.String(), -1) {
+		if !real[m[1]] {
+			t.Errorf("-h lists -%s, which is not a real flag", m[1])
+		}
+	}
+}
+
+// Every key the generated config file contains must be documented in the man
+// page. The generated file is the authoritative list of user-facing settings,
+// so this catches a new key that never reached the documentation.
+func TestEveryConfigKeyIsDocumented(t *testing.T) {
+	path := t.TempDir() + "/config"
+	if err := config.WriteDefault(path, config.Default()); err != nil {
+		t.Fatal(err)
+	}
+	generated, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	man := readRepoFile(t, "man/prognosis.1")
+
+	seen := map[string]bool{}
+	for _, line := range strings.Split(string(generated), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		key, _, ok := strings.Cut(line, "=")
+		if !ok || seen[key] {
+			continue
+		}
+		seen[key] = true
+		if !strings.Contains(man, key) {
+			t.Errorf("config key %q is written into the generated config but not documented in the man page", key)
+		}
+	}
+	if len(seen) < 8 {
+		t.Fatalf("only found %d config keys; the parser above is probably wrong", len(seen))
+	}
+}
+
+// The columns a user can name must all be documented, or the error message
+// listing them points at something the man page never explains.
+func TestEveryColumnIsDocumented(t *testing.T) {
+	man := readRepoFile(t, "man/prognosis.1")
+	readme := readRepoFile(t, "README.md")
+	for _, col := range config.ValidColumns() {
+		if !strings.Contains(man, col) {
+			t.Errorf("column %q is not documented in the man page", col)
+		}
+		if !strings.Contains(readme, col) {
+			t.Errorf("column %q is not documented in the README", col)
+		}
+	}
+}