US09-05: Automate Documentation Acceptance (#111)

tests/integration/test_docs_gate.py

@@ -0,0 +1,196 @@
+"""US09-05: the documentation gate's own contract, checked without a browser.
+
+The gate itself needs a browser and a server and takes minutes; what is checkable
+offline is what makes it a *gate* rather than a long test run:
+
+* it selects every documentation check by marker, so adding one is enough to put it
+  in front of a release;
+* it accepts no skip at all — a check that did not run is a page nobody compared
+  with the code;
+* it retains checksummed evidence per run;
+* CI runs it on pull requests, which is when a fix is still cheap.
+
+The checks it runs are the four suites written by US09-01 through US09-04.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+import pytest
+import yaml
+
+from photo_pipeline.config import Config
+from photo_pipeline.services import release
+
+REPO = Path(__file__).resolve().parents[2]
+DOCS = REPO / "docs"
+TEST_WORKFLOW = yaml.safe_load((REPO / ".gitea" / "workflows" / "test.yml").read_text())
+
+pytestmark = pytest.mark.phase_i
+
+# The suites the gate exists to run. Each is mapped to a story in the traceability
+# matrix, so this list is also what stops one being quietly dropped.
+DOCUMENTATION_SUITES = (
+    "tests/integration/test_documentation.py",
+    "tests/integration/test_installation_manual.py",
+    "tests/integration/test_architecture_overview.py",
+    "tests/integration/test_user_manual.py",
+    "tests/integration/test_docs_gate.py",
+    "tests/e2e/test_docs_ui.py",
+    "tests/e2e/test_user_manual_screenshots.py",
+)
+
+
+def _config(tmp_path) -> Config:
+    (tmp_path / "data").mkdir(exist_ok=True)
+    return Config.from_env({"PHOTO_PIPELINE_DATA_DIR": str(tmp_path / "data")})
+
+
+# ── what the gate runs ───────────────────────────────────────────────────────
+
+
+def test_the_gate_selects_every_documentation_check_by_marker():
+    assert release.DOCS_STAGES == (("documentation", ("tests", "-m", "phase_i")),)
+    declared = [
+        line
+        for line in (REPO / "pyproject.toml").read_text().splitlines()
+        if line.strip().startswith('"phase_i:')
+    ]
+    assert declared, "an unregistered marker selects nothing and fails no gate"
+
+
+def test_every_documentation_suite_carries_the_marker():
+    """A documentation test without the marker is a check the gate never runs."""
+    unmarked = [
+        suite
+        for suite in DOCUMENTATION_SUITES
+        if "pytest.mark.phase_i" not in (REPO / suite).read_text()
+    ]
+    assert unmarked == [], f"suites missing the phase_i marker: {unmarked}"
+
+
+def test_the_marker_selects_nothing_outside_the_documentation_suites():
+    """Scope matters: the gate's promise is that it ran *the documentation checks*."""
+    marked = sorted(
+        str(path.relative_to(REPO))
+        for path in (REPO / "tests").rglob("test_*.py")
+        if "pytest.mark.phase_i" in path.read_text()
+    )
+    assert marked == sorted(DOCUMENTATION_SUITES)
+
+
+# ── no skip is an environment limit here ─────────────────────────────────────
+
+
+def test_the_gate_accepts_no_skipped_check_at_all():
+    assert release.DOCS_ALLOWED_SKIP_REASONS == ()
+    skipped = release.StageResult(
+        "documentation", [], 0, 0.1, "1 skipped", ["SKIPPED [1] x.py:1: exiftool not installed"]
+    )
+    # The release gate tolerates this one because it describes the machine. The
+    # documentation gate cannot: nothing here depends on the machine.
+    assert release.unexpected_skips([skipped]) == []
+    assert release.unexpected_skips([skipped], release.DOCS_ALLOWED_SKIP_REASONS) == [
+        "SKIPPED [1] x.py:1: exiftool not installed"
+    ]
+
+
+def test_a_skipped_check_fails_the_run_and_the_evidence_says_so(tmp_path):
+    skipping = tmp_path / "test_skipping.py"
+    skipping.write_text(
+        "import pytest\n\ndef test_x():\n    pytest.skip('exiftool not installed')\n"
+    )
+    evidence = tmp_path / "evidence"
+
+    report = release.run_gate(
+        _config(tmp_path),
+        output=evidence,
+        stages=(("documentation", (str(skipping),)),),
+        allowed_skips=release.DOCS_ALLOWED_SKIP_REASONS,
+    )
+
+    assert report["ok"] is False
+    assert report["failures"] == [], "the stage passed; the skip is what fails the gate"
+    assert report["unexpected_skips"], report
+    written = json.loads((evidence / release.REPORT_NAME).read_text())
+    assert written["ok"] is False
+    assert (evidence / "logs" / "documentation.log").exists()
+    for line in (evidence / release.CHECKSUMS_NAME).read_text().splitlines():
+        digest, name = line.split("  ", 1)
+        assert release.sha256_file(evidence / name) == digest
+
+
+def test_a_broken_documentation_check_fails_the_gate(tmp_path):
+    """The failure mode that matters: a page and the code disagreeing."""
+    failing = tmp_path / "test_disagreement.py"
+    failing.write_text(
+        "def test_the_manual_matches_the_code():\n"
+        "    documented = {'lock_held'}\n"
+        "    emitted = {'lock_held', 'renamed_since'}\n"
+        "    assert emitted <= documented\n"
+    )
+
+    report = release.run_gate(
+        _config(tmp_path),
+        output=tmp_path / "evidence",
+        stages=(("documentation", (str(failing),)),),
+        allowed_skips=release.DOCS_ALLOWED_SKIP_REASONS,
+    )
+
+    assert report["ok"] is False
+    assert report["failures"], report
+
+
+def test_a_clean_tree_passes(tmp_path):
+    passing = tmp_path / "test_passing.py"
+    passing.write_text("def test_x():\n    assert True\n")
+
+    report = release.run_gate(
+        _config(tmp_path),
+        output=tmp_path / "evidence",
+        stages=(("documentation", (str(passing),)),),
+        allowed_skips=release.DOCS_ALLOWED_SKIP_REASONS,
+    )
+
+    assert report["ok"] is True and report["unexpected_skips"] == []
+    assert report["revision"], "the evidence must say which commit it covers"
+
+
+# ── the command, and CI ──────────────────────────────────────────────────────
+
+
+def test_the_command_is_documented_and_wired():
+    assert "docs-gate" in (REPO / "photo_pipeline" / "__main__.py").read_text()
+    assert "docs-gate" in (DOCS / "index.md").read_text()
+
+
+def test_ci_runs_the_gate_on_pull_requests_and_keeps_its_evidence():
+    job = TEST_WORKFLOW["jobs"]["documentation"]
+    # No `if:` — documentation drifts on the same commits that change behaviour, and
+    # the pull request is when saying so is still cheap.
+    assert "if" not in job
+    assert "pull_request" in TEST_WORKFLOW[True] or "pull_request" in TEST_WORKFLOW.get("on", {})
+    script = "\n".join(step["run"] for step in job["steps"] if "run" in step)
+    assert "photo_pipeline docs-gate" in script
+    evidence = next(step for step in job["steps"] if "upload-artifact" in str(step.get("uses")))
+    assert evidence["if"] == "always()", "a failed gate's evidence is the one worth keeping"
+
+
+# ── the pages and the repository point at each other ─────────────────────────
+
+
+def test_the_readme_and_the_documentation_index_link_to_each_other():
+    """Neither may become the forgotten copy."""
+    readme = (REPO / "README.md").read_text()
+    index = (DOCS / "index.md").read_text()
+    assert "docs/index.md" in readme
+    assert re.search(r"\.\./README\.md", index), "the index does not link back to the README"
+
+
+def test_every_documentation_story_is_mapped_in_the_traceability_matrix():
+    matrix = json.loads((REPO / "tests" / "story_traceability.json").read_text())["stories"]
+    for story in ("US09-01", "US09-02", "US09-03", "US09-04", "US09-05"):
+        assert matrix.get(story), f"{story} is not mapped to any test"