US09-05: Automate Documentation Acceptance

tests/e2e/test_docs_ui.py

@@ -18,6 +18,9 @@ import pytest
 
 from tests.e2e._pipeline_harness import Server, seed_library
 
+# Every check here belongs to the documentation gate (US09-05).
+pytestmark = pytest.mark.phase_i
+
 
 @pytest.fixture(scope="module")
 def server(tmp_path_factory):

tests/e2e/test_user_manual_screenshots.py

@@ -24,10 +24,13 @@ from contextlib import closing
 from pathlib import Path
 
 import pytest
+from PIL import Image
+from playwright.sync_api import TimeoutError as PlaywrightTimeout
 
 from tests.conftest import session_client
 from tests.e2e._pipeline_harness import (
     Server,
+    approve_album,
     seed_album,
     start_worker,
     wait_until,
@@ -39,21 +42,29 @@ IMAGES = DOCS / "images"
 ALBUM = "rome"
 VIEWPORT = {"width": 1280, "height": 900}
 
-# One per stage page of the manual: the route to visit, and the element whose
-# presence means the view has actually finished rendering.
+# One per stage page of the manual: the route, the heading that view renders, and the
+# element whose presence means it has finished.
+#
+# The heading matters more than it looks. Every route replaces the same container, so
+# waiting for "an h1" matches the *previous* view's heading and photographs the screen
+# you just left — which is how the statistics page first shipped a picture of the
+# archive view.
 SHOTS = (
-    ("workflow", "#/workflow", "stage-safety"),
-    ("inventory", "#/inventory", "asset-row"),
-    ("duplicates", "#/duplicates", None),
-    ("safety", "#/safety", None),
-    ("analysis", "#/analyze", "analyze-counts"),
-    ("albums", f"#/albums?album={ALBUM}", "suggested-name"),
-    ("renames", "#/renames", None),
-    ("uploads", "#/uploads", "upload-scope"),
-    ("archive", "#/archive", "archive-locations"),
-    ("statistics", "#/stats", None),
+    ("workflow", "#/workflow", "Workflow", "stage-safety"),
+    ("inventory", "#/inventory", "Inventory", "asset-row"),
+    ("duplicates", "#/duplicates", "Duplicate clusters", None),
+    ("safety", "#/safety", "Safety review", None),
+    ("analysis", "#/analyze", "Analyze", "analyze-counts"),
+    ("albums", f"#/albums?album={ALBUM}", "Albums", "suggested-name"),
+    ("renames", "#/renames", "Renames", None),
+    ("uploads", "#/uploads", "Upload", "upload-scope"),
+    ("archive", "#/archive", "Archive", "archive-locations"),
+    ("statistics", "#/stats", "Stats", None),
 )
 
+# Every check here belongs to the documentation gate (US09-05).
+pytestmark = pytest.mark.phase_i
+
 
 def writing() -> bool:
     return os.environ.get("PHOTO_PIPELINE_WRITE_SCREENSHOTS") == "1"
@@ -90,17 +101,23 @@ def _prepare(base: str, seeded) -> None:
         ).raise_for_status()
         wait_until(lambda: client.get("/api/v1/workflow").status_code == 200)
 
+    # An approved name and a built — deliberately unapplied — plan, so the renames
+    # screenshot shows the preview its page describes rather than an empty state.
+    # Building a plan moves nothing; applying it is what would, and nothing here does.
+    approve_album(base, album=ALBUM, name="2019 — Rome")
+    with closing(session_client(base, timeout=30)) as client:
+        client.post("/api/v1/rename-plans").raise_for_status()
+
 
 def _capture(page, server, target: Path) -> list[str]:
     target.mkdir(parents=True, exist_ok=True)
     page.set_viewport_size(VIEWPORT)
     written = []
-    for name, route, ready in SHOTS:
+    for name, route, heading, ready in SHOTS:
         page.goto(f"{server.base}/app/{route}")
+        page.locator("main h1", has_text=heading).first.wait_for(timeout=30_000)
         if ready:
             page.get_by_test_id(ready).first.wait_for(timeout=30_000)
-        else:
-            page.locator("main h1").first.wait_for(timeout=30_000)
         page.screenshot(path=str(target / f"{name}.png"))
         written.append(name)
     return written
@@ -110,7 +127,7 @@ def test_the_generator_produces_every_screenshot_the_manual_references(page, sta
     destination = IMAGES if writing() else tmp_path / "images"
     written = _capture(page, stack, destination)
 
-    assert sorted(written) == sorted(name for name, _, _ in SHOTS)
+    assert sorted(written) == sorted(name for name, *_ in SHOTS)
     for name in written:
         produced = destination / f"{name}.png"
         assert produced.stat().st_size > 5_000, f"{name}.png is too small to be a view"
@@ -133,6 +150,60 @@ def test_no_screenshot_shows_a_real_path_or_a_secret(page, stack, tmp_path):
     assert all(ALBUM in path or "images" in path for path in paths)
 
 
+def test_the_committed_screenshots_still_show_what_the_application_shows(page, stack, tmp_path):
+    """A UI change that invalidates the manual should be a red build, not a discovery
+    months later by somebody following a picture of a screen that no longer exists.
+
+    The tolerance is on *content*, not pixels. Comparing the committed PNGs with a
+    fresh capture was tried first and rejected: PNG output is not reproducible, font
+    rasterisation differs between the machine that generated an image and the machine
+    running the gate, and several views legitimately print the fixture library's
+    absolute path, which is a fresh temporary directory every run. A pixel or
+    perceptual comparison therefore fails for reasons that have nothing to do with
+    the manual being wrong — and a check that cries wolf gets deleted.
+
+    What actually invalidates a screenshot is the view no longer showing what its
+    page says it shows. That is what is compared here: the heading, the elements the
+    page describes, and the fact that the image was captured at the same size.
+
+    ponytail: content comparison, not pixels. A visual-diff service with per-platform
+    baselines is the upgrade if cosmetic regressions ever need catching too.
+    """
+    if not IMAGES.is_dir():
+        pytest.skip("no screenshots have been generated into the repository yet")
+
+    fresh = tmp_path / "fresh"
+    _capture(page, stack, fresh)
+
+    drifted = []
+    for name, route, heading, ready in SHOTS:
+        committed = IMAGES / f"{name}.png"
+        if not committed.is_file():
+            drifted.append(f"{name}: never committed")
+            continue
+        with Image.open(committed) as image:
+            if image.size != (VIEWPORT["width"], VIEWPORT["height"]):
+                drifted.append(f"{name}: committed at {image.size}, captured at {VIEWPORT}")
+        # The views render asynchronously, so each check waits rather than asking a
+        # question the page has not finished answering.
+        page.goto(f"{stack.base}/app/{route}")
+        try:
+            page.locator("main h1", has_text=heading).first.wait_for(timeout=15_000)
+        except PlaywrightTimeout:
+            drifted.append(f"{name}: the view no longer shows the heading '{heading}'")
+            continue
+        if ready:
+            try:
+                page.get_by_test_id(ready).first.wait_for(timeout=15_000)
+            except PlaywrightTimeout:
+                drifted.append(f"{name}: the view no longer renders '{ready}'")
+
+    assert drifted == [], (
+        f"the manual's screenshots no longer match the application: {drifted}. "
+        "Regenerate them with PHOTO_PIPELINE_WRITE_SCREENSHOTS=1 and review the result."
+    )
+
+
 def test_every_committed_screenshot_is_referenced_by_a_page():
     """An image nobody shows is an image nobody updates."""
     if not IMAGES.is_dir():