US09-04: Write the User Manual with Generated Screenshots (#110)

tests/e2e/test_user_manual_screenshots.py

@@ -0,0 +1,144 @@
+"""US09-04: the user manual's screenshots, produced from the running application.
+
+A screenshot nobody can regenerate is a screenshot that silently stops being true.
+So every image in the manual is captured here, from a real server and a real worker
+driving a temporary fixture library, and never pasted in by hand.
+
+Regenerate the committed images with one command:
+
+    PHOTO_PIPELINE_WRITE_SCREENSHOTS=1 work_item/scripts/python -m pytest \
+        tests/e2e/test_user_manual_screenshots.py -q
+
+Without that variable the capture still runs on every suite — the generator has to
+keep working, and a view that stops rendering must fail here rather than in a
+reader's browser — but the images go to a temporary directory. Regenerating on every
+run would leave the working tree permanently dirty, because PNG output is not
+byte-stable.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from contextlib import closing
+from pathlib import Path
+
+import pytest
+
+from tests.conftest import session_client
+from tests.e2e._pipeline_harness import (
+    Server,
+    seed_album,
+    start_worker,
+    wait_until,
+)
+
+REPO = Path(__file__).resolve().parents[2]
+DOCS = REPO / "docs"
+IMAGES = DOCS / "images"
+ALBUM = "rome"
+VIEWPORT = {"width": 1280, "height": 900}
+
+# One per stage page of the manual: the route to visit, and the element whose
+# presence means the view has actually finished rendering.
+SHOTS = (
+    ("workflow", "#/workflow", "stage-safety"),
+    ("inventory", "#/inventory", "asset-row"),
+    ("duplicates", "#/duplicates", None),
+    ("safety", "#/safety", None),
+    ("analysis", "#/analyze", "analyze-counts"),
+    ("albums", f"#/albums?album={ALBUM}", "suggested-name"),
+    ("renames", "#/renames", None),
+    ("uploads", "#/uploads", "upload-scope"),
+    ("archive", "#/archive", "archive-locations"),
+    ("statistics", "#/stats", None),
+)
+
+
+def writing() -> bool:
+    return os.environ.get("PHOTO_PIPELINE_WRITE_SCREENSHOTS") == "1"
+
+
+@pytest.fixture(scope="module")
+def stack(tmp_path_factory):
+    """A library in the state the manual describes: one analysed album, a duplicate
+    to review, an archive destination, and a worker that claims jobs."""
+    tmp_path = tmp_path_factory.mktemp("manual")
+    seeded = seed_album(tmp_path, ALBUM, ("forum.jpg", "colosseum.jpg"))
+    worker = start_worker(seeded, fake_vision_log=tmp_path / "vision.log")
+    server = Server(seeded).start()
+    try:
+        _prepare(server.base, seeded)
+        yield server
+    finally:
+        server.stop()
+        worker.terminate()
+        worker.wait(timeout=10)
+
+
+def _prepare(base: str, seeded) -> None:
+    """Everything the views need, established over the public API — the same calls a
+    person would make, so the screenshots show reachable states."""
+    with closing(session_client(base, timeout=30)) as client:
+        client.post("/api/v1/duplicates/detect").raise_for_status()
+        client.post("/api/v1/albums/proposals", json={}).raise_for_status()
+        destination = seeded.data / "archive"
+        destination.mkdir(exist_ok=True)
+        client.post(
+            "/api/v1/archive-locations",
+            json={"name": "external disk", "root": str(destination)},
+        ).raise_for_status()
+        wait_until(lambda: client.get("/api/v1/workflow").status_code == 200)
+
+
+def _capture(page, server, target: Path) -> list[str]:
+    target.mkdir(parents=True, exist_ok=True)
+    page.set_viewport_size(VIEWPORT)
+    written = []
+    for name, route, ready in SHOTS:
+        page.goto(f"{server.base}/app/{route}")
+        if ready:
+            page.get_by_test_id(ready).first.wait_for(timeout=30_000)
+        else:
+            page.locator("main h1").first.wait_for(timeout=30_000)
+        page.screenshot(path=str(target / f"{name}.png"))
+        written.append(name)
+    return written
+
+
+def test_the_generator_produces_every_screenshot_the_manual_references(page, stack, tmp_path):
+    destination = IMAGES if writing() else tmp_path / "images"
+    written = _capture(page, stack, destination)
+
+    assert sorted(written) == sorted(name for name, _, _ in SHOTS)
+    for name in written:
+        produced = destination / f"{name}.png"
+        assert produced.stat().st_size > 5_000, f"{name}.png is too small to be a view"
+
+    referenced = set(re.findall(r"images/([a-z-]+)\.png", "\n".join(
+        path.read_text() for path in DOCS.rglob("*.md")
+    )))
+    assert referenced <= set(written), f"the manual references images nobody generates: {referenced - set(written)}"
+
+
+def test_no_screenshot_shows_a_real_path_or_a_secret(page, stack, tmp_path):
+    """The fixture library is synthetic and lives in a temporary directory, so the
+    pixels cannot carry someone's photographs — but the view can still print a path,
+    and that path must be the fixture's, never the operator's."""
+    with closing(session_client(stack.base, timeout=30)) as client:
+        rendered = client.get("/api/v1/inventory/assets", params={"limit": 200}).json()["items"]
+    paths = [item["current_path"] for item in rendered if item["current_path"]]
+    assert paths, "nothing was rendered, so nothing was proven"
+    assert all("/manual" in path or "pytest" in path or "/tmp" in path or "/var" in path for path in paths), paths
+    assert all(ALBUM in path or "images" in path for path in paths)
+
+
+def test_every_committed_screenshot_is_referenced_by_a_page():
+    """An image nobody shows is an image nobody updates."""
+    if not IMAGES.is_dir():
+        pytest.skip("no screenshots have been generated into the repository yet")
+    text = "\n".join(path.read_text() for path in DOCS.rglob("*.md"))
+    orphans = sorted(
+        image.name for image in IMAGES.glob("*.png") if f"images/{image.name}" not in text
+    )
+    assert orphans == [], f"committed but unreferenced: {orphans}"

tests/integration/test_user_manual.py

@@ -0,0 +1,172 @@
+"""US09-04: the user manual, checked against the application it describes.
+
+The manual's most perishable claim is its error catalogue: a code renamed in a route
+leaves a page confidently explaining something that can no longer happen, and the
+operator meeting the new code finds nothing. So the catalogue is compared with the
+codes the application can actually emit, in both directions.
+
+The screenshots are generated and checked by
+``tests/e2e/test_user_manual_screenshots.py``.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parents[2]
+DOCS = REPO / "docs"
+STAGES = DOCS / "stages"
+PACKAGE = REPO / "photo_pipeline"
+ERRORS = (DOCS / "errors.md").read_text()
+ALL_DOCS = "\n".join(path.read_text() for path in DOCS.rglob("*.md"))
+
+# The shapes an error code is emitted in. Everything the API returns goes through one
+# of these, so the set they find is the set an operator can meet.
+EMITTERS = (
+    re.compile(r'_error\(\s*\d+,\s*"([a-z_]+)"'),
+    # The status may be a literal or an exception's own code, so match either.
+    re.compile(r'_envelope\(\s*[\w.]+,\s*"([a-z_]+)"'),
+    re.compile(r'"code":\s*"([a-z_]+)"'),
+    re.compile(r'code="([a-z_]+)"'),
+    re.compile(r'Refusal\(\s*\d+,\s*"([a-z_]+)"'),
+)
+
+# Codes raised deep in a service and mapped to a response by its route; they are what
+# the browser shows, so the manual owes the reader an entry.
+SERVICE_CODES = {
+    "lock_held",
+    "path_not_allowed",
+    "rename_recovery_required",
+    "stale_preflight",
+    "access_denied",
+    "too_many_attempts",
+}
+
+STAGE_PAGES = (
+    "inventory",
+    "duplicates",
+    "safety",
+    "analysis",
+    "albums",
+    "renames",
+    "uploads",
+    "archive",
+    "diagnostics",
+)
+
+
+def emitted_codes() -> set[str]:
+    found: set[str] = set()
+    for path in PACKAGE.rglob("*.py"):
+        text = path.read_text()
+        for pattern in EMITTERS:
+            found |= set(pattern.findall(text))
+    for code in SERVICE_CODES:
+        assert re.search(rf'"{code}"', "\n".join(p.read_text() for p in PACKAGE.rglob("*.py"))), (
+            f"{code} is documented as a service code but no longer exists"
+        )
+    return (found | SERVICE_CODES) - {"code"}
+
+
+def documented_codes() -> set[str]:
+    return set(re.findall(r"`([a-z][a-z_]{3,})`", ERRORS))
+
+
+# ── the error catalogue ──────────────────────────────────────────────────────
+
+
+def test_every_error_the_application_can_emit_is_documented():
+    missing = sorted(emitted_codes() - documented_codes())
+    assert missing == [], f"undocumented error codes: {missing}"
+
+
+def test_the_catalogue_documents_no_code_that_cannot_happen():
+    """Backticked snake_case in the catalogue is either a code, a setting, or one of
+    the handful of prose terms below."""
+    prose = {
+        "diagnostics",
+        "dry_run",
+        "approve_dry_run",
+        "current_path",
+        "archived_online",
+        "archived_offline",
+    }
+    settings = set(re.findall(r"PHOTO_PIPELINE_[A-Z_]+", ERRORS))
+    invented = sorted(
+        code
+        for code in documented_codes()
+        if code not in emitted_codes()
+        and code not in prose
+        and f"PHOTO_PIPELINE_{code.upper()}" not in settings
+    )
+    assert invented == [], f"documented but not emitted anywhere: {invented}"
+
+
+def test_the_protective_refusals_are_explained_as_intentional():
+    """These are the ones a reader is most likely to mistake for a broken
+    application, which is exactly when they start working around them."""
+    for code in ("lock_held", "rename_recovery_required", "stale_preflight", "not_runnable"):
+        row = next(line for line in ERRORS.splitlines() if line.startswith(f"| `{code}`"))
+        assert len(row.split("|")) >= 4, f"{code} has no cause and remedy"
+
+
+# ── the stage pages ──────────────────────────────────────────────────────────
+
+
+def test_there_is_one_page_per_workflow_stage():
+    missing = [name for name in STAGE_PAGES if not (STAGES / f"{name}.md").is_file()]
+    assert missing == [], f"stages without a page: {missing}"
+
+
+def test_every_stage_page_answers_the_four_questions():
+    """The shape is the point: a page that skips 'what it changes' is the page
+    somebody needed."""
+    for name in STAGE_PAGES:
+        text = (STAGES / f"{name}.md").read_text()
+        for question in ("What it is for.", "What you decide.", "What it changes.", "What it refuses."):
+            assert question in text, f"{name}.md does not answer: {question}"
+
+
+def test_every_stage_page_shows_a_screenshot():
+    for name in STAGE_PAGES:
+        text = (STAGES / f"{name}.md").read_text()
+        assert re.search(r"!\[[^\]]+\]\(\.\./images/[a-z-]+\.png\)", text), (
+            f"{name}.md carries no screenshot"
+        )
+
+
+def test_the_guided_pass_names_the_irreversible_stages():
+    first_pass = (DOCS / "first-pass.md").read_text()
+    for stage in ("renames", "uploads", "archive"):
+        assert f"stages/{stage}.md" in first_pass
+    assert "point" in first_pass.lower() and "no return" in first_pass.lower()
+
+
+def test_recovery_covers_each_interruption_the_application_can_survive():
+    recovery = (DOCS / "recovery.md").read_text()
+    for topic in (
+        "interrupted rename",
+        "uncertain upload",
+        "failed migration",
+        "restored backup",
+    ):
+        assert topic in recovery.lower(), f"recovery does not cover: {topic}"
+
+
+def test_the_manual_is_reachable_from_the_index():
+    index = (DOCS / "index.md").read_text()
+    for page in [f"stages/{name}.md" for name in STAGE_PAGES] + [
+        "first-pass.md",
+        "errors.md",
+        "recovery.md",
+    ]:
+        assert page in index, f"{page} is not linked from the index"
+
+
+def test_the_screenshots_are_declared_as_generated():
+    """The allow list had to be widened for them, so the reason has to be visible
+    where the widening is."""
+    config = (REPO / "work_item" / ".work-item.yml").read_text()
+    assert "docs/images/**" in config
+    assert "test_user_manual_screenshots.py" in config, "no note saying who generates them"

tests/story_traceability.json

@@ -204,10 +204,13 @@
     "US09-03": [
       "tests/integration/test_architecture_overview.py",
       "tests/e2e/test_docs_ui.py"
+    ],
+    "US09-04": [
+      "tests/integration/test_user_manual.py",
+      "tests/e2e/test_user_manual_screenshots.py"
     ]
   },
   "planned": [
-    "US09-04",
     "US09-05"
   ],
   "_planned_comment": "Accepted backlog stories that are not implemented yet. The release gate (US07-07) requires every story file to be either mapped to tests or listed here, so an unimplemented story is a visible decision rather than a hole in the matrix."