US09-04: Write the User Manual with Generated Screenshots

tests/integration/test_user_manual.py

@@ -0,0 +1,172 @@
+"""US09-04: the user manual, checked against the application it describes.
+
+The manual's most perishable claim is its error catalogue: a code renamed in a route
+leaves a page confidently explaining something that can no longer happen, and the
+operator meeting the new code finds nothing. So the catalogue is compared with the
+codes the application can actually emit, in both directions.
+
+The screenshots are generated and checked by
+``tests/e2e/test_user_manual_screenshots.py``.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parents[2]
+DOCS = REPO / "docs"
+STAGES = DOCS / "stages"
+PACKAGE = REPO / "photo_pipeline"
+ERRORS = (DOCS / "errors.md").read_text()
+ALL_DOCS = "\n".join(path.read_text() for path in DOCS.rglob("*.md"))
+
+# The shapes an error code is emitted in. Everything the API returns goes through one
+# of these, so the set they find is the set an operator can meet.
+EMITTERS = (
+    re.compile(r'_error\(\s*\d+,\s*"([a-z_]+)"'),
+    # The status may be a literal or an exception's own code, so match either.
+    re.compile(r'_envelope\(\s*[\w.]+,\s*"([a-z_]+)"'),
+    re.compile(r'"code":\s*"([a-z_]+)"'),
+    re.compile(r'code="([a-z_]+)"'),
+    re.compile(r'Refusal\(\s*\d+,\s*"([a-z_]+)"'),
+)
+
+# Codes raised deep in a service and mapped to a response by its route; they are what
+# the browser shows, so the manual owes the reader an entry.
+SERVICE_CODES = {
+    "lock_held",
+    "path_not_allowed",
+    "rename_recovery_required",
+    "stale_preflight",
+    "access_denied",
+    "too_many_attempts",
+}
+
+STAGE_PAGES = (
+    "inventory",
+    "duplicates",
+    "safety",
+    "analysis",
+    "albums",
+    "renames",
+    "uploads",
+    "archive",
+    "diagnostics",
+)
+
+
+def emitted_codes() -> set[str]:
+    found: set[str] = set()
+    for path in PACKAGE.rglob("*.py"):
+        text = path.read_text()
+        for pattern in EMITTERS:
+            found |= set(pattern.findall(text))
+    for code in SERVICE_CODES:
+        assert re.search(rf'"{code}"', "\n".join(p.read_text() for p in PACKAGE.rglob("*.py"))), (
+            f"{code} is documented as a service code but no longer exists"
+        )
+    return (found | SERVICE_CODES) - {"code"}
+
+
+def documented_codes() -> set[str]:
+    return set(re.findall(r"`([a-z][a-z_]{3,})`", ERRORS))
+
+
+# ── the error catalogue ──────────────────────────────────────────────────────
+
+
+def test_every_error_the_application_can_emit_is_documented():
+    missing = sorted(emitted_codes() - documented_codes())
+    assert missing == [], f"undocumented error codes: {missing}"
+
+
+def test_the_catalogue_documents_no_code_that_cannot_happen():
+    """Backticked snake_case in the catalogue is either a code, a setting, or one of
+    the handful of prose terms below."""
+    prose = {
+        "diagnostics",
+        "dry_run",
+        "approve_dry_run",
+        "current_path",
+        "archived_online",
+        "archived_offline",
+    }
+    settings = set(re.findall(r"PHOTO_PIPELINE_[A-Z_]+", ERRORS))
+    invented = sorted(
+        code
+        for code in documented_codes()
+        if code not in emitted_codes()
+        and code not in prose
+        and f"PHOTO_PIPELINE_{code.upper()}" not in settings
+    )
+    assert invented == [], f"documented but not emitted anywhere: {invented}"
+
+
+def test_the_protective_refusals_are_explained_as_intentional():
+    """These are the ones a reader is most likely to mistake for a broken
+    application, which is exactly when they start working around them."""
+    for code in ("lock_held", "rename_recovery_required", "stale_preflight", "not_runnable"):
+        row = next(line for line in ERRORS.splitlines() if line.startswith(f"| `{code}`"))
+        assert len(row.split("|")) >= 4, f"{code} has no cause and remedy"
+
+
+# ── the stage pages ──────────────────────────────────────────────────────────
+
+
+def test_there_is_one_page_per_workflow_stage():
+    missing = [name for name in STAGE_PAGES if not (STAGES / f"{name}.md").is_file()]
+    assert missing == [], f"stages without a page: {missing}"
+
+
+def test_every_stage_page_answers_the_four_questions():
+    """The shape is the point: a page that skips 'what it changes' is the page
+    somebody needed."""
+    for name in STAGE_PAGES:
+        text = (STAGES / f"{name}.md").read_text()
+        for question in ("What it is for.", "What you decide.", "What it changes.", "What it refuses."):
+            assert question in text, f"{name}.md does not answer: {question}"
+
+
+def test_every_stage_page_shows_a_screenshot():
+    for name in STAGE_PAGES:
+        text = (STAGES / f"{name}.md").read_text()
+        assert re.search(r"!\[[^\]]+\]\(\.\./images/[a-z-]+\.png\)", text), (
+            f"{name}.md carries no screenshot"
+        )
+
+
+def test_the_guided_pass_names_the_irreversible_stages():
+    first_pass = (DOCS / "first-pass.md").read_text()
+    for stage in ("renames", "uploads", "archive"):
+        assert f"stages/{stage}.md" in first_pass
+    assert "point" in first_pass.lower() and "no return" in first_pass.lower()
+
+
+def test_recovery_covers_each_interruption_the_application_can_survive():
+    recovery = (DOCS / "recovery.md").read_text()
+    for topic in (
+        "interrupted rename",
+        "uncertain upload",
+        "failed migration",
+        "restored backup",
+    ):
+        assert topic in recovery.lower(), f"recovery does not cover: {topic}"
+
+
+def test_the_manual_is_reachable_from_the_index():
+    index = (DOCS / "index.md").read_text()
+    for page in [f"stages/{name}.md" for name in STAGE_PAGES] + [
+        "first-pass.md",
+        "errors.md",
+        "recovery.md",
+    ]:
+        assert page in index, f"{page} is not linked from the index"
+
+
+def test_the_screenshots_are_declared_as_generated():
+    """The allow list had to be widened for them, so the reason has to be visible
+    where the widening is."""
+    config = (REPO / "work_item" / ".work-item.yml").read_text()
+    assert "docs/images/**" in config
+    assert "test_user_manual_screenshots.py" in config, "no note saying who generates them"