US08-03: Compose the Runtime and Mount the Library Safely (#98)

tests/e2e/test_compose_stack.py

@@ -0,0 +1,337 @@
+"""US08-03: the composition, actually composed.
+
+Nothing here is faked below the process boundary: Docker builds the image, Compose
+starts the migrate/API/worker containers against a temporary fixture library on a
+real bind mount, and every assertion is made over HTTP or against what the stack
+left in its data volume. The vision provider is the deterministic fake seam the
+other end-to-end suites use, because an upload of real photos to a real model is not
+what this story is about — the mount, the lock, the volume, and the restart are.
+
+The file contract (one API, one worker, migrations first, no committed values) is
+checked without a daemon in ``tests/integration/test_compose_runtime.py``; only the
+running proof needs Docker, and CI is where it runs unskipped (US08-04).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import socket
+import subprocess
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+import httpx
+import pytest
+from PIL import Image
+
+REPO = Path(__file__).resolve().parents[2]
+PROJECT = "photo-pipeline-us0803"
+IMAGE = "photo-pipeline-test:us08-03"
+SECRET = "compose-acceptance-secret"
+CONTAINER_LIBRARY = "/library"
+READY_TIMEOUT_SECONDS = 180
+JOB_TIMEOUT_SECONDS = 180
+UP_TIMEOUT_SECONDS = 30 * 60
+
+pytestmark = pytest.mark.container
+
+
+def compose_available() -> bool:
+    try:
+        return (
+            subprocess.run(
+                ["docker", "compose", "version"], capture_output=True, timeout=60
+            ).returncode
+            == 0
+        )
+    except (OSError, subprocess.SubprocessError):
+        return False
+
+
+needs_compose = pytest.mark.skipif(
+    not compose_available(), reason="no Docker daemon with the compose plugin"
+)
+
+
+def free_port() -> int:
+    with socket.socket() as sock:
+        sock.bind(("127.0.0.1", 0))
+        return sock.getsockname()[1]
+
+
+class Stack:
+    """The composition under test, plus the environment it was started with."""
+
+    def __init__(self, library: Path, env_file: Path) -> None:
+        self.library = library
+        self.port = free_port()
+        self.base = f"http://127.0.0.1:{self.port}"
+        # Compose reads the repository's own .env for substitution; the process
+        # environment wins over it, so the test's values are the ones that apply.
+        self.env = {
+            **os.environ,
+            "PHOTO_PIPELINE_IMAGE": IMAGE,
+            "PHOTO_PIPELINE_ENV_FILE": str(env_file),
+            "PHOTO_PIPELINE_LIBRARY_HOST_PATH": str(library),
+            "PHOTO_PIPELINE_LIBRARY_ROOTS": CONTAINER_LIBRARY,
+            "PHOTO_PIPELINE_PORT": str(self.port),
+            "PHOTO_PIPELINE_UID": str(os.getuid()),
+            "PHOTO_PIPELINE_GID": str(os.getgid()),
+        }
+
+    def compose(self, *args: str, check: bool = True, timeout: int = 300):
+        result = subprocess.run(
+            ["docker", "compose", "-p", PROJECT, "-f", str(REPO / "docker-compose.yml"), *args],
+            capture_output=True,
+            text=True,
+            env=self.env,
+            cwd=REPO,
+            timeout=timeout,
+        )
+        if check and result.returncode != 0:
+            raise AssertionError(
+                f"docker compose {' '.join(args)} failed:\n{result.stdout}\n{result.stderr}\n"
+                f"{self.compose('logs', '--tail', '80', check=False).stdout}"
+            )
+        return result
+
+    def wait_until_ready(self) -> None:
+        deadline = time.monotonic() + READY_TIMEOUT_SECONDS
+        while time.monotonic() < deadline:
+            try:
+                if httpx.get(f"{self.base}/api/v1/health/ready", timeout=5).status_code == 200:
+                    return
+            except httpx.HTTPError:
+                pass
+            time.sleep(0.5)
+        logs = self.compose("logs", "--tail", "120", check=False)
+        raise AssertionError(f"the stack never became ready:\n{logs.stdout}\n{logs.stderr}")
+
+    def client(self) -> httpx.Client:
+        client = httpx.Client(base_url=f"{self.base}/api/v1", timeout=60)
+        bootstrap = client.get("/session", headers={"X-Access-Secret": SECRET})
+        assert bootstrap.status_code == 200, bootstrap.text
+        client.headers["X-CSRF-Token"] = bootstrap.json()["csrf_token"]
+        return client
+
+
+@pytest.fixture(scope="module")
+def library() -> Path:
+    """A small fixture library on the host, mounted into both containers.
+
+    Not under pytest's ``tmp_path``: on macOS that is ``/var/folders/...``, which a
+    Docker VM (Colima, Docker Desktop) does not share, so the bind mount would arrive
+    empty and every assertion below would be about nothing. ``$HOME`` is shared by
+    every default configuration.
+    """
+    base = Path(
+        os.environ.get("PHOTO_PIPELINE_TEST_MOUNT_BASE", Path.home() / ".cache" / "photo-pipeline")
+    )
+    base.mkdir(parents=True, exist_ok=True)
+    root = Path(tempfile.mkdtemp(prefix="library-", dir=base))
+    for album, count in (("01_day", 2), ("02_night", 1)):
+        (root / album).mkdir()
+        for index in range(count):
+            colour = (40 * (index + 1), 90, 160)
+            Image.new("RGB", (64, 48), colour).save(root / album / f"{album}_{index}.jpg")
+    # The exclusion sentinel: it must never be discovered, counted, or analyzed.
+    (root / "_IGNORE").mkdir()
+    Image.new("RGB", (32, 32), (0, 0, 0)).save(root / "_IGNORE" / "sentinel.jpg")
+    try:
+        yield root
+    finally:
+        shutil.rmtree(root, ignore_errors=True)
+
+
+@pytest.fixture(scope="module")
+def env_file(tmp_path_factory) -> Path:
+    """Configuration and secrets come from the environment, so the test writes its
+    own file rather than borrowing the operator's."""
+    path = tmp_path_factory.mktemp("config") / "compose.env"
+    path.write_text(
+        "\n".join(
+            [
+                f"PHOTO_PIPELINE_ACCESS_SECRET={SECRET}",
+                "PHOTO_PIPELINE_LOG_FORMAT=text",
+                # The deterministic vision seam, in the data volume so both roles and
+                # the test can see it (concept §18).
+                "PHOTO_PIPELINE_FAKE_VISION_LOG=/data/vision.log",
+            ]
+        )
+        + "\n"
+    )
+    return path
+
+
+@pytest.fixture(scope="module")
+def stack(library, env_file):
+    if not compose_available():
+        pytest.skip("no Docker daemon with the compose plugin")
+    running = Stack(library, env_file)
+    running.compose("down", "--volumes", "--remove-orphans", check=False)
+    running.compose("up", "--detach", "--build", timeout=UP_TIMEOUT_SECONDS)
+    try:
+        running.wait_until_ready()
+        yield running
+    finally:
+        running.compose("down", "--volumes", "--remove-orphans", check=False, timeout=300)
+
+
+def await_job(client: httpx.Client, job_id: str, states=("succeeded",)) -> dict:
+    deadline = time.monotonic() + JOB_TIMEOUT_SECONDS
+    snapshot: dict = {}
+    while time.monotonic() < deadline:
+        response = client.get(f"/jobs/{job_id}")
+        if response.status_code == 200:
+            snapshot = response.json()
+            if snapshot["state"] in states:
+                return snapshot
+        time.sleep(0.5)
+    raise AssertionError(f"job {job_id} never reached {states}: {snapshot}")
+
+
+# ── the mounted library ──────────────────────────────────────────────────────
+
+
+@needs_compose
+def test_the_stack_scans_the_bind_mounted_library_at_its_container_paths(stack):
+    client = stack.client()
+    try:
+        scanned = client.post("/inventory/scan")
+        assert scanned.status_code == 200, scanned.text
+        assets = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
+    finally:
+        client.close()
+
+    assert len(assets) == 3, assets
+    paths = {asset["current_path"] for asset in assets}
+    assert all(path.startswith(CONTAINER_LIBRARY + "/") for path in paths), paths
+    assert not any("_IGNORE" in path or "sentinel" in path for path in paths)
+    # The host paths are what the operator mounted, and they are not what the
+    # application records: the roots are the container's.
+    assert not any(str(stack.library) in path for path in paths)
+
+
+@needs_compose
+def test_a_library_root_that_is_not_mounted_is_refused_at_startup(stack):
+    """The container-specific failure: configured roots that name nothing mounted."""
+    refused = stack.compose(
+        "run",
+        "--rm",
+        "--no-deps",
+        "--env",
+        "PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos",
+        "api",
+        "serve",
+        check=False,
+    )
+    assert refused.returncode == 5, refused.stdout + refused.stderr
+    assert "library root /srv/photos" in refused.stdout + refused.stderr
+
+
+@needs_compose
+def test_a_second_worker_is_refused_by_the_library_lock(stack):
+    """Not by convention: the running worker's lock is in the shared data volume."""
+    refused = stack.compose(
+        "run", "--rm", "--no-deps", "worker", "worker", "--id", "worker-2", check=False
+    )
+    assert refused.returncode == 2, refused.stdout + refused.stderr
+    assert "worker is already running" in refused.stdout + refused.stderr
+
+
+# ── restart, resume, and the data volume ─────────────────────────────────────
+
+
+@needs_compose
+def test_a_queued_job_resumes_after_both_containers_restart(stack):
+    client = stack.client()
+    try:
+        client.post("/inventory/scan").raise_for_status()
+        assets = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
+        for asset in assets:
+            decided = client.post(
+                "/safety/decisions", json={"asset_id": asset["id"], "decision": "sfw"}
+            )
+            assert decided.status_code == 200, decided.text
+
+        # Stop the worker first, so the job is provably still queued when the restart
+        # happens: a job that finished before the restart would prove nothing.
+        stack.compose("stop", "worker")
+        job = client.post("/analysis/jobs").json()
+        assert client.get(f"/jobs/{job['id']}").json()["state"] == "queued"
+    finally:
+        client.close()
+
+    stack.compose("restart", "api", "worker")
+    stack.wait_until_ready()
+
+    client = stack.client()  # the session is per API process, so it is re-bootstrapped
+    try:
+        finished = await_job(client, job["id"])
+        assert finished["state"] == "succeeded", finished
+        # The database is intact and the work is durable, not merely reported.
+        after = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
+        assert {asset["id"] for asset in after} == {asset["id"] for asset in assets}
+        analysed = client.get(f"/analysis/results/{assets[0]['id']}")
+        assert analysed.status_code == 200, analysed.text
+        assert analysed.json()["description"]
+        # Only the mounted library's own photos were analysed: the sentinel under
+        # _IGNORE is not an asset, so it can never have become an item of this job.
+        assert finished["progress"]["total"] == len(assets)
+    finally:
+        client.close()
+
+
+@needs_compose
+def test_the_data_volume_survives_recreating_the_containers(stack):
+    """`down` without `--volumes` then `up` is the upgrade path: state stays."""
+    client = stack.client()
+    try:
+        client.post("/inventory/scan").raise_for_status()
+        before = {a["id"] for a in client.get("/inventory/assets").json()["items"]}
+    finally:
+        client.close()
+
+    stack.compose("down", "--remove-orphans", timeout=300)
+    stack.compose("up", "--detach", timeout=UP_TIMEOUT_SECONDS)
+    stack.wait_until_ready()
+
+    client = stack.client()
+    try:
+        after = {a["id"] for a in client.get("/inventory/assets").json()["items"]}
+    finally:
+        client.close()
+    assert after == before, "the same assets, from the same database, on the same volume"
+
+
+# ── operating it ─────────────────────────────────────────────────────────────
+
+
+@needs_compose
+def test_backup_verify_and_diagnostics_run_as_container_commands(stack):
+    backup = stack.compose("run", "--rm", "--no-deps", "api", "backup", "--reason", "compose")
+    manifest = json.loads(backup.stdout[backup.stdout.index("{") :])
+    assert manifest["name"].startswith("2")
+
+    verified = stack.compose(
+        "run", "--rm", "--no-deps", "api", "verify-backup", f"/data/backups/{manifest['name']}"
+    )
+    assert json.loads(verified.stdout[verified.stdout.index("{") :])["ok"] is True
+
+    report = stack.compose("run", "--rm", "--no-deps", "api", "diagnostics")
+    diagnostics = json.loads(report.stdout[report.stdout.index("{") :])
+    components = {c["name"]: c["path"] for c in diagnostics["components"]}
+    # Database, WAL, thumbnail cache, and backups all live in the mounted volume.
+    for name in ("database", "write_ahead_log", "thumbnail_cache", "backups"):
+        assert components[name].startswith("/data/"), (name, components[name])
+    assert {tool["name"] for tool in diagnostics["tools"]} >= {"exiftool", "immich-go"}
+    # And the worker running beside this command is visible as the lock's holder.
+    assert diagnostics["locks"]["worker"]["role"] == "worker"
+
+
+if __name__ == "__main__":  # a quick way to run just this file
+    raise SystemExit(pytest.main([__file__, "-v", *sys.argv[1:]]))