261 lines
9.1 KiB
Python
261 lines
9.1 KiB
Python
"""The composed stack, as a fixture: build, provision, drive, destroy (US08-03/US08-05).
|
|
|
|
Extracted from ``tests/e2e/test_compose_stack.py`` when the container acceptance gate
|
|
needed the same stack under a different image, project, and library. One
|
|
implementation, because two would drift on exactly the details that make a container
|
|
test worth anything — the mount, the volume, the ports, and the teardown.
|
|
|
|
Nothing here fakes anything below the process boundary: Docker builds the image,
|
|
Compose starts the real containers, and every helper talks to them over HTTP or the
|
|
Docker CLI.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import shutil
|
|
import socket
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import httpx
|
|
import pytest
|
|
from PIL import Image
|
|
|
|
REPO = Path(__file__).resolve().parents[2]
|
|
COMPOSE_FILE = REPO / "docker-compose.yml"
|
|
CONTAINER_LIBRARY = "/library"
|
|
READY_TIMEOUT_SECONDS = 180
|
|
JOB_TIMEOUT_SECONDS = 300
|
|
UP_TIMEOUT_SECONDS = 30 * 60
|
|
|
|
|
|
def compose_available() -> bool:
|
|
try:
|
|
return (
|
|
subprocess.run(
|
|
["docker", "compose", "version"], capture_output=True, timeout=60
|
|
).returncode
|
|
== 0
|
|
)
|
|
except (OSError, subprocess.SubprocessError):
|
|
return False
|
|
|
|
|
|
def free_port() -> int:
|
|
with socket.socket() as sock:
|
|
sock.bind(("127.0.0.1", 0))
|
|
return sock.getsockname()[1]
|
|
|
|
|
|
def mount_base() -> Path:
|
|
"""A directory a Docker VM shares with the host.
|
|
|
|
Not pytest's ``tmp_path``: on macOS that is ``/var/folders/...``, which a Docker VM
|
|
(Colima, Docker Desktop) does not share, so the bind mount would arrive empty and
|
|
every assertion would be about nothing. ``$HOME`` is shared by every default
|
|
configuration.
|
|
"""
|
|
base = Path(
|
|
os.environ.get("PHOTO_PIPELINE_TEST_MOUNT_BASE", Path.home() / ".cache" / "photo-pipeline")
|
|
)
|
|
base.mkdir(parents=True, exist_ok=True)
|
|
return base
|
|
|
|
|
|
def temporary_library(albums: dict[str, int], *, prefix: str = "library-") -> Path:
|
|
"""A fixture library on a shareable host path, with the exclusion sentinel in it."""
|
|
root = Path(tempfile.mkdtemp(prefix=prefix, dir=mount_base()))
|
|
for position, (album, count) in enumerate(albums.items()):
|
|
(root / album).mkdir(parents=True, exist_ok=True)
|
|
for index in range(count):
|
|
# Distinct per album *and* index: two solid images of the same colour are
|
|
# byte-identical, which would make them a duplicate cluster by accident.
|
|
colour = (17 + 7 * index, 31 + 29 * position, 160 - 3 * index)
|
|
Image.new("RGB", (64, 48), colour).save(root / album / f"{album}_{index}.jpg")
|
|
# Never discovered, counted, analyzed, or uploaded — asserted from outside it.
|
|
(root / "_IGNORE").mkdir(exist_ok=True)
|
|
Image.new("RGB", (32, 32), (0, 0, 0)).save(root / "_IGNORE" / "sentinel.jpg")
|
|
return root
|
|
|
|
|
|
def remove_library(root: Path) -> None:
|
|
shutil.rmtree(root, ignore_errors=True)
|
|
|
|
|
|
def write_env_file(path: Path, secret: str, extra: dict[str, str] | None = None) -> Path:
|
|
"""Configuration and secrets come from the environment, so a test writes its own
|
|
file rather than borrowing the operator's ``.env``."""
|
|
lines = [
|
|
f"PHOTO_PIPELINE_ACCESS_SECRET={secret}",
|
|
"PHOTO_PIPELINE_LOG_FORMAT=text",
|
|
# The deterministic vision seam, in the data volume so both roles and the test
|
|
# can read it (concept §18).
|
|
"PHOTO_PIPELINE_FAKE_VISION_LOG=/data/vision.log",
|
|
*(f"{key}={value}" for key, value in (extra or {}).items()),
|
|
]
|
|
path.write_text("\n".join(lines) + "\n")
|
|
return path
|
|
|
|
|
|
class Stack:
|
|
"""The composition under test, plus the environment it was started with."""
|
|
|
|
def __init__(
|
|
self,
|
|
library: Path,
|
|
env_file: Path,
|
|
*,
|
|
project: str,
|
|
image: str,
|
|
secret: str,
|
|
) -> None:
|
|
self.library = library
|
|
self.project = project
|
|
self.secret = secret
|
|
self.port = free_port()
|
|
self.base = f"http://127.0.0.1:{self.port}"
|
|
# Compose reads the repository's own .env for substitution; the process
|
|
# environment wins over it, so the test's values are the ones that apply.
|
|
self.env = {
|
|
**os.environ,
|
|
"PHOTO_PIPELINE_IMAGE": image,
|
|
"PHOTO_PIPELINE_ENV_FILE": str(env_file),
|
|
"PHOTO_PIPELINE_LIBRARY_HOST_PATH": str(library),
|
|
"PHOTO_PIPELINE_LIBRARY_ROOTS": CONTAINER_LIBRARY,
|
|
"PHOTO_PIPELINE_PORT": str(self.port),
|
|
"PHOTO_PIPELINE_UID": str(os.getuid()),
|
|
"PHOTO_PIPELINE_GID": str(os.getgid()),
|
|
}
|
|
|
|
# ── the compose lifecycle ────────────────────────────────────────────────
|
|
|
|
def compose(self, *args: str, check: bool = True, timeout: int = 300):
|
|
result = subprocess.run(
|
|
["docker", "compose", "-p", self.project, "-f", str(COMPOSE_FILE), *args],
|
|
capture_output=True,
|
|
text=True,
|
|
env=self.env,
|
|
cwd=REPO,
|
|
timeout=timeout,
|
|
)
|
|
if check and result.returncode != 0:
|
|
raise AssertionError(
|
|
f"docker compose {' '.join(args)} failed:\n{result.stdout}\n{result.stderr}\n"
|
|
f"{self.compose('logs', '--tail', '80', check=False).stdout}"
|
|
)
|
|
return result
|
|
|
|
def up(self, *extra: str) -> None:
|
|
self.compose("up", "--detach", *extra, timeout=UP_TIMEOUT_SECONDS)
|
|
|
|
def down(self, *, volumes: bool = True) -> None:
|
|
self.compose(
|
|
"down",
|
|
*(("--volumes",) if volumes else ()),
|
|
"--remove-orphans",
|
|
check=False,
|
|
timeout=300,
|
|
)
|
|
|
|
def use_image(self, image: str) -> None:
|
|
"""Point the composition at another tag — the upgrade path (US08-05)."""
|
|
self.env["PHOTO_PIPELINE_IMAGE"] = image
|
|
|
|
def logs(self, *services: str) -> str:
|
|
result = self.compose("logs", *services, check=False)
|
|
return result.stdout + result.stderr
|
|
|
|
def wait_until_ready(self) -> None:
|
|
deadline = time.monotonic() + READY_TIMEOUT_SECONDS
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
if httpx.get(f"{self.base}/api/v1/health/ready", timeout=5).status_code == 200:
|
|
return
|
|
except httpx.HTTPError:
|
|
pass
|
|
time.sleep(0.5)
|
|
raise AssertionError(f"the stack never became ready:\n{self.logs()}")
|
|
|
|
# ── talking to it ────────────────────────────────────────────────────────
|
|
|
|
def client(self) -> httpx.Client:
|
|
"""A browser that has loaded the app: session cookie in the jar, token in a
|
|
header. The session belongs to the API process, so it is re-bootstrapped after
|
|
every restart."""
|
|
client = httpx.Client(base_url=f"{self.base}/api/v1", timeout=60)
|
|
bootstrap = client.get("/session", headers={"X-Access-Secret": self.secret})
|
|
assert bootstrap.status_code == 200, bootstrap.text
|
|
client.headers["X-CSRF-Token"] = bootstrap.json()["csrf_token"]
|
|
return client
|
|
|
|
|
|
def await_job(client: httpx.Client, job_id: str, states=("succeeded",)) -> dict:
|
|
deadline = time.monotonic() + JOB_TIMEOUT_SECONDS
|
|
snapshot: dict = {}
|
|
while time.monotonic() < deadline:
|
|
response = client.get(f"/jobs/{job_id}")
|
|
if response.status_code == 200:
|
|
snapshot = response.json()
|
|
if snapshot["state"] in states:
|
|
return snapshot
|
|
time.sleep(0.5)
|
|
raise AssertionError(f"job {job_id} never reached {states}: {snapshot}")
|
|
|
|
|
|
def build_image(tag: str, *, revision: str | None = None) -> str:
|
|
"""Build the application image; from a git revision's tree when one is named.
|
|
|
|
``revision`` is how the upgrade journey gets the *previous* version without a
|
|
registry: the tree of that commit is the build context, so what it produces is the
|
|
image that commit would have published.
|
|
"""
|
|
if revision is None:
|
|
subprocess.run(
|
|
[
|
|
"docker",
|
|
"build",
|
|
"--build-arg",
|
|
f"UID={os.getuid()}",
|
|
"--build-arg",
|
|
f"GID={os.getgid()}",
|
|
"-t",
|
|
tag,
|
|
str(REPO),
|
|
],
|
|
check=True,
|
|
timeout=UP_TIMEOUT_SECONDS,
|
|
)
|
|
return tag
|
|
archive = subprocess.run(
|
|
["git", "archive", "--format=tar", revision],
|
|
cwd=REPO,
|
|
capture_output=True,
|
|
check=True,
|
|
timeout=300,
|
|
).stdout
|
|
subprocess.run(
|
|
[
|
|
"docker",
|
|
"build",
|
|
"--build-arg",
|
|
f"UID={os.getuid()}",
|
|
"--build-arg",
|
|
f"GID={os.getgid()}",
|
|
"-t",
|
|
tag,
|
|
"-",
|
|
],
|
|
input=archive,
|
|
check=True,
|
|
timeout=UP_TIMEOUT_SECONDS,
|
|
)
|
|
return tag
|
|
|
|
|
|
needs_compose = pytest.mark.skipif(
|
|
not compose_available(), reason="no Docker daemon with the compose plugin"
|
|
)
|