338 lines
13 KiB
Python
338 lines
13 KiB
Python
"""US08-03: the composition, actually composed.
|
|
|
|
Nothing here is faked below the process boundary: Docker builds the image, Compose
|
|
starts the migrate/API/worker containers against a temporary fixture library on a
|
|
real bind mount, and every assertion is made over HTTP or against what the stack
|
|
left in its data volume. The vision provider is the deterministic fake seam the
|
|
other end-to-end suites use, because an upload of real photos to a real model is not
|
|
what this story is about — the mount, the lock, the volume, and the restart are.
|
|
|
|
The file contract (one API, one worker, migrations first, no committed values) is
|
|
checked without a daemon in ``tests/integration/test_compose_runtime.py``; only the
|
|
running proof needs Docker, and CI is where it runs unskipped (US08-04).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import shutil
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import httpx
|
|
import pytest
|
|
from PIL import Image
|
|
|
|
REPO = Path(__file__).resolve().parents[2]
|
|
PROJECT = "photo-pipeline-us0803"
|
|
IMAGE = "photo-pipeline-test:us08-03"
|
|
SECRET = "compose-acceptance-secret"
|
|
CONTAINER_LIBRARY = "/library"
|
|
READY_TIMEOUT_SECONDS = 180
|
|
JOB_TIMEOUT_SECONDS = 180
|
|
UP_TIMEOUT_SECONDS = 30 * 60
|
|
|
|
pytestmark = pytest.mark.container
|
|
|
|
|
|
def compose_available() -> bool:
|
|
try:
|
|
return (
|
|
subprocess.run(
|
|
["docker", "compose", "version"], capture_output=True, timeout=60
|
|
).returncode
|
|
== 0
|
|
)
|
|
except (OSError, subprocess.SubprocessError):
|
|
return False
|
|
|
|
|
|
needs_compose = pytest.mark.skipif(
|
|
not compose_available(), reason="no Docker daemon with the compose plugin"
|
|
)
|
|
|
|
|
|
def free_port() -> int:
|
|
with socket.socket() as sock:
|
|
sock.bind(("127.0.0.1", 0))
|
|
return sock.getsockname()[1]
|
|
|
|
|
|
class Stack:
|
|
"""The composition under test, plus the environment it was started with."""
|
|
|
|
def __init__(self, library: Path, env_file: Path) -> None:
|
|
self.library = library
|
|
self.port = free_port()
|
|
self.base = f"http://127.0.0.1:{self.port}"
|
|
# Compose reads the repository's own .env for substitution; the process
|
|
# environment wins over it, so the test's values are the ones that apply.
|
|
self.env = {
|
|
**os.environ,
|
|
"PHOTO_PIPELINE_IMAGE": IMAGE,
|
|
"PHOTO_PIPELINE_ENV_FILE": str(env_file),
|
|
"PHOTO_PIPELINE_LIBRARY_HOST_PATH": str(library),
|
|
"PHOTO_PIPELINE_LIBRARY_ROOTS": CONTAINER_LIBRARY,
|
|
"PHOTO_PIPELINE_PORT": str(self.port),
|
|
"PHOTO_PIPELINE_UID": str(os.getuid()),
|
|
"PHOTO_PIPELINE_GID": str(os.getgid()),
|
|
}
|
|
|
|
def compose(self, *args: str, check: bool = True, timeout: int = 300):
|
|
result = subprocess.run(
|
|
["docker", "compose", "-p", PROJECT, "-f", str(REPO / "docker-compose.yml"), *args],
|
|
capture_output=True,
|
|
text=True,
|
|
env=self.env,
|
|
cwd=REPO,
|
|
timeout=timeout,
|
|
)
|
|
if check and result.returncode != 0:
|
|
raise AssertionError(
|
|
f"docker compose {' '.join(args)} failed:\n{result.stdout}\n{result.stderr}\n"
|
|
f"{self.compose('logs', '--tail', '80', check=False).stdout}"
|
|
)
|
|
return result
|
|
|
|
def wait_until_ready(self) -> None:
|
|
deadline = time.monotonic() + READY_TIMEOUT_SECONDS
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
if httpx.get(f"{self.base}/api/v1/health/ready", timeout=5).status_code == 200:
|
|
return
|
|
except httpx.HTTPError:
|
|
pass
|
|
time.sleep(0.5)
|
|
logs = self.compose("logs", "--tail", "120", check=False)
|
|
raise AssertionError(f"the stack never became ready:\n{logs.stdout}\n{logs.stderr}")
|
|
|
|
def client(self) -> httpx.Client:
|
|
client = httpx.Client(base_url=f"{self.base}/api/v1", timeout=60)
|
|
bootstrap = client.get("/session", headers={"X-Access-Secret": SECRET})
|
|
assert bootstrap.status_code == 200, bootstrap.text
|
|
client.headers["X-CSRF-Token"] = bootstrap.json()["csrf_token"]
|
|
return client
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def library() -> Path:
|
|
"""A small fixture library on the host, mounted into both containers.
|
|
|
|
Not under pytest's ``tmp_path``: on macOS that is ``/var/folders/...``, which a
|
|
Docker VM (Colima, Docker Desktop) does not share, so the bind mount would arrive
|
|
empty and every assertion below would be about nothing. ``$HOME`` is shared by
|
|
every default configuration.
|
|
"""
|
|
base = Path(
|
|
os.environ.get("PHOTO_PIPELINE_TEST_MOUNT_BASE", Path.home() / ".cache" / "photo-pipeline")
|
|
)
|
|
base.mkdir(parents=True, exist_ok=True)
|
|
root = Path(tempfile.mkdtemp(prefix="library-", dir=base))
|
|
for album, count in (("01_day", 2), ("02_night", 1)):
|
|
(root / album).mkdir()
|
|
for index in range(count):
|
|
colour = (40 * (index + 1), 90, 160)
|
|
Image.new("RGB", (64, 48), colour).save(root / album / f"{album}_{index}.jpg")
|
|
# The exclusion sentinel: it must never be discovered, counted, or analyzed.
|
|
(root / "_IGNORE").mkdir()
|
|
Image.new("RGB", (32, 32), (0, 0, 0)).save(root / "_IGNORE" / "sentinel.jpg")
|
|
try:
|
|
yield root
|
|
finally:
|
|
shutil.rmtree(root, ignore_errors=True)
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def env_file(tmp_path_factory) -> Path:
|
|
"""Configuration and secrets come from the environment, so the test writes its
|
|
own file rather than borrowing the operator's."""
|
|
path = tmp_path_factory.mktemp("config") / "compose.env"
|
|
path.write_text(
|
|
"\n".join(
|
|
[
|
|
f"PHOTO_PIPELINE_ACCESS_SECRET={SECRET}",
|
|
"PHOTO_PIPELINE_LOG_FORMAT=text",
|
|
# The deterministic vision seam, in the data volume so both roles and
|
|
# the test can see it (concept §18).
|
|
"PHOTO_PIPELINE_FAKE_VISION_LOG=/data/vision.log",
|
|
]
|
|
)
|
|
+ "\n"
|
|
)
|
|
return path
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def stack(library, env_file):
|
|
if not compose_available():
|
|
pytest.skip("no Docker daemon with the compose plugin")
|
|
running = Stack(library, env_file)
|
|
running.compose("down", "--volumes", "--remove-orphans", check=False)
|
|
running.compose("up", "--detach", "--build", timeout=UP_TIMEOUT_SECONDS)
|
|
try:
|
|
running.wait_until_ready()
|
|
yield running
|
|
finally:
|
|
running.compose("down", "--volumes", "--remove-orphans", check=False, timeout=300)
|
|
|
|
|
|
def await_job(client: httpx.Client, job_id: str, states=("succeeded",)) -> dict:
|
|
deadline = time.monotonic() + JOB_TIMEOUT_SECONDS
|
|
snapshot: dict = {}
|
|
while time.monotonic() < deadline:
|
|
response = client.get(f"/jobs/{job_id}")
|
|
if response.status_code == 200:
|
|
snapshot = response.json()
|
|
if snapshot["state"] in states:
|
|
return snapshot
|
|
time.sleep(0.5)
|
|
raise AssertionError(f"job {job_id} never reached {states}: {snapshot}")
|
|
|
|
|
|
# ── the mounted library ──────────────────────────────────────────────────────
|
|
|
|
|
|
@needs_compose
|
|
def test_the_stack_scans_the_bind_mounted_library_at_its_container_paths(stack):
|
|
client = stack.client()
|
|
try:
|
|
scanned = client.post("/inventory/scan")
|
|
assert scanned.status_code == 200, scanned.text
|
|
assets = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
|
|
finally:
|
|
client.close()
|
|
|
|
assert len(assets) == 3, assets
|
|
paths = {asset["current_path"] for asset in assets}
|
|
assert all(path.startswith(CONTAINER_LIBRARY + "/") for path in paths), paths
|
|
assert not any("_IGNORE" in path or "sentinel" in path for path in paths)
|
|
# The host paths are what the operator mounted, and they are not what the
|
|
# application records: the roots are the container's.
|
|
assert not any(str(stack.library) in path for path in paths)
|
|
|
|
|
|
@needs_compose
|
|
def test_a_library_root_that_is_not_mounted_is_refused_at_startup(stack):
|
|
"""The container-specific failure: configured roots that name nothing mounted."""
|
|
refused = stack.compose(
|
|
"run",
|
|
"--rm",
|
|
"--no-deps",
|
|
"--env",
|
|
"PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos",
|
|
"api",
|
|
"serve",
|
|
check=False,
|
|
)
|
|
assert refused.returncode == 5, refused.stdout + refused.stderr
|
|
assert "library root /srv/photos" in refused.stdout + refused.stderr
|
|
|
|
|
|
@needs_compose
|
|
def test_a_second_worker_is_refused_by_the_library_lock(stack):
|
|
"""Not by convention: the running worker's lock is in the shared data volume."""
|
|
refused = stack.compose(
|
|
"run", "--rm", "--no-deps", "worker", "worker", "--id", "worker-2", check=False
|
|
)
|
|
assert refused.returncode == 2, refused.stdout + refused.stderr
|
|
assert "worker is already running" in refused.stdout + refused.stderr
|
|
|
|
|
|
# ── restart, resume, and the data volume ─────────────────────────────────────
|
|
|
|
|
|
@needs_compose
|
|
def test_a_queued_job_resumes_after_both_containers_restart(stack):
|
|
client = stack.client()
|
|
try:
|
|
client.post("/inventory/scan").raise_for_status()
|
|
assets = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
|
|
for asset in assets:
|
|
decided = client.post(
|
|
"/safety/decisions", json={"asset_id": asset["id"], "decision": "sfw"}
|
|
)
|
|
assert decided.status_code == 200, decided.text
|
|
|
|
# Stop the worker first, so the job is provably still queued when the restart
|
|
# happens: a job that finished before the restart would prove nothing.
|
|
stack.compose("stop", "worker")
|
|
job = client.post("/analysis/jobs").json()
|
|
assert client.get(f"/jobs/{job['id']}").json()["state"] == "queued"
|
|
finally:
|
|
client.close()
|
|
|
|
stack.compose("restart", "api", "worker")
|
|
stack.wait_until_ready()
|
|
|
|
client = stack.client() # the session is per API process, so it is re-bootstrapped
|
|
try:
|
|
finished = await_job(client, job["id"])
|
|
assert finished["state"] == "succeeded", finished
|
|
# The database is intact and the work is durable, not merely reported.
|
|
after = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
|
|
assert {asset["id"] for asset in after} == {asset["id"] for asset in assets}
|
|
analysed = client.get(f"/analysis/results/{assets[0]['id']}")
|
|
assert analysed.status_code == 200, analysed.text
|
|
assert analysed.json()["description"]
|
|
# Only the mounted library's own photos were analysed: the sentinel under
|
|
# _IGNORE is not an asset, so it can never have become an item of this job.
|
|
assert finished["progress"]["total"] == len(assets)
|
|
finally:
|
|
client.close()
|
|
|
|
|
|
@needs_compose
|
|
def test_the_data_volume_survives_recreating_the_containers(stack):
|
|
"""`down` without `--volumes` then `up` is the upgrade path: state stays."""
|
|
client = stack.client()
|
|
try:
|
|
client.post("/inventory/scan").raise_for_status()
|
|
before = {a["id"] for a in client.get("/inventory/assets").json()["items"]}
|
|
finally:
|
|
client.close()
|
|
|
|
stack.compose("down", "--remove-orphans", timeout=300)
|
|
stack.compose("up", "--detach", timeout=UP_TIMEOUT_SECONDS)
|
|
stack.wait_until_ready()
|
|
|
|
client = stack.client()
|
|
try:
|
|
after = {a["id"] for a in client.get("/inventory/assets").json()["items"]}
|
|
finally:
|
|
client.close()
|
|
assert after == before, "the same assets, from the same database, on the same volume"
|
|
|
|
|
|
# ── operating it ─────────────────────────────────────────────────────────────
|
|
|
|
|
|
@needs_compose
|
|
def test_backup_verify_and_diagnostics_run_as_container_commands(stack):
|
|
backup = stack.compose("run", "--rm", "--no-deps", "api", "backup", "--reason", "compose")
|
|
manifest = json.loads(backup.stdout[backup.stdout.index("{") :])
|
|
assert manifest["name"].startswith("2")
|
|
|
|
verified = stack.compose(
|
|
"run", "--rm", "--no-deps", "api", "verify-backup", f"/data/backups/{manifest['name']}"
|
|
)
|
|
assert json.loads(verified.stdout[verified.stdout.index("{") :])["ok"] is True
|
|
|
|
report = stack.compose("run", "--rm", "--no-deps", "api", "diagnostics")
|
|
diagnostics = json.loads(report.stdout[report.stdout.index("{") :])
|
|
components = {c["name"]: c["path"] for c in diagnostics["components"]}
|
|
# Database, WAL, thumbnail cache, and backups all live in the mounted volume.
|
|
for name in ("database", "write_ahead_log", "thumbnail_cache", "backups"):
|
|
assert components[name].startswith("/data/"), (name, components[name])
|
|
assert {tool["name"] for tool in diagnostics["tools"]} >= {"exiftool", "immich-go"}
|
|
# And the worker running beside this command is visible as the lock's holder.
|
|
assert diagnostics["locks"]["worker"]["role"] == "worker"
|
|
|
|
|
|
if __name__ == "__main__": # a quick way to run just this file
|
|
raise SystemExit(pytest.main([__file__, "-v", *sys.argv[1:]]))
|