diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..5ac697b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,22 @@ +# Deny-by-default build context (US08-02): the image must contain no secrets, no +# photos, no database, no logs, and no .git. An allow list is the only version of this +# rule that stays true when a new file appears in the working copy. +* + +!pyproject.toml +!alembic.ini +!README.md +!photo_pipeline +!migrations +!frontend +!docker + +# Nothing generated, even under an allowed directory. +**/__pycache__ +**/*.py[cod] +**/.DS_Store +**/*.env +**/*.log +**/*.db +**/*.db-* +**/*.sqlite* diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..fd3b57a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,115 @@ +# One image, two roles (US08-02). +# +# The application is not self-contained Python: it shells out to `exiftool` for every +# EXIF checkpoint and to `immich-go` for every upload, and it serves the static +# frontend from `frontend/`. All three are installed here at pinned versions, because +# an image whose external tools drift is an image whose metadata checkpoints and +# upload reports drift with them (concept §15, "External integration risks"). +# +# Everything is pinned: +# * the base image by tag *and* digest, so a moved tag cannot change the runtime; +# * exiftool by its Debian package version, verified against `exiftool -ver`; +# * immich-go by release version and per-architecture SHA-256 of the release asset. +# The verified versions become image labels and /etc/photo-pipeline/versions.json, +# which `python -m photo_pipeline diagnostics` reports — so a running container can +# prove what it contains instead of being trusted about it. +# +# The project is installed editable on purpose: `photo_pipeline.db` resolves +# `alembic.ini` and `migrations/`, and the API resolves `frontend/`, relative to the +# repository root. An editable install keeps that one layout instead of scattering the +# same files across site-packages and a source tree. + +ARG PYTHON_IMAGE=python:3.12.14-slim-trixie@sha256:2c941e860699f878900b0edc2403613c234d4b32eda3cc9fa7036991a2a63c4a + +# ── the uploader, fetched and verified outside the final layer ──────────────── +FROM ${PYTHON_IMAGE} AS uploader + +ARG IMMICH_GO_VERSION=0.32.0 +ARG IMMICH_GO_SHA256_AMD64=6e2ad86bafdadb9466d6515de7cb882726c0aea1a21d51164dff361d7d480a97 +ARG IMMICH_GO_SHA256_ARM64=2c35d9284baae407ef9540bdac5f488971b0bdc7be758a4d7c05ab270af09fdb + +COPY docker/fetch-immich-go.py /tmp/fetch-immich-go.py +RUN python /tmp/fetch-immich-go.py \ + --version "${IMMICH_GO_VERSION}" \ + --sha256-amd64 "${IMMICH_GO_SHA256_AMD64}" \ + --sha256-arm64 "${IMMICH_GO_SHA256_ARM64}" \ + --into /usr/local/bin \ + && /usr/local/bin/immich-go version + +# ── the application ────────────────────────────────────────────────────────── +FROM ${PYTHON_IMAGE} AS runtime + +ARG EXIFTOOL_VERSION=13.25+dfsg-1 +ARG IMMICH_GO_VERSION=0.32.0 +# The library is mounted from the host, so the container's identity must match the +# ownership that library already has: everything this application renames, writes +# EXIF into, or archives has to stay owned by the host user afterwards. +ARG UID=1000 +ARG GID=1000 + +LABEL org.opencontainers.image.title="photo_pipeline" \ + org.opencontainers.image.source="https://github.com/domverse/photoanalyzer" \ + io.photoanalyzer.exiftool.version="${EXIFTOOL_VERSION}" \ + io.photoanalyzer.immich-go.version="${IMMICH_GO_VERSION}" + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PATH=/opt/venv/bin:$PATH \ + PHOTO_PIPELINE_DATA_DIR=/data + +RUN set -eu; \ + apt-get update; \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + "libimage-exiftool-perl=${EXIFTOOL_VERSION}"; \ + rm -rf /var/lib/apt/lists/* + +COPY --from=uploader /usr/local/bin/immich-go /usr/local/bin/immich-go + +WORKDIR /app +COPY pyproject.toml alembic.ini README.md ./ +COPY photo_pipeline ./photo_pipeline +COPY migrations ./migrations +COPY frontend ./frontend +COPY docker/entrypoint.sh docker/healthcheck.sh /usr/local/bin/ + +# Runtime dependencies only: the `test` extra (pytest, playwright) and the `vision` +# extra stay out, and pip's build isolation leaves no build tooling behind. +RUN set -eu; \ + python -m venv /opt/venv; \ + /opt/venv/bin/pip install --no-cache-dir -e . + +# What is installed must be what was pinned, or the labels and the version record +# would be a claim rather than a fact. +RUN set -eu; \ + mkdir -p /etc/photo-pipeline; \ + exiftool_version="$(exiftool -ver)"; \ + immich_go_version="$(immich-go version | head -n 1 | tr -d '\r')"; \ + expected_exiftool="$(printf '%s' "${EXIFTOOL_VERSION}" | cut -d+ -f1 | cut -d- -f1)"; \ + [ "${exiftool_version}" = "${expected_exiftool}" ] \ + || { echo "exiftool ${exiftool_version} is not the pinned ${expected_exiftool}" >&2; exit 1; }; \ + case "${immich_go_version}" in \ + *"${IMMICH_GO_VERSION}"*) ;; \ + *) echo "immich-go '${immich_go_version}' is not pinned ${IMMICH_GO_VERSION}" >&2; exit 1 ;; \ + esac; \ + printf '{\n "exiftool": "%s",\n "immich-go": "%s"\n}\n' \ + "${exiftool_version}" "${IMMICH_GO_VERSION}" > /etc/photo-pipeline/versions.json + +# Non-root, with the host library's ownership. /data is the persistent volume; the +# photo library itself is mounted by the deployment (US08-03), never baked in. +RUN set -eu; \ + groupadd --gid "${GID}" --non-unique app; \ + useradd --uid "${UID}" --gid "${GID}" --non-unique --no-create-home --home-dir /app app; \ + mkdir -p /data; \ + chown "${UID}:${GID}" /data +USER ${UID}:${GID} + +EXPOSE 8000 + +# Readiness, not liveness: an unmigrated or misconfigured database answers +# /api/v1/health/ready with 503, and a container that cannot serve must not be +# reported healthy. The worker role has no endpoint, so its check is a no-op here. +HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \ + CMD ["/usr/local/bin/healthcheck.sh"] + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] +CMD ["serve"] diff --git a/README.md b/README.md index d7207fd..4cb63b4 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,55 @@ and the session cookie is marked `Secure` when that external scheme is HTTPS. He endpoints stay reachable without the secret so an orchestrator can restart the container; nothing else does. +## Container image (US08-02) + +One image runs either role. It is built from a clean checkout with no arguments: + +```bash +docker build -t photo-pipeline:dev . +``` + +Everything external is pinned, and the build fails rather than drifting: the Python +base image by tag *and* digest, `exiftool` by its Debian package version (verified +against `exiftool -ver`), and `immich-go` by release version and per-architecture +SHA-256 of the release asset. The verified versions become image labels and +`/etc/photo-pipeline/versions.json`, which `diagnostics` reports as `tools[].pinned` +beside the version actually installed — so a replaced binary shows up as a +`tool_version_drift` warning instead of as a misparsed upload report. + +| build argument | default | why change it | +|---|---|---| +| `UID` / `GID` | `1000` | must match the owner of the mounted photo library | +| `PYTHON_IMAGE` | pinned digest | upgrading the base image | +| `EXIFTOOL_VERSION` | Debian package version | upgrading exiftool | +| `IMMICH_GO_VERSION` + `IMMICH_GO_SHA256_AMD64`/`_ARM64` | pinned release | upgrading the uploader (take the digests from that release's `checksums.txt`) | + +The first argument is the role, and every other management command still works: + +```bash +docker run --rm -v /srv/photos:/srv/photos -v pp-data:/data \ + -e PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos photo-pipeline:dev migrate + +docker run -d -p 127.0.0.1:8000:8000 -v /srv/photos:/srv/photos -v pp-data:/data \ + -e PHOTO_PIPELINE_HOST=0.0.0.0 -e PHOTO_PIPELINE_ACCESS_SECRET=... \ + -e PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos photo-pipeline:dev serve + +docker run -d -v /srv/photos:/srv/photos -v pp-data:/data \ + -e PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos photo-pipeline:dev worker +``` + +One role per container: `serve` and `worker` each take the library process lock for +their role (US07-05), so no supervisor starts both. The container refuses to run as +UID 0 — files it renames or writes must keep the ownership the host library expects — +and `/data` is the persistent volume holding the database, journals, backups, and +thumbnail cache. Binding to `0.0.0.0` makes the access secret mandatory +([above](#reaching-it-through-a-hostname-or-proxy-us08-01)); `serve` refuses to start +without it. The declared `HEALTHCHECK` polls `/api/v1/health/ready`, so a container +whose database is unmigrated or misconfigured is never reported healthy. + +Composing the runtime and mounting the library is US08-03; publishing and deploying +the image is US08-04. + ## Testing One offline command runs the whole suite (unit, integration, and browser diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 0000000..7abd884 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# One entrypoint, one role per container (US08-02). +# +# The first argument is the management command the image runs — `serve` and `worker` +# are the two roles, and every other `python -m photo_pipeline` command (migrate, +# diagnostics, backup, restore, dry-run) is passed through unchanged so operating the +# container is operating the same CLI. No supervisor: two roles in one container would +# share a process lock they are each meant to hold alone (US07-05). +set -eu + +if [ "$(id -u)" = "0" ]; then + echo "refusing to run as root: start this image with a non-root UID/GID so files" \ + "it renames or writes keep the ownership the mounted library expects" >&2 + exit 1 +fi + +role="${1:-serve}" +# The health check has to know which role it is checking, and only the API has an +# endpoint to check. /tmp is writable for the unprivileged user; /run may not be. +printf '%s' "${role}" > "${PHOTO_PIPELINE_ROLE_FILE:-/tmp/photo-pipeline-role}" 2>/dev/null || true + +exec python -m photo_pipeline "$@" diff --git a/docker/fetch-immich-go.py b/docker/fetch-immich-go.py new file mode 100644 index 0000000..5cf8ae1 --- /dev/null +++ b/docker/fetch-immich-go.py @@ -0,0 +1,72 @@ +"""Download one pinned immich-go release and verify it before unpacking (US08-02). + +Run at image build time by the `uploader` stage, with the interpreter that is already +in the base image: no curl, no wget, and no download tooling in the layer that ships. +The checksum is not advisory — a release asset that does not match the pinned digest +is a failed build, not a warning, because the uploader's flags and report format are +what the upload parser is written against (concept §15). +""" + +from __future__ import annotations + +import argparse +import hashlib +import platform +import tarfile +import tempfile +import urllib.request +from pathlib import Path + +RELEASE_URL = "https://github.com/simulot/immich-go/releases/download/v{version}/{asset}" +# Debian/BuildKit architecture as the interpreter sees it → release asset name. +ASSETS = { + "x86_64": ("immich-go_Linux_x86_64.tar.gz", "amd64"), + "amd64": ("immich-go_Linux_x86_64.tar.gz", "amd64"), + "aarch64": ("immich-go_Linux_arm64.tar.gz", "arm64"), + "arm64": ("immich-go_Linux_arm64.tar.gz", "arm64"), +} +TIMEOUT_SECONDS = 300 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--version", required=True, help="immich-go release, without the v") + parser.add_argument("--sha256-amd64", required=True) + parser.add_argument("--sha256-arm64", required=True) + parser.add_argument("--into", default="/usr/local/bin") + args = parser.parse_args() + + machine = platform.machine().lower() + if machine not in ASSETS: + raise SystemExit(f"unsupported architecture: {machine}") + asset, arch = ASSETS[machine] + expected = {"amd64": args.sha256_amd64, "arm64": args.sha256_arm64}[arch] + url = RELEASE_URL.format(version=args.version, asset=asset) + + with urllib.request.urlopen(url, timeout=TIMEOUT_SECONDS) as response: # noqa: S310 + payload = response.read() + digest = hashlib.sha256(payload).hexdigest() + if digest != expected: + raise SystemExit(f"checksum mismatch for {url}: {digest} != {expected}") + + target = Path(args.into) + target.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory() as work: + archive = Path(work) / asset + archive.write_bytes(payload) + with tarfile.open(archive) as tar: + member = tar.getmember("immich-go") + # Extract exactly the one file this pin is about, by name, so nothing + # else in the archive can decide where it lands. + extracted = tar.extractfile(member) + if extracted is None: + raise SystemExit("release archive contains no immich-go binary") + binary = target / "immich-go" + binary.write_bytes(extracted.read()) + binary.chmod(0o755) + print(f"immich-go {args.version} ({arch}) verified {digest}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docker/healthcheck.sh b/docker/healthcheck.sh new file mode 100755 index 0000000..8e5688a --- /dev/null +++ b/docker/healthcheck.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# Container health for the `serve` role: readiness, not liveness (US08-02). +# +# /api/v1/health/ready is 503 until the database is reachable, migrated, and in WAL +# mode with foreign keys on, so an unmigrated or misconfigured container never reports +# healthy. Health endpoints need no session and no access secret, which is what lets an +# orchestrator restart a container it holds no credentials for (US08-01). +set -eu + +role="$(cat "${PHOTO_PIPELINE_ROLE_FILE:-/tmp/photo-pipeline-role}" 2>/dev/null || echo unknown)" +if [ "${role}" != "serve" ]; then + # ponytail: the worker has no endpoint to probe; its liveness is its lease and job + # heartbeat in the database. Add a `worker --health` command if a restart policy + # ever needs to act on it. + exit 0 +fi + +port="${PHOTO_PIPELINE_PORT:-8000}" +exec python - "${port}" <<'PY' +import sys +import urllib.error +import urllib.request + +url = f"http://127.0.0.1:{sys.argv[1]}/api/v1/health/ready" +try: + with urllib.request.urlopen(url, timeout=5) as response: # noqa: S310 — loopback + sys.exit(0 if response.status == 200 else 1) +except urllib.error.HTTPError as error: + print(f"not ready: HTTP {error.code}", file=sys.stderr) + sys.exit(1) +except OSError as error: + print(f"not ready: {error}", file=sys.stderr) + sys.exit(1) +PY diff --git a/photo_pipeline/integrations/exiftool.py b/photo_pipeline/integrations/exiftool.py index 780b04e..03b6ffb 100644 --- a/photo_pipeline/integrations/exiftool.py +++ b/photo_pipeline/integrations/exiftool.py @@ -12,8 +12,10 @@ nt-apply-list, pa-nsfw-filter). No dependency on the archived entry points. from __future__ import annotations +import functools import json import os +import shutil import subprocess from collections.abc import Iterable @@ -32,6 +34,27 @@ def _timeout() -> float: return DEFAULT_TIMEOUT_SECONDS +def find_binary(binary: str = "exiftool") -> str | None: + """Absolute path of exiftool, or ``None`` when it is not installed.""" + return shutil.which(binary) + + +@functools.lru_cache(maxsize=1) +def version() -> str | None: + """Reported exiftool version, or ``None`` when it is missing or unusable. + + Cached: it cannot change inside a running process, and diagnostics asks for it + on every report (US08-02, where a container image pins this version). + """ + try: + result = subprocess.run( + ["exiftool", "-ver"], capture_output=True, text=True, timeout=_timeout() + ) + except (OSError, subprocess.SubprocessError): + return None + return (result.stdout or "").strip() or None + + def read_keyword_sets(paths: Iterable[str]) -> dict[str, set[str]]: """Map each path to its lowercased set of ``Keywords`` + ``Subject`` values. diff --git a/photo_pipeline/services/diagnostics.py b/photo_pipeline/services/diagnostics.py index 9055cca..82537bc 100644 --- a/photo_pipeline/services/diagnostics.py +++ b/photo_pipeline/services/diagnostics.py @@ -15,10 +15,13 @@ application. from __future__ import annotations +import functools +import json import shutil from pathlib import Path from photo_pipeline.config import Config +from photo_pipeline.integrations import exiftool, immich_go from photo_pipeline.services import app_lock # Below this much free space, mutating stages should stop rather than risk a @@ -26,6 +29,12 @@ from photo_pipeline.services import app_lock LOW_DISK_BYTES = 1_000_000_000 CRITICAL_DISK_BYTES = 200_000_000 +# Written into the container image at build time (US08-02). The image pins exiftool +# and immich-go, and this file is how a running container reports which versions it +# was built with — so a drifted or missing binary is visible here rather than in a +# failed EXIF checkpoint or a misparsed upload report. +IMAGE_VERSIONS_FILE = Path("/etc/photo-pipeline/versions.json") + def _tree_bytes(path: Path) -> int: if not path.exists(): @@ -69,8 +78,47 @@ def disk(path: Path) -> dict: } +def _pinned_versions() -> dict[str, str]: + """The versions this image recorded at build time; empty outside a container.""" + try: + recorded = json.loads(IMAGE_VERSIONS_FILE.read_text()) + except (OSError, ValueError): + return {} + if not isinstance(recorded, dict): + return {} + return {str(name): str(value) for name, value in recorded.items()} + + +@functools.lru_cache(maxsize=4) +def _uploader_version(binary: str) -> str | None: + """Cached: the uploader cannot change version inside one process.""" + return immich_go.version(binary) + + +def tools(config: Config) -> list[dict]: + """The external executables the pipeline shells out to, and their versions. + + ``pinned`` is what the image was built against, ``version`` is what is actually + installed. They differ only when the binary was replaced or mounted over. + """ + return [ + { + "name": "exiftool", + "path": exiftool.find_binary(), + "version": exiftool.version(), + "pinned": _pinned_versions().get("exiftool"), + }, + { + "name": "immich-go", + "path": immich_go.find_binary(config.immich_go_binary), + "version": _uploader_version(config.immich_go_binary), + "pinned": _pinned_versions().get("immich-go"), + }, + ] + + def report(config: Config) -> dict: - """Sizes, disk headroom, warnings, and who currently holds the library lock.""" + """Sizes, disk headroom, tool versions, warnings, and who holds the library lock.""" database = config.database_path components = [ _component("database", database), @@ -132,6 +180,23 @@ def report(config: Config) -> dict: } ) + installed_tools = tools(config) + for tool in installed_tools: + # A missing tool is reported as ``version: null`` rather than warned about: on a + # development machine the uploader is legitimately absent, and the stages that + # need it already refuse to run. A *drifted* tool is different — the image pinned + # a version and something replaced it. + if tool["version"] and tool["pinned"] and tool["pinned"] not in tool["version"]: + warnings.append( + { + "code": "tool_version_drift", + "message": ( + f"{tool['name']} reports {tool['version']} but this image pinned " + f"{tool['pinned']}" + ), + } + ) + locks = {} for role in ("api", "worker"): holder = app_lock.LibraryLock(config, role).holder() @@ -151,6 +216,7 @@ def report(config: Config) -> dict: "components": components, "total_bytes": sum(component["bytes"] for component in components), "disk": space, + "tools": installed_tools, "warnings": warnings, "locks": locks, "legacy_activity": legacy, diff --git a/photo_pipeline/services/release.py b/photo_pipeline/services/release.py index 0c030e6..ed1f04d 100644 --- a/photo_pipeline/services/release.py +++ b/photo_pipeline/services/release.py @@ -49,8 +49,16 @@ STAGES: tuple[tuple[str, tuple[str, ...]], ...] = ( ("browser", ("tests/e2e",)), ) -# Skips the gate accepts, because they describe the machine rather than the code. -ALLOWED_SKIP_REASONS = ("exiftool not installed", "root ignores directory permissions") +# Skips the gate accepts, because they describe the machine rather than the code. The +# container ones (US08-02) belong here for the same reason exiftool does: the image +# build needs a Docker daemon and the network, and its definition is still checked +# offline in tests/integration/test_container_image.py. +ALLOWED_SKIP_REASONS = ( + "exiftool not installed", + "root ignores directory permissions", + "no Docker daemon available", + "bind-mount ownership is virtualised", +) class ReleaseError(RuntimeError): diff --git a/pyproject.toml b/pyproject.toml index 8ec5e04..0954922 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,4 +49,5 @@ markers = [ "phase_d: Phase D end-to-end acceptance (US04-06) — guarded rename API, fault, and browser journeys", "phase_e: Phase E end-to-end acceptance (US05-06) — upload preflight, uploader, and browser journeys", "phase_f: Phase F end-to-end acceptance (US06-06) — archive destination, transfer, and restore journeys", + "container: builds and runs the container image (US08-02) — needs a Docker daemon and network", ] diff --git a/tests/e2e/test_container_runtime.py b/tests/e2e/test_container_runtime.py new file mode 100644 index 0000000..e4a7c65 --- /dev/null +++ b/tests/e2e/test_container_runtime.py @@ -0,0 +1,306 @@ +"""US08-02: the built image, actually built and actually run. + +This is the acceptance test for the image itself, so nothing here is faked: Docker +builds from a clean context, the container starts under a chosen UID/GID against a +mounted data directory, and the assertions are made over HTTP and against the files +the container left on the host. + +It is skipped without a Docker daemon — the build also needs the network for the base +image, the pinned exiftool package, and the pinned uploader release. The contract the +Dockerfile itself has to keep (pins, non-root, health target, build context) is checked +offline in ``tests/integration/test_container_image.py``, so a machine without Docker +still fails on a broken image definition; only the running proof needs the daemon. CI +builds the image on every change (US08-04), which is where this runs unskipped. +""" + +from __future__ import annotations + +import json +import os +import platform +import re +import socket +import subprocess +import sys +import time +from pathlib import Path + +import httpx +import pytest + +REPO = Path(__file__).resolve().parents[2] +IMAGE = "photo-pipeline-test:us08-02" +SECRET = "container-acceptance-secret" +HOSTNAME = "photos.test" +READY_TIMEOUT_SECONDS = 120 +BUILD_TIMEOUT_SECONDS = 30 * 60 + +pytestmark = pytest.mark.container + + +def docker_available() -> bool: + try: + return subprocess.run(["docker", "info"], capture_output=True, timeout=60).returncode == 0 + except (OSError, subprocess.SubprocessError): + return False + + +needs_docker = pytest.mark.skipif(not docker_available(), reason="no Docker daemon available") + + +def docker(*args: str, check: bool = True, timeout: int = 120) -> subprocess.CompletedProcess: + result = subprocess.run( + ["docker", *args], capture_output=True, text=True, timeout=timeout + ) + if check and result.returncode != 0: + raise AssertionError(f"docker {' '.join(args)} failed:\n{result.stdout}\n{result.stderr}") + return result + + +def pins() -> dict[str, str]: + """The pinned versions, read from the Dockerfile that produced the image.""" + text = (REPO / "Dockerfile").read_text() + found = dict(re.findall(r"^ARG\s+([A-Z0-9_]+)=(.+)$", text, re.MULTILINE)) + return { + # The Debian package version carries a packaging suffix; exiftool reports the + # upstream version only. + "exiftool": found["EXIFTOOL_VERSION"].split("+")[0].split("-")[0], + "immich-go": found["IMMICH_GO_VERSION"], + } + + +def free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +@pytest.fixture(scope="module") +def image() -> str: + """Build from a clean checkout: the build context is the repository, unmodified.""" + if not docker_available(): + pytest.skip("no Docker daemon available") + docker( + "build", + "--build-arg", + f"UID={os.getuid()}", + "--build-arg", + f"GID={os.getgid()}", + "-t", + IMAGE, + str(REPO), + timeout=BUILD_TIMEOUT_SECONDS, + ) + return IMAGE + + +@pytest.fixture +def data_dir(tmp_path) -> Path: + data = tmp_path / "data" + data.mkdir() + return data + + +def run_detached(image: str, port: int, *args: str, data: Path | None = None) -> str: + """Start a container. ``data`` bind-mounts the host's data directory when the test + is about the files themselves; otherwise the image's own /data is used, because a + macOS bind mount arrives with an ownership the container did not choose.""" + result = docker( + "run", + "--detach", + "--rm", + "--publish", + f"127.0.0.1:{port}:8000", + *(("--volume", f"{data}:/data") if data is not None else ()), + "--env", + # Reachable from outside the container means reachable from another machine as + # far as the application is concerned, so the access secret is mandatory + # (US08-01) — the image must not weaken that. + "PHOTO_PIPELINE_HOST=0.0.0.0", + "--env", + f"PHOTO_PIPELINE_ACCESS_SECRET={SECRET}", + "--env", + f"PHOTO_PIPELINE_ALLOWED_HOSTS={HOSTNAME}", + image, + *args, + ) + return result.stdout.strip() + + +def wait_until_ready(base: str, container: str) -> None: + deadline = time.monotonic() + READY_TIMEOUT_SECONDS + while time.monotonic() < deadline: + try: + if httpx.get(f"{base}/api/v1/health/ready", timeout=5).status_code == 200: + return + except httpx.HTTPError: + pass + if docker("inspect", "-f", "{{.State.Running}}", container, check=False).stdout.strip() in ( + "false", + "", + ): + break + time.sleep(0.5) + logs = docker("logs", container, check=False) + raise AssertionError(f"container never became ready:\n{logs.stdout}\n{logs.stderr}") + + +@pytest.fixture +def serving(image): + port = free_port() + container = run_detached(image, port, "serve") + try: + base = f"http://127.0.0.1:{port}" + wait_until_ready(base, container) + yield base, container + finally: + docker("rm", "--force", container, check=False) + + +def session(base: str) -> httpx.Client: + client = httpx.Client(base_url=base, timeout=30) + bootstrap = client.get("/api/v1/session", headers={"X-Access-Secret": SECRET}) + assert bootstrap.status_code == 200, bootstrap.text + client.headers["X-CSRF-Token"] = bootstrap.json()["csrf_token"] + return client + + +# ── the image serves, and says what it contains ────────────────────────────── + + +@needs_docker +def test_the_container_serves_the_frontend_and_the_pinned_tool_versions(serving): + base, container = serving + + index = httpx.get(f"{base}/app/index.html", timeout=30) + assert index.status_code == 200 + assert " list[str]: + """The lines that do something: comments explain, they do not build.""" + return [line.strip() for line in text.splitlines() if line.strip() and not line.startswith("#")] + + +def build_args() -> dict[str, str]: + """Every ``ARG name=default`` in the Dockerfile — the pins, in other words.""" + found = {} + for match in re.finditer(r"^ARG\s+([A-Z0-9_]+)=(.+)$", DOCKERFILE, re.MULTILINE): + found[match.group(1)] = match.group(2).strip() + return found + + +# ── pins ───────────────────────────────────────────────────────────────────── + + +def test_the_base_image_is_pinned_by_version_and_digest(): + base = build_args()["PYTHON_IMAGE"] + assert base.startswith("python:3.12.") + assert "@sha256:" in base, "a tag can be moved; a digest cannot" + assert ":latest" not in DOCKERFILE + # Both stages build from the same pinned base, so the tool that was verified in one + # is the tool that ships in the other. + assert DOCKERFILE.count("FROM ${PYTHON_IMAGE}") == 2 + + +def test_exiftool_and_the_uploader_are_pinned_and_verified(): + args = build_args() + assert re.match(r"^\d+\.\d+", args["EXIFTOOL_VERSION"]) + assert re.match(r"^\d+\.\d+\.\d+$", args["IMMICH_GO_VERSION"]) + for arch in ("AMD64", "ARM64"): + assert SHA256.match(args[f"IMMICH_GO_SHA256_{arch}"]), arch + # The pinned exiftool package is installed by version, not by name alone. + assert 'libimage-exiftool-perl=${EXIFTOOL_VERSION}"' in DOCKERFILE + # And the build fails if what got installed is not what was pinned. + assert "is not the pinned" in DOCKERFILE and "is not pinned" in DOCKERFILE + + +def test_the_uploader_download_refuses_a_mismatching_checksum(tmp_path): + """The verification is the point of pinning a URL, so it is run, not read.""" + script = REPO / "docker" / "fetch-immich-go.py" + result = subprocess.run( + [ + sys.executable, + str(script), + "--version", + "0.0.0-does-not-exist", + "--sha256-amd64", + "0" * 64, + "--sha256-arm64", + "0" * 64, + "--into", + str(tmp_path), + ], + capture_output=True, + text=True, + ) + assert result.returncode != 0 + assert not list(tmp_path.iterdir()), "nothing is written before it is verified" + + +def test_the_recorded_versions_are_reported_by_diagnostics(tmp_path, monkeypatch): + """What the image records is what `diagnostics` answers with (acceptance criterion 2).""" + recorded = tmp_path / "versions.json" + recorded.write_text(json.dumps({"exiftool": "13.25", "immich-go": "0.32.0"})) + monkeypatch.setattr(diagnostics, "IMAGE_VERSIONS_FILE", recorded) + monkeypatch.setattr(diagnostics.exiftool, "version", lambda: "13.25") + monkeypatch.setattr(diagnostics, "_uploader_version", lambda _binary: "immich-go 0.32.0") + + config = Config(data_dir=tmp_path / "data") + reported = {tool["name"]: tool for tool in diagnostics.tools(config)} + + assert reported["exiftool"]["pinned"] == "13.25" + assert reported["immich-go"]["pinned"] == "0.32.0" + assert "0.32.0" in reported["immich-go"]["version"] + assert diagnostics.report(config)["tools"] == list(reported.values()) + assert "tool_version_drift" not in {w["code"] for w in diagnostics.report(config)["warnings"]} + + +def test_a_replaced_tool_is_reported_as_drift(tmp_path, monkeypatch): + recorded = tmp_path / "versions.json" + recorded.write_text(json.dumps({"exiftool": "13.25"})) + monkeypatch.setattr(diagnostics, "IMAGE_VERSIONS_FILE", recorded) + monkeypatch.setattr(diagnostics.exiftool, "version", lambda: "12.57") + + report = diagnostics.report(Config(data_dir=tmp_path / "data")) + + drift = [w for w in report["warnings"] if w["code"] == "tool_version_drift"] + assert drift and "13.25" in drift[0]["message"] and "12.57" in drift[0]["message"] + + +def test_versions_are_absent_rather_than_invented_outside_a_container(tmp_path, monkeypatch): + monkeypatch.setattr(diagnostics, "IMAGE_VERSIONS_FILE", tmp_path / "nothing.json") + for tool in diagnostics.tools(Config(data_dir=tmp_path / "data")): + assert tool["pinned"] is None + + +# ── the final layer ────────────────────────────────────────────────────────── + + +def test_no_test_or_build_tooling_is_installed_in_the_image(): + runtime = "\n".join(instructions(DOCKERFILE.split("AS runtime", 1)[1])) + for unwanted in ("[test]", "pytest", "playwright", "build-essential", "gcc"): + assert unwanted not in runtime, unwanted + assert "pip install --no-cache-dir -e ." in runtime + + +def test_neither_secrets_nor_library_data_can_enter_the_build_context(): + lines = instructions(DOCKERIGNORE) + assert lines[0] == "*", "the context is deny-by-default" + allowed = {line[1:] for line in lines if line.startswith("!")} + # Everything the Dockerfile copies has to be allowed, and nothing else is. + copied = { + source + for match in re.finditer(r"^COPY (?!--from)(.+)$", DOCKERFILE, re.MULTILINE) + for source in match.group(1).split()[:-1] + } + assert {Path(source).parts[0] for source in copied} <= allowed + assert not {"data", ".git", ".env", "tests", ".venv"} & allowed + for generated in ("**/*.env", "**/*.db", "**/*.log", "**/__pycache__"): + assert generated in lines, generated + + +def test_the_image_runs_as_a_non_root_user_whose_ids_are_build_arguments(): + args = build_args() + assert args["UID"] == "1000" and args["GID"] == "1000" + assert "USER ${UID}:${GID}" in DOCKERFILE + assert re.search(r"^USER (root|0)", DOCKERFILE, re.MULTILINE) is None + assert 'useradd --uid "${UID}" --gid "${GID}"' in DOCKERFILE + + +def test_the_health_check_is_readiness_and_the_default_role_is_serve(): + assert "HEALTHCHECK" in DOCKERFILE + assert "/usr/local/bin/healthcheck.sh" in DOCKERFILE + assert 'CMD ["serve"]' in DOCKERFILE + assert 'ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]' in DOCKERFILE + assert "/api/v1/health/ready" in HEALTHCHECK.read_text() + assert "/api/v1/health/live" not in HEALTHCHECK.read_text() + # No supervisor: one role per container (acceptance criterion 4). + for supervisor in ("supervisord", "s6-overlay", "runit"): + assert supervisor not in DOCKERFILE + + +@pytest.mark.parametrize("script", [ENTRYPOINT, HEALTHCHECK]) +def test_the_scripts_are_executable(script): + assert script.stat().st_mode & stat.S_IXUSR, f"{script.name} must be executable in git" + + +# ── the entrypoint, run as shell ───────────────────────────────────────────── + + +@pytest.fixture +def stubs(tmp_path): + """A PATH where ``python`` records its arguments and ``id`` can be told a UID.""" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + recorded = tmp_path / "argv" + python = bin_dir / "python" + python.write_text(f'#!/bin/sh\nprintf "%s\\n" "$@" > {recorded}\nexit 0\n') + python.chmod(0o755) + (bin_dir / "id").write_text('#!/bin/sh\nprintf "%s" "${STUB_UID:-1000}"\n') + (bin_dir / "id").chmod(0o755) + return bin_dir, recorded, tmp_path / "role" + + +def run_script(script: Path, *args, stubs, env=None): + bin_dir, recorded, role_file = stubs + result = subprocess.run( + ["/bin/sh", str(script), *args], + capture_output=True, + text=True, + env={ + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "PHOTO_PIPELINE_ROLE_FILE": str(role_file), + **(env or {}), + }, + ) + argv = recorded.read_text().splitlines() if recorded.exists() else [] + return result, argv + + +def test_the_container_refuses_to_run_as_root(stubs): + result, argv = run_script(ENTRYPOINT, "serve", stubs=stubs, env={"STUB_UID": "0"}) + assert result.returncode == 1 + assert "refusing to run as root" in result.stderr + assert argv == [], "the application is never started as root" + assert not stubs[2].exists(), "not even the role marker is written" + + +@pytest.mark.parametrize( + "given,expected", + [ + (["serve"], ["-m", "photo_pipeline", "serve"]), + (["worker", "--id", "worker-2"], ["-m", "photo_pipeline", "worker", "--id", "worker-2"]), + # Every other management command stays reachable: operating the container is + # operating the same CLI. + (["diagnostics"], ["-m", "photo_pipeline", "diagnostics"]), + ([], ["-m", "photo_pipeline"]), + ], +) +def test_the_role_selects_the_command_and_arguments_pass_through(given, expected, stubs): + result, argv = run_script(ENTRYPOINT, *given, stubs=stubs) + assert result.returncode == 0, result.stderr + assert argv == expected + + +def test_the_role_is_recorded_for_the_health_check(stubs): + run_script(ENTRYPOINT, "worker", stubs=stubs) + assert stubs[2].read_text() == "worker" + + +def test_the_health_check_only_probes_the_serving_role(stubs): + stubs[2].write_text("worker") + result, argv = run_script(HEALTHCHECK, stubs=stubs) + assert result.returncode == 0 and argv == [], "a worker has no endpoint to probe" + + stubs[2].write_text("serve") + result, argv = run_script(HEALTHCHECK, stubs=stubs, env={"PHOTO_PIPELINE_PORT": "9123"}) + assert result.returncode == 0, result.stderr + assert argv == ["-", "9123"], "the configured port is the one probed" + + +def test_the_health_check_fails_while_the_api_is_not_ready(stubs): + """No python stub: the real interpreter probes a port nothing is listening on.""" + stubs[2].write_text("serve") + result = subprocess.run( + ["/bin/sh", str(HEALTHCHECK)], + capture_output=True, + text=True, + env={ + "PATH": os.path.dirname(sys.executable) + os.pathsep + os.environ["PATH"], + "PHOTO_PIPELINE_ROLE_FILE": str(stubs[2]), + "PHOTO_PIPELINE_PORT": "1", + }, + ) + assert result.returncode == 1 + assert "not ready" in result.stderr + + +def test_the_scripts_are_posix_shell(): + """They run in the image's /bin/sh, which is dash — not bash.""" + shells = ["/bin/sh"] + ([dash] if (dash := shutil.which("dash")) else []) + for shell in shells: + for script in (ENTRYPOINT, HEALTHCHECK): + checked = subprocess.run([shell, "-n", str(script)], capture_output=True, text=True) + assert checked.returncode == 0, f"{shell} {script.name}: {checked.stderr}" diff --git a/tests/story_traceability.json b/tests/story_traceability.json index 24f9b13..ffa4051 100644 --- a/tests/story_traceability.json +++ b/tests/story_traceability.json @@ -178,10 +178,13 @@ "US08-01": [ "tests/unit/test_security_policy.py", "tests/integration/test_trusted_hosts.py" + ], + "US08-02": [ + "tests/integration/test_container_image.py", + "tests/e2e/test_container_runtime.py" ] }, "planned": [ - "US08-02", "US08-03", "US08-04", "US08-05"