Compare commits
1 Commits
us/US08-03
...
us/US08-02
| Author | SHA1 | Date | |
|---|---|---|---|
| e77af8c8b4 |
22
.dockerignore
Normal file
22
.dockerignore
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
# Deny-by-default build context (US08-02): the image must contain no secrets, no
|
||||||
|
# photos, no database, no logs, and no .git. An allow list is the only version of this
|
||||||
|
# rule that stays true when a new file appears in the working copy.
|
||||||
|
*
|
||||||
|
|
||||||
|
!pyproject.toml
|
||||||
|
!alembic.ini
|
||||||
|
!README.md
|
||||||
|
!photo_pipeline
|
||||||
|
!migrations
|
||||||
|
!frontend
|
||||||
|
!docker
|
||||||
|
|
||||||
|
# Nothing generated, even under an allowed directory.
|
||||||
|
**/__pycache__
|
||||||
|
**/*.py[cod]
|
||||||
|
**/.DS_Store
|
||||||
|
**/*.env
|
||||||
|
**/*.log
|
||||||
|
**/*.db
|
||||||
|
**/*.db-*
|
||||||
|
**/*.sqlite*
|
||||||
115
Dockerfile
Normal file
115
Dockerfile
Normal file
@@ -0,0 +1,115 @@
|
|||||||
|
# One image, two roles (US08-02).
|
||||||
|
#
|
||||||
|
# The application is not self-contained Python: it shells out to `exiftool` for every
|
||||||
|
# EXIF checkpoint and to `immich-go` for every upload, and it serves the static
|
||||||
|
# frontend from `frontend/`. All three are installed here at pinned versions, because
|
||||||
|
# an image whose external tools drift is an image whose metadata checkpoints and
|
||||||
|
# upload reports drift with them (concept §15, "External integration risks").
|
||||||
|
#
|
||||||
|
# Everything is pinned:
|
||||||
|
# * the base image by tag *and* digest, so a moved tag cannot change the runtime;
|
||||||
|
# * exiftool by its Debian package version, verified against `exiftool -ver`;
|
||||||
|
# * immich-go by release version and per-architecture SHA-256 of the release asset.
|
||||||
|
# The verified versions become image labels and /etc/photo-pipeline/versions.json,
|
||||||
|
# which `python -m photo_pipeline diagnostics` reports — so a running container can
|
||||||
|
# prove what it contains instead of being trusted about it.
|
||||||
|
#
|
||||||
|
# The project is installed editable on purpose: `photo_pipeline.db` resolves
|
||||||
|
# `alembic.ini` and `migrations/`, and the API resolves `frontend/`, relative to the
|
||||||
|
# repository root. An editable install keeps that one layout instead of scattering the
|
||||||
|
# same files across site-packages and a source tree.
|
||||||
|
|
||||||
|
ARG PYTHON_IMAGE=python:3.12.14-slim-trixie@sha256:2c941e860699f878900b0edc2403613c234d4b32eda3cc9fa7036991a2a63c4a
|
||||||
|
|
||||||
|
# ── the uploader, fetched and verified outside the final layer ────────────────
|
||||||
|
FROM ${PYTHON_IMAGE} AS uploader
|
||||||
|
|
||||||
|
ARG IMMICH_GO_VERSION=0.32.0
|
||||||
|
ARG IMMICH_GO_SHA256_AMD64=6e2ad86bafdadb9466d6515de7cb882726c0aea1a21d51164dff361d7d480a97
|
||||||
|
ARG IMMICH_GO_SHA256_ARM64=2c35d9284baae407ef9540bdac5f488971b0bdc7be758a4d7c05ab270af09fdb
|
||||||
|
|
||||||
|
COPY docker/fetch-immich-go.py /tmp/fetch-immich-go.py
|
||||||
|
RUN python /tmp/fetch-immich-go.py \
|
||||||
|
--version "${IMMICH_GO_VERSION}" \
|
||||||
|
--sha256-amd64 "${IMMICH_GO_SHA256_AMD64}" \
|
||||||
|
--sha256-arm64 "${IMMICH_GO_SHA256_ARM64}" \
|
||||||
|
--into /usr/local/bin \
|
||||||
|
&& /usr/local/bin/immich-go version
|
||||||
|
|
||||||
|
# ── the application ──────────────────────────────────────────────────────────
|
||||||
|
FROM ${PYTHON_IMAGE} AS runtime
|
||||||
|
|
||||||
|
ARG EXIFTOOL_VERSION=13.25+dfsg-1
|
||||||
|
ARG IMMICH_GO_VERSION=0.32.0
|
||||||
|
# The library is mounted from the host, so the container's identity must match the
|
||||||
|
# ownership that library already has: everything this application renames, writes
|
||||||
|
# EXIF into, or archives has to stay owned by the host user afterwards.
|
||||||
|
ARG UID=1000
|
||||||
|
ARG GID=1000
|
||||||
|
|
||||||
|
LABEL org.opencontainers.image.title="photo_pipeline" \
|
||||||
|
org.opencontainers.image.source="https://github.com/domverse/photoanalyzer" \
|
||||||
|
io.photoanalyzer.exiftool.version="${EXIFTOOL_VERSION}" \
|
||||||
|
io.photoanalyzer.immich-go.version="${IMMICH_GO_VERSION}"
|
||||||
|
|
||||||
|
ENV PYTHONUNBUFFERED=1 \
|
||||||
|
PYTHONDONTWRITEBYTECODE=1 \
|
||||||
|
PATH=/opt/venv/bin:$PATH \
|
||||||
|
PHOTO_PIPELINE_DATA_DIR=/data
|
||||||
|
|
||||||
|
RUN set -eu; \
|
||||||
|
apt-get update; \
|
||||||
|
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||||
|
"libimage-exiftool-perl=${EXIFTOOL_VERSION}"; \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
COPY --from=uploader /usr/local/bin/immich-go /usr/local/bin/immich-go
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
COPY pyproject.toml alembic.ini README.md ./
|
||||||
|
COPY photo_pipeline ./photo_pipeline
|
||||||
|
COPY migrations ./migrations
|
||||||
|
COPY frontend ./frontend
|
||||||
|
COPY docker/entrypoint.sh docker/healthcheck.sh /usr/local/bin/
|
||||||
|
|
||||||
|
# Runtime dependencies only: the `test` extra (pytest, playwright) and the `vision`
|
||||||
|
# extra stay out, and pip's build isolation leaves no build tooling behind.
|
||||||
|
RUN set -eu; \
|
||||||
|
python -m venv /opt/venv; \
|
||||||
|
/opt/venv/bin/pip install --no-cache-dir -e .
|
||||||
|
|
||||||
|
# What is installed must be what was pinned, or the labels and the version record
|
||||||
|
# would be a claim rather than a fact.
|
||||||
|
RUN set -eu; \
|
||||||
|
mkdir -p /etc/photo-pipeline; \
|
||||||
|
exiftool_version="$(exiftool -ver)"; \
|
||||||
|
immich_go_version="$(immich-go version | head -n 1 | tr -d '\r')"; \
|
||||||
|
expected_exiftool="$(printf '%s' "${EXIFTOOL_VERSION}" | cut -d+ -f1 | cut -d- -f1)"; \
|
||||||
|
[ "${exiftool_version}" = "${expected_exiftool}" ] \
|
||||||
|
|| { echo "exiftool ${exiftool_version} is not the pinned ${expected_exiftool}" >&2; exit 1; }; \
|
||||||
|
case "${immich_go_version}" in \
|
||||||
|
*"${IMMICH_GO_VERSION}"*) ;; \
|
||||||
|
*) echo "immich-go '${immich_go_version}' is not pinned ${IMMICH_GO_VERSION}" >&2; exit 1 ;; \
|
||||||
|
esac; \
|
||||||
|
printf '{\n "exiftool": "%s",\n "immich-go": "%s"\n}\n' \
|
||||||
|
"${exiftool_version}" "${IMMICH_GO_VERSION}" > /etc/photo-pipeline/versions.json
|
||||||
|
|
||||||
|
# Non-root, with the host library's ownership. /data is the persistent volume; the
|
||||||
|
# photo library itself is mounted by the deployment (US08-03), never baked in.
|
||||||
|
RUN set -eu; \
|
||||||
|
groupadd --gid "${GID}" --non-unique app; \
|
||||||
|
useradd --uid "${UID}" --gid "${GID}" --non-unique --no-create-home --home-dir /app app; \
|
||||||
|
mkdir -p /data; \
|
||||||
|
chown "${UID}:${GID}" /data
|
||||||
|
USER ${UID}:${GID}
|
||||||
|
|
||||||
|
EXPOSE 8000
|
||||||
|
|
||||||
|
# Readiness, not liveness: an unmigrated or misconfigured database answers
|
||||||
|
# /api/v1/health/ready with 503, and a container that cannot serve must not be
|
||||||
|
# reported healthy. The worker role has no endpoint, so its check is a no-op here.
|
||||||
|
HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
|
||||||
|
CMD ["/usr/local/bin/healthcheck.sh"]
|
||||||
|
|
||||||
|
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
|
||||||
|
CMD ["serve"]
|
||||||
49
README.md
49
README.md
@@ -97,6 +97,55 @@ and the session cookie is marked `Secure` when that external scheme is HTTPS. He
|
|||||||
endpoints stay reachable without the secret so an orchestrator can restart the
|
endpoints stay reachable without the secret so an orchestrator can restart the
|
||||||
container; nothing else does.
|
container; nothing else does.
|
||||||
|
|
||||||
|
## Container image (US08-02)
|
||||||
|
|
||||||
|
One image runs either role. It is built from a clean checkout with no arguments:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker build -t photo-pipeline:dev .
|
||||||
|
```
|
||||||
|
|
||||||
|
Everything external is pinned, and the build fails rather than drifting: the Python
|
||||||
|
base image by tag *and* digest, `exiftool` by its Debian package version (verified
|
||||||
|
against `exiftool -ver`), and `immich-go` by release version and per-architecture
|
||||||
|
SHA-256 of the release asset. The verified versions become image labels and
|
||||||
|
`/etc/photo-pipeline/versions.json`, which `diagnostics` reports as `tools[].pinned`
|
||||||
|
beside the version actually installed — so a replaced binary shows up as a
|
||||||
|
`tool_version_drift` warning instead of as a misparsed upload report.
|
||||||
|
|
||||||
|
| build argument | default | why change it |
|
||||||
|
|---|---|---|
|
||||||
|
| `UID` / `GID` | `1000` | must match the owner of the mounted photo library |
|
||||||
|
| `PYTHON_IMAGE` | pinned digest | upgrading the base image |
|
||||||
|
| `EXIFTOOL_VERSION` | Debian package version | upgrading exiftool |
|
||||||
|
| `IMMICH_GO_VERSION` + `IMMICH_GO_SHA256_AMD64`/`_ARM64` | pinned release | upgrading the uploader (take the digests from that release's `checksums.txt`) |
|
||||||
|
|
||||||
|
The first argument is the role, and every other management command still works:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker run --rm -v /srv/photos:/srv/photos -v pp-data:/data \
|
||||||
|
-e PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos photo-pipeline:dev migrate
|
||||||
|
|
||||||
|
docker run -d -p 127.0.0.1:8000:8000 -v /srv/photos:/srv/photos -v pp-data:/data \
|
||||||
|
-e PHOTO_PIPELINE_HOST=0.0.0.0 -e PHOTO_PIPELINE_ACCESS_SECRET=... \
|
||||||
|
-e PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos photo-pipeline:dev serve
|
||||||
|
|
||||||
|
docker run -d -v /srv/photos:/srv/photos -v pp-data:/data \
|
||||||
|
-e PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos photo-pipeline:dev worker
|
||||||
|
```
|
||||||
|
|
||||||
|
One role per container: `serve` and `worker` each take the library process lock for
|
||||||
|
their role (US07-05), so no supervisor starts both. The container refuses to run as
|
||||||
|
UID 0 — files it renames or writes must keep the ownership the host library expects —
|
||||||
|
and `/data` is the persistent volume holding the database, journals, backups, and
|
||||||
|
thumbnail cache. Binding to `0.0.0.0` makes the access secret mandatory
|
||||||
|
([above](#reaching-it-through-a-hostname-or-proxy-us08-01)); `serve` refuses to start
|
||||||
|
without it. The declared `HEALTHCHECK` polls `/api/v1/health/ready`, so a container
|
||||||
|
whose database is unmigrated or misconfigured is never reported healthy.
|
||||||
|
|
||||||
|
Composing the runtime and mounting the library is US08-03; publishing and deploying
|
||||||
|
the image is US08-04.
|
||||||
|
|
||||||
## Testing
|
## Testing
|
||||||
|
|
||||||
One offline command runs the whole suite (unit, integration, and browser
|
One offline command runs the whole suite (unit, integration, and browser
|
||||||
|
|||||||
22
docker/entrypoint.sh
Executable file
22
docker/entrypoint.sh
Executable file
@@ -0,0 +1,22 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# One entrypoint, one role per container (US08-02).
|
||||||
|
#
|
||||||
|
# The first argument is the management command the image runs — `serve` and `worker`
|
||||||
|
# are the two roles, and every other `python -m photo_pipeline` command (migrate,
|
||||||
|
# diagnostics, backup, restore, dry-run) is passed through unchanged so operating the
|
||||||
|
# container is operating the same CLI. No supervisor: two roles in one container would
|
||||||
|
# share a process lock they are each meant to hold alone (US07-05).
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
if [ "$(id -u)" = "0" ]; then
|
||||||
|
echo "refusing to run as root: start this image with a non-root UID/GID so files" \
|
||||||
|
"it renames or writes keep the ownership the mounted library expects" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
role="${1:-serve}"
|
||||||
|
# The health check has to know which role it is checking, and only the API has an
|
||||||
|
# endpoint to check. /tmp is writable for the unprivileged user; /run may not be.
|
||||||
|
printf '%s' "${role}" > "${PHOTO_PIPELINE_ROLE_FILE:-/tmp/photo-pipeline-role}" 2>/dev/null || true
|
||||||
|
|
||||||
|
exec python -m photo_pipeline "$@"
|
||||||
72
docker/fetch-immich-go.py
Normal file
72
docker/fetch-immich-go.py
Normal file
@@ -0,0 +1,72 @@
|
|||||||
|
"""Download one pinned immich-go release and verify it before unpacking (US08-02).
|
||||||
|
|
||||||
|
Run at image build time by the `uploader` stage, with the interpreter that is already
|
||||||
|
in the base image: no curl, no wget, and no download tooling in the layer that ships.
|
||||||
|
The checksum is not advisory — a release asset that does not match the pinned digest
|
||||||
|
is a failed build, not a warning, because the uploader's flags and report format are
|
||||||
|
what the upload parser is written against (concept §15).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import platform
|
||||||
|
import tarfile
|
||||||
|
import tempfile
|
||||||
|
import urllib.request
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
RELEASE_URL = "https://github.com/simulot/immich-go/releases/download/v{version}/{asset}"
|
||||||
|
# Debian/BuildKit architecture as the interpreter sees it → release asset name.
|
||||||
|
ASSETS = {
|
||||||
|
"x86_64": ("immich-go_Linux_x86_64.tar.gz", "amd64"),
|
||||||
|
"amd64": ("immich-go_Linux_x86_64.tar.gz", "amd64"),
|
||||||
|
"aarch64": ("immich-go_Linux_arm64.tar.gz", "arm64"),
|
||||||
|
"arm64": ("immich-go_Linux_arm64.tar.gz", "arm64"),
|
||||||
|
}
|
||||||
|
TIMEOUT_SECONDS = 300
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--version", required=True, help="immich-go release, without the v")
|
||||||
|
parser.add_argument("--sha256-amd64", required=True)
|
||||||
|
parser.add_argument("--sha256-arm64", required=True)
|
||||||
|
parser.add_argument("--into", default="/usr/local/bin")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
machine = platform.machine().lower()
|
||||||
|
if machine not in ASSETS:
|
||||||
|
raise SystemExit(f"unsupported architecture: {machine}")
|
||||||
|
asset, arch = ASSETS[machine]
|
||||||
|
expected = {"amd64": args.sha256_amd64, "arm64": args.sha256_arm64}[arch]
|
||||||
|
url = RELEASE_URL.format(version=args.version, asset=asset)
|
||||||
|
|
||||||
|
with urllib.request.urlopen(url, timeout=TIMEOUT_SECONDS) as response: # noqa: S310
|
||||||
|
payload = response.read()
|
||||||
|
digest = hashlib.sha256(payload).hexdigest()
|
||||||
|
if digest != expected:
|
||||||
|
raise SystemExit(f"checksum mismatch for {url}: {digest} != {expected}")
|
||||||
|
|
||||||
|
target = Path(args.into)
|
||||||
|
target.mkdir(parents=True, exist_ok=True)
|
||||||
|
with tempfile.TemporaryDirectory() as work:
|
||||||
|
archive = Path(work) / asset
|
||||||
|
archive.write_bytes(payload)
|
||||||
|
with tarfile.open(archive) as tar:
|
||||||
|
member = tar.getmember("immich-go")
|
||||||
|
# Extract exactly the one file this pin is about, by name, so nothing
|
||||||
|
# else in the archive can decide where it lands.
|
||||||
|
extracted = tar.extractfile(member)
|
||||||
|
if extracted is None:
|
||||||
|
raise SystemExit("release archive contains no immich-go binary")
|
||||||
|
binary = target / "immich-go"
|
||||||
|
binary.write_bytes(extracted.read())
|
||||||
|
binary.chmod(0o755)
|
||||||
|
print(f"immich-go {args.version} ({arch}) verified {digest}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
34
docker/healthcheck.sh
Executable file
34
docker/healthcheck.sh
Executable file
@@ -0,0 +1,34 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# Container health for the `serve` role: readiness, not liveness (US08-02).
|
||||||
|
#
|
||||||
|
# /api/v1/health/ready is 503 until the database is reachable, migrated, and in WAL
|
||||||
|
# mode with foreign keys on, so an unmigrated or misconfigured container never reports
|
||||||
|
# healthy. Health endpoints need no session and no access secret, which is what lets an
|
||||||
|
# orchestrator restart a container it holds no credentials for (US08-01).
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
role="$(cat "${PHOTO_PIPELINE_ROLE_FILE:-/tmp/photo-pipeline-role}" 2>/dev/null || echo unknown)"
|
||||||
|
if [ "${role}" != "serve" ]; then
|
||||||
|
# ponytail: the worker has no endpoint to probe; its liveness is its lease and job
|
||||||
|
# heartbeat in the database. Add a `worker --health` command if a restart policy
|
||||||
|
# ever needs to act on it.
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
port="${PHOTO_PIPELINE_PORT:-8000}"
|
||||||
|
exec python - "${port}" <<'PY'
|
||||||
|
import sys
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
url = f"http://127.0.0.1:{sys.argv[1]}/api/v1/health/ready"
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(url, timeout=5) as response: # noqa: S310 — loopback
|
||||||
|
sys.exit(0 if response.status == 200 else 1)
|
||||||
|
except urllib.error.HTTPError as error:
|
||||||
|
print(f"not ready: HTTP {error.code}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
except OSError as error:
|
||||||
|
print(f"not ready: {error}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
PY
|
||||||
@@ -12,8 +12,10 @@ nt-apply-list, pa-nsfw-filter). No dependency on the archived entry points.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import functools
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
|
|
||||||
@@ -32,6 +34,27 @@ def _timeout() -> float:
|
|||||||
return DEFAULT_TIMEOUT_SECONDS
|
return DEFAULT_TIMEOUT_SECONDS
|
||||||
|
|
||||||
|
|
||||||
|
def find_binary(binary: str = "exiftool") -> str | None:
|
||||||
|
"""Absolute path of exiftool, or ``None`` when it is not installed."""
|
||||||
|
return shutil.which(binary)
|
||||||
|
|
||||||
|
|
||||||
|
@functools.lru_cache(maxsize=1)
|
||||||
|
def version() -> str | None:
|
||||||
|
"""Reported exiftool version, or ``None`` when it is missing or unusable.
|
||||||
|
|
||||||
|
Cached: it cannot change inside a running process, and diagnostics asks for it
|
||||||
|
on every report (US08-02, where a container image pins this version).
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
["exiftool", "-ver"], capture_output=True, text=True, timeout=_timeout()
|
||||||
|
)
|
||||||
|
except (OSError, subprocess.SubprocessError):
|
||||||
|
return None
|
||||||
|
return (result.stdout or "").strip() or None
|
||||||
|
|
||||||
|
|
||||||
def read_keyword_sets(paths: Iterable[str]) -> dict[str, set[str]]:
|
def read_keyword_sets(paths: Iterable[str]) -> dict[str, set[str]]:
|
||||||
"""Map each path to its lowercased set of ``Keywords`` + ``Subject`` values.
|
"""Map each path to its lowercased set of ``Keywords`` + ``Subject`` values.
|
||||||
|
|
||||||
|
|||||||
@@ -15,10 +15,13 @@ application.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import functools
|
||||||
|
import json
|
||||||
import shutil
|
import shutil
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from photo_pipeline.config import Config
|
from photo_pipeline.config import Config
|
||||||
|
from photo_pipeline.integrations import exiftool, immich_go
|
||||||
from photo_pipeline.services import app_lock
|
from photo_pipeline.services import app_lock
|
||||||
|
|
||||||
# Below this much free space, mutating stages should stop rather than risk a
|
# Below this much free space, mutating stages should stop rather than risk a
|
||||||
@@ -26,6 +29,12 @@ from photo_pipeline.services import app_lock
|
|||||||
LOW_DISK_BYTES = 1_000_000_000
|
LOW_DISK_BYTES = 1_000_000_000
|
||||||
CRITICAL_DISK_BYTES = 200_000_000
|
CRITICAL_DISK_BYTES = 200_000_000
|
||||||
|
|
||||||
|
# Written into the container image at build time (US08-02). The image pins exiftool
|
||||||
|
# and immich-go, and this file is how a running container reports which versions it
|
||||||
|
# was built with — so a drifted or missing binary is visible here rather than in a
|
||||||
|
# failed EXIF checkpoint or a misparsed upload report.
|
||||||
|
IMAGE_VERSIONS_FILE = Path("/etc/photo-pipeline/versions.json")
|
||||||
|
|
||||||
|
|
||||||
def _tree_bytes(path: Path) -> int:
|
def _tree_bytes(path: Path) -> int:
|
||||||
if not path.exists():
|
if not path.exists():
|
||||||
@@ -69,8 +78,47 @@ def disk(path: Path) -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _pinned_versions() -> dict[str, str]:
|
||||||
|
"""The versions this image recorded at build time; empty outside a container."""
|
||||||
|
try:
|
||||||
|
recorded = json.loads(IMAGE_VERSIONS_FILE.read_text())
|
||||||
|
except (OSError, ValueError):
|
||||||
|
return {}
|
||||||
|
if not isinstance(recorded, dict):
|
||||||
|
return {}
|
||||||
|
return {str(name): str(value) for name, value in recorded.items()}
|
||||||
|
|
||||||
|
|
||||||
|
@functools.lru_cache(maxsize=4)
|
||||||
|
def _uploader_version(binary: str) -> str | None:
|
||||||
|
"""Cached: the uploader cannot change version inside one process."""
|
||||||
|
return immich_go.version(binary)
|
||||||
|
|
||||||
|
|
||||||
|
def tools(config: Config) -> list[dict]:
|
||||||
|
"""The external executables the pipeline shells out to, and their versions.
|
||||||
|
|
||||||
|
``pinned`` is what the image was built against, ``version`` is what is actually
|
||||||
|
installed. They differ only when the binary was replaced or mounted over.
|
||||||
|
"""
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"name": "exiftool",
|
||||||
|
"path": exiftool.find_binary(),
|
||||||
|
"version": exiftool.version(),
|
||||||
|
"pinned": _pinned_versions().get("exiftool"),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "immich-go",
|
||||||
|
"path": immich_go.find_binary(config.immich_go_binary),
|
||||||
|
"version": _uploader_version(config.immich_go_binary),
|
||||||
|
"pinned": _pinned_versions().get("immich-go"),
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def report(config: Config) -> dict:
|
def report(config: Config) -> dict:
|
||||||
"""Sizes, disk headroom, warnings, and who currently holds the library lock."""
|
"""Sizes, disk headroom, tool versions, warnings, and who holds the library lock."""
|
||||||
database = config.database_path
|
database = config.database_path
|
||||||
components = [
|
components = [
|
||||||
_component("database", database),
|
_component("database", database),
|
||||||
@@ -132,6 +180,23 @@ def report(config: Config) -> dict:
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
installed_tools = tools(config)
|
||||||
|
for tool in installed_tools:
|
||||||
|
# A missing tool is reported as ``version: null`` rather than warned about: on a
|
||||||
|
# development machine the uploader is legitimately absent, and the stages that
|
||||||
|
# need it already refuse to run. A *drifted* tool is different — the image pinned
|
||||||
|
# a version and something replaced it.
|
||||||
|
if tool["version"] and tool["pinned"] and tool["pinned"] not in tool["version"]:
|
||||||
|
warnings.append(
|
||||||
|
{
|
||||||
|
"code": "tool_version_drift",
|
||||||
|
"message": (
|
||||||
|
f"{tool['name']} reports {tool['version']} but this image pinned "
|
||||||
|
f"{tool['pinned']}"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
locks = {}
|
locks = {}
|
||||||
for role in ("api", "worker"):
|
for role in ("api", "worker"):
|
||||||
holder = app_lock.LibraryLock(config, role).holder()
|
holder = app_lock.LibraryLock(config, role).holder()
|
||||||
@@ -151,6 +216,7 @@ def report(config: Config) -> dict:
|
|||||||
"components": components,
|
"components": components,
|
||||||
"total_bytes": sum(component["bytes"] for component in components),
|
"total_bytes": sum(component["bytes"] for component in components),
|
||||||
"disk": space,
|
"disk": space,
|
||||||
|
"tools": installed_tools,
|
||||||
"warnings": warnings,
|
"warnings": warnings,
|
||||||
"locks": locks,
|
"locks": locks,
|
||||||
"legacy_activity": legacy,
|
"legacy_activity": legacy,
|
||||||
|
|||||||
@@ -49,8 +49,16 @@ STAGES: tuple[tuple[str, tuple[str, ...]], ...] = (
|
|||||||
("browser", ("tests/e2e",)),
|
("browser", ("tests/e2e",)),
|
||||||
)
|
)
|
||||||
|
|
||||||
# Skips the gate accepts, because they describe the machine rather than the code.
|
# Skips the gate accepts, because they describe the machine rather than the code. The
|
||||||
ALLOWED_SKIP_REASONS = ("exiftool not installed", "root ignores directory permissions")
|
# container ones (US08-02) belong here for the same reason exiftool does: the image
|
||||||
|
# build needs a Docker daemon and the network, and its definition is still checked
|
||||||
|
# offline in tests/integration/test_container_image.py.
|
||||||
|
ALLOWED_SKIP_REASONS = (
|
||||||
|
"exiftool not installed",
|
||||||
|
"root ignores directory permissions",
|
||||||
|
"no Docker daemon available",
|
||||||
|
"bind-mount ownership is virtualised",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class ReleaseError(RuntimeError):
|
class ReleaseError(RuntimeError):
|
||||||
|
|||||||
@@ -49,4 +49,5 @@ markers = [
|
|||||||
"phase_d: Phase D end-to-end acceptance (US04-06) — guarded rename API, fault, and browser journeys",
|
"phase_d: Phase D end-to-end acceptance (US04-06) — guarded rename API, fault, and browser journeys",
|
||||||
"phase_e: Phase E end-to-end acceptance (US05-06) — upload preflight, uploader, and browser journeys",
|
"phase_e: Phase E end-to-end acceptance (US05-06) — upload preflight, uploader, and browser journeys",
|
||||||
"phase_f: Phase F end-to-end acceptance (US06-06) — archive destination, transfer, and restore journeys",
|
"phase_f: Phase F end-to-end acceptance (US06-06) — archive destination, transfer, and restore journeys",
|
||||||
|
"container: builds and runs the container image (US08-02) — needs a Docker daemon and network",
|
||||||
]
|
]
|
||||||
|
|||||||
306
tests/e2e/test_container_runtime.py
Normal file
306
tests/e2e/test_container_runtime.py
Normal file
@@ -0,0 +1,306 @@
|
|||||||
|
"""US08-02: the built image, actually built and actually run.
|
||||||
|
|
||||||
|
This is the acceptance test for the image itself, so nothing here is faked: Docker
|
||||||
|
builds from a clean context, the container starts under a chosen UID/GID against a
|
||||||
|
mounted data directory, and the assertions are made over HTTP and against the files
|
||||||
|
the container left on the host.
|
||||||
|
|
||||||
|
It is skipped without a Docker daemon — the build also needs the network for the base
|
||||||
|
image, the pinned exiftool package, and the pinned uploader release. The contract the
|
||||||
|
Dockerfile itself has to keep (pins, non-root, health target, build context) is checked
|
||||||
|
offline in ``tests/integration/test_container_image.py``, so a machine without Docker
|
||||||
|
still fails on a broken image definition; only the running proof needs the daemon. CI
|
||||||
|
builds the image on every change (US08-04), which is where this runs unskipped.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import platform
|
||||||
|
import re
|
||||||
|
import socket
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
REPO = Path(__file__).resolve().parents[2]
|
||||||
|
IMAGE = "photo-pipeline-test:us08-02"
|
||||||
|
SECRET = "container-acceptance-secret"
|
||||||
|
HOSTNAME = "photos.test"
|
||||||
|
READY_TIMEOUT_SECONDS = 120
|
||||||
|
BUILD_TIMEOUT_SECONDS = 30 * 60
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.container
|
||||||
|
|
||||||
|
|
||||||
|
def docker_available() -> bool:
|
||||||
|
try:
|
||||||
|
return subprocess.run(["docker", "info"], capture_output=True, timeout=60).returncode == 0
|
||||||
|
except (OSError, subprocess.SubprocessError):
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
needs_docker = pytest.mark.skipif(not docker_available(), reason="no Docker daemon available")
|
||||||
|
|
||||||
|
|
||||||
|
def docker(*args: str, check: bool = True, timeout: int = 120) -> subprocess.CompletedProcess:
|
||||||
|
result = subprocess.run(
|
||||||
|
["docker", *args], capture_output=True, text=True, timeout=timeout
|
||||||
|
)
|
||||||
|
if check and result.returncode != 0:
|
||||||
|
raise AssertionError(f"docker {' '.join(args)} failed:\n{result.stdout}\n{result.stderr}")
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def pins() -> dict[str, str]:
|
||||||
|
"""The pinned versions, read from the Dockerfile that produced the image."""
|
||||||
|
text = (REPO / "Dockerfile").read_text()
|
||||||
|
found = dict(re.findall(r"^ARG\s+([A-Z0-9_]+)=(.+)$", text, re.MULTILINE))
|
||||||
|
return {
|
||||||
|
# The Debian package version carries a packaging suffix; exiftool reports the
|
||||||
|
# upstream version only.
|
||||||
|
"exiftool": found["EXIFTOOL_VERSION"].split("+")[0].split("-")[0],
|
||||||
|
"immich-go": found["IMMICH_GO_VERSION"],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def free_port() -> int:
|
||||||
|
with socket.socket() as sock:
|
||||||
|
sock.bind(("127.0.0.1", 0))
|
||||||
|
return sock.getsockname()[1]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module")
|
||||||
|
def image() -> str:
|
||||||
|
"""Build from a clean checkout: the build context is the repository, unmodified."""
|
||||||
|
if not docker_available():
|
||||||
|
pytest.skip("no Docker daemon available")
|
||||||
|
docker(
|
||||||
|
"build",
|
||||||
|
"--build-arg",
|
||||||
|
f"UID={os.getuid()}",
|
||||||
|
"--build-arg",
|
||||||
|
f"GID={os.getgid()}",
|
||||||
|
"-t",
|
||||||
|
IMAGE,
|
||||||
|
str(REPO),
|
||||||
|
timeout=BUILD_TIMEOUT_SECONDS,
|
||||||
|
)
|
||||||
|
return IMAGE
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def data_dir(tmp_path) -> Path:
|
||||||
|
data = tmp_path / "data"
|
||||||
|
data.mkdir()
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def run_detached(image: str, port: int, *args: str, data: Path | None = None) -> str:
|
||||||
|
"""Start a container. ``data`` bind-mounts the host's data directory when the test
|
||||||
|
is about the files themselves; otherwise the image's own /data is used, because a
|
||||||
|
macOS bind mount arrives with an ownership the container did not choose."""
|
||||||
|
result = docker(
|
||||||
|
"run",
|
||||||
|
"--detach",
|
||||||
|
"--rm",
|
||||||
|
"--publish",
|
||||||
|
f"127.0.0.1:{port}:8000",
|
||||||
|
*(("--volume", f"{data}:/data") if data is not None else ()),
|
||||||
|
"--env",
|
||||||
|
# Reachable from outside the container means reachable from another machine as
|
||||||
|
# far as the application is concerned, so the access secret is mandatory
|
||||||
|
# (US08-01) — the image must not weaken that.
|
||||||
|
"PHOTO_PIPELINE_HOST=0.0.0.0",
|
||||||
|
"--env",
|
||||||
|
f"PHOTO_PIPELINE_ACCESS_SECRET={SECRET}",
|
||||||
|
"--env",
|
||||||
|
f"PHOTO_PIPELINE_ALLOWED_HOSTS={HOSTNAME}",
|
||||||
|
image,
|
||||||
|
*args,
|
||||||
|
)
|
||||||
|
return result.stdout.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def wait_until_ready(base: str, container: str) -> None:
|
||||||
|
deadline = time.monotonic() + READY_TIMEOUT_SECONDS
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
try:
|
||||||
|
if httpx.get(f"{base}/api/v1/health/ready", timeout=5).status_code == 200:
|
||||||
|
return
|
||||||
|
except httpx.HTTPError:
|
||||||
|
pass
|
||||||
|
if docker("inspect", "-f", "{{.State.Running}}", container, check=False).stdout.strip() in (
|
||||||
|
"false",
|
||||||
|
"",
|
||||||
|
):
|
||||||
|
break
|
||||||
|
time.sleep(0.5)
|
||||||
|
logs = docker("logs", container, check=False)
|
||||||
|
raise AssertionError(f"container never became ready:\n{logs.stdout}\n{logs.stderr}")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def serving(image):
|
||||||
|
port = free_port()
|
||||||
|
container = run_detached(image, port, "serve")
|
||||||
|
try:
|
||||||
|
base = f"http://127.0.0.1:{port}"
|
||||||
|
wait_until_ready(base, container)
|
||||||
|
yield base, container
|
||||||
|
finally:
|
||||||
|
docker("rm", "--force", container, check=False)
|
||||||
|
|
||||||
|
|
||||||
|
def session(base: str) -> httpx.Client:
|
||||||
|
client = httpx.Client(base_url=base, timeout=30)
|
||||||
|
bootstrap = client.get("/api/v1/session", headers={"X-Access-Secret": SECRET})
|
||||||
|
assert bootstrap.status_code == 200, bootstrap.text
|
||||||
|
client.headers["X-CSRF-Token"] = bootstrap.json()["csrf_token"]
|
||||||
|
return client
|
||||||
|
|
||||||
|
|
||||||
|
# ── the image serves, and says what it contains ──────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@needs_docker
|
||||||
|
def test_the_container_serves_the_frontend_and_the_pinned_tool_versions(serving):
|
||||||
|
base, container = serving
|
||||||
|
|
||||||
|
index = httpx.get(f"{base}/app/index.html", timeout=30)
|
||||||
|
assert index.status_code == 200
|
||||||
|
assert "<title" in index.text.lower(), "the application shell, not an API error"
|
||||||
|
|
||||||
|
client = session(base)
|
||||||
|
try:
|
||||||
|
tools = {tool["name"]: tool for tool in client.get("/api/v1/diagnostics").json()["tools"]}
|
||||||
|
finally:
|
||||||
|
client.close()
|
||||||
|
for name, pinned in pins().items():
|
||||||
|
assert tools[name]["pinned"] == pinned, name
|
||||||
|
# Recorded *and* installed: the reported version comes from running the binary.
|
||||||
|
assert pinned in tools[name]["version"], (name, tools[name])
|
||||||
|
assert tools[name]["path"], f"{name} is not on PATH inside the image"
|
||||||
|
|
||||||
|
logs = docker("logs", container, check=False)
|
||||||
|
assert SECRET not in logs.stdout + logs.stderr, "the access secret never reaches the log"
|
||||||
|
|
||||||
|
|
||||||
|
@needs_docker
|
||||||
|
def test_the_declared_health_check_reports_readiness(serving):
|
||||||
|
"""The declared HEALTHCHECK is readiness, so Docker's own verdict is the assertion."""
|
||||||
|
_, container = serving
|
||||||
|
deadline = time.monotonic() + READY_TIMEOUT_SECONDS
|
||||||
|
status = ""
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
status = docker(
|
||||||
|
"inspect", "-f", "{{.State.Health.Status}}", container, check=False
|
||||||
|
).stdout.strip()
|
||||||
|
if status == "healthy":
|
||||||
|
break
|
||||||
|
time.sleep(1)
|
||||||
|
assert status == "healthy"
|
||||||
|
|
||||||
|
probe = docker("exec", container, "/usr/local/bin/healthcheck.sh", check=False)
|
||||||
|
assert probe.returncode == 0
|
||||||
|
# Point the probe at a port nothing serves: the same script must fail, which is
|
||||||
|
# what makes the healthy verdict above evidence rather than a default.
|
||||||
|
unready = docker(
|
||||||
|
"exec",
|
||||||
|
"--env",
|
||||||
|
"PHOTO_PIPELINE_PORT=1",
|
||||||
|
container,
|
||||||
|
"/usr/local/bin/healthcheck.sh",
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
assert unready.returncode != 0
|
||||||
|
|
||||||
|
|
||||||
|
# ── identity: never root, always the configured owner ────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@needs_docker
|
||||||
|
def test_the_container_refuses_to_run_as_root(image, data_dir):
|
||||||
|
result = docker(
|
||||||
|
"run",
|
||||||
|
"--rm",
|
||||||
|
"--user",
|
||||||
|
"0:0",
|
||||||
|
"--volume",
|
||||||
|
f"{data_dir}:/data",
|
||||||
|
image,
|
||||||
|
"diagnostics",
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
assert result.returncode != 0
|
||||||
|
assert "refusing to run as root" in result.stderr + result.stdout
|
||||||
|
assert not list(data_dir.iterdir()), "a refused container writes nothing"
|
||||||
|
|
||||||
|
|
||||||
|
@needs_docker
|
||||||
|
def test_what_the_container_writes_is_owned_by_the_build_arguments(image):
|
||||||
|
"""The identity the image was built with is the identity on disk afterwards.
|
||||||
|
|
||||||
|
Asserted from inside the container so it holds on every host: a macOS bind mount
|
||||||
|
reports an ownership the container never chose. The host-side proof, which is what
|
||||||
|
the mounted library actually needs, is the Linux test below.
|
||||||
|
"""
|
||||||
|
port = free_port()
|
||||||
|
container = run_detached(image, port, "serve")
|
||||||
|
try:
|
||||||
|
wait_until_ready(f"http://127.0.0.1:{port}", container)
|
||||||
|
owner = docker(
|
||||||
|
"exec", container, "stat", "-c", "%u:%g", "/data/photo_pipeline.db"
|
||||||
|
).stdout.strip()
|
||||||
|
assert owner == f"{os.getuid()}:{os.getgid()}"
|
||||||
|
assert docker("exec", container, "id", "-u").stdout.strip() == str(os.getuid())
|
||||||
|
finally:
|
||||||
|
docker("rm", "--force", container, check=False)
|
||||||
|
|
||||||
|
|
||||||
|
@needs_docker
|
||||||
|
@pytest.mark.skipif(
|
||||||
|
platform.system() != "Linux",
|
||||||
|
reason="bind-mount ownership is virtualised by Docker Desktop on macOS/Windows",
|
||||||
|
)
|
||||||
|
def test_files_the_container_writes_keep_the_configured_ownership(image, data_dir):
|
||||||
|
docker("run", "--rm", "--volume", f"{data_dir}:/data", image, "migrate", timeout=300)
|
||||||
|
|
||||||
|
written = sorted(path for path in data_dir.rglob("*") if path.is_file())
|
||||||
|
assert written, "migrate creates the database in the mounted data directory"
|
||||||
|
for path in written:
|
||||||
|
assert (path.stat().st_uid, path.stat().st_gid) == (os.getuid(), os.getgid()), path
|
||||||
|
|
||||||
|
|
||||||
|
@needs_docker
|
||||||
|
def test_the_worker_role_runs_from_the_same_image(image, data_dir):
|
||||||
|
"""One image, two roles: the worker is the same entrypoint with another argument."""
|
||||||
|
port = free_port()
|
||||||
|
container = run_detached(image, port, "worker", "--id", "container-worker")
|
||||||
|
try:
|
||||||
|
# Taking the worker's library lock is the observable proof that it started,
|
||||||
|
# migrated, and reached its job loop — no sleep required (US07-05).
|
||||||
|
deadline = time.monotonic() + READY_TIMEOUT_SECONDS
|
||||||
|
lock = ""
|
||||||
|
while not lock and time.monotonic() < deadline:
|
||||||
|
assert docker("inspect", "-f", "{{.State.Running}}", container).stdout.strip() == (
|
||||||
|
"true"
|
||||||
|
), docker("logs", container, check=False).stdout
|
||||||
|
lock = docker("exec", container, "cat", "/data/worker.lock.json", check=False).stdout
|
||||||
|
time.sleep(0.5)
|
||||||
|
assert lock, docker("logs", container, check=False).stdout
|
||||||
|
assert json.loads(lock)["role"] == "worker"
|
||||||
|
|
||||||
|
role = docker("exec", container, "cat", "/tmp/photo-pipeline-role").stdout.strip()
|
||||||
|
assert role == "worker", "the health check can tell which role this container is"
|
||||||
|
finally:
|
||||||
|
docker("rm", "--force", container, check=False)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__": # a quick way to run just this file
|
||||||
|
raise SystemExit(pytest.main([__file__, "-v", *sys.argv[1:]]))
|
||||||
284
tests/integration/test_container_image.py
Normal file
284
tests/integration/test_container_image.py
Normal file
@@ -0,0 +1,284 @@
|
|||||||
|
"""US08-02: the image's build contract, its entrypoint, and its health check.
|
||||||
|
|
||||||
|
Building the image needs a Docker daemon and the network, which is what
|
||||||
|
``tests/e2e/test_container_runtime.py`` does. Everything that can be checked without
|
||||||
|
either is checked here, because the parts most likely to rot silently — a pin that
|
||||||
|
stopped being a pin, a build context that started including the library, a health
|
||||||
|
check pointed at liveness instead of readiness — are all readable from the files.
|
||||||
|
|
||||||
|
The entrypoint and health check are shell, so they are exercised as shell: run with a
|
||||||
|
stubbed ``id`` and ``python`` on ``PATH``, which is enough to prove the refusal, the
|
||||||
|
role marker, and the argument pass-through without a container.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import stat
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from photo_pipeline.config import Config
|
||||||
|
from photo_pipeline.services import diagnostics
|
||||||
|
|
||||||
|
REPO = Path(__file__).resolve().parents[2]
|
||||||
|
DOCKERFILE = (REPO / "Dockerfile").read_text()
|
||||||
|
DOCKERIGNORE = (REPO / ".dockerignore").read_text()
|
||||||
|
ENTRYPOINT = REPO / "docker" / "entrypoint.sh"
|
||||||
|
HEALTHCHECK = REPO / "docker" / "healthcheck.sh"
|
||||||
|
|
||||||
|
SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
||||||
|
|
||||||
|
|
||||||
|
def instructions(text: str) -> list[str]:
|
||||||
|
"""The lines that do something: comments explain, they do not build."""
|
||||||
|
return [line.strip() for line in text.splitlines() if line.strip() and not line.startswith("#")]
|
||||||
|
|
||||||
|
|
||||||
|
def build_args() -> dict[str, str]:
|
||||||
|
"""Every ``ARG name=default`` in the Dockerfile — the pins, in other words."""
|
||||||
|
found = {}
|
||||||
|
for match in re.finditer(r"^ARG\s+([A-Z0-9_]+)=(.+)$", DOCKERFILE, re.MULTILINE):
|
||||||
|
found[match.group(1)] = match.group(2).strip()
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
|
# ── pins ─────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_base_image_is_pinned_by_version_and_digest():
|
||||||
|
base = build_args()["PYTHON_IMAGE"]
|
||||||
|
assert base.startswith("python:3.12.")
|
||||||
|
assert "@sha256:" in base, "a tag can be moved; a digest cannot"
|
||||||
|
assert ":latest" not in DOCKERFILE
|
||||||
|
# Both stages build from the same pinned base, so the tool that was verified in one
|
||||||
|
# is the tool that ships in the other.
|
||||||
|
assert DOCKERFILE.count("FROM ${PYTHON_IMAGE}") == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_exiftool_and_the_uploader_are_pinned_and_verified():
|
||||||
|
args = build_args()
|
||||||
|
assert re.match(r"^\d+\.\d+", args["EXIFTOOL_VERSION"])
|
||||||
|
assert re.match(r"^\d+\.\d+\.\d+$", args["IMMICH_GO_VERSION"])
|
||||||
|
for arch in ("AMD64", "ARM64"):
|
||||||
|
assert SHA256.match(args[f"IMMICH_GO_SHA256_{arch}"]), arch
|
||||||
|
# The pinned exiftool package is installed by version, not by name alone.
|
||||||
|
assert 'libimage-exiftool-perl=${EXIFTOOL_VERSION}"' in DOCKERFILE
|
||||||
|
# And the build fails if what got installed is not what was pinned.
|
||||||
|
assert "is not the pinned" in DOCKERFILE and "is not pinned" in DOCKERFILE
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_uploader_download_refuses_a_mismatching_checksum(tmp_path):
|
||||||
|
"""The verification is the point of pinning a URL, so it is run, not read."""
|
||||||
|
script = REPO / "docker" / "fetch-immich-go.py"
|
||||||
|
result = subprocess.run(
|
||||||
|
[
|
||||||
|
sys.executable,
|
||||||
|
str(script),
|
||||||
|
"--version",
|
||||||
|
"0.0.0-does-not-exist",
|
||||||
|
"--sha256-amd64",
|
||||||
|
"0" * 64,
|
||||||
|
"--sha256-arm64",
|
||||||
|
"0" * 64,
|
||||||
|
"--into",
|
||||||
|
str(tmp_path),
|
||||||
|
],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
assert result.returncode != 0
|
||||||
|
assert not list(tmp_path.iterdir()), "nothing is written before it is verified"
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_recorded_versions_are_reported_by_diagnostics(tmp_path, monkeypatch):
|
||||||
|
"""What the image records is what `diagnostics` answers with (acceptance criterion 2)."""
|
||||||
|
recorded = tmp_path / "versions.json"
|
||||||
|
recorded.write_text(json.dumps({"exiftool": "13.25", "immich-go": "0.32.0"}))
|
||||||
|
monkeypatch.setattr(diagnostics, "IMAGE_VERSIONS_FILE", recorded)
|
||||||
|
monkeypatch.setattr(diagnostics.exiftool, "version", lambda: "13.25")
|
||||||
|
monkeypatch.setattr(diagnostics, "_uploader_version", lambda _binary: "immich-go 0.32.0")
|
||||||
|
|
||||||
|
config = Config(data_dir=tmp_path / "data")
|
||||||
|
reported = {tool["name"]: tool for tool in diagnostics.tools(config)}
|
||||||
|
|
||||||
|
assert reported["exiftool"]["pinned"] == "13.25"
|
||||||
|
assert reported["immich-go"]["pinned"] == "0.32.0"
|
||||||
|
assert "0.32.0" in reported["immich-go"]["version"]
|
||||||
|
assert diagnostics.report(config)["tools"] == list(reported.values())
|
||||||
|
assert "tool_version_drift" not in {w["code"] for w in diagnostics.report(config)["warnings"]}
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_replaced_tool_is_reported_as_drift(tmp_path, monkeypatch):
|
||||||
|
recorded = tmp_path / "versions.json"
|
||||||
|
recorded.write_text(json.dumps({"exiftool": "13.25"}))
|
||||||
|
monkeypatch.setattr(diagnostics, "IMAGE_VERSIONS_FILE", recorded)
|
||||||
|
monkeypatch.setattr(diagnostics.exiftool, "version", lambda: "12.57")
|
||||||
|
|
||||||
|
report = diagnostics.report(Config(data_dir=tmp_path / "data"))
|
||||||
|
|
||||||
|
drift = [w for w in report["warnings"] if w["code"] == "tool_version_drift"]
|
||||||
|
assert drift and "13.25" in drift[0]["message"] and "12.57" in drift[0]["message"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_versions_are_absent_rather_than_invented_outside_a_container(tmp_path, monkeypatch):
|
||||||
|
monkeypatch.setattr(diagnostics, "IMAGE_VERSIONS_FILE", tmp_path / "nothing.json")
|
||||||
|
for tool in diagnostics.tools(Config(data_dir=tmp_path / "data")):
|
||||||
|
assert tool["pinned"] is None
|
||||||
|
|
||||||
|
|
||||||
|
# ── the final layer ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_test_or_build_tooling_is_installed_in_the_image():
|
||||||
|
runtime = "\n".join(instructions(DOCKERFILE.split("AS runtime", 1)[1]))
|
||||||
|
for unwanted in ("[test]", "pytest", "playwright", "build-essential", "gcc"):
|
||||||
|
assert unwanted not in runtime, unwanted
|
||||||
|
assert "pip install --no-cache-dir -e ." in runtime
|
||||||
|
|
||||||
|
|
||||||
|
def test_neither_secrets_nor_library_data_can_enter_the_build_context():
|
||||||
|
lines = instructions(DOCKERIGNORE)
|
||||||
|
assert lines[0] == "*", "the context is deny-by-default"
|
||||||
|
allowed = {line[1:] for line in lines if line.startswith("!")}
|
||||||
|
# Everything the Dockerfile copies has to be allowed, and nothing else is.
|
||||||
|
copied = {
|
||||||
|
source
|
||||||
|
for match in re.finditer(r"^COPY (?!--from)(.+)$", DOCKERFILE, re.MULTILINE)
|
||||||
|
for source in match.group(1).split()[:-1]
|
||||||
|
}
|
||||||
|
assert {Path(source).parts[0] for source in copied} <= allowed
|
||||||
|
assert not {"data", ".git", ".env", "tests", ".venv"} & allowed
|
||||||
|
for generated in ("**/*.env", "**/*.db", "**/*.log", "**/__pycache__"):
|
||||||
|
assert generated in lines, generated
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_image_runs_as_a_non_root_user_whose_ids_are_build_arguments():
|
||||||
|
args = build_args()
|
||||||
|
assert args["UID"] == "1000" and args["GID"] == "1000"
|
||||||
|
assert "USER ${UID}:${GID}" in DOCKERFILE
|
||||||
|
assert re.search(r"^USER (root|0)", DOCKERFILE, re.MULTILINE) is None
|
||||||
|
assert 'useradd --uid "${UID}" --gid "${GID}"' in DOCKERFILE
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_health_check_is_readiness_and_the_default_role_is_serve():
|
||||||
|
assert "HEALTHCHECK" in DOCKERFILE
|
||||||
|
assert "/usr/local/bin/healthcheck.sh" in DOCKERFILE
|
||||||
|
assert 'CMD ["serve"]' in DOCKERFILE
|
||||||
|
assert 'ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]' in DOCKERFILE
|
||||||
|
assert "/api/v1/health/ready" in HEALTHCHECK.read_text()
|
||||||
|
assert "/api/v1/health/live" not in HEALTHCHECK.read_text()
|
||||||
|
# No supervisor: one role per container (acceptance criterion 4).
|
||||||
|
for supervisor in ("supervisord", "s6-overlay", "runit"):
|
||||||
|
assert supervisor not in DOCKERFILE
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("script", [ENTRYPOINT, HEALTHCHECK])
|
||||||
|
def test_the_scripts_are_executable(script):
|
||||||
|
assert script.stat().st_mode & stat.S_IXUSR, f"{script.name} must be executable in git"
|
||||||
|
|
||||||
|
|
||||||
|
# ── the entrypoint, run as shell ─────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def stubs(tmp_path):
|
||||||
|
"""A PATH where ``python`` records its arguments and ``id`` can be told a UID."""
|
||||||
|
bin_dir = tmp_path / "bin"
|
||||||
|
bin_dir.mkdir()
|
||||||
|
recorded = tmp_path / "argv"
|
||||||
|
python = bin_dir / "python"
|
||||||
|
python.write_text(f'#!/bin/sh\nprintf "%s\\n" "$@" > {recorded}\nexit 0\n')
|
||||||
|
python.chmod(0o755)
|
||||||
|
(bin_dir / "id").write_text('#!/bin/sh\nprintf "%s" "${STUB_UID:-1000}"\n')
|
||||||
|
(bin_dir / "id").chmod(0o755)
|
||||||
|
return bin_dir, recorded, tmp_path / "role"
|
||||||
|
|
||||||
|
|
||||||
|
def run_script(script: Path, *args, stubs, env=None):
|
||||||
|
bin_dir, recorded, role_file = stubs
|
||||||
|
result = subprocess.run(
|
||||||
|
["/bin/sh", str(script), *args],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
env={
|
||||||
|
"PATH": f"{bin_dir}:{os.environ['PATH']}",
|
||||||
|
"PHOTO_PIPELINE_ROLE_FILE": str(role_file),
|
||||||
|
**(env or {}),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
argv = recorded.read_text().splitlines() if recorded.exists() else []
|
||||||
|
return result, argv
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_container_refuses_to_run_as_root(stubs):
|
||||||
|
result, argv = run_script(ENTRYPOINT, "serve", stubs=stubs, env={"STUB_UID": "0"})
|
||||||
|
assert result.returncode == 1
|
||||||
|
assert "refusing to run as root" in result.stderr
|
||||||
|
assert argv == [], "the application is never started as root"
|
||||||
|
assert not stubs[2].exists(), "not even the role marker is written"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"given,expected",
|
||||||
|
[
|
||||||
|
(["serve"], ["-m", "photo_pipeline", "serve"]),
|
||||||
|
(["worker", "--id", "worker-2"], ["-m", "photo_pipeline", "worker", "--id", "worker-2"]),
|
||||||
|
# Every other management command stays reachable: operating the container is
|
||||||
|
# operating the same CLI.
|
||||||
|
(["diagnostics"], ["-m", "photo_pipeline", "diagnostics"]),
|
||||||
|
([], ["-m", "photo_pipeline"]),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_the_role_selects_the_command_and_arguments_pass_through(given, expected, stubs):
|
||||||
|
result, argv = run_script(ENTRYPOINT, *given, stubs=stubs)
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
assert argv == expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_role_is_recorded_for_the_health_check(stubs):
|
||||||
|
run_script(ENTRYPOINT, "worker", stubs=stubs)
|
||||||
|
assert stubs[2].read_text() == "worker"
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_health_check_only_probes_the_serving_role(stubs):
|
||||||
|
stubs[2].write_text("worker")
|
||||||
|
result, argv = run_script(HEALTHCHECK, stubs=stubs)
|
||||||
|
assert result.returncode == 0 and argv == [], "a worker has no endpoint to probe"
|
||||||
|
|
||||||
|
stubs[2].write_text("serve")
|
||||||
|
result, argv = run_script(HEALTHCHECK, stubs=stubs, env={"PHOTO_PIPELINE_PORT": "9123"})
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
assert argv == ["-", "9123"], "the configured port is the one probed"
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_health_check_fails_while_the_api_is_not_ready(stubs):
|
||||||
|
"""No python stub: the real interpreter probes a port nothing is listening on."""
|
||||||
|
stubs[2].write_text("serve")
|
||||||
|
result = subprocess.run(
|
||||||
|
["/bin/sh", str(HEALTHCHECK)],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
env={
|
||||||
|
"PATH": os.path.dirname(sys.executable) + os.pathsep + os.environ["PATH"],
|
||||||
|
"PHOTO_PIPELINE_ROLE_FILE": str(stubs[2]),
|
||||||
|
"PHOTO_PIPELINE_PORT": "1",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert result.returncode == 1
|
||||||
|
assert "not ready" in result.stderr
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_scripts_are_posix_shell():
|
||||||
|
"""They run in the image's /bin/sh, which is dash — not bash."""
|
||||||
|
shells = ["/bin/sh"] + ([dash] if (dash := shutil.which("dash")) else [])
|
||||||
|
for shell in shells:
|
||||||
|
for script in (ENTRYPOINT, HEALTHCHECK):
|
||||||
|
checked = subprocess.run([shell, "-n", str(script)], capture_output=True, text=True)
|
||||||
|
assert checked.returncode == 0, f"{shell} {script.name}: {checked.stderr}"
|
||||||
@@ -178,10 +178,13 @@
|
|||||||
"US08-01": [
|
"US08-01": [
|
||||||
"tests/unit/test_security_policy.py",
|
"tests/unit/test_security_policy.py",
|
||||||
"tests/integration/test_trusted_hosts.py"
|
"tests/integration/test_trusted_hosts.py"
|
||||||
|
],
|
||||||
|
"US08-02": [
|
||||||
|
"tests/integration/test_container_image.py",
|
||||||
|
"tests/e2e/test_container_runtime.py"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"planned": [
|
"planned": [
|
||||||
"US08-02",
|
|
||||||
"US08-03",
|
"US08-03",
|
||||||
"US08-04",
|
"US08-04",
|
||||||
"US08-05"
|
"US08-05"
|
||||||
|
|||||||
Reference in New Issue
Block a user