Compare commits

..

4 Commits

27 changed files with 2637 additions and 42 deletions

22
.dockerignore Normal file
View File

@@ -0,0 +1,22 @@
# Deny-by-default build context (US08-02): the image must contain no secrets, no
# photos, no database, no logs, and no .git. An allow list is the only version of this
# rule that stays true when a new file appears in the working copy.
*
!pyproject.toml
!alembic.ini
!README.md
!photo_pipeline
!migrations
!frontend
!docker
# Nothing generated, even under an allowed directory.
**/__pycache__
**/*.py[cod]
**/.DS_Store
**/*.env
**/*.log
**/*.db
**/*.db-*
**/*.sqlite*

85
.env.example Normal file
View File

@@ -0,0 +1,85 @@
# Every setting the application and its composition read, with no values in it.
# Copy to `.env`, fill in what you need, and keep that copy out of git (it is
# gitignored, and the work-item safety checks refuse to stage it).
#
# cp .env.example .env
#
# Empty means "use the default noted beside it". Anything already exported in the
# shell wins over this file, both for the app and for `docker compose`.
# ── the composition (host side; read by docker-compose.yml only) ─────────────
# The photo library on this host. Bind-mounted at PHOTO_PIPELINE_LIBRARY_ROOTS.
PHOTO_PIPELINE_LIBRARY_HOST_PATH=
# Which image to run. Default: photo-pipeline:dev (what `up --build` builds).
PHOTO_PIPELINE_IMAGE=
# Must be the owner of the library above: what the containers rename and rewrite
# keeps this ownership. Default: 1000 / 1000.
PHOTO_PIPELINE_UID=
PHOTO_PIPELINE_GID=
# Host address the API port is published on. Default: 127.0.0.1. Anything else
# exposes the app beyond this machine — then ALLOWED_HOSTS and ACCESS_SECRET below
# are what stand in for the loopback boundary (US08-01).
PHOTO_PIPELINE_PUBLISH_ADDRESS=
# This file's own path, if it is not ./.env. Default: .env.
PHOTO_PIPELINE_ENV_FILE=
# ── library and data ────────────────────────────────────────────────────────
# os.pathsep-separated. In a container these are the *container-side* mount paths,
# and `serve`/`worker` refuse to start when they are not mounted.
PHOTO_PIPELINE_LIBRARY_ROOTS=
# Database, WAL, thumbnail cache, journals, backups. Default: data (the container
# sets /data, which is the persistent volume; a network mount is unsupported).
PHOTO_PIPELINE_DATA_DIR=
# Database file, if it should not live in the data directory. Default:
# <data dir>/photo_pipeline.db.
PHOTO_PIPELINE_DB_PATH=
# ── serving ─────────────────────────────────────────────────────────────────
# Bind address. Default: 127.0.0.1. The composition sets 0.0.0.0 inside the
# container and publishes to loopback on the host instead.
PHOTO_PIPELINE_HOST=
# Under the composition this is the *published* host port; the container serves
# 8000. Default: 8000.
PHOTO_PIPELINE_PORT=
# Comma-separated hostnames the app answers to besides loopback. Empty means
# loopback only. Naming one makes the access secret mandatory.
PHOTO_PIPELINE_ALLOWED_HOSTS=
# Traded for the session cookie at GET /api/v1/session via X-Access-Secret.
# Required as soon as the app is reachable from anywhere but loopback. Generate
# one with: python -c 'import secrets; print(secrets.token_urlsafe(32))'
PHOTO_PIPELINE_ACCESS_SECRET=
# Comma-separated peer addresses whose X-Forwarded-Proto/-Host may be believed.
# Only the reverse proxy's address belongs here. Default: none.
PHOTO_PIPELINE_TRUSTED_PROXIES=
# Largest request body accepted, in bytes. Default: 1048576.
PHOTO_PIPELINE_MAX_REQUEST_BYTES=
# ── logging ─────────────────────────────────────────────────────────────────
# Default: INFO.
PHOTO_PIPELINE_LOG_LEVEL=
# json or text. Default: json.
PHOTO_PIPELINE_LOG_FORMAT=
# ── limits ──────────────────────────────────────────────────────────────────
# Thumbnail cache quota in bytes. Default: 500000000.
PHOTO_PIPELINE_THUMBNAIL_CACHE_QUOTA_BYTES=
# Refuse to decode images larger than this many pixels. Default: 100000000.
PHOTO_PIPELINE_THUMBNAIL_MAX_PIXELS=
# Free space an archive destination must keep beyond the transfer. Default:
# 1000000000.
PHOTO_PIPELINE_ARCHIVE_FREE_SPACE_RESERVE_BYTES=
# ── safety gate ─────────────────────────────────────────────────────────────
# true refuses every mutating request until a read-only dry run of this library has
# been produced and approved (US07-07). Default: false. Turn it on before pointing
# the app at photos that cannot be replaced.
PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL=
# ── external services ───────────────────────────────────────────────────────
# Vision provider key for the analysis stage. Without it, analysis cannot run.
PHOTO_PIPELINE_VISION_API_KEY=
# Immich server and its API key, for the upload stage.
PHOTO_PIPELINE_IMMICH_SERVER_URL=
PHOTO_PIPELINE_IMMICH_API_KEY=
# Uploader binary. Default: immich-go (on PATH; pinned inside the image).
PHOTO_PIPELINE_IMMICH_GO_BINARY=

115
Dockerfile Normal file
View File

@@ -0,0 +1,115 @@
# One image, two roles (US08-02).
#
# The application is not self-contained Python: it shells out to `exiftool` for every
# EXIF checkpoint and to `immich-go` for every upload, and it serves the static
# frontend from `frontend/`. All three are installed here at pinned versions, because
# an image whose external tools drift is an image whose metadata checkpoints and
# upload reports drift with them (concept §15, "External integration risks").
#
# Everything is pinned:
# * the base image by tag *and* digest, so a moved tag cannot change the runtime;
# * exiftool by its Debian package version, verified against `exiftool -ver`;
# * immich-go by release version and per-architecture SHA-256 of the release asset.
# The verified versions become image labels and /etc/photo-pipeline/versions.json,
# which `python -m photo_pipeline diagnostics` reports — so a running container can
# prove what it contains instead of being trusted about it.
#
# The project is installed editable on purpose: `photo_pipeline.db` resolves
# `alembic.ini` and `migrations/`, and the API resolves `frontend/`, relative to the
# repository root. An editable install keeps that one layout instead of scattering the
# same files across site-packages and a source tree.
ARG PYTHON_IMAGE=python:3.12.14-slim-trixie@sha256:2c941e860699f878900b0edc2403613c234d4b32eda3cc9fa7036991a2a63c4a
# ── the uploader, fetched and verified outside the final layer ────────────────
FROM ${PYTHON_IMAGE} AS uploader
ARG IMMICH_GO_VERSION=0.32.0
ARG IMMICH_GO_SHA256_AMD64=6e2ad86bafdadb9466d6515de7cb882726c0aea1a21d51164dff361d7d480a97
ARG IMMICH_GO_SHA256_ARM64=2c35d9284baae407ef9540bdac5f488971b0bdc7be758a4d7c05ab270af09fdb
COPY docker/fetch-immich-go.py /tmp/fetch-immich-go.py
RUN python /tmp/fetch-immich-go.py \
--version "${IMMICH_GO_VERSION}" \
--sha256-amd64 "${IMMICH_GO_SHA256_AMD64}" \
--sha256-arm64 "${IMMICH_GO_SHA256_ARM64}" \
--into /usr/local/bin \
&& /usr/local/bin/immich-go version
# ── the application ──────────────────────────────────────────────────────────
FROM ${PYTHON_IMAGE} AS runtime
ARG EXIFTOOL_VERSION=13.25+dfsg-1
ARG IMMICH_GO_VERSION=0.32.0
# The library is mounted from the host, so the container's identity must match the
# ownership that library already has: everything this application renames, writes
# EXIF into, or archives has to stay owned by the host user afterwards.
ARG UID=1000
ARG GID=1000
LABEL org.opencontainers.image.title="photo_pipeline" \
org.opencontainers.image.source="https://github.com/domverse/photoanalyzer" \
io.photoanalyzer.exiftool.version="${EXIFTOOL_VERSION}" \
io.photoanalyzer.immich-go.version="${IMMICH_GO_VERSION}"
ENV PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
PATH=/opt/venv/bin:$PATH \
PHOTO_PIPELINE_DATA_DIR=/data
RUN set -eu; \
apt-get update; \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
"libimage-exiftool-perl=${EXIFTOOL_VERSION}"; \
rm -rf /var/lib/apt/lists/*
COPY --from=uploader /usr/local/bin/immich-go /usr/local/bin/immich-go
WORKDIR /app
COPY pyproject.toml alembic.ini README.md ./
COPY photo_pipeline ./photo_pipeline
COPY migrations ./migrations
COPY frontend ./frontend
COPY docker/entrypoint.sh docker/healthcheck.sh /usr/local/bin/
# Runtime dependencies only: the `test` extra (pytest, playwright) and the `vision`
# extra stay out, and pip's build isolation leaves no build tooling behind.
RUN set -eu; \
python -m venv /opt/venv; \
/opt/venv/bin/pip install --no-cache-dir -e .
# What is installed must be what was pinned, or the labels and the version record
# would be a claim rather than a fact.
RUN set -eu; \
mkdir -p /etc/photo-pipeline; \
exiftool_version="$(exiftool -ver)"; \
immich_go_version="$(immich-go version | head -n 1 | tr -d '\r')"; \
expected_exiftool="$(printf '%s' "${EXIFTOOL_VERSION}" | cut -d+ -f1 | cut -d- -f1)"; \
[ "${exiftool_version}" = "${expected_exiftool}" ] \
|| { echo "exiftool ${exiftool_version} is not the pinned ${expected_exiftool}" >&2; exit 1; }; \
case "${immich_go_version}" in \
*"${IMMICH_GO_VERSION}"*) ;; \
*) echo "immich-go '${immich_go_version}' is not pinned ${IMMICH_GO_VERSION}" >&2; exit 1 ;; \
esac; \
printf '{\n "exiftool": "%s",\n "immich-go": "%s"\n}\n' \
"${exiftool_version}" "${IMMICH_GO_VERSION}" > /etc/photo-pipeline/versions.json
# Non-root, with the host library's ownership. /data is the persistent volume; the
# photo library itself is mounted by the deployment (US08-03), never baked in.
RUN set -eu; \
groupadd --gid "${GID}" --non-unique app; \
useradd --uid "${UID}" --gid "${GID}" --non-unique --no-create-home --home-dir /app app; \
mkdir -p /data; \
chown "${UID}:${GID}" /data
USER ${UID}:${GID}
EXPOSE 8000
# Readiness, not liveness: an unmigrated or misconfigured database answers
# /api/v1/health/ready with 503, and a container that cannot serve must not be
# reported healthy. The worker role has no endpoint, so its check is a no-op here.
HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
CMD ["/usr/local/bin/healthcheck.sh"]
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
CMD ["serve"]

125
README.md
View File

@@ -69,6 +69,131 @@ not a loopback name (DNS rebinding), when `Origin` is any other origin, when
thumbnail), or when the body exceeds `PHOTO_PIPELINE_MAX_REQUEST_BYTES`. There is no
CORS middleware at all, so no other origin can read a response.
### Reaching it through a hostname or proxy (US08-01)
| variable | meaning |
|---|---|
| `PHOTO_PIPELINE_ALLOWED_HOSTS` | comma-separated extra names the app answers to; empty means loopback only |
| `PHOTO_PIPELINE_ACCESS_SECRET` | traded for the session cookie at `GET /api/v1/session` via `X-Access-Secret` |
| `PHOTO_PIPELINE_TRUSTED_PROXIES` | comma-separated peer addresses whose `X-Forwarded-Proto`/`X-Forwarded-Host` are believed |
Being reachable *was* the authentication: whoever could open `127.0.0.1:8000` owned
the library. So naming any non-loopback host — or binding to one, `0.0.0.0` included
— makes the access secret mandatory, and `serve` refuses to start without it rather
than publishing the library. Loopback-only deployments need no secret and behave
exactly as before.
```bash
curl -sc /tmp/pp.jar -H "X-Access-Secret: $PHOTO_PIPELINE_ACCESS_SECRET" \
https://photos.example.com/api/v1/session
```
The browser asks for the secret once per tab and keeps it in `sessionStorage`.
Wrong secrets are rate-limited (5 per minute) and logged with the caller's address
only. `Host` and `Origin` are judged against the configured names; the *external*
scheme and host come from the forwarded headers only when the request arrived from a
`PHOTO_PIPELINE_TRUSTED_PROXIES` address, so a client cannot declare its own origin,
and the session cookie is marked `Secure` when that external scheme is HTTPS. Health
endpoints stay reachable without the secret so an orchestrator can restart the
container; nothing else does.
## Container image (US08-02)
One image runs either role. It is built from a clean checkout with no arguments:
```bash
docker build -t photo-pipeline:dev .
```
Everything external is pinned, and the build fails rather than drifting: the Python
base image by tag *and* digest, `exiftool` by its Debian package version (verified
against `exiftool -ver`), and `immich-go` by release version and per-architecture
SHA-256 of the release asset. The verified versions become image labels and
`/etc/photo-pipeline/versions.json`, which `diagnostics` reports as `tools[].pinned`
beside the version actually installed — so a replaced binary shows up as a
`tool_version_drift` warning instead of as a misparsed upload report.
| build argument | default | why change it |
|---|---|---|
| `UID` / `GID` | `1000` | must match the owner of the mounted photo library |
| `PYTHON_IMAGE` | pinned digest | upgrading the base image |
| `EXIFTOOL_VERSION` | Debian package version | upgrading exiftool |
| `IMMICH_GO_VERSION` + `IMMICH_GO_SHA256_AMD64`/`_ARM64` | pinned release | upgrading the uploader (take the digests from that release's `checksums.txt`) |
The first argument is the role, and every other management command still works:
```bash
docker run --rm -v /srv/photos:/srv/photos -v pp-data:/data \
-e PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos photo-pipeline:dev migrate
docker run -d -p 127.0.0.1:8000:8000 -v /srv/photos:/srv/photos -v pp-data:/data \
-e PHOTO_PIPELINE_HOST=0.0.0.0 -e PHOTO_PIPELINE_ACCESS_SECRET=... \
-e PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos photo-pipeline:dev serve
docker run -d -v /srv/photos:/srv/photos -v pp-data:/data \
-e PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos photo-pipeline:dev worker
```
One role per container: `serve` and `worker` each take the library process lock for
their role (US07-05), so no supervisor starts both. The container refuses to run as
UID 0 — files it renames or writes must keep the ownership the host library expects —
and `/data` is the persistent volume holding the database, journals, backups, and
thumbnail cache. Binding to `0.0.0.0` makes the access secret mandatory
([above](#reaching-it-through-a-hostname-or-proxy-us08-01)); `serve` refuses to start
without it. The declared `HEALTHCHECK` polls `/api/v1/health/ready`, so a container
whose database is unmigrated or misconfigured is never reported healthy.
## Composed runtime (US08-03)
`docker-compose.yml` is the deployment: one `serve` container, one `worker`
container, one bind-mounted library, one data volume, and a one-shot `migrate` that
both roles wait for.
```bash
cp .env.example .env && $EDITOR .env # nothing has a value in it; fill in yours
docker compose up -d --build
```
`.env.example` lists every `PHOTO_PIPELINE_*` variable with its default in a comment
and no values at all. The three the composition cannot start without are
`PHOTO_PIPELINE_LIBRARY_HOST_PATH` (the library on this host),
`PHOTO_PIPELINE_LIBRARY_ROOTS` (where it is mounted *inside* the container), and
`PHOTO_PIPELINE_ACCESS_SECRET` — publishing the port means the app is reachable from
outside the container, so US08-01 makes the secret mandatory. Set
`PHOTO_PIPELINE_UID`/`GID` to the owner of the library: what the containers rename
and rewrite keeps that ownership.
| invariant | how the composition keeps it |
|---|---|
| one writer | `serve` and `worker` take their role's library lock (US07-05) in the shared `/data` volume, so `--scale worker=2` is refused by the lock, not by convention |
| migrations first | `migrate` runs the backup-then-migrate path and must exit 0 before `api` and `worker` start; a failed upgrade leaves the previous database and its pre-migration backup intact |
| container paths | `PHOTO_PIPELINE_LIBRARY_ROOTS` is both the mount target and the configured root; a root that is not mounted makes `serve`/`worker` exit 5 at startup instead of writing into the container's throwaway layer |
| loopback by default | the port is published to `127.0.0.1` unless `PHOTO_PIPELINE_PUBLISH_ADDRESS` says otherwise, and exposing it needs the hostname in `PHOTO_PIPELINE_ALLOWED_HOSTS` plus the access secret |
| restart safety | both roles are `restart: unless-stopped` with a 30 s stop grace period, and a job interrupted by a restart resumes exactly as it does on a host restart |
The `data` volume holds the database, its write-ahead log, the thumbnail cache,
journals, and backups. **It must stay on a local filesystem** — SQLite in WAL mode
needs real local locking, so NFS, SMB, and network volume drivers are unsupported
there and corrupt the database rather than slow it down. The library bind mount has
no such restriction.
Operating the deployment is operating the same CLI:
```bash
docker compose run --rm --no-deps api diagnostics
docker compose run --rm --no-deps api backup --reason pre-upgrade
docker compose run --rm --no-deps api verify-backup /data/backups/<name>
docker compose run --rm --no-deps api restore /data/backups/<name> --into /data/restored
docker compose logs -f worker
```
`--no-deps` keeps a one-off command from starting a second stack; `api` is only the
service the command borrows the image and mounts from. `restore` is deliberately not
an API call: it replaces the state of an installation and belongs to a stopped one,
so stop `api` and `worker` first and restart them against the restored directory.
Publishing and deploying the image is US08-04.
## Testing
One offline command runs the whole suite (unit, integration, and browser

105
docker-compose.yml Normal file
View File

@@ -0,0 +1,105 @@
# The deployed runtime (US08-03): one API, one worker, one library, one volume.
#
# The same image (US08-02) runs both roles, so what is composed here is process
# topology, not a second application. Three invariants shape it:
#
# * one writer — `serve` and `worker` each take the library process lock for their
# role (US07-05), and both containers mount the *same* data volume, which is what
# makes the lock file visible to both. Scaling `worker` past 1 is refused by that
# lock rather than by anyone remembering not to;
# * one local filesystem — the database, its write-ahead log, the thumbnail cache,
# and the backups live in the `data` volume, and SQLite in WAL mode requires real
# local-filesystem locking. A network mount (NFS, SMB, or a cloud volume driver)
# is unsupported for it; that is a corrupted database, not a slow one;
# * one library path vocabulary — `PHOTO_PIPELINE_LIBRARY_ROOTS` names the path
# *inside* the container, which is also the bind mount's target below. A root
# that is not mounted there makes `serve` and `worker` refuse at startup instead
# of writing into the container's throwaway layer.
#
# Configuration and secrets come from the environment only: copy `.env.example` to
# `.env` and fill it in. Nothing is baked into the image and nothing with a value in
# it is committed.
#
# cp .env.example .env && $EDITOR .env
# docker compose up -d --build
#
# Operating it is operating the same CLI — `docker compose run --rm --no-deps api
# <command>` — see README, "Composed runtime".
name: photo-pipeline
x-runtime: &runtime
image: ${PHOTO_PIPELINE_IMAGE:-photo-pipeline:dev}
build:
context: .
args:
# Everything the app renames or rewrites has to stay owned by the host user
# the library already belongs to.
UID: ${PHOTO_PIPELINE_UID:-1000}
GID: ${PHOTO_PIPELINE_GID:-1000}
user: "${PHOTO_PIPELINE_UID:-1000}:${PHOTO_PIPELINE_GID:-1000}"
env_file:
- ${PHOTO_PIPELINE_ENV_FILE:-.env}
volumes:
- data:/data
- "${PHOTO_PIPELINE_LIBRARY_HOST_PATH:?set PHOTO_PIPELINE_LIBRARY_HOST_PATH to the photo library on this host}:${PHOTO_PIPELINE_LIBRARY_ROOTS:?set PHOTO_PIPELINE_LIBRARY_ROOTS to the container-side library path}"
# Jobs check for cancellation between items and leave a resumable record; a
# too-short grace period turns an orderly stop into a recovery on next start.
stop_grace_period: 30s
x-environment: &environment
# Set here rather than left to the file: these two are what the composition itself
# promises, and an `.env` that disagreed would move the database off the volume or
# the library off its mount.
PHOTO_PIPELINE_DATA_DIR: /data
PHOTO_PIPELINE_LIBRARY_ROOTS: ${PHOTO_PIPELINE_LIBRARY_ROOTS}
services:
# Migrations run to completion before either role accepts work, through the same
# backup-then-migrate path the roles use (US07-05): a pending upgrade is snapshotted
# first, and a failed one exits non-zero with the backup named — so `api` and
# `worker` never start, and the previous database is left intact and restorable.
migrate:
<<: *runtime
command: ["migrate"]
environment: *environment
restart: "no"
api:
<<: *runtime
command: ["serve"]
environment:
<<: *environment
# Published to host loopback below. Inside the container the server must bind
# the container's own interface for that publish to reach it, which is exactly
# what makes the access secret mandatory (US08-01) — `serve` refuses to start
# without one. Exposing the port beyond loopback additionally needs
# PHOTO_PIPELINE_ALLOWED_HOSTS to name the hostname it is reached under.
PHOTO_PIPELINE_HOST: 0.0.0.0
PHOTO_PIPELINE_PORT: 8000
ports:
# Host side only: PHOTO_PIPELINE_PORT in `.env` moves the *published* port, and
# the container always serves 8000, which is what the image's health check probes.
- "${PHOTO_PIPELINE_PUBLISH_ADDRESS:-127.0.0.1}:${PHOTO_PIPELINE_PORT:-8000}:8000"
depends_on:
migrate:
condition: service_completed_successfully
restart: unless-stopped
# One worker. A second one is refused by the library lock in the shared data
# volume, which is the point: `docker compose up --scale worker=2` fails loudly
# instead of running two writers against one library.
worker:
<<: *runtime
command: ["worker", "--id", "worker-1"]
environment: *environment
depends_on:
migrate:
condition: service_completed_successfully
restart: unless-stopped
volumes:
# Local driver on purpose: the database, WAL, thumbnail cache, and backups need a
# real local filesystem. Do not point this at NFS, SMB, or a network volume driver.
data:
driver: local

22
docker/entrypoint.sh Executable file
View File

@@ -0,0 +1,22 @@
#!/bin/sh
# One entrypoint, one role per container (US08-02).
#
# The first argument is the management command the image runs — `serve` and `worker`
# are the two roles, and every other `python -m photo_pipeline` command (migrate,
# diagnostics, backup, restore, dry-run) is passed through unchanged so operating the
# container is operating the same CLI. No supervisor: two roles in one container would
# share a process lock they are each meant to hold alone (US07-05).
set -eu
if [ "$(id -u)" = "0" ]; then
echo "refusing to run as root: start this image with a non-root UID/GID so files" \
"it renames or writes keep the ownership the mounted library expects" >&2
exit 1
fi
role="${1:-serve}"
# The health check has to know which role it is checking, and only the API has an
# endpoint to check. /tmp is writable for the unprivileged user; /run may not be.
printf '%s' "${role}" > "${PHOTO_PIPELINE_ROLE_FILE:-/tmp/photo-pipeline-role}" 2>/dev/null || true
exec python -m photo_pipeline "$@"

72
docker/fetch-immich-go.py Normal file
View File

@@ -0,0 +1,72 @@
"""Download one pinned immich-go release and verify it before unpacking (US08-02).
Run at image build time by the `uploader` stage, with the interpreter that is already
in the base image: no curl, no wget, and no download tooling in the layer that ships.
The checksum is not advisory — a release asset that does not match the pinned digest
is a failed build, not a warning, because the uploader's flags and report format are
what the upload parser is written against (concept §15).
"""
from __future__ import annotations
import argparse
import hashlib
import platform
import tarfile
import tempfile
import urllib.request
from pathlib import Path
RELEASE_URL = "https://github.com/simulot/immich-go/releases/download/v{version}/{asset}"
# Debian/BuildKit architecture as the interpreter sees it → release asset name.
ASSETS = {
"x86_64": ("immich-go_Linux_x86_64.tar.gz", "amd64"),
"amd64": ("immich-go_Linux_x86_64.tar.gz", "amd64"),
"aarch64": ("immich-go_Linux_arm64.tar.gz", "arm64"),
"arm64": ("immich-go_Linux_arm64.tar.gz", "arm64"),
}
TIMEOUT_SECONDS = 300
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--version", required=True, help="immich-go release, without the v")
parser.add_argument("--sha256-amd64", required=True)
parser.add_argument("--sha256-arm64", required=True)
parser.add_argument("--into", default="/usr/local/bin")
args = parser.parse_args()
machine = platform.machine().lower()
if machine not in ASSETS:
raise SystemExit(f"unsupported architecture: {machine}")
asset, arch = ASSETS[machine]
expected = {"amd64": args.sha256_amd64, "arm64": args.sha256_arm64}[arch]
url = RELEASE_URL.format(version=args.version, asset=asset)
with urllib.request.urlopen(url, timeout=TIMEOUT_SECONDS) as response: # noqa: S310
payload = response.read()
digest = hashlib.sha256(payload).hexdigest()
if digest != expected:
raise SystemExit(f"checksum mismatch for {url}: {digest} != {expected}")
target = Path(args.into)
target.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory() as work:
archive = Path(work) / asset
archive.write_bytes(payload)
with tarfile.open(archive) as tar:
member = tar.getmember("immich-go")
# Extract exactly the one file this pin is about, by name, so nothing
# else in the archive can decide where it lands.
extracted = tar.extractfile(member)
if extracted is None:
raise SystemExit("release archive contains no immich-go binary")
binary = target / "immich-go"
binary.write_bytes(extracted.read())
binary.chmod(0o755)
print(f"immich-go {args.version} ({arch}) verified {digest}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

34
docker/healthcheck.sh Executable file
View File

@@ -0,0 +1,34 @@
#!/bin/sh
# Container health for the `serve` role: readiness, not liveness (US08-02).
#
# /api/v1/health/ready is 503 until the database is reachable, migrated, and in WAL
# mode with foreign keys on, so an unmigrated or misconfigured container never reports
# healthy. Health endpoints need no session and no access secret, which is what lets an
# orchestrator restart a container it holds no credentials for (US08-01).
set -eu
role="$(cat "${PHOTO_PIPELINE_ROLE_FILE:-/tmp/photo-pipeline-role}" 2>/dev/null || echo unknown)"
if [ "${role}" != "serve" ]; then
# ponytail: the worker has no endpoint to probe; its liveness is its lease and job
# heartbeat in the database. Add a `worker --health` command if a restart policy
# ever needs to act on it.
exit 0
fi
port="${PHOTO_PIPELINE_PORT:-8000}"
exec python - "${port}" <<'PY'
import sys
import urllib.error
import urllib.request
url = f"http://127.0.0.1:{sys.argv[1]}/api/v1/health/ready"
try:
with urllib.request.urlopen(url, timeout=5) as response: # noqa: S310 — loopback
sys.exit(0 if response.status == 200 else 1)
except urllib.error.HTTPError as error:
print(f"not ready: HTTP {error.code}", file=sys.stderr)
sys.exit(1)
except OSError as error:
print(f"not ready: {error}", file=sys.stderr)
sys.exit(1)
PY

View File

@@ -7,9 +7,28 @@ export const BASE = "/api/v1";
// what makes it proof that the caller is this app and not another page.
let csrfToken = null;
// A deployment reachable through a proxy trades an operator secret for that cookie.
// Kept per tab: sessionStorage dies with the tab, and the secret never enters a URL.
const SECRET_KEY = "pp_access_secret";
async function bootstrap(secret) {
return fetch(BASE + "/session", {
credentials: "same-origin",
headers: secret ? { "X-Access-Secret": secret } : {},
});
}
async function session() {
if (csrfToken === null) {
const response = await fetch(BASE + "/session", { credentials: "same-origin" });
let response = await bootstrap(sessionStorage.getItem(SECRET_KEY));
if (response.status === 401) {
sessionStorage.removeItem(SECRET_KEY);
const secret = prompt("Access secret");
if (secret) {
response = await bootstrap(secret);
if (response.ok) sessionStorage.setItem(SECRET_KEY, secret);
}
}
const body = await response.json().catch(() => null);
csrfToken = (body && body.csrf_token) || null;
}

View File

@@ -11,9 +11,11 @@ from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from typing import Sequence
from photo_pipeline import path_policy
from photo_pipeline.config import Config
from photo_pipeline.services.app_lock import LegacyProcessActive, LibraryLock, LockHeld
from photo_pipeline.services.backup import BackupError, BackupService, migrate_with_backup
@@ -83,6 +85,18 @@ def main(argv: Sequence[str] | None = None) -> int:
config = Config.from_env()
config.database_path.parent.mkdir(parents=True, exist_ok=True)
# In a container the two roles that touch the library check their boundary before
# they take a lock or bind a port: the configured roots must name the mount paths,
# and a mismatch is cheaper to refuse than to discover at the first write (US08-03).
# Only in a container — on a host an unmounted root is an ordinary Tuesday (an
# archive medium that is not plugged in), and refusing to serve would take the
# offline half of the library away with it.
if args.command in ("serve", "worker") and path_policy.in_container():
refusal = path_policy.roots_refusal(config.library_roots, require_mount=True)
if refusal is not None:
print(refusal, file=sys.stderr)
return 5
if args.command == "migrate":
manifest = migrate_with_backup(config)
if manifest:
@@ -211,13 +225,21 @@ def main(argv: Sequence[str] | None = None) -> int:
import uvicorn
from photo_pipeline.api.app import create_app
from photo_pipeline.api.app import ConfigurationRefused, create_app
# An exposed deployment without an access secret must not reach the port at all,
# and the operator needs a sentence, not a traceback (US08-01).
try:
app = create_app(config)
except ConfigurationRefused as error:
print(str(error), file=sys.stderr)
return 4
lock = LibraryLock(config, "api")
if (held := _acquire(lock, allow_legacy=args.allow_legacy)) is not None:
return held
try:
uvicorn.run(create_app(config), host=config.host, port=config.port)
uvicorn.run(app, host=config.host, port=config.port)
finally:
lock.release()
return 0
@@ -228,8 +250,6 @@ def _acquire(lock: LibraryLock, *, allow_legacy: bool) -> int | None:
Returns an exit code to return, or ``None`` when the lock was acquired.
"""
import sys
try:
lock.acquire(allow_legacy=allow_legacy)
except LockHeld as error:

View File

@@ -35,7 +35,13 @@ from photo_pipeline.api.routes import (
uploads,
workflow,
)
from photo_pipeline.api.security import DEFAULT_HEADERS, SecurityMiddleware, Session
from photo_pipeline.api.security import (
DEFAULT_HEADERS,
FailureLimiter,
SecurityMiddleware,
Session,
trust_refusal,
)
# Registers the safety_score / analysis job handlers on import.
import photo_pipeline.jobs.domain_handlers # noqa: F401
@@ -84,9 +90,16 @@ def _install_error_handlers(app: FastAPI) -> None:
return _envelope(500, "internal_error", "internal error")
class ConfigurationRefused(RuntimeError):
"""The configuration would serve the library to callers it cannot authenticate."""
def create_app(config: Config | None = None) -> FastAPI:
config = config or Config.from_env()
configure_logging(config.log_level, config.log_format)
# Before anything is built, let alone bound to a port (US08-01).
if (why := trust_refusal(config)) is not None:
raise ConfigurationRefused(why)
@asynccontextmanager
async def lifespan(app: FastAPI):
@@ -114,6 +127,10 @@ def create_app(config: Config | None = None) -> FastAPI:
# One session per process: the browser exchanges it for a cookie + CSRF token,
# and every other origin is refused before a route ever runs (US07-02).
app.state.session = Session.create()
app.state.access_limiter = FailureLimiter()
# Also set in the lifespan, but the bootstrap route reads it, and a caller can
# arrive before anything else has touched app.state.
app.state.config = config
app.add_middleware(SecurityMiddleware, session=app.state.session, config=config)
_install_error_handlers(app)
app.include_router(session_routes.router, prefix="/api/v1")

View File

@@ -3,21 +3,52 @@
It sets the ``HttpOnly``/``SameSite=Strict`` session cookie and returns the CSRF
token in the body. A foreign page can call this — it just cannot read the answer,
because the app sends no CORS headers — and the cookie it received is never attached
to a request that foreign page initiates.
to a request that foreign page initiated.
When an access secret is configured (mandatory as soon as the app is reachable from
another machine, US08-01) this is also the authentication gate: the secret buys the
cookie, and every route behind it keeps asking for exactly the session and CSRF token
it asked for before. Wrong secrets are counted, and a burst of them stops being
answered — otherwise a proxy-exposed deployment could be guessed at indefinitely.
"""
from __future__ import annotations
import logging
import secrets
from fastapi import APIRouter, Request
from fastapi.responses import JSONResponse
from photo_pipeline.api.security import SESSION_COOKIE
from photo_pipeline.api.security import ACCESS_SECRET_HEADER, SESSION_COOKIE
router = APIRouter(tags=["session"])
log = logging.getLogger(__name__)
def _refuse(status: int, code: str, message: str) -> JSONResponse:
return JSONResponse(status_code=status, content={"error": {"code": code, "message": message}})
@router.get("/session")
def start_session(request: Request) -> JSONResponse:
config = request.app.state.config
secret = config.access_secret
if secret is not None:
limiter = request.app.state.access_limiter
if limiter.blocked():
return _refuse(429, "too_many_attempts", "too many failed attempts; retry later")
offered = request.headers.get(ACCESS_SECRET_HEADER, "")
if not secrets.compare_digest(offered, secret.get_secret_value()):
limiter.record_failure()
# The client address is the whole record: the offered secret, the issued
# session, and the request body all stay out of the log.
log.warning(
"access secret rejected", extra={"client": _client(request), "path": "/session"}
)
return _refuse(401, "access_denied", "a valid access secret is required")
session = request.app.state.session
response = JSONResponse({"csrf_token": session.csrf_token})
response.set_cookie(
@@ -25,6 +56,13 @@ def start_session(request: Request) -> JSONResponse:
session.id,
httponly=True,
samesite="strict",
# HTTPS outside means the cookie must never travel over a plain hop, even one
# this process cannot see. Loopback http keeps working unchanged.
secure=request.scope.get("state", {}).get("external_scheme") == "https",
path="/",
)
return response
def _client(request: Request) -> str:
return request.client.host if request.client else "unknown"

View File

@@ -19,6 +19,14 @@ The defenses stack, because each one alone has a hole:
only in the bootstrap response body, which a foreign page cannot read (no CORS) —
so possessing it proves the caller is same-origin.
Behind a reverse proxy (US08-01) the same stack holds with two substitutions: the
allowed host set comes from configuration instead of being the loopback names, and
the host/scheme the policy judges is the *external* one, which is only read from
``X-Forwarded-*`` when the request actually arrived from a configured proxy. The
loopback check was standing in for authentication, so naming a non-loopback host
also makes an access secret mandatory — ``trust_refusal`` refuses to start without
one, and the secret is what the bootstrap endpoint trades for the session cookie.
``evaluate`` is a pure function over the request metadata: the whole policy is one
table that a unit test can enumerate, and the middleware only applies its verdict.
"""
@@ -26,6 +34,7 @@ table that a unit test can enumerate, and the middleware only applies its verdic
from __future__ import annotations
import secrets
import time
from collections.abc import Mapping
from dataclasses import dataclass
from urllib.parse import urlsplit
@@ -35,6 +44,7 @@ from starlette.responses import JSONResponse
SESSION_COOKIE = "pp_session"
CSRF_HEADER = "x-csrf-token"
ACCESS_SECRET_HEADER = "x-access-secret"
API_PREFIX = "/api/v1"
SAFE_METHODS = frozenset({"GET", "HEAD", "OPTIONS"})
# Reachable without a session: liveness/readiness (an orchestrator has no cookie)
@@ -96,6 +106,28 @@ def split_host(value: str) -> tuple[str, str]:
return host, port
def external_view(
*,
client: str | None,
headers: Mapping[str, str],
scheme: str,
trusted_proxies: frozenset[str],
) -> tuple[str, str]:
"""The ``(scheme, host)`` the caller used, as opposed to the one this hop saw.
Forwarded headers are a client-supplied claim. Believing them from anyone lets a
request declare its own origin — and origin is half of this module's evidence —
so they count only when the connection came from a configured proxy.
"""
host = headers.get("host", "")
if client is None or client not in trusted_proxies:
return scheme, host
# A chain appends: the first entry is what the original client asked for.
forwarded_proto = headers.get("x-forwarded-proto", "").split(",")[0].strip().lower()
forwarded_host = headers.get("x-forwarded-host", "").split(",")[0].strip()
return forwarded_proto or scheme, forwarded_host or host
def evaluate(
*,
method: str,
@@ -103,20 +135,26 @@ def evaluate(
headers: Mapping[str, str],
session: Session,
allowed_hosts: frozenset[str] = LOOPBACK_HOSTS,
scheme: str = "http",
max_request_bytes: int,
) -> Refusal | None:
"""Why this request must be refused, or ``None`` when it may proceed."""
"""Why this request must be refused, or ``None`` when it may proceed.
``headers["host"]`` and ``scheme`` are the external ones (see ``external_view``);
the allowed origins are the allowed hosts under that scheme and port, so there is
no second list that can drift away from the first.
"""
host_header = headers.get("host", "")
host, port = split_host(host_header)
if host.lower() not in allowed_hosts:
return Refusal(403, "host_not_allowed", "request host is not a local address")
return Refusal(403, "host_not_allowed", "request host is not an allowed address")
origin = headers.get("origin")
if origin is not None and origin != "":
parts = urlsplit(origin)
origin_host, origin_port = split_host(parts.netloc)
if (
parts.scheme not in ("http", "https")
parts.scheme != scheme
or origin_host.lower() not in allowed_hosts
or origin_port != port
):
@@ -142,6 +180,52 @@ def evaluate(
return None
def exposed_hosts(config) -> list[str]:
"""Configured names by which this application is reachable from another machine."""
names = {str(config.host).lower()}
names.update(split_host(name)[0].lower() for name in config.allowed_hosts)
return sorted(names - LOOPBACK_HOSTS)
def trust_refusal(config) -> str | None:
"""Why this configuration must not serve at all, or ``None``.
Reaching the app used to prove ownership of it. The moment a configuration makes
it reachable from elsewhere that stops being true, so serving without a secret
would publish the library — refuse at startup rather than at the first request,
when the operator is no longer watching (US08-01).
"""
exposed = exposed_hosts(config)
if exposed and config.access_secret is None:
return (
f"refusing to serve: {', '.join(exposed)} is reachable from outside this "
"machine, so PHOTO_PIPELINE_ACCESS_SECRET must be set"
)
return None
class FailureLimiter:
"""Bounded failed access-secret attempts, so the secret cannot be guessed online.
ponytail: one counter for the whole process rather than per client address —
behind a proxy every attempt arrives from the same address anyway. Per-caller
buckets if the app is ever exposed without one.
"""
def __init__(self, limit: int = 5, window: float = 60.0) -> None:
self.limit = limit
self.window = window
self._failures: list[float] = []
def blocked(self) -> bool:
now = time.monotonic()
self._failures = [at for at in self._failures if now - at < self.window]
return len(self._failures) >= self.limit
def record_failure(self) -> None:
self._failures.append(time.monotonic())
class SecurityMiddleware:
"""Pure-ASGI so the SSE stream keeps streaming (BaseHTTPMiddleware buffers)."""
@@ -150,7 +234,12 @@ class SecurityMiddleware:
self.session = session
self.config = config
self.max_request_bytes = config.max_request_bytes
self.allowed_hosts = frozenset(LOOPBACK_HOSTS | {str(config.host).lower()})
self.allowed_hosts = frozenset(
LOOPBACK_HOSTS
| {str(config.host).lower()}
| {split_host(name)[0].lower() for name in config.allowed_hosts}
)
self.trusted_proxies = frozenset(config.trusted_proxies)
async def __call__(self, scope, receive, send) -> None:
if scope["type"] != "http":
@@ -161,12 +250,22 @@ class SecurityMiddleware:
# policy never has to parse a Cookie header.
lookup = dict(headers)
lookup["cookie-session"] = _cookie(headers.get("cookie", ""), SESSION_COOKIE)
client = scope.get("client")
scheme, lookup["host"] = external_view(
client=client[0] if client else None,
headers=headers,
scheme=scope.get("scheme", "http"),
trusted_proxies=self.trusted_proxies,
)
# What the session cookie's Secure flag is decided from, one hop later.
scope.setdefault("state", {})["external_scheme"] = scheme
refusal = evaluate(
method=scope.get("method", "GET"),
path=scope.get("path", "/"),
headers=lookup,
session=self.session,
allowed_hosts=self.allowed_hosts,
scheme=scheme,
max_request_bytes=self.max_request_bytes,
)
if refusal is None:

View File

@@ -7,6 +7,9 @@ real external call needs them.
pydantic-settings would do this too, but a prefix-scan over the declared fields
is a few lines and one fewer dependency.
Tuple-valued settings are lists in one variable: library roots are ``os.pathsep``
separated because they are paths, everything else is comma separated.
"""
from __future__ import annotations
@@ -20,6 +23,7 @@ from pydantic import BaseModel, ConfigDict, SecretStr
ENV_PREFIX = "PHOTO_PIPELINE_"
ENV_FILE_VAR = f"{ENV_PREFIX}ENV_FILE"
DEFAULT_ENV_FILE = Path(".env")
COMMA_LIST_FIELDS = frozenset({"allowed_hosts", "trusted_proxies"})
# The archived CLI's variable names, so the configuration file an operator already
# has keeps working. The vision provider reads the OpenAI SDK's names, and the
@@ -85,6 +89,19 @@ class Config(BaseModel):
log_level: str = "INFO"
log_format: str = "json" # "json" or "text"
# Trust boundary (US08-01). Empty means loopback only, which is what the app did
# before there was a setting: a request whose Host is not a loopback name is
# refused, and no secret is needed because nothing outside this machine can call.
# Naming a real hostname here is what makes the app reachable through a reverse
# proxy, and it is exactly then that ``access_secret`` becomes mandatory.
allowed_hosts: tuple[str, ...] = ()
# Addresses whose ``X-Forwarded-Proto``/``X-Forwarded-Host`` may be believed. A
# client that is not the proxy can otherwise declare its own origin.
trusted_proxies: tuple[str, ...] = ()
# Exchanged for the session cookie at the bootstrap endpoint. Once set it is
# required even on loopback, so a development setup cannot half-enable it.
access_secret: SecretStr | None = None
# Largest request body the API accepts. Every endpoint takes small JSON commands;
# anything larger is a mistake or an attempt to exhaust memory (US07-02).
max_request_bytes: int = 1_048_576
@@ -131,5 +148,10 @@ class Config(BaseModel):
raw = env.get(ENV_PREFIX + name.upper())
if not raw:
continue
data[name] = raw.split(os.pathsep) if name == "library_roots" else raw
if name == "library_roots":
data[name] = raw.split(os.pathsep)
elif name in COMMA_LIST_FIELDS:
data[name] = [part.strip() for part in raw.split(",") if part.strip()]
else:
data[name] = raw
return cls(**data)

View File

@@ -12,8 +12,10 @@ nt-apply-list, pa-nsfw-filter). No dependency on the archived entry points.
from __future__ import annotations
import functools
import json
import os
import shutil
import subprocess
from collections.abc import Iterable
@@ -32,6 +34,27 @@ def _timeout() -> float:
return DEFAULT_TIMEOUT_SECONDS
def find_binary(binary: str = "exiftool") -> str | None:
"""Absolute path of exiftool, or ``None`` when it is not installed."""
return shutil.which(binary)
@functools.lru_cache(maxsize=1)
def version() -> str | None:
"""Reported exiftool version, or ``None`` when it is missing or unusable.
Cached: it cannot change inside a running process, and diagnostics asks for it
on every report (US08-02, where a container image pins this version).
"""
try:
result = subprocess.run(
["exiftool", "-ver"], capture_output=True, text=True, timeout=_timeout()
)
except (OSError, subprocess.SubprocessError):
return None
return (result.stdout or "").strip() or None
def read_keyword_sets(paths: Iterable[str]) -> dict[str, set[str]]:
"""Map each path to its lowercased set of ``Keywords`` + ``Subject`` values.

View File

@@ -15,9 +15,7 @@ import os
from pathlib import Path
from typing import Iterable, Iterator
SUPPORTED_EXTENSIONS = {
".jpg", ".jpeg", ".png", ".webp", ".heic", ".heif", ".tiff", ".tif"
}
SUPPORTED_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp", ".heic", ".heif", ".tiff", ".tif"}
EXCLUDED_DIR_NAMES = {"_IGNORE", ".@__thumb"}
@@ -72,6 +70,63 @@ def resolve_in_roots(roots: Iterable[os.PathLike | str], path: os.PathLike | str
raise PathPolicyError("path is outside the configured library roots")
def in_container() -> bool:
"""Whether this process is running inside a container image build of the app."""
return Path("/.dockerenv").exists()
def _under_mount(path: Path) -> bool:
"""Whether ``path`` or one of its parents below ``/`` is a mounted filesystem."""
current = path.resolve()
while current != current.parent:
if os.path.ismount(current):
return True
current = current.parent
return False
def roots_refusal(roots: Iterable[os.PathLike | str], *, require_mount: bool = False) -> str | None:
"""Why the configured library roots cannot be worked with, or ``None`` (US08-03).
The roots name the directories this installation renames folders in, rewrites
EXIF in, and archives from. Container path policy is the same problem as host
path policy with one new failure mode: the configured roots must name the
*container-side* mount paths. A host path configured inside a container is
either absent or an ordinary directory of the image, so the library looks empty
and the first write lands in the container's throwaway layer instead of in the
library. That is worth refusing at startup, while an operator is still watching,
rather than at the first write.
``require_mount`` is the container-only half of that: inside a container a real
library arrives through a bind mount, so a root that is not on (or under) a
mount point is not the library the deployment meant.
Writability is deliberately *not* checked: a bind mount's ownership is
virtualised by Docker Desktop and Colima (it arrives as ``root:root``), so
``os.access`` there is evidence about the virtio layer rather than about the
library. A wrong UID/GID surfaces as a refused rename with the real errno, which
is at least true; a refusal here would be false on two supported platforms.
"""
for root in roots:
path = Path(root)
if not path.exists():
return (
f"library root {path} does not exist: PHOTO_PIPELINE_LIBRARY_ROOTS must "
"name paths that exist here, and in a container that means the mount path"
)
if not path.is_dir():
return f"library root {path} is not a directory"
if not os.access(path, os.R_OK | os.X_OK):
return f"library root {path} is not readable by this process"
if require_mount and not _under_mount(path):
return (
f"library root {path} is not on a mounted filesystem in this container: "
"the library was not bind-mounted there, so PHOTO_PIPELINE_LIBRARY_ROOTS "
"names a directory of the image rather than the library"
)
return None
def iter_supported_files(root: os.PathLike | str) -> Iterator[Path]:
"""Yield supported, non-excluded files under ``root`` in deterministic order.

View File

@@ -15,9 +15,17 @@ package:
"started_at": "...", "library_roots": ["..."]}
One holder per role: an API and a worker are designed to run together, a second
worker is not. A lock whose process is gone is stale and is taken over with the
takeover recorded — refusing to start because of a crashed predecessor would turn
one outage into two.
worker is not. A lock whose process is gone is stale and is taken over — refusing
to start because of a crashed predecessor would turn one outage into two.
Ownership is an advisory ``flock`` on that file, not the record inside it. The
record says *who*; the kernel says *whether*. That distinction is what makes the
lock work in containers (US08-03), where a PID and a hostname are namespaced: a
lock left behind by a container that no longer exists names a pid that still
"exists" in the new container and a host that cannot be probed, so believing the
file would deadlock every restart. A flock is released when its holder dies however
it dies, and is seen by every process that can open the file — which for a local
data directory is every container of this deployment.
Legacy detection is deliberately a heuristic, not a promise: the archived CLI has
no lock of its own, so what can be observed is its state files being written right
@@ -27,6 +35,7 @@ mutating stage should refuse until it stops.
from __future__ import annotations
import fcntl
import json
import os
import socket
@@ -109,6 +118,12 @@ def _now() -> datetime:
return datetime.now(timezone.utc)
def _in_container() -> bool:
from photo_pipeline import path_policy
return path_policy.in_container()
def legacy_activity(config: Config) -> dict:
"""Legacy state files written within the activity window, if any."""
seen: list[dict] = []
@@ -139,6 +154,7 @@ class LibraryLock:
self.role = role
self.path = Path(config.data_dir) / f"{role}{LOCK_SUFFIX}"
self._acquired = False
self._handle = None
# ── inspection ────────────────────────────────────────────────────────────
@@ -178,12 +194,31 @@ class LibraryLock:
"stop it before running the application"
)
self.path.parent.mkdir(parents=True, exist_ok=True)
# The kernel decides, because the file cannot: a container's PID and hostname
# are namespaced, so a lock left by a container that no longer exists names a
# pid that "exists" and a host that cannot be probed (US08-03). An advisory
# flock is held by a live process or by nobody, is released when that process
# dies however it dies, and is shared by every process that can open this
# file — which, for a local data directory, is every role in every container
# of this deployment.
handle = open(self.path, "a+", encoding="utf-8")
try:
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError:
handle.close()
raise LockHeld(self.holder() or Holder(self.role, -1, "unknown", "unknown")) from None
current = self.holder()
if current is not None:
if current.alive:
raise LockHeld(current)
# Stale: its process is gone. Take over, and say so.
self.path.unlink(missing_ok=True)
if current is not None and current.host != socket.gethostname() and not _in_container():
# We hold the kernel's lock, so nothing on *this* machine holds the file.
# On a host that still leaves one case open: a data directory shared with
# another machine, whose flock we cannot trust. Believe its record rather
# than run two writers. In a container the data volume is local by
# construction (US08-03), and a foreign hostname is only a dead container.
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
handle.close()
raise LockHeld(current)
mine = Holder(
role=self.role,
@@ -192,15 +227,14 @@ class LibraryLock:
started_at=_now().isoformat(),
library_roots=tuple(str(root) for root in self._config.library_roots),
)
self.path.parent.mkdir(parents=True, exist_ok=True)
payload = {k: v for k, v in mine.as_dict().items() if k != "alive"}
# Exclusive create, so two processes racing here cannot both believe they won.
try:
with open(self.path, "x", encoding="utf-8") as handle:
json.dump(payload, handle, indent=2)
except FileExistsError:
winner = self.holder()
raise LockHeld(winner or mine) from None
handle.seek(0)
handle.truncate()
json.dump(payload, handle, indent=2)
handle.flush()
# Held open on purpose: closing it is what releases the lock, and that must
# happen when this process ends, not when this method returns.
self._handle = handle
self._acquired = True
return mine
@@ -208,9 +242,10 @@ class LibraryLock:
"""Give up a lock this process owns. Another holder's lock is left alone."""
if not self._acquired:
return
current = self.holder()
if current is not None and current.pid == os.getpid():
self.path.unlink(missing_ok=True)
self.path.unlink(missing_ok=True)
if self._handle is not None:
self._handle.close() # closing the descriptor releases the kernel lock
self._handle = None
self._acquired = False
def __enter__(self) -> "LibraryLock":

View File

@@ -15,10 +15,13 @@ application.
from __future__ import annotations
import functools
import json
import shutil
from pathlib import Path
from photo_pipeline.config import Config
from photo_pipeline.integrations import exiftool, immich_go
from photo_pipeline.services import app_lock
# Below this much free space, mutating stages should stop rather than risk a
@@ -26,6 +29,12 @@ from photo_pipeline.services import app_lock
LOW_DISK_BYTES = 1_000_000_000
CRITICAL_DISK_BYTES = 200_000_000
# Written into the container image at build time (US08-02). The image pins exiftool
# and immich-go, and this file is how a running container reports which versions it
# was built with — so a drifted or missing binary is visible here rather than in a
# failed EXIF checkpoint or a misparsed upload report.
IMAGE_VERSIONS_FILE = Path("/etc/photo-pipeline/versions.json")
def _tree_bytes(path: Path) -> int:
if not path.exists():
@@ -69,8 +78,47 @@ def disk(path: Path) -> dict:
}
def _pinned_versions() -> dict[str, str]:
"""The versions this image recorded at build time; empty outside a container."""
try:
recorded = json.loads(IMAGE_VERSIONS_FILE.read_text())
except (OSError, ValueError):
return {}
if not isinstance(recorded, dict):
return {}
return {str(name): str(value) for name, value in recorded.items()}
@functools.lru_cache(maxsize=4)
def _uploader_version(binary: str) -> str | None:
"""Cached: the uploader cannot change version inside one process."""
return immich_go.version(binary)
def tools(config: Config) -> list[dict]:
"""The external executables the pipeline shells out to, and their versions.
``pinned`` is what the image was built against, ``version`` is what is actually
installed. They differ only when the binary was replaced or mounted over.
"""
return [
{
"name": "exiftool",
"path": exiftool.find_binary(),
"version": exiftool.version(),
"pinned": _pinned_versions().get("exiftool"),
},
{
"name": "immich-go",
"path": immich_go.find_binary(config.immich_go_binary),
"version": _uploader_version(config.immich_go_binary),
"pinned": _pinned_versions().get("immich-go"),
},
]
def report(config: Config) -> dict:
"""Sizes, disk headroom, warnings, and who currently holds the library lock."""
"""Sizes, disk headroom, tool versions, warnings, and who holds the library lock."""
database = config.database_path
components = [
_component("database", database),
@@ -132,6 +180,23 @@ def report(config: Config) -> dict:
}
)
installed_tools = tools(config)
for tool in installed_tools:
# A missing tool is reported as ``version: null`` rather than warned about: on a
# development machine the uploader is legitimately absent, and the stages that
# need it already refuse to run. A *drifted* tool is different — the image pinned
# a version and something replaced it.
if tool["version"] and tool["pinned"] and tool["pinned"] not in tool["version"]:
warnings.append(
{
"code": "tool_version_drift",
"message": (
f"{tool['name']} reports {tool['version']} but this image pinned "
f"{tool['pinned']}"
),
}
)
locks = {}
for role in ("api", "worker"):
holder = app_lock.LibraryLock(config, role).holder()
@@ -151,6 +216,7 @@ def report(config: Config) -> dict:
"components": components,
"total_bytes": sum(component["bytes"] for component in components),
"disk": space,
"tools": installed_tools,
"warnings": warnings,
"locks": locks,
"legacy_activity": legacy,

View File

@@ -49,8 +49,16 @@ STAGES: tuple[tuple[str, tuple[str, ...]], ...] = (
("browser", ("tests/e2e",)),
)
# Skips the gate accepts, because they describe the machine rather than the code.
ALLOWED_SKIP_REASONS = ("exiftool not installed", "root ignores directory permissions")
# Skips the gate accepts, because they describe the machine rather than the code. The
# container ones (US08-02) belong here for the same reason exiftool does: the image
# build needs a Docker daemon and the network, and its definition is still checked
# offline in tests/integration/test_container_image.py.
ALLOWED_SKIP_REASONS = (
"exiftool not installed",
"root ignores directory permissions",
"no Docker daemon available",
"bind-mount ownership is virtualised",
)
class ReleaseError(RuntimeError):

View File

@@ -49,4 +49,5 @@ markers = [
"phase_d: Phase D end-to-end acceptance (US04-06) — guarded rename API, fault, and browser journeys",
"phase_e: Phase E end-to-end acceptance (US05-06) — upload preflight, uploader, and browser journeys",
"phase_f: Phase F end-to-end acceptance (US06-06) — archive destination, transfer, and restore journeys",
"container: builds and runs the container image and its composition (US08-02, US08-03) — needs a Docker daemon, the compose plugin, and the network",
]

View File

@@ -0,0 +1,337 @@
"""US08-03: the composition, actually composed.
Nothing here is faked below the process boundary: Docker builds the image, Compose
starts the migrate/API/worker containers against a temporary fixture library on a
real bind mount, and every assertion is made over HTTP or against what the stack
left in its data volume. The vision provider is the deterministic fake seam the
other end-to-end suites use, because an upload of real photos to a real model is not
what this story is about — the mount, the lock, the volume, and the restart are.
The file contract (one API, one worker, migrations first, no committed values) is
checked without a daemon in ``tests/integration/test_compose_runtime.py``; only the
running proof needs Docker, and CI is where it runs unskipped (US08-04).
"""
from __future__ import annotations
import json
import os
import shutil
import socket
import subprocess
import sys
import tempfile
import time
from pathlib import Path
import httpx
import pytest
from PIL import Image
REPO = Path(__file__).resolve().parents[2]
PROJECT = "photo-pipeline-us0803"
IMAGE = "photo-pipeline-test:us08-03"
SECRET = "compose-acceptance-secret"
CONTAINER_LIBRARY = "/library"
READY_TIMEOUT_SECONDS = 180
JOB_TIMEOUT_SECONDS = 180
UP_TIMEOUT_SECONDS = 30 * 60
pytestmark = pytest.mark.container
def compose_available() -> bool:
try:
return (
subprocess.run(
["docker", "compose", "version"], capture_output=True, timeout=60
).returncode
== 0
)
except (OSError, subprocess.SubprocessError):
return False
needs_compose = pytest.mark.skipif(
not compose_available(), reason="no Docker daemon with the compose plugin"
)
def free_port() -> int:
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
return sock.getsockname()[1]
class Stack:
"""The composition under test, plus the environment it was started with."""
def __init__(self, library: Path, env_file: Path) -> None:
self.library = library
self.port = free_port()
self.base = f"http://127.0.0.1:{self.port}"
# Compose reads the repository's own .env for substitution; the process
# environment wins over it, so the test's values are the ones that apply.
self.env = {
**os.environ,
"PHOTO_PIPELINE_IMAGE": IMAGE,
"PHOTO_PIPELINE_ENV_FILE": str(env_file),
"PHOTO_PIPELINE_LIBRARY_HOST_PATH": str(library),
"PHOTO_PIPELINE_LIBRARY_ROOTS": CONTAINER_LIBRARY,
"PHOTO_PIPELINE_PORT": str(self.port),
"PHOTO_PIPELINE_UID": str(os.getuid()),
"PHOTO_PIPELINE_GID": str(os.getgid()),
}
def compose(self, *args: str, check: bool = True, timeout: int = 300):
result = subprocess.run(
["docker", "compose", "-p", PROJECT, "-f", str(REPO / "docker-compose.yml"), *args],
capture_output=True,
text=True,
env=self.env,
cwd=REPO,
timeout=timeout,
)
if check and result.returncode != 0:
raise AssertionError(
f"docker compose {' '.join(args)} failed:\n{result.stdout}\n{result.stderr}\n"
f"{self.compose('logs', '--tail', '80', check=False).stdout}"
)
return result
def wait_until_ready(self) -> None:
deadline = time.monotonic() + READY_TIMEOUT_SECONDS
while time.monotonic() < deadline:
try:
if httpx.get(f"{self.base}/api/v1/health/ready", timeout=5).status_code == 200:
return
except httpx.HTTPError:
pass
time.sleep(0.5)
logs = self.compose("logs", "--tail", "120", check=False)
raise AssertionError(f"the stack never became ready:\n{logs.stdout}\n{logs.stderr}")
def client(self) -> httpx.Client:
client = httpx.Client(base_url=f"{self.base}/api/v1", timeout=60)
bootstrap = client.get("/session", headers={"X-Access-Secret": SECRET})
assert bootstrap.status_code == 200, bootstrap.text
client.headers["X-CSRF-Token"] = bootstrap.json()["csrf_token"]
return client
@pytest.fixture(scope="module")
def library() -> Path:
"""A small fixture library on the host, mounted into both containers.
Not under pytest's ``tmp_path``: on macOS that is ``/var/folders/...``, which a
Docker VM (Colima, Docker Desktop) does not share, so the bind mount would arrive
empty and every assertion below would be about nothing. ``$HOME`` is shared by
every default configuration.
"""
base = Path(
os.environ.get("PHOTO_PIPELINE_TEST_MOUNT_BASE", Path.home() / ".cache" / "photo-pipeline")
)
base.mkdir(parents=True, exist_ok=True)
root = Path(tempfile.mkdtemp(prefix="library-", dir=base))
for album, count in (("01_day", 2), ("02_night", 1)):
(root / album).mkdir()
for index in range(count):
colour = (40 * (index + 1), 90, 160)
Image.new("RGB", (64, 48), colour).save(root / album / f"{album}_{index}.jpg")
# The exclusion sentinel: it must never be discovered, counted, or analyzed.
(root / "_IGNORE").mkdir()
Image.new("RGB", (32, 32), (0, 0, 0)).save(root / "_IGNORE" / "sentinel.jpg")
try:
yield root
finally:
shutil.rmtree(root, ignore_errors=True)
@pytest.fixture(scope="module")
def env_file(tmp_path_factory) -> Path:
"""Configuration and secrets come from the environment, so the test writes its
own file rather than borrowing the operator's."""
path = tmp_path_factory.mktemp("config") / "compose.env"
path.write_text(
"\n".join(
[
f"PHOTO_PIPELINE_ACCESS_SECRET={SECRET}",
"PHOTO_PIPELINE_LOG_FORMAT=text",
# The deterministic vision seam, in the data volume so both roles and
# the test can see it (concept §18).
"PHOTO_PIPELINE_FAKE_VISION_LOG=/data/vision.log",
]
)
+ "\n"
)
return path
@pytest.fixture(scope="module")
def stack(library, env_file):
if not compose_available():
pytest.skip("no Docker daemon with the compose plugin")
running = Stack(library, env_file)
running.compose("down", "--volumes", "--remove-orphans", check=False)
running.compose("up", "--detach", "--build", timeout=UP_TIMEOUT_SECONDS)
try:
running.wait_until_ready()
yield running
finally:
running.compose("down", "--volumes", "--remove-orphans", check=False, timeout=300)
def await_job(client: httpx.Client, job_id: str, states=("succeeded",)) -> dict:
deadline = time.monotonic() + JOB_TIMEOUT_SECONDS
snapshot: dict = {}
while time.monotonic() < deadline:
response = client.get(f"/jobs/{job_id}")
if response.status_code == 200:
snapshot = response.json()
if snapshot["state"] in states:
return snapshot
time.sleep(0.5)
raise AssertionError(f"job {job_id} never reached {states}: {snapshot}")
# ── the mounted library ──────────────────────────────────────────────────────
@needs_compose
def test_the_stack_scans_the_bind_mounted_library_at_its_container_paths(stack):
client = stack.client()
try:
scanned = client.post("/inventory/scan")
assert scanned.status_code == 200, scanned.text
assets = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
finally:
client.close()
assert len(assets) == 3, assets
paths = {asset["current_path"] for asset in assets}
assert all(path.startswith(CONTAINER_LIBRARY + "/") for path in paths), paths
assert not any("_IGNORE" in path or "sentinel" in path for path in paths)
# The host paths are what the operator mounted, and they are not what the
# application records: the roots are the container's.
assert not any(str(stack.library) in path for path in paths)
@needs_compose
def test_a_library_root_that_is_not_mounted_is_refused_at_startup(stack):
"""The container-specific failure: configured roots that name nothing mounted."""
refused = stack.compose(
"run",
"--rm",
"--no-deps",
"--env",
"PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos",
"api",
"serve",
check=False,
)
assert refused.returncode == 5, refused.stdout + refused.stderr
assert "library root /srv/photos" in refused.stdout + refused.stderr
@needs_compose
def test_a_second_worker_is_refused_by_the_library_lock(stack):
"""Not by convention: the running worker's lock is in the shared data volume."""
refused = stack.compose(
"run", "--rm", "--no-deps", "worker", "worker", "--id", "worker-2", check=False
)
assert refused.returncode == 2, refused.stdout + refused.stderr
assert "worker is already running" in refused.stdout + refused.stderr
# ── restart, resume, and the data volume ─────────────────────────────────────
@needs_compose
def test_a_queued_job_resumes_after_both_containers_restart(stack):
client = stack.client()
try:
client.post("/inventory/scan").raise_for_status()
assets = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
for asset in assets:
decided = client.post(
"/safety/decisions", json={"asset_id": asset["id"], "decision": "sfw"}
)
assert decided.status_code == 200, decided.text
# Stop the worker first, so the job is provably still queued when the restart
# happens: a job that finished before the restart would prove nothing.
stack.compose("stop", "worker")
job = client.post("/analysis/jobs").json()
assert client.get(f"/jobs/{job['id']}").json()["state"] == "queued"
finally:
client.close()
stack.compose("restart", "api", "worker")
stack.wait_until_ready()
client = stack.client() # the session is per API process, so it is re-bootstrapped
try:
finished = await_job(client, job["id"])
assert finished["state"] == "succeeded", finished
# The database is intact and the work is durable, not merely reported.
after = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
assert {asset["id"] for asset in after} == {asset["id"] for asset in assets}
analysed = client.get(f"/analysis/results/{assets[0]['id']}")
assert analysed.status_code == 200, analysed.text
assert analysed.json()["description"]
# Only the mounted library's own photos were analysed: the sentinel under
# _IGNORE is not an asset, so it can never have become an item of this job.
assert finished["progress"]["total"] == len(assets)
finally:
client.close()
@needs_compose
def test_the_data_volume_survives_recreating_the_containers(stack):
"""`down` without `--volumes` then `up` is the upgrade path: state stays."""
client = stack.client()
try:
client.post("/inventory/scan").raise_for_status()
before = {a["id"] for a in client.get("/inventory/assets").json()["items"]}
finally:
client.close()
stack.compose("down", "--remove-orphans", timeout=300)
stack.compose("up", "--detach", timeout=UP_TIMEOUT_SECONDS)
stack.wait_until_ready()
client = stack.client()
try:
after = {a["id"] for a in client.get("/inventory/assets").json()["items"]}
finally:
client.close()
assert after == before, "the same assets, from the same database, on the same volume"
# ── operating it ─────────────────────────────────────────────────────────────
@needs_compose
def test_backup_verify_and_diagnostics_run_as_container_commands(stack):
backup = stack.compose("run", "--rm", "--no-deps", "api", "backup", "--reason", "compose")
manifest = json.loads(backup.stdout[backup.stdout.index("{") :])
assert manifest["name"].startswith("2")
verified = stack.compose(
"run", "--rm", "--no-deps", "api", "verify-backup", f"/data/backups/{manifest['name']}"
)
assert json.loads(verified.stdout[verified.stdout.index("{") :])["ok"] is True
report = stack.compose("run", "--rm", "--no-deps", "api", "diagnostics")
diagnostics = json.loads(report.stdout[report.stdout.index("{") :])
components = {c["name"]: c["path"] for c in diagnostics["components"]}
# Database, WAL, thumbnail cache, and backups all live in the mounted volume.
for name in ("database", "write_ahead_log", "thumbnail_cache", "backups"):
assert components[name].startswith("/data/"), (name, components[name])
assert {tool["name"] for tool in diagnostics["tools"]} >= {"exiftool", "immich-go"}
# And the worker running beside this command is visible as the lock's holder.
assert diagnostics["locks"]["worker"]["role"] == "worker"
if __name__ == "__main__": # a quick way to run just this file
raise SystemExit(pytest.main([__file__, "-v", *sys.argv[1:]]))

View File

@@ -0,0 +1,306 @@
"""US08-02: the built image, actually built and actually run.
This is the acceptance test for the image itself, so nothing here is faked: Docker
builds from a clean context, the container starts under a chosen UID/GID against a
mounted data directory, and the assertions are made over HTTP and against the files
the container left on the host.
It is skipped without a Docker daemon — the build also needs the network for the base
image, the pinned exiftool package, and the pinned uploader release. The contract the
Dockerfile itself has to keep (pins, non-root, health target, build context) is checked
offline in ``tests/integration/test_container_image.py``, so a machine without Docker
still fails on a broken image definition; only the running proof needs the daemon. CI
builds the image on every change (US08-04), which is where this runs unskipped.
"""
from __future__ import annotations
import json
import os
import platform
import re
import socket
import subprocess
import sys
import time
from pathlib import Path
import httpx
import pytest
REPO = Path(__file__).resolve().parents[2]
IMAGE = "photo-pipeline-test:us08-02"
SECRET = "container-acceptance-secret"
HOSTNAME = "photos.test"
READY_TIMEOUT_SECONDS = 120
BUILD_TIMEOUT_SECONDS = 30 * 60
pytestmark = pytest.mark.container
def docker_available() -> bool:
try:
return subprocess.run(["docker", "info"], capture_output=True, timeout=60).returncode == 0
except (OSError, subprocess.SubprocessError):
return False
needs_docker = pytest.mark.skipif(not docker_available(), reason="no Docker daemon available")
def docker(*args: str, check: bool = True, timeout: int = 120) -> subprocess.CompletedProcess:
result = subprocess.run(
["docker", *args], capture_output=True, text=True, timeout=timeout
)
if check and result.returncode != 0:
raise AssertionError(f"docker {' '.join(args)} failed:\n{result.stdout}\n{result.stderr}")
return result
def pins() -> dict[str, str]:
"""The pinned versions, read from the Dockerfile that produced the image."""
text = (REPO / "Dockerfile").read_text()
found = dict(re.findall(r"^ARG\s+([A-Z0-9_]+)=(.+)$", text, re.MULTILINE))
return {
# The Debian package version carries a packaging suffix; exiftool reports the
# upstream version only.
"exiftool": found["EXIFTOOL_VERSION"].split("+")[0].split("-")[0],
"immich-go": found["IMMICH_GO_VERSION"],
}
def free_port() -> int:
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
return sock.getsockname()[1]
@pytest.fixture(scope="module")
def image() -> str:
"""Build from a clean checkout: the build context is the repository, unmodified."""
if not docker_available():
pytest.skip("no Docker daemon available")
docker(
"build",
"--build-arg",
f"UID={os.getuid()}",
"--build-arg",
f"GID={os.getgid()}",
"-t",
IMAGE,
str(REPO),
timeout=BUILD_TIMEOUT_SECONDS,
)
return IMAGE
@pytest.fixture
def data_dir(tmp_path) -> Path:
data = tmp_path / "data"
data.mkdir()
return data
def run_detached(image: str, port: int, *args: str, data: Path | None = None) -> str:
"""Start a container. ``data`` bind-mounts the host's data directory when the test
is about the files themselves; otherwise the image's own /data is used, because a
macOS bind mount arrives with an ownership the container did not choose."""
result = docker(
"run",
"--detach",
"--rm",
"--publish",
f"127.0.0.1:{port}:8000",
*(("--volume", f"{data}:/data") if data is not None else ()),
"--env",
# Reachable from outside the container means reachable from another machine as
# far as the application is concerned, so the access secret is mandatory
# (US08-01) — the image must not weaken that.
"PHOTO_PIPELINE_HOST=0.0.0.0",
"--env",
f"PHOTO_PIPELINE_ACCESS_SECRET={SECRET}",
"--env",
f"PHOTO_PIPELINE_ALLOWED_HOSTS={HOSTNAME}",
image,
*args,
)
return result.stdout.strip()
def wait_until_ready(base: str, container: str) -> None:
deadline = time.monotonic() + READY_TIMEOUT_SECONDS
while time.monotonic() < deadline:
try:
if httpx.get(f"{base}/api/v1/health/ready", timeout=5).status_code == 200:
return
except httpx.HTTPError:
pass
if docker("inspect", "-f", "{{.State.Running}}", container, check=False).stdout.strip() in (
"false",
"",
):
break
time.sleep(0.5)
logs = docker("logs", container, check=False)
raise AssertionError(f"container never became ready:\n{logs.stdout}\n{logs.stderr}")
@pytest.fixture
def serving(image):
port = free_port()
container = run_detached(image, port, "serve")
try:
base = f"http://127.0.0.1:{port}"
wait_until_ready(base, container)
yield base, container
finally:
docker("rm", "--force", container, check=False)
def session(base: str) -> httpx.Client:
client = httpx.Client(base_url=base, timeout=30)
bootstrap = client.get("/api/v1/session", headers={"X-Access-Secret": SECRET})
assert bootstrap.status_code == 200, bootstrap.text
client.headers["X-CSRF-Token"] = bootstrap.json()["csrf_token"]
return client
# ── the image serves, and says what it contains ──────────────────────────────
@needs_docker
def test_the_container_serves_the_frontend_and_the_pinned_tool_versions(serving):
base, container = serving
index = httpx.get(f"{base}/app/index.html", timeout=30)
assert index.status_code == 200
assert "<title" in index.text.lower(), "the application shell, not an API error"
client = session(base)
try:
tools = {tool["name"]: tool for tool in client.get("/api/v1/diagnostics").json()["tools"]}
finally:
client.close()
for name, pinned in pins().items():
assert tools[name]["pinned"] == pinned, name
# Recorded *and* installed: the reported version comes from running the binary.
assert pinned in tools[name]["version"], (name, tools[name])
assert tools[name]["path"], f"{name} is not on PATH inside the image"
logs = docker("logs", container, check=False)
assert SECRET not in logs.stdout + logs.stderr, "the access secret never reaches the log"
@needs_docker
def test_the_declared_health_check_reports_readiness(serving):
"""The declared HEALTHCHECK is readiness, so Docker's own verdict is the assertion."""
_, container = serving
deadline = time.monotonic() + READY_TIMEOUT_SECONDS
status = ""
while time.monotonic() < deadline:
status = docker(
"inspect", "-f", "{{.State.Health.Status}}", container, check=False
).stdout.strip()
if status == "healthy":
break
time.sleep(1)
assert status == "healthy"
probe = docker("exec", container, "/usr/local/bin/healthcheck.sh", check=False)
assert probe.returncode == 0
# Point the probe at a port nothing serves: the same script must fail, which is
# what makes the healthy verdict above evidence rather than a default.
unready = docker(
"exec",
"--env",
"PHOTO_PIPELINE_PORT=1",
container,
"/usr/local/bin/healthcheck.sh",
check=False,
)
assert unready.returncode != 0
# ── identity: never root, always the configured owner ────────────────────────
@needs_docker
def test_the_container_refuses_to_run_as_root(image, data_dir):
result = docker(
"run",
"--rm",
"--user",
"0:0",
"--volume",
f"{data_dir}:/data",
image,
"diagnostics",
check=False,
)
assert result.returncode != 0
assert "refusing to run as root" in result.stderr + result.stdout
assert not list(data_dir.iterdir()), "a refused container writes nothing"
@needs_docker
def test_what_the_container_writes_is_owned_by_the_build_arguments(image):
"""The identity the image was built with is the identity on disk afterwards.
Asserted from inside the container so it holds on every host: a macOS bind mount
reports an ownership the container never chose. The host-side proof, which is what
the mounted library actually needs, is the Linux test below.
"""
port = free_port()
container = run_detached(image, port, "serve")
try:
wait_until_ready(f"http://127.0.0.1:{port}", container)
owner = docker(
"exec", container, "stat", "-c", "%u:%g", "/data/photo_pipeline.db"
).stdout.strip()
assert owner == f"{os.getuid()}:{os.getgid()}"
assert docker("exec", container, "id", "-u").stdout.strip() == str(os.getuid())
finally:
docker("rm", "--force", container, check=False)
@needs_docker
@pytest.mark.skipif(
platform.system() != "Linux",
reason="bind-mount ownership is virtualised by Docker Desktop on macOS/Windows",
)
def test_files_the_container_writes_keep_the_configured_ownership(image, data_dir):
docker("run", "--rm", "--volume", f"{data_dir}:/data", image, "migrate", timeout=300)
written = sorted(path for path in data_dir.rglob("*") if path.is_file())
assert written, "migrate creates the database in the mounted data directory"
for path in written:
assert (path.stat().st_uid, path.stat().st_gid) == (os.getuid(), os.getgid()), path
@needs_docker
def test_the_worker_role_runs_from_the_same_image(image, data_dir):
"""One image, two roles: the worker is the same entrypoint with another argument."""
port = free_port()
container = run_detached(image, port, "worker", "--id", "container-worker")
try:
# Taking the worker's library lock is the observable proof that it started,
# migrated, and reached its job loop — no sleep required (US07-05).
deadline = time.monotonic() + READY_TIMEOUT_SECONDS
lock = ""
while not lock and time.monotonic() < deadline:
assert docker("inspect", "-f", "{{.State.Running}}", container).stdout.strip() == (
"true"
), docker("logs", container, check=False).stdout
lock = docker("exec", container, "cat", "/data/worker.lock.json", check=False).stdout
time.sleep(0.5)
assert lock, docker("logs", container, check=False).stdout
assert json.loads(lock)["role"] == "worker"
role = docker("exec", container, "cat", "/tmp/photo-pipeline-role").stdout.strip()
assert role == "worker", "the health check can tell which role this container is"
finally:
docker("rm", "--force", container, check=False)
if __name__ == "__main__": # a quick way to run just this file
raise SystemExit(pytest.main([__file__, "-v", *sys.argv[1:]]))

View File

@@ -0,0 +1,299 @@
"""US08-03: the composition's contract, and the library-root check it depends on.
Bringing the stack up needs a Docker daemon and the network, which is what
``tests/e2e/test_compose_stack.py`` does. What can be checked without either is
checked here, because the parts that rot silently — a second writer that is only
prevented by convention, a data volume that stopped being the same volume for both
roles, migrations that stopped running first, a committed value in a file that must
carry none — are all readable from the files.
The startup refusal is the other half: in a container the configured library roots
must name the mount paths, and a mismatch has to fail before the lock is taken, not
at the first rename.
"""
from __future__ import annotations
import json
import os
import re
import stat
from pathlib import Path
import pytest
import yaml
from photo_pipeline import path_policy
from photo_pipeline.__main__ import main
from photo_pipeline.config import Config
REPO = Path(__file__).resolve().parents[2]
COMPOSE_FILE = REPO / "docker-compose.yml"
COMPOSE = yaml.safe_load(COMPOSE_FILE.read_text())
ENV_EXAMPLE = REPO / ".env.example"
SERVICES = COMPOSE["services"]
DATA_VOLUME = "data:/data"
def env_example_keys() -> list[str]:
"""The variables the example file declares, in file order."""
return [
line.split("=", 1)[0]
for line in ENV_EXAMPLE.read_text().splitlines()
if "=" in line and not line.lstrip().startswith("#")
]
# ── one API, one worker, one library, one volume ─────────────────────────────
def test_exactly_one_serving_and_one_working_container_from_the_same_image():
roles = {name: service["command"][0] for name, service in SERVICES.items()}
assert sorted(roles.values()) == ["migrate", "serve", "worker"]
assert [name for name, role in roles.items() if role == "serve"] == ["api"]
assert [name for name, role in roles.items() if role == "worker"] == ["worker"]
images = {service["image"] for service in SERVICES.values()}
assert len(images) == 1, "both roles must run the same build of the application"
assert "latest" not in images.pop()
# No `replicas`/`scale` key promising a second worker is fine; the lock decides.
assert not any("deploy" in service for service in SERVICES.values())
def test_both_roles_share_the_data_volume_so_the_lock_is_visible_to_both():
"""A second worker is refused by the library lock (US07-05) only if it can see it."""
for name, service in SERVICES.items():
assert DATA_VOLUME in service["volumes"], name
assert COMPOSE["volumes"]["data"]["driver"] == "local"
text = COMPOSE_FILE.read_text()
# The composition has to say why, because the failure is silent corruption.
assert "WAL" in text and re.search(r"NFS|SMB|network", text)
def test_the_library_is_a_bind_mount_whose_target_is_the_configured_root():
for name, service in SERVICES.items():
mounts = [volume for volume in service["volumes"] if volume != DATA_VOLUME]
assert len(mounts) == 1, name
source, target = re.match(r"^(\$\{.*?\}):(\$\{.*?\})$", mounts[0]).groups()
# An unset host path fails the composition rather than mounting something else.
assert source.startswith("${PHOTO_PIPELINE_LIBRARY_HOST_PATH:?")
# The container-side path and the configured root are one variable, so they
# cannot drift apart into a library that is mounted but not configured.
assert target.startswith("${PHOTO_PIPELINE_LIBRARY_ROOTS:?")
assert service["environment"]["PHOTO_PIPELINE_LIBRARY_ROOTS"] == (
"${PHOTO_PIPELINE_LIBRARY_ROOTS}"
)
assert service["environment"]["PHOTO_PIPELINE_DATA_DIR"] == "/data"
def test_migrations_run_to_completion_before_either_role_accepts_work():
"""`migrate` runs the backup-then-migrate path, and a failed upgrade exits
non-zero with its pre-migration backup intact — proven in
tests/integration/test_backup_recovery.py. What the composition adds is that
neither role starts until it succeeded."""
assert SERVICES["migrate"]["command"] == ["migrate"]
assert SERVICES["migrate"]["restart"] == "no", "a one-shot that retries is not a gate"
for role in ("api", "worker"):
assert SERVICES[role]["depends_on"] == {
"migrate": {"condition": "service_completed_successfully"}
}, role
def test_the_api_port_is_published_to_host_loopback_by_default():
published = SERVICES["api"]["ports"]
assert published == [
"${PHOTO_PIPELINE_PUBLISH_ADDRESS:-127.0.0.1}:${PHOTO_PIPELINE_PORT:-8000}:8000"
]
# Reachable from the host means reachable from elsewhere as far as the app is
# concerned, so the access secret stays mandatory (US08-01).
assert SERVICES["api"]["environment"]["PHOTO_PIPELINE_HOST"] == "0.0.0.0"
assert SERVICES["api"]["environment"]["PHOTO_PIPELINE_PORT"] == 8000
assert "PHOTO_PIPELINE_ACCESS_SECRET" not in SERVICES["api"]["environment"]
def test_containers_restart_by_themselves_and_stop_with_time_to_drain():
for role in ("api", "worker"):
assert SERVICES[role]["restart"] == "unless-stopped", role
assert SERVICES[role]["stop_grace_period"] == "30s", role
def test_the_containers_run_as_the_library_owner_and_never_as_root():
for name, service in SERVICES.items():
assert service["user"] == "${PHOTO_PIPELINE_UID:-1000}:${PHOTO_PIPELINE_GID:-1000}", name
assert service["build"]["args"]["UID"] == "${PHOTO_PIPELINE_UID:-1000}", name
# ── configuration comes from the environment, never from a committed file ────
def test_configuration_and_secrets_come_from_the_environment_only():
for name, service in SERVICES.items():
assert service["env_file"] == ["${PHOTO_PIPELINE_ENV_FILE:-.env}"], name
for key, value in service["environment"].items():
# Every value is either a variable reference or a property of the
# composition itself (the volume path, the container's own port).
composed = isinstance(value, int) or value in ("/data", "0.0.0.0")
assert composed or value.startswith("${"), (name, key, value)
assert not (REPO / ".env").is_file() or ".env" in (REPO / ".gitignore").read_text()
def test_the_example_file_lists_every_setting_and_carries_no_values():
declared = env_example_keys()
assert declared == sorted(set(declared), key=declared.index), "no variable twice"
for line in ENV_EXAMPLE.read_text().splitlines():
if "=" in line and not line.lstrip().startswith("#"):
assert line.endswith("="), f"a value in the example file: {line}"
expected = {f"PHOTO_PIPELINE_{name.upper()}" for name in Config.model_fields}
assert expected <= set(declared), sorted(expected - set(declared))
# And every variable the composition substitutes is documented there too.
substituted = set(re.findall(r"\$\{(PHOTO_PIPELINE_[A-Z_]+)", COMPOSE_FILE.read_text()))
assert substituted <= set(declared), sorted(substituted - set(declared))
def test_the_example_file_is_not_a_dotenv_that_could_be_loaded_by_accident():
"""`.env.example` must not be what `.env` is: no values means nothing to leak."""
assert ENV_EXAMPLE.name != ".env"
parsed = {k: v for k, v in _parse(ENV_EXAMPLE.read_text()).items() if v}
assert parsed == {}
def _parse(text: str) -> dict[str, str]:
from photo_pipeline.config import parse_env_file
return parse_env_file(text)
# ── the lock across container lifetimes ──────────────────────────────────────
def test_a_lock_left_by_a_container_that_is_gone_does_not_block_the_restart(
tmp_path, monkeypatch
):
"""A restarted container is a new hostname and a recycled pid 1, so the record
in the lock file proves nothing; the kernel's flock does (US08-03)."""
from photo_pipeline.services.app_lock import LibraryLock
config = Config(data_dir=tmp_path / "data", library_roots=(tmp_path,))
(tmp_path / "data").mkdir()
(tmp_path / "data" / "worker.lock.json").write_text(
json.dumps(
{
"lock_version": 1,
"role": "worker",
"pid": 1, # pid 1 of a container that no longer exists
"host": "3f2a1b9c4d5e", # its hostname was its container id
"started_at": "2026-01-01T00:00:00+00:00",
"library_roots": ["/library"],
}
)
)
monkeypatch.setattr(path_policy, "in_container", lambda: True)
taken = LibraryLock(config, "worker").acquire()
assert taken.pid == os.getpid(), "the worker must come back after a restart"
def test_a_second_worker_is_still_refused_while_the_first_holds_the_lock(tmp_path, monkeypatch):
"""The other half: the same flock refuses a concurrent second writer, whether it
is a process or another container of the same composition."""
from photo_pipeline.services.app_lock import LibraryLock, LockHeld
config = Config(data_dir=tmp_path / "data", library_roots=(tmp_path,))
monkeypatch.setattr(path_policy, "in_container", lambda: True)
first = LibraryLock(config, "worker")
first.acquire()
with pytest.raises(LockHeld, match="worker is already running"):
LibraryLock(config, "worker").acquire()
first.release()
LibraryLock(config, "worker").acquire() # free again
# ── the startup check the mount depends on ───────────────────────────────────
def test_configured_roots_that_are_mounted_and_writable_are_accepted(tmp_path):
assert path_policy.roots_refusal([tmp_path]) is None
assert path_policy.roots_refusal([]) is None, "no roots is a configuration, not a fault"
def test_an_unmounted_library_root_is_refused_by_name(tmp_path):
refusal = path_policy.roots_refusal([tmp_path / "srv" / "photos"])
assert refusal is not None
assert "does not exist" in refusal and "PHOTO_PIPELINE_LIBRARY_ROOTS" in refusal
def test_a_root_that_is_not_a_directory_or_not_readable_is_refused(tmp_path):
a_file = tmp_path / "photos.txt"
a_file.write_text("not a library")
assert "not a directory" in path_policy.roots_refusal([a_file])
unreadable = tmp_path / "unreadable"
unreadable.mkdir()
unreadable.chmod(0o000)
try:
refusal = path_policy.roots_refusal([unreadable])
finally:
unreadable.chmod(0o755)
if os.getuid() != 0: # root ignores the mode, and CI may well be root
assert refusal is not None and "not readable" in refusal
def test_an_unwritable_root_is_not_refused_here(tmp_path):
"""A bind mount's ownership is virtualised on macOS and Windows, so os.access
would refuse a working deployment. The real errno at the first rename is at
least true; this check is about the mount, not the mode."""
read_only = tmp_path / "read-only"
read_only.mkdir()
read_only.chmod(stat.S_IRUSR | stat.S_IXUSR)
try:
assert path_policy.roots_refusal([read_only]) is None
finally:
read_only.chmod(0o755)
def test_in_a_container_a_root_that_was_never_mounted_is_refused(tmp_path, monkeypatch):
"""The container-only failure: the path exists, but it belongs to the image."""
unmounted = tmp_path / "library"
(unmounted / "album").mkdir(parents=True)
refusal = path_policy.roots_refusal([unmounted], require_mount=True)
assert refusal is not None and "not on a mounted filesystem" in refusal
# A bind mount is a mount point, and a root *below* one is mounted too: a
# deployment may mount /srv and configure /srv/photos.
monkeypatch.setattr(os.path, "ismount", lambda path: Path(path) == unmounted.resolve())
assert path_policy.roots_refusal([unmounted], require_mount=True) is None
assert path_policy.roots_refusal([unmounted / "album"], require_mount=True) is None
@pytest.mark.parametrize("role", ["serve", "worker"])
def test_a_root_mismatch_refuses_at_startup_before_any_lock_is_taken(
role, tmp_path, monkeypatch, capsys
):
data = tmp_path / "data"
monkeypatch.setenv("PHOTO_PIPELINE_DATA_DIR", str(data))
monkeypatch.setenv("PHOTO_PIPELINE_LIBRARY_ROOTS", str(tmp_path / "not-mounted"))
monkeypatch.setattr(path_policy, "in_container", lambda: True)
assert main([role]) == 5
assert "does not exist" in capsys.readouterr().err
assert not list(data.glob("*.lock.json")), "nothing started, so nothing is locked"
@pytest.mark.parametrize("role", ["serve", "worker"])
def test_an_unmounted_root_on_a_host_is_not_a_reason_to_refuse(role, tmp_path, monkeypatch):
"""An archive medium that is not plugged in is a Tuesday, not a misconfiguration:
refusing would take the offline half of the library away with it (concept §9)."""
monkeypatch.setenv("PHOTO_PIPELINE_DATA_DIR", str(tmp_path / "data"))
monkeypatch.setenv("PHOTO_PIPELINE_LIBRARY_ROOTS", str(tmp_path / "not-mounted"))
monkeypatch.setenv("PHOTO_PIPELINE_ACCESS_SECRET", "unused-on-loopback")
monkeypatch.setattr(path_policy, "in_container", lambda: False)
# Reaching the lock is the proof: that is the next thing either role does, and
# stopping there keeps the test out of a uvicorn/worker loop.
monkeypatch.setattr("photo_pipeline.__main__._acquire", lambda *_, **__: 99)
assert main([role]) == 99

View File

@@ -0,0 +1,284 @@
"""US08-02: the image's build contract, its entrypoint, and its health check.
Building the image needs a Docker daemon and the network, which is what
``tests/e2e/test_container_runtime.py`` does. Everything that can be checked without
either is checked here, because the parts most likely to rot silently — a pin that
stopped being a pin, a build context that started including the library, a health
check pointed at liveness instead of readiness — are all readable from the files.
The entrypoint and health check are shell, so they are exercised as shell: run with a
stubbed ``id`` and ``python`` on ``PATH``, which is enough to prove the refusal, the
role marker, and the argument pass-through without a container.
"""
from __future__ import annotations
import json
import os
import re
import shutil
import stat
import subprocess
import sys
from pathlib import Path
import pytest
from photo_pipeline.config import Config
from photo_pipeline.services import diagnostics
REPO = Path(__file__).resolve().parents[2]
DOCKERFILE = (REPO / "Dockerfile").read_text()
DOCKERIGNORE = (REPO / ".dockerignore").read_text()
ENTRYPOINT = REPO / "docker" / "entrypoint.sh"
HEALTHCHECK = REPO / "docker" / "healthcheck.sh"
SHA256 = re.compile(r"^[0-9a-f]{64}$")
def instructions(text: str) -> list[str]:
"""The lines that do something: comments explain, they do not build."""
return [line.strip() for line in text.splitlines() if line.strip() and not line.startswith("#")]
def build_args() -> dict[str, str]:
"""Every ``ARG name=default`` in the Dockerfile — the pins, in other words."""
found = {}
for match in re.finditer(r"^ARG\s+([A-Z0-9_]+)=(.+)$", DOCKERFILE, re.MULTILINE):
found[match.group(1)] = match.group(2).strip()
return found
# ── pins ─────────────────────────────────────────────────────────────────────
def test_the_base_image_is_pinned_by_version_and_digest():
base = build_args()["PYTHON_IMAGE"]
assert base.startswith("python:3.12.")
assert "@sha256:" in base, "a tag can be moved; a digest cannot"
assert ":latest" not in DOCKERFILE
# Both stages build from the same pinned base, so the tool that was verified in one
# is the tool that ships in the other.
assert DOCKERFILE.count("FROM ${PYTHON_IMAGE}") == 2
def test_exiftool_and_the_uploader_are_pinned_and_verified():
args = build_args()
assert re.match(r"^\d+\.\d+", args["EXIFTOOL_VERSION"])
assert re.match(r"^\d+\.\d+\.\d+$", args["IMMICH_GO_VERSION"])
for arch in ("AMD64", "ARM64"):
assert SHA256.match(args[f"IMMICH_GO_SHA256_{arch}"]), arch
# The pinned exiftool package is installed by version, not by name alone.
assert 'libimage-exiftool-perl=${EXIFTOOL_VERSION}"' in DOCKERFILE
# And the build fails if what got installed is not what was pinned.
assert "is not the pinned" in DOCKERFILE and "is not pinned" in DOCKERFILE
def test_the_uploader_download_refuses_a_mismatching_checksum(tmp_path):
"""The verification is the point of pinning a URL, so it is run, not read."""
script = REPO / "docker" / "fetch-immich-go.py"
result = subprocess.run(
[
sys.executable,
str(script),
"--version",
"0.0.0-does-not-exist",
"--sha256-amd64",
"0" * 64,
"--sha256-arm64",
"0" * 64,
"--into",
str(tmp_path),
],
capture_output=True,
text=True,
)
assert result.returncode != 0
assert not list(tmp_path.iterdir()), "nothing is written before it is verified"
def test_the_recorded_versions_are_reported_by_diagnostics(tmp_path, monkeypatch):
"""What the image records is what `diagnostics` answers with (acceptance criterion 2)."""
recorded = tmp_path / "versions.json"
recorded.write_text(json.dumps({"exiftool": "13.25", "immich-go": "0.32.0"}))
monkeypatch.setattr(diagnostics, "IMAGE_VERSIONS_FILE", recorded)
monkeypatch.setattr(diagnostics.exiftool, "version", lambda: "13.25")
monkeypatch.setattr(diagnostics, "_uploader_version", lambda _binary: "immich-go 0.32.0")
config = Config(data_dir=tmp_path / "data")
reported = {tool["name"]: tool for tool in diagnostics.tools(config)}
assert reported["exiftool"]["pinned"] == "13.25"
assert reported["immich-go"]["pinned"] == "0.32.0"
assert "0.32.0" in reported["immich-go"]["version"]
assert diagnostics.report(config)["tools"] == list(reported.values())
assert "tool_version_drift" not in {w["code"] for w in diagnostics.report(config)["warnings"]}
def test_a_replaced_tool_is_reported_as_drift(tmp_path, monkeypatch):
recorded = tmp_path / "versions.json"
recorded.write_text(json.dumps({"exiftool": "13.25"}))
monkeypatch.setattr(diagnostics, "IMAGE_VERSIONS_FILE", recorded)
monkeypatch.setattr(diagnostics.exiftool, "version", lambda: "12.57")
report = diagnostics.report(Config(data_dir=tmp_path / "data"))
drift = [w for w in report["warnings"] if w["code"] == "tool_version_drift"]
assert drift and "13.25" in drift[0]["message"] and "12.57" in drift[0]["message"]
def test_versions_are_absent_rather_than_invented_outside_a_container(tmp_path, monkeypatch):
monkeypatch.setattr(diagnostics, "IMAGE_VERSIONS_FILE", tmp_path / "nothing.json")
for tool in diagnostics.tools(Config(data_dir=tmp_path / "data")):
assert tool["pinned"] is None
# ── the final layer ──────────────────────────────────────────────────────────
def test_no_test_or_build_tooling_is_installed_in_the_image():
runtime = "\n".join(instructions(DOCKERFILE.split("AS runtime", 1)[1]))
for unwanted in ("[test]", "pytest", "playwright", "build-essential", "gcc"):
assert unwanted not in runtime, unwanted
assert "pip install --no-cache-dir -e ." in runtime
def test_neither_secrets_nor_library_data_can_enter_the_build_context():
lines = instructions(DOCKERIGNORE)
assert lines[0] == "*", "the context is deny-by-default"
allowed = {line[1:] for line in lines if line.startswith("!")}
# Everything the Dockerfile copies has to be allowed, and nothing else is.
copied = {
source
for match in re.finditer(r"^COPY (?!--from)(.+)$", DOCKERFILE, re.MULTILINE)
for source in match.group(1).split()[:-1]
}
assert {Path(source).parts[0] for source in copied} <= allowed
assert not {"data", ".git", ".env", "tests", ".venv"} & allowed
for generated in ("**/*.env", "**/*.db", "**/*.log", "**/__pycache__"):
assert generated in lines, generated
def test_the_image_runs_as_a_non_root_user_whose_ids_are_build_arguments():
args = build_args()
assert args["UID"] == "1000" and args["GID"] == "1000"
assert "USER ${UID}:${GID}" in DOCKERFILE
assert re.search(r"^USER (root|0)", DOCKERFILE, re.MULTILINE) is None
assert 'useradd --uid "${UID}" --gid "${GID}"' in DOCKERFILE
def test_the_health_check_is_readiness_and_the_default_role_is_serve():
assert "HEALTHCHECK" in DOCKERFILE
assert "/usr/local/bin/healthcheck.sh" in DOCKERFILE
assert 'CMD ["serve"]' in DOCKERFILE
assert 'ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]' in DOCKERFILE
assert "/api/v1/health/ready" in HEALTHCHECK.read_text()
assert "/api/v1/health/live" not in HEALTHCHECK.read_text()
# No supervisor: one role per container (acceptance criterion 4).
for supervisor in ("supervisord", "s6-overlay", "runit"):
assert supervisor not in DOCKERFILE
@pytest.mark.parametrize("script", [ENTRYPOINT, HEALTHCHECK])
def test_the_scripts_are_executable(script):
assert script.stat().st_mode & stat.S_IXUSR, f"{script.name} must be executable in git"
# ── the entrypoint, run as shell ─────────────────────────────────────────────
@pytest.fixture
def stubs(tmp_path):
"""A PATH where ``python`` records its arguments and ``id`` can be told a UID."""
bin_dir = tmp_path / "bin"
bin_dir.mkdir()
recorded = tmp_path / "argv"
python = bin_dir / "python"
python.write_text(f'#!/bin/sh\nprintf "%s\\n" "$@" > {recorded}\nexit 0\n')
python.chmod(0o755)
(bin_dir / "id").write_text('#!/bin/sh\nprintf "%s" "${STUB_UID:-1000}"\n')
(bin_dir / "id").chmod(0o755)
return bin_dir, recorded, tmp_path / "role"
def run_script(script: Path, *args, stubs, env=None):
bin_dir, recorded, role_file = stubs
result = subprocess.run(
["/bin/sh", str(script), *args],
capture_output=True,
text=True,
env={
"PATH": f"{bin_dir}:{os.environ['PATH']}",
"PHOTO_PIPELINE_ROLE_FILE": str(role_file),
**(env or {}),
},
)
argv = recorded.read_text().splitlines() if recorded.exists() else []
return result, argv
def test_the_container_refuses_to_run_as_root(stubs):
result, argv = run_script(ENTRYPOINT, "serve", stubs=stubs, env={"STUB_UID": "0"})
assert result.returncode == 1
assert "refusing to run as root" in result.stderr
assert argv == [], "the application is never started as root"
assert not stubs[2].exists(), "not even the role marker is written"
@pytest.mark.parametrize(
"given,expected",
[
(["serve"], ["-m", "photo_pipeline", "serve"]),
(["worker", "--id", "worker-2"], ["-m", "photo_pipeline", "worker", "--id", "worker-2"]),
# Every other management command stays reachable: operating the container is
# operating the same CLI.
(["diagnostics"], ["-m", "photo_pipeline", "diagnostics"]),
([], ["-m", "photo_pipeline"]),
],
)
def test_the_role_selects_the_command_and_arguments_pass_through(given, expected, stubs):
result, argv = run_script(ENTRYPOINT, *given, stubs=stubs)
assert result.returncode == 0, result.stderr
assert argv == expected
def test_the_role_is_recorded_for_the_health_check(stubs):
run_script(ENTRYPOINT, "worker", stubs=stubs)
assert stubs[2].read_text() == "worker"
def test_the_health_check_only_probes_the_serving_role(stubs):
stubs[2].write_text("worker")
result, argv = run_script(HEALTHCHECK, stubs=stubs)
assert result.returncode == 0 and argv == [], "a worker has no endpoint to probe"
stubs[2].write_text("serve")
result, argv = run_script(HEALTHCHECK, stubs=stubs, env={"PHOTO_PIPELINE_PORT": "9123"})
assert result.returncode == 0, result.stderr
assert argv == ["-", "9123"], "the configured port is the one probed"
def test_the_health_check_fails_while_the_api_is_not_ready(stubs):
"""No python stub: the real interpreter probes a port nothing is listening on."""
stubs[2].write_text("serve")
result = subprocess.run(
["/bin/sh", str(HEALTHCHECK)],
capture_output=True,
text=True,
env={
"PATH": os.path.dirname(sys.executable) + os.pathsep + os.environ["PATH"],
"PHOTO_PIPELINE_ROLE_FILE": str(stubs[2]),
"PHOTO_PIPELINE_PORT": "1",
},
)
assert result.returncode == 1
assert "not ready" in result.stderr
def test_the_scripts_are_posix_shell():
"""They run in the image's /bin/sh, which is dash — not bash."""
shells = ["/bin/sh"] + ([dash] if (dash := shutil.which("dash")) else [])
for shell in shells:
for script in (ENTRYPOINT, HEALTHCHECK):
checked = subprocess.run([shell, "-n", str(script)], capture_output=True, text=True)
assert checked.returncode == 0, f"{shell} {script.name}: {checked.stderr}"

View File

@@ -0,0 +1,253 @@
"""US08-01: the configurable trust boundary and its authentication gate.
Until now, reaching the app proved ownership of it: it answered only to loopback
names. A container behind a reverse proxy answers to a real hostname, so these tests
pin the two halves that replace that proof — the app refuses to start exposed without
an access secret, and the secret is the only way to obtain the session every other
route already required (US07-02, unchanged and re-asserted here).
The suite's ``conftest`` bootstraps a session for any ``TestClient`` automatically,
which is precisely what an unauthenticated caller does not get; ``raw_client``
pre-seeds a placeholder CSRF header to opt out of that convenience.
"""
from __future__ import annotations
import pytest
from starlette.testclient import TestClient
from photo_pipeline.api.app import ConfigurationRefused, create_app
from photo_pipeline.api.security import ACCESS_SECRET_HEADER, CSRF_HEADER, SESSION_COOKIE
from photo_pipeline.config import Config
SECRET = "operator-secret-value"
HOSTNAME = "photos.example.com"
# What Starlette reports as the peer address of an in-process request.
TESTCLIENT_ADDRESS = "testclient"
# One of each route class: a read, a mutation, and a media endpoint.
PROTECTED = [
("GET", "/api/v1/workflow", None),
("POST", "/api/v1/albums/proposals", {}),
("GET", "/api/v1/assets/unknown-asset/thumbnail?size=256", None),
]
def config(tmp_path, **overrides) -> Config:
return Config(data_dir=tmp_path / "data", **overrides)
def raw_client(app, base_url="http://127.0.0.1") -> TestClient:
client = TestClient(app, base_url=base_url)
client.headers[CSRF_HEADER] = "placeholder"
return client
def exchange(client, secret=SECRET, headers=None):
return client.get("/api/v1/session", headers={ACCESS_SECRET_HEADER: secret, **(headers or {})})
# ── startup: exposure without a secret is refused, loopback is unchanged ──────
@pytest.mark.parametrize(
"exposure,exposed",
[({"allowed_hosts": (HOSTNAME,)}, HOSTNAME), ({"host": "0.0.0.0"}, "0.0.0.0")],
)
def test_an_exposed_configuration_refuses_to_serve_without_a_secret(tmp_path, exposure, exposed):
with pytest.raises(ConfigurationRefused) as refused:
create_app(config(tmp_path, **exposure))
assert "PHOTO_PIPELINE_ACCESS_SECRET" in str(refused.value)
# The message names what is exposed, so the operator knows which setting did it.
assert exposed in str(refused.value)
def test_the_serve_command_reports_the_refusal_instead_of_binding(tmp_path, monkeypatch, capsys):
"""Exit before the port, the lock, and the database, with a sentence not a trace."""
from photo_pipeline.__main__ import main
monkeypatch.setenv("PHOTO_PIPELINE_DATA_DIR", str(tmp_path / "data"))
monkeypatch.setenv("PHOTO_PIPELINE_ALLOWED_HOSTS", HOSTNAME)
monkeypatch.delenv("PHOTO_PIPELINE_ACCESS_SECRET", raising=False)
assert main(["serve"]) == 4
assert "PHOTO_PIPELINE_ACCESS_SECRET" in capsys.readouterr().err
def test_an_exposed_configuration_with_a_secret_starts(tmp_path):
app = create_app(config(tmp_path, allowed_hosts=(HOSTNAME,), access_secret=SECRET))
with raw_client(app, base_url=f"http://{HOSTNAME}") as client:
assert exchange(client).status_code == 200
def test_a_loopback_configuration_still_needs_no_secret(tmp_path):
"""An unset trust boundary must behave exactly as it did before this story."""
with raw_client(create_app(config(tmp_path))) as client:
response = client.get("/api/v1/session")
assert response.status_code == 200
assert response.json()["csrf_token"]
assert "secure" not in response.headers["set-cookie"].lower()
# ── the exchange: secret in, session out ─────────────────────────────────────
@pytest.fixture
def gated(tmp_path):
app = create_app(
config(
tmp_path,
allowed_hosts=(HOSTNAME,),
access_secret=SECRET,
trusted_proxies=(TESTCLIENT_ADDRESS,),
)
)
with raw_client(app, base_url=f"http://{HOSTNAME}") as client:
yield client
def test_the_secret_buys_the_session_and_the_session_buys_the_routes(gated):
response = exchange(gated)
assert response.status_code == 200
cookie = response.headers["set-cookie"].lower()
assert "httponly" in cookie and "samesite=strict" in cookie
gated.headers[CSRF_HEADER] = response.json()["csrf_token"]
# The session and CSRF requirements behind the gate are the ones US07-02 set.
assert gated.get("/api/v1/workflow").status_code == 200
assert gated.post("/api/v1/albums/proposals", json={}).status_code == 200
refused = gated.post("/api/v1/albums/proposals", json={}, headers={CSRF_HEADER: "guessed"})
assert refused.json()["error"]["code"] == "csrf_failed"
@pytest.mark.parametrize("offered", ["", "wrong-secret", SECRET + "x", SECRET.upper()])
def test_a_wrong_secret_buys_nothing(gated, offered):
response = exchange(gated, secret=offered)
assert response.status_code == 401
assert response.json()["error"]["code"] == "access_denied"
assert "set-cookie" not in response.headers
def test_a_refusal_never_echoes_the_secret_or_the_session(gated, caplog):
with caplog.at_level("WARNING"):
response = exchange(gated, secret="wrong-secret")
assert SECRET not in response.text and "wrong-secret" not in response.text
assert SECRET not in caplog.text
# Logged as an event with its caller, without the session it did not get.
assert "access secret rejected" in caplog.text
def test_guessing_is_rate_limited(gated):
codes = [exchange(gated, secret=f"guess-{n}").status_code for n in range(6)]
assert codes.count(401) == 5 and codes[-1] == 429
assert gated.get("/api/v1/session").status_code == 429
# The right secret is refused too while the limiter holds: that is the point.
blocked = exchange(gated)
assert blocked.status_code == 429
assert SECRET not in blocked.text
def test_every_route_class_is_unreachable_without_the_secret(gated):
for method, path, body in PROTECTED:
response = gated.request(method, path, json=body)
assert response.status_code == 401, path
assert response.json()["error"]["code"] == "unauthenticated", path
# Health stays open: an orchestrator restarting the container holds no secret.
assert gated.get("/api/v1/health/live").status_code == 200
assert gated.get("/api/v1/health/ready").status_code == 200
def test_a_session_from_another_process_is_not_replayable(tmp_path):
"""Sessions live in the process, so a cookie captured from a previous one — a
restarted container, or a second deployment — must not open this one."""
settings = dict(allowed_hosts=(HOSTNAME,), access_secret=SECRET)
first, second = (create_app(config(tmp_path / str(n), **settings)) for n in (1, 2))
with raw_client(first, base_url=f"http://{HOSTNAME}") as client:
exchange(client)
stolen = client.cookies[SESSION_COOKIE]
with raw_client(second, base_url=f"http://{HOSTNAME}") as client:
client.cookies.set(SESSION_COOKIE, stolen, domain=HOSTNAME)
response = client.get("/api/v1/workflow")
assert response.status_code == 401
assert response.json()["error"]["code"] == "unauthenticated"
def test_a_cross_site_request_is_still_refused_behind_the_gate(gated):
gated.headers[CSRF_HEADER] = exchange(gated).json()["csrf_token"]
refused = gated.post(
"/api/v1/albums/proposals", json={}, headers={"Origin": "https://evil.example"}
)
assert refused.json()["error"]["code"] == "origin_not_allowed"
embedded = gated.get(
"/api/v1/assets/unknown-asset/thumbnail?size=256", headers={"Sec-Fetch-Site": "cross-site"}
)
assert embedded.json()["error"]["code"] == "cross_site_blocked"
def test_an_unconfigured_host_is_refused_even_with_a_valid_session(gated):
gated.headers[CSRF_HEADER] = exchange(gated).json()["csrf_token"]
for host in ("other.example.com", "192.168.1.10"):
response = gated.get("/api/v1/workflow", headers={"Host": host})
assert response.status_code == 403, host
assert response.json()["error"]["code"] == "host_not_allowed", host
# ── forwarded headers: believed from the proxy, ignored from anyone else ──────
def test_a_trusted_proxys_https_makes_the_cookie_secure(gated):
"""The proxy speaks HTTPS outward and HTTP to this app, so only the header knows."""
assert "secure" in exchange(gated, headers={"X-Forwarded-Proto": "https"}).headers[
"set-cookie"
].lower()
assert "secure" not in exchange(gated).headers["set-cookie"].lower()
def test_the_external_scheme_is_part_of_the_accepted_origin(gated):
gated.headers[CSRF_HEADER] = exchange(gated).json()["csrf_token"]
allowed = gated.post(
"/api/v1/albums/proposals",
json={},
headers={"X-Forwarded-Proto": "https", "Origin": f"https://{HOSTNAME}"},
)
assert allowed.status_code == 200
# The scheme is part of the origin: the same name over plain HTTP is not it.
refused = gated.post(
"/api/v1/albums/proposals",
json={},
headers={"X-Forwarded-Proto": "https", "Origin": f"http://{HOSTNAME}"},
)
assert refused.json()["error"]["code"] == "origin_not_allowed"
def test_a_trusted_proxys_forwarded_host_is_the_host_that_is_judged(tmp_path):
"""The proxy terminates the operator's hostname and dials this app by address."""
app = create_app(
config(
tmp_path,
allowed_hosts=(HOSTNAME,),
access_secret=SECRET,
trusted_proxies=(TESTCLIENT_ADDRESS,),
)
)
with raw_client(app, base_url="http://10.0.0.5") as client:
forwarded = {"X-Forwarded-Host": HOSTNAME}
assert exchange(client, headers=forwarded).status_code == 200
# Without the header the address it was dialled by is not an allowed name.
assert exchange(client).json()["error"]["code"] == "host_not_allowed"
def test_forwarded_headers_from_an_untrusted_client_are_ignored(tmp_path):
"""Otherwise any caller could declare the hostname and scheme of its choosing."""
app = create_app(config(tmp_path, allowed_hosts=(HOSTNAME,), access_secret=SECRET))
with raw_client(app, base_url="http://evil.example") as client:
forged = exchange(client, headers={"X-Forwarded-Host": HOSTNAME})
assert forged.json()["error"]["code"] == "host_not_allowed"
with raw_client(app, base_url=f"http://{HOSTNAME}") as client:
# A forged scheme would flip the cookie's Secure flag on a plain connection,
# which is how a cookie gets set and then never sent again.
response = exchange(client, headers={"X-Forwarded-Proto": "https"})
assert response.status_code == 200
assert "secure" not in response.headers["set-cookie"].lower()

View File

@@ -174,12 +174,21 @@
"US07-07": [
"tests/e2e/test_release_gate.py",
"tests/e2e/test_release_journey.py"
],
"US08-01": [
"tests/unit/test_security_policy.py",
"tests/integration/test_trusted_hosts.py"
],
"US08-02": [
"tests/integration/test_container_image.py",
"tests/e2e/test_container_runtime.py"
],
"US08-03": [
"tests/integration/test_compose_runtime.py",
"tests/e2e/test_compose_stack.py"
]
},
"planned": [
"US08-01",
"US08-02",
"US08-03",
"US08-04",
"US08-05"
],

View File

@@ -13,18 +13,31 @@ import pytest
from photo_pipeline.api.security import (
CSRF_HEADER,
LOOPBACK_HOSTS,
PUBLIC_PATHS,
FailureLimiter,
Session,
evaluate,
exposed_hosts,
external_view,
split_host,
trust_refusal,
)
from photo_pipeline.config import Config
SESSION = Session(id="session-id", csrf_token="csrf-token")
HOST = "127.0.0.1:8000"
LIMIT = 1024
HOSTNAME = "photos.example.com"
def check(method="GET", path="/api/v1/workflow", **headers):
def check(
method="GET",
path="/api/v1/workflow",
allowed_hosts=LOOPBACK_HOSTS,
scheme="http",
**headers,
):
"""Evaluate a request that is authenticated and same-origin unless overridden."""
sent = {
"host": HOST,
@@ -38,6 +51,8 @@ def check(method="GET", path="/api/v1/workflow", **headers):
path=path,
headers=sent,
session=SESSION,
allowed_hosts=allowed_hosts,
scheme=scheme,
max_request_bytes=LIMIT,
)
@@ -155,6 +170,115 @@ def test_refusals_name_no_path_secret_or_internal():
assert "/" not in refusal.message
# ── US08-01: the same table with a configured trust boundary ─────────────────
CONFIGURED = frozenset(LOOPBACK_HOSTS | {HOSTNAME})
def test_a_configured_host_is_accepted_and_its_neighbours_are_not():
assert check(host=HOSTNAME, allowed_hosts=CONFIGURED) is None
for host in ("other.example.com", f"evil-{HOSTNAME}", "192.168.1.10"):
refusal = check(host=host, allowed_hosts=CONFIGURED)
assert (refusal.status, refusal.code) == (403, "host_not_allowed"), host
def test_the_loopback_default_refuses_a_host_nobody_configured():
"""The default set is what the app enforced before there was a setting."""
refusal = check(host=HOSTNAME)
assert (refusal.status, refusal.code) == (403, "host_not_allowed")
def test_the_origin_must_match_the_external_scheme():
for scheme in ("http", "https"):
assert (
check(
method="POST",
host=HOSTNAME,
origin=f"{scheme}://{HOSTNAME}",
allowed_hosts=CONFIGURED,
scheme=scheme,
)
is None
)
# An HTTPS deployment whose caller claims plain HTTP is a different origin.
refusal = check(
method="POST",
host=HOSTNAME,
origin=f"http://{HOSTNAME}",
allowed_hosts=CONFIGURED,
scheme="https",
)
assert (refusal.status, refusal.code) == (403, "origin_not_allowed")
def view(client, *, trusted=(), **headers):
sent = {name.replace("_", "-"): value for name, value in headers.items()}
return external_view(
client=client,
headers={"host": HOST, **sent},
scheme="http",
trusted_proxies=frozenset(trusted),
)
def test_forwarded_headers_are_ignored_without_a_trusted_proxy():
forged = {"x_forwarded_proto": "https", "x_forwarded_host": HOSTNAME}
assert view("10.0.0.9", **forged) == ("http", HOST)
assert view(None, **forged) == ("http", HOST)
# Configuring *a* proxy does not trust a caller that is not it.
assert view("10.0.0.9", trusted=("10.0.0.1",), **forged) == ("http", HOST)
def test_a_trusted_proxy_defines_the_external_scheme_and_host():
assert view(
"10.0.0.1", trusted=("10.0.0.1",), x_forwarded_proto="https", x_forwarded_host=HOSTNAME
) == ("https", HOSTNAME)
# A chain: the first entry is what the original client asked for.
assert view(
"10.0.0.1",
trusted=("10.0.0.1",),
x_forwarded_proto="https, http",
x_forwarded_host=f"{HOSTNAME}, inner.internal",
) == ("https", HOSTNAME)
# Trusted but silent: this hop's own view stands.
assert view("10.0.0.1", trusted=("10.0.0.1",)) == ("http", HOST)
@pytest.mark.parametrize(
"settings,exposed",
[
({}, []),
({"host": "127.0.0.1"}, []),
({"allowed_hosts": ("localhost", "127.0.0.1")}, []),
({"allowed_hosts": (f"{HOSTNAME}:8443",)}, [HOSTNAME]),
({"host": "0.0.0.0", "allowed_hosts": (HOSTNAME,)}, ["0.0.0.0", HOSTNAME]),
],
)
def test_exposed_hosts_names_only_what_another_machine_can_reach(settings, exposed):
assert exposed_hosts(Config(**settings)) == exposed
def test_an_exposed_configuration_without_a_secret_must_not_serve():
refusal = trust_refusal(Config(allowed_hosts=(HOSTNAME,)))
assert HOSTNAME in refusal and "PHOTO_PIPELINE_ACCESS_SECRET" in refusal
assert trust_refusal(Config(allowed_hosts=(HOSTNAME,), access_secret="s")) is None
# Loopback-only, with and without a secret, is unchanged.
assert trust_refusal(Config()) is None
assert trust_refusal(Config(access_secret="s")) is None
def test_failed_attempts_are_bounded_per_window():
limiter = FailureLimiter(limit=2, window=60.0)
assert not limiter.blocked()
limiter.record_failure()
assert not limiter.blocked()
limiter.record_failure()
assert limiter.blocked()
# Attempts age out, so a locked-out operator is not locked out forever.
limiter._failures = [-120.0, -120.0]
assert not limiter.blocked()
@pytest.mark.parametrize(
"value,expected",
[