Compare commits

..

4 Commits

Author SHA1 Message Date
6282123780 US08-05: Automate Container Deployment Acceptance
Some checks failed
Test / suites (pull_request) Failing after 2m38s
Test / container (pull_request) Has been skipped
2026-08-21 10:49:05 +02:00
a19dd280c9 US08-04: Publish and Deploy from Gitea Actions (#99)
Some checks failed
Test / suites (push) Failing after 2m29s
2026-08-19 20:37:32 +02:00
1632544132 US08-03: Compose the Runtime and Mount the Library Safely (#98) 2026-08-19 19:47:42 +02:00
888d859e93 US08-02: Build a Reproducible Application Image (#97) 2026-08-18 23:11:15 +02:00
17 changed files with 2357 additions and 39 deletions

85
.env.example Normal file
View File

@@ -0,0 +1,85 @@
# Every setting the application and its composition read, with no values in it.
# Copy to `.env`, fill in what you need, and keep that copy out of git (it is
# gitignored, and the work-item safety checks refuse to stage it).
#
# cp .env.example .env
#
# Empty means "use the default noted beside it". Anything already exported in the
# shell wins over this file, both for the app and for `docker compose`.
# ── the composition (host side; read by docker-compose.yml only) ─────────────
# The photo library on this host. Bind-mounted at PHOTO_PIPELINE_LIBRARY_ROOTS.
PHOTO_PIPELINE_LIBRARY_HOST_PATH=
# Which image to run. Default: photo-pipeline:dev (what `up --build` builds).
PHOTO_PIPELINE_IMAGE=
# Must be the owner of the library above: what the containers rename and rewrite
# keeps this ownership. Default: 1000 / 1000.
PHOTO_PIPELINE_UID=
PHOTO_PIPELINE_GID=
# Host address the API port is published on. Default: 127.0.0.1. Anything else
# exposes the app beyond this machine — then ALLOWED_HOSTS and ACCESS_SECRET below
# are what stand in for the loopback boundary (US08-01).
PHOTO_PIPELINE_PUBLISH_ADDRESS=
# This file's own path, if it is not ./.env. Default: .env.
PHOTO_PIPELINE_ENV_FILE=
# ── library and data ────────────────────────────────────────────────────────
# os.pathsep-separated. In a container these are the *container-side* mount paths,
# and `serve`/`worker` refuse to start when they are not mounted.
PHOTO_PIPELINE_LIBRARY_ROOTS=
# Database, WAL, thumbnail cache, journals, backups. Default: data (the container
# sets /data, which is the persistent volume; a network mount is unsupported).
PHOTO_PIPELINE_DATA_DIR=
# Database file, if it should not live in the data directory. Default:
# <data dir>/photo_pipeline.db.
PHOTO_PIPELINE_DB_PATH=
# ── serving ─────────────────────────────────────────────────────────────────
# Bind address. Default: 127.0.0.1. The composition sets 0.0.0.0 inside the
# container and publishes to loopback on the host instead.
PHOTO_PIPELINE_HOST=
# Under the composition this is the *published* host port; the container serves
# 8000. Default: 8000.
PHOTO_PIPELINE_PORT=
# Comma-separated hostnames the app answers to besides loopback. Empty means
# loopback only. Naming one makes the access secret mandatory.
PHOTO_PIPELINE_ALLOWED_HOSTS=
# Traded for the session cookie at GET /api/v1/session via X-Access-Secret.
# Required as soon as the app is reachable from anywhere but loopback. Generate
# one with: python -c 'import secrets; print(secrets.token_urlsafe(32))'
PHOTO_PIPELINE_ACCESS_SECRET=
# Comma-separated peer addresses whose X-Forwarded-Proto/-Host may be believed.
# Only the reverse proxy's address belongs here. Default: none.
PHOTO_PIPELINE_TRUSTED_PROXIES=
# Largest request body accepted, in bytes. Default: 1048576.
PHOTO_PIPELINE_MAX_REQUEST_BYTES=
# ── logging ─────────────────────────────────────────────────────────────────
# Default: INFO.
PHOTO_PIPELINE_LOG_LEVEL=
# json or text. Default: json.
PHOTO_PIPELINE_LOG_FORMAT=
# ── limits ──────────────────────────────────────────────────────────────────
# Thumbnail cache quota in bytes. Default: 500000000.
PHOTO_PIPELINE_THUMBNAIL_CACHE_QUOTA_BYTES=
# Refuse to decode images larger than this many pixels. Default: 100000000.
PHOTO_PIPELINE_THUMBNAIL_MAX_PIXELS=
# Free space an archive destination must keep beyond the transfer. Default:
# 1000000000.
PHOTO_PIPELINE_ARCHIVE_FREE_SPACE_RESERVE_BYTES=
# ── safety gate ─────────────────────────────────────────────────────────────
# true refuses every mutating request until a read-only dry run of this library has
# been produced and approved (US07-07). Default: false. Turn it on before pointing
# the app at photos that cannot be replaced.
PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL=
# ── external services ───────────────────────────────────────────────────────
# Vision provider key for the analysis stage. Without it, analysis cannot run.
PHOTO_PIPELINE_VISION_API_KEY=
# Immich server and its API key, for the upload stage.
PHOTO_PIPELINE_IMMICH_SERVER_URL=
PHOTO_PIPELINE_IMMICH_API_KEY=
# Uploader binary. Default: immich-go (on PATH; pinned inside the image).
PHOTO_PIPELINE_IMMICH_GO_BINARY=

View File

@@ -0,0 +1,99 @@
# Publish and redeploy (US08-04). Adapted from the crowdsec-admin deploy workflow:
# build, log in to the Gitea registry, push, trigger the Portainer webhook, prune.
# The difference is the gate — this project has a required suite that must not be
# skipped, so publishing happens only after `Test` (`.gitea/workflows/test.yml`)
# succeeded on `main`, never on the push itself.
#
# The stack is managed by Portainer from git (`docker-compose.yml`), and the runtime
# secrets it needs — vision key, Immich key, access secret — live in the Portainer
# stack's environment. They are deliberately not repository secrets and are not in the
# image: rotation stays in one place, and a repository read never discloses them.
#
# Repository secrets required:
# REGISTRY_USER user with write:package on the registry
# REGISTRY_TOKEN that user's token
# PORTAINER_WEBHOOK_URL POST URL from the stack's auto-update setting
name: Deploy
on:
workflow_run:
workflows:
- Test
types:
- completed
branches:
- main
workflow_dispatch:
inputs:
dry_run:
description: Build and push a scratch tag only — leave `latest` and the running stack alone
type: boolean
default: true
concurrency:
# Deliberately not keyed by commit: the point is that two deploys of *different*
# commits cannot overlap. Queued, not cancelled — a half-pushed tag set is worse
# than a late one.
group: deploy
cancel-in-progress: false
env:
IMAGE: git.domverse-berlin.eu/domverse/photoanalyzer
# The commit that was tested, not whatever `main` points at by the time this starts.
SHA: ${{ gitea.event.workflow_run.head_sha || gitea.sha }}
jobs:
publish:
# A completed `Test` run is not a passing one.
if: >-
(gitea.event_name == 'workflow_run' && gitea.event.workflow_run.conclusion == 'success')
|| (gitea.event_name == 'workflow_dispatch' && inputs.dry_run == false)
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
ref: ${{ env.SHA }}
- name: Log in to the Gitea registry
run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login git.domverse-berlin.eu -u "${{ secrets.REGISTRY_USER }}" --password-stdin
- name: Build and push
# The commit tag is pushed first, so a `latest` that exists is always a tag
# that also exists under its own commit — which is what makes a rollback a
# tag change rather than a rebuild.
run: |
docker build -t "$IMAGE:$SHA" -t "$IMAGE:latest" .
docker push "$IMAGE:$SHA"
docker push "$IMAGE:latest"
- name: Trigger the Portainer redeploy
# --fail turns an HTTP error into a non-zero exit: a redeploy that did not
# happen must not read as a green deploy.
run: curl -sS --fail -X POST "${{ secrets.PORTAINER_WEBHOOK_URL }}"
- name: Prune dangling images
# Untagged layers only. Published tags are the rollback history; `-a` would
# delete exactly the images this workflow exists to keep.
run: docker image prune -f
dry-run:
# Manual only, and the default: prove the image still builds and the registry
# still accepts it without moving `latest` or touching the running stack.
if: gitea.event_name == 'workflow_dispatch' && inputs.dry_run
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Log in to the Gitea registry
run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login git.domverse-berlin.eu -u "${{ secrets.REGISTRY_USER }}" --password-stdin
- name: Build and push a scratch tag
run: |
docker build -t "$IMAGE:scratch-$SHA" .
docker push "$IMAGE:scratch-$SHA"
- name: Prune dangling images
run: docker image prune -f

103
.gitea/workflows/test.yml Normal file
View File

@@ -0,0 +1,103 @@
# The test gate. Deploy waits for this workflow by name (`.gitea/workflows/deploy.yml`
# triggers on `workflow_run: [Test]`), so renaming it here without renaming it there
# would leave `main` publishing without a suite. `tests/integration/test_deploy_workflows.py`
# asserts both halves of that link, and that the commands below are still the ones
# configured in `work_item/.work-item.yml`.
name: Test
on:
pull_request:
push:
branches:
- main
concurrency:
# One run per branch; a newer push makes the older run's answer irrelevant.
group: test-${{ gitea.ref }}
cancel-in-progress: true
jobs:
suites:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install exiftool
# The EXIF checkpoints are the safety invariant of every metadata stage; a run
# without exiftool would skip them for a reason the release gate accepts.
run: |
SUDO=
if [ "$(id -u)" -ne 0 ]; then SUDO=sudo; fi
$SUDO apt-get update
$SUDO apt-get install -y --no-install-recommends libimage-exiftool-perl
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install the application and its test dependencies
run: |
python -m pip install --upgrade pip
python -m pip install -e '.[test]'
python -m playwright install --with-deps chromium
- name: Helper suite
run: work_item/scripts/python -m unittest discover -s work_item/tests -v
- name: Application suite
run: work_item/scripts/python -m pytest tests -q
# The deployed container, verified the way the host application is (US08-05). It
# runs on `main` only — a pull request has nothing published to upgrade *from*, and
# building two images per push would pay for that on every commit. Deploy waits for
# this whole workflow, so a red container gate is a deploy that does not happen.
container:
if: gitea.event_name == 'push'
runs-on: ubuntu-latest
steps:
- name: Checkout
# The upgrade journey builds the previous commit's tree when no published
# image is named, so the history has to be there.
uses: actions/checkout@v4
with:
fetch-depth: 2
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install the application and its test dependencies
run: |
python -m pip install --upgrade pip
python -m pip install -e '.[test]'
python -m playwright install --with-deps chromium
- name: Container acceptance gate
# One command: it builds the image, provisions the composition against a
# temporary fixture library and an isolated volume, runs the phase_h journeys,
# destroys the stack, and writes the evidence. Any skipped check fails it.
env:
# The fixture libraries are bind-mounted into the containers, so this path
# has to be one the Docker daemon can see. On a runner that talks to a
# sibling daemon, point it at a shared host path instead of the workspace —
# an unshared path arrives as an empty mount and the journeys fail on the
# scan, which is the symptom to recognise.
PHOTO_PIPELINE_DATA_DIR: ${{ gitea.workspace }}/gate-data
PHOTO_PIPELINE_LIBRARY_ROOTS: ${{ gitea.workspace }}/gate-library
PHOTO_PIPELINE_TEST_MOUNT_BASE: ${{ gitea.workspace }}/gate-mounts
run: |
mkdir -p "$PHOTO_PIPELINE_LIBRARY_ROOTS" "$PHOTO_PIPELINE_TEST_MOUNT_BASE"
python -m photo_pipeline container-gate --output gate-evidence
- name: Keep the evidence
# Retained per run, and retained on failure especially: the logs are the only
# account of what the containers did.
if: always()
uses: actions/upload-artifact@v3
with:
name: container-gate-${{ gitea.sha }}
path: gate-evidence

133
README.md
View File

@@ -143,8 +143,137 @@ thumbnail cache. Binding to `0.0.0.0` makes the access secret mandatory
without it. The declared `HEALTHCHECK` polls `/api/v1/health/ready`, so a container
whose database is unmigrated or misconfigured is never reported healthy.
Composing the runtime and mounting the library is US08-03; publishing and deploying
the image is US08-04.
## Composed runtime (US08-03)
`docker-compose.yml` is the deployment: one `serve` container, one `worker`
container, one bind-mounted library, one data volume, and a one-shot `migrate` that
both roles wait for.
```bash
cp .env.example .env && $EDITOR .env # nothing has a value in it; fill in yours
docker compose up -d --build
```
`.env.example` lists every `PHOTO_PIPELINE_*` variable with its default in a comment
and no values at all. The three the composition cannot start without are
`PHOTO_PIPELINE_LIBRARY_HOST_PATH` (the library on this host),
`PHOTO_PIPELINE_LIBRARY_ROOTS` (where it is mounted *inside* the container), and
`PHOTO_PIPELINE_ACCESS_SECRET` — publishing the port means the app is reachable from
outside the container, so US08-01 makes the secret mandatory. Set
`PHOTO_PIPELINE_UID`/`GID` to the owner of the library: what the containers rename
and rewrite keeps that ownership.
| invariant | how the composition keeps it |
|---|---|
| one writer | `serve` and `worker` take their role's library lock (US07-05) in the shared `/data` volume, so `--scale worker=2` is refused by the lock, not by convention |
| migrations first | `migrate` runs the backup-then-migrate path and must exit 0 before `api` and `worker` start; a failed upgrade leaves the previous database and its pre-migration backup intact |
| container paths | `PHOTO_PIPELINE_LIBRARY_ROOTS` is both the mount target and the configured root; a root that is not mounted makes `serve`/`worker` exit 5 at startup instead of writing into the container's throwaway layer |
| loopback by default | the port is published to `127.0.0.1` unless `PHOTO_PIPELINE_PUBLISH_ADDRESS` says otherwise, and exposing it needs the hostname in `PHOTO_PIPELINE_ALLOWED_HOSTS` plus the access secret |
| restart safety | both roles are `restart: unless-stopped` with a 30 s stop grace period, and a job interrupted by a restart resumes exactly as it does on a host restart |
The `data` volume holds the database, its write-ahead log, the thumbnail cache,
journals, and backups. **It must stay on a local filesystem** — SQLite in WAL mode
needs real local locking, so NFS, SMB, and network volume drivers are unsupported
there and corrupt the database rather than slow it down. The library bind mount has
no such restriction.
Operating the deployment is operating the same CLI:
```bash
docker compose run --rm --no-deps api diagnostics
docker compose run --rm --no-deps api backup --reason pre-upgrade
docker compose run --rm --no-deps api verify-backup /data/backups/<name>
docker compose run --rm --no-deps api restore /data/backups/<name> --into /data/restored
docker compose logs -f worker
```
`--no-deps` keeps a one-off command from starting a second stack; `api` is only the
service the command borrows the image and mounts from. `restore` is deliberately not
an API call: it replaces the state of an installation and belongs to a stopped one,
so stop `api` and `worker` first and restart them against the restored directory.
## Publishing and deploying (US08-04)
Two workflows in `.gitea/workflows/` make `main` the only path to the running stack:
| workflow | runs on | does |
|---|---|---|
| `test.yml` (**Test**) | every pull request, and every push to `main` | installs exiftool, the package, and Chromium, then runs the suites configured in `work_item/.work-item.yml` |
| `deploy.yml` (**Deploy**) | a **successful** `Test` run on `main`, or manual dispatch | builds the image, pushes it, triggers the Portainer webhook, prunes dangling layers |
Deploy waits for `Test` through `workflow_run`, so publishing is downstream of a green
suite rather than of a push. `completed` is not `success`: the publish job runs only on
`conclusion == 'success'`. Because an unmatched `workflow_run` filter does not fail but
simply never fires, `tests/integration/test_deploy_workflows.py` asserts the link from
both ends — renaming either workflow, or dropping one of the configured suites out of
the test job, fails that test rather than silently unhooking the gate.
The image is `git.domverse-berlin.eu/domverse/photoanalyzer`, tagged with the commit
SHA and `latest`. The commit tag is pushed first, so a `latest` that exists always has
a commit tag beside it and **a rollback is a tag change, not a rebuild**:
```bash
docker pull git.domverse-berlin.eu/domverse/photoanalyzer:<known-good-sha>
docker tag git.domverse-berlin.eu/domverse/photoanalyzer:<known-good-sha> \
git.domverse-berlin.eu/domverse/photoanalyzer:latest
docker push git.domverse-berlin.eu/domverse/photoanalyzer:latest
# then trigger the Portainer webhook, or redeploy the stack from Portainer
```
Manual dispatch defaults to **dry run**: it builds and pushes `scratch-<sha>` only, and
touches neither `latest` nor the running stack. Clear the `dry_run` input to publish by
hand. A workflow-level `concurrency: deploy` group with `cancel-in-progress: false`
queues deploys instead of overlapping them, so two commits can never race to `latest`.
### Secrets
Repository secrets — used only by the pipeline:
| secret | purpose |
|---|---|
| `REGISTRY_USER` | registry user with `write:package` |
| `REGISTRY_TOKEN` | that user's token, passed on stdin, never as an argument |
| `PORTAINER_WEBHOOK_URL` | the stack's auto-update POST URL; `curl --fail` makes a refused redeploy a failed workflow |
Runtime secrets — `PHOTO_PIPELINE_VISION_API_KEY`, `PHOTO_PIPELINE_IMMICH_API_KEY`,
`PHOTO_PIPELINE_ACCESS_SECRET` — are **not** repository secrets and are not in the
image. The stack is managed by Portainer from git (`docker-compose.yml`), and those
values live in the Portainer stack's environment, so rotation is one place and a
repository read discloses nothing. A test asserts the workflows never name them.
## Container acceptance gate (US08-05)
The deployed container is verified the way the host application is, by one command:
```bash
work_item/scripts/python -m photo_pipeline container-gate --output gate-evidence
```
It builds the image, provisions the composition against a **temporary fixture library
on a bind mount and an isolated data volume**, runs the `phase_h` journeys against it,
destroys every stack afterwards, and writes `release-report.json`, `logs/container.log`,
and `CHECKSUMS.sha256` into the evidence directory. It exits non-zero when a journey
fails, when the story matrix has a hole, **or when any check skipped at all** — unlike
the release gate, this one accepts no environment excuse: a run that never reached the
containers proved nothing about them.
The journeys (`tests/e2e/test_phase_h_container.py`) are:
| journey | what it proves |
|---|---|
| browser | discovery, duplicate review, analysis, album proposal, rename, upload preflight, and archive views, driven through the containerized frontend — including a rename that really moves the operator's folder on the bind mount |
| upgrade | the previous version's image runs first, then this one against the same volume: schema at the new head, assets, analysis results, job progress, the unapplied rename plan, and the thumbnail cache all survive |
| restart | `docker kill` on both containers mid-job; the job resumes, every photo ends with exactly one stored result, and only the in-flight item ever reaches the provider twice |
| security | no session refused, a forged `X-Forwarded-Host` cannot smuggle an allowed hostname past the check, a symlink out of the mounted library is refused, an unmounted library root refuses startup, and no secret appears in `docker compose logs` |
The upgrade journey builds the previous commit's tree when no published image is named;
point it at the real one with `PHOTO_PIPELINE_PREVIOUS_IMAGE`. On a Docker VM (Colima,
Docker Desktop) the fixture library must live on a shared path — it defaults to
`~/.cache/photo-pipeline`, overridable with `PHOTO_PIPELINE_TEST_MOUNT_BASE`.
CI runs this gate as the `container` job of **Test** on pushes to `main` and keeps its
evidence as a run artefact. Deploy waits for the whole `Test` workflow, so a red
container gate is a publish that does not happen.
## Testing

105
docker-compose.yml Normal file
View File

@@ -0,0 +1,105 @@
# The deployed runtime (US08-03): one API, one worker, one library, one volume.
#
# The same image (US08-02) runs both roles, so what is composed here is process
# topology, not a second application. Three invariants shape it:
#
# * one writer — `serve` and `worker` each take the library process lock for their
# role (US07-05), and both containers mount the *same* data volume, which is what
# makes the lock file visible to both. Scaling `worker` past 1 is refused by that
# lock rather than by anyone remembering not to;
# * one local filesystem — the database, its write-ahead log, the thumbnail cache,
# and the backups live in the `data` volume, and SQLite in WAL mode requires real
# local-filesystem locking. A network mount (NFS, SMB, or a cloud volume driver)
# is unsupported for it; that is a corrupted database, not a slow one;
# * one library path vocabulary — `PHOTO_PIPELINE_LIBRARY_ROOTS` names the path
# *inside* the container, which is also the bind mount's target below. A root
# that is not mounted there makes `serve` and `worker` refuse at startup instead
# of writing into the container's throwaway layer.
#
# Configuration and secrets come from the environment only: copy `.env.example` to
# `.env` and fill it in. Nothing is baked into the image and nothing with a value in
# it is committed.
#
# cp .env.example .env && $EDITOR .env
# docker compose up -d --build
#
# Operating it is operating the same CLI — `docker compose run --rm --no-deps api
# <command>` — see README, "Composed runtime".
name: photo-pipeline
x-runtime: &runtime
image: ${PHOTO_PIPELINE_IMAGE:-photo-pipeline:dev}
build:
context: .
args:
# Everything the app renames or rewrites has to stay owned by the host user
# the library already belongs to.
UID: ${PHOTO_PIPELINE_UID:-1000}
GID: ${PHOTO_PIPELINE_GID:-1000}
user: "${PHOTO_PIPELINE_UID:-1000}:${PHOTO_PIPELINE_GID:-1000}"
env_file:
- ${PHOTO_PIPELINE_ENV_FILE:-.env}
volumes:
- data:/data
- "${PHOTO_PIPELINE_LIBRARY_HOST_PATH:?set PHOTO_PIPELINE_LIBRARY_HOST_PATH to the photo library on this host}:${PHOTO_PIPELINE_LIBRARY_ROOTS:?set PHOTO_PIPELINE_LIBRARY_ROOTS to the container-side library path}"
# Jobs check for cancellation between items and leave a resumable record; a
# too-short grace period turns an orderly stop into a recovery on next start.
stop_grace_period: 30s
x-environment: &environment
# Set here rather than left to the file: these two are what the composition itself
# promises, and an `.env` that disagreed would move the database off the volume or
# the library off its mount.
PHOTO_PIPELINE_DATA_DIR: /data
PHOTO_PIPELINE_LIBRARY_ROOTS: ${PHOTO_PIPELINE_LIBRARY_ROOTS}
services:
# Migrations run to completion before either role accepts work, through the same
# backup-then-migrate path the roles use (US07-05): a pending upgrade is snapshotted
# first, and a failed one exits non-zero with the backup named — so `api` and
# `worker` never start, and the previous database is left intact and restorable.
migrate:
<<: *runtime
command: ["migrate"]
environment: *environment
restart: "no"
api:
<<: *runtime
command: ["serve"]
environment:
<<: *environment
# Published to host loopback below. Inside the container the server must bind
# the container's own interface for that publish to reach it, which is exactly
# what makes the access secret mandatory (US08-01) — `serve` refuses to start
# without one. Exposing the port beyond loopback additionally needs
# PHOTO_PIPELINE_ALLOWED_HOSTS to name the hostname it is reached under.
PHOTO_PIPELINE_HOST: 0.0.0.0
PHOTO_PIPELINE_PORT: 8000
ports:
# Host side only: PHOTO_PIPELINE_PORT in `.env` moves the *published* port, and
# the container always serves 8000, which is what the image's health check probes.
- "${PHOTO_PIPELINE_PUBLISH_ADDRESS:-127.0.0.1}:${PHOTO_PIPELINE_PORT:-8000}:8000"
depends_on:
migrate:
condition: service_completed_successfully
restart: unless-stopped
# One worker. A second one is refused by the library lock in the shared data
# volume, which is the point: `docker compose up --scale worker=2` fails loudly
# instead of running two writers against one library.
worker:
<<: *runtime
command: ["worker", "--id", "worker-1"]
environment: *environment
depends_on:
migrate:
condition: service_completed_successfully
restart: unless-stopped
volumes:
# Local driver on purpose: the database, WAL, thumbnail cache, and backups need a
# real local filesystem. Do not point this at NFS, SMB, or a network volume driver.
data:
driver: local

View File

@@ -1,5 +1,5 @@
"""Application management CLI:
``python -m photo_pipeline {serve,migrate,worker,import-legacy-scores,backup,verify-backup,restore,diagnostics}``.
``python -m photo_pipeline {serve,migrate,worker,import-legacy-scores,backup,verify-backup,restore,diagnostics,benchmark,release-gate,container-gate,dry-run,approve-dry-run}``.
``serve`` and ``worker`` take the library process lock for their role (US07-05):
two workers, or the frozen CLI running beside the app, would each be safe on their
@@ -11,9 +11,11 @@ from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from typing import Sequence
from photo_pipeline import path_policy
from photo_pipeline.config import Config
from photo_pipeline.services.app_lock import LegacyProcessActive, LibraryLock, LockHeld
from photo_pipeline.services.backup import BackupError, BackupService, migrate_with_backup
@@ -68,6 +70,14 @@ def main(argv: Sequence[str] | None = None) -> int:
"release-gate", help="Run every suite in an isolated stack and keep the evidence"
)
gate_cmd.add_argument("--output", help="Evidence directory (default: data/release/<stamp>)")
container_cmd = commands.add_parser(
"container-gate",
help="Provision the composition from the built image, run the phase_h "
"acceptance suite against it, destroy it, and keep the evidence (US08-05)",
)
container_cmd.add_argument(
"--output", help="Evidence directory (default: data/container-gate/<stamp>)"
)
dry_cmd = commands.add_parser(
"dry-run", help="Read-only reconciliation of the configured library (US07-07)"
)
@@ -83,6 +93,18 @@ def main(argv: Sequence[str] | None = None) -> int:
config = Config.from_env()
config.database_path.parent.mkdir(parents=True, exist_ok=True)
# In a container the two roles that touch the library check their boundary before
# they take a lock or bind a port: the configured roots must name the mount paths,
# and a mismatch is cheaper to refuse than to discover at the first write (US08-03).
# Only in a container — on a host an unmounted root is an ordinary Tuesday (an
# archive medium that is not plugged in), and refusing to serve would take the
# offline half of the library away with it.
if args.command in ("serve", "worker") and path_policy.in_container():
refusal = path_policy.roots_refusal(config.library_roots, require_mount=True)
if refusal is not None:
print(refusal, file=sys.stderr)
return 5
if args.command == "migrate":
manifest = migrate_with_backup(config)
if manifest:
@@ -141,6 +163,31 @@ def main(argv: Sequence[str] | None = None) -> int:
)
return 0 if report["ok"] else 1
if args.command == "container-gate":
from datetime import datetime, timezone
from photo_pipeline.services import release
# The suite provisions and destroys the composition itself; what this command
# adds is the single entry point and the retained evidence. No skip is an
# environment limit here: a gate that did not reach the containers proved
# nothing about them.
output = args.output or Path(config.data_dir) / "container-gate" / datetime.now(
timezone.utc
).strftime("%Y%m%dT%H%M%SZ")
report = release.run_gate(
config,
output=output,
stages=release.CONTAINER_STAGES,
allowed_skips=release.CONTAINER_ALLOWED_SKIP_REASONS,
)
print(
json.dumps(
{k: v for k, v in report.items() if k not in ("stages", "matrix")}, indent=2
)
)
return 0 if report["ok"] else 1
if args.command == "dry-run":
from photo_pipeline.services import release
@@ -209,8 +256,6 @@ def main(argv: Sequence[str] | None = None) -> int:
lock.release()
return 0
import sys
import uvicorn
from photo_pipeline.api.app import ConfigurationRefused, create_app
@@ -238,8 +283,6 @@ def _acquire(lock: LibraryLock, *, allow_legacy: bool) -> int | None:
Returns an exit code to return, or ``None`` when the lock was acquired.
"""
import sys
try:
lock.acquire(allow_legacy=allow_legacy)
except LockHeld as error:

View File

@@ -15,9 +15,7 @@ import os
from pathlib import Path
from typing import Iterable, Iterator
SUPPORTED_EXTENSIONS = {
".jpg", ".jpeg", ".png", ".webp", ".heic", ".heif", ".tiff", ".tif"
}
SUPPORTED_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp", ".heic", ".heif", ".tiff", ".tif"}
EXCLUDED_DIR_NAMES = {"_IGNORE", ".@__thumb"}
@@ -72,6 +70,63 @@ def resolve_in_roots(roots: Iterable[os.PathLike | str], path: os.PathLike | str
raise PathPolicyError("path is outside the configured library roots")
def in_container() -> bool:
"""Whether this process is running inside a container image build of the app."""
return Path("/.dockerenv").exists()
def _under_mount(path: Path) -> bool:
"""Whether ``path`` or one of its parents below ``/`` is a mounted filesystem."""
current = path.resolve()
while current != current.parent:
if os.path.ismount(current):
return True
current = current.parent
return False
def roots_refusal(roots: Iterable[os.PathLike | str], *, require_mount: bool = False) -> str | None:
"""Why the configured library roots cannot be worked with, or ``None`` (US08-03).
The roots name the directories this installation renames folders in, rewrites
EXIF in, and archives from. Container path policy is the same problem as host
path policy with one new failure mode: the configured roots must name the
*container-side* mount paths. A host path configured inside a container is
either absent or an ordinary directory of the image, so the library looks empty
and the first write lands in the container's throwaway layer instead of in the
library. That is worth refusing at startup, while an operator is still watching,
rather than at the first write.
``require_mount`` is the container-only half of that: inside a container a real
library arrives through a bind mount, so a root that is not on (or under) a
mount point is not the library the deployment meant.
Writability is deliberately *not* checked: a bind mount's ownership is
virtualised by Docker Desktop and Colima (it arrives as ``root:root``), so
``os.access`` there is evidence about the virtio layer rather than about the
library. A wrong UID/GID surfaces as a refused rename with the real errno, which
is at least true; a refusal here would be false on two supported platforms.
"""
for root in roots:
path = Path(root)
if not path.exists():
return (
f"library root {path} does not exist: PHOTO_PIPELINE_LIBRARY_ROOTS must "
"name paths that exist here, and in a container that means the mount path"
)
if not path.is_dir():
return f"library root {path} is not a directory"
if not os.access(path, os.R_OK | os.X_OK):
return f"library root {path} is not readable by this process"
if require_mount and not _under_mount(path):
return (
f"library root {path} is not on a mounted filesystem in this container: "
"the library was not bind-mounted there, so PHOTO_PIPELINE_LIBRARY_ROOTS "
"names a directory of the image rather than the library"
)
return None
def iter_supported_files(root: os.PathLike | str) -> Iterator[Path]:
"""Yield supported, non-excluded files under ``root`` in deterministic order.

View File

@@ -15,9 +15,17 @@ package:
"started_at": "...", "library_roots": ["..."]}
One holder per role: an API and a worker are designed to run together, a second
worker is not. A lock whose process is gone is stale and is taken over with the
takeover recorded — refusing to start because of a crashed predecessor would turn
one outage into two.
worker is not. A lock whose process is gone is stale and is taken over — refusing
to start because of a crashed predecessor would turn one outage into two.
Ownership is an advisory ``flock`` on that file, not the record inside it. The
record says *who*; the kernel says *whether*. That distinction is what makes the
lock work in containers (US08-03), where a PID and a hostname are namespaced: a
lock left behind by a container that no longer exists names a pid that still
"exists" in the new container and a host that cannot be probed, so believing the
file would deadlock every restart. A flock is released when its holder dies however
it dies, and is seen by every process that can open the file — which for a local
data directory is every container of this deployment.
Legacy detection is deliberately a heuristic, not a promise: the archived CLI has
no lock of its own, so what can be observed is its state files being written right
@@ -27,6 +35,7 @@ mutating stage should refuse until it stops.
from __future__ import annotations
import fcntl
import json
import os
import socket
@@ -109,6 +118,12 @@ def _now() -> datetime:
return datetime.now(timezone.utc)
def _in_container() -> bool:
from photo_pipeline import path_policy
return path_policy.in_container()
def legacy_activity(config: Config) -> dict:
"""Legacy state files written within the activity window, if any."""
seen: list[dict] = []
@@ -139,6 +154,7 @@ class LibraryLock:
self.role = role
self.path = Path(config.data_dir) / f"{role}{LOCK_SUFFIX}"
self._acquired = False
self._handle = None
# ── inspection ────────────────────────────────────────────────────────────
@@ -178,12 +194,31 @@ class LibraryLock:
"stop it before running the application"
)
self.path.parent.mkdir(parents=True, exist_ok=True)
# The kernel decides, because the file cannot: a container's PID and hostname
# are namespaced, so a lock left by a container that no longer exists names a
# pid that "exists" and a host that cannot be probed (US08-03). An advisory
# flock is held by a live process or by nobody, is released when that process
# dies however it dies, and is shared by every process that can open this
# file — which, for a local data directory, is every role in every container
# of this deployment.
handle = open(self.path, "a+", encoding="utf-8")
try:
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError:
handle.close()
raise LockHeld(self.holder() or Holder(self.role, -1, "unknown", "unknown")) from None
current = self.holder()
if current is not None:
if current.alive:
raise LockHeld(current)
# Stale: its process is gone. Take over, and say so.
self.path.unlink(missing_ok=True)
if current is not None and current.host != socket.gethostname() and not _in_container():
# We hold the kernel's lock, so nothing on *this* machine holds the file.
# On a host that still leaves one case open: a data directory shared with
# another machine, whose flock we cannot trust. Believe its record rather
# than run two writers. In a container the data volume is local by
# construction (US08-03), and a foreign hostname is only a dead container.
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
handle.close()
raise LockHeld(current)
mine = Holder(
role=self.role,
@@ -192,15 +227,14 @@ class LibraryLock:
started_at=_now().isoformat(),
library_roots=tuple(str(root) for root in self._config.library_roots),
)
self.path.parent.mkdir(parents=True, exist_ok=True)
payload = {k: v for k, v in mine.as_dict().items() if k != "alive"}
# Exclusive create, so two processes racing here cannot both believe they won.
try:
with open(self.path, "x", encoding="utf-8") as handle:
json.dump(payload, handle, indent=2)
except FileExistsError:
winner = self.holder()
raise LockHeld(winner or mine) from None
handle.seek(0)
handle.truncate()
json.dump(payload, handle, indent=2)
handle.flush()
# Held open on purpose: closing it is what releases the lock, and that must
# happen when this process ends, not when this method returns.
self._handle = handle
self._acquired = True
return mine
@@ -208,9 +242,10 @@ class LibraryLock:
"""Give up a lock this process owns. Another holder's lock is left alone."""
if not self._acquired:
return
current = self.holder()
if current is not None and current.pid == os.getpid():
self.path.unlink(missing_ok=True)
self.path.unlink(missing_ok=True)
if self._handle is not None:
self._handle.close() # closing the descriptor releases the kernel lock
self._handle = None
self._acquired = False
def __enter__(self) -> "LibraryLock":

View File

@@ -60,6 +60,17 @@ ALLOWED_SKIP_REASONS = (
"bind-mount ownership is virtualised",
)
# The container acceptance gate (US08-05): the deployed application, verified the way
# the host application is. Deliberately one stage selected by marker, so adding a
# phase_h test is enough to put it in front of a deploy.
CONTAINER_STAGES: tuple[tuple[str, tuple[str, ...]], ...] = (
("container", ("tests/e2e", "-m", "phase_h")),
)
# Nothing. This gate exists to prove the *deployed* runtime, and every reason a check
# would skip here — no daemon, no compose, no browser — means it was not proven.
CONTAINER_ALLOWED_SKIP_REASONS: tuple[str, ...] = ()
class ReleaseError(RuntimeError):
pass
@@ -175,13 +186,15 @@ def _run_stage(label: str, paths: tuple[str, ...], *, repo: Path, log_dir: Path)
return StageResult(label, command, result.returncode, elapsed, summary, skipped)
def unexpected_skips(results: list[StageResult]) -> list[str]:
def unexpected_skips(
results: list[StageResult], allowed: tuple[str, ...] = ALLOWED_SKIP_REASONS
) -> list[str]:
"""Skips the gate will not accept: everything but the documented environment ones."""
return [
line
for result in results
for line in result.skipped
if not any(reason in line for reason in ALLOWED_SKIP_REASONS)
if not any(reason in line for reason in allowed)
]
@@ -190,6 +203,7 @@ def run_gate(
*,
output: Path | str | None = None,
stages: tuple[tuple[str, tuple[str, ...]], ...] = STAGES,
allowed_skips: tuple[str, ...] = ALLOWED_SKIP_REASONS,
repo: Path | None = None,
) -> dict:
"""Run every suite in an isolated stack and retain checksummed evidence.
@@ -206,13 +220,14 @@ def run_gate(
logs = directory / "logs"
logs.mkdir(parents=True, exist_ok=True)
started_at = _now()
matrix = story_matrix(repo)
results = [_run_stage(label, paths, repo=repo, log_dir=logs) for label, paths in stages]
skips = unexpected_skips(results)
skips = unexpected_skips(results, allowed_skips)
report = {
"schema_version": SCHEMA_VERSION,
"started_at": _now().isoformat(),
"started_at": started_at.isoformat(),
"revision": revision(),
"python": sys.version.split()[0],
"platform": os.uname().sysname,

View File

@@ -24,6 +24,9 @@ test = [
"httpx>=0.27",
"playwright>=1.40",
"pytest-playwright>=0.4",
# The composition and the workflows are configuration, and the tests that hold
# them to their promises read them (US08-03, US08-04).
"pyyaml>=6",
]
[build-system]
@@ -49,5 +52,6 @@ markers = [
"phase_d: Phase D end-to-end acceptance (US04-06) — guarded rename API, fault, and browser journeys",
"phase_e: Phase E end-to-end acceptance (US05-06) — upload preflight, uploader, and browser journeys",
"phase_f: Phase F end-to-end acceptance (US06-06) — archive destination, transfer, and restore journeys",
"container: builds and runs the container image (US08-02) — needs a Docker daemon and network",
"container: builds and runs the container image and its composition (US08-02, US08-03) — needs a Docker daemon, the compose plugin, and the network",
"phase_h: Phase H container deployment acceptance (US08-05) — the browser, upgrade, restart, and security journeys against the composed stack",
]

View File

@@ -0,0 +1,260 @@
"""The composed stack, as a fixture: build, provision, drive, destroy (US08-03/US08-05).
Extracted from ``tests/e2e/test_compose_stack.py`` when the container acceptance gate
needed the same stack under a different image, project, and library. One
implementation, because two would drift on exactly the details that make a container
test worth anything — the mount, the volume, the ports, and the teardown.
Nothing here fakes anything below the process boundary: Docker builds the image,
Compose starts the real containers, and every helper talks to them over HTTP or the
Docker CLI.
"""
from __future__ import annotations
import os
import shutil
import socket
import subprocess
import tempfile
import time
from pathlib import Path
import httpx
import pytest
from PIL import Image
REPO = Path(__file__).resolve().parents[2]
COMPOSE_FILE = REPO / "docker-compose.yml"
CONTAINER_LIBRARY = "/library"
READY_TIMEOUT_SECONDS = 180
JOB_TIMEOUT_SECONDS = 300
UP_TIMEOUT_SECONDS = 30 * 60
def compose_available() -> bool:
try:
return (
subprocess.run(
["docker", "compose", "version"], capture_output=True, timeout=60
).returncode
== 0
)
except (OSError, subprocess.SubprocessError):
return False
def free_port() -> int:
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
return sock.getsockname()[1]
def mount_base() -> Path:
"""A directory a Docker VM shares with the host.
Not pytest's ``tmp_path``: on macOS that is ``/var/folders/...``, which a Docker VM
(Colima, Docker Desktop) does not share, so the bind mount would arrive empty and
every assertion would be about nothing. ``$HOME`` is shared by every default
configuration.
"""
base = Path(
os.environ.get("PHOTO_PIPELINE_TEST_MOUNT_BASE", Path.home() / ".cache" / "photo-pipeline")
)
base.mkdir(parents=True, exist_ok=True)
return base
def temporary_library(albums: dict[str, int], *, prefix: str = "library-") -> Path:
"""A fixture library on a shareable host path, with the exclusion sentinel in it."""
root = Path(tempfile.mkdtemp(prefix=prefix, dir=mount_base()))
for position, (album, count) in enumerate(albums.items()):
(root / album).mkdir(parents=True, exist_ok=True)
for index in range(count):
# Distinct per album *and* index: two solid images of the same colour are
# byte-identical, which would make them a duplicate cluster by accident.
colour = (17 + 7 * index, 31 + 29 * position, 160 - 3 * index)
Image.new("RGB", (64, 48), colour).save(root / album / f"{album}_{index}.jpg")
# Never discovered, counted, analyzed, or uploaded — asserted from outside it.
(root / "_IGNORE").mkdir(exist_ok=True)
Image.new("RGB", (32, 32), (0, 0, 0)).save(root / "_IGNORE" / "sentinel.jpg")
return root
def remove_library(root: Path) -> None:
shutil.rmtree(root, ignore_errors=True)
def write_env_file(path: Path, secret: str, extra: dict[str, str] | None = None) -> Path:
"""Configuration and secrets come from the environment, so a test writes its own
file rather than borrowing the operator's ``.env``."""
lines = [
f"PHOTO_PIPELINE_ACCESS_SECRET={secret}",
"PHOTO_PIPELINE_LOG_FORMAT=text",
# The deterministic vision seam, in the data volume so both roles and the test
# can read it (concept §18).
"PHOTO_PIPELINE_FAKE_VISION_LOG=/data/vision.log",
*(f"{key}={value}" for key, value in (extra or {}).items()),
]
path.write_text("\n".join(lines) + "\n")
return path
class Stack:
"""The composition under test, plus the environment it was started with."""
def __init__(
self,
library: Path,
env_file: Path,
*,
project: str,
image: str,
secret: str,
) -> None:
self.library = library
self.project = project
self.secret = secret
self.port = free_port()
self.base = f"http://127.0.0.1:{self.port}"
# Compose reads the repository's own .env for substitution; the process
# environment wins over it, so the test's values are the ones that apply.
self.env = {
**os.environ,
"PHOTO_PIPELINE_IMAGE": image,
"PHOTO_PIPELINE_ENV_FILE": str(env_file),
"PHOTO_PIPELINE_LIBRARY_HOST_PATH": str(library),
"PHOTO_PIPELINE_LIBRARY_ROOTS": CONTAINER_LIBRARY,
"PHOTO_PIPELINE_PORT": str(self.port),
"PHOTO_PIPELINE_UID": str(os.getuid()),
"PHOTO_PIPELINE_GID": str(os.getgid()),
}
# ── the compose lifecycle ────────────────────────────────────────────────
def compose(self, *args: str, check: bool = True, timeout: int = 300):
result = subprocess.run(
["docker", "compose", "-p", self.project, "-f", str(COMPOSE_FILE), *args],
capture_output=True,
text=True,
env=self.env,
cwd=REPO,
timeout=timeout,
)
if check and result.returncode != 0:
raise AssertionError(
f"docker compose {' '.join(args)} failed:\n{result.stdout}\n{result.stderr}\n"
f"{self.compose('logs', '--tail', '80', check=False).stdout}"
)
return result
def up(self, *extra: str) -> None:
self.compose("up", "--detach", *extra, timeout=UP_TIMEOUT_SECONDS)
def down(self, *, volumes: bool = True) -> None:
self.compose(
"down",
*(("--volumes",) if volumes else ()),
"--remove-orphans",
check=False,
timeout=300,
)
def use_image(self, image: str) -> None:
"""Point the composition at another tag — the upgrade path (US08-05)."""
self.env["PHOTO_PIPELINE_IMAGE"] = image
def logs(self, *services: str) -> str:
result = self.compose("logs", *services, check=False)
return result.stdout + result.stderr
def wait_until_ready(self) -> None:
deadline = time.monotonic() + READY_TIMEOUT_SECONDS
while time.monotonic() < deadline:
try:
if httpx.get(f"{self.base}/api/v1/health/ready", timeout=5).status_code == 200:
return
except httpx.HTTPError:
pass
time.sleep(0.5)
raise AssertionError(f"the stack never became ready:\n{self.logs()}")
# ── talking to it ────────────────────────────────────────────────────────
def client(self) -> httpx.Client:
"""A browser that has loaded the app: session cookie in the jar, token in a
header. The session belongs to the API process, so it is re-bootstrapped after
every restart."""
client = httpx.Client(base_url=f"{self.base}/api/v1", timeout=60)
bootstrap = client.get("/session", headers={"X-Access-Secret": self.secret})
assert bootstrap.status_code == 200, bootstrap.text
client.headers["X-CSRF-Token"] = bootstrap.json()["csrf_token"]
return client
def await_job(client: httpx.Client, job_id: str, states=("succeeded",)) -> dict:
deadline = time.monotonic() + JOB_TIMEOUT_SECONDS
snapshot: dict = {}
while time.monotonic() < deadline:
response = client.get(f"/jobs/{job_id}")
if response.status_code == 200:
snapshot = response.json()
if snapshot["state"] in states:
return snapshot
time.sleep(0.5)
raise AssertionError(f"job {job_id} never reached {states}: {snapshot}")
def build_image(tag: str, *, revision: str | None = None) -> str:
"""Build the application image; from a git revision's tree when one is named.
``revision`` is how the upgrade journey gets the *previous* version without a
registry: the tree of that commit is the build context, so what it produces is the
image that commit would have published.
"""
if revision is None:
subprocess.run(
[
"docker",
"build",
"--build-arg",
f"UID={os.getuid()}",
"--build-arg",
f"GID={os.getgid()}",
"-t",
tag,
str(REPO),
],
check=True,
timeout=UP_TIMEOUT_SECONDS,
)
return tag
archive = subprocess.run(
["git", "archive", "--format=tar", revision],
cwd=REPO,
capture_output=True,
check=True,
timeout=300,
).stdout
subprocess.run(
[
"docker",
"build",
"--build-arg",
f"UID={os.getuid()}",
"--build-arg",
f"GID={os.getgid()}",
"-t",
tag,
"-",
],
input=archive,
check=True,
timeout=UP_TIMEOUT_SECONDS,
)
return tag
needs_compose = pytest.mark.skipif(
not compose_available(), reason="no Docker daemon with the compose plugin"
)

View File

@@ -0,0 +1,212 @@
"""US08-03: the composition, actually composed.
Nothing here is faked below the process boundary: Docker builds the image, Compose
starts the migrate/API/worker containers against a temporary fixture library on a
real bind mount, and every assertion is made over HTTP or against what the stack
left in its data volume. The vision provider is the deterministic fake seam the
other end-to-end suites use, because an upload of real photos to a real model is not
what this story is about — the mount, the lock, the volume, and the restart are.
The file contract (one API, one worker, migrations first, no committed values) is
checked without a daemon in ``tests/integration/test_compose_runtime.py``; only the
running proof needs Docker, and CI is where it runs unskipped (US08-04).
The stack itself lives in ``tests/e2e/_container_harness.py``, shared with the
container acceptance gate (US08-05).
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
import pytest
from tests.e2e._container_harness import (
CONTAINER_LIBRARY,
Stack,
await_job,
compose_available,
needs_compose,
remove_library,
temporary_library,
write_env_file,
)
PROJECT = "photo-pipeline-us0803"
IMAGE = "photo-pipeline-test:us08-03"
SECRET = "compose-acceptance-secret"
pytestmark = pytest.mark.container
@pytest.fixture(scope="module")
def library() -> Path:
root = temporary_library({"01_day": 2, "02_night": 1})
try:
yield root
finally:
remove_library(root)
@pytest.fixture(scope="module")
def env_file(tmp_path_factory) -> Path:
return write_env_file(tmp_path_factory.mktemp("config") / "compose.env", SECRET)
@pytest.fixture(scope="module")
def stack(library, env_file):
if not compose_available():
pytest.skip("no Docker daemon with the compose plugin")
running = Stack(library, env_file, project=PROJECT, image=IMAGE, secret=SECRET)
running.down()
running.up("--build")
try:
running.wait_until_ready()
yield running
finally:
running.down()
# ── the mounted library ──────────────────────────────────────────────────────
@needs_compose
def test_the_stack_scans_the_bind_mounted_library_at_its_container_paths(stack):
client = stack.client()
try:
scanned = client.post("/inventory/scan")
assert scanned.status_code == 200, scanned.text
assets = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
finally:
client.close()
assert len(assets) == 3, assets
paths = {asset["current_path"] for asset in assets}
assert all(path.startswith(CONTAINER_LIBRARY + "/") for path in paths), paths
assert not any("_IGNORE" in path or "sentinel" in path for path in paths)
# The host paths are what the operator mounted, and they are not what the
# application records: the roots are the container's.
assert not any(str(stack.library) in path for path in paths)
@needs_compose
def test_a_library_root_that_is_not_mounted_is_refused_at_startup(stack):
"""The container-specific failure: configured roots that name nothing mounted."""
refused = stack.compose(
"run",
"--rm",
"--no-deps",
"--env",
"PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos",
"api",
"serve",
check=False,
)
assert refused.returncode == 5, refused.stdout + refused.stderr
assert "library root /srv/photos" in refused.stdout + refused.stderr
@needs_compose
def test_a_second_worker_is_refused_by_the_library_lock(stack):
"""Not by convention: the running worker's lock is in the shared data volume."""
refused = stack.compose(
"run", "--rm", "--no-deps", "worker", "worker", "--id", "worker-2", check=False
)
assert refused.returncode == 2, refused.stdout + refused.stderr
assert "worker is already running" in refused.stdout + refused.stderr
# ── restart, resume, and the data volume ─────────────────────────────────────
@needs_compose
def test_a_queued_job_resumes_after_both_containers_restart(stack):
client = stack.client()
try:
client.post("/inventory/scan").raise_for_status()
assets = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
for asset in assets:
decided = client.post(
"/safety/decisions", json={"asset_id": asset["id"], "decision": "sfw"}
)
assert decided.status_code == 200, decided.text
# Stop the worker first, so the job is provably still queued when the restart
# happens: a job that finished before the restart would prove nothing.
stack.compose("stop", "worker")
job = client.post("/analysis/jobs").json()
assert client.get(f"/jobs/{job['id']}").json()["state"] == "queued"
finally:
client.close()
stack.compose("restart", "api", "worker")
stack.wait_until_ready()
client = stack.client() # the session is per API process, so it is re-bootstrapped
try:
finished = await_job(client, job["id"])
assert finished["state"] == "succeeded", finished
# The database is intact and the work is durable, not merely reported.
after = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
assert {asset["id"] for asset in after} == {asset["id"] for asset in assets}
analysed = client.get(f"/analysis/results/{assets[0]['id']}")
assert analysed.status_code == 200, analysed.text
assert analysed.json()["description"]
# Only the mounted library's own photos were analysed: the sentinel under
# _IGNORE is not an asset, so it can never have become an item of this job.
assert finished["progress"]["total"] == len(assets)
finally:
client.close()
@needs_compose
def test_the_data_volume_survives_recreating_the_containers(stack):
"""`down` without `--volumes` then `up` is the upgrade path: state stays."""
client = stack.client()
try:
client.post("/inventory/scan").raise_for_status()
before = {a["id"] for a in client.get("/inventory/assets").json()["items"]}
finally:
client.close()
stack.down(volumes=False)
stack.up()
stack.wait_until_ready()
client = stack.client()
try:
after = {a["id"] for a in client.get("/inventory/assets").json()["items"]}
finally:
client.close()
assert after == before, "the same assets, from the same database, on the same volume"
# ── operating it ─────────────────────────────────────────────────────────────
@needs_compose
def test_backup_verify_and_diagnostics_run_as_container_commands(stack):
backup = stack.compose("run", "--rm", "--no-deps", "api", "backup", "--reason", "compose")
manifest = json.loads(backup.stdout[backup.stdout.index("{") :])
assert manifest["name"].startswith("2")
verified = stack.compose(
"run", "--rm", "--no-deps", "api", "verify-backup", f"/data/backups/{manifest['name']}"
)
assert json.loads(verified.stdout[verified.stdout.index("{") :])["ok"] is True
report = stack.compose("run", "--rm", "--no-deps", "api", "diagnostics")
diagnostics = json.loads(report.stdout[report.stdout.index("{") :])
components = {c["name"]: c["path"] for c in diagnostics["components"]}
# Database, WAL, thumbnail cache, and backups all live in the mounted volume.
for name in ("database", "write_ahead_log", "thumbnail_cache", "backups"):
assert components[name].startswith("/data/"), (name, components[name])
assert {tool["name"] for tool in diagnostics["tools"]} >= {"exiftool", "immich-go"}
# And the worker running beside this command is visible as the lock's holder.
assert diagnostics["locks"]["worker"]["role"] == "worker"
if __name__ == "__main__": # a quick way to run just this file
raise SystemExit(pytest.main([__file__, "-v", *sys.argv[1:]]))

View File

@@ -0,0 +1,510 @@
"""Phase H — the container acceptance gate (US08-05).
The deployed application is verified the way the host application is: real image,
real composition, real browser, real restarts. Everything below runs against
containers that this suite provisions from the built image, against a temporary
fixture library on a bind mount and an isolated data volume, and destroys afterwards.
Four journeys, one per thing a deployment can get wrong:
* **the browser journey** — discovery, duplicate review, analysis, album proposal,
rename, upload preflight, and archive, driven through the containerized frontend;
* **the upgrade journey** — the previous version's image runs first, then this one,
and the database, its migrations, the rename journal, the job history, and the
thumbnail cache have to still be there;
* **the restart journey** — both containers are killed mid-job and the work resumes
without doing anything twice;
* **the security gates** — the refusals a loopback deployment made are still made
behind a published port: no session, forged forwarded headers, a path that leaves
the mounted library, and a secret in the logs.
Run it as one command, with evidence: ``python -m photo_pipeline container-gate``.
"""
from __future__ import annotations
import collections
from contextlib import closing
import json
import os
import shutil
import subprocess
import sys
import time
from pathlib import Path
import httpx
import pytest
from playwright.sync_api import expect
from photo_pipeline.db import head_revision
from tests.e2e._container_harness import (
Stack,
await_job,
build_image,
compose_available,
needs_compose,
remove_library,
temporary_library,
write_env_file,
)
pytestmark = [pytest.mark.phase_h, pytest.mark.container]
PROJECT = "photo-pipeline-us0805"
IMAGE = "photo-pipeline-test:us08-05"
# The version being upgraded *from*. CI passes the tag it last published; without one,
# the previous commit's tree is built, which is the same claim without a registry.
PREVIOUS_IMAGE = os.environ.get("PHOTO_PIPELINE_PREVIOUS_IMAGE")
PREVIOUS_TAG = "photo-pipeline-test:us08-05-previous"
SECRET = "container-gate-access-secret"
IMMICH_SENTINEL = "immich-sentinel-9f3a2b"
HOSTNAME = "photos.test"
ALBUM = "rome"
RENAMED = "2019 Rome"
BURST = 30
# ── the image and the stacks ─────────────────────────────────────────────────
@pytest.fixture(scope="session")
def image() -> str:
if not compose_available():
pytest.skip("no Docker daemon with the compose plugin")
return build_image(IMAGE)
@pytest.fixture(scope="module")
def journey(image, tmp_path_factory):
"""One album, one exact duplicate of a photo in it, and the exclusion sentinel."""
library = temporary_library({ALBUM: 3}, prefix="us0805-journey-")
shutil.copyfile(library / ALBUM / f"{ALBUM}_0.jpg", library / ALBUM / "copy.jpg")
env_file = write_env_file(
tmp_path_factory.mktemp("journey") / "gate.env",
SECRET,
{
"PHOTO_PIPELINE_ALLOWED_HOSTS": HOSTNAME,
# Configured but never reachable: the upload view's job here is to show the
# preflight blockers, and the key's job is to be absent from every log.
"PHOTO_PIPELINE_IMMICH_API_KEY": IMMICH_SENTINEL,
"PHOTO_PIPELINE_IMMICH_SERVER_URL": "http://127.0.0.1:1",
},
)
stack = Stack(library, env_file, project=f"{PROJECT}-journey", image=image, secret=SECRET)
try:
stack.down()
stack.up()
stack.wait_until_ready()
_seed_journey(stack)
yield stack
finally:
stack.down()
remove_library(library)
def _seed_journey(stack: Stack) -> None:
"""Everything the views need, established over the public API before they render."""
with closing(stack.client()) as client:
client.post("/inventory/scan").raise_for_status()
client.post("/duplicates/detect").raise_for_status()
queue = client.get("/safety/queue", params={"limit": 100}).json()["items"]
for item in queue:
decided = client.post(
"/safety/decisions", json={"asset_id": item["asset_id"], "decision": "sfw"}
)
assert decided.status_code == 200, decided.text
job = client.post("/analysis/jobs").json()
await_job(client, job["id"])
client.post("/albums/proposals", json={}).raise_for_status()
# An archive destination inside the data volume: a second mount would prove
# nothing more, and the view needs a location to have something to show.
stack.compose("exec", "-T", "api", "mkdir", "-p", "/data/archive")
client.post(
"/archive-locations", json={"name": "external", "root": "/data/archive"}
).raise_for_status()
# ── the browser journey ──────────────────────────────────────────────────────
@needs_compose
def test_the_browser_journey_covers_every_stage_view_of_the_deployed_app(page, journey):
"""One pass through the deployed frontend, in workflow order.
The access secret is supplied the way a person supplies it — the app asks, the
answer is kept for the tab — so what is proven is the authenticated deployment,
not a test-only bypass.
"""
page.on("dialog", lambda dialog: dialog.accept(SECRET))
errors: list[str] = []
page.on("pageerror", lambda error: errors.append(str(error)))
base = journey.base
with closing(journey.client()) as client:
cluster = client.get("/duplicates/clusters").json()["items"][0]
# ── discovery ────────────────────────────────────────────────────────────
page.goto(f"{base}/app/#/workflow")
page.get_by_test_id("stage-safety").wait_for()
expect(page.get_by_test_id("stage-analysis")).to_be_visible()
page.goto(f"{base}/app/#/inventory")
rows = page.get_by_test_id("asset-row")
rows.first.wait_for()
assert rows.count() == 4, "three photos and the duplicate copy; never the sentinel"
assert "sentinel" not in page.content() and "_IGNORE" not in page.content()
# ── duplicate review ─────────────────────────────────────────────────────
page.goto(f"{base}/app/#/duplicates/{cluster['id']}")
page.get_by_test_id("cluster-state").wait_for()
# Exact bytes: the cluster arrives decided, and the review surface has to show
# both members and the evidence the decision was made on.
expect(page.get_by_test_id("cluster-state")).to_contain_text("decided")
expect(page.get_by_test_id("member")).to_have_count(2)
expect(page.get_by_test_id("member").first).to_contain_text("/library/")
# ── analysis ─────────────────────────────────────────────────────────────
page.goto(f"{base}/app/#/analyze")
page.get_by_test_id("analyze-counts").wait_for()
expect(page.get_by_test_id("run-analysis")).to_be_visible()
# ── album proposal ───────────────────────────────────────────────────────
page.goto(f"{base}/app/#/albums?album={ALBUM}")
page.get_by_test_id("suggested-name").wait_for()
page.get_by_test_id("final-name").fill(RENAMED)
page.get_by_test_id("save-name").click()
page.get_by_test_id("approve").click()
expect(page.get_by_test_id("proposal-status")).to_contain_text("approved")
# ── rename, applied against the bind mount ───────────────────────────────
page.goto(f"{base}/app/#/renames")
page.get_by_test_id("build-plan").click()
page.get_by_test_id("operations").wait_for()
expect(page.get_by_test_id("op-destination").first).to_contain_text(RENAMED)
page.get_by_test_id("apply-plan").click()
expect(page.get_by_test_id("apply-result")).to_contain_text("Applied 1, failed 0")
# The mounted library is the host's directory: the container renamed the operator's
# folder, not a copy inside its own layer.
assert (journey.library / RENAMED).is_dir()
assert not (journey.library / ALBUM).exists()
# ── upload preflight ─────────────────────────────────────────────────────
page.goto(f"{base}/app/#/uploads")
page.get_by_test_id("upload-scope").wait_for()
expect(page.get_by_test_id("album-row").first).to_be_visible()
assert IMMICH_SENTINEL not in page.content(), "the API key never reaches the browser"
# ── archive ──────────────────────────────────────────────────────────────
page.goto(f"{base}/app/#/archive")
page.get_by_test_id("archive-locations").wait_for()
expect(page.get_by_test_id("location-row").first).to_contain_text("external")
assert errors == [], f"the deployed frontend raised page errors: {errors}"
# ── the upgrade journey ──────────────────────────────────────────────────────
@pytest.fixture(scope="module")
def previous_image(image) -> str:
"""The image the deployment is upgrading *from*.
The published tag when there is one. Before the first publish there is nothing to
pull, and refusing then would mean the gate could never let the first deploy
through — so the previous commit's tree is built instead, which is the same claim
without a registry.
"""
if PREVIOUS_IMAGE:
pulled = subprocess.run(
["docker", "pull", PREVIOUS_IMAGE], capture_output=True, timeout=1800
)
if pulled.returncode == 0:
return PREVIOUS_IMAGE
return build_image(PREVIOUS_TAG, revision="HEAD~1")
@needs_compose
def test_an_upgrade_keeps_the_database_the_journal_the_jobs_and_the_cache(
image, previous_image, tmp_path_factory
):
library = temporary_library({ALBUM: 2}, prefix="us0805-upgrade-")
env_file = write_env_file(tmp_path_factory.mktemp("upgrade") / "gate.env", SECRET)
stack = Stack(
library, env_file, project=f"{PROJECT}-upgrade", image=previous_image, secret=SECRET
)
try:
stack.down()
stack.up()
stack.wait_until_ready()
# ── what the previous version leaves behind ──────────────────────────
with closing(stack.client()) as client:
client.post("/inventory/scan").raise_for_status()
assets = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
for item in client.get("/safety/queue", params={"limit": 100}).json()["items"]:
client.post(
"/safety/decisions", json={"asset_id": item["asset_id"], "decision": "sfw"}
).raise_for_status()
job = client.post("/analysis/jobs").json()
finished = await_job(client, job["id"])
for asset in assets: # populate the thumbnail cache
thumbnail = client.get(f"/assets/{asset['id']}/thumbnail", params={"size": 256})
assert thumbnail.status_code == 200, thumbnail.text
# A rename plan, left unapplied: the journal has to survive the upgrade
# exactly as it was, or a half-applied one could not be recovered.
client.post("/albums/proposals", json={}).raise_for_status()
_approve(client, RENAMED)
plan = client.post("/rename-plans").json()
before = {
"assets": sorted(asset["id"] for asset in assets),
"job": finished["progress"],
"plan": (plan["id"], plan["checksum"]),
"thumbnails": _cache_files(stack),
"revision": _revision(stack),
}
assert before["thumbnails"], "no thumbnail was cached, so nothing would be proven"
# ── the upgrade: same volume, new image ──────────────────────────────
stack.down(volumes=False)
stack.use_image(image)
stack.up()
stack.wait_until_ready()
after_revision = _revision(stack)
assert after_revision == head_revision(), "the new image did not migrate the volume"
if after_revision != before["revision"]:
# A schema change is snapshotted before it is applied (US07-05), so a
# failed upgrade is restorable rather than a lost library.
assert _ls(stack, "/data/backups"), "a migration ran without a backup"
with closing(stack.client()) as client:
assets = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
assert sorted(asset["id"] for asset in assets) == before["assets"]
assert client.get(f"/jobs/{job['id']}").json()["progress"] == before["job"]
plans = client.get("/rename-plans").json()["items"]
assert (plans[0]["id"], plans[0]["checksum"]) == before["plan"]
for asset in assets:
assert (
client.get(f"/analysis/results/{asset['id']}").status_code == 200
), "an analysis result did not survive the upgrade"
# The cache is keyed by pixel hash and thumbnail version, so an upgrade that
# kept the volume must keep the files: regenerating them is work nobody asked
# for, and losing them silently is how a cache stops being one.
assert set(before["thumbnails"]) <= set(_cache_files(stack))
finally:
stack.down()
remove_library(library)
def _approve(client: httpx.Client, name: str, *, album: str = ALBUM) -> None:
for payload, route in (({"name": name}, "edit"), ({}, "approve")):
current = client.get(f"/albums/proposals/{album}").json()
client.post(
f"/albums/proposals/{album}/{route}",
json={**payload, "expected_version": current["version"]},
).raise_for_status()
def _exec(stack: Stack, *args: str) -> str:
return stack.compose("exec", "-T", "api", *args).stdout
def _ls(stack: Stack, directory: str) -> list[str]:
listing = stack.compose("exec", "-T", "api", "ls", directory, check=False)
return [line for line in listing.stdout.split() if line]
def _cache_files(stack: Stack) -> list[str]:
return sorted(
_exec(stack, "find", "/data/cache", "-type", "f", "-name", "*.webp").split()
)
def _revision(stack: Stack) -> str:
"""The schema revision the volume's database is actually at."""
return _exec(
stack,
"python",
"-c",
"import sqlite3;print(sqlite3.connect('/data/photo_pipeline.db')"
".execute('select version_num from alembic_version').fetchone()[0])",
).strip()
# ── the restart journey ──────────────────────────────────────────────────────
@needs_compose
def test_killing_both_containers_mid_job_resumes_without_doing_anything_twice(
image, tmp_path_factory
):
"""`docker kill` is the honest restart: no grace period, no orderly stop, no
chance for either process to write a tidy final state."""
library = temporary_library({"burst": BURST}, prefix="us0805-restart-")
env_file = write_env_file(tmp_path_factory.mktemp("restart") / "gate.env", SECRET)
stack = Stack(library, env_file, project=f"{PROJECT}-restart", image=image, secret=SECRET)
try:
stack.down()
stack.up()
stack.wait_until_ready()
with closing(stack.client()) as client:
client.post("/inventory/scan").raise_for_status()
assets = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
assert len(assets) == BURST
for item in client.get("/safety/queue", params={"limit": 100}).json()["items"]:
client.post(
"/safety/decisions", json={"asset_id": item["asset_id"], "decision": "sfw"}
).raise_for_status()
job = client.post("/analysis/jobs").json()
progress = _wait_for_progress(client, job["id"])
assert 0 < progress["done"] < progress["total"], progress
stack.compose("kill", "api", "worker")
stack.up()
stack.wait_until_ready()
with closing(stack.client()) as client:
finished = await_job(client, job["id"])
assert finished["progress"]["done"] == BURST, finished
results = [
client.get(f"/analysis/results/{asset['id']}").json() for asset in assets
]
# Exactly one stored result per asset: at-least-once execution, idempotent
# recovery — a retried item overwrites its own attempt, it does not add one.
assert len(results) == BURST
assert all(result["description"] for result in results)
# And the side effect nobody can take back — the call to the provider — happened
# again only for whatever was in flight when the containers died.
analysed = collections.Counter(_exec(stack, "cat", "/data/vision.log").split())
assert len(analysed) == BURST, "every photo was analysed, and only the library's"
assert max(analysed.values()) <= 2, dict(analysed)
assert sum(1 for count in analysed.values() if count > 1) <= 1, dict(analysed)
finally:
stack.down()
remove_library(library)
def _wait_for_progress(client: httpx.Client, job_id: str, *, timeout: float = 120) -> dict:
"""Wait until the job is provably under way but provably unfinished."""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
snapshot = client.get(f"/jobs/{job_id}").json()
progress = snapshot["progress"]
if progress["done"] and progress["done"] < progress["total"]:
return progress
if snapshot["state"] in ("succeeded", "failed"):
raise AssertionError(f"the job finished before it could be interrupted: {snapshot}")
time.sleep(0.05)
raise AssertionError(f"the job never started: {client.get(f'/jobs/{job_id}').json()}")
# ── the security gates ───────────────────────────────────────────────────────
@needs_compose
def test_the_deployed_instance_refuses_a_caller_without_a_session(journey):
with httpx.Client(base_url=f"{journey.base}/api/v1", timeout=30) as client:
for method, path in (("GET", "/workflow"), ("POST", "/inventory/scan")):
response = client.request(method, path)
assert response.status_code == 401, path
assert response.json()["error"]["code"] == "unauthenticated"
# A session still has to be paid for with the operator's secret.
assert client.get("/session", headers={"X-Access-Secret": "guessed"}).status_code == 401
# Readiness stays open: the orchestrator's health check holds no session.
assert client.get("/health/ready").status_code == 200
@needs_compose
def test_forged_forwarded_headers_cannot_smuggle_an_allowed_host_past_the_check(journey):
"""Behind a proxy the app believes ``X-Forwarded-*`` — but only from the proxy.
Nothing in this composition is a trusted proxy, so the claim is the client's."""
with httpx.Client(base_url=f"{journey.base}/api/v1", timeout=30) as client:
client.headers["X-CSRF-Token"] = (
client.get("/session", headers={"X-Access-Secret": SECRET}).json()["csrf_token"]
)
# The hostname this deployment is reached under is accepted.
assert client.get("/workflow", headers={"Host": HOSTNAME}).status_code == 200
forged = client.get(
"/workflow",
headers={"Host": "photos.evil.example", "X-Forwarded-Host": HOSTNAME},
)
assert forged.status_code == 403
assert forged.json()["error"]["code"] == "host_not_allowed"
# A forged protocol claim must not mark the session cookie as HTTPS-only
# either — that would strand the operator's real, plain-HTTP session.
bootstrap = httpx.get(
f"{journey.base}/api/v1/session",
headers={"X-Access-Secret": SECRET, "X-Forwarded-Proto": "https"},
timeout=30,
)
assert "secure" not in bootstrap.headers["set-cookie"].lower()
@needs_compose
def test_a_path_that_leaves_the_mounted_library_is_refused(journey):
"""The container's own filesystem is not the library. A symlink swapped under a
known asset is the sharpest version of the question, because the database still
points at a path inside the mount."""
with closing(journey.client()) as client:
asset = client.get("/inventory/assets", params={"limit": 200}).json()["items"][0]
original = journey.library / Path(asset["current_path"]).relative_to("/library")
kept = original.read_bytes()
original.unlink()
original.symlink_to("/etc/passwd")
try:
escaped = client.get(f"/assets/{asset['id']}/thumbnail", params={"size": 256})
finally:
original.unlink()
original.write_bytes(kept)
assert escaped.status_code == 403
assert escaped.json()["error"]["code"] == "path_not_allowed"
assert "root:" not in escaped.text
# And a root that names nothing mounted is refused before the process serves.
refused = journey.compose(
"run",
"--rm",
"--no-deps",
"--env",
"PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos",
"api",
"serve",
check=False,
)
assert refused.returncode == 5, refused.stdout + refused.stderr
@needs_compose
def test_no_secret_reaches_the_container_logs(journey):
with httpx.Client(base_url=f"{journey.base}/api/v1", timeout=30) as client:
client.get("/session", headers={"X-Access-Secret": "wrong-secret-attempt"})
client.get("/session", headers={"X-Access-Secret": SECRET})
logs = journey.logs()
assert SECRET not in logs
assert IMMICH_SENTINEL not in logs
assert "wrong-secret-attempt" not in logs, "a rejected secret is still a secret"
# The refusal itself is logged, so an operator can see the attempt.
assert "access secret rejected" in logs
@needs_compose
def test_the_evidence_of_this_run_names_the_stack_it_was_produced_from(journey):
"""A gate that cannot say what it ran against is an opinion. `docker compose ps`
is the record: one API, one worker, one completed migration."""
listing = [
json.loads(line)
for line in journey.compose("ps", "--all", "--format", "json").stdout.splitlines()
if line.strip()
]
services = {entry["Service"]: entry["State"] for entry in listing}
assert services["api"] == "running" and services["worker"] == "running"
assert services["migrate"] == "exited"
if __name__ == "__main__": # a quick way to run just this file
raise SystemExit(pytest.main([__file__, "-v", *sys.argv[1:]]))

View File

@@ -0,0 +1,299 @@
"""US08-03: the composition's contract, and the library-root check it depends on.
Bringing the stack up needs a Docker daemon and the network, which is what
``tests/e2e/test_compose_stack.py`` does. What can be checked without either is
checked here, because the parts that rot silently — a second writer that is only
prevented by convention, a data volume that stopped being the same volume for both
roles, migrations that stopped running first, a committed value in a file that must
carry none — are all readable from the files.
The startup refusal is the other half: in a container the configured library roots
must name the mount paths, and a mismatch has to fail before the lock is taken, not
at the first rename.
"""
from __future__ import annotations
import json
import os
import re
import stat
from pathlib import Path
import pytest
import yaml
from photo_pipeline import path_policy
from photo_pipeline.__main__ import main
from photo_pipeline.config import Config
REPO = Path(__file__).resolve().parents[2]
COMPOSE_FILE = REPO / "docker-compose.yml"
COMPOSE = yaml.safe_load(COMPOSE_FILE.read_text())
ENV_EXAMPLE = REPO / ".env.example"
SERVICES = COMPOSE["services"]
DATA_VOLUME = "data:/data"
def env_example_keys() -> list[str]:
"""The variables the example file declares, in file order."""
return [
line.split("=", 1)[0]
for line in ENV_EXAMPLE.read_text().splitlines()
if "=" in line and not line.lstrip().startswith("#")
]
# ── one API, one worker, one library, one volume ─────────────────────────────
def test_exactly_one_serving_and_one_working_container_from_the_same_image():
roles = {name: service["command"][0] for name, service in SERVICES.items()}
assert sorted(roles.values()) == ["migrate", "serve", "worker"]
assert [name for name, role in roles.items() if role == "serve"] == ["api"]
assert [name for name, role in roles.items() if role == "worker"] == ["worker"]
images = {service["image"] for service in SERVICES.values()}
assert len(images) == 1, "both roles must run the same build of the application"
assert "latest" not in images.pop()
# No `replicas`/`scale` key promising a second worker is fine; the lock decides.
assert not any("deploy" in service for service in SERVICES.values())
def test_both_roles_share_the_data_volume_so_the_lock_is_visible_to_both():
"""A second worker is refused by the library lock (US07-05) only if it can see it."""
for name, service in SERVICES.items():
assert DATA_VOLUME in service["volumes"], name
assert COMPOSE["volumes"]["data"]["driver"] == "local"
text = COMPOSE_FILE.read_text()
# The composition has to say why, because the failure is silent corruption.
assert "WAL" in text and re.search(r"NFS|SMB|network", text)
def test_the_library_is_a_bind_mount_whose_target_is_the_configured_root():
for name, service in SERVICES.items():
mounts = [volume for volume in service["volumes"] if volume != DATA_VOLUME]
assert len(mounts) == 1, name
source, target = re.match(r"^(\$\{.*?\}):(\$\{.*?\})$", mounts[0]).groups()
# An unset host path fails the composition rather than mounting something else.
assert source.startswith("${PHOTO_PIPELINE_LIBRARY_HOST_PATH:?")
# The container-side path and the configured root are one variable, so they
# cannot drift apart into a library that is mounted but not configured.
assert target.startswith("${PHOTO_PIPELINE_LIBRARY_ROOTS:?")
assert service["environment"]["PHOTO_PIPELINE_LIBRARY_ROOTS"] == (
"${PHOTO_PIPELINE_LIBRARY_ROOTS}"
)
assert service["environment"]["PHOTO_PIPELINE_DATA_DIR"] == "/data"
def test_migrations_run_to_completion_before_either_role_accepts_work():
"""`migrate` runs the backup-then-migrate path, and a failed upgrade exits
non-zero with its pre-migration backup intact — proven in
tests/integration/test_backup_recovery.py. What the composition adds is that
neither role starts until it succeeded."""
assert SERVICES["migrate"]["command"] == ["migrate"]
assert SERVICES["migrate"]["restart"] == "no", "a one-shot that retries is not a gate"
for role in ("api", "worker"):
assert SERVICES[role]["depends_on"] == {
"migrate": {"condition": "service_completed_successfully"}
}, role
def test_the_api_port_is_published_to_host_loopback_by_default():
published = SERVICES["api"]["ports"]
assert published == [
"${PHOTO_PIPELINE_PUBLISH_ADDRESS:-127.0.0.1}:${PHOTO_PIPELINE_PORT:-8000}:8000"
]
# Reachable from the host means reachable from elsewhere as far as the app is
# concerned, so the access secret stays mandatory (US08-01).
assert SERVICES["api"]["environment"]["PHOTO_PIPELINE_HOST"] == "0.0.0.0"
assert SERVICES["api"]["environment"]["PHOTO_PIPELINE_PORT"] == 8000
assert "PHOTO_PIPELINE_ACCESS_SECRET" not in SERVICES["api"]["environment"]
def test_containers_restart_by_themselves_and_stop_with_time_to_drain():
for role in ("api", "worker"):
assert SERVICES[role]["restart"] == "unless-stopped", role
assert SERVICES[role]["stop_grace_period"] == "30s", role
def test_the_containers_run_as_the_library_owner_and_never_as_root():
for name, service in SERVICES.items():
assert service["user"] == "${PHOTO_PIPELINE_UID:-1000}:${PHOTO_PIPELINE_GID:-1000}", name
assert service["build"]["args"]["UID"] == "${PHOTO_PIPELINE_UID:-1000}", name
# ── configuration comes from the environment, never from a committed file ────
def test_configuration_and_secrets_come_from_the_environment_only():
for name, service in SERVICES.items():
assert service["env_file"] == ["${PHOTO_PIPELINE_ENV_FILE:-.env}"], name
for key, value in service["environment"].items():
# Every value is either a variable reference or a property of the
# composition itself (the volume path, the container's own port).
composed = isinstance(value, int) or value in ("/data", "0.0.0.0")
assert composed or value.startswith("${"), (name, key, value)
assert not (REPO / ".env").is_file() or ".env" in (REPO / ".gitignore").read_text()
def test_the_example_file_lists_every_setting_and_carries_no_values():
declared = env_example_keys()
assert declared == sorted(set(declared), key=declared.index), "no variable twice"
for line in ENV_EXAMPLE.read_text().splitlines():
if "=" in line and not line.lstrip().startswith("#"):
assert line.endswith("="), f"a value in the example file: {line}"
expected = {f"PHOTO_PIPELINE_{name.upper()}" for name in Config.model_fields}
assert expected <= set(declared), sorted(expected - set(declared))
# And every variable the composition substitutes is documented there too.
substituted = set(re.findall(r"\$\{(PHOTO_PIPELINE_[A-Z_]+)", COMPOSE_FILE.read_text()))
assert substituted <= set(declared), sorted(substituted - set(declared))
def test_the_example_file_is_not_a_dotenv_that_could_be_loaded_by_accident():
"""`.env.example` must not be what `.env` is: no values means nothing to leak."""
assert ENV_EXAMPLE.name != ".env"
parsed = {k: v for k, v in _parse(ENV_EXAMPLE.read_text()).items() if v}
assert parsed == {}
def _parse(text: str) -> dict[str, str]:
from photo_pipeline.config import parse_env_file
return parse_env_file(text)
# ── the lock across container lifetimes ──────────────────────────────────────
def test_a_lock_left_by_a_container_that_is_gone_does_not_block_the_restart(
tmp_path, monkeypatch
):
"""A restarted container is a new hostname and a recycled pid 1, so the record
in the lock file proves nothing; the kernel's flock does (US08-03)."""
from photo_pipeline.services.app_lock import LibraryLock
config = Config(data_dir=tmp_path / "data", library_roots=(tmp_path,))
(tmp_path / "data").mkdir()
(tmp_path / "data" / "worker.lock.json").write_text(
json.dumps(
{
"lock_version": 1,
"role": "worker",
"pid": 1, # pid 1 of a container that no longer exists
"host": "3f2a1b9c4d5e", # its hostname was its container id
"started_at": "2026-01-01T00:00:00+00:00",
"library_roots": ["/library"],
}
)
)
monkeypatch.setattr(path_policy, "in_container", lambda: True)
taken = LibraryLock(config, "worker").acquire()
assert taken.pid == os.getpid(), "the worker must come back after a restart"
def test_a_second_worker_is_still_refused_while_the_first_holds_the_lock(tmp_path, monkeypatch):
"""The other half: the same flock refuses a concurrent second writer, whether it
is a process or another container of the same composition."""
from photo_pipeline.services.app_lock import LibraryLock, LockHeld
config = Config(data_dir=tmp_path / "data", library_roots=(tmp_path,))
monkeypatch.setattr(path_policy, "in_container", lambda: True)
first = LibraryLock(config, "worker")
first.acquire()
with pytest.raises(LockHeld, match="worker is already running"):
LibraryLock(config, "worker").acquire()
first.release()
LibraryLock(config, "worker").acquire() # free again
# ── the startup check the mount depends on ───────────────────────────────────
def test_configured_roots_that_are_mounted_and_writable_are_accepted(tmp_path):
assert path_policy.roots_refusal([tmp_path]) is None
assert path_policy.roots_refusal([]) is None, "no roots is a configuration, not a fault"
def test_an_unmounted_library_root_is_refused_by_name(tmp_path):
refusal = path_policy.roots_refusal([tmp_path / "srv" / "photos"])
assert refusal is not None
assert "does not exist" in refusal and "PHOTO_PIPELINE_LIBRARY_ROOTS" in refusal
def test_a_root_that_is_not_a_directory_or_not_readable_is_refused(tmp_path):
a_file = tmp_path / "photos.txt"
a_file.write_text("not a library")
assert "not a directory" in path_policy.roots_refusal([a_file])
unreadable = tmp_path / "unreadable"
unreadable.mkdir()
unreadable.chmod(0o000)
try:
refusal = path_policy.roots_refusal([unreadable])
finally:
unreadable.chmod(0o755)
if os.getuid() != 0: # root ignores the mode, and CI may well be root
assert refusal is not None and "not readable" in refusal
def test_an_unwritable_root_is_not_refused_here(tmp_path):
"""A bind mount's ownership is virtualised on macOS and Windows, so os.access
would refuse a working deployment. The real errno at the first rename is at
least true; this check is about the mount, not the mode."""
read_only = tmp_path / "read-only"
read_only.mkdir()
read_only.chmod(stat.S_IRUSR | stat.S_IXUSR)
try:
assert path_policy.roots_refusal([read_only]) is None
finally:
read_only.chmod(0o755)
def test_in_a_container_a_root_that_was_never_mounted_is_refused(tmp_path, monkeypatch):
"""The container-only failure: the path exists, but it belongs to the image."""
unmounted = tmp_path / "library"
(unmounted / "album").mkdir(parents=True)
refusal = path_policy.roots_refusal([unmounted], require_mount=True)
assert refusal is not None and "not on a mounted filesystem" in refusal
# A bind mount is a mount point, and a root *below* one is mounted too: a
# deployment may mount /srv and configure /srv/photos.
monkeypatch.setattr(os.path, "ismount", lambda path: Path(path) == unmounted.resolve())
assert path_policy.roots_refusal([unmounted], require_mount=True) is None
assert path_policy.roots_refusal([unmounted / "album"], require_mount=True) is None
@pytest.mark.parametrize("role", ["serve", "worker"])
def test_a_root_mismatch_refuses_at_startup_before_any_lock_is_taken(
role, tmp_path, monkeypatch, capsys
):
data = tmp_path / "data"
monkeypatch.setenv("PHOTO_PIPELINE_DATA_DIR", str(data))
monkeypatch.setenv("PHOTO_PIPELINE_LIBRARY_ROOTS", str(tmp_path / "not-mounted"))
monkeypatch.setattr(path_policy, "in_container", lambda: True)
assert main([role]) == 5
assert "does not exist" in capsys.readouterr().err
assert not list(data.glob("*.lock.json")), "nothing started, so nothing is locked"
@pytest.mark.parametrize("role", ["serve", "worker"])
def test_an_unmounted_root_on_a_host_is_not_a_reason_to_refuse(role, tmp_path, monkeypatch):
"""An archive medium that is not plugged in is a Tuesday, not a misconfiguration:
refusing would take the offline half of the library away with it (concept §9)."""
monkeypatch.setenv("PHOTO_PIPELINE_DATA_DIR", str(tmp_path / "data"))
monkeypatch.setenv("PHOTO_PIPELINE_LIBRARY_ROOTS", str(tmp_path / "not-mounted"))
monkeypatch.setenv("PHOTO_PIPELINE_ACCESS_SECRET", "unused-on-loopback")
monkeypatch.setattr(path_policy, "in_container", lambda: False)
# Reaching the lock is the proof: that is the next thing either role does, and
# stopping there keeps the test out of a uvicorn/worker loop.
monkeypatch.setattr("photo_pipeline.__main__._acquire", lambda *_, **__: 99)
assert main([role]) == 99

View File

@@ -0,0 +1,147 @@
"""US08-05: the container acceptance gate's own contract, checked without a daemon.
The gate itself needs Docker, a browser, and several minutes; running it from inside
the suite would be a fork bomb with better manners. What is checkable offline is what
makes it a *gate* rather than a long test run:
* it selects the container journeys by marker, so adding one is enough to put it in
front of a deploy;
* it accepts no skip at all — every reason a check would skip here (no daemon, no
compose plugin, no browser) means the deployed runtime was not proven;
* it retains checksummed evidence per run;
* and CI runs it on `main`, which is what the publish step waits for.
The running proof is ``tests/e2e/test_phase_h_container.py``.
"""
from __future__ import annotations
import json
import re
from pathlib import Path
import yaml
from photo_pipeline.config import Config
from photo_pipeline.services import release
REPO = Path(__file__).resolve().parents[2]
SUITE = REPO / "tests" / "e2e" / "test_phase_h_container.py"
TEST_WORKFLOW = yaml.safe_load((REPO / ".gitea" / "workflows" / "test.yml").read_text())
def _config(tmp_path) -> Config:
for name in ("data", "lib"):
(tmp_path / name).mkdir(exist_ok=True)
return Config.from_env(
{
"PHOTO_PIPELINE_DATA_DIR": str(tmp_path / "data"),
"PHOTO_PIPELINE_LIBRARY_ROOTS": str(tmp_path / "lib"),
}
)
# ── what the gate runs ───────────────────────────────────────────────────────
def test_the_gate_selects_the_container_journeys_by_their_marker():
assert release.CONTAINER_STAGES == (("container", ("tests/e2e", "-m", "phase_h")),)
declared = [line for line in (REPO / "pyproject.toml").read_text().splitlines()
if line.strip().startswith('"phase_h:')]
assert declared, "an unregistered marker selects nothing and fails no gate"
def test_every_journey_in_the_suite_carries_the_marker():
"""A test in that file without the marker is a check the gate never runs."""
source = SUITE.read_text()
assert "pytestmark = [pytest.mark.phase_h, pytest.mark.container]" in source
journeys = re.findall(r"^def (test_[a-z_]+)", source, re.MULTILINE)
assert len(journeys) >= 4, journeys
for required in ("browser", "upgrade", "kill", "secret"):
assert any(required in name for name in journeys), (required, journeys)
# ── no skip is an environment limit here ─────────────────────────────────────
def test_the_gate_accepts_no_skipped_check_at_all():
assert release.CONTAINER_ALLOWED_SKIP_REASONS == ()
docker_missing = release.StageResult(
"container", [], 0, 0.1, "1 skipped", ["SKIPPED [1] x.py:1: no Docker daemon available"]
)
# The release gate tolerates exactly this one, because the image definition is
# still checked offline. The container gate cannot: it is the deployment's proof.
assert release.unexpected_skips([docker_missing]) == []
assert release.unexpected_skips(
[docker_missing], release.CONTAINER_ALLOWED_SKIP_REASONS
) == ["SKIPPED [1] x.py:1: no Docker daemon available"]
def test_a_skipped_check_fails_the_run_and_the_evidence_says_so(tmp_path):
skipping = tmp_path / "test_skipping.py"
skipping.write_text(
"import pytest\n\n"
"def test_x():\n"
" pytest.skip('no Docker daemon available')\n"
)
evidence = tmp_path / "evidence"
report = release.run_gate(
_config(tmp_path),
output=evidence,
stages=(("container", (str(skipping),)),),
allowed_skips=release.CONTAINER_ALLOWED_SKIP_REASONS,
)
assert report["ok"] is False
assert report["failures"] == [], "the stage passed; the skip is what fails the gate"
assert report["unexpected_skips"], report
written = json.loads((evidence / release.REPORT_NAME).read_text())
assert written["ok"] is False
assert (evidence / "logs" / "container.log").exists()
for line in (evidence / release.CHECKSUMS_NAME).read_text().splitlines():
digest, name = line.split(" ", 1)
assert release.sha256_file(evidence / name) == digest
def test_the_same_run_passes_when_nothing_skips(tmp_path):
passing = tmp_path / "test_passing.py"
passing.write_text("def test_x():\n assert True\n")
report = release.run_gate(
_config(tmp_path),
output=tmp_path / "evidence",
stages=(("container", (str(passing),)),),
allowed_skips=release.CONTAINER_ALLOWED_SKIP_REASONS,
)
assert report["ok"] is True and report["unexpected_skips"] == []
assert report["revision"], "the evidence must say which commit it covers"
# ── the command, and CI ──────────────────────────────────────────────────────
def test_the_command_is_documented_and_wired():
from photo_pipeline.__main__ import main # noqa: F401 (import proves it loads)
assert "container-gate" in (REPO / "photo_pipeline" / "__main__.py").read_text()
assert "container-gate" in (REPO / "README.md").read_text()
def test_ci_runs_the_gate_on_main_and_keeps_its_evidence():
job = TEST_WORKFLOW["jobs"]["container"]
assert job["if"] == "gitea.event_name == 'push'", "pull requests have nothing to upgrade from"
script = "\n".join(step["run"] for step in job["steps"] if "run" in step)
assert "photo_pipeline container-gate" in script
evidence = next(step for step in job["steps"] if "upload-artifact" in str(step.get("uses")))
assert evidence["if"] == "always()", "a failed gate's logs are the ones worth keeping"
assert evidence["with"]["path"] == "gate-evidence"
def test_the_upgrade_journey_can_still_reach_the_previous_version():
"""Without depth, `git archive HEAD~1` has nothing to build."""
checkout = next(
step for step in TEST_WORKFLOW["jobs"]["container"]["steps"] if "checkout" in str(step.get("uses"))
)
assert checkout["with"]["fetch-depth"] >= 2

View File

@@ -0,0 +1,210 @@
"""US08-04: the deployment pipeline's contract, read from the workflow files.
A CI/CD pipeline is the one piece of this repository whose failure mode is silence.
A renamed workflow, a renamed job, a trigger that quietly stopped matching, and the
suite is no longer between `main` and the registry — with nothing red anywhere to say
so. So the link is asserted from both ends here: the deploy workflow waits for the
test workflow *by name*, and that name is read out of the test workflow itself.
The same applies to the commands: `work_item/.work-item.yml` is where the required
suites are configured, and the test workflow has to run those, not a copy of them that
was true once.
Nothing here talks to Gitea, the registry, or Portainer. What is checkable offline is
what the files promise; whether the runner honours them is the deployment's own
evidence (US08-05).
"""
from __future__ import annotations
import re
from pathlib import Path
import yaml
REPO = Path(__file__).resolve().parents[2]
WORKFLOWS = REPO / ".gitea" / "workflows"
TEST_FILE = WORKFLOWS / "test.yml"
DEPLOY_FILE = WORKFLOWS / "deploy.yml"
TEST_WORKFLOW = yaml.safe_load(TEST_FILE.read_text())
DEPLOY_WORKFLOW = yaml.safe_load(DEPLOY_FILE.read_text())
WORK_ITEM = yaml.safe_load((REPO / "work_item" / ".work-item.yml").read_text())
REGISTRY = "git.domverse-berlin.eu"
def triggers(workflow: dict) -> dict:
"""The `on:` block. PyYAML resolves the bare key `on` to ``True``."""
return workflow[True] if True in workflow else workflow["on"]
def steps(workflow: dict, job: str) -> list[dict]:
return workflow["jobs"][job]["steps"]
def run_script(workflow: dict, job: str) -> str:
"""Every `run:` in a job, as one blob — what the runner would execute."""
return "\n".join(step["run"] for step in steps(workflow, job) if "run" in step)
# ── the gate ─────────────────────────────────────────────────────────────────
def test_the_test_workflow_runs_on_pull_requests_and_on_main():
on = triggers(TEST_WORKFLOW)
assert "pull_request" in on
assert on["push"]["branches"] == ["main"]
def test_the_test_workflow_runs_the_configured_required_suites():
required = WORK_ITEM["workflow"]["required_tests"]
assert required, "the helper's required tests are what CI exists to run"
script = run_script(TEST_WORKFLOW, "suites")
for command in required:
assert command in script, f"CI does not run the configured suite: {command}"
def test_the_deploy_workflow_waits_for_the_test_workflow_by_its_real_name():
# Renaming either side without the other breaks this, which is the point: an
# unmatched `workflow_run` filter does not fail, it simply never fires.
awaited = triggers(DEPLOY_WORKFLOW)["workflow_run"]
assert awaited["workflows"] == [TEST_WORKFLOW["name"]]
assert awaited["types"] == ["completed"]
assert awaited["branches"] == ["main"], "only main is deployable"
def test_publishing_requires_the_test_run_to_have_succeeded_and_not_merely_finished():
condition = DEPLOY_WORKFLOW["jobs"]["publish"]["if"]
assert "workflow_run.conclusion == 'success'" in condition
# `completed` includes failure and cancellation; the conclusion check is the gate.
assert "gitea.event_name == 'workflow_run'" in condition
def test_the_deploy_workflow_can_also_be_dispatched_by_hand():
assert "workflow_dispatch" in triggers(DEPLOY_WORKFLOW)
# ── what is published ────────────────────────────────────────────────────────
def test_the_image_is_published_under_this_project_s_own_path():
slug = WORK_ITEM["repository"]["slug"]
assert DEPLOY_WORKFLOW["env"]["IMAGE"] == f"{REGISTRY}/{slug}"
def test_both_latest_and_the_commit_are_pushed_so_a_rollback_is_a_tag_change():
script = run_script(DEPLOY_WORKFLOW, "publish")
assert 'docker push "$IMAGE:$SHA"' in script
assert 'docker push "$IMAGE:latest"' in script
# A `latest` without its commit tag would be a version that cannot be rolled back to.
assert script.index('docker push "$IMAGE:$SHA"') < script.index('docker push "$IMAGE:latest"')
def test_the_published_commit_is_the_one_that_was_tested():
assert DEPLOY_WORKFLOW["env"]["SHA"].startswith("${{ gitea.event.workflow_run.head_sha")
checkout = steps(DEPLOY_WORKFLOW, "publish")[0]
assert checkout["with"]["ref"] == "${{ env.SHA }}"
# ── secrets ──────────────────────────────────────────────────────────────────
def test_registry_credentials_and_the_webhook_come_from_repository_secrets():
script = run_script(DEPLOY_WORKFLOW, "publish")
for secret in ("REGISTRY_USER", "REGISTRY_TOKEN", "PORTAINER_WEBHOOK_URL"):
assert f"secrets.{secret}" in script
def test_no_runtime_secret_is_named_by_the_pipeline_at_all():
# The vision key, the Immich key, and the access secret belong to the Portainer
# stack. A workflow that mentions one is a workflow that could carry one.
text = TEST_FILE.read_text() + DEPLOY_FILE.read_text()
for runtime_secret in (
"PHOTO_PIPELINE_VISION_API_KEY",
"PHOTO_PIPELINE_IMMICH_API_KEY",
"PHOTO_PIPELINE_ACCESS_SECRET",
):
assert runtime_secret not in text
def test_the_workflows_carry_no_credential_values():
text = TEST_FILE.read_text() + DEPLOY_FILE.read_text()
for line in text.splitlines():
if re.search(r"(?i)(token|secret|password|api[_-]?key)\s*[:=]", line):
assert "${{ secrets." in line or line.lstrip().startswith("#"), line
def test_the_registry_password_is_never_an_argument():
# An argument is visible in the runner's process list and in `docker login`'s own
# warning; stdin is not.
script = run_script(DEPLOY_WORKFLOW, "publish") + run_script(DEPLOY_WORKFLOW, "dry-run")
assert "--password-stdin" in script
assert "--password " not in script
# ── redeploy, pruning, and overlap ───────────────────────────────────────────
def test_a_failed_webhook_call_fails_the_workflow():
webhook = next(
step
for step in steps(DEPLOY_WORKFLOW, "publish")
if "PORTAINER_WEBHOOK_URL" in step.get("run", "")
)
assert "--fail" in webhook["run"], "curl exits 0 on an HTTP 500 without it"
assert "-k" not in webhook["run"].split(), "the redeploy call verifies TLS"
def test_pruning_removes_dangling_layers_and_never_published_tags():
for job in ("publish", "dry-run"):
script = run_script(DEPLOY_WORKFLOW, job)
assert "docker image prune -f" in script
assert "prune -a" not in script and "--all" not in script
def test_two_deploys_of_different_commits_cannot_overlap():
guard = DEPLOY_WORKFLOW["concurrency"]
assert "sha" not in guard["group"].lower(), "a per-commit group guards nothing"
assert guard["cancel-in-progress"] is False, "a cancelled push leaves half a tag set"
# ── the dry run ──────────────────────────────────────────────────────────────
def test_the_dry_run_is_manual_only_and_defaults_to_on():
condition = DEPLOY_WORKFLOW["jobs"]["dry-run"]["if"]
assert condition == "gitea.event_name == 'workflow_dispatch' && inputs.dry_run"
dispatch = triggers(DEPLOY_WORKFLOW)["workflow_dispatch"]
assert dispatch["inputs"]["dry_run"]["default"] is True
assert dispatch["inputs"]["dry_run"]["type"] == "boolean"
def test_the_dry_run_publishes_a_scratch_tag_and_nothing_else():
script = run_script(DEPLOY_WORKFLOW, "dry-run")
assert 'docker push "$IMAGE:scratch-$SHA"' in script
assert ":latest" not in script, "a dry run that moves latest is a deploy"
assert "PORTAINER_WEBHOOK_URL" not in script, "a dry run never redeploys the stack"
def test_a_manual_deploy_is_the_deliberate_choice_not_the_default():
publish = DEPLOY_WORKFLOW["jobs"]["publish"]["if"]
assert "inputs.dry_run == false" in publish
# ── the files themselves ─────────────────────────────────────────────────────
def test_every_workflow_file_parses_and_declares_a_name_a_trigger_and_a_job():
files = sorted(WORKFLOWS.glob("*.yml")) + sorted(WORKFLOWS.glob("*.yaml"))
assert {path.name for path in files} == {"test.yml", "deploy.yml"}
for path in files:
workflow = yaml.safe_load(path.read_text())
assert workflow["name"], path
assert triggers(workflow), path
assert workflow["jobs"], path
for job_name, job in workflow["jobs"].items():
assert job["runs-on"], f"{path}:{job_name}"
assert job["steps"], f"{path}:{job_name}"
for step in job["steps"]:
assert step.get("name"), f"{path}:{job_name} has an unnamed step"

View File

@@ -182,12 +182,19 @@
"US08-02": [
"tests/integration/test_container_image.py",
"tests/e2e/test_container_runtime.py"
],
"US08-03": [
"tests/integration/test_compose_runtime.py",
"tests/e2e/test_compose_stack.py"
],
"US08-04": [
"tests/integration/test_deploy_workflows.py"
],
"US08-05": [
"tests/integration/test_container_gate.py",
"tests/e2e/test_phase_h_container.py"
]
},
"planned": [
"US08-03",
"US08-04",
"US08-05"
],
"planned": [],
"_planned_comment": "Accepted backlog stories that are not implemented yet. The release gate (US07-07) requires every story file to be either mapped to tests or listed here, so an unimplemented story is a visible decision rather than a hole in the matrix."
}