Compare commits
3 Commits
us/US08-03
...
us/US08-05
| Author | SHA1 | Date | |
|---|---|---|---|
| 6282123780 | |||
| a19dd280c9 | |||
| 1632544132 |
85
.env.example
Normal file
85
.env.example
Normal file
@@ -0,0 +1,85 @@
|
|||||||
|
# Every setting the application and its composition read, with no values in it.
|
||||||
|
# Copy to `.env`, fill in what you need, and keep that copy out of git (it is
|
||||||
|
# gitignored, and the work-item safety checks refuse to stage it).
|
||||||
|
#
|
||||||
|
# cp .env.example .env
|
||||||
|
#
|
||||||
|
# Empty means "use the default noted beside it". Anything already exported in the
|
||||||
|
# shell wins over this file, both for the app and for `docker compose`.
|
||||||
|
|
||||||
|
# ── the composition (host side; read by docker-compose.yml only) ─────────────
|
||||||
|
# The photo library on this host. Bind-mounted at PHOTO_PIPELINE_LIBRARY_ROOTS.
|
||||||
|
PHOTO_PIPELINE_LIBRARY_HOST_PATH=
|
||||||
|
# Which image to run. Default: photo-pipeline:dev (what `up --build` builds).
|
||||||
|
PHOTO_PIPELINE_IMAGE=
|
||||||
|
# Must be the owner of the library above: what the containers rename and rewrite
|
||||||
|
# keeps this ownership. Default: 1000 / 1000.
|
||||||
|
PHOTO_PIPELINE_UID=
|
||||||
|
PHOTO_PIPELINE_GID=
|
||||||
|
# Host address the API port is published on. Default: 127.0.0.1. Anything else
|
||||||
|
# exposes the app beyond this machine — then ALLOWED_HOSTS and ACCESS_SECRET below
|
||||||
|
# are what stand in for the loopback boundary (US08-01).
|
||||||
|
PHOTO_PIPELINE_PUBLISH_ADDRESS=
|
||||||
|
# This file's own path, if it is not ./.env. Default: .env.
|
||||||
|
PHOTO_PIPELINE_ENV_FILE=
|
||||||
|
|
||||||
|
# ── library and data ────────────────────────────────────────────────────────
|
||||||
|
# os.pathsep-separated. In a container these are the *container-side* mount paths,
|
||||||
|
# and `serve`/`worker` refuse to start when they are not mounted.
|
||||||
|
PHOTO_PIPELINE_LIBRARY_ROOTS=
|
||||||
|
# Database, WAL, thumbnail cache, journals, backups. Default: data (the container
|
||||||
|
# sets /data, which is the persistent volume; a network mount is unsupported).
|
||||||
|
PHOTO_PIPELINE_DATA_DIR=
|
||||||
|
# Database file, if it should not live in the data directory. Default:
|
||||||
|
# <data dir>/photo_pipeline.db.
|
||||||
|
PHOTO_PIPELINE_DB_PATH=
|
||||||
|
|
||||||
|
# ── serving ─────────────────────────────────────────────────────────────────
|
||||||
|
# Bind address. Default: 127.0.0.1. The composition sets 0.0.0.0 inside the
|
||||||
|
# container and publishes to loopback on the host instead.
|
||||||
|
PHOTO_PIPELINE_HOST=
|
||||||
|
# Under the composition this is the *published* host port; the container serves
|
||||||
|
# 8000. Default: 8000.
|
||||||
|
PHOTO_PIPELINE_PORT=
|
||||||
|
# Comma-separated hostnames the app answers to besides loopback. Empty means
|
||||||
|
# loopback only. Naming one makes the access secret mandatory.
|
||||||
|
PHOTO_PIPELINE_ALLOWED_HOSTS=
|
||||||
|
# Traded for the session cookie at GET /api/v1/session via X-Access-Secret.
|
||||||
|
# Required as soon as the app is reachable from anywhere but loopback. Generate
|
||||||
|
# one with: python -c 'import secrets; print(secrets.token_urlsafe(32))'
|
||||||
|
PHOTO_PIPELINE_ACCESS_SECRET=
|
||||||
|
# Comma-separated peer addresses whose X-Forwarded-Proto/-Host may be believed.
|
||||||
|
# Only the reverse proxy's address belongs here. Default: none.
|
||||||
|
PHOTO_PIPELINE_TRUSTED_PROXIES=
|
||||||
|
# Largest request body accepted, in bytes. Default: 1048576.
|
||||||
|
PHOTO_PIPELINE_MAX_REQUEST_BYTES=
|
||||||
|
|
||||||
|
# ── logging ─────────────────────────────────────────────────────────────────
|
||||||
|
# Default: INFO.
|
||||||
|
PHOTO_PIPELINE_LOG_LEVEL=
|
||||||
|
# json or text. Default: json.
|
||||||
|
PHOTO_PIPELINE_LOG_FORMAT=
|
||||||
|
|
||||||
|
# ── limits ──────────────────────────────────────────────────────────────────
|
||||||
|
# Thumbnail cache quota in bytes. Default: 500000000.
|
||||||
|
PHOTO_PIPELINE_THUMBNAIL_CACHE_QUOTA_BYTES=
|
||||||
|
# Refuse to decode images larger than this many pixels. Default: 100000000.
|
||||||
|
PHOTO_PIPELINE_THUMBNAIL_MAX_PIXELS=
|
||||||
|
# Free space an archive destination must keep beyond the transfer. Default:
|
||||||
|
# 1000000000.
|
||||||
|
PHOTO_PIPELINE_ARCHIVE_FREE_SPACE_RESERVE_BYTES=
|
||||||
|
|
||||||
|
# ── safety gate ─────────────────────────────────────────────────────────────
|
||||||
|
# true refuses every mutating request until a read-only dry run of this library has
|
||||||
|
# been produced and approved (US07-07). Default: false. Turn it on before pointing
|
||||||
|
# the app at photos that cannot be replaced.
|
||||||
|
PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL=
|
||||||
|
|
||||||
|
# ── external services ───────────────────────────────────────────────────────
|
||||||
|
# Vision provider key for the analysis stage. Without it, analysis cannot run.
|
||||||
|
PHOTO_PIPELINE_VISION_API_KEY=
|
||||||
|
# Immich server and its API key, for the upload stage.
|
||||||
|
PHOTO_PIPELINE_IMMICH_SERVER_URL=
|
||||||
|
PHOTO_PIPELINE_IMMICH_API_KEY=
|
||||||
|
# Uploader binary. Default: immich-go (on PATH; pinned inside the image).
|
||||||
|
PHOTO_PIPELINE_IMMICH_GO_BINARY=
|
||||||
99
.gitea/workflows/deploy.yml
Normal file
99
.gitea/workflows/deploy.yml
Normal file
@@ -0,0 +1,99 @@
|
|||||||
|
# Publish and redeploy (US08-04). Adapted from the crowdsec-admin deploy workflow:
|
||||||
|
# build, log in to the Gitea registry, push, trigger the Portainer webhook, prune.
|
||||||
|
# The difference is the gate — this project has a required suite that must not be
|
||||||
|
# skipped, so publishing happens only after `Test` (`.gitea/workflows/test.yml`)
|
||||||
|
# succeeded on `main`, never on the push itself.
|
||||||
|
#
|
||||||
|
# The stack is managed by Portainer from git (`docker-compose.yml`), and the runtime
|
||||||
|
# secrets it needs — vision key, Immich key, access secret — live in the Portainer
|
||||||
|
# stack's environment. They are deliberately not repository secrets and are not in the
|
||||||
|
# image: rotation stays in one place, and a repository read never discloses them.
|
||||||
|
#
|
||||||
|
# Repository secrets required:
|
||||||
|
# REGISTRY_USER user with write:package on the registry
|
||||||
|
# REGISTRY_TOKEN that user's token
|
||||||
|
# PORTAINER_WEBHOOK_URL POST URL from the stack's auto-update setting
|
||||||
|
|
||||||
|
name: Deploy
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_run:
|
||||||
|
workflows:
|
||||||
|
- Test
|
||||||
|
types:
|
||||||
|
- completed
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
dry_run:
|
||||||
|
description: Build and push a scratch tag only — leave `latest` and the running stack alone
|
||||||
|
type: boolean
|
||||||
|
default: true
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
# Deliberately not keyed by commit: the point is that two deploys of *different*
|
||||||
|
# commits cannot overlap. Queued, not cancelled — a half-pushed tag set is worse
|
||||||
|
# than a late one.
|
||||||
|
group: deploy
|
||||||
|
cancel-in-progress: false
|
||||||
|
|
||||||
|
env:
|
||||||
|
IMAGE: git.domverse-berlin.eu/domverse/photoanalyzer
|
||||||
|
# The commit that was tested, not whatever `main` points at by the time this starts.
|
||||||
|
SHA: ${{ gitea.event.workflow_run.head_sha || gitea.sha }}
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
publish:
|
||||||
|
# A completed `Test` run is not a passing one.
|
||||||
|
if: >-
|
||||||
|
(gitea.event_name == 'workflow_run' && gitea.event.workflow_run.conclusion == 'success')
|
||||||
|
|| (gitea.event_name == 'workflow_dispatch' && inputs.dry_run == false)
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
ref: ${{ env.SHA }}
|
||||||
|
|
||||||
|
- name: Log in to the Gitea registry
|
||||||
|
run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login git.domverse-berlin.eu -u "${{ secrets.REGISTRY_USER }}" --password-stdin
|
||||||
|
|
||||||
|
- name: Build and push
|
||||||
|
# The commit tag is pushed first, so a `latest` that exists is always a tag
|
||||||
|
# that also exists under its own commit — which is what makes a rollback a
|
||||||
|
# tag change rather than a rebuild.
|
||||||
|
run: |
|
||||||
|
docker build -t "$IMAGE:$SHA" -t "$IMAGE:latest" .
|
||||||
|
docker push "$IMAGE:$SHA"
|
||||||
|
docker push "$IMAGE:latest"
|
||||||
|
|
||||||
|
- name: Trigger the Portainer redeploy
|
||||||
|
# --fail turns an HTTP error into a non-zero exit: a redeploy that did not
|
||||||
|
# happen must not read as a green deploy.
|
||||||
|
run: curl -sS --fail -X POST "${{ secrets.PORTAINER_WEBHOOK_URL }}"
|
||||||
|
|
||||||
|
- name: Prune dangling images
|
||||||
|
# Untagged layers only. Published tags are the rollback history; `-a` would
|
||||||
|
# delete exactly the images this workflow exists to keep.
|
||||||
|
run: docker image prune -f
|
||||||
|
|
||||||
|
dry-run:
|
||||||
|
# Manual only, and the default: prove the image still builds and the registry
|
||||||
|
# still accepts it without moving `latest` or touching the running stack.
|
||||||
|
if: gitea.event_name == 'workflow_dispatch' && inputs.dry_run
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Log in to the Gitea registry
|
||||||
|
run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login git.domverse-berlin.eu -u "${{ secrets.REGISTRY_USER }}" --password-stdin
|
||||||
|
|
||||||
|
- name: Build and push a scratch tag
|
||||||
|
run: |
|
||||||
|
docker build -t "$IMAGE:scratch-$SHA" .
|
||||||
|
docker push "$IMAGE:scratch-$SHA"
|
||||||
|
|
||||||
|
- name: Prune dangling images
|
||||||
|
run: docker image prune -f
|
||||||
103
.gitea/workflows/test.yml
Normal file
103
.gitea/workflows/test.yml
Normal file
@@ -0,0 +1,103 @@
|
|||||||
|
# The test gate. Deploy waits for this workflow by name (`.gitea/workflows/deploy.yml`
|
||||||
|
# triggers on `workflow_run: [Test]`), so renaming it here without renaming it there
|
||||||
|
# would leave `main` publishing without a suite. `tests/integration/test_deploy_workflows.py`
|
||||||
|
# asserts both halves of that link, and that the commands below are still the ones
|
||||||
|
# configured in `work_item/.work-item.yml`.
|
||||||
|
|
||||||
|
name: Test
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
# One run per branch; a newer push makes the older run's answer irrelevant.
|
||||||
|
group: test-${{ gitea.ref }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
suites:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Install exiftool
|
||||||
|
# The EXIF checkpoints are the safety invariant of every metadata stage; a run
|
||||||
|
# without exiftool would skip them for a reason the release gate accepts.
|
||||||
|
run: |
|
||||||
|
SUDO=
|
||||||
|
if [ "$(id -u)" -ne 0 ]; then SUDO=sudo; fi
|
||||||
|
$SUDO apt-get update
|
||||||
|
$SUDO apt-get install -y --no-install-recommends libimage-exiftool-perl
|
||||||
|
|
||||||
|
- name: Set up Python
|
||||||
|
uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: '3.12'
|
||||||
|
|
||||||
|
- name: Install the application and its test dependencies
|
||||||
|
run: |
|
||||||
|
python -m pip install --upgrade pip
|
||||||
|
python -m pip install -e '.[test]'
|
||||||
|
python -m playwright install --with-deps chromium
|
||||||
|
|
||||||
|
- name: Helper suite
|
||||||
|
run: work_item/scripts/python -m unittest discover -s work_item/tests -v
|
||||||
|
|
||||||
|
- name: Application suite
|
||||||
|
run: work_item/scripts/python -m pytest tests -q
|
||||||
|
|
||||||
|
# The deployed container, verified the way the host application is (US08-05). It
|
||||||
|
# runs on `main` only — a pull request has nothing published to upgrade *from*, and
|
||||||
|
# building two images per push would pay for that on every commit. Deploy waits for
|
||||||
|
# this whole workflow, so a red container gate is a deploy that does not happen.
|
||||||
|
container:
|
||||||
|
if: gitea.event_name == 'push'
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
# The upgrade journey builds the previous commit's tree when no published
|
||||||
|
# image is named, so the history has to be there.
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
fetch-depth: 2
|
||||||
|
|
||||||
|
- name: Set up Python
|
||||||
|
uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: '3.12'
|
||||||
|
|
||||||
|
- name: Install the application and its test dependencies
|
||||||
|
run: |
|
||||||
|
python -m pip install --upgrade pip
|
||||||
|
python -m pip install -e '.[test]'
|
||||||
|
python -m playwright install --with-deps chromium
|
||||||
|
|
||||||
|
- name: Container acceptance gate
|
||||||
|
# One command: it builds the image, provisions the composition against a
|
||||||
|
# temporary fixture library and an isolated volume, runs the phase_h journeys,
|
||||||
|
# destroys the stack, and writes the evidence. Any skipped check fails it.
|
||||||
|
env:
|
||||||
|
# The fixture libraries are bind-mounted into the containers, so this path
|
||||||
|
# has to be one the Docker daemon can see. On a runner that talks to a
|
||||||
|
# sibling daemon, point it at a shared host path instead of the workspace —
|
||||||
|
# an unshared path arrives as an empty mount and the journeys fail on the
|
||||||
|
# scan, which is the symptom to recognise.
|
||||||
|
PHOTO_PIPELINE_DATA_DIR: ${{ gitea.workspace }}/gate-data
|
||||||
|
PHOTO_PIPELINE_LIBRARY_ROOTS: ${{ gitea.workspace }}/gate-library
|
||||||
|
PHOTO_PIPELINE_TEST_MOUNT_BASE: ${{ gitea.workspace }}/gate-mounts
|
||||||
|
run: |
|
||||||
|
mkdir -p "$PHOTO_PIPELINE_LIBRARY_ROOTS" "$PHOTO_PIPELINE_TEST_MOUNT_BASE"
|
||||||
|
python -m photo_pipeline container-gate --output gate-evidence
|
||||||
|
|
||||||
|
- name: Keep the evidence
|
||||||
|
# Retained per run, and retained on failure especially: the logs are the only
|
||||||
|
# account of what the containers did.
|
||||||
|
if: always()
|
||||||
|
uses: actions/upload-artifact@v3
|
||||||
|
with:
|
||||||
|
name: container-gate-${{ gitea.sha }}
|
||||||
|
path: gate-evidence
|
||||||
133
README.md
133
README.md
@@ -143,8 +143,137 @@ thumbnail cache. Binding to `0.0.0.0` makes the access secret mandatory
|
|||||||
without it. The declared `HEALTHCHECK` polls `/api/v1/health/ready`, so a container
|
without it. The declared `HEALTHCHECK` polls `/api/v1/health/ready`, so a container
|
||||||
whose database is unmigrated or misconfigured is never reported healthy.
|
whose database is unmigrated or misconfigured is never reported healthy.
|
||||||
|
|
||||||
Composing the runtime and mounting the library is US08-03; publishing and deploying
|
## Composed runtime (US08-03)
|
||||||
the image is US08-04.
|
|
||||||
|
`docker-compose.yml` is the deployment: one `serve` container, one `worker`
|
||||||
|
container, one bind-mounted library, one data volume, and a one-shot `migrate` that
|
||||||
|
both roles wait for.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cp .env.example .env && $EDITOR .env # nothing has a value in it; fill in yours
|
||||||
|
docker compose up -d --build
|
||||||
|
```
|
||||||
|
|
||||||
|
`.env.example` lists every `PHOTO_PIPELINE_*` variable with its default in a comment
|
||||||
|
and no values at all. The three the composition cannot start without are
|
||||||
|
`PHOTO_PIPELINE_LIBRARY_HOST_PATH` (the library on this host),
|
||||||
|
`PHOTO_PIPELINE_LIBRARY_ROOTS` (where it is mounted *inside* the container), and
|
||||||
|
`PHOTO_PIPELINE_ACCESS_SECRET` — publishing the port means the app is reachable from
|
||||||
|
outside the container, so US08-01 makes the secret mandatory. Set
|
||||||
|
`PHOTO_PIPELINE_UID`/`GID` to the owner of the library: what the containers rename
|
||||||
|
and rewrite keeps that ownership.
|
||||||
|
|
||||||
|
| invariant | how the composition keeps it |
|
||||||
|
|---|---|
|
||||||
|
| one writer | `serve` and `worker` take their role's library lock (US07-05) in the shared `/data` volume, so `--scale worker=2` is refused by the lock, not by convention |
|
||||||
|
| migrations first | `migrate` runs the backup-then-migrate path and must exit 0 before `api` and `worker` start; a failed upgrade leaves the previous database and its pre-migration backup intact |
|
||||||
|
| container paths | `PHOTO_PIPELINE_LIBRARY_ROOTS` is both the mount target and the configured root; a root that is not mounted makes `serve`/`worker` exit 5 at startup instead of writing into the container's throwaway layer |
|
||||||
|
| loopback by default | the port is published to `127.0.0.1` unless `PHOTO_PIPELINE_PUBLISH_ADDRESS` says otherwise, and exposing it needs the hostname in `PHOTO_PIPELINE_ALLOWED_HOSTS` plus the access secret |
|
||||||
|
| restart safety | both roles are `restart: unless-stopped` with a 30 s stop grace period, and a job interrupted by a restart resumes exactly as it does on a host restart |
|
||||||
|
|
||||||
|
The `data` volume holds the database, its write-ahead log, the thumbnail cache,
|
||||||
|
journals, and backups. **It must stay on a local filesystem** — SQLite in WAL mode
|
||||||
|
needs real local locking, so NFS, SMB, and network volume drivers are unsupported
|
||||||
|
there and corrupt the database rather than slow it down. The library bind mount has
|
||||||
|
no such restriction.
|
||||||
|
|
||||||
|
Operating the deployment is operating the same CLI:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose run --rm --no-deps api diagnostics
|
||||||
|
docker compose run --rm --no-deps api backup --reason pre-upgrade
|
||||||
|
docker compose run --rm --no-deps api verify-backup /data/backups/<name>
|
||||||
|
docker compose run --rm --no-deps api restore /data/backups/<name> --into /data/restored
|
||||||
|
docker compose logs -f worker
|
||||||
|
```
|
||||||
|
|
||||||
|
`--no-deps` keeps a one-off command from starting a second stack; `api` is only the
|
||||||
|
service the command borrows the image and mounts from. `restore` is deliberately not
|
||||||
|
an API call: it replaces the state of an installation and belongs to a stopped one,
|
||||||
|
so stop `api` and `worker` first and restart them against the restored directory.
|
||||||
|
|
||||||
|
## Publishing and deploying (US08-04)
|
||||||
|
|
||||||
|
Two workflows in `.gitea/workflows/` make `main` the only path to the running stack:
|
||||||
|
|
||||||
|
| workflow | runs on | does |
|
||||||
|
|---|---|---|
|
||||||
|
| `test.yml` (**Test**) | every pull request, and every push to `main` | installs exiftool, the package, and Chromium, then runs the suites configured in `work_item/.work-item.yml` |
|
||||||
|
| `deploy.yml` (**Deploy**) | a **successful** `Test` run on `main`, or manual dispatch | builds the image, pushes it, triggers the Portainer webhook, prunes dangling layers |
|
||||||
|
|
||||||
|
Deploy waits for `Test` through `workflow_run`, so publishing is downstream of a green
|
||||||
|
suite rather than of a push. `completed` is not `success`: the publish job runs only on
|
||||||
|
`conclusion == 'success'`. Because an unmatched `workflow_run` filter does not fail but
|
||||||
|
simply never fires, `tests/integration/test_deploy_workflows.py` asserts the link from
|
||||||
|
both ends — renaming either workflow, or dropping one of the configured suites out of
|
||||||
|
the test job, fails that test rather than silently unhooking the gate.
|
||||||
|
|
||||||
|
The image is `git.domverse-berlin.eu/domverse/photoanalyzer`, tagged with the commit
|
||||||
|
SHA and `latest`. The commit tag is pushed first, so a `latest` that exists always has
|
||||||
|
a commit tag beside it and **a rollback is a tag change, not a rebuild**:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker pull git.domverse-berlin.eu/domverse/photoanalyzer:<known-good-sha>
|
||||||
|
docker tag git.domverse-berlin.eu/domverse/photoanalyzer:<known-good-sha> \
|
||||||
|
git.domverse-berlin.eu/domverse/photoanalyzer:latest
|
||||||
|
docker push git.domverse-berlin.eu/domverse/photoanalyzer:latest
|
||||||
|
# then trigger the Portainer webhook, or redeploy the stack from Portainer
|
||||||
|
```
|
||||||
|
|
||||||
|
Manual dispatch defaults to **dry run**: it builds and pushes `scratch-<sha>` only, and
|
||||||
|
touches neither `latest` nor the running stack. Clear the `dry_run` input to publish by
|
||||||
|
hand. A workflow-level `concurrency: deploy` group with `cancel-in-progress: false`
|
||||||
|
queues deploys instead of overlapping them, so two commits can never race to `latest`.
|
||||||
|
|
||||||
|
### Secrets
|
||||||
|
|
||||||
|
Repository secrets — used only by the pipeline:
|
||||||
|
|
||||||
|
| secret | purpose |
|
||||||
|
|---|---|
|
||||||
|
| `REGISTRY_USER` | registry user with `write:package` |
|
||||||
|
| `REGISTRY_TOKEN` | that user's token, passed on stdin, never as an argument |
|
||||||
|
| `PORTAINER_WEBHOOK_URL` | the stack's auto-update POST URL; `curl --fail` makes a refused redeploy a failed workflow |
|
||||||
|
|
||||||
|
Runtime secrets — `PHOTO_PIPELINE_VISION_API_KEY`, `PHOTO_PIPELINE_IMMICH_API_KEY`,
|
||||||
|
`PHOTO_PIPELINE_ACCESS_SECRET` — are **not** repository secrets and are not in the
|
||||||
|
image. The stack is managed by Portainer from git (`docker-compose.yml`), and those
|
||||||
|
values live in the Portainer stack's environment, so rotation is one place and a
|
||||||
|
repository read discloses nothing. A test asserts the workflows never name them.
|
||||||
|
|
||||||
|
## Container acceptance gate (US08-05)
|
||||||
|
|
||||||
|
The deployed container is verified the way the host application is, by one command:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
work_item/scripts/python -m photo_pipeline container-gate --output gate-evidence
|
||||||
|
```
|
||||||
|
|
||||||
|
It builds the image, provisions the composition against a **temporary fixture library
|
||||||
|
on a bind mount and an isolated data volume**, runs the `phase_h` journeys against it,
|
||||||
|
destroys every stack afterwards, and writes `release-report.json`, `logs/container.log`,
|
||||||
|
and `CHECKSUMS.sha256` into the evidence directory. It exits non-zero when a journey
|
||||||
|
fails, when the story matrix has a hole, **or when any check skipped at all** — unlike
|
||||||
|
the release gate, this one accepts no environment excuse: a run that never reached the
|
||||||
|
containers proved nothing about them.
|
||||||
|
|
||||||
|
The journeys (`tests/e2e/test_phase_h_container.py`) are:
|
||||||
|
|
||||||
|
| journey | what it proves |
|
||||||
|
|---|---|
|
||||||
|
| browser | discovery, duplicate review, analysis, album proposal, rename, upload preflight, and archive views, driven through the containerized frontend — including a rename that really moves the operator's folder on the bind mount |
|
||||||
|
| upgrade | the previous version's image runs first, then this one against the same volume: schema at the new head, assets, analysis results, job progress, the unapplied rename plan, and the thumbnail cache all survive |
|
||||||
|
| restart | `docker kill` on both containers mid-job; the job resumes, every photo ends with exactly one stored result, and only the in-flight item ever reaches the provider twice |
|
||||||
|
| security | no session refused, a forged `X-Forwarded-Host` cannot smuggle an allowed hostname past the check, a symlink out of the mounted library is refused, an unmounted library root refuses startup, and no secret appears in `docker compose logs` |
|
||||||
|
|
||||||
|
The upgrade journey builds the previous commit's tree when no published image is named;
|
||||||
|
point it at the real one with `PHOTO_PIPELINE_PREVIOUS_IMAGE`. On a Docker VM (Colima,
|
||||||
|
Docker Desktop) the fixture library must live on a shared path — it defaults to
|
||||||
|
`~/.cache/photo-pipeline`, overridable with `PHOTO_PIPELINE_TEST_MOUNT_BASE`.
|
||||||
|
|
||||||
|
CI runs this gate as the `container` job of **Test** on pushes to `main` and keeps its
|
||||||
|
evidence as a run artefact. Deploy waits for the whole `Test` workflow, so a red
|
||||||
|
container gate is a publish that does not happen.
|
||||||
|
|
||||||
## Testing
|
## Testing
|
||||||
|
|
||||||
|
|||||||
105
docker-compose.yml
Normal file
105
docker-compose.yml
Normal file
@@ -0,0 +1,105 @@
|
|||||||
|
# The deployed runtime (US08-03): one API, one worker, one library, one volume.
|
||||||
|
#
|
||||||
|
# The same image (US08-02) runs both roles, so what is composed here is process
|
||||||
|
# topology, not a second application. Three invariants shape it:
|
||||||
|
#
|
||||||
|
# * one writer — `serve` and `worker` each take the library process lock for their
|
||||||
|
# role (US07-05), and both containers mount the *same* data volume, which is what
|
||||||
|
# makes the lock file visible to both. Scaling `worker` past 1 is refused by that
|
||||||
|
# lock rather than by anyone remembering not to;
|
||||||
|
# * one local filesystem — the database, its write-ahead log, the thumbnail cache,
|
||||||
|
# and the backups live in the `data` volume, and SQLite in WAL mode requires real
|
||||||
|
# local-filesystem locking. A network mount (NFS, SMB, or a cloud volume driver)
|
||||||
|
# is unsupported for it; that is a corrupted database, not a slow one;
|
||||||
|
# * one library path vocabulary — `PHOTO_PIPELINE_LIBRARY_ROOTS` names the path
|
||||||
|
# *inside* the container, which is also the bind mount's target below. A root
|
||||||
|
# that is not mounted there makes `serve` and `worker` refuse at startup instead
|
||||||
|
# of writing into the container's throwaway layer.
|
||||||
|
#
|
||||||
|
# Configuration and secrets come from the environment only: copy `.env.example` to
|
||||||
|
# `.env` and fill it in. Nothing is baked into the image and nothing with a value in
|
||||||
|
# it is committed.
|
||||||
|
#
|
||||||
|
# cp .env.example .env && $EDITOR .env
|
||||||
|
# docker compose up -d --build
|
||||||
|
#
|
||||||
|
# Operating it is operating the same CLI — `docker compose run --rm --no-deps api
|
||||||
|
# <command>` — see README, "Composed runtime".
|
||||||
|
|
||||||
|
name: photo-pipeline
|
||||||
|
|
||||||
|
x-runtime: &runtime
|
||||||
|
image: ${PHOTO_PIPELINE_IMAGE:-photo-pipeline:dev}
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
args:
|
||||||
|
# Everything the app renames or rewrites has to stay owned by the host user
|
||||||
|
# the library already belongs to.
|
||||||
|
UID: ${PHOTO_PIPELINE_UID:-1000}
|
||||||
|
GID: ${PHOTO_PIPELINE_GID:-1000}
|
||||||
|
user: "${PHOTO_PIPELINE_UID:-1000}:${PHOTO_PIPELINE_GID:-1000}"
|
||||||
|
env_file:
|
||||||
|
- ${PHOTO_PIPELINE_ENV_FILE:-.env}
|
||||||
|
volumes:
|
||||||
|
- data:/data
|
||||||
|
- "${PHOTO_PIPELINE_LIBRARY_HOST_PATH:?set PHOTO_PIPELINE_LIBRARY_HOST_PATH to the photo library on this host}:${PHOTO_PIPELINE_LIBRARY_ROOTS:?set PHOTO_PIPELINE_LIBRARY_ROOTS to the container-side library path}"
|
||||||
|
# Jobs check for cancellation between items and leave a resumable record; a
|
||||||
|
# too-short grace period turns an orderly stop into a recovery on next start.
|
||||||
|
stop_grace_period: 30s
|
||||||
|
|
||||||
|
x-environment: &environment
|
||||||
|
# Set here rather than left to the file: these two are what the composition itself
|
||||||
|
# promises, and an `.env` that disagreed would move the database off the volume or
|
||||||
|
# the library off its mount.
|
||||||
|
PHOTO_PIPELINE_DATA_DIR: /data
|
||||||
|
PHOTO_PIPELINE_LIBRARY_ROOTS: ${PHOTO_PIPELINE_LIBRARY_ROOTS}
|
||||||
|
|
||||||
|
services:
|
||||||
|
# Migrations run to completion before either role accepts work, through the same
|
||||||
|
# backup-then-migrate path the roles use (US07-05): a pending upgrade is snapshotted
|
||||||
|
# first, and a failed one exits non-zero with the backup named — so `api` and
|
||||||
|
# `worker` never start, and the previous database is left intact and restorable.
|
||||||
|
migrate:
|
||||||
|
<<: *runtime
|
||||||
|
command: ["migrate"]
|
||||||
|
environment: *environment
|
||||||
|
restart: "no"
|
||||||
|
|
||||||
|
api:
|
||||||
|
<<: *runtime
|
||||||
|
command: ["serve"]
|
||||||
|
environment:
|
||||||
|
<<: *environment
|
||||||
|
# Published to host loopback below. Inside the container the server must bind
|
||||||
|
# the container's own interface for that publish to reach it, which is exactly
|
||||||
|
# what makes the access secret mandatory (US08-01) — `serve` refuses to start
|
||||||
|
# without one. Exposing the port beyond loopback additionally needs
|
||||||
|
# PHOTO_PIPELINE_ALLOWED_HOSTS to name the hostname it is reached under.
|
||||||
|
PHOTO_PIPELINE_HOST: 0.0.0.0
|
||||||
|
PHOTO_PIPELINE_PORT: 8000
|
||||||
|
ports:
|
||||||
|
# Host side only: PHOTO_PIPELINE_PORT in `.env` moves the *published* port, and
|
||||||
|
# the container always serves 8000, which is what the image's health check probes.
|
||||||
|
- "${PHOTO_PIPELINE_PUBLISH_ADDRESS:-127.0.0.1}:${PHOTO_PIPELINE_PORT:-8000}:8000"
|
||||||
|
depends_on:
|
||||||
|
migrate:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
# One worker. A second one is refused by the library lock in the shared data
|
||||||
|
# volume, which is the point: `docker compose up --scale worker=2` fails loudly
|
||||||
|
# instead of running two writers against one library.
|
||||||
|
worker:
|
||||||
|
<<: *runtime
|
||||||
|
command: ["worker", "--id", "worker-1"]
|
||||||
|
environment: *environment
|
||||||
|
depends_on:
|
||||||
|
migrate:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
# Local driver on purpose: the database, WAL, thumbnail cache, and backups need a
|
||||||
|
# real local filesystem. Do not point this at NFS, SMB, or a network volume driver.
|
||||||
|
data:
|
||||||
|
driver: local
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
"""Application management CLI:
|
"""Application management CLI:
|
||||||
``python -m photo_pipeline {serve,migrate,worker,import-legacy-scores,backup,verify-backup,restore,diagnostics}``.
|
``python -m photo_pipeline {serve,migrate,worker,import-legacy-scores,backup,verify-backup,restore,diagnostics,benchmark,release-gate,container-gate,dry-run,approve-dry-run}``.
|
||||||
|
|
||||||
``serve`` and ``worker`` take the library process lock for their role (US07-05):
|
``serve`` and ``worker`` take the library process lock for their role (US07-05):
|
||||||
two workers, or the frozen CLI running beside the app, would each be safe on their
|
two workers, or the frozen CLI running beside the app, would each be safe on their
|
||||||
@@ -11,9 +11,11 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import json
|
import json
|
||||||
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Sequence
|
from typing import Sequence
|
||||||
|
|
||||||
|
from photo_pipeline import path_policy
|
||||||
from photo_pipeline.config import Config
|
from photo_pipeline.config import Config
|
||||||
from photo_pipeline.services.app_lock import LegacyProcessActive, LibraryLock, LockHeld
|
from photo_pipeline.services.app_lock import LegacyProcessActive, LibraryLock, LockHeld
|
||||||
from photo_pipeline.services.backup import BackupError, BackupService, migrate_with_backup
|
from photo_pipeline.services.backup import BackupError, BackupService, migrate_with_backup
|
||||||
@@ -68,6 +70,14 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|||||||
"release-gate", help="Run every suite in an isolated stack and keep the evidence"
|
"release-gate", help="Run every suite in an isolated stack and keep the evidence"
|
||||||
)
|
)
|
||||||
gate_cmd.add_argument("--output", help="Evidence directory (default: data/release/<stamp>)")
|
gate_cmd.add_argument("--output", help="Evidence directory (default: data/release/<stamp>)")
|
||||||
|
container_cmd = commands.add_parser(
|
||||||
|
"container-gate",
|
||||||
|
help="Provision the composition from the built image, run the phase_h "
|
||||||
|
"acceptance suite against it, destroy it, and keep the evidence (US08-05)",
|
||||||
|
)
|
||||||
|
container_cmd.add_argument(
|
||||||
|
"--output", help="Evidence directory (default: data/container-gate/<stamp>)"
|
||||||
|
)
|
||||||
dry_cmd = commands.add_parser(
|
dry_cmd = commands.add_parser(
|
||||||
"dry-run", help="Read-only reconciliation of the configured library (US07-07)"
|
"dry-run", help="Read-only reconciliation of the configured library (US07-07)"
|
||||||
)
|
)
|
||||||
@@ -83,6 +93,18 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|||||||
config = Config.from_env()
|
config = Config.from_env()
|
||||||
config.database_path.parent.mkdir(parents=True, exist_ok=True)
|
config.database_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# In a container the two roles that touch the library check their boundary before
|
||||||
|
# they take a lock or bind a port: the configured roots must name the mount paths,
|
||||||
|
# and a mismatch is cheaper to refuse than to discover at the first write (US08-03).
|
||||||
|
# Only in a container — on a host an unmounted root is an ordinary Tuesday (an
|
||||||
|
# archive medium that is not plugged in), and refusing to serve would take the
|
||||||
|
# offline half of the library away with it.
|
||||||
|
if args.command in ("serve", "worker") and path_policy.in_container():
|
||||||
|
refusal = path_policy.roots_refusal(config.library_roots, require_mount=True)
|
||||||
|
if refusal is not None:
|
||||||
|
print(refusal, file=sys.stderr)
|
||||||
|
return 5
|
||||||
|
|
||||||
if args.command == "migrate":
|
if args.command == "migrate":
|
||||||
manifest = migrate_with_backup(config)
|
manifest = migrate_with_backup(config)
|
||||||
if manifest:
|
if manifest:
|
||||||
@@ -141,6 +163,31 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|||||||
)
|
)
|
||||||
return 0 if report["ok"] else 1
|
return 0 if report["ok"] else 1
|
||||||
|
|
||||||
|
if args.command == "container-gate":
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
from photo_pipeline.services import release
|
||||||
|
|
||||||
|
# The suite provisions and destroys the composition itself; what this command
|
||||||
|
# adds is the single entry point and the retained evidence. No skip is an
|
||||||
|
# environment limit here: a gate that did not reach the containers proved
|
||||||
|
# nothing about them.
|
||||||
|
output = args.output or Path(config.data_dir) / "container-gate" / datetime.now(
|
||||||
|
timezone.utc
|
||||||
|
).strftime("%Y%m%dT%H%M%SZ")
|
||||||
|
report = release.run_gate(
|
||||||
|
config,
|
||||||
|
output=output,
|
||||||
|
stages=release.CONTAINER_STAGES,
|
||||||
|
allowed_skips=release.CONTAINER_ALLOWED_SKIP_REASONS,
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{k: v for k, v in report.items() if k not in ("stages", "matrix")}, indent=2
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return 0 if report["ok"] else 1
|
||||||
|
|
||||||
if args.command == "dry-run":
|
if args.command == "dry-run":
|
||||||
from photo_pipeline.services import release
|
from photo_pipeline.services import release
|
||||||
|
|
||||||
@@ -209,8 +256,6 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|||||||
lock.release()
|
lock.release()
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
import sys
|
|
||||||
|
|
||||||
import uvicorn
|
import uvicorn
|
||||||
|
|
||||||
from photo_pipeline.api.app import ConfigurationRefused, create_app
|
from photo_pipeline.api.app import ConfigurationRefused, create_app
|
||||||
@@ -238,8 +283,6 @@ def _acquire(lock: LibraryLock, *, allow_legacy: bool) -> int | None:
|
|||||||
|
|
||||||
Returns an exit code to return, or ``None`` when the lock was acquired.
|
Returns an exit code to return, or ``None`` when the lock was acquired.
|
||||||
"""
|
"""
|
||||||
import sys
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
lock.acquire(allow_legacy=allow_legacy)
|
lock.acquire(allow_legacy=allow_legacy)
|
||||||
except LockHeld as error:
|
except LockHeld as error:
|
||||||
|
|||||||
@@ -15,9 +15,7 @@ import os
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Iterable, Iterator
|
from typing import Iterable, Iterator
|
||||||
|
|
||||||
SUPPORTED_EXTENSIONS = {
|
SUPPORTED_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp", ".heic", ".heif", ".tiff", ".tif"}
|
||||||
".jpg", ".jpeg", ".png", ".webp", ".heic", ".heif", ".tiff", ".tif"
|
|
||||||
}
|
|
||||||
EXCLUDED_DIR_NAMES = {"_IGNORE", ".@__thumb"}
|
EXCLUDED_DIR_NAMES = {"_IGNORE", ".@__thumb"}
|
||||||
|
|
||||||
|
|
||||||
@@ -72,6 +70,63 @@ def resolve_in_roots(roots: Iterable[os.PathLike | str], path: os.PathLike | str
|
|||||||
raise PathPolicyError("path is outside the configured library roots")
|
raise PathPolicyError("path is outside the configured library roots")
|
||||||
|
|
||||||
|
|
||||||
|
def in_container() -> bool:
|
||||||
|
"""Whether this process is running inside a container image build of the app."""
|
||||||
|
return Path("/.dockerenv").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def _under_mount(path: Path) -> bool:
|
||||||
|
"""Whether ``path`` or one of its parents below ``/`` is a mounted filesystem."""
|
||||||
|
current = path.resolve()
|
||||||
|
while current != current.parent:
|
||||||
|
if os.path.ismount(current):
|
||||||
|
return True
|
||||||
|
current = current.parent
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def roots_refusal(roots: Iterable[os.PathLike | str], *, require_mount: bool = False) -> str | None:
|
||||||
|
"""Why the configured library roots cannot be worked with, or ``None`` (US08-03).
|
||||||
|
|
||||||
|
The roots name the directories this installation renames folders in, rewrites
|
||||||
|
EXIF in, and archives from. Container path policy is the same problem as host
|
||||||
|
path policy with one new failure mode: the configured roots must name the
|
||||||
|
*container-side* mount paths. A host path configured inside a container is
|
||||||
|
either absent or an ordinary directory of the image, so the library looks empty
|
||||||
|
and the first write lands in the container's throwaway layer instead of in the
|
||||||
|
library. That is worth refusing at startup, while an operator is still watching,
|
||||||
|
rather than at the first write.
|
||||||
|
|
||||||
|
``require_mount`` is the container-only half of that: inside a container a real
|
||||||
|
library arrives through a bind mount, so a root that is not on (or under) a
|
||||||
|
mount point is not the library the deployment meant.
|
||||||
|
|
||||||
|
Writability is deliberately *not* checked: a bind mount's ownership is
|
||||||
|
virtualised by Docker Desktop and Colima (it arrives as ``root:root``), so
|
||||||
|
``os.access`` there is evidence about the virtio layer rather than about the
|
||||||
|
library. A wrong UID/GID surfaces as a refused rename with the real errno, which
|
||||||
|
is at least true; a refusal here would be false on two supported platforms.
|
||||||
|
"""
|
||||||
|
for root in roots:
|
||||||
|
path = Path(root)
|
||||||
|
if not path.exists():
|
||||||
|
return (
|
||||||
|
f"library root {path} does not exist: PHOTO_PIPELINE_LIBRARY_ROOTS must "
|
||||||
|
"name paths that exist here, and in a container that means the mount path"
|
||||||
|
)
|
||||||
|
if not path.is_dir():
|
||||||
|
return f"library root {path} is not a directory"
|
||||||
|
if not os.access(path, os.R_OK | os.X_OK):
|
||||||
|
return f"library root {path} is not readable by this process"
|
||||||
|
if require_mount and not _under_mount(path):
|
||||||
|
return (
|
||||||
|
f"library root {path} is not on a mounted filesystem in this container: "
|
||||||
|
"the library was not bind-mounted there, so PHOTO_PIPELINE_LIBRARY_ROOTS "
|
||||||
|
"names a directory of the image rather than the library"
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def iter_supported_files(root: os.PathLike | str) -> Iterator[Path]:
|
def iter_supported_files(root: os.PathLike | str) -> Iterator[Path]:
|
||||||
"""Yield supported, non-excluded files under ``root`` in deterministic order.
|
"""Yield supported, non-excluded files under ``root`` in deterministic order.
|
||||||
|
|
||||||
|
|||||||
@@ -15,9 +15,17 @@ package:
|
|||||||
"started_at": "...", "library_roots": ["..."]}
|
"started_at": "...", "library_roots": ["..."]}
|
||||||
|
|
||||||
One holder per role: an API and a worker are designed to run together, a second
|
One holder per role: an API and a worker are designed to run together, a second
|
||||||
worker is not. A lock whose process is gone is stale and is taken over with the
|
worker is not. A lock whose process is gone is stale and is taken over — refusing
|
||||||
takeover recorded — refusing to start because of a crashed predecessor would turn
|
to start because of a crashed predecessor would turn one outage into two.
|
||||||
one outage into two.
|
|
||||||
|
Ownership is an advisory ``flock`` on that file, not the record inside it. The
|
||||||
|
record says *who*; the kernel says *whether*. That distinction is what makes the
|
||||||
|
lock work in containers (US08-03), where a PID and a hostname are namespaced: a
|
||||||
|
lock left behind by a container that no longer exists names a pid that still
|
||||||
|
"exists" in the new container and a host that cannot be probed, so believing the
|
||||||
|
file would deadlock every restart. A flock is released when its holder dies however
|
||||||
|
it dies, and is seen by every process that can open the file — which for a local
|
||||||
|
data directory is every container of this deployment.
|
||||||
|
|
||||||
Legacy detection is deliberately a heuristic, not a promise: the archived CLI has
|
Legacy detection is deliberately a heuristic, not a promise: the archived CLI has
|
||||||
no lock of its own, so what can be observed is its state files being written right
|
no lock of its own, so what can be observed is its state files being written right
|
||||||
@@ -27,6 +35,7 @@ mutating stage should refuse until it stops.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import fcntl
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import socket
|
import socket
|
||||||
@@ -109,6 +118,12 @@ def _now() -> datetime:
|
|||||||
return datetime.now(timezone.utc)
|
return datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def _in_container() -> bool:
|
||||||
|
from photo_pipeline import path_policy
|
||||||
|
|
||||||
|
return path_policy.in_container()
|
||||||
|
|
||||||
|
|
||||||
def legacy_activity(config: Config) -> dict:
|
def legacy_activity(config: Config) -> dict:
|
||||||
"""Legacy state files written within the activity window, if any."""
|
"""Legacy state files written within the activity window, if any."""
|
||||||
seen: list[dict] = []
|
seen: list[dict] = []
|
||||||
@@ -139,6 +154,7 @@ class LibraryLock:
|
|||||||
self.role = role
|
self.role = role
|
||||||
self.path = Path(config.data_dir) / f"{role}{LOCK_SUFFIX}"
|
self.path = Path(config.data_dir) / f"{role}{LOCK_SUFFIX}"
|
||||||
self._acquired = False
|
self._acquired = False
|
||||||
|
self._handle = None
|
||||||
|
|
||||||
# ── inspection ────────────────────────────────────────────────────────────
|
# ── inspection ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -178,12 +194,31 @@ class LibraryLock:
|
|||||||
"stop it before running the application"
|
"stop it before running the application"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
# The kernel decides, because the file cannot: a container's PID and hostname
|
||||||
|
# are namespaced, so a lock left by a container that no longer exists names a
|
||||||
|
# pid that "exists" and a host that cannot be probed (US08-03). An advisory
|
||||||
|
# flock is held by a live process or by nobody, is released when that process
|
||||||
|
# dies however it dies, and is shared by every process that can open this
|
||||||
|
# file — which, for a local data directory, is every role in every container
|
||||||
|
# of this deployment.
|
||||||
|
handle = open(self.path, "a+", encoding="utf-8")
|
||||||
|
try:
|
||||||
|
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
except OSError:
|
||||||
|
handle.close()
|
||||||
|
raise LockHeld(self.holder() or Holder(self.role, -1, "unknown", "unknown")) from None
|
||||||
|
|
||||||
current = self.holder()
|
current = self.holder()
|
||||||
if current is not None:
|
if current is not None and current.host != socket.gethostname() and not _in_container():
|
||||||
if current.alive:
|
# We hold the kernel's lock, so nothing on *this* machine holds the file.
|
||||||
raise LockHeld(current)
|
# On a host that still leaves one case open: a data directory shared with
|
||||||
# Stale: its process is gone. Take over, and say so.
|
# another machine, whose flock we cannot trust. Believe its record rather
|
||||||
self.path.unlink(missing_ok=True)
|
# than run two writers. In a container the data volume is local by
|
||||||
|
# construction (US08-03), and a foreign hostname is only a dead container.
|
||||||
|
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
||||||
|
handle.close()
|
||||||
|
raise LockHeld(current)
|
||||||
|
|
||||||
mine = Holder(
|
mine = Holder(
|
||||||
role=self.role,
|
role=self.role,
|
||||||
@@ -192,15 +227,14 @@ class LibraryLock:
|
|||||||
started_at=_now().isoformat(),
|
started_at=_now().isoformat(),
|
||||||
library_roots=tuple(str(root) for root in self._config.library_roots),
|
library_roots=tuple(str(root) for root in self._config.library_roots),
|
||||||
)
|
)
|
||||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
payload = {k: v for k, v in mine.as_dict().items() if k != "alive"}
|
payload = {k: v for k, v in mine.as_dict().items() if k != "alive"}
|
||||||
# Exclusive create, so two processes racing here cannot both believe they won.
|
handle.seek(0)
|
||||||
try:
|
handle.truncate()
|
||||||
with open(self.path, "x", encoding="utf-8") as handle:
|
json.dump(payload, handle, indent=2)
|
||||||
json.dump(payload, handle, indent=2)
|
handle.flush()
|
||||||
except FileExistsError:
|
# Held open on purpose: closing it is what releases the lock, and that must
|
||||||
winner = self.holder()
|
# happen when this process ends, not when this method returns.
|
||||||
raise LockHeld(winner or mine) from None
|
self._handle = handle
|
||||||
self._acquired = True
|
self._acquired = True
|
||||||
return mine
|
return mine
|
||||||
|
|
||||||
@@ -208,9 +242,10 @@ class LibraryLock:
|
|||||||
"""Give up a lock this process owns. Another holder's lock is left alone."""
|
"""Give up a lock this process owns. Another holder's lock is left alone."""
|
||||||
if not self._acquired:
|
if not self._acquired:
|
||||||
return
|
return
|
||||||
current = self.holder()
|
self.path.unlink(missing_ok=True)
|
||||||
if current is not None and current.pid == os.getpid():
|
if self._handle is not None:
|
||||||
self.path.unlink(missing_ok=True)
|
self._handle.close() # closing the descriptor releases the kernel lock
|
||||||
|
self._handle = None
|
||||||
self._acquired = False
|
self._acquired = False
|
||||||
|
|
||||||
def __enter__(self) -> "LibraryLock":
|
def __enter__(self) -> "LibraryLock":
|
||||||
|
|||||||
@@ -60,6 +60,17 @@ ALLOWED_SKIP_REASONS = (
|
|||||||
"bind-mount ownership is virtualised",
|
"bind-mount ownership is virtualised",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# The container acceptance gate (US08-05): the deployed application, verified the way
|
||||||
|
# the host application is. Deliberately one stage selected by marker, so adding a
|
||||||
|
# phase_h test is enough to put it in front of a deploy.
|
||||||
|
CONTAINER_STAGES: tuple[tuple[str, tuple[str, ...]], ...] = (
|
||||||
|
("container", ("tests/e2e", "-m", "phase_h")),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Nothing. This gate exists to prove the *deployed* runtime, and every reason a check
|
||||||
|
# would skip here — no daemon, no compose, no browser — means it was not proven.
|
||||||
|
CONTAINER_ALLOWED_SKIP_REASONS: tuple[str, ...] = ()
|
||||||
|
|
||||||
|
|
||||||
class ReleaseError(RuntimeError):
|
class ReleaseError(RuntimeError):
|
||||||
pass
|
pass
|
||||||
@@ -175,13 +186,15 @@ def _run_stage(label: str, paths: tuple[str, ...], *, repo: Path, log_dir: Path)
|
|||||||
return StageResult(label, command, result.returncode, elapsed, summary, skipped)
|
return StageResult(label, command, result.returncode, elapsed, summary, skipped)
|
||||||
|
|
||||||
|
|
||||||
def unexpected_skips(results: list[StageResult]) -> list[str]:
|
def unexpected_skips(
|
||||||
|
results: list[StageResult], allowed: tuple[str, ...] = ALLOWED_SKIP_REASONS
|
||||||
|
) -> list[str]:
|
||||||
"""Skips the gate will not accept: everything but the documented environment ones."""
|
"""Skips the gate will not accept: everything but the documented environment ones."""
|
||||||
return [
|
return [
|
||||||
line
|
line
|
||||||
for result in results
|
for result in results
|
||||||
for line in result.skipped
|
for line in result.skipped
|
||||||
if not any(reason in line for reason in ALLOWED_SKIP_REASONS)
|
if not any(reason in line for reason in allowed)
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@@ -190,6 +203,7 @@ def run_gate(
|
|||||||
*,
|
*,
|
||||||
output: Path | str | None = None,
|
output: Path | str | None = None,
|
||||||
stages: tuple[tuple[str, tuple[str, ...]], ...] = STAGES,
|
stages: tuple[tuple[str, tuple[str, ...]], ...] = STAGES,
|
||||||
|
allowed_skips: tuple[str, ...] = ALLOWED_SKIP_REASONS,
|
||||||
repo: Path | None = None,
|
repo: Path | None = None,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""Run every suite in an isolated stack and retain checksummed evidence.
|
"""Run every suite in an isolated stack and retain checksummed evidence.
|
||||||
@@ -206,13 +220,14 @@ def run_gate(
|
|||||||
logs = directory / "logs"
|
logs = directory / "logs"
|
||||||
logs.mkdir(parents=True, exist_ok=True)
|
logs.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
started_at = _now()
|
||||||
matrix = story_matrix(repo)
|
matrix = story_matrix(repo)
|
||||||
results = [_run_stage(label, paths, repo=repo, log_dir=logs) for label, paths in stages]
|
results = [_run_stage(label, paths, repo=repo, log_dir=logs) for label, paths in stages]
|
||||||
skips = unexpected_skips(results)
|
skips = unexpected_skips(results, allowed_skips)
|
||||||
|
|
||||||
report = {
|
report = {
|
||||||
"schema_version": SCHEMA_VERSION,
|
"schema_version": SCHEMA_VERSION,
|
||||||
"started_at": _now().isoformat(),
|
"started_at": started_at.isoformat(),
|
||||||
"revision": revision(),
|
"revision": revision(),
|
||||||
"python": sys.version.split()[0],
|
"python": sys.version.split()[0],
|
||||||
"platform": os.uname().sysname,
|
"platform": os.uname().sysname,
|
||||||
|
|||||||
@@ -24,6 +24,9 @@ test = [
|
|||||||
"httpx>=0.27",
|
"httpx>=0.27",
|
||||||
"playwright>=1.40",
|
"playwright>=1.40",
|
||||||
"pytest-playwright>=0.4",
|
"pytest-playwright>=0.4",
|
||||||
|
# The composition and the workflows are configuration, and the tests that hold
|
||||||
|
# them to their promises read them (US08-03, US08-04).
|
||||||
|
"pyyaml>=6",
|
||||||
]
|
]
|
||||||
|
|
||||||
[build-system]
|
[build-system]
|
||||||
@@ -49,5 +52,6 @@ markers = [
|
|||||||
"phase_d: Phase D end-to-end acceptance (US04-06) — guarded rename API, fault, and browser journeys",
|
"phase_d: Phase D end-to-end acceptance (US04-06) — guarded rename API, fault, and browser journeys",
|
||||||
"phase_e: Phase E end-to-end acceptance (US05-06) — upload preflight, uploader, and browser journeys",
|
"phase_e: Phase E end-to-end acceptance (US05-06) — upload preflight, uploader, and browser journeys",
|
||||||
"phase_f: Phase F end-to-end acceptance (US06-06) — archive destination, transfer, and restore journeys",
|
"phase_f: Phase F end-to-end acceptance (US06-06) — archive destination, transfer, and restore journeys",
|
||||||
"container: builds and runs the container image (US08-02) — needs a Docker daemon and network",
|
"container: builds and runs the container image and its composition (US08-02, US08-03) — needs a Docker daemon, the compose plugin, and the network",
|
||||||
|
"phase_h: Phase H container deployment acceptance (US08-05) — the browser, upgrade, restart, and security journeys against the composed stack",
|
||||||
]
|
]
|
||||||
|
|||||||
260
tests/e2e/_container_harness.py
Normal file
260
tests/e2e/_container_harness.py
Normal file
@@ -0,0 +1,260 @@
|
|||||||
|
"""The composed stack, as a fixture: build, provision, drive, destroy (US08-03/US08-05).
|
||||||
|
|
||||||
|
Extracted from ``tests/e2e/test_compose_stack.py`` when the container acceptance gate
|
||||||
|
needed the same stack under a different image, project, and library. One
|
||||||
|
implementation, because two would drift on exactly the details that make a container
|
||||||
|
test worth anything — the mount, the volume, the ports, and the teardown.
|
||||||
|
|
||||||
|
Nothing here fakes anything below the process boundary: Docker builds the image,
|
||||||
|
Compose starts the real containers, and every helper talks to them over HTTP or the
|
||||||
|
Docker CLI.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import socket
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
REPO = Path(__file__).resolve().parents[2]
|
||||||
|
COMPOSE_FILE = REPO / "docker-compose.yml"
|
||||||
|
CONTAINER_LIBRARY = "/library"
|
||||||
|
READY_TIMEOUT_SECONDS = 180
|
||||||
|
JOB_TIMEOUT_SECONDS = 300
|
||||||
|
UP_TIMEOUT_SECONDS = 30 * 60
|
||||||
|
|
||||||
|
|
||||||
|
def compose_available() -> bool:
|
||||||
|
try:
|
||||||
|
return (
|
||||||
|
subprocess.run(
|
||||||
|
["docker", "compose", "version"], capture_output=True, timeout=60
|
||||||
|
).returncode
|
||||||
|
== 0
|
||||||
|
)
|
||||||
|
except (OSError, subprocess.SubprocessError):
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def free_port() -> int:
|
||||||
|
with socket.socket() as sock:
|
||||||
|
sock.bind(("127.0.0.1", 0))
|
||||||
|
return sock.getsockname()[1]
|
||||||
|
|
||||||
|
|
||||||
|
def mount_base() -> Path:
|
||||||
|
"""A directory a Docker VM shares with the host.
|
||||||
|
|
||||||
|
Not pytest's ``tmp_path``: on macOS that is ``/var/folders/...``, which a Docker VM
|
||||||
|
(Colima, Docker Desktop) does not share, so the bind mount would arrive empty and
|
||||||
|
every assertion would be about nothing. ``$HOME`` is shared by every default
|
||||||
|
configuration.
|
||||||
|
"""
|
||||||
|
base = Path(
|
||||||
|
os.environ.get("PHOTO_PIPELINE_TEST_MOUNT_BASE", Path.home() / ".cache" / "photo-pipeline")
|
||||||
|
)
|
||||||
|
base.mkdir(parents=True, exist_ok=True)
|
||||||
|
return base
|
||||||
|
|
||||||
|
|
||||||
|
def temporary_library(albums: dict[str, int], *, prefix: str = "library-") -> Path:
|
||||||
|
"""A fixture library on a shareable host path, with the exclusion sentinel in it."""
|
||||||
|
root = Path(tempfile.mkdtemp(prefix=prefix, dir=mount_base()))
|
||||||
|
for position, (album, count) in enumerate(albums.items()):
|
||||||
|
(root / album).mkdir(parents=True, exist_ok=True)
|
||||||
|
for index in range(count):
|
||||||
|
# Distinct per album *and* index: two solid images of the same colour are
|
||||||
|
# byte-identical, which would make them a duplicate cluster by accident.
|
||||||
|
colour = (17 + 7 * index, 31 + 29 * position, 160 - 3 * index)
|
||||||
|
Image.new("RGB", (64, 48), colour).save(root / album / f"{album}_{index}.jpg")
|
||||||
|
# Never discovered, counted, analyzed, or uploaded — asserted from outside it.
|
||||||
|
(root / "_IGNORE").mkdir(exist_ok=True)
|
||||||
|
Image.new("RGB", (32, 32), (0, 0, 0)).save(root / "_IGNORE" / "sentinel.jpg")
|
||||||
|
return root
|
||||||
|
|
||||||
|
|
||||||
|
def remove_library(root: Path) -> None:
|
||||||
|
shutil.rmtree(root, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
|
def write_env_file(path: Path, secret: str, extra: dict[str, str] | None = None) -> Path:
|
||||||
|
"""Configuration and secrets come from the environment, so a test writes its own
|
||||||
|
file rather than borrowing the operator's ``.env``."""
|
||||||
|
lines = [
|
||||||
|
f"PHOTO_PIPELINE_ACCESS_SECRET={secret}",
|
||||||
|
"PHOTO_PIPELINE_LOG_FORMAT=text",
|
||||||
|
# The deterministic vision seam, in the data volume so both roles and the test
|
||||||
|
# can read it (concept §18).
|
||||||
|
"PHOTO_PIPELINE_FAKE_VISION_LOG=/data/vision.log",
|
||||||
|
*(f"{key}={value}" for key, value in (extra or {}).items()),
|
||||||
|
]
|
||||||
|
path.write_text("\n".join(lines) + "\n")
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
class Stack:
|
||||||
|
"""The composition under test, plus the environment it was started with."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
library: Path,
|
||||||
|
env_file: Path,
|
||||||
|
*,
|
||||||
|
project: str,
|
||||||
|
image: str,
|
||||||
|
secret: str,
|
||||||
|
) -> None:
|
||||||
|
self.library = library
|
||||||
|
self.project = project
|
||||||
|
self.secret = secret
|
||||||
|
self.port = free_port()
|
||||||
|
self.base = f"http://127.0.0.1:{self.port}"
|
||||||
|
# Compose reads the repository's own .env for substitution; the process
|
||||||
|
# environment wins over it, so the test's values are the ones that apply.
|
||||||
|
self.env = {
|
||||||
|
**os.environ,
|
||||||
|
"PHOTO_PIPELINE_IMAGE": image,
|
||||||
|
"PHOTO_PIPELINE_ENV_FILE": str(env_file),
|
||||||
|
"PHOTO_PIPELINE_LIBRARY_HOST_PATH": str(library),
|
||||||
|
"PHOTO_PIPELINE_LIBRARY_ROOTS": CONTAINER_LIBRARY,
|
||||||
|
"PHOTO_PIPELINE_PORT": str(self.port),
|
||||||
|
"PHOTO_PIPELINE_UID": str(os.getuid()),
|
||||||
|
"PHOTO_PIPELINE_GID": str(os.getgid()),
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── the compose lifecycle ────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def compose(self, *args: str, check: bool = True, timeout: int = 300):
|
||||||
|
result = subprocess.run(
|
||||||
|
["docker", "compose", "-p", self.project, "-f", str(COMPOSE_FILE), *args],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
env=self.env,
|
||||||
|
cwd=REPO,
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
if check and result.returncode != 0:
|
||||||
|
raise AssertionError(
|
||||||
|
f"docker compose {' '.join(args)} failed:\n{result.stdout}\n{result.stderr}\n"
|
||||||
|
f"{self.compose('logs', '--tail', '80', check=False).stdout}"
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
|
def up(self, *extra: str) -> None:
|
||||||
|
self.compose("up", "--detach", *extra, timeout=UP_TIMEOUT_SECONDS)
|
||||||
|
|
||||||
|
def down(self, *, volumes: bool = True) -> None:
|
||||||
|
self.compose(
|
||||||
|
"down",
|
||||||
|
*(("--volumes",) if volumes else ()),
|
||||||
|
"--remove-orphans",
|
||||||
|
check=False,
|
||||||
|
timeout=300,
|
||||||
|
)
|
||||||
|
|
||||||
|
def use_image(self, image: str) -> None:
|
||||||
|
"""Point the composition at another tag — the upgrade path (US08-05)."""
|
||||||
|
self.env["PHOTO_PIPELINE_IMAGE"] = image
|
||||||
|
|
||||||
|
def logs(self, *services: str) -> str:
|
||||||
|
result = self.compose("logs", *services, check=False)
|
||||||
|
return result.stdout + result.stderr
|
||||||
|
|
||||||
|
def wait_until_ready(self) -> None:
|
||||||
|
deadline = time.monotonic() + READY_TIMEOUT_SECONDS
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
try:
|
||||||
|
if httpx.get(f"{self.base}/api/v1/health/ready", timeout=5).status_code == 200:
|
||||||
|
return
|
||||||
|
except httpx.HTTPError:
|
||||||
|
pass
|
||||||
|
time.sleep(0.5)
|
||||||
|
raise AssertionError(f"the stack never became ready:\n{self.logs()}")
|
||||||
|
|
||||||
|
# ── talking to it ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def client(self) -> httpx.Client:
|
||||||
|
"""A browser that has loaded the app: session cookie in the jar, token in a
|
||||||
|
header. The session belongs to the API process, so it is re-bootstrapped after
|
||||||
|
every restart."""
|
||||||
|
client = httpx.Client(base_url=f"{self.base}/api/v1", timeout=60)
|
||||||
|
bootstrap = client.get("/session", headers={"X-Access-Secret": self.secret})
|
||||||
|
assert bootstrap.status_code == 200, bootstrap.text
|
||||||
|
client.headers["X-CSRF-Token"] = bootstrap.json()["csrf_token"]
|
||||||
|
return client
|
||||||
|
|
||||||
|
|
||||||
|
def await_job(client: httpx.Client, job_id: str, states=("succeeded",)) -> dict:
|
||||||
|
deadline = time.monotonic() + JOB_TIMEOUT_SECONDS
|
||||||
|
snapshot: dict = {}
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
response = client.get(f"/jobs/{job_id}")
|
||||||
|
if response.status_code == 200:
|
||||||
|
snapshot = response.json()
|
||||||
|
if snapshot["state"] in states:
|
||||||
|
return snapshot
|
||||||
|
time.sleep(0.5)
|
||||||
|
raise AssertionError(f"job {job_id} never reached {states}: {snapshot}")
|
||||||
|
|
||||||
|
|
||||||
|
def build_image(tag: str, *, revision: str | None = None) -> str:
|
||||||
|
"""Build the application image; from a git revision's tree when one is named.
|
||||||
|
|
||||||
|
``revision`` is how the upgrade journey gets the *previous* version without a
|
||||||
|
registry: the tree of that commit is the build context, so what it produces is the
|
||||||
|
image that commit would have published.
|
||||||
|
"""
|
||||||
|
if revision is None:
|
||||||
|
subprocess.run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"build",
|
||||||
|
"--build-arg",
|
||||||
|
f"UID={os.getuid()}",
|
||||||
|
"--build-arg",
|
||||||
|
f"GID={os.getgid()}",
|
||||||
|
"-t",
|
||||||
|
tag,
|
||||||
|
str(REPO),
|
||||||
|
],
|
||||||
|
check=True,
|
||||||
|
timeout=UP_TIMEOUT_SECONDS,
|
||||||
|
)
|
||||||
|
return tag
|
||||||
|
archive = subprocess.run(
|
||||||
|
["git", "archive", "--format=tar", revision],
|
||||||
|
cwd=REPO,
|
||||||
|
capture_output=True,
|
||||||
|
check=True,
|
||||||
|
timeout=300,
|
||||||
|
).stdout
|
||||||
|
subprocess.run(
|
||||||
|
[
|
||||||
|
"docker",
|
||||||
|
"build",
|
||||||
|
"--build-arg",
|
||||||
|
f"UID={os.getuid()}",
|
||||||
|
"--build-arg",
|
||||||
|
f"GID={os.getgid()}",
|
||||||
|
"-t",
|
||||||
|
tag,
|
||||||
|
"-",
|
||||||
|
],
|
||||||
|
input=archive,
|
||||||
|
check=True,
|
||||||
|
timeout=UP_TIMEOUT_SECONDS,
|
||||||
|
)
|
||||||
|
return tag
|
||||||
|
|
||||||
|
|
||||||
|
needs_compose = pytest.mark.skipif(
|
||||||
|
not compose_available(), reason="no Docker daemon with the compose plugin"
|
||||||
|
)
|
||||||
212
tests/e2e/test_compose_stack.py
Normal file
212
tests/e2e/test_compose_stack.py
Normal file
@@ -0,0 +1,212 @@
|
|||||||
|
"""US08-03: the composition, actually composed.
|
||||||
|
|
||||||
|
Nothing here is faked below the process boundary: Docker builds the image, Compose
|
||||||
|
starts the migrate/API/worker containers against a temporary fixture library on a
|
||||||
|
real bind mount, and every assertion is made over HTTP or against what the stack
|
||||||
|
left in its data volume. The vision provider is the deterministic fake seam the
|
||||||
|
other end-to-end suites use, because an upload of real photos to a real model is not
|
||||||
|
what this story is about — the mount, the lock, the volume, and the restart are.
|
||||||
|
|
||||||
|
The file contract (one API, one worker, migrations first, no committed values) is
|
||||||
|
checked without a daemon in ``tests/integration/test_compose_runtime.py``; only the
|
||||||
|
running proof needs Docker, and CI is where it runs unskipped (US08-04).
|
||||||
|
|
||||||
|
The stack itself lives in ``tests/e2e/_container_harness.py``, shared with the
|
||||||
|
container acceptance gate (US08-05).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from tests.e2e._container_harness import (
|
||||||
|
CONTAINER_LIBRARY,
|
||||||
|
Stack,
|
||||||
|
await_job,
|
||||||
|
compose_available,
|
||||||
|
needs_compose,
|
||||||
|
remove_library,
|
||||||
|
temporary_library,
|
||||||
|
write_env_file,
|
||||||
|
)
|
||||||
|
|
||||||
|
PROJECT = "photo-pipeline-us0803"
|
||||||
|
IMAGE = "photo-pipeline-test:us08-03"
|
||||||
|
SECRET = "compose-acceptance-secret"
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.container
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module")
|
||||||
|
def library() -> Path:
|
||||||
|
root = temporary_library({"01_day": 2, "02_night": 1})
|
||||||
|
try:
|
||||||
|
yield root
|
||||||
|
finally:
|
||||||
|
remove_library(root)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module")
|
||||||
|
def env_file(tmp_path_factory) -> Path:
|
||||||
|
return write_env_file(tmp_path_factory.mktemp("config") / "compose.env", SECRET)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module")
|
||||||
|
def stack(library, env_file):
|
||||||
|
if not compose_available():
|
||||||
|
pytest.skip("no Docker daemon with the compose plugin")
|
||||||
|
running = Stack(library, env_file, project=PROJECT, image=IMAGE, secret=SECRET)
|
||||||
|
running.down()
|
||||||
|
running.up("--build")
|
||||||
|
try:
|
||||||
|
running.wait_until_ready()
|
||||||
|
yield running
|
||||||
|
finally:
|
||||||
|
running.down()
|
||||||
|
|
||||||
|
|
||||||
|
# ── the mounted library ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@needs_compose
|
||||||
|
def test_the_stack_scans_the_bind_mounted_library_at_its_container_paths(stack):
|
||||||
|
client = stack.client()
|
||||||
|
try:
|
||||||
|
scanned = client.post("/inventory/scan")
|
||||||
|
assert scanned.status_code == 200, scanned.text
|
||||||
|
assets = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
|
||||||
|
finally:
|
||||||
|
client.close()
|
||||||
|
|
||||||
|
assert len(assets) == 3, assets
|
||||||
|
paths = {asset["current_path"] for asset in assets}
|
||||||
|
assert all(path.startswith(CONTAINER_LIBRARY + "/") for path in paths), paths
|
||||||
|
assert not any("_IGNORE" in path or "sentinel" in path for path in paths)
|
||||||
|
# The host paths are what the operator mounted, and they are not what the
|
||||||
|
# application records: the roots are the container's.
|
||||||
|
assert not any(str(stack.library) in path for path in paths)
|
||||||
|
|
||||||
|
|
||||||
|
@needs_compose
|
||||||
|
def test_a_library_root_that_is_not_mounted_is_refused_at_startup(stack):
|
||||||
|
"""The container-specific failure: configured roots that name nothing mounted."""
|
||||||
|
refused = stack.compose(
|
||||||
|
"run",
|
||||||
|
"--rm",
|
||||||
|
"--no-deps",
|
||||||
|
"--env",
|
||||||
|
"PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos",
|
||||||
|
"api",
|
||||||
|
"serve",
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
assert refused.returncode == 5, refused.stdout + refused.stderr
|
||||||
|
assert "library root /srv/photos" in refused.stdout + refused.stderr
|
||||||
|
|
||||||
|
|
||||||
|
@needs_compose
|
||||||
|
def test_a_second_worker_is_refused_by_the_library_lock(stack):
|
||||||
|
"""Not by convention: the running worker's lock is in the shared data volume."""
|
||||||
|
refused = stack.compose(
|
||||||
|
"run", "--rm", "--no-deps", "worker", "worker", "--id", "worker-2", check=False
|
||||||
|
)
|
||||||
|
assert refused.returncode == 2, refused.stdout + refused.stderr
|
||||||
|
assert "worker is already running" in refused.stdout + refused.stderr
|
||||||
|
|
||||||
|
|
||||||
|
# ── restart, resume, and the data volume ─────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@needs_compose
|
||||||
|
def test_a_queued_job_resumes_after_both_containers_restart(stack):
|
||||||
|
client = stack.client()
|
||||||
|
try:
|
||||||
|
client.post("/inventory/scan").raise_for_status()
|
||||||
|
assets = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
|
||||||
|
for asset in assets:
|
||||||
|
decided = client.post(
|
||||||
|
"/safety/decisions", json={"asset_id": asset["id"], "decision": "sfw"}
|
||||||
|
)
|
||||||
|
assert decided.status_code == 200, decided.text
|
||||||
|
|
||||||
|
# Stop the worker first, so the job is provably still queued when the restart
|
||||||
|
# happens: a job that finished before the restart would prove nothing.
|
||||||
|
stack.compose("stop", "worker")
|
||||||
|
job = client.post("/analysis/jobs").json()
|
||||||
|
assert client.get(f"/jobs/{job['id']}").json()["state"] == "queued"
|
||||||
|
finally:
|
||||||
|
client.close()
|
||||||
|
|
||||||
|
stack.compose("restart", "api", "worker")
|
||||||
|
stack.wait_until_ready()
|
||||||
|
|
||||||
|
client = stack.client() # the session is per API process, so it is re-bootstrapped
|
||||||
|
try:
|
||||||
|
finished = await_job(client, job["id"])
|
||||||
|
assert finished["state"] == "succeeded", finished
|
||||||
|
# The database is intact and the work is durable, not merely reported.
|
||||||
|
after = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
|
||||||
|
assert {asset["id"] for asset in after} == {asset["id"] for asset in assets}
|
||||||
|
analysed = client.get(f"/analysis/results/{assets[0]['id']}")
|
||||||
|
assert analysed.status_code == 200, analysed.text
|
||||||
|
assert analysed.json()["description"]
|
||||||
|
# Only the mounted library's own photos were analysed: the sentinel under
|
||||||
|
# _IGNORE is not an asset, so it can never have become an item of this job.
|
||||||
|
assert finished["progress"]["total"] == len(assets)
|
||||||
|
finally:
|
||||||
|
client.close()
|
||||||
|
|
||||||
|
|
||||||
|
@needs_compose
|
||||||
|
def test_the_data_volume_survives_recreating_the_containers(stack):
|
||||||
|
"""`down` without `--volumes` then `up` is the upgrade path: state stays."""
|
||||||
|
client = stack.client()
|
||||||
|
try:
|
||||||
|
client.post("/inventory/scan").raise_for_status()
|
||||||
|
before = {a["id"] for a in client.get("/inventory/assets").json()["items"]}
|
||||||
|
finally:
|
||||||
|
client.close()
|
||||||
|
|
||||||
|
stack.down(volumes=False)
|
||||||
|
stack.up()
|
||||||
|
stack.wait_until_ready()
|
||||||
|
|
||||||
|
client = stack.client()
|
||||||
|
try:
|
||||||
|
after = {a["id"] for a in client.get("/inventory/assets").json()["items"]}
|
||||||
|
finally:
|
||||||
|
client.close()
|
||||||
|
assert after == before, "the same assets, from the same database, on the same volume"
|
||||||
|
|
||||||
|
|
||||||
|
# ── operating it ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@needs_compose
|
||||||
|
def test_backup_verify_and_diagnostics_run_as_container_commands(stack):
|
||||||
|
backup = stack.compose("run", "--rm", "--no-deps", "api", "backup", "--reason", "compose")
|
||||||
|
manifest = json.loads(backup.stdout[backup.stdout.index("{") :])
|
||||||
|
assert manifest["name"].startswith("2")
|
||||||
|
|
||||||
|
verified = stack.compose(
|
||||||
|
"run", "--rm", "--no-deps", "api", "verify-backup", f"/data/backups/{manifest['name']}"
|
||||||
|
)
|
||||||
|
assert json.loads(verified.stdout[verified.stdout.index("{") :])["ok"] is True
|
||||||
|
|
||||||
|
report = stack.compose("run", "--rm", "--no-deps", "api", "diagnostics")
|
||||||
|
diagnostics = json.loads(report.stdout[report.stdout.index("{") :])
|
||||||
|
components = {c["name"]: c["path"] for c in diagnostics["components"]}
|
||||||
|
# Database, WAL, thumbnail cache, and backups all live in the mounted volume.
|
||||||
|
for name in ("database", "write_ahead_log", "thumbnail_cache", "backups"):
|
||||||
|
assert components[name].startswith("/data/"), (name, components[name])
|
||||||
|
assert {tool["name"] for tool in diagnostics["tools"]} >= {"exiftool", "immich-go"}
|
||||||
|
# And the worker running beside this command is visible as the lock's holder.
|
||||||
|
assert diagnostics["locks"]["worker"]["role"] == "worker"
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__": # a quick way to run just this file
|
||||||
|
raise SystemExit(pytest.main([__file__, "-v", *sys.argv[1:]]))
|
||||||
510
tests/e2e/test_phase_h_container.py
Normal file
510
tests/e2e/test_phase_h_container.py
Normal file
@@ -0,0 +1,510 @@
|
|||||||
|
"""Phase H — the container acceptance gate (US08-05).
|
||||||
|
|
||||||
|
The deployed application is verified the way the host application is: real image,
|
||||||
|
real composition, real browser, real restarts. Everything below runs against
|
||||||
|
containers that this suite provisions from the built image, against a temporary
|
||||||
|
fixture library on a bind mount and an isolated data volume, and destroys afterwards.
|
||||||
|
|
||||||
|
Four journeys, one per thing a deployment can get wrong:
|
||||||
|
|
||||||
|
* **the browser journey** — discovery, duplicate review, analysis, album proposal,
|
||||||
|
rename, upload preflight, and archive, driven through the containerized frontend;
|
||||||
|
* **the upgrade journey** — the previous version's image runs first, then this one,
|
||||||
|
and the database, its migrations, the rename journal, the job history, and the
|
||||||
|
thumbnail cache have to still be there;
|
||||||
|
* **the restart journey** — both containers are killed mid-job and the work resumes
|
||||||
|
without doing anything twice;
|
||||||
|
* **the security gates** — the refusals a loopback deployment made are still made
|
||||||
|
behind a published port: no session, forged forwarded headers, a path that leaves
|
||||||
|
the mounted library, and a secret in the logs.
|
||||||
|
|
||||||
|
Run it as one command, with evidence: ``python -m photo_pipeline container-gate``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import collections
|
||||||
|
from contextlib import closing
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
from playwright.sync_api import expect
|
||||||
|
|
||||||
|
from photo_pipeline.db import head_revision
|
||||||
|
from tests.e2e._container_harness import (
|
||||||
|
Stack,
|
||||||
|
await_job,
|
||||||
|
build_image,
|
||||||
|
compose_available,
|
||||||
|
needs_compose,
|
||||||
|
remove_library,
|
||||||
|
temporary_library,
|
||||||
|
write_env_file,
|
||||||
|
)
|
||||||
|
|
||||||
|
pytestmark = [pytest.mark.phase_h, pytest.mark.container]
|
||||||
|
|
||||||
|
PROJECT = "photo-pipeline-us0805"
|
||||||
|
IMAGE = "photo-pipeline-test:us08-05"
|
||||||
|
# The version being upgraded *from*. CI passes the tag it last published; without one,
|
||||||
|
# the previous commit's tree is built, which is the same claim without a registry.
|
||||||
|
PREVIOUS_IMAGE = os.environ.get("PHOTO_PIPELINE_PREVIOUS_IMAGE")
|
||||||
|
PREVIOUS_TAG = "photo-pipeline-test:us08-05-previous"
|
||||||
|
SECRET = "container-gate-access-secret"
|
||||||
|
IMMICH_SENTINEL = "immich-sentinel-9f3a2b"
|
||||||
|
HOSTNAME = "photos.test"
|
||||||
|
ALBUM = "rome"
|
||||||
|
RENAMED = "2019 Rome"
|
||||||
|
BURST = 30
|
||||||
|
|
||||||
|
|
||||||
|
# ── the image and the stacks ─────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def image() -> str:
|
||||||
|
if not compose_available():
|
||||||
|
pytest.skip("no Docker daemon with the compose plugin")
|
||||||
|
return build_image(IMAGE)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module")
|
||||||
|
def journey(image, tmp_path_factory):
|
||||||
|
"""One album, one exact duplicate of a photo in it, and the exclusion sentinel."""
|
||||||
|
library = temporary_library({ALBUM: 3}, prefix="us0805-journey-")
|
||||||
|
shutil.copyfile(library / ALBUM / f"{ALBUM}_0.jpg", library / ALBUM / "copy.jpg")
|
||||||
|
env_file = write_env_file(
|
||||||
|
tmp_path_factory.mktemp("journey") / "gate.env",
|
||||||
|
SECRET,
|
||||||
|
{
|
||||||
|
"PHOTO_PIPELINE_ALLOWED_HOSTS": HOSTNAME,
|
||||||
|
# Configured but never reachable: the upload view's job here is to show the
|
||||||
|
# preflight blockers, and the key's job is to be absent from every log.
|
||||||
|
"PHOTO_PIPELINE_IMMICH_API_KEY": IMMICH_SENTINEL,
|
||||||
|
"PHOTO_PIPELINE_IMMICH_SERVER_URL": "http://127.0.0.1:1",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
stack = Stack(library, env_file, project=f"{PROJECT}-journey", image=image, secret=SECRET)
|
||||||
|
try:
|
||||||
|
stack.down()
|
||||||
|
stack.up()
|
||||||
|
stack.wait_until_ready()
|
||||||
|
_seed_journey(stack)
|
||||||
|
yield stack
|
||||||
|
finally:
|
||||||
|
stack.down()
|
||||||
|
remove_library(library)
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_journey(stack: Stack) -> None:
|
||||||
|
"""Everything the views need, established over the public API before they render."""
|
||||||
|
with closing(stack.client()) as client:
|
||||||
|
client.post("/inventory/scan").raise_for_status()
|
||||||
|
client.post("/duplicates/detect").raise_for_status()
|
||||||
|
queue = client.get("/safety/queue", params={"limit": 100}).json()["items"]
|
||||||
|
for item in queue:
|
||||||
|
decided = client.post(
|
||||||
|
"/safety/decisions", json={"asset_id": item["asset_id"], "decision": "sfw"}
|
||||||
|
)
|
||||||
|
assert decided.status_code == 200, decided.text
|
||||||
|
job = client.post("/analysis/jobs").json()
|
||||||
|
await_job(client, job["id"])
|
||||||
|
client.post("/albums/proposals", json={}).raise_for_status()
|
||||||
|
# An archive destination inside the data volume: a second mount would prove
|
||||||
|
# nothing more, and the view needs a location to have something to show.
|
||||||
|
stack.compose("exec", "-T", "api", "mkdir", "-p", "/data/archive")
|
||||||
|
client.post(
|
||||||
|
"/archive-locations", json={"name": "external", "root": "/data/archive"}
|
||||||
|
).raise_for_status()
|
||||||
|
|
||||||
|
|
||||||
|
# ── the browser journey ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@needs_compose
|
||||||
|
def test_the_browser_journey_covers_every_stage_view_of_the_deployed_app(page, journey):
|
||||||
|
"""One pass through the deployed frontend, in workflow order.
|
||||||
|
|
||||||
|
The access secret is supplied the way a person supplies it — the app asks, the
|
||||||
|
answer is kept for the tab — so what is proven is the authenticated deployment,
|
||||||
|
not a test-only bypass.
|
||||||
|
"""
|
||||||
|
page.on("dialog", lambda dialog: dialog.accept(SECRET))
|
||||||
|
errors: list[str] = []
|
||||||
|
page.on("pageerror", lambda error: errors.append(str(error)))
|
||||||
|
base = journey.base
|
||||||
|
with closing(journey.client()) as client:
|
||||||
|
cluster = client.get("/duplicates/clusters").json()["items"][0]
|
||||||
|
|
||||||
|
# ── discovery ────────────────────────────────────────────────────────────
|
||||||
|
page.goto(f"{base}/app/#/workflow")
|
||||||
|
page.get_by_test_id("stage-safety").wait_for()
|
||||||
|
expect(page.get_by_test_id("stage-analysis")).to_be_visible()
|
||||||
|
|
||||||
|
page.goto(f"{base}/app/#/inventory")
|
||||||
|
rows = page.get_by_test_id("asset-row")
|
||||||
|
rows.first.wait_for()
|
||||||
|
assert rows.count() == 4, "three photos and the duplicate copy; never the sentinel"
|
||||||
|
assert "sentinel" not in page.content() and "_IGNORE" not in page.content()
|
||||||
|
|
||||||
|
# ── duplicate review ─────────────────────────────────────────────────────
|
||||||
|
page.goto(f"{base}/app/#/duplicates/{cluster['id']}")
|
||||||
|
page.get_by_test_id("cluster-state").wait_for()
|
||||||
|
# Exact bytes: the cluster arrives decided, and the review surface has to show
|
||||||
|
# both members and the evidence the decision was made on.
|
||||||
|
expect(page.get_by_test_id("cluster-state")).to_contain_text("decided")
|
||||||
|
expect(page.get_by_test_id("member")).to_have_count(2)
|
||||||
|
expect(page.get_by_test_id("member").first).to_contain_text("/library/")
|
||||||
|
|
||||||
|
# ── analysis ─────────────────────────────────────────────────────────────
|
||||||
|
page.goto(f"{base}/app/#/analyze")
|
||||||
|
page.get_by_test_id("analyze-counts").wait_for()
|
||||||
|
expect(page.get_by_test_id("run-analysis")).to_be_visible()
|
||||||
|
|
||||||
|
# ── album proposal ───────────────────────────────────────────────────────
|
||||||
|
page.goto(f"{base}/app/#/albums?album={ALBUM}")
|
||||||
|
page.get_by_test_id("suggested-name").wait_for()
|
||||||
|
page.get_by_test_id("final-name").fill(RENAMED)
|
||||||
|
page.get_by_test_id("save-name").click()
|
||||||
|
page.get_by_test_id("approve").click()
|
||||||
|
expect(page.get_by_test_id("proposal-status")).to_contain_text("approved")
|
||||||
|
|
||||||
|
# ── rename, applied against the bind mount ───────────────────────────────
|
||||||
|
page.goto(f"{base}/app/#/renames")
|
||||||
|
page.get_by_test_id("build-plan").click()
|
||||||
|
page.get_by_test_id("operations").wait_for()
|
||||||
|
expect(page.get_by_test_id("op-destination").first).to_contain_text(RENAMED)
|
||||||
|
page.get_by_test_id("apply-plan").click()
|
||||||
|
expect(page.get_by_test_id("apply-result")).to_contain_text("Applied 1, failed 0")
|
||||||
|
# The mounted library is the host's directory: the container renamed the operator's
|
||||||
|
# folder, not a copy inside its own layer.
|
||||||
|
assert (journey.library / RENAMED).is_dir()
|
||||||
|
assert not (journey.library / ALBUM).exists()
|
||||||
|
|
||||||
|
# ── upload preflight ─────────────────────────────────────────────────────
|
||||||
|
page.goto(f"{base}/app/#/uploads")
|
||||||
|
page.get_by_test_id("upload-scope").wait_for()
|
||||||
|
expect(page.get_by_test_id("album-row").first).to_be_visible()
|
||||||
|
assert IMMICH_SENTINEL not in page.content(), "the API key never reaches the browser"
|
||||||
|
|
||||||
|
# ── archive ──────────────────────────────────────────────────────────────
|
||||||
|
page.goto(f"{base}/app/#/archive")
|
||||||
|
page.get_by_test_id("archive-locations").wait_for()
|
||||||
|
expect(page.get_by_test_id("location-row").first).to_contain_text("external")
|
||||||
|
|
||||||
|
assert errors == [], f"the deployed frontend raised page errors: {errors}"
|
||||||
|
|
||||||
|
|
||||||
|
# ── the upgrade journey ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module")
|
||||||
|
def previous_image(image) -> str:
|
||||||
|
"""The image the deployment is upgrading *from*.
|
||||||
|
|
||||||
|
The published tag when there is one. Before the first publish there is nothing to
|
||||||
|
pull, and refusing then would mean the gate could never let the first deploy
|
||||||
|
through — so the previous commit's tree is built instead, which is the same claim
|
||||||
|
without a registry.
|
||||||
|
"""
|
||||||
|
if PREVIOUS_IMAGE:
|
||||||
|
pulled = subprocess.run(
|
||||||
|
["docker", "pull", PREVIOUS_IMAGE], capture_output=True, timeout=1800
|
||||||
|
)
|
||||||
|
if pulled.returncode == 0:
|
||||||
|
return PREVIOUS_IMAGE
|
||||||
|
return build_image(PREVIOUS_TAG, revision="HEAD~1")
|
||||||
|
|
||||||
|
|
||||||
|
@needs_compose
|
||||||
|
def test_an_upgrade_keeps_the_database_the_journal_the_jobs_and_the_cache(
|
||||||
|
image, previous_image, tmp_path_factory
|
||||||
|
):
|
||||||
|
library = temporary_library({ALBUM: 2}, prefix="us0805-upgrade-")
|
||||||
|
env_file = write_env_file(tmp_path_factory.mktemp("upgrade") / "gate.env", SECRET)
|
||||||
|
stack = Stack(
|
||||||
|
library, env_file, project=f"{PROJECT}-upgrade", image=previous_image, secret=SECRET
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
stack.down()
|
||||||
|
stack.up()
|
||||||
|
stack.wait_until_ready()
|
||||||
|
|
||||||
|
# ── what the previous version leaves behind ──────────────────────────
|
||||||
|
with closing(stack.client()) as client:
|
||||||
|
client.post("/inventory/scan").raise_for_status()
|
||||||
|
assets = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
|
||||||
|
for item in client.get("/safety/queue", params={"limit": 100}).json()["items"]:
|
||||||
|
client.post(
|
||||||
|
"/safety/decisions", json={"asset_id": item["asset_id"], "decision": "sfw"}
|
||||||
|
).raise_for_status()
|
||||||
|
job = client.post("/analysis/jobs").json()
|
||||||
|
finished = await_job(client, job["id"])
|
||||||
|
for asset in assets: # populate the thumbnail cache
|
||||||
|
thumbnail = client.get(f"/assets/{asset['id']}/thumbnail", params={"size": 256})
|
||||||
|
assert thumbnail.status_code == 200, thumbnail.text
|
||||||
|
# A rename plan, left unapplied: the journal has to survive the upgrade
|
||||||
|
# exactly as it was, or a half-applied one could not be recovered.
|
||||||
|
client.post("/albums/proposals", json={}).raise_for_status()
|
||||||
|
_approve(client, RENAMED)
|
||||||
|
plan = client.post("/rename-plans").json()
|
||||||
|
|
||||||
|
before = {
|
||||||
|
"assets": sorted(asset["id"] for asset in assets),
|
||||||
|
"job": finished["progress"],
|
||||||
|
"plan": (plan["id"], plan["checksum"]),
|
||||||
|
"thumbnails": _cache_files(stack),
|
||||||
|
"revision": _revision(stack),
|
||||||
|
}
|
||||||
|
assert before["thumbnails"], "no thumbnail was cached, so nothing would be proven"
|
||||||
|
|
||||||
|
# ── the upgrade: same volume, new image ──────────────────────────────
|
||||||
|
stack.down(volumes=False)
|
||||||
|
stack.use_image(image)
|
||||||
|
stack.up()
|
||||||
|
stack.wait_until_ready()
|
||||||
|
|
||||||
|
after_revision = _revision(stack)
|
||||||
|
assert after_revision == head_revision(), "the new image did not migrate the volume"
|
||||||
|
if after_revision != before["revision"]:
|
||||||
|
# A schema change is snapshotted before it is applied (US07-05), so a
|
||||||
|
# failed upgrade is restorable rather than a lost library.
|
||||||
|
assert _ls(stack, "/data/backups"), "a migration ran without a backup"
|
||||||
|
|
||||||
|
with closing(stack.client()) as client:
|
||||||
|
assets = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
|
||||||
|
assert sorted(asset["id"] for asset in assets) == before["assets"]
|
||||||
|
assert client.get(f"/jobs/{job['id']}").json()["progress"] == before["job"]
|
||||||
|
plans = client.get("/rename-plans").json()["items"]
|
||||||
|
assert (plans[0]["id"], plans[0]["checksum"]) == before["plan"]
|
||||||
|
for asset in assets:
|
||||||
|
assert (
|
||||||
|
client.get(f"/analysis/results/{asset['id']}").status_code == 200
|
||||||
|
), "an analysis result did not survive the upgrade"
|
||||||
|
# The cache is keyed by pixel hash and thumbnail version, so an upgrade that
|
||||||
|
# kept the volume must keep the files: regenerating them is work nobody asked
|
||||||
|
# for, and losing them silently is how a cache stops being one.
|
||||||
|
assert set(before["thumbnails"]) <= set(_cache_files(stack))
|
||||||
|
finally:
|
||||||
|
stack.down()
|
||||||
|
remove_library(library)
|
||||||
|
|
||||||
|
|
||||||
|
def _approve(client: httpx.Client, name: str, *, album: str = ALBUM) -> None:
|
||||||
|
for payload, route in (({"name": name}, "edit"), ({}, "approve")):
|
||||||
|
current = client.get(f"/albums/proposals/{album}").json()
|
||||||
|
client.post(
|
||||||
|
f"/albums/proposals/{album}/{route}",
|
||||||
|
json={**payload, "expected_version": current["version"]},
|
||||||
|
).raise_for_status()
|
||||||
|
|
||||||
|
|
||||||
|
def _exec(stack: Stack, *args: str) -> str:
|
||||||
|
return stack.compose("exec", "-T", "api", *args).stdout
|
||||||
|
|
||||||
|
|
||||||
|
def _ls(stack: Stack, directory: str) -> list[str]:
|
||||||
|
listing = stack.compose("exec", "-T", "api", "ls", directory, check=False)
|
||||||
|
return [line for line in listing.stdout.split() if line]
|
||||||
|
|
||||||
|
|
||||||
|
def _cache_files(stack: Stack) -> list[str]:
|
||||||
|
return sorted(
|
||||||
|
_exec(stack, "find", "/data/cache", "-type", "f", "-name", "*.webp").split()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _revision(stack: Stack) -> str:
|
||||||
|
"""The schema revision the volume's database is actually at."""
|
||||||
|
return _exec(
|
||||||
|
stack,
|
||||||
|
"python",
|
||||||
|
"-c",
|
||||||
|
"import sqlite3;print(sqlite3.connect('/data/photo_pipeline.db')"
|
||||||
|
".execute('select version_num from alembic_version').fetchone()[0])",
|
||||||
|
).strip()
|
||||||
|
|
||||||
|
|
||||||
|
# ── the restart journey ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@needs_compose
|
||||||
|
def test_killing_both_containers_mid_job_resumes_without_doing_anything_twice(
|
||||||
|
image, tmp_path_factory
|
||||||
|
):
|
||||||
|
"""`docker kill` is the honest restart: no grace period, no orderly stop, no
|
||||||
|
chance for either process to write a tidy final state."""
|
||||||
|
library = temporary_library({"burst": BURST}, prefix="us0805-restart-")
|
||||||
|
env_file = write_env_file(tmp_path_factory.mktemp("restart") / "gate.env", SECRET)
|
||||||
|
stack = Stack(library, env_file, project=f"{PROJECT}-restart", image=image, secret=SECRET)
|
||||||
|
try:
|
||||||
|
stack.down()
|
||||||
|
stack.up()
|
||||||
|
stack.wait_until_ready()
|
||||||
|
|
||||||
|
with closing(stack.client()) as client:
|
||||||
|
client.post("/inventory/scan").raise_for_status()
|
||||||
|
assets = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
|
||||||
|
assert len(assets) == BURST
|
||||||
|
for item in client.get("/safety/queue", params={"limit": 100}).json()["items"]:
|
||||||
|
client.post(
|
||||||
|
"/safety/decisions", json={"asset_id": item["asset_id"], "decision": "sfw"}
|
||||||
|
).raise_for_status()
|
||||||
|
job = client.post("/analysis/jobs").json()
|
||||||
|
progress = _wait_for_progress(client, job["id"])
|
||||||
|
|
||||||
|
assert 0 < progress["done"] < progress["total"], progress
|
||||||
|
stack.compose("kill", "api", "worker")
|
||||||
|
|
||||||
|
stack.up()
|
||||||
|
stack.wait_until_ready()
|
||||||
|
with closing(stack.client()) as client:
|
||||||
|
finished = await_job(client, job["id"])
|
||||||
|
assert finished["progress"]["done"] == BURST, finished
|
||||||
|
results = [
|
||||||
|
client.get(f"/analysis/results/{asset['id']}").json() for asset in assets
|
||||||
|
]
|
||||||
|
|
||||||
|
# Exactly one stored result per asset: at-least-once execution, idempotent
|
||||||
|
# recovery — a retried item overwrites its own attempt, it does not add one.
|
||||||
|
assert len(results) == BURST
|
||||||
|
assert all(result["description"] for result in results)
|
||||||
|
# And the side effect nobody can take back — the call to the provider — happened
|
||||||
|
# again only for whatever was in flight when the containers died.
|
||||||
|
analysed = collections.Counter(_exec(stack, "cat", "/data/vision.log").split())
|
||||||
|
assert len(analysed) == BURST, "every photo was analysed, and only the library's"
|
||||||
|
assert max(analysed.values()) <= 2, dict(analysed)
|
||||||
|
assert sum(1 for count in analysed.values() if count > 1) <= 1, dict(analysed)
|
||||||
|
finally:
|
||||||
|
stack.down()
|
||||||
|
remove_library(library)
|
||||||
|
|
||||||
|
|
||||||
|
def _wait_for_progress(client: httpx.Client, job_id: str, *, timeout: float = 120) -> dict:
|
||||||
|
"""Wait until the job is provably under way but provably unfinished."""
|
||||||
|
deadline = time.monotonic() + timeout
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
snapshot = client.get(f"/jobs/{job_id}").json()
|
||||||
|
progress = snapshot["progress"]
|
||||||
|
if progress["done"] and progress["done"] < progress["total"]:
|
||||||
|
return progress
|
||||||
|
if snapshot["state"] in ("succeeded", "failed"):
|
||||||
|
raise AssertionError(f"the job finished before it could be interrupted: {snapshot}")
|
||||||
|
time.sleep(0.05)
|
||||||
|
raise AssertionError(f"the job never started: {client.get(f'/jobs/{job_id}').json()}")
|
||||||
|
|
||||||
|
|
||||||
|
# ── the security gates ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@needs_compose
|
||||||
|
def test_the_deployed_instance_refuses_a_caller_without_a_session(journey):
|
||||||
|
with httpx.Client(base_url=f"{journey.base}/api/v1", timeout=30) as client:
|
||||||
|
for method, path in (("GET", "/workflow"), ("POST", "/inventory/scan")):
|
||||||
|
response = client.request(method, path)
|
||||||
|
assert response.status_code == 401, path
|
||||||
|
assert response.json()["error"]["code"] == "unauthenticated"
|
||||||
|
# A session still has to be paid for with the operator's secret.
|
||||||
|
assert client.get("/session", headers={"X-Access-Secret": "guessed"}).status_code == 401
|
||||||
|
# Readiness stays open: the orchestrator's health check holds no session.
|
||||||
|
assert client.get("/health/ready").status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
@needs_compose
|
||||||
|
def test_forged_forwarded_headers_cannot_smuggle_an_allowed_host_past_the_check(journey):
|
||||||
|
"""Behind a proxy the app believes ``X-Forwarded-*`` — but only from the proxy.
|
||||||
|
Nothing in this composition is a trusted proxy, so the claim is the client's."""
|
||||||
|
with httpx.Client(base_url=f"{journey.base}/api/v1", timeout=30) as client:
|
||||||
|
client.headers["X-CSRF-Token"] = (
|
||||||
|
client.get("/session", headers={"X-Access-Secret": SECRET}).json()["csrf_token"]
|
||||||
|
)
|
||||||
|
# The hostname this deployment is reached under is accepted.
|
||||||
|
assert client.get("/workflow", headers={"Host": HOSTNAME}).status_code == 200
|
||||||
|
|
||||||
|
forged = client.get(
|
||||||
|
"/workflow",
|
||||||
|
headers={"Host": "photos.evil.example", "X-Forwarded-Host": HOSTNAME},
|
||||||
|
)
|
||||||
|
assert forged.status_code == 403
|
||||||
|
assert forged.json()["error"]["code"] == "host_not_allowed"
|
||||||
|
# A forged protocol claim must not mark the session cookie as HTTPS-only
|
||||||
|
# either — that would strand the operator's real, plain-HTTP session.
|
||||||
|
bootstrap = httpx.get(
|
||||||
|
f"{journey.base}/api/v1/session",
|
||||||
|
headers={"X-Access-Secret": SECRET, "X-Forwarded-Proto": "https"},
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
assert "secure" not in bootstrap.headers["set-cookie"].lower()
|
||||||
|
|
||||||
|
|
||||||
|
@needs_compose
|
||||||
|
def test_a_path_that_leaves_the_mounted_library_is_refused(journey):
|
||||||
|
"""The container's own filesystem is not the library. A symlink swapped under a
|
||||||
|
known asset is the sharpest version of the question, because the database still
|
||||||
|
points at a path inside the mount."""
|
||||||
|
with closing(journey.client()) as client:
|
||||||
|
asset = client.get("/inventory/assets", params={"limit": 200}).json()["items"][0]
|
||||||
|
original = journey.library / Path(asset["current_path"]).relative_to("/library")
|
||||||
|
kept = original.read_bytes()
|
||||||
|
original.unlink()
|
||||||
|
original.symlink_to("/etc/passwd")
|
||||||
|
try:
|
||||||
|
escaped = client.get(f"/assets/{asset['id']}/thumbnail", params={"size": 256})
|
||||||
|
finally:
|
||||||
|
original.unlink()
|
||||||
|
original.write_bytes(kept)
|
||||||
|
|
||||||
|
assert escaped.status_code == 403
|
||||||
|
assert escaped.json()["error"]["code"] == "path_not_allowed"
|
||||||
|
assert "root:" not in escaped.text
|
||||||
|
|
||||||
|
# And a root that names nothing mounted is refused before the process serves.
|
||||||
|
refused = journey.compose(
|
||||||
|
"run",
|
||||||
|
"--rm",
|
||||||
|
"--no-deps",
|
||||||
|
"--env",
|
||||||
|
"PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos",
|
||||||
|
"api",
|
||||||
|
"serve",
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
assert refused.returncode == 5, refused.stdout + refused.stderr
|
||||||
|
|
||||||
|
|
||||||
|
@needs_compose
|
||||||
|
def test_no_secret_reaches_the_container_logs(journey):
|
||||||
|
with httpx.Client(base_url=f"{journey.base}/api/v1", timeout=30) as client:
|
||||||
|
client.get("/session", headers={"X-Access-Secret": "wrong-secret-attempt"})
|
||||||
|
client.get("/session", headers={"X-Access-Secret": SECRET})
|
||||||
|
logs = journey.logs()
|
||||||
|
assert SECRET not in logs
|
||||||
|
assert IMMICH_SENTINEL not in logs
|
||||||
|
assert "wrong-secret-attempt" not in logs, "a rejected secret is still a secret"
|
||||||
|
# The refusal itself is logged, so an operator can see the attempt.
|
||||||
|
assert "access secret rejected" in logs
|
||||||
|
|
||||||
|
|
||||||
|
@needs_compose
|
||||||
|
def test_the_evidence_of_this_run_names_the_stack_it_was_produced_from(journey):
|
||||||
|
"""A gate that cannot say what it ran against is an opinion. `docker compose ps`
|
||||||
|
is the record: one API, one worker, one completed migration."""
|
||||||
|
listing = [
|
||||||
|
json.loads(line)
|
||||||
|
for line in journey.compose("ps", "--all", "--format", "json").stdout.splitlines()
|
||||||
|
if line.strip()
|
||||||
|
]
|
||||||
|
services = {entry["Service"]: entry["State"] for entry in listing}
|
||||||
|
assert services["api"] == "running" and services["worker"] == "running"
|
||||||
|
assert services["migrate"] == "exited"
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__": # a quick way to run just this file
|
||||||
|
raise SystemExit(pytest.main([__file__, "-v", *sys.argv[1:]]))
|
||||||
299
tests/integration/test_compose_runtime.py
Normal file
299
tests/integration/test_compose_runtime.py
Normal file
@@ -0,0 +1,299 @@
|
|||||||
|
"""US08-03: the composition's contract, and the library-root check it depends on.
|
||||||
|
|
||||||
|
Bringing the stack up needs a Docker daemon and the network, which is what
|
||||||
|
``tests/e2e/test_compose_stack.py`` does. What can be checked without either is
|
||||||
|
checked here, because the parts that rot silently — a second writer that is only
|
||||||
|
prevented by convention, a data volume that stopped being the same volume for both
|
||||||
|
roles, migrations that stopped running first, a committed value in a file that must
|
||||||
|
carry none — are all readable from the files.
|
||||||
|
|
||||||
|
The startup refusal is the other half: in a container the configured library roots
|
||||||
|
must name the mount paths, and a mismatch has to fail before the lock is taken, not
|
||||||
|
at the first rename.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import stat
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
from photo_pipeline import path_policy
|
||||||
|
from photo_pipeline.__main__ import main
|
||||||
|
from photo_pipeline.config import Config
|
||||||
|
|
||||||
|
REPO = Path(__file__).resolve().parents[2]
|
||||||
|
COMPOSE_FILE = REPO / "docker-compose.yml"
|
||||||
|
COMPOSE = yaml.safe_load(COMPOSE_FILE.read_text())
|
||||||
|
ENV_EXAMPLE = REPO / ".env.example"
|
||||||
|
|
||||||
|
SERVICES = COMPOSE["services"]
|
||||||
|
DATA_VOLUME = "data:/data"
|
||||||
|
|
||||||
|
|
||||||
|
def env_example_keys() -> list[str]:
|
||||||
|
"""The variables the example file declares, in file order."""
|
||||||
|
return [
|
||||||
|
line.split("=", 1)[0]
|
||||||
|
for line in ENV_EXAMPLE.read_text().splitlines()
|
||||||
|
if "=" in line and not line.lstrip().startswith("#")
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
# ── one API, one worker, one library, one volume ─────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_exactly_one_serving_and_one_working_container_from_the_same_image():
|
||||||
|
roles = {name: service["command"][0] for name, service in SERVICES.items()}
|
||||||
|
assert sorted(roles.values()) == ["migrate", "serve", "worker"]
|
||||||
|
assert [name for name, role in roles.items() if role == "serve"] == ["api"]
|
||||||
|
assert [name for name, role in roles.items() if role == "worker"] == ["worker"]
|
||||||
|
|
||||||
|
images = {service["image"] for service in SERVICES.values()}
|
||||||
|
assert len(images) == 1, "both roles must run the same build of the application"
|
||||||
|
assert "latest" not in images.pop()
|
||||||
|
# No `replicas`/`scale` key promising a second worker is fine; the lock decides.
|
||||||
|
assert not any("deploy" in service for service in SERVICES.values())
|
||||||
|
|
||||||
|
|
||||||
|
def test_both_roles_share_the_data_volume_so_the_lock_is_visible_to_both():
|
||||||
|
"""A second worker is refused by the library lock (US07-05) only if it can see it."""
|
||||||
|
for name, service in SERVICES.items():
|
||||||
|
assert DATA_VOLUME in service["volumes"], name
|
||||||
|
assert COMPOSE["volumes"]["data"]["driver"] == "local"
|
||||||
|
text = COMPOSE_FILE.read_text()
|
||||||
|
# The composition has to say why, because the failure is silent corruption.
|
||||||
|
assert "WAL" in text and re.search(r"NFS|SMB|network", text)
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_library_is_a_bind_mount_whose_target_is_the_configured_root():
|
||||||
|
for name, service in SERVICES.items():
|
||||||
|
mounts = [volume for volume in service["volumes"] if volume != DATA_VOLUME]
|
||||||
|
assert len(mounts) == 1, name
|
||||||
|
source, target = re.match(r"^(\$\{.*?\}):(\$\{.*?\})$", mounts[0]).groups()
|
||||||
|
# An unset host path fails the composition rather than mounting something else.
|
||||||
|
assert source.startswith("${PHOTO_PIPELINE_LIBRARY_HOST_PATH:?")
|
||||||
|
# The container-side path and the configured root are one variable, so they
|
||||||
|
# cannot drift apart into a library that is mounted but not configured.
|
||||||
|
assert target.startswith("${PHOTO_PIPELINE_LIBRARY_ROOTS:?")
|
||||||
|
assert service["environment"]["PHOTO_PIPELINE_LIBRARY_ROOTS"] == (
|
||||||
|
"${PHOTO_PIPELINE_LIBRARY_ROOTS}"
|
||||||
|
)
|
||||||
|
assert service["environment"]["PHOTO_PIPELINE_DATA_DIR"] == "/data"
|
||||||
|
|
||||||
|
|
||||||
|
def test_migrations_run_to_completion_before_either_role_accepts_work():
|
||||||
|
"""`migrate` runs the backup-then-migrate path, and a failed upgrade exits
|
||||||
|
non-zero with its pre-migration backup intact — proven in
|
||||||
|
tests/integration/test_backup_recovery.py. What the composition adds is that
|
||||||
|
neither role starts until it succeeded."""
|
||||||
|
assert SERVICES["migrate"]["command"] == ["migrate"]
|
||||||
|
assert SERVICES["migrate"]["restart"] == "no", "a one-shot that retries is not a gate"
|
||||||
|
for role in ("api", "worker"):
|
||||||
|
assert SERVICES[role]["depends_on"] == {
|
||||||
|
"migrate": {"condition": "service_completed_successfully"}
|
||||||
|
}, role
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_api_port_is_published_to_host_loopback_by_default():
|
||||||
|
published = SERVICES["api"]["ports"]
|
||||||
|
assert published == [
|
||||||
|
"${PHOTO_PIPELINE_PUBLISH_ADDRESS:-127.0.0.1}:${PHOTO_PIPELINE_PORT:-8000}:8000"
|
||||||
|
]
|
||||||
|
# Reachable from the host means reachable from elsewhere as far as the app is
|
||||||
|
# concerned, so the access secret stays mandatory (US08-01).
|
||||||
|
assert SERVICES["api"]["environment"]["PHOTO_PIPELINE_HOST"] == "0.0.0.0"
|
||||||
|
assert SERVICES["api"]["environment"]["PHOTO_PIPELINE_PORT"] == 8000
|
||||||
|
assert "PHOTO_PIPELINE_ACCESS_SECRET" not in SERVICES["api"]["environment"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_containers_restart_by_themselves_and_stop_with_time_to_drain():
|
||||||
|
for role in ("api", "worker"):
|
||||||
|
assert SERVICES[role]["restart"] == "unless-stopped", role
|
||||||
|
assert SERVICES[role]["stop_grace_period"] == "30s", role
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_containers_run_as_the_library_owner_and_never_as_root():
|
||||||
|
for name, service in SERVICES.items():
|
||||||
|
assert service["user"] == "${PHOTO_PIPELINE_UID:-1000}:${PHOTO_PIPELINE_GID:-1000}", name
|
||||||
|
assert service["build"]["args"]["UID"] == "${PHOTO_PIPELINE_UID:-1000}", name
|
||||||
|
|
||||||
|
|
||||||
|
# ── configuration comes from the environment, never from a committed file ────
|
||||||
|
|
||||||
|
|
||||||
|
def test_configuration_and_secrets_come_from_the_environment_only():
|
||||||
|
for name, service in SERVICES.items():
|
||||||
|
assert service["env_file"] == ["${PHOTO_PIPELINE_ENV_FILE:-.env}"], name
|
||||||
|
for key, value in service["environment"].items():
|
||||||
|
# Every value is either a variable reference or a property of the
|
||||||
|
# composition itself (the volume path, the container's own port).
|
||||||
|
composed = isinstance(value, int) or value in ("/data", "0.0.0.0")
|
||||||
|
assert composed or value.startswith("${"), (name, key, value)
|
||||||
|
assert not (REPO / ".env").is_file() or ".env" in (REPO / ".gitignore").read_text()
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_example_file_lists_every_setting_and_carries_no_values():
|
||||||
|
declared = env_example_keys()
|
||||||
|
assert declared == sorted(set(declared), key=declared.index), "no variable twice"
|
||||||
|
for line in ENV_EXAMPLE.read_text().splitlines():
|
||||||
|
if "=" in line and not line.lstrip().startswith("#"):
|
||||||
|
assert line.endswith("="), f"a value in the example file: {line}"
|
||||||
|
|
||||||
|
expected = {f"PHOTO_PIPELINE_{name.upper()}" for name in Config.model_fields}
|
||||||
|
assert expected <= set(declared), sorted(expected - set(declared))
|
||||||
|
# And every variable the composition substitutes is documented there too.
|
||||||
|
substituted = set(re.findall(r"\$\{(PHOTO_PIPELINE_[A-Z_]+)", COMPOSE_FILE.read_text()))
|
||||||
|
assert substituted <= set(declared), sorted(substituted - set(declared))
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_example_file_is_not_a_dotenv_that_could_be_loaded_by_accident():
|
||||||
|
"""`.env.example` must not be what `.env` is: no values means nothing to leak."""
|
||||||
|
assert ENV_EXAMPLE.name != ".env"
|
||||||
|
parsed = {k: v for k, v in _parse(ENV_EXAMPLE.read_text()).items() if v}
|
||||||
|
assert parsed == {}
|
||||||
|
|
||||||
|
|
||||||
|
def _parse(text: str) -> dict[str, str]:
|
||||||
|
from photo_pipeline.config import parse_env_file
|
||||||
|
|
||||||
|
return parse_env_file(text)
|
||||||
|
|
||||||
|
|
||||||
|
# ── the lock across container lifetimes ──────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_lock_left_by_a_container_that_is_gone_does_not_block_the_restart(
|
||||||
|
tmp_path, monkeypatch
|
||||||
|
):
|
||||||
|
"""A restarted container is a new hostname and a recycled pid 1, so the record
|
||||||
|
in the lock file proves nothing; the kernel's flock does (US08-03)."""
|
||||||
|
from photo_pipeline.services.app_lock import LibraryLock
|
||||||
|
|
||||||
|
config = Config(data_dir=tmp_path / "data", library_roots=(tmp_path,))
|
||||||
|
(tmp_path / "data").mkdir()
|
||||||
|
(tmp_path / "data" / "worker.lock.json").write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"lock_version": 1,
|
||||||
|
"role": "worker",
|
||||||
|
"pid": 1, # pid 1 of a container that no longer exists
|
||||||
|
"host": "3f2a1b9c4d5e", # its hostname was its container id
|
||||||
|
"started_at": "2026-01-01T00:00:00+00:00",
|
||||||
|
"library_roots": ["/library"],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(path_policy, "in_container", lambda: True)
|
||||||
|
|
||||||
|
taken = LibraryLock(config, "worker").acquire()
|
||||||
|
|
||||||
|
assert taken.pid == os.getpid(), "the worker must come back after a restart"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_second_worker_is_still_refused_while_the_first_holds_the_lock(tmp_path, monkeypatch):
|
||||||
|
"""The other half: the same flock refuses a concurrent second writer, whether it
|
||||||
|
is a process or another container of the same composition."""
|
||||||
|
from photo_pipeline.services.app_lock import LibraryLock, LockHeld
|
||||||
|
|
||||||
|
config = Config(data_dir=tmp_path / "data", library_roots=(tmp_path,))
|
||||||
|
monkeypatch.setattr(path_policy, "in_container", lambda: True)
|
||||||
|
first = LibraryLock(config, "worker")
|
||||||
|
first.acquire()
|
||||||
|
|
||||||
|
with pytest.raises(LockHeld, match="worker is already running"):
|
||||||
|
LibraryLock(config, "worker").acquire()
|
||||||
|
|
||||||
|
first.release()
|
||||||
|
LibraryLock(config, "worker").acquire() # free again
|
||||||
|
|
||||||
|
|
||||||
|
# ── the startup check the mount depends on ───────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_configured_roots_that_are_mounted_and_writable_are_accepted(tmp_path):
|
||||||
|
assert path_policy.roots_refusal([tmp_path]) is None
|
||||||
|
assert path_policy.roots_refusal([]) is None, "no roots is a configuration, not a fault"
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_unmounted_library_root_is_refused_by_name(tmp_path):
|
||||||
|
refusal = path_policy.roots_refusal([tmp_path / "srv" / "photos"])
|
||||||
|
assert refusal is not None
|
||||||
|
assert "does not exist" in refusal and "PHOTO_PIPELINE_LIBRARY_ROOTS" in refusal
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_root_that_is_not_a_directory_or_not_readable_is_refused(tmp_path):
|
||||||
|
a_file = tmp_path / "photos.txt"
|
||||||
|
a_file.write_text("not a library")
|
||||||
|
assert "not a directory" in path_policy.roots_refusal([a_file])
|
||||||
|
|
||||||
|
unreadable = tmp_path / "unreadable"
|
||||||
|
unreadable.mkdir()
|
||||||
|
unreadable.chmod(0o000)
|
||||||
|
try:
|
||||||
|
refusal = path_policy.roots_refusal([unreadable])
|
||||||
|
finally:
|
||||||
|
unreadable.chmod(0o755)
|
||||||
|
if os.getuid() != 0: # root ignores the mode, and CI may well be root
|
||||||
|
assert refusal is not None and "not readable" in refusal
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_unwritable_root_is_not_refused_here(tmp_path):
|
||||||
|
"""A bind mount's ownership is virtualised on macOS and Windows, so os.access
|
||||||
|
would refuse a working deployment. The real errno at the first rename is at
|
||||||
|
least true; this check is about the mount, not the mode."""
|
||||||
|
read_only = tmp_path / "read-only"
|
||||||
|
read_only.mkdir()
|
||||||
|
read_only.chmod(stat.S_IRUSR | stat.S_IXUSR)
|
||||||
|
try:
|
||||||
|
assert path_policy.roots_refusal([read_only]) is None
|
||||||
|
finally:
|
||||||
|
read_only.chmod(0o755)
|
||||||
|
|
||||||
|
|
||||||
|
def test_in_a_container_a_root_that_was_never_mounted_is_refused(tmp_path, monkeypatch):
|
||||||
|
"""The container-only failure: the path exists, but it belongs to the image."""
|
||||||
|
unmounted = tmp_path / "library"
|
||||||
|
(unmounted / "album").mkdir(parents=True)
|
||||||
|
refusal = path_policy.roots_refusal([unmounted], require_mount=True)
|
||||||
|
assert refusal is not None and "not on a mounted filesystem" in refusal
|
||||||
|
|
||||||
|
# A bind mount is a mount point, and a root *below* one is mounted too: a
|
||||||
|
# deployment may mount /srv and configure /srv/photos.
|
||||||
|
monkeypatch.setattr(os.path, "ismount", lambda path: Path(path) == unmounted.resolve())
|
||||||
|
assert path_policy.roots_refusal([unmounted], require_mount=True) is None
|
||||||
|
assert path_policy.roots_refusal([unmounted / "album"], require_mount=True) is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("role", ["serve", "worker"])
|
||||||
|
def test_a_root_mismatch_refuses_at_startup_before_any_lock_is_taken(
|
||||||
|
role, tmp_path, monkeypatch, capsys
|
||||||
|
):
|
||||||
|
data = tmp_path / "data"
|
||||||
|
monkeypatch.setenv("PHOTO_PIPELINE_DATA_DIR", str(data))
|
||||||
|
monkeypatch.setenv("PHOTO_PIPELINE_LIBRARY_ROOTS", str(tmp_path / "not-mounted"))
|
||||||
|
monkeypatch.setattr(path_policy, "in_container", lambda: True)
|
||||||
|
|
||||||
|
assert main([role]) == 5
|
||||||
|
assert "does not exist" in capsys.readouterr().err
|
||||||
|
assert not list(data.glob("*.lock.json")), "nothing started, so nothing is locked"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("role", ["serve", "worker"])
|
||||||
|
def test_an_unmounted_root_on_a_host_is_not_a_reason_to_refuse(role, tmp_path, monkeypatch):
|
||||||
|
"""An archive medium that is not plugged in is a Tuesday, not a misconfiguration:
|
||||||
|
refusing would take the offline half of the library away with it (concept §9)."""
|
||||||
|
monkeypatch.setenv("PHOTO_PIPELINE_DATA_DIR", str(tmp_path / "data"))
|
||||||
|
monkeypatch.setenv("PHOTO_PIPELINE_LIBRARY_ROOTS", str(tmp_path / "not-mounted"))
|
||||||
|
monkeypatch.setenv("PHOTO_PIPELINE_ACCESS_SECRET", "unused-on-loopback")
|
||||||
|
monkeypatch.setattr(path_policy, "in_container", lambda: False)
|
||||||
|
# Reaching the lock is the proof: that is the next thing either role does, and
|
||||||
|
# stopping there keeps the test out of a uvicorn/worker loop.
|
||||||
|
monkeypatch.setattr("photo_pipeline.__main__._acquire", lambda *_, **__: 99)
|
||||||
|
|
||||||
|
assert main([role]) == 99
|
||||||
147
tests/integration/test_container_gate.py
Normal file
147
tests/integration/test_container_gate.py
Normal file
@@ -0,0 +1,147 @@
|
|||||||
|
"""US08-05: the container acceptance gate's own contract, checked without a daemon.
|
||||||
|
|
||||||
|
The gate itself needs Docker, a browser, and several minutes; running it from inside
|
||||||
|
the suite would be a fork bomb with better manners. What is checkable offline is what
|
||||||
|
makes it a *gate* rather than a long test run:
|
||||||
|
|
||||||
|
* it selects the container journeys by marker, so adding one is enough to put it in
|
||||||
|
front of a deploy;
|
||||||
|
* it accepts no skip at all — every reason a check would skip here (no daemon, no
|
||||||
|
compose plugin, no browser) means the deployed runtime was not proven;
|
||||||
|
* it retains checksummed evidence per run;
|
||||||
|
* and CI runs it on `main`, which is what the publish step waits for.
|
||||||
|
|
||||||
|
The running proof is ``tests/e2e/test_phase_h_container.py``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
from photo_pipeline.config import Config
|
||||||
|
from photo_pipeline.services import release
|
||||||
|
|
||||||
|
REPO = Path(__file__).resolve().parents[2]
|
||||||
|
SUITE = REPO / "tests" / "e2e" / "test_phase_h_container.py"
|
||||||
|
TEST_WORKFLOW = yaml.safe_load((REPO / ".gitea" / "workflows" / "test.yml").read_text())
|
||||||
|
|
||||||
|
|
||||||
|
def _config(tmp_path) -> Config:
|
||||||
|
for name in ("data", "lib"):
|
||||||
|
(tmp_path / name).mkdir(exist_ok=True)
|
||||||
|
return Config.from_env(
|
||||||
|
{
|
||||||
|
"PHOTO_PIPELINE_DATA_DIR": str(tmp_path / "data"),
|
||||||
|
"PHOTO_PIPELINE_LIBRARY_ROOTS": str(tmp_path / "lib"),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ── what the gate runs ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_gate_selects_the_container_journeys_by_their_marker():
|
||||||
|
assert release.CONTAINER_STAGES == (("container", ("tests/e2e", "-m", "phase_h")),)
|
||||||
|
declared = [line for line in (REPO / "pyproject.toml").read_text().splitlines()
|
||||||
|
if line.strip().startswith('"phase_h:')]
|
||||||
|
assert declared, "an unregistered marker selects nothing and fails no gate"
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_journey_in_the_suite_carries_the_marker():
|
||||||
|
"""A test in that file without the marker is a check the gate never runs."""
|
||||||
|
source = SUITE.read_text()
|
||||||
|
assert "pytestmark = [pytest.mark.phase_h, pytest.mark.container]" in source
|
||||||
|
journeys = re.findall(r"^def (test_[a-z_]+)", source, re.MULTILINE)
|
||||||
|
assert len(journeys) >= 4, journeys
|
||||||
|
for required in ("browser", "upgrade", "kill", "secret"):
|
||||||
|
assert any(required in name for name in journeys), (required, journeys)
|
||||||
|
|
||||||
|
|
||||||
|
# ── no skip is an environment limit here ─────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_gate_accepts_no_skipped_check_at_all():
|
||||||
|
assert release.CONTAINER_ALLOWED_SKIP_REASONS == ()
|
||||||
|
docker_missing = release.StageResult(
|
||||||
|
"container", [], 0, 0.1, "1 skipped", ["SKIPPED [1] x.py:1: no Docker daemon available"]
|
||||||
|
)
|
||||||
|
# The release gate tolerates exactly this one, because the image definition is
|
||||||
|
# still checked offline. The container gate cannot: it is the deployment's proof.
|
||||||
|
assert release.unexpected_skips([docker_missing]) == []
|
||||||
|
assert release.unexpected_skips(
|
||||||
|
[docker_missing], release.CONTAINER_ALLOWED_SKIP_REASONS
|
||||||
|
) == ["SKIPPED [1] x.py:1: no Docker daemon available"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_skipped_check_fails_the_run_and_the_evidence_says_so(tmp_path):
|
||||||
|
skipping = tmp_path / "test_skipping.py"
|
||||||
|
skipping.write_text(
|
||||||
|
"import pytest\n\n"
|
||||||
|
"def test_x():\n"
|
||||||
|
" pytest.skip('no Docker daemon available')\n"
|
||||||
|
)
|
||||||
|
evidence = tmp_path / "evidence"
|
||||||
|
|
||||||
|
report = release.run_gate(
|
||||||
|
_config(tmp_path),
|
||||||
|
output=evidence,
|
||||||
|
stages=(("container", (str(skipping),)),),
|
||||||
|
allowed_skips=release.CONTAINER_ALLOWED_SKIP_REASONS,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert report["ok"] is False
|
||||||
|
assert report["failures"] == [], "the stage passed; the skip is what fails the gate"
|
||||||
|
assert report["unexpected_skips"], report
|
||||||
|
written = json.loads((evidence / release.REPORT_NAME).read_text())
|
||||||
|
assert written["ok"] is False
|
||||||
|
assert (evidence / "logs" / "container.log").exists()
|
||||||
|
for line in (evidence / release.CHECKSUMS_NAME).read_text().splitlines():
|
||||||
|
digest, name = line.split(" ", 1)
|
||||||
|
assert release.sha256_file(evidence / name) == digest
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_same_run_passes_when_nothing_skips(tmp_path):
|
||||||
|
passing = tmp_path / "test_passing.py"
|
||||||
|
passing.write_text("def test_x():\n assert True\n")
|
||||||
|
|
||||||
|
report = release.run_gate(
|
||||||
|
_config(tmp_path),
|
||||||
|
output=tmp_path / "evidence",
|
||||||
|
stages=(("container", (str(passing),)),),
|
||||||
|
allowed_skips=release.CONTAINER_ALLOWED_SKIP_REASONS,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert report["ok"] is True and report["unexpected_skips"] == []
|
||||||
|
assert report["revision"], "the evidence must say which commit it covers"
|
||||||
|
|
||||||
|
|
||||||
|
# ── the command, and CI ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_command_is_documented_and_wired():
|
||||||
|
from photo_pipeline.__main__ import main # noqa: F401 (import proves it loads)
|
||||||
|
|
||||||
|
assert "container-gate" in (REPO / "photo_pipeline" / "__main__.py").read_text()
|
||||||
|
assert "container-gate" in (REPO / "README.md").read_text()
|
||||||
|
|
||||||
|
|
||||||
|
def test_ci_runs_the_gate_on_main_and_keeps_its_evidence():
|
||||||
|
job = TEST_WORKFLOW["jobs"]["container"]
|
||||||
|
assert job["if"] == "gitea.event_name == 'push'", "pull requests have nothing to upgrade from"
|
||||||
|
script = "\n".join(step["run"] for step in job["steps"] if "run" in step)
|
||||||
|
assert "photo_pipeline container-gate" in script
|
||||||
|
evidence = next(step for step in job["steps"] if "upload-artifact" in str(step.get("uses")))
|
||||||
|
assert evidence["if"] == "always()", "a failed gate's logs are the ones worth keeping"
|
||||||
|
assert evidence["with"]["path"] == "gate-evidence"
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_upgrade_journey_can_still_reach_the_previous_version():
|
||||||
|
"""Without depth, `git archive HEAD~1` has nothing to build."""
|
||||||
|
checkout = next(
|
||||||
|
step for step in TEST_WORKFLOW["jobs"]["container"]["steps"] if "checkout" in str(step.get("uses"))
|
||||||
|
)
|
||||||
|
assert checkout["with"]["fetch-depth"] >= 2
|
||||||
210
tests/integration/test_deploy_workflows.py
Normal file
210
tests/integration/test_deploy_workflows.py
Normal file
@@ -0,0 +1,210 @@
|
|||||||
|
"""US08-04: the deployment pipeline's contract, read from the workflow files.
|
||||||
|
|
||||||
|
A CI/CD pipeline is the one piece of this repository whose failure mode is silence.
|
||||||
|
A renamed workflow, a renamed job, a trigger that quietly stopped matching, and the
|
||||||
|
suite is no longer between `main` and the registry — with nothing red anywhere to say
|
||||||
|
so. So the link is asserted from both ends here: the deploy workflow waits for the
|
||||||
|
test workflow *by name*, and that name is read out of the test workflow itself.
|
||||||
|
|
||||||
|
The same applies to the commands: `work_item/.work-item.yml` is where the required
|
||||||
|
suites are configured, and the test workflow has to run those, not a copy of them that
|
||||||
|
was true once.
|
||||||
|
|
||||||
|
Nothing here talks to Gitea, the registry, or Portainer. What is checkable offline is
|
||||||
|
what the files promise; whether the runner honours them is the deployment's own
|
||||||
|
evidence (US08-05).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
REPO = Path(__file__).resolve().parents[2]
|
||||||
|
WORKFLOWS = REPO / ".gitea" / "workflows"
|
||||||
|
TEST_FILE = WORKFLOWS / "test.yml"
|
||||||
|
DEPLOY_FILE = WORKFLOWS / "deploy.yml"
|
||||||
|
|
||||||
|
TEST_WORKFLOW = yaml.safe_load(TEST_FILE.read_text())
|
||||||
|
DEPLOY_WORKFLOW = yaml.safe_load(DEPLOY_FILE.read_text())
|
||||||
|
WORK_ITEM = yaml.safe_load((REPO / "work_item" / ".work-item.yml").read_text())
|
||||||
|
|
||||||
|
REGISTRY = "git.domverse-berlin.eu"
|
||||||
|
|
||||||
|
|
||||||
|
def triggers(workflow: dict) -> dict:
|
||||||
|
"""The `on:` block. PyYAML resolves the bare key `on` to ``True``."""
|
||||||
|
return workflow[True] if True in workflow else workflow["on"]
|
||||||
|
|
||||||
|
|
||||||
|
def steps(workflow: dict, job: str) -> list[dict]:
|
||||||
|
return workflow["jobs"][job]["steps"]
|
||||||
|
|
||||||
|
|
||||||
|
def run_script(workflow: dict, job: str) -> str:
|
||||||
|
"""Every `run:` in a job, as one blob — what the runner would execute."""
|
||||||
|
return "\n".join(step["run"] for step in steps(workflow, job) if "run" in step)
|
||||||
|
|
||||||
|
|
||||||
|
# ── the gate ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_test_workflow_runs_on_pull_requests_and_on_main():
|
||||||
|
on = triggers(TEST_WORKFLOW)
|
||||||
|
assert "pull_request" in on
|
||||||
|
assert on["push"]["branches"] == ["main"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_test_workflow_runs_the_configured_required_suites():
|
||||||
|
required = WORK_ITEM["workflow"]["required_tests"]
|
||||||
|
assert required, "the helper's required tests are what CI exists to run"
|
||||||
|
script = run_script(TEST_WORKFLOW, "suites")
|
||||||
|
for command in required:
|
||||||
|
assert command in script, f"CI does not run the configured suite: {command}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_deploy_workflow_waits_for_the_test_workflow_by_its_real_name():
|
||||||
|
# Renaming either side without the other breaks this, which is the point: an
|
||||||
|
# unmatched `workflow_run` filter does not fail, it simply never fires.
|
||||||
|
awaited = triggers(DEPLOY_WORKFLOW)["workflow_run"]
|
||||||
|
assert awaited["workflows"] == [TEST_WORKFLOW["name"]]
|
||||||
|
assert awaited["types"] == ["completed"]
|
||||||
|
assert awaited["branches"] == ["main"], "only main is deployable"
|
||||||
|
|
||||||
|
|
||||||
|
def test_publishing_requires_the_test_run_to_have_succeeded_and_not_merely_finished():
|
||||||
|
condition = DEPLOY_WORKFLOW["jobs"]["publish"]["if"]
|
||||||
|
assert "workflow_run.conclusion == 'success'" in condition
|
||||||
|
# `completed` includes failure and cancellation; the conclusion check is the gate.
|
||||||
|
assert "gitea.event_name == 'workflow_run'" in condition
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_deploy_workflow_can_also_be_dispatched_by_hand():
|
||||||
|
assert "workflow_dispatch" in triggers(DEPLOY_WORKFLOW)
|
||||||
|
|
||||||
|
|
||||||
|
# ── what is published ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_image_is_published_under_this_project_s_own_path():
|
||||||
|
slug = WORK_ITEM["repository"]["slug"]
|
||||||
|
assert DEPLOY_WORKFLOW["env"]["IMAGE"] == f"{REGISTRY}/{slug}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_both_latest_and_the_commit_are_pushed_so_a_rollback_is_a_tag_change():
|
||||||
|
script = run_script(DEPLOY_WORKFLOW, "publish")
|
||||||
|
assert 'docker push "$IMAGE:$SHA"' in script
|
||||||
|
assert 'docker push "$IMAGE:latest"' in script
|
||||||
|
# A `latest` without its commit tag would be a version that cannot be rolled back to.
|
||||||
|
assert script.index('docker push "$IMAGE:$SHA"') < script.index('docker push "$IMAGE:latest"')
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_published_commit_is_the_one_that_was_tested():
|
||||||
|
assert DEPLOY_WORKFLOW["env"]["SHA"].startswith("${{ gitea.event.workflow_run.head_sha")
|
||||||
|
checkout = steps(DEPLOY_WORKFLOW, "publish")[0]
|
||||||
|
assert checkout["with"]["ref"] == "${{ env.SHA }}"
|
||||||
|
|
||||||
|
|
||||||
|
# ── secrets ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_registry_credentials_and_the_webhook_come_from_repository_secrets():
|
||||||
|
script = run_script(DEPLOY_WORKFLOW, "publish")
|
||||||
|
for secret in ("REGISTRY_USER", "REGISTRY_TOKEN", "PORTAINER_WEBHOOK_URL"):
|
||||||
|
assert f"secrets.{secret}" in script
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_runtime_secret_is_named_by_the_pipeline_at_all():
|
||||||
|
# The vision key, the Immich key, and the access secret belong to the Portainer
|
||||||
|
# stack. A workflow that mentions one is a workflow that could carry one.
|
||||||
|
text = TEST_FILE.read_text() + DEPLOY_FILE.read_text()
|
||||||
|
for runtime_secret in (
|
||||||
|
"PHOTO_PIPELINE_VISION_API_KEY",
|
||||||
|
"PHOTO_PIPELINE_IMMICH_API_KEY",
|
||||||
|
"PHOTO_PIPELINE_ACCESS_SECRET",
|
||||||
|
):
|
||||||
|
assert runtime_secret not in text
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_workflows_carry_no_credential_values():
|
||||||
|
text = TEST_FILE.read_text() + DEPLOY_FILE.read_text()
|
||||||
|
for line in text.splitlines():
|
||||||
|
if re.search(r"(?i)(token|secret|password|api[_-]?key)\s*[:=]", line):
|
||||||
|
assert "${{ secrets." in line or line.lstrip().startswith("#"), line
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_registry_password_is_never_an_argument():
|
||||||
|
# An argument is visible in the runner's process list and in `docker login`'s own
|
||||||
|
# warning; stdin is not.
|
||||||
|
script = run_script(DEPLOY_WORKFLOW, "publish") + run_script(DEPLOY_WORKFLOW, "dry-run")
|
||||||
|
assert "--password-stdin" in script
|
||||||
|
assert "--password " not in script
|
||||||
|
|
||||||
|
|
||||||
|
# ── redeploy, pruning, and overlap ───────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_failed_webhook_call_fails_the_workflow():
|
||||||
|
webhook = next(
|
||||||
|
step
|
||||||
|
for step in steps(DEPLOY_WORKFLOW, "publish")
|
||||||
|
if "PORTAINER_WEBHOOK_URL" in step.get("run", "")
|
||||||
|
)
|
||||||
|
assert "--fail" in webhook["run"], "curl exits 0 on an HTTP 500 without it"
|
||||||
|
assert "-k" not in webhook["run"].split(), "the redeploy call verifies TLS"
|
||||||
|
|
||||||
|
|
||||||
|
def test_pruning_removes_dangling_layers_and_never_published_tags():
|
||||||
|
for job in ("publish", "dry-run"):
|
||||||
|
script = run_script(DEPLOY_WORKFLOW, job)
|
||||||
|
assert "docker image prune -f" in script
|
||||||
|
assert "prune -a" not in script and "--all" not in script
|
||||||
|
|
||||||
|
|
||||||
|
def test_two_deploys_of_different_commits_cannot_overlap():
|
||||||
|
guard = DEPLOY_WORKFLOW["concurrency"]
|
||||||
|
assert "sha" not in guard["group"].lower(), "a per-commit group guards nothing"
|
||||||
|
assert guard["cancel-in-progress"] is False, "a cancelled push leaves half a tag set"
|
||||||
|
|
||||||
|
|
||||||
|
# ── the dry run ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_dry_run_is_manual_only_and_defaults_to_on():
|
||||||
|
condition = DEPLOY_WORKFLOW["jobs"]["dry-run"]["if"]
|
||||||
|
assert condition == "gitea.event_name == 'workflow_dispatch' && inputs.dry_run"
|
||||||
|
dispatch = triggers(DEPLOY_WORKFLOW)["workflow_dispatch"]
|
||||||
|
assert dispatch["inputs"]["dry_run"]["default"] is True
|
||||||
|
assert dispatch["inputs"]["dry_run"]["type"] == "boolean"
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_dry_run_publishes_a_scratch_tag_and_nothing_else():
|
||||||
|
script = run_script(DEPLOY_WORKFLOW, "dry-run")
|
||||||
|
assert 'docker push "$IMAGE:scratch-$SHA"' in script
|
||||||
|
assert ":latest" not in script, "a dry run that moves latest is a deploy"
|
||||||
|
assert "PORTAINER_WEBHOOK_URL" not in script, "a dry run never redeploys the stack"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_manual_deploy_is_the_deliberate_choice_not_the_default():
|
||||||
|
publish = DEPLOY_WORKFLOW["jobs"]["publish"]["if"]
|
||||||
|
assert "inputs.dry_run == false" in publish
|
||||||
|
|
||||||
|
|
||||||
|
# ── the files themselves ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_workflow_file_parses_and_declares_a_name_a_trigger_and_a_job():
|
||||||
|
files = sorted(WORKFLOWS.glob("*.yml")) + sorted(WORKFLOWS.glob("*.yaml"))
|
||||||
|
assert {path.name for path in files} == {"test.yml", "deploy.yml"}
|
||||||
|
for path in files:
|
||||||
|
workflow = yaml.safe_load(path.read_text())
|
||||||
|
assert workflow["name"], path
|
||||||
|
assert triggers(workflow), path
|
||||||
|
assert workflow["jobs"], path
|
||||||
|
for job_name, job in workflow["jobs"].items():
|
||||||
|
assert job["runs-on"], f"{path}:{job_name}"
|
||||||
|
assert job["steps"], f"{path}:{job_name}"
|
||||||
|
for step in job["steps"]:
|
||||||
|
assert step.get("name"), f"{path}:{job_name} has an unnamed step"
|
||||||
@@ -182,12 +182,19 @@
|
|||||||
"US08-02": [
|
"US08-02": [
|
||||||
"tests/integration/test_container_image.py",
|
"tests/integration/test_container_image.py",
|
||||||
"tests/e2e/test_container_runtime.py"
|
"tests/e2e/test_container_runtime.py"
|
||||||
|
],
|
||||||
|
"US08-03": [
|
||||||
|
"tests/integration/test_compose_runtime.py",
|
||||||
|
"tests/e2e/test_compose_stack.py"
|
||||||
|
],
|
||||||
|
"US08-04": [
|
||||||
|
"tests/integration/test_deploy_workflows.py"
|
||||||
|
],
|
||||||
|
"US08-05": [
|
||||||
|
"tests/integration/test_container_gate.py",
|
||||||
|
"tests/e2e/test_phase_h_container.py"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"planned": [
|
"planned": [],
|
||||||
"US08-03",
|
|
||||||
"US08-04",
|
|
||||||
"US08-05"
|
|
||||||
],
|
|
||||||
"_planned_comment": "Accepted backlog stories that are not implemented yet. The release gate (US07-07) requires every story file to be either mapped to tests or listed here, so an unimplemented story is a visible decision rather than a hole in the matrix."
|
"_planned_comment": "Accepted backlog stories that are not implemented yet. The release gate (US07-07) requires every story file to be either mapped to tests or listed here, so an unimplemented story is a visible decision rather than a hole in the matrix."
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user