domverse/photoanalyzer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Compare commits

base: domverse:us/US09-05-automate-documentation-acceptance
Branches Tags
domverse:main domverse:us/US09-05-automate-documentation-acceptance domverse:us/US09-04-write-the-user-manual-with-generated-screenshots domverse:us/US09-03-write-the-architecture-overview domverse:us/US09-02-write-the-installation-and-operations-manual domverse:us/US09-01-serve-the-documentation-inside-the-application domverse:chore/E09-documentation-backlog domverse:us/US08-05-automate-container-deployment-acceptance domverse:us/US08-04-publish-and-deploy-from-gitea-actions domverse:us/US08-03-compose-the-runtime-and-mount-the-library-safely domverse:us/US08-02-build-a-reproducible-application-image domverse:us/US08-01-make-the-trust-boundary-configurable-and-authent domverse:chore/local-run-and-env-file domverse:us/US07-07-automate-full-release-acceptance domverse:us/US07-06-validate-performance-and-resource-bounds domverse:us/US07-05-deliver-backup-and-operational-recovery domverse:us/US07-04-prove-concurrency-and-crash-recovery domverse:us/US07-03-harden-media-and-metadata-edge-cases domverse:us/US07-02-harden-api-authorization-and-path-boundaries domverse:us/US07-01-complete-donor-migration-and-freeze-the-cli-arch domverse:us/US06-06-automate-phase-f-end-to-end-acceptance domverse:us/US06-05-operate-archive-and-restore-in-the-browser domverse:us/US06-04-plan-and-execute-safe-restores domverse:us/US06-03-preserve-offline-identity-and-review-evidence domverse:us/US06-02-transfer-verify-and-remove-active-sources domverse:us/US06-01-configure-and-preflight-archive-destinations domverse:us/US05-06-automate-phase-e-end-to-end-acceptance domverse:us/US05-05-operate-uploads-in-the-browser domverse:us/US05-04-verify-retry-and-resolve-uncertain-uploads domverse:us/US05-03-parse-and-persist-uploader-outcomes domverse:us/US05-02-orchestrate-album-upload-batches domverse:us/US05-01-validate-credentials-and-upload-readiness domverse:us/US04-06-automate-phase-d-end-to-end-acceptance domverse:us/US04-05-operate-rename-plans-in-the-browser domverse:us/US04-03-apply-and-verify-guarded-renames
..
compare: domverse:us/US07-04-prove-concurrency-and-crash-recovery
Branches Tags
domverse:main domverse:us/US09-05-automate-documentation-acceptance domverse:us/US09-04-write-the-user-manual-with-generated-screenshots domverse:us/US09-03-write-the-architecture-overview domverse:us/US09-02-write-the-installation-and-operations-manual domverse:us/US09-01-serve-the-documentation-inside-the-application domverse:chore/E09-documentation-backlog domverse:us/US08-05-automate-container-deployment-acceptance domverse:us/US08-04-publish-and-deploy-from-gitea-actions domverse:us/US08-03-compose-the-runtime-and-mount-the-library-safely domverse:us/US08-02-build-a-reproducible-application-image domverse:us/US08-01-make-the-trust-boundary-configurable-and-authent domverse:chore/local-run-and-env-file domverse:us/US07-07-automate-full-release-acceptance domverse:us/US07-06-validate-performance-and-resource-bounds domverse:us/US07-05-deliver-backup-and-operational-recovery domverse:us/US07-04-prove-concurrency-and-crash-recovery domverse:us/US07-03-harden-media-and-metadata-edge-cases domverse:us/US07-02-harden-api-authorization-and-path-boundaries domverse:us/US07-01-complete-donor-migration-and-freeze-the-cli-arch domverse:us/US06-06-automate-phase-f-end-to-end-acceptance domverse:us/US06-05-operate-archive-and-restore-in-the-browser domverse:us/US06-04-plan-and-execute-safe-restores domverse:us/US06-03-preserve-offline-identity-and-review-evidence domverse:us/US06-02-transfer-verify-and-remove-active-sources domverse:us/US06-01-configure-and-preflight-archive-destinations domverse:us/US05-06-automate-phase-e-end-to-end-acceptance domverse:us/US05-05-operate-uploads-in-the-browser domverse:us/US05-04-verify-retry-and-resolve-uncertain-uploads domverse:us/US05-03-parse-and-persist-uploader-outcomes domverse:us/US05-02-orchestrate-album-upload-batches domverse:us/US05-01-validate-credentials-and-upload-readiness domverse:us/US04-06-automate-phase-d-end-to-end-acceptance domverse:us/US04-05-operate-rename-plans-in-the-browser domverse:us/US04-03-apply-and-verify-guarded-renames

1 Commits
us/US09-05 ... us/US07-04

Author SHA1 Message Date
domverse
c4e64d2b9a US07-04: Prove Concurrency and Crash Recovery 2026-08-17 11:08:51 +02:00
114 changed files with 197 additions and 15906 deletions
Expand all files Collapse all files

23
.dockerignore
View File

@@ -1,23 +0,0 @@
# Deny-by-default build context (US08-02): the image must contain no secrets, no
# photos, no database, no logs, and no .git. An allow list is the only version of this
# rule that stays true when a new file appears in the working copy.
*
!pyproject.toml
!alembic.ini
!README.md
!photo_pipeline
!migrations
!frontend
!docs
!docker
# Nothing generated, even under an allowed directory.
**/__pycache__
**/*.py[cod]
**/.DS_Store
**/*.env
**/*.log
**/*.db
**/*.db-*
**/*.sqlite*

85
.env.example
View File

@@ -1,85 +0,0 @@
# Every setting the application and its composition read, with no values in it.
# Copy to `.env`, fill in what you need, and keep that copy out of git (it is
# gitignored, and the work-item safety checks refuse to stage it).
#
# cp .env.example .env
#
# Empty means "use the default noted beside it". Anything already exported in the
# shell wins over this file, both for the app and for `docker compose`.
# ── the composition (host side; read by docker-compose.yml only) ─────────────
# The photo library on this host. Bind-mounted at PHOTO_PIPELINE_LIBRARY_ROOTS.
PHOTO_PIPELINE_LIBRARY_HOST_PATH=
# Which image to run. Default: photo-pipeline:dev (what `up --build` builds).
PHOTO_PIPELINE_IMAGE=
# Must be the owner of the library above: what the containers rename and rewrite
# keeps this ownership. Default: 1000 / 1000.
PHOTO_PIPELINE_UID=
PHOTO_PIPELINE_GID=
# Host address the API port is published on. Default: 127.0.0.1. Anything else
# exposes the app beyond this machine — then ALLOWED_HOSTS and ACCESS_SECRET below
# are what stand in for the loopback boundary (US08-01).
PHOTO_PIPELINE_PUBLISH_ADDRESS=
# This file's own path, if it is not ./.env. Default: .env.
PHOTO_PIPELINE_ENV_FILE=
# ── library and data ────────────────────────────────────────────────────────
# os.pathsep-separated. In a container these are the *container-side* mount paths,
# and `serve`/`worker` refuse to start when they are not mounted.
PHOTO_PIPELINE_LIBRARY_ROOTS=
# Database, WAL, thumbnail cache, journals, backups. Default: data (the container
# sets /data, which is the persistent volume; a network mount is unsupported).
PHOTO_PIPELINE_DATA_DIR=
# Database file, if it should not live in the data directory. Default:
# <data dir>/photo_pipeline.db.
PHOTO_PIPELINE_DB_PATH=
# ── serving ─────────────────────────────────────────────────────────────────
# Bind address. Default: 127.0.0.1. The composition sets 0.0.0.0 inside the
# container and publishes to loopback on the host instead.
PHOTO_PIPELINE_HOST=
# Under the composition this is the *published* host port; the container serves
# 8000. Default: 8000.
PHOTO_PIPELINE_PORT=
# Comma-separated hostnames the app answers to besides loopback. Empty means
# loopback only. Naming one makes the access secret mandatory.
PHOTO_PIPELINE_ALLOWED_HOSTS=
# Traded for the session cookie at GET /api/v1/session via X-Access-Secret.
# Required as soon as the app is reachable from anywhere but loopback. Generate
# one with: python -c 'import secrets; print(secrets.token_urlsafe(32))'
PHOTO_PIPELINE_ACCESS_SECRET=
# Comma-separated peer addresses whose X-Forwarded-Proto/-Host may be believed.
# Only the reverse proxy's address belongs here. Default: none.
PHOTO_PIPELINE_TRUSTED_PROXIES=
# Largest request body accepted, in bytes. Default: 1048576.
PHOTO_PIPELINE_MAX_REQUEST_BYTES=
# ── logging ─────────────────────────────────────────────────────────────────
# Default: INFO.
PHOTO_PIPELINE_LOG_LEVEL=
# json or text. Default: json.
PHOTO_PIPELINE_LOG_FORMAT=
# ── limits ──────────────────────────────────────────────────────────────────
# Thumbnail cache quota in bytes. Default: 500000000.
PHOTO_PIPELINE_THUMBNAIL_CACHE_QUOTA_BYTES=
# Refuse to decode images larger than this many pixels. Default: 100000000.
PHOTO_PIPELINE_THUMBNAIL_MAX_PIXELS=
# Free space an archive destination must keep beyond the transfer. Default:
# 1000000000.
PHOTO_PIPELINE_ARCHIVE_FREE_SPACE_RESERVE_BYTES=
# ── safety gate ─────────────────────────────────────────────────────────────
# true refuses every mutating request until a read-only dry run of this library has
# been produced and approved (US07-07). Default: false. Turn it on before pointing
# the app at photos that cannot be replaced.
PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL=
# ── external services ───────────────────────────────────────────────────────
# Vision provider key for the analysis stage. Without it, analysis cannot run.
PHOTO_PIPELINE_VISION_API_KEY=
# Immich server and its API key, for the upload stage.
PHOTO_PIPELINE_IMMICH_SERVER_URL=
PHOTO_PIPELINE_IMMICH_API_KEY=
# Uploader binary. Default: immich-go (on PATH; pinned inside the image).
PHOTO_PIPELINE_IMMICH_GO_BINARY=

9
.gitattributes vendored
View File

@@ -1,9 +0,0 @@
# Vendored third-party browser bundles (US09-01). Minified upstream files carry
# trailing whitespace and very long lines; they are not ours to reformat, and the
# submit gate's `git diff --check` would refuse them forever. Their integrity is
# controlled where it belongs instead: a pinned version and a recorded sha256 in
# frontend/js/vendor/VERSIONS.json, asserted by tests/integration/test_documentation.py.
#
# Only the whitespace check is turned off. They stay text, so a credential scan or a
# search still reads them.
frontend/js/vendor/*.js -whitespace linguist-vendored

99
.gitea/workflows/deploy.yml
View File

@@ -1,99 +0,0 @@
# Publish and redeploy (US08-04). Adapted from the crowdsec-admin deploy workflow:
# build, log in to the Gitea registry, push, trigger the Portainer webhook, prune.
# The difference is the gate — this project has a required suite that must not be
# skipped, so publishing happens only after `Test` (`.gitea/workflows/test.yml`)
# succeeded on `main`, never on the push itself.
#
# The stack is managed by Portainer from git (`docker-compose.yml`), and the runtime
# secrets it needs — vision key, Immich key, access secret — live in the Portainer
# stack's environment. They are deliberately not repository secrets and are not in the
# image: rotation stays in one place, and a repository read never discloses them.
#
# Repository secrets required:
# REGISTRY_USER user with write:package on the registry
# REGISTRY_TOKEN that user's token
# PORTAINER_WEBHOOK_URL POST URL from the stack's auto-update setting
name: Deploy
on:
workflow_run:
workflows:
- Test
types:
- completed
branches:
- main
workflow_dispatch:
inputs:
dry_run:
description: Build and push a scratch tag only — leave `latest` and the running stack alone
type: boolean
default: true
concurrency:
# Deliberately not keyed by commit: the point is that two deploys of *different*
# commits cannot overlap. Queued, not cancelled — a half-pushed tag set is worse
# than a late one.
group: deploy
cancel-in-progress: false
env:
IMAGE: git.domverse-berlin.eu/domverse/photoanalyzer
# The commit that was tested, not whatever `main` points at by the time this starts.
SHA: ${{ gitea.event.workflow_run.head_sha || gitea.sha }}
jobs:
publish:
# A completed `Test` run is not a passing one.
if: >-
(gitea.event_name == 'workflow_run' && gitea.event.workflow_run.conclusion == 'success')
|| (gitea.event_name == 'workflow_dispatch' && inputs.dry_run == false)
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
ref: ${{ env.SHA }}
- name: Log in to the Gitea registry
run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login git.domverse-berlin.eu -u "${{ secrets.REGISTRY_USER }}" --password-stdin
- name: Build and push
# The commit tag is pushed first, so a `latest` that exists is always a tag
# that also exists under its own commit — which is what makes a rollback a
# tag change rather than a rebuild.
run: |
docker build -t "$IMAGE:$SHA" -t "$IMAGE:latest" .
docker push "$IMAGE:$SHA"
docker push "$IMAGE:latest"
- name: Trigger the Portainer redeploy
# --fail turns an HTTP error into a non-zero exit: a redeploy that did not
# happen must not read as a green deploy.
run: curl -sS --fail -X POST "${{ secrets.PORTAINER_WEBHOOK_URL }}"
- name: Prune dangling images
# Untagged layers only. Published tags are the rollback history; `-a` would
# delete exactly the images this workflow exists to keep.
run: docker image prune -f
dry-run:
# Manual only, and the default: prove the image still builds and the registry
# still accepts it without moving `latest` or touching the running stack.
if: gitea.event_name == 'workflow_dispatch' && inputs.dry_run
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Log in to the Gitea registry
run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login git.domverse-berlin.eu -u "${{ secrets.REGISTRY_USER }}" --password-stdin
- name: Build and push a scratch tag
run: |
docker build -t "$IMAGE:scratch-$SHA" .
docker push "$IMAGE:scratch-$SHA"
- name: Prune dangling images
run: docker image prune -f

139
.gitea/workflows/test.yml
View File

@@ -1,139 +0,0 @@
# The test gate. Deploy waits for this workflow by name (`.gitea/workflows/deploy.yml`
# triggers on `workflow_run: [Test]`), so renaming it here without renaming it there
# would leave `main` publishing without a suite. `tests/integration/test_deploy_workflows.py`
# asserts both halves of that link, and that the commands below are still the ones
# configured in `work_item/.work-item.yml`.
name: Test
on:
pull_request:
push:
branches:
- main
concurrency:
# One run per branch; a newer push makes the older run's answer irrelevant.
group: test-${{ gitea.ref }}
cancel-in-progress: true
jobs:
suites:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install exiftool
# The EXIF checkpoints are the safety invariant of every metadata stage; a run
# without exiftool would skip them for a reason the release gate accepts.
run: |
SUDO=
if [ "$(id -u)" -ne 0 ]; then SUDO=sudo; fi
$SUDO apt-get update
$SUDO apt-get install -y --no-install-recommends libimage-exiftool-perl
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install the application and its test dependencies
run: |
python -m pip install --upgrade pip
python -m pip install -e '.[test]'
python -m playwright install --with-deps chromium
- name: Helper suite
run: work_item/scripts/python -m unittest discover -s work_item/tests -v
- name: Application suite
run: work_item/scripts/python -m pytest tests -q
# The deployed container, verified the way the host application is (US08-05). It
# runs on `main` only — a pull request has nothing published to upgrade *from*, and
# building two images per push would pay for that on every commit. Deploy waits for
# this whole workflow, so a red container gate is a deploy that does not happen.
container:
if: gitea.event_name == 'push'
runs-on: ubuntu-latest
steps:
- name: Checkout
# The upgrade journey builds the previous commit's tree when no published
# image is named, so the history has to be there.
uses: actions/checkout@v4
with:
fetch-depth: 2
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install the application and its test dependencies
run: |
python -m pip install --upgrade pip
python -m pip install -e '.[test]'
python -m playwright install --with-deps chromium
- name: Container acceptance gate
# One command: it builds the image, provisions the composition against a
# temporary fixture library and an isolated volume, runs the phase_h journeys,
# destroys the stack, and writes the evidence. Any skipped check fails it.
env:
# The fixture libraries are bind-mounted into the containers, so this path
# has to be one the Docker daemon can see. On a runner that talks to a
# sibling daemon, point it at a shared host path instead of the workspace —
# an unshared path arrives as an empty mount and the journeys fail on the
# scan, which is the symptom to recognise.
PHOTO_PIPELINE_DATA_DIR: ${{ gitea.workspace }}/gate-data
PHOTO_PIPELINE_LIBRARY_ROOTS: ${{ gitea.workspace }}/gate-library
PHOTO_PIPELINE_TEST_MOUNT_BASE: ${{ gitea.workspace }}/gate-mounts
run: |
mkdir -p "$PHOTO_PIPELINE_LIBRARY_ROOTS" "$PHOTO_PIPELINE_TEST_MOUNT_BASE"
python -m photo_pipeline container-gate --output gate-evidence
- name: Keep the evidence
# Retained per run, and retained on failure especially: the logs are the only
# account of what the containers did.
if: always()
uses: actions/upload-artifact@v3
with:
name: container-gate-${{ gitea.sha }}
path: gate-evidence
# The manuals, checked against the application they describe (US09-05). Runs on
# every pull request as well as `main`: documentation drifts by the same commits
# that change behaviour, and telling the author while the change is still open is
# the only time the fix is cheap.
documentation:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install the application and its test dependencies
run: |
python -m pip install --upgrade pip
python -m pip install -e '.[test]'
python -m playwright install --with-deps chromium
- name: Documentation gate
# One command: every phase_i check, offline and in the browser, with the
# evidence retained. It accepts no skip — a check that did not run is a page
# nobody compared with the code.
env:
PHOTO_PIPELINE_DATA_DIR: ${{ gitea.workspace }}/docs-gate-data
run: python -m photo_pipeline docs-gate --output docs-evidence
- name: Keep the evidence
if: always()
uses: actions/upload-artifact@v3
with:
name: docs-gate-${{ gitea.sha }}
path: docs-evidence

9
.gitignore vendored
View File

@@ -20,12 +20,3 @@ _IGNORE/
# Test failure evidence (US07-04)
.artifacts/
# Any dotenv, not only the default name.
*.env
# Local virtualenv for running the app.
.venv/
# setuptools editable-install metadata.
*.egg-info/

117
Dockerfile
View File

@@ -1,117 +0,0 @@
# One image, two roles (US08-02).
#
# The application is not self-contained Python: it shells out to `exiftool` for every
# EXIF checkpoint and to `immich-go` for every upload, and it serves the static
# frontend from `frontend/`. All three are installed here at pinned versions, because
# an image whose external tools drift is an image whose metadata checkpoints and
# upload reports drift with them (concept §15, "External integration risks").
#
# Everything is pinned:
# * the base image by tag *and* digest, so a moved tag cannot change the runtime;
# * exiftool by its Debian package version, verified against `exiftool -ver`;
# * immich-go by release version and per-architecture SHA-256 of the release asset.
# The verified versions become image labels and /etc/photo-pipeline/versions.json,
# which `python -m photo_pipeline diagnostics` reports — so a running container can
# prove what it contains instead of being trusted about it.
#
# The project is installed editable on purpose: `photo_pipeline.db` resolves
# `alembic.ini` and `migrations/`, and the API resolves `frontend/`, relative to the
# repository root. An editable install keeps that one layout instead of scattering the
# same files across site-packages and a source tree.
ARG PYTHON_IMAGE=python:3.12.14-slim-trixie@sha256:2c941e860699f878900b0edc2403613c234d4b32eda3cc9fa7036991a2a63c4a
# ── the uploader, fetched and verified outside the final layer ────────────────
FROM ${PYTHON_IMAGE} AS uploader
ARG IMMICH_GO_VERSION=0.32.0
ARG IMMICH_GO_SHA256_AMD64=6e2ad86bafdadb9466d6515de7cb882726c0aea1a21d51164dff361d7d480a97
ARG IMMICH_GO_SHA256_ARM64=2c35d9284baae407ef9540bdac5f488971b0bdc7be758a4d7c05ab270af09fdb
COPY docker/fetch-immich-go.py /tmp/fetch-immich-go.py
RUN python /tmp/fetch-immich-go.py \
--version "${IMMICH_GO_VERSION}" \
--sha256-amd64 "${IMMICH_GO_SHA256_AMD64}" \
--sha256-arm64 "${IMMICH_GO_SHA256_ARM64}" \
--into /usr/local/bin \
&& /usr/local/bin/immich-go version
# ── the application ──────────────────────────────────────────────────────────
FROM ${PYTHON_IMAGE} AS runtime
ARG EXIFTOOL_VERSION=13.25+dfsg-1
ARG IMMICH_GO_VERSION=0.32.0
# The library is mounted from the host, so the container's identity must match the
# ownership that library already has: everything this application renames, writes
# EXIF into, or archives has to stay owned by the host user afterwards.
ARG UID=1000
ARG GID=1000
LABEL org.opencontainers.image.title="photo_pipeline" \
org.opencontainers.image.source="https://github.com/domverse/photoanalyzer" \
io.photoanalyzer.exiftool.version="${EXIFTOOL_VERSION}" \
io.photoanalyzer.immich-go.version="${IMMICH_GO_VERSION}"
ENV PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
PATH=/opt/venv/bin:$PATH \
PHOTO_PIPELINE_DATA_DIR=/data
RUN set -eu; \
apt-get update; \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
"libimage-exiftool-perl=${EXIFTOOL_VERSION}"; \
rm -rf /var/lib/apt/lists/*
COPY --from=uploader /usr/local/bin/immich-go /usr/local/bin/immich-go
WORKDIR /app
COPY pyproject.toml alembic.ini README.md ./
COPY photo_pipeline ./photo_pipeline
COPY migrations ./migrations
COPY frontend ./frontend
# The manuals are served by the application itself (US09-01), so they ship with it.
COPY docs ./docs
COPY docker/entrypoint.sh docker/healthcheck.sh /usr/local/bin/
# Runtime dependencies only: the `test` extra (pytest, playwright) and the `vision`
# extra stay out, and pip's build isolation leaves no build tooling behind.
RUN set -eu; \
python -m venv /opt/venv; \
/opt/venv/bin/pip install --no-cache-dir -e .
# What is installed must be what was pinned, or the labels and the version record
# would be a claim rather than a fact.
RUN set -eu; \
mkdir -p /etc/photo-pipeline; \
exiftool_version="$(exiftool -ver)"; \
immich_go_version="$(immich-go version | head -n 1 | tr -d '\r')"; \
expected_exiftool="$(printf '%s' "${EXIFTOOL_VERSION}" | cut -d+ -f1 | cut -d- -f1)"; \
[ "${exiftool_version}" = "${expected_exiftool}" ] \
|| { echo "exiftool ${exiftool_version} is not the pinned ${expected_exiftool}" >&2; exit 1; }; \
case "${immich_go_version}" in \
*"${IMMICH_GO_VERSION}"*) ;; \
*) echo "immich-go '${immich_go_version}' is not pinned ${IMMICH_GO_VERSION}" >&2; exit 1 ;; \
esac; \
printf '{\n "exiftool": "%s",\n "immich-go": "%s"\n}\n' \
"${exiftool_version}" "${IMMICH_GO_VERSION}" > /etc/photo-pipeline/versions.json
# Non-root, with the host library's ownership. /data is the persistent volume; the
# photo library itself is mounted by the deployment (US08-03), never baked in.
RUN set -eu; \
groupadd --gid "${GID}" --non-unique app; \
useradd --uid "${UID}" --gid "${GID}" --non-unique --no-create-home --home-dir /app app; \
mkdir -p /data; \
chown "${UID}:${GID}" /data
USER ${UID}:${GID}
EXPOSE 8000
# Readiness, not liveness: an unmigrated or misconfigured database answers
# /api/v1/health/ready with 503, and a container that cannot serve must not be
# reported healthy. The worker role has no endpoint, so its check is a no-op here.
HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
CMD ["/usr/local/bin/healthcheck.sh"]
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
CMD ["serve"]

441
README.md
View File

@@ -4,70 +4,19 @@ Integrated, restart-safe photo analysis, duplicate review, metadata, upload, and
archive workflow. Planning lives in `INTEGRATED_PIPELINE_CONCEPT.md` and
`delivery_backlog/`.
## Documentation
The manuals live in [`docs/`](docs/index.md) and are the same files the running
application serves at `/app/#/docs`:
| | |
|---|---|
| [Overview](docs/overview.md) | what it does and the rules it will not break |
| [Installation and operations](docs/installation.md) | host and container install, every setting, backup and restore, every refusal |
| [A guided first pass](docs/first-pass.md) | one library from scan to verified upload |
| [The stages](docs/index.md) | one illustrated page each, from inventory to archive |
| [Errors and refusals](docs/errors.md) | every error code, cause, and remedy |
| [Architecture](docs/architecture.md) | diagrams, module map, journals, invariants |
This README stays the repository's own notes — how the code is built, tested, and
released. Anything an operator or a user needs belongs in `docs/`, and one gate
(`python -m photo_pipeline docs-gate`) keeps those pages honest against the code.
## Application (`photo_pipeline`)
The target application lives in `photo_pipeline/` (FastAPI + SQLAlchemy + Alembic).
Install it into a virtualenv once:
Run it with:
```bash
python3.12 -m venv .venv
.venv/bin/pip install -e ".[vision]" # drop [vision] for a review-only install
python -m photo_pipeline migrate # apply database migrations
python -m photo_pipeline serve # start the API + static review UI (127.0.0.1:8000)
```
Then run the two processes:
```bash
.venv/bin/python -m photo_pipeline migrate # apply database migrations
.venv/bin/python -m photo_pipeline serve # API + review UI at 127.0.0.1:8000/app/
.venv/bin/python -m photo_pipeline worker # second terminal: runs the jobs
```
The server enqueues work and serves the UI; nothing actually scans, scores,
analyses, uploads, or archives without a worker. `work_item/scripts/python` is the
*helper's* launcher — it prefers Conda base and falls back to a bare system
interpreter, so it is not how the application is run.
Configuration comes from `PHOTO_PIPELINE_*` environment variables (see
`photo_pipeline/config.py`); secrets are referenced, never logged.
### Configuration file
`.env` in the working directory is read at startup, or any path named by
`PHOTO_PIPELINE_ENV_FILE`. It is parsed, never executed: `KEY=value` lines,
`#` comments, optional quotes — no interpolation and no `export`. **Anything already
exported wins**, so the file is the standing configuration and the shell is the
override for one run.
The archived CLI's variable names still work, so an existing `photo_analyzer.env`
can be used as-is:
| in the file | applied as |
|---|---|
| `LLM_API_KEY` / `GEMINI_API_KEY` | `OPENAI_API_KEY` |
| `LLM_BASE_URL` | `OPENAI_BASE_URL` |
| `LIBRARY` | `PHOTO_PIPELINE_LIBRARY_ROOTS` |
`.env` and `*.env` are gitignored and denied by the work-item safety checks: the
file holds a real key and must never be committed.
### API access (US07-02)
The app listens on loopback, so its attacker is another page in the same browser.
@@ -87,212 +36,6 @@ not a loopback name (DNS rebinding), when `Origin` is any other origin, when
thumbnail), or when the body exceeds `PHOTO_PIPELINE_MAX_REQUEST_BYTES`. There is no
CORS middleware at all, so no other origin can read a response.
### Reaching it through a hostname or proxy (US08-01)
| variable | meaning |
|---|---|
| `PHOTO_PIPELINE_ALLOWED_HOSTS` | comma-separated extra names the app answers to; empty means loopback only |
| `PHOTO_PIPELINE_ACCESS_SECRET` | traded for the session cookie at `GET /api/v1/session` via `X-Access-Secret` |
| `PHOTO_PIPELINE_TRUSTED_PROXIES` | comma-separated peer addresses whose `X-Forwarded-Proto`/`X-Forwarded-Host` are believed |
Being reachable *was* the authentication: whoever could open `127.0.0.1:8000` owned
the library. So naming any non-loopback host — or binding to one, `0.0.0.0` included
— makes the access secret mandatory, and `serve` refuses to start without it rather
than publishing the library. Loopback-only deployments need no secret and behave
exactly as before.
```bash
curl -sc /tmp/pp.jar -H "X-Access-Secret: $PHOTO_PIPELINE_ACCESS_SECRET" \
https://photos.example.com/api/v1/session
```
The browser asks for the secret once per tab and keeps it in `sessionStorage`.
Wrong secrets are rate-limited (5 per minute) and logged with the caller's address
only. `Host` and `Origin` are judged against the configured names; the *external*
scheme and host come from the forwarded headers only when the request arrived from a
`PHOTO_PIPELINE_TRUSTED_PROXIES` address, so a client cannot declare its own origin,
and the session cookie is marked `Secure` when that external scheme is HTTPS. Health
endpoints stay reachable without the secret so an orchestrator can restart the
container; nothing else does.
## Container image (US08-02)
One image runs either role. It is built from a clean checkout with no arguments:
```bash
docker build -t photo-pipeline:dev .
```
Everything external is pinned, and the build fails rather than drifting: the Python
base image by tag *and* digest, `exiftool` by its Debian package version (verified
against `exiftool -ver`), and `immich-go` by release version and per-architecture
SHA-256 of the release asset. The verified versions become image labels and
`/etc/photo-pipeline/versions.json`, which `diagnostics` reports as `tools[].pinned`
beside the version actually installed — so a replaced binary shows up as a
`tool_version_drift` warning instead of as a misparsed upload report.
| build argument | default | why change it |
|---|---|---|
| `UID` / `GID` | `1000` | must match the owner of the mounted photo library |
| `PYTHON_IMAGE` | pinned digest | upgrading the base image |
| `EXIFTOOL_VERSION` | Debian package version | upgrading exiftool |
| `IMMICH_GO_VERSION` + `IMMICH_GO_SHA256_AMD64`/`_ARM64` | pinned release | upgrading the uploader (take the digests from that release's `checksums.txt`) |
The first argument is the role, and every other management command still works:
```bash
docker run --rm -v /srv/photos:/srv/photos -v pp-data:/data \
-e PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos photo-pipeline:dev migrate
docker run -d -p 127.0.0.1:8000:8000 -v /srv/photos:/srv/photos -v pp-data:/data \
-e PHOTO_PIPELINE_HOST=0.0.0.0 -e PHOTO_PIPELINE_ACCESS_SECRET=... \
-e PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos photo-pipeline:dev serve
docker run -d -v /srv/photos:/srv/photos -v pp-data:/data \
-e PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos photo-pipeline:dev worker
```
One role per container: `serve` and `worker` each take the library process lock for
their role (US07-05), so no supervisor starts both. The container refuses to run as
UID 0 — files it renames or writes must keep the ownership the host library expects —
and `/data` is the persistent volume holding the database, journals, backups, and
thumbnail cache. Binding to `0.0.0.0` makes the access secret mandatory
([above](#reaching-it-through-a-hostname-or-proxy-us08-01)); `serve` refuses to start
without it. The declared `HEALTHCHECK` polls `/api/v1/health/ready`, so a container
whose database is unmigrated or misconfigured is never reported healthy.
## Composed runtime (US08-03)
`docker-compose.yml` is the deployment: one `serve` container, one `worker`
container, one bind-mounted library, one data volume, and a one-shot `migrate` that
both roles wait for.
```bash
cp .env.example .env && $EDITOR .env # nothing has a value in it; fill in yours
docker compose up -d --build
```
`.env.example` lists every `PHOTO_PIPELINE_*` variable with its default in a comment
and no values at all. The three the composition cannot start without are
`PHOTO_PIPELINE_LIBRARY_HOST_PATH` (the library on this host),
`PHOTO_PIPELINE_LIBRARY_ROOTS` (where it is mounted *inside* the container), and
`PHOTO_PIPELINE_ACCESS_SECRET` — publishing the port means the app is reachable from
outside the container, so US08-01 makes the secret mandatory. Set
`PHOTO_PIPELINE_UID`/`GID` to the owner of the library: what the containers rename
and rewrite keeps that ownership.
| invariant | how the composition keeps it |
|---|---|
| one writer | `serve` and `worker` take their role's library lock (US07-05) in the shared `/data` volume, so `--scale worker=2` is refused by the lock, not by convention |
| migrations first | `migrate` runs the backup-then-migrate path and must exit 0 before `api` and `worker` start; a failed upgrade leaves the previous database and its pre-migration backup intact |
| container paths | `PHOTO_PIPELINE_LIBRARY_ROOTS` is both the mount target and the configured root; a root that is not mounted makes `serve`/`worker` exit 5 at startup instead of writing into the container's throwaway layer |
| loopback by default | the port is published to `127.0.0.1` unless `PHOTO_PIPELINE_PUBLISH_ADDRESS` says otherwise, and exposing it needs the hostname in `PHOTO_PIPELINE_ALLOWED_HOSTS` plus the access secret |
| restart safety | both roles are `restart: unless-stopped` with a 30 s stop grace period, and a job interrupted by a restart resumes exactly as it does on a host restart |
The `data` volume holds the database, its write-ahead log, the thumbnail cache,
journals, and backups. **It must stay on a local filesystem** — SQLite in WAL mode
needs real local locking, so NFS, SMB, and network volume drivers are unsupported
there and corrupt the database rather than slow it down. The library bind mount has
no such restriction.
Operating the deployment is operating the same CLI:
```bash
docker compose run --rm --no-deps api diagnostics
docker compose run --rm --no-deps api backup --reason pre-upgrade
docker compose run --rm --no-deps api verify-backup /data/backups/<name>
docker compose run --rm --no-deps api restore /data/backups/<name> --into /data/restored
docker compose logs -f worker
```
`--no-deps` keeps a one-off command from starting a second stack; `api` is only the
service the command borrows the image and mounts from. `restore` is deliberately not
an API call: it replaces the state of an installation and belongs to a stopped one,
so stop `api` and `worker` first and restart them against the restored directory.
## Publishing and deploying (US08-04)
Two workflows in `.gitea/workflows/` make `main` the only path to the running stack:
| workflow | runs on | does |
|---|---|---|
| `test.yml` (**Test**) | every pull request, and every push to `main` | installs exiftool, the package, and Chromium, then runs the suites configured in `work_item/.work-item.yml` |
| `deploy.yml` (**Deploy**) | a **successful** `Test` run on `main`, or manual dispatch | builds the image, pushes it, triggers the Portainer webhook, prunes dangling layers |
Deploy waits for `Test` through `workflow_run`, so publishing is downstream of a green
suite rather than of a push. `completed` is not `success`: the publish job runs only on
`conclusion == 'success'`. Because an unmatched `workflow_run` filter does not fail but
simply never fires, `tests/integration/test_deploy_workflows.py` asserts the link from
both ends — renaming either workflow, or dropping one of the configured suites out of
the test job, fails that test rather than silently unhooking the gate.
The image is `git.domverse-berlin.eu/domverse/photoanalyzer`, tagged with the commit
SHA and `latest`. The commit tag is pushed first, so a `latest` that exists always has
a commit tag beside it and **a rollback is a tag change, not a rebuild**:
```bash
docker pull git.domverse-berlin.eu/domverse/photoanalyzer:<known-good-sha>
docker tag git.domverse-berlin.eu/domverse/photoanalyzer:<known-good-sha> \
git.domverse-berlin.eu/domverse/photoanalyzer:latest
docker push git.domverse-berlin.eu/domverse/photoanalyzer:latest
# then trigger the Portainer webhook, or redeploy the stack from Portainer
```
Manual dispatch defaults to **dry run**: it builds and pushes `scratch-<sha>` only, and
touches neither `latest` nor the running stack. Clear the `dry_run` input to publish by
hand. A workflow-level `concurrency: deploy` group with `cancel-in-progress: false`
queues deploys instead of overlapping them, so two commits can never race to `latest`.
### Secrets
Repository secrets — used only by the pipeline:
| secret | purpose |
|---|---|
| `REGISTRY_USER` | registry user with `write:package` |
| `REGISTRY_TOKEN` | that user's token, passed on stdin, never as an argument |
| `PORTAINER_WEBHOOK_URL` | the stack's auto-update POST URL; `curl --fail` makes a refused redeploy a failed workflow |
Runtime secrets — `PHOTO_PIPELINE_VISION_API_KEY`, `PHOTO_PIPELINE_IMMICH_API_KEY`,
`PHOTO_PIPELINE_ACCESS_SECRET` — are **not** repository secrets and are not in the
image. The stack is managed by Portainer from git (`docker-compose.yml`), and those
values live in the Portainer stack's environment, so rotation is one place and a
repository read discloses nothing. A test asserts the workflows never name them.
## Container acceptance gate (US08-05)
The deployed container is verified the way the host application is, by one command:
```bash
work_item/scripts/python -m photo_pipeline container-gate --output gate-evidence
```
It builds the image, provisions the composition against a **temporary fixture library
on a bind mount and an isolated data volume**, runs the `phase_h` journeys against it,
destroys every stack afterwards, and writes `release-report.json`, `logs/container.log`,
and `CHECKSUMS.sha256` into the evidence directory. It exits non-zero when a journey
fails, when the story matrix has a hole, **or when any check skipped at all** — unlike
the release gate, this one accepts no environment excuse: a run that never reached the
containers proved nothing about them.
The journeys (`tests/e2e/test_phase_h_container.py`) are:
| journey | what it proves |
|---|---|
| browser | discovery, duplicate review, analysis, album proposal, rename, upload preflight, and archive views, driven through the containerized frontend — including a rename that really moves the operator's folder on the bind mount |
| upgrade | the previous version's image runs first, then this one against the same volume: schema at the new head, assets, analysis results, job progress, the unapplied rename plan, and the thumbnail cache all survive |
| restart | `docker kill` on both containers mid-job; the job resumes, every photo ends with exactly one stored result, and only the in-flight item ever reaches the provider twice |
| security | no session refused, a forged `X-Forwarded-Host` cannot smuggle an allowed hostname past the check, a symlink out of the mounted library is refused, an unmounted library root refuses startup, and no secret appears in `docker compose logs` |
The upgrade journey builds the previous commit's tree when no published image is named;
point it at the real one with `PHOTO_PIPELINE_PREVIOUS_IMAGE`. On a Docker VM (Colima,
Docker Desktop) the fixture library must live on a shared path — it defaults to
`~/.cache/photo-pipeline`, overridable with `PHOTO_PIPELINE_TEST_MOUNT_BASE`.
CI runs this gate as the `container` job of **Test** on pushes to `main` and keeps its
evidence as a run artefact. Deploy waits for the whole `Test` workflow, so a red
container gate is a publish that does not happen.
## Testing
One offline command runs the whole suite (unit, integration, and browser
@@ -538,184 +281,6 @@ file in the temporary library are copied to `.artifacts/<test id>/` before pytes
deletes the directory. Point `PHOTO_PIPELINE_TEST_ARTIFACTS` elsewhere to collect
them from CI.
## Release gate (US07-07)
One command runs every suite in an isolated stack and keeps the evidence:
```bash
work_item/scripts/python -m photo_pipeline release-gate --output data/release/$(date -u +%Y%m%dT%H%M%SZ)
```
It fails — and exits non-zero — when any stage fails, when a suite skips a test for
a reason that is not a documented environment limit (`exiftool not installed`,
`root ignores directory permissions`), or when the story matrix has a hole. The
evidence directory holds `release-report.json` (revision, per-stage result, timings,
summaries), `logs/<stage>.log`, and `CHECKSUMS.sha256` over both.
**The story matrix** lives in `tests/story_traceability.json`: every story under
`delivery_backlog/stories/` is either mapped to test files that exist, or listed in
`planned` as an accepted but unimplemented story. A story that is neither, or a
mapping to a file that has been deleted, fails the gate.
**The journey** (`tests/e2e/test_release_journey.py`) takes one fresh library through
discovery, duplicate review, safety, analysis, EXIF verification, album proposal,
guarded rename, rescan, upload with server-side verification, archive, offline
deduplication, and restore — over HTTP against real server and worker processes,
with a full restart in the middle and at the end.
### Real-library dry run and approval
Before the application is pointed at photos that cannot be replaced:
```bash
work_item/scripts/python -m photo_pipeline dry-run --output dry-run.json
work_item/scripts/python -m photo_pipeline approve-dry-run dry-run.json --approver "$(whoami)"
```
The dry run is strictly read-only: it opens no file for writing, writes no database
row, and reports what it found — file counts by extension, folders, bytes, unreadable
files, excluded directories, and a reconciliation against what the database already
knows (already registered, new, recorded but absent). Set
`PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL=1` and **every mutating API request is
refused with `403 dry_run_not_approved`** until a report for exactly those library
roots has been approved. Reading stays open — you have to be able to see what was
found in order to approve it — and so does taking a backup. Change the library roots
and the approval no longer applies: it approves that reconciliation, not the idea of
mutating.
## Performance budgets (US07-06)
Budgets are measured, not asserted in prose. `python -m photo_pipeline benchmark`
builds a synthetic library of a stated size, runs the same scenarios every time,
writes a machine-readable report, and **exits non-zero when a budget is breached**.
```bash
work_item/scripts/python -m photo_pipeline benchmark --profile smoke # ~2 s, runs in CI
work_item/scripts/python -m photo_pipeline benchmark --profile short # 25k assets
work_item/scripts/python -m photo_pipeline benchmark --profile full \
--output data/benchmarks/full.json # 25k + 100k
work_item/scripts/python -m photo_pipeline benchmark --profile huge \
--soak-seconds 3600 --output data/benchmarks/soak.json # 500k + soak
```
| Metric | Budget | Why |
|---|---|---|
| `latency_p95_ms` | 250 ms | a list or search page must feel immediate |
| `latency_max_ms` | 2 000 ms | no single page may stall the review flow |
| `rss_growth_bytes` | 400 MB | a run must not leak the library |
| `open_files` | 256 | file descriptors are a hard operating-system limit |
| `wal_bytes` | 200 MB | a growing write-ahead log means checkpoints are starving |
| `queue_depth` | 1 000 | an unbounded queue is an out-of-memory in waiting |
| `cache_over_quota_bytes` | 0 | the thumbnail cache has to respect its quota |
Measured on the reference machine (Apple Silicon, SQLite WAL), p95 per scenario:
| Scenario | 25k | 100k |
|---|---|---|
| `inventory_page` | 0.5 ms | 0.6 ms |
| `library_search` | 4.8 ms | 17.1 ms |
| `library_stats` | 56.8 ms | 197.4 ms |
| `workflow_readiness` | 52.6 ms | 235.4 ms |
| `duplicate_cluster_list` | 0.6 ms | 0.5 ms |
| `duplicate_cluster_page` | 1.7 ms | 1.7 ms |
CI runs the `smoke` profile through `tests/integration/test_performance_budgets.py`;
the 25k/100k/500k matrix and the multi-hour soak belong to scheduled infrastructure,
because minutes of build time do not belong in the suite that runs on every change.
**Exceptions.** A budget that cannot be met is not a warning to ignore: it goes into
`APPROVED_EXCEPTIONS` in `photo_pipeline/services/benchmarks.py` with its raised
limit, who approved it, why, and a review date. Every report lists the exceptions it
applied, so a release review sees them.
Approved today, both for the 500k `huge` profile only, review by 2027-02-17:
| Scenario | Measured at 500k | Raised limit |
|---|---|---|
| `library_stats` | 1.08 s p95 · 3.2 s max | 1.5 s p95 · 4 s max |
| `workflow_readiness` | 1.40 s p95 · 3.3 s max | 1.8 s p95 · 4 s max |
Both are library-wide aggregates — the current safety decision of every asset, and
the album/tag/year breakdown of every analysis row — and both meet the 250 ms budget
at the 100k rows the concept sets it for. Beyond that they are linear against one
SQLite writer; the fix is denormalized totals or the planned PostgreSQL transition,
not a query tweak. Everything else at 500k is inside budget, and a soak at that size
grows neither resident memory nor the job queue.
## Backup and recovery (US07-05)
Backups go through SQLite's online backup API, never a file copy: with WAL enabled
the `.db` file alone is missing every committed page still in the write-ahead log.
Each backup is a directory under `data/backups/` holding the snapshot and a
`manifest.json` describing it — schema revision, SHA-256, row counts, the archive
media the library depends on, and which configuration was set. Secrets are recorded
as `configured`, never as values, so a manifest is safe to attach to a bug report.
```bash
work_item/scripts/python -m photo_pipeline backup --reason before-upgrade --keep 7
work_item/scripts/python -m photo_pipeline verify-backup data/backups/<name>
work_item/scripts/python -m photo_pipeline diagnostics
```
The same is available at `GET /api/v1/diagnostics`, `GET|POST /api/v1/backups`,
`GET /api/v1/backups/{name}/verify`, and `POST /api/v1/backups/prune`. **Restore is
not an endpoint** — it replaces the state of an installation, so it belongs to a
stopped one and a person at a terminal.
### Integrity check
`verify-backup` runs `PRAGMA integrity_check` (structure) *and*
`PRAGMA foreign_key_check` (references), compares the snapshot's SHA-256 with the
manifest, and re-counts every table the manifest recorded. Any mismatch — bit rot, a
truncated copy, a "repaired" snapshot — fails the check, and `restore` refuses a
backup that does not verify.
### Restore drill
1. Stop the server and the worker.
2. `python -m photo_pipeline verify-backup data/backups/<name>` — never restore an
unverified snapshot.
3. `python -m photo_pipeline restore data/backups/<name> --into /path/to/fresh-data`
(a target that already holds a database is refused; recovering in place means
moving the old data directory aside first).
4. Point `PHOTO_PIPELINE_DATA_DIR` at the restored directory and run
`python -m photo_pipeline migrate`.
5. Run an inventory scan so paths are reconciled against the real library.
6. Mount every archive location named in the manifest before archiving again — the
database records where archived originals are, but it does not contain them.
Practise this against a copy before you need it; the drill is exercised
automatically by `tests/integration/test_backup_recovery.py`.
### Failed migration
A pending schema upgrade is snapshotted first (`reason: pre-migration`), by both the
API startup and `python -m photo_pipeline migrate`. If a migration fails, the error
log names the backup directory: stop everything and run the restore drill against
it. An up-to-date database is not backed up again on every start.
### Archive media
Archived originals live on their medium, not in the backup. The manifest lists every
archive location with its `media_id` and whether it was mounted when the backup was
taken. Keep one copy of each medium off-site, and remount a location before
restoring assets from it.
### Retention and disk
`--keep N` (default 7) prunes the oldest backups and never the newest.
`diagnostics` reports the database, write-ahead log, thumbnail cache, uploader
reports, backups, and logs separately, with free space and warnings for low disk
(`disk_low`, `disk_critical`), a cache over its quota, a write-ahead log outgrowing
its database, and a legacy CLI writing the library.
### Process locking
`serve` and `worker` take a JSON lock in the data directory (`api.lock.json`,
`worker.lock.json`). A second worker exits `2` and names the holder; a lock whose
process is gone is taken over. If the frozen CLI's state files are being written,
both refuse with exit `3` — `--allow-legacy` overrides, and you own the outcome.
## Legacy CLI archive
The command-line tools this application was extracted from are frozen in

32
delivery_backlog/E08-container-deployment.md
View File

@@ -1,32 +0,0 @@
# E08 — Container Deployment
Concept phase: none. This epic is a delivery-format addition on top of the concept:
the same application, same safety invariants, packaged as a Docker image and deployed
continuously from Gitea Actions instead of being started by hand from a working copy.
It does not change the product scope in
[`INTEGRATED_PIPELINE_CONCEPT.md`](../INTEGRATED_PIPELINE_CONCEPT.md). SQLite stays the
store, one worker stays the writer, the library process lock stays authoritative, and
no path outside the configured library roots becomes reachable because the process now
runs in a container.
One decision does extend the concept and is made here explicitly: the application may
be reached through a reverse proxy under a real hostname, not only over loopback. That
requires a configurable trust boundary and an authentication gate, because the
loopback-only checks of US07-02 are what currently stand in for authentication.
## Stories
1. [US08-01 — Make the trust boundary configurable and authenticated](stories/US08-01-trusted-hosts-auth.md)
2. [US08-02 — Build a reproducible application image](stories/US08-02-container-image.md)
3. [US08-03 — Compose the runtime and mount the library safely](stories/US08-03-compose-runtime.md)
4. [US08-04 — Publish and deploy from Gitea Actions](stories/US08-04-gitea-cicd.md)
5. [US08-05 — Automate container deployment acceptance](stories/US08-05-container-e2e.md)
## Epic outcome
A tagged image built from `main` runs the API and the worker as separate containers
against a mounted library and a persistent data volume, is published to the Gitea
registry, is redeployed by webhook, survives restart and upgrade with its database and
journals intact, and refuses every request that a loopback deployment would have
refused.

77
delivery_backlog/E09-documentation.md
View File

@@ -1,77 +0,0 @@
# E09 — Product Documentation
Concept phase: none. Like [E08](E08-container-deployment.md), this epic is a delivery
addition rather than a product-scope change: the same application, documented well
enough that somebody who did not build it can install it, understand it, and operate it
without reading the source.
It does not change the product scope in
[`INTEGRATED_PIPELINE_CONCEPT.md`](../INTEGRATED_PIPELINE_CONCEPT.md). No safety
invariant moves, no schema changes, no new runtime capability. The one change to the
running application is a documentation view and the static assets it needs.
## Why the application serves its own documentation
The manuals describe an application that is reached over HTTP behind an access secret.
Documentation that lives only in the repository is unreachable from the deployment it
describes: an operator who has just been handed a URL and a secret has no Gitea account
in front of them. So the same markdown files are both the repository's documentation and
the deployment's `/docs` view, and neither is a copy of the other.
## Decisions made in this epic
**Markdown is the source.** Everything is written as markdown under `docs/`, so it is
reviewable in a diff, readable on Gitea, and renderable in the app. No documentation
format that only a tool can read.
**The renderer is vendored, not written and not fetched.** `marked` (MIT, no
dependencies, ships an ES module) is pinned and committed under
`frontend/js/vendor/`. A CDN is not an option: the application is deployed to a network
whose outbound access is not assumed, and `default-src 'self'` forbids it.
**Diagrams are mermaid, rendered client-side, with the script boundary intact.** The
claim that mermaid requires `'unsafe-eval'` was tested rather than believed: mermaid 11's
bundle contains no `eval(` and no `new Function` — only a lodash `Function("return this")`
global-detection fallback that short-circuits on `globalThis` and never executes.
Rendered under this application's exact CSP, a flowchart produced a 15.8 KB SVG and
raised **no `script-src` violation**. What it does raise is `style-src`: mermaid styles
its output with an injected `<style>` element and `style=` attributes.
So `script-src 'self'` stays exactly as US07-02 and US08-01 left it, and `style-src`
gains `'unsafe-inline'`. That relaxation is bounded on purpose: with `script-src` intact
no injected markup can execute, and with `img-src`, `connect-src`, and `font-src` all
still `'self'`, the CSS-based exfiltration channels stay closed. What remains is
defacement of a page the operator is already authenticated on.
The rejected alternative is recorded because it may become the better trade later:
pre-rendering each diagram to a committed SVG with the already-installed Playwright
(no Node toolchain needed) and serving that, which would leave CSP untouched entirely at
the cost of a generator and a freshness check. If `style-src 'unsafe-inline'` is ever
judged too much, that is the migration, and it does not change a single markdown file.
**Screenshots are produced, not pasted.** Every screenshot in the user manual is captured
by Playwright from the real application against a temporary fixture library, by the same
kind of code that already drives the browser suites. A screenshot nobody can regenerate
is a screenshot that silently stops being true.
**The documentation is held to the code by tests.** Configuration keys, CLI commands,
exit codes, API error codes, job and journal states, and module names all appear in both
the code and the manuals. Each of those is cross-checked, so the failure mode of stale
documentation is a red test rather than a misled operator.
## Stories
1. [US09-01 — Serve the documentation inside the application](stories/US09-01-docs-in-app.md)
2. [US09-02 — Write the installation and operations manual](stories/US09-02-installation-manual.md)
3. [US09-03 — Write the architecture overview](stories/US09-03-architecture-overview.md)
4. [US09-04 — Write the user manual with generated screenshots](stories/US09-04-user-manual.md)
5. [US09-05 — Automate documentation acceptance](stories/US09-05-docs-gate.md)
## Epic outcome
A deployed instance serves its own installation manual, architecture overview, and
illustrated user manual at `/app/#/docs`, rendered from the same markdown files that are
readable in the repository. Screenshots are regenerated from the running application,
every documented command, setting, state, and error code is cross-checked against the
code that implements it, and one gate fails the build when documentation and application
disagree.

9
delivery_backlog/README.md
View File

@@ -2,14 +2,11 @@
This backlog decomposes the phases in
[`INTEGRATED_PIPELINE_CONCEPT.md`](../INTEGRATED_PIPELINE_CONCEPT.md) into seven
epics and small, independently verifiable user stories, plus two delivery-format
epics: E08 packages the released application as a deployable container, and E09
documents it for the people who install, operate, and use it.
epics and small, independently verifiable user stories.
## Numbering and file naming
- Epics: `E01` through `E07`, matching concept Phases A through G; `E08` and `E09`
have no concept phase and must not change product scope.
- Epics: `E01` through `E07`, matching concept Phases A through G.
- Stories: `US<epic>-<sequence>`, for example `US03-02`.
- Epic files: `E01-<slug>.md`.
- Story files: `stories/US01-01-<slug>.md`.
@@ -39,8 +36,6 @@ documents it for the people who install, operate, and use it.
5. [E05 — Immich upload](E05-immich-upload.md)
6. [E06 — Archive lifecycle](E06-archive-lifecycle.md)
7. [E07 — Hardening and release](E07-hardening-release.md)
8. [E08 — Container deployment](E08-container-deployment.md)
9. [E09 — Product documentation](E09-documentation.md)
## Shared definition of done

42
delivery_backlog/stories/US08-01-trusted-hosts-auth.md
View File

@@ -1,42 +0,0 @@
# US08-01 — Make the Trust Boundary Configurable and Authenticated
Epic: [E08](../E08-container-deployment.md)
As an operator, I want to reach the application through my own hostname without
weakening it, so a container behind a reverse proxy is as safe as the loopback
deployment it replaces.
## Context
`photo_pipeline/api/security.py` refuses any request whose `Host` or `Origin` is not
loopback. That check is the current stand-in for authentication: whoever can reach
`127.0.0.1:8000` is the owner. Behind a proxy the hostname is no longer loopback, so
relaxing the check without adding an authentication gate would publish the library.
## Acceptance criteria
- Allowed hosts and origins come from configuration (`PHOTO_PIPELINE_*`), default to
the current loopback set, and an unset configuration behaves exactly as today.
- Whenever a non-loopback host is configured, startup requires an access secret and
refuses to serve without one; loopback-only deployments keep working with no secret.
- The secret is exchanged for the existing session cookie and CSRF token through the
bootstrap endpoint; every protected route keeps its current session and CSRF
requirements unchanged.
- Forwarded headers (`X-Forwarded-Proto`, `X-Forwarded-Host`) are honored only from a
configured trusted proxy and ignored otherwise, so a client cannot forge its origin.
- Cookies are marked `Secure` when the effective external scheme is HTTPS.
- Failed authentication is rate-limited and logged without the secret, the session id,
or any request body.
- Health endpoints stay reachable without the secret; nothing else does.
## Automated tests
- Unit tests for host/origin evaluation across loopback default, configured host,
unconfigured host, forged forwarded headers, and trusted-proxy forwarded headers.
- Integration tests: startup refusal without a secret, successful exchange, wrong
secret, replay of an old session, cross-site request, and unauthenticated access to
every route class.
## Dependencies
- US07-02

41
delivery_backlog/stories/US08-02-container-image.md
View File

@@ -1,41 +0,0 @@
# US08-02 — Build a Reproducible Application Image
Epic: [E08](../E08-container-deployment.md)
As an operator, I want one image that can run either application role, so deployment is
a pull instead of a Python environment I have to reproduce by hand.
## Context
The application shells out to `exiftool` and `immich-go`, writes into the library as a
normal filesystem user, and serves a static frontend from `frontend/`. All three have to
be true inside the image, or the container starts and then fails on the first real
operation.
## Acceptance criteria
- A `Dockerfile` builds from a pinned Python base, installs the project and its runtime
dependencies, and contains no test, playwright, or build-only tooling in the final
layer.
- `exiftool` and `immich-go` are present at pinned versions, and their versions are
recorded in the image and reported by `python -m photo_pipeline diagnostics`.
- The image runs as a non-root user whose UID/GID are build-time arguments, so files
the application renames or writes keep the ownership the host library expects.
- One entrypoint selects the role: `serve` or `worker`, passing through the existing
CLI arguments; no supervisor runs two roles in one container.
- `serve` containers declare a `HEALTHCHECK` against `/api/v1/health/ready`, so an
unmigrated or misconfigured database is not reported healthy.
- The image contains no secrets, no library data, no database, and no `.git`; the build
context is constrained by `.dockerignore`.
- Image build is reproducible from a clean checkout and documented in `README.md`.
## Automated tests
- A build-and-run test asserts the image starts, reports ready, serves the frontend
index, and returns the pinned `exiftool` and `immich-go` versions.
- A test asserts the container refuses to run as UID 0 and that a file created by the
container is owned by the configured UID/GID.
## Dependencies
- US07-05

47
delivery_backlog/stories/US08-03-compose-runtime.md
View File

@@ -1,47 +0,0 @@
# US08-03 — Compose the Runtime and Mount the Library Safely
Epic: [E08](../E08-container-deployment.md)
As an operator, I want a single compose file that runs the API and the worker against my
real library, so a deployment is one command and the safety invariants survive it.
## Context
The library process lock (US07-05) assumes both roles see the same lock file, and SQLite
in WAL mode assumes a real local filesystem. Container path policy is the same problem
as host path policy with a new failure mode: the configured library roots must name the
in-container mount paths, not the host paths.
## Acceptance criteria
- `docker-compose.yml` runs exactly one `serve` and one `worker` container from the same
image and the same data volume, and a second worker is refused by the existing lock
rather than by convention.
- The library is a bind mount; `PHOTO_PIPELINE_LIBRARY_ROOTS` names the container-side
paths, and a mismatch between mounted and configured roots fails at startup with a
clear message instead of at the first write.
- The data volume holds the database, WAL, thumbnail cache, and backups on a local
filesystem; the composition documents that a network mount is unsupported for it.
- Migrations run before `serve` and `worker` accept work, using the existing backup-then-
migrate path, and an upgrade that fails leaves the previous database intact.
- Configuration and secrets come from the environment, never from the image or a
committed file; a `.env.example` lists every `PHOTO_PIPELINE_*` variable with safe
defaults and no values.
- The API port is published to host loopback by default; exposing it publicly requires
the configured hostname and access secret from US08-01.
- Containers restart automatically, and a restart mid-job resumes exactly as a host
restart does today.
- Backup, verify-backup, restore, and diagnostics are documented as container commands
and work against the mounted volumes.
## Automated tests
- An integration test brings the composition up against a temporary fixture library,
runs a job, restarts both containers, and asserts the job resumes and the database is
intact.
- Tests for: second worker refused, library-root mismatch refused at startup, failed
migration leaving the previous database restorable.
## Dependencies
- US08-01, US08-02

40
delivery_backlog/stories/US08-04-gitea-cicd.md
View File

@@ -1,40 +0,0 @@
# US08-04 — Publish and Deploy from Gitea Actions
Epic: [E08](../E08-container-deployment.md)
As a release owner, I want `main` to build, publish, and redeploy the image
automatically, so deployment is the same reproducible path every time.
## Context
The workflow is adapted from the `crowdsec-admin` deployment workflow
(`.gitea/workflows/deploy.yml` in that repository): build, log in to the Gitea registry,
push, trigger a Portainer webhook, prune. This project needs the same shape plus a test
gate, because unlike that project it has a required suite that must not be skipped.
## Acceptance criteria
- `.gitea/workflows/` contains a test workflow that runs on pull requests and on `main`,
executing the configured required suites, and a deploy workflow that runs only after
the tests pass on `main` and on manual dispatch.
- The deploy workflow publishes to `git.domverse-berlin.eu` under this project's own
image path, tagged `latest` and the commit SHA, so a rollback is a tag change.
- Registry credentials and the Portainer webhook come from repository secrets; runtime
secrets (vision key, Immich key, access secret) stay in the Portainer stack and never
enter the repository or the image.
- Redeploy is triggered by webhook and the workflow fails when the webhook call fails.
- Dangling images are pruned; published tags are not.
- A concurrency guard prevents two deploys of different commits overlapping.
- `README.md` documents the required secrets, the image path, the rollback procedure,
and that the stack is managed by Portainer from git.
## Automated tests
- Workflow files are validated (syntax and required job/step names) by a repository test
so a rename cannot silently disable the test gate.
- A dry-run job builds and pushes to a scratch tag on manual dispatch without touching
`latest` or triggering a redeploy.
## Dependencies
- US08-02, US08-03

30
delivery_backlog/stories/US08-05-container-e2e.md
View File

@@ -1,30 +0,0 @@
# US08-05 — Automate Container Deployment Acceptance
Epic: [E08](../E08-container-deployment.md)
As a release owner, I want one automated gate that proves the deployed container, so the
packaged application is verified the same way the host application is.
## Acceptance criteria
- One documented command provisions the composition from the built image against a
temporary fixture library and an isolated data volume, and destroys it afterwards.
- A browser journey against the containerized application covers discovery, duplicate
review, analysis, album proposal, rename, upload preflight, and archive views.
- An upgrade journey runs the previous published image, then the new one, and asserts
migrations, journals, jobs, and the thumbnail cache survive.
- A restart journey kills both containers mid-job and asserts resume without duplicate
side effects.
- Security gates run against the deployed instance: unauthenticated access refused,
forged forwarded headers refused, paths outside the mounted library roots refused, and
no secret in container logs.
- Evidence is retained per run and the gate fails on any skipped required check.
## Automated tests
- The container acceptance suite runs on a `phase_h` marker in CI on `main` and before a
published deploy; earlier epic suites keep running unchanged.
## Dependencies
- US08-01 through US08-04

49
delivery_backlog/stories/US09-01-docs-in-app.md
View File

@@ -1,49 +0,0 @@
# US09-01 — Serve the Documentation Inside the Application
Epic: [E09](../E09-documentation.md)
As an operator who was handed a URL and an access secret, I want the manuals inside the
application I am looking at, so that understanding it does not require a repository
checkout or an internet connection.
## Acceptance criteria
- Documentation lives as markdown under `docs/`, with an index that names every page and
the order it is meant to be read in. The same files render on Gitea without
modification.
- The application serves a `/docs` view reachable from the main navigation, listing the
pages and rendering the selected one.
- Rendering uses a vendored, version-pinned markdown library committed under
`frontend/js/vendor/`. It is not fetched from a CDN, not bundled by a build step, and
not written by hand. The build records the version and a checksum of the vendored file.
- Diagrams written as ```mermaid``` fenced blocks render as diagrams. Mermaid is vendored
the same way.
- `script-src` in the Content-Security-Policy is unchanged: no `'unsafe-eval'`, no
`'unsafe-inline'`. Only `style-src` gains `'unsafe-inline'`, and the reason is recorded
where the policy is defined.
- Links between documents work in the app: a relative `../foo.md#section` link navigates
to that page and heading rather than downloading a file or leaving the application.
Every heading has a stable anchor, and a deep link to an anchor scrolls to it.
- An unknown page renders a documentation-specific not-found message with a link back to
the index, and never exposes a filesystem path.
- Documentation is readable without an access secret **or** behind the same session as
the rest of the application — whichever is chosen is stated explicitly in the story's
implementation notes and covered by a test, because an operator locked out by a
configuration mistake is exactly who needs the troubleshooting page.
- The view works with no outbound network access at all.
## Automated tests
- Unit: heading-anchor slugs (duplicates, punctuation, non-ASCII), and the rewriting of
relative markdown links into in-app routes.
- Integration: every markdown file under `docs/` is reachable from the index; every
internal link in every document resolves to a file and, where an anchor is given, to a
heading that exists; the vendored library files match their recorded checksums.
- Browser: the documentation view renders a page, follows an internal link, deep-links to
an anchor, renders a mermaid diagram to an SVG, and shows the not-found message for an
unknown page — all with **zero console errors and zero CSP violations**, asserted, not
eyeballed.
## Dependencies
- None beyond the delivered application.

46
delivery_backlog/stories/US09-02-installation-manual.md
View File

@@ -1,46 +0,0 @@
# US09-02 — Write the Installation and Operations Manual
Epic: [E09](../E09-documentation.md)
As somebody installing this application for the first time, I want one manual that takes
me from nothing to a running instance pointed at my photo library, so that I do not have
to reconstruct the procedure from the README, the compose file, and the test suite.
## Acceptance criteria
- A **host installation** path: prerequisites and their versions (Python, exiftool,
immich-go, Playwright's browser for the test suite), virtual environment, dependency
install, `.env`, database migration, starting `serve` and `worker`, and how to verify
the install succeeded.
- A **container installation** path: the published image, the compose file, the data
volume, the library bind mount and why it is mounted the way it is, published ports,
running behind a reverse proxy with `PHOTO_PIPELINE_ALLOWED_HOSTS` and
`PHOTO_PIPELINE_TRUSTED_PROXIES`, and the Portainer/webhook deployment already in use.
- A **configuration reference**: every `PHOTO_PIPELINE_*` setting with its meaning,
default, accepted values, and whether it is a secret. Secrets are described, never
exemplified with a real-looking value.
- A **first run checklist** that ends in a verified state: library discovered, worker
claiming jobs, readiness endpoint green, diagnostics clean.
- **Operations**: upgrading (including what the migration backup does), backup, verify,
restore, pruning, diagnostics, and the log locations for each role.
- **Troubleshooting**: each refusal an operator can hit at startup — every non-zero exit
code of the CLI, the trust-boundary refusal, the unmounted library root refusal, the
library lock being held, and a legacy CLI still running — with the cause and the fix.
- The manual states the safety invariants an installer must not work around: one worker
writes, `_IGNORE/` is never read, EXIF is verified before upload, and the library lock
is authoritative.
## Automated tests
- Every `PHOTO_PIPELINE_*` setting documented exists as a field on `Config`, and every
field on `Config` is documented — both directions, so a new setting cannot ship
undocumented.
- Every CLI command and flag named in the manual exists in the argument parser, and every
subcommand the parser accepts appears in the manual.
- Every exit code documented is one the CLI can actually return.
- No documented example value collides with a real secret pattern (the repository's own
credential scanner runs over `docs/`).
## Dependencies
- US09-01

47
delivery_backlog/stories/US09-03-architecture-overview.md
View File

@@ -1,47 +0,0 @@
# US09-03 — Write the Architecture Overview
Epic: [E09](../E09-documentation.md)
As a developer or reviewer new to this repository, I want an architecture overview that
explains what the pieces are and which rules they enforce, so that I can find the right
module and not violate an invariant I never knew existed.
## Acceptance criteria
- A **context diagram**: the operator, the browser application, the API and worker, the
photo library, the Immich server, the vision provider, and the archive destination —
with the direction and nature of every interaction, including which ones leave the
machine.
- A **runtime diagram**: the migrate/api/worker composition, the data volume, the library
bind mount, the SQLite database, and the process lock — showing what is shared and what
is exclusive.
- A **module map**: every package under `photo_pipeline/` with its responsibility and the
boundary it must not cross (which modules may touch the filesystem, which may call an
external provider, which own schema).
- **Key flows**, each as a diagram plus prose: the durable job lifecycle from enqueue to
recovery; the rename journal state machine including the rollback and manual-resolution
paths; the upload lifecycle through preflight, run, report ingestion, and verification.
- **The invariants and where they are enforced**, each pointing at the module that owns
it: one writer at a time, the library process lock, the path policy and library roots,
`_IGNORE/` exclusion, verified EXIF before upload, safety decisions gating the vision
provider, and restart-safety of every mutation.
- **The data model**: the tables, what identity means for an asset, and how a moved file
keeps its identity.
- A short **history and donor** section: what was migrated out of the legacy CLIs, where
the archive and its ledger live, and why they are kept.
- Diagrams are ```mermaid``` blocks so the source is diffable and Gitea renders them
natively.
## Automated tests
- Every package and top-level module under `photo_pipeline/` appears in the module map,
and every module named in the map exists — a new service cannot appear on the map's
blind side.
- Every job state, journal state, and upload batch state named in the document exists in
the code, and every state the code defines is named in the document.
- Every mermaid block parses (rendered in the browser test without an error node).
- Every code path referenced by file name exists.
## Dependencies
- US09-01

52
delivery_backlog/stories/US09-04-user-manual.md
View File

@@ -1,52 +0,0 @@
# US09-04 — Write the User Manual with Generated Screenshots
Epic: [E09](../E09-documentation.md)
As the person actually sorting a photo library, I want a manual that walks the workflow
screen by screen and tells me what each refusal means, so that I can use the application
confidently and know what it is about to do to my files before it does it.
## Acceptance criteria
- One page per workflow stage, in the order the application presents them: discovery and
inventory, duplicate review, safety review, analysis, album proposals, renames, upload,
archive, and diagnostics. Each page answers the same four questions: what this stage is
for, what I have to decide, **what it changes on disk or on the server**, and what it
refuses to do.
- A **guided first pass** that takes a new library from scan to a verified upload, naming
the point of no return in each stage and what is reversible after it.
- Every stage page carries at least one screenshot of the real application showing the
state being described.
- Screenshots are **generated** by a committed Playwright script that seeds a temporary
fixture library, drives the application, and writes the images. Regenerating them is one
documented command. No screenshot is captured by hand.
- Screenshots contain no real library paths, no personal photos, and no secrets — the
fixture library is synthetic, and this is asserted rather than assumed.
- An **error and refusal catalogue**: every `error.code` the API can return, with what
causes it, what the application did or refused to do, and what the operator should do
next. Refusals that protect data (`lock_held`, `rename_recovery_required`,
`stale_preflight`, `path_not_allowed`, `host_not_allowed`, `dry_run_not_approved`,
conflict codes) are explained as intentional, not as faults.
- A **recovery** page: an interrupted rename, an uncertain upload, a failed migration, a
restored backup — what the application does by itself and what needs a decision.
- The work-item safety allow list is extended to permit `docs/images/**`, since the
repository denies image files by default. This is an explicit, reviewed change, not a
quiet one, and it stays narrow enough that a real photo still cannot be committed.
## Automated tests
- Every `error.code` the application can emit is documented, and every code documented
exists in the code — both directions.
- Every image referenced by a documentation page exists, and every image under
`docs/images/` is referenced by a page.
- The screenshot generator runs end to end in the test environment and produces every
image the manual references, against a temporary fixture library that is destroyed
afterwards.
- Generated screenshots are checked for library paths outside the fixture root and for
the configured secrets.
- Browser: each stage page renders in the documentation view with its screenshot loaded
and no console error.
## Dependencies
- US09-01

38
delivery_backlog/stories/US09-05-docs-gate.md
View File

@@ -1,38 +0,0 @@
# US09-05 — Automate Documentation Acceptance
Epic: [E09](../E09-documentation.md)
As a release owner, I want one gate that proves the documentation still describes the
application, so that a change to the code cannot quietly make the manuals wrong.
## Acceptance criteria
- One documented command runs every documentation check and retains its evidence, in the
shape the release and container gates already use.
- The gate fails when: an internal link or anchor is dead; a page is unreachable from the
index; an image is referenced but missing or present but unreferenced; a documented
setting, command, exit code, error code, or state does not exist in the code; a code
path exists that the documentation is required to cover and does not.
- The gate regenerates the screenshots and fails when a regenerated image no longer
matches the committed one beyond a stated tolerance, so a UI change that invalidates the
manual is a red build rather than a discovery months later.
- The gate renders every documentation page in a real browser and fails on any console
error or CSP violation, and asserts that `script-src` contains neither `'unsafe-eval'`
nor `'unsafe-inline'`.
- The checks run on a `phase_i` marker; CI runs the gate, and the earlier epic suites keep
running unchanged.
- The README and the application's documentation index point at each other, so neither is
the forgotten copy.
- Documentation stories are mapped in the story traceability matrix like every other
story.
## Automated tests
- The gate's own contract is testable without a browser: a seeded broken link, a missing
image, an undocumented error code, and a stale screenshot each fail it, and a clean tree
passes.
- The full documentation suite runs on `phase_i` in CI.
## Dependencies
- US09-01 through US09-04

105
docker-compose.yml
View File

@@ -1,105 +0,0 @@
# The deployed runtime (US08-03): one API, one worker, one library, one volume.
#
# The same image (US08-02) runs both roles, so what is composed here is process
# topology, not a second application. Three invariants shape it:
#
# * one writer — `serve` and `worker` each take the library process lock for their
# role (US07-05), and both containers mount the *same* data volume, which is what
# makes the lock file visible to both. Scaling `worker` past 1 is refused by that
# lock rather than by anyone remembering not to;
# * one local filesystem — the database, its write-ahead log, the thumbnail cache,
# and the backups live in the `data` volume, and SQLite in WAL mode requires real
# local-filesystem locking. A network mount (NFS, SMB, or a cloud volume driver)
# is unsupported for it; that is a corrupted database, not a slow one;
# * one library path vocabulary — `PHOTO_PIPELINE_LIBRARY_ROOTS` names the path
# *inside* the container, which is also the bind mount's target below. A root
# that is not mounted there makes `serve` and `worker` refuse at startup instead
# of writing into the container's throwaway layer.
#
# Configuration and secrets come from the environment only: copy `.env.example` to
# `.env` and fill it in. Nothing is baked into the image and nothing with a value in
# it is committed.
#
# cp .env.example .env && $EDITOR .env
# docker compose up -d --build
#
# Operating it is operating the same CLI — `docker compose run --rm --no-deps api
# <command>` — see README, "Composed runtime".
name: photo-pipeline
x-runtime: &runtime
image: ${PHOTO_PIPELINE_IMAGE:-photo-pipeline:dev}
build:
context: .
args:
# Everything the app renames or rewrites has to stay owned by the host user
# the library already belongs to.
UID: ${PHOTO_PIPELINE_UID:-1000}
GID: ${PHOTO_PIPELINE_GID:-1000}
user: "${PHOTO_PIPELINE_UID:-1000}:${PHOTO_PIPELINE_GID:-1000}"
env_file:
- ${PHOTO_PIPELINE_ENV_FILE:-.env}
volumes:
- data:/data
- "${PHOTO_PIPELINE_LIBRARY_HOST_PATH:?set PHOTO_PIPELINE_LIBRARY_HOST_PATH to the photo library on this host}:${PHOTO_PIPELINE_LIBRARY_ROOTS:?set PHOTO_PIPELINE_LIBRARY_ROOTS to the container-side library path}"
# Jobs check for cancellation between items and leave a resumable record; a
# too-short grace period turns an orderly stop into a recovery on next start.
stop_grace_period: 30s
x-environment: &environment
# Set here rather than left to the file: these two are what the composition itself
# promises, and an `.env` that disagreed would move the database off the volume or
# the library off its mount.
PHOTO_PIPELINE_DATA_DIR: /data
PHOTO_PIPELINE_LIBRARY_ROOTS: ${PHOTO_PIPELINE_LIBRARY_ROOTS}
services:
# Migrations run to completion before either role accepts work, through the same
# backup-then-migrate path the roles use (US07-05): a pending upgrade is snapshotted
# first, and a failed one exits non-zero with the backup named — so `api` and
# `worker` never start, and the previous database is left intact and restorable.
migrate:
<<: *runtime
command: ["migrate"]
environment: *environment
restart: "no"
api:
<<: *runtime
command: ["serve"]
environment:
<<: *environment
# Published to host loopback below. Inside the container the server must bind
# the container's own interface for that publish to reach it, which is exactly
# what makes the access secret mandatory (US08-01) — `serve` refuses to start
# without one. Exposing the port beyond loopback additionally needs
# PHOTO_PIPELINE_ALLOWED_HOSTS to name the hostname it is reached under.
PHOTO_PIPELINE_HOST: 0.0.0.0
PHOTO_PIPELINE_PORT: 8000
ports:
# Host side only: PHOTO_PIPELINE_PORT in `.env` moves the *published* port, and
# the container always serves 8000, which is what the image's health check probes.
- "${PHOTO_PIPELINE_PUBLISH_ADDRESS:-127.0.0.1}:${PHOTO_PIPELINE_PORT:-8000}:8000"
depends_on:
migrate:
condition: service_completed_successfully
restart: unless-stopped
# One worker. A second one is refused by the library lock in the shared data
# volume, which is the point: `docker compose up --scale worker=2` fails loudly
# instead of running two writers against one library.
worker:
<<: *runtime
command: ["worker", "--id", "worker-1"]
environment: *environment
depends_on:
migrate:
condition: service_completed_successfully
restart: unless-stopped
volumes:
# Local driver on purpose: the database, WAL, thumbnail cache, and backups need a
# real local filesystem. Do not point this at NFS, SMB, or a network volume driver.
data:
driver: local

22
docker/entrypoint.sh
View File

@@ -1,22 +0,0 @@
#!/bin/sh
# One entrypoint, one role per container (US08-02).
#
# The first argument is the management command the image runs — `serve` and `worker`
# are the two roles, and every other `python -m photo_pipeline` command (migrate,
# diagnostics, backup, restore, dry-run) is passed through unchanged so operating the
# container is operating the same CLI. No supervisor: two roles in one container would
# share a process lock they are each meant to hold alone (US07-05).
set -eu
if [ "$(id -u)" = "0" ]; then
echo "refusing to run as root: start this image with a non-root UID/GID so files" \
"it renames or writes keep the ownership the mounted library expects" >&2
exit 1
fi
role="${1:-serve}"
# The health check has to know which role it is checking, and only the API has an
# endpoint to check. /tmp is writable for the unprivileged user; /run may not be.
printf '%s' "${role}" > "${PHOTO_PIPELINE_ROLE_FILE:-/tmp/photo-pipeline-role}" 2>/dev/null || true
exec python -m photo_pipeline "$@"

72
docker/fetch-immich-go.py
View File

@@ -1,72 +0,0 @@
"""Download one pinned immich-go release and verify it before unpacking (US08-02).
Run at image build time by the `uploader` stage, with the interpreter that is already
in the base image: no curl, no wget, and no download tooling in the layer that ships.
The checksum is not advisory — a release asset that does not match the pinned digest
is a failed build, not a warning, because the uploader's flags and report format are
what the upload parser is written against (concept §15).
"""
from __future__ import annotations
import argparse
import hashlib
import platform
import tarfile
import tempfile
import urllib.request
from pathlib import Path
RELEASE_URL = "https://github.com/simulot/immich-go/releases/download/v{version}/{asset}"
# Debian/BuildKit architecture as the interpreter sees it → release asset name.
ASSETS = {
"x86_64": ("immich-go_Linux_x86_64.tar.gz", "amd64"),
"amd64": ("immich-go_Linux_x86_64.tar.gz", "amd64"),
"aarch64": ("immich-go_Linux_arm64.tar.gz", "arm64"),
"arm64": ("immich-go_Linux_arm64.tar.gz", "arm64"),
}
TIMEOUT_SECONDS = 300
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--version", required=True, help="immich-go release, without the v")
parser.add_argument("--sha256-amd64", required=True)
parser.add_argument("--sha256-arm64", required=True)
parser.add_argument("--into", default="/usr/local/bin")
args = parser.parse_args()
machine = platform.machine().lower()
if machine not in ASSETS:
raise SystemExit(f"unsupported architecture: {machine}")
asset, arch = ASSETS[machine]
expected = {"amd64": args.sha256_amd64, "arm64": args.sha256_arm64}[arch]
url = RELEASE_URL.format(version=args.version, asset=asset)
with urllib.request.urlopen(url, timeout=TIMEOUT_SECONDS) as response: # noqa: S310
payload = response.read()
digest = hashlib.sha256(payload).hexdigest()
if digest != expected:
raise SystemExit(f"checksum mismatch for {url}: {digest} != {expected}")
target = Path(args.into)
target.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory() as work:
archive = Path(work) / asset
archive.write_bytes(payload)
with tarfile.open(archive) as tar:
member = tar.getmember("immich-go")
# Extract exactly the one file this pin is about, by name, so nothing
# else in the archive can decide where it lands.
extracted = tar.extractfile(member)
if extracted is None:
raise SystemExit("release archive contains no immich-go binary")
binary = target / "immich-go"
binary.write_bytes(extracted.read())
binary.chmod(0o755)
print(f"immich-go {args.version} ({arch}) verified {digest}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

34
docker/healthcheck.sh
View File

@@ -1,34 +0,0 @@
#!/bin/sh
# Container health for the `serve` role: readiness, not liveness (US08-02).
#
# /api/v1/health/ready is 503 until the database is reachable, migrated, and in WAL
# mode with foreign keys on, so an unmigrated or misconfigured container never reports
# healthy. Health endpoints need no session and no access secret, which is what lets an
# orchestrator restart a container it holds no credentials for (US08-01).
set -eu
role="$(cat "${PHOTO_PIPELINE_ROLE_FILE:-/tmp/photo-pipeline-role}" 2>/dev/null || echo unknown)"
if [ "${role}" != "serve" ]; then
# ponytail: the worker has no endpoint to probe; its liveness is its lease and job
# heartbeat in the database. Add a `worker --health` command if a restart policy
# ever needs to act on it.
exit 0
fi
port="${PHOTO_PIPELINE_PORT:-8000}"
exec python - "${port}" <<'PY'
import sys
import urllib.error
import urllib.request
url = f"http://127.0.0.1:{sys.argv[1]}/api/v1/health/ready"
try:
with urllib.request.urlopen(url, timeout=5) as response: # noqa: S310 — loopback
sys.exit(0 if response.status == 200 else 1)
except urllib.error.HTTPError as error:
print(f"not ready: HTTP {error.code}", file=sys.stderr)
sys.exit(1)
except OSError as error:
print(f"not ready: {error}", file=sys.stderr)
sys.exit(1)
PY

224
docs/architecture.md
View File

@@ -1,224 +0,0 @@
# Architecture
[← Documentation index](index.md)
For whoever has to change this code without breaking somebody's photo library. It
explains what the pieces are, which rules each one keeps, and where to look when a
stage refuses.
The product decisions behind all of it live in `INTEGRATED_PIPELINE_CONCEPT.md`; this
page describes what was built.
## Context
Five things outside the application, and what actually crosses each boundary.
```mermaid
flowchart LR
operator([Operator]):::person -->|browser, one session| app
app[Photo Pipeline]:::system -->|read, rename, EXIF write| library[(Photo library<br/>bind mount)]
app -->|database, cache, journals, backups| data[(Data directory<br/>local filesystem)]
app -->|confirmed-SFW images only| vision[Vision provider]:::ext
app -->|immich-go, verified bytes| immich[Immich server]:::ext
app -->|copy, verify, then remove| archive[(Archive medium)]
classDef person fill:#1f6feb,stroke:#58a6ff,color:#fff
classDef system fill:#238636,stroke:#3fb950,color:#fff
classDef ext fill:#6e40c9,stroke:#a371f7,color:#fff
```
| boundary | leaves the machine? | carries |
|---|---|---|
| operator → app | no (loopback, or a proxy you configured) | commands against stable ids, never paths |
| app → library | no | reads, folder renames, EXIF merges |
| app → data directory | no | SQLite in WAL mode, thumbnails, journals, backups |
| app → vision provider | **yes** | image bytes of confirmed-SFW canonical assets only |
| app → Immich | **yes** | the exact verified bytes of an approved album |
| app → archive medium | no | copies, verified before the source is removed |
## Runtime
Two long-lived processes and a one-shot migration, sharing one database and one
library.
```mermaid
flowchart TD
browser([Browser]) -->|JSON + SSE| api
migrate["migrate<br/>backup, then upgrade"] -->|must exit 0| api
migrate --> worker
api["api · serve<br/>enqueues, serves, reads"] -->|jobs table| db[(SQLite WAL)]
worker["worker<br/>claims and does the work"] -->|jobs table| db
worker --> tools["exiftool · vision API · immich-go"]
api -.->|api.lock.json| lock{{library lock}}
worker -.->|worker.lock.json| lock
api --> lib[(library)]
worker --> lib
```
`serve` enqueues and renders; **it does not do the work**. Everything that scans,
scores, analyses, renames, uploads, or archives happens in the worker, which claims
a queued job atomically with a fencing token. One mutating job runs at a time, and
the lock file in the data directory is what makes a second worker impossible rather
than merely discouraged.
## The modules
| package | owns | must not |
|---|---|---|
| `api/` | HTTP surface, error envelope, security policy | contain SQL or business logic |
| `api/routes/` | one module per resource group, all under `/api/v1` | accept a filesystem path from the browser |
| `api/security.py` | host/origin/CSRF/session/size policy as one pure `evaluate`, plus the ASGI middleware | be bypassed per-route |
| `schemas/` | Pydantic request and response contracts | reach the database |
| `services/` | all domain logic; the only place a decision is made | be imported by the frozen CLI archive |
| `models/` | SQLAlchemy tables | hold behaviour |
| `jobs/` | durable job lifecycle, worker loop, lock ranks, handler registry | run work in the API process |
| `integrations/` | the real outside world: `exiftool`, vision, NSFW model, `immich-go` and its report grammars | be called without a version recorded |
| `path_policy.py` | the library boundary, `_IGNORE/` exclusion, symlink-escape refusal | be duplicated anywhere |
| `imaging.py` | the single bounded-decode door | let a caller open an image directly |
| `faults.py` | the crash barriers the tests fire | read anything but its one env var |
| `config.py` | typed settings, secrets as `SecretStr` | log or return a value |
| `db.py` | engine, sessions, WAL and foreign keys, migration entry | be used to bypass a repository |
Services worth knowing by name: `inventory`, `duplicates`, `safety`, `analysis`,
`albums`/`proposals`/`naming`, `renames`/`rename_apply`/`rename_journal`,
`uploads`/`upload_batches`/`upload_reports`/`upload_verification`,
`archives`/`archive_transfer`/`archive_journal`/`restores`/`availability`,
`thumbnails`, `hashing`, `exif_checkpoint`, `jobs`, `workflow`, `backup`,
`diagnostics`, `app_lock`, `release`, `benchmarks`, `library`, `legacy_import`.
## The state machines
Three journals decide what a restart is allowed to assume. All three are read from
the database plus the real world — never guessed from a missing file.
### Durable jobs
```mermaid
stateDiagram-v2
[*] --> queued
queued --> running: claimed with a fencing token
queued --> cancelled
queued --> cancelling
running --> succeeded
running --> failed
running --> cancelling
cancelling --> cancelled
failed --> retry_queued
retry_queued --> running
succeeded --> [*]
cancelled --> [*]
```
`succeeded` and `cancelled` are terminal, so a duplicate delivery cannot move a
finished job. Claiming is compare-and-set on the row version; a worker whose lease
expired cannot commit after another has taken over.
### The rename journal
The only state machine that moves somebody's folders.
```mermaid
stateDiagram-v2
[*] --> planned
planned --> moving
moving --> moved
moving --> planned: proven untouched
moving --> rollback_required
moved --> database_updated
database_updated --> verified
verified --> complete
moved --> rollback_required
database_updated --> rollback_required
verified --> rollback_required
planned --> failed
failed --> planned
rollback_required --> rolled_back
complete --> [*]
rolled_back --> [*]
```
Intent is written **before** the disk is touched, which is why `moving` can resolve
backwards: the evidence decides. `moving`, `moved`, `database_updated`, and
`rollback_required` are the unsafe states — while any operation sits in one, the
library may be half-renamed, so unrelated mutations are refused with
`409 rename_recovery_required` until a person resolves it.
### Upload batches
```mermaid
stateDiagram-v2
[*] --> planned
planned --> running
running --> succeeded
running --> failed
running --> cancelling
running --> unknown_requires_verification: process died after acceptance
cancelling --> cancelled
failed --> running: retry
cancelled --> running: retry
```
`unknown_requires_verification` is deliberately **not** restartable. The server may
already hold the files; the answer is to ask Immich for the recorded SHA-1, not to
upload again and hope.
Archive transfers use the same shape — `planned → transferring → verified →
removing → complete` — and the source is removed only after the archived bytes are
verified.
## Identity and the data model
A path is metadata. The identity is `assets.id`, a UUID that never changes, and
`asset_paths` records every path an asset has ever had with the reason it changed.
That is what makes a rename cheap: nothing else in the database has to move.
Tables: `assets`, `asset_paths`, `thumbnails`, `jobs`, `job_items`, `job_events`,
`safety_reviews`, `analysis_results`, `exif_projections`, `duplicate_clusters`,
`duplicate_members`, `duplicate_negative_links`, `album_proposals`, `rename_plans`,
`rename_operations`, `upload_batches`, `upload_items`, `upload_verifications`,
`archive_locations`, `archive_plans`, `archive_operations`.
Availability is independent of workflow progress: `active`, `archiving`,
`archived_online`, `archived_offline`, `restoring`, `missing_unexpected`. An
unmounted archive disk is `archived_offline`, never `missing` — the scanner is not
allowed to conclude that a photo is gone because a disk is unplugged.
EXIF is a projection with its own verified state (`verified`, `divergent`,
`failed`): the desired values are written, read back, and compared, and anything the
stage does not own having changed makes the asset `divergent` and blocks the next
mutating stage.
## Where each invariant lives
| invariant | enforced in |
|---|---|
| `_IGNORE/` is never traversed, counted, or opened | `path_policy.is_excluded`, used by every discovery path |
| no path outside the library roots is reachable | `path_policy.resolve_in_roots` — it returns the resolved path, because validating one name and opening another is the symlink race |
| one writer per library | `services/app_lock.py` (an `flock` on a JSON lock file, per role) |
| one mutating job at a time | `services/jobs.py` lock keys plus `jobs/locks.py` rank ordering |
| only confirmed-SFW assets reach the vision provider | `services/analysis.py`, re-checked *after* the provider call so a decision that flipped mid-flight discards the result |
| EXIF is verified, and other fields preserved | `services/exif_checkpoint.py` |
| uploads carry the exact verified bytes | `services/uploads.py` preflight, re-proved immediately before the uploader runs |
| an uncertain upload is not a failure | `services/upload_batches.py`, `services/upload_verification.py` |
| the archive source outlives its copy until verified | `services/archive_transfer.py` |
| decoding is bounded | `imaging.py` |
| the trust boundary and CSRF | `api/security.py` |
## Concurrency
Locks are taken broad to narrow — `library → stage/job → album/folder → asset` — and
never the other way, which is what makes deadlock structural rather than lucky.
Ranks live in `jobs/locks.py`.
Exactly-once execution is not achievable across SQLite, a filesystem, subprocesses,
and a remote server. The design promises **at-least-once with idempotent recovery**:
every handler may run twice, and running twice must not produce two side effects.
That is why the rename journal records intent before moving, why EXIF writes merge
and verify, and why upload retries consult both local history and Immich.
## History
This application was extracted from two command-line tools rather than written from
nothing. Their sources are frozen in `legacy_cli_archive/` with the ledger mapping
each donated behaviour to the service that now owns it, the characterization tests
that pinned it, and every intentional difference. Production code must not import
them; they are provenance and rollback evidence.

95
docs/errors.md
View File

@@ -1,95 +0,0 @@
# Errors and refusals
[← Documentation index](index.md)
Every error the API returns carries a code:
```json
{"error": {"code": "lock_held", "message": "a library_write job is already running"}}
```
Most of them are **refusals, not faults**. This application would rather stop and
explain than guess about somebody's photographs, so a code below usually means it
protected something. Each row says what caused it and what to do.
## Access and the trust boundary
| code | cause | what to do |
|---|---|---|
| `unauthenticated` | no application session | reload the page; the browser bootstraps one |
| `access_denied` | wrong or missing access secret | check `PHOTO_PIPELINE_ACCESS_SECRET`. Attempts are logged with the caller's address only |
| `too_many_attempts` | more than five failed secret attempts in a minute | wait. This is the rate limit, not a lockout |
| `csrf_failed` | a mutation without the session's token | reload; a stale tab has an old token |
| `host_not_allowed` | the `Host` header is not a configured name | add the real hostname to `PHOTO_PIPELINE_ALLOWED_HOSTS` |
| `origin_not_allowed` | the request came from another origin | not something a browser tab of this app produces |
| `cross_site_blocked` | another site triggered the request | expected — this is the protection working |
| `payload_too_large` | the body exceeds `PHOTO_PIPELINE_MAX_REQUEST_BYTES` | send less; every endpoint takes small commands |
| `path_not_allowed` | a path resolved outside the library roots | usually a symlink. Nothing outside the roots is reachable, by design |
## The safety gate
| code | cause | what to do |
|---|---|---|
| `dry_run_not_approved` | mutation is gated until a dry run is approved | run `dry-run`, read it, then `approve-dry-run` |
| `approval_scope_mismatch` | the approval covers different library roots | approve a report for the roots actually configured |
| `approval_unreadable` | the approval record cannot be read | re-approve; do not edit it by hand |
## Concurrency and staleness
| code | cause | what to do |
|---|---|---|
| `lock_held` | a mutating job already holds the lane | wait for it. One at a time is what makes a crash recoverable |
| `version_conflict` | you acted on a version that changed underneath you | the view re-renders with the server's truth; decide again |
| `invalid_transition` | a state change that the machine does not allow | usually a stale tab; reload |
| `job_error` | the job service refused the request | the message says why |
| `rename_recovery_required` | an interrupted rename is unresolved | resolve it in [Renames](stages/renames.md). Unrelated mutations stay blocked on purpose |
## Stage refusals
| code | cause | what to do |
|---|---|---|
| `nothing_to_score` | no photo is eligible for safety scoring | the queue is already decided |
| `nothing_eligible` | no photo is eligible for analysis | resolve safety decisions first |
| `nothing_to_plan` | no approved album name to build a plan from | approve a proposal first |
| `invalid_decision` | the decision is not one this cluster accepts | reload the cluster |
| `invalid_proposal` | the name is empty, reserved, or has forbidden characters | fix the name; the rules are in [Album proposals](stages/albums.md) |
| `unknown_album` | the album is not a folder the library knows | rescan |
| `cannot_apply` | the plan is invalid, stale, or another plan is unresolved | read the blockers in the preview |
| `cannot_rollback` | the operation is complete or its preconditions no longer hold | rollback is recovery, not undo |
## Upload
| code | cause | what to do |
|---|---|---|
| `stale_preflight` | bytes changed between approval and running | re-run preflight; this is the check that stops the wrong bytes being uploaded |
| `not_runnable` | the batch is in a state that must not be re-run | an uncertain batch is verified, never retried |
| `requires_verification` | the outcome is unknown; the server may hold the files | verify against Immich, then resolve |
| `changed_after_upload` | the file changed after a successful upload | uploading again may create or upgrade an asset |
## Operations
| code | cause | what to do |
|---|---|---|
| `backup_failed` | the snapshot could not be taken or verified | check free space and the message |
| `invalid_retention` | a `--keep` value that is not a positive count | the newest backup is never pruned |
| `not_found` | no such id | usually a stale link |
## Generic
| code | cause |
|---|---|
| `invalid_request` | the request body failed validation. Field names only — never the values, which end up in logs and screenshots |
| `http_error` | a plain HTTP-level refusal, with its status |
| `internal_error` | an unhandled error. The code is all you get; the traceback is in the server log, because it can carry paths and credentials |
## Diagnostics warnings
Not errors — reported by `diagnostics` and worth acting on. They are listed with
their meanings in [Diagnostics](stages/diagnostics.md): `disk_low`, `disk_critical`,
`cache_over_quota`, `wal_growth`, `tool_version_drift`, `legacy_process_active`, and
`no_roots`.
## Startup refusals
Exit codes 2 to 5 are covered by the installation manual under
[when it refuses to start](installation.md#when-it-refuses-to-start).

66
docs/first-pass.md
View File

@@ -1,66 +0,0 @@
# A guided first pass
[← Documentation index](index.md)
One library, from nothing to a verified upload. Each stage links to its own page; this
is the order, and — more importantly — where the points of no return are.
![The workflow home, with every stage and its counts](images/workflow.png)
The workflow page is the home view and the honest summary: each stage shows its state,
its counts, why it is blocked if it is, and the one action that moves it forward.
| # | stage | reversible afterwards? |
|---|---|---|
| 0 | [Inventory](stages/inventory.md) | nothing was changed |
| 1 | [Duplicate review](stages/duplicates.md) | yes — decisions can be changed, no file is deleted |
| 2 | [Safety review](stages/safety.md) | the decision, yes; the EXIF keyword is a file write |
| 3 | [Analysis](stages/analysis.md) | the result, yes; the caption and keywords are a file write |
| 4 | [Album proposals](stages/albums.md) | yes — approving renames nothing |
| 5 | [Renames](stages/renames.md) | **your folders move.** Recoverable, journaled, but real |
| 6 | [Upload](stages/uploads.md) | **assets exist on the Immich server.** Not undone from here |
| 7 | [Archive](stages/archive.md) | **originals leave active storage.** Restorable while the medium is reachable |
| — | [Diagnostics](stages/diagnostics.md) | read-only, any time |
## Before the first run
Take a backup of the photo library itself. This application is careful — it previews,
journals, and verifies — but it is the first time you are pointing it at your
photographs, and a backup is cheaper than confidence.
If the library is irreplaceable, turn on `PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL`
before anything else. Every mutating request is then refused until you have produced
a read-only reconciliation report and approved it by name.
## The pass
1. **Scan.** Inventory → *Rescan*. Reads only. Check the count, and check that
nothing under `_IGNORE/` appears.
2. **Resolve duplicates.** Exact byte matches can be accepted as recommended; anything
fuzzy gets looked at. Resolving first is what stops you paying to analyse the same
photo twice.
3. **Decide safety.** Every canonical photo becomes `sfw` or `nsfw`. Nothing reaches
the vision provider until it is confirmed SFW — this is the gate the whole design
exists around.
4. **Analyse.** Only confirmed-SFW photos. Each result is written to the database and
projected into EXIF, then read back and verified.
5. **Propose album names.** Evidence from what was analysed. Edit anything you disagree
with. Approving changes no file.
6. **Build the rename plan, read it, apply it.** The preview shows every old → new
path. This is where folders move.
7. **Rescan.** Paths are reconciled; identities are unchanged.
8. **Upload.** Preflight lists every blocker. The command is previewed without the API
key. One album at a time.
9. **Verify.** An uncertain upload is not a failure and not a success — ask the server.
10. **Archive** (optional). Copy, verify, and only then reclaim the space.
Stop anywhere. Every stage is resumable, and closing the browser cancels nothing.
## Two rules worth internalising
**The application refuses more than it warns.** A refusal is not a fault; it is the
design working. [Errors and refusals](errors.md) explains each one.
**One long job at a time.** Browsing stays available while a job runs, but a second
mutating job is refused — that is what makes a crash recoverable rather than
ambiguous.

BIN
docs/images/albums.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 64 KiB

BIN
docs/images/analysis.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 31 KiB

BIN
docs/images/archive.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 128 KiB

BIN
docs/images/duplicates.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 24 KiB

BIN
docs/images/inventory.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 55 KiB

BIN
docs/images/renames.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 78 KiB

BIN
docs/images/safety.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 31 KiB

BIN
docs/images/statistics.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 33 KiB

BIN
docs/images/uploads.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 101 KiB

BIN
docs/images/workflow.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 63 KiB

68
docs/index.md
View File

@@ -1,68 +0,0 @@
# Photo Pipeline documentation
One local application that takes a photo library from discovery to a verified Immich
upload: duplicate detection, safety review, content analysis, album naming, guarded
renaming, upload, and archive — one visible, resume-safe workflow.
These pages are readable three ways, and they are the same files each time: in the
repository under `docs/`, on Gitea, and inside the running application under
**Docs**. There is no separate copy to fall out of date.
## Read in this order
1. [Overview](overview.md) — what the application does, the stages it moves a photo
through, and the rules it will not break.
2. [Installation and operations](installation.md) — host and container installation,
every setting, the first-run checklist, upgrades, backup and restore, and what
each refusal at startup means.
3. [A guided first pass](first-pass.md) — one library from scan to verified upload,
with the point of no return named in each stage.
4. [Architecture](architecture.md) — the context and runtime diagrams, what each
module owns, the three journals a restart reads, and where every invariant is
enforced.
## The stages, one page each
Every page answers the same four questions: what the stage is for, what you decide,
what it changes on disk or on the server, and what it refuses.
1. [Inventory and discovery](stages/inventory.md)
2. [Duplicate review](stages/duplicates.md)
3. [Safety review](stages/safety.md)
4. [Analysis](stages/analysis.md)
5. [Album proposals](stages/albums.md)
6. [Renames](stages/renames.md)
7. [Upload](stages/uploads.md)
8. [Archive](stages/archive.md)
9. [Diagnostics and library statistics](stages/diagnostics.md)
## When something goes wrong
- [Errors and refusals](errors.md) — every error code, what caused it, what to do.
- [Recovery](recovery.md) — what the application resolves by itself, and what needs
you.
## Keeping these pages true
Documentation that disagrees with the application is worse than none, because it is
trusted. So one command compares them:
```bash
python -m photo_pipeline docs-gate
```
It runs every `phase_i` check — dead links and anchors, pages nobody links to, images
nobody shows, settings, commands, exit codes, error codes, and states that no longer
exist in the code, the pages rendering in a real browser without a console or policy
error, and the screenshots still showing what the application shows. It retains its
evidence and **accepts no skipped check**: a check that did not run is a page nobody
compared. CI runs it on every pull request.
The repository's own build, test, and release notes live in the
[README](../README.md).
## Conventions
A page tells you what a stage **changes on disk or on the server** before it tells
you how to run it. Refusals are documented as intentional: this application would
rather stop and explain than guess about somebody's photographs.

336
docs/installation.md
View File

@@ -1,336 +0,0 @@
# Installation and operations
[← Documentation index](index.md)
From nothing to a running instance pointed at your photo library, and everything you
need afterwards: upgrading, backing up, restoring, and understanding a refusal.
Two ways to install. **Container** is the deployment this project builds and ships;
**host** is what you want for development or a single machine you already manage.
Both run the same two processes against the same database.
> Read [what it will not do](overview.md#what-it-will-not-do) first. Several of the
> steps below only make sense once you know which rules the application is keeping.
## Before you start
| you need | version | why |
|---|---|---|
| Python | 3.12 or newer | the application |
| `exiftool` | 13.x (the image pins `13.25+dfsg-1`) | every EXIF read and write |
| `immich-go` | 0.32.0 (pinned in the image) | the upload stage only |
| Docker + the compose plugin | any current | the container installation only |
| a vision provider key | — | the analysis stage only |
You also need a photo library you can afford to be wrong about. Take a backup of it
before pointing anything at it for the first time, and consider running with
[`PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL`](#the-settings) on.
## Host installation
```bash
python3.12 -m venv .venv
.venv/bin/pip install -e ".[vision]" # drop [vision] for a review-only install
```
Configure it. Every setting is an environment variable; a `.env` file in the working
directory is read at startup:
```bash
cp .env.example .env && $EDITOR .env
```
`.env.example` lists every variable with its default and no values at all. For a
loopback installation you need exactly one:
```bash
PHOTO_PIPELINE_LIBRARY_ROOTS=/home/you/Pictures
```
Then migrate and start the two processes:
```bash
.venv/bin/python -m photo_pipeline migrate # create or upgrade the database
.venv/bin/python -m photo_pipeline serve # API and browser app on 127.0.0.1:8000
.venv/bin/python -m photo_pipeline worker # second terminal
```
**Both processes are required.** `serve` enqueues work and renders the application;
nothing is scanned, scored, analysed, renamed, uploaded, or archived without a
worker. Open <http://127.0.0.1:8000/app/> and confirm the workflow page loads.
### The configuration file
`.env`, or any path named by `PHOTO_PIPELINE_ENV_FILE`. It is parsed, never executed:
`KEY=value` lines, `#` comments, optional quotes, no interpolation and no `export`. A
configuration file that can run code is a configuration file that can be a
vulnerability.
**Anything already exported in the shell wins.** The file is your standing
configuration; the environment is the override for one run.
The archived CLI's names still work, so an existing `photo_analyzer.env` can be used
as it is:
| in the file | applied as |
|---|---|
| `LLM_API_KEY` / `GEMINI_API_KEY` | `OPENAI_API_KEY` |
| `LLM_BASE_URL` | `OPENAI_BASE_URL` |
| `LIBRARY` | `PHOTO_PIPELINE_LIBRARY_ROOTS` |
`.env` and `*.env` are gitignored and refused by the repository's safety checks. The
file holds a real key; it must never be committed.
## Container installation
```bash
cp .env.example .env && $EDITOR .env
docker compose up -d --build
```
The composition is one `serve` container, one `worker` container, a one-shot
`migrate` that both wait for, one bind-mounted library, and one named data volume.
Three variables it cannot start without:
| variable | is |
|---|---|
| `PHOTO_PIPELINE_LIBRARY_HOST_PATH` | the library on this host |
| `PHOTO_PIPELINE_LIBRARY_ROOTS` | where that library is mounted **inside** the container |
| `PHOTO_PIPELINE_ACCESS_SECRET` | required, because publishing a port means the app is reachable from outside the container |
Set `PHOTO_PIPELINE_UID` and `PHOTO_PIPELINE_GID` to the owner of the library: what
the containers rename and rewrite keeps that ownership.
### The data volume
The `data` volume holds the database, its write-ahead log, the thumbnail cache, the
operation journals, and the backups. **It must stay on a local filesystem.** SQLite
in WAL mode needs real local locking, so NFS, SMB, and network volume drivers do not
slow it down — they corrupt it. The library bind mount has no such restriction.
### Ports and reaching it
The API port is published to `127.0.0.1` unless `PHOTO_PIPELINE_PUBLISH_ADDRESS` says
otherwise. Being reachable *was* the authentication in earlier versions: whoever could
open the port owned the library. So the moment the application answers to anything but
loopback — a hostname, a proxy, `0.0.0.0` — the access secret becomes mandatory and
`serve` refuses to start without it rather than publishing your photographs.
Behind a reverse proxy:
```bash
PHOTO_PIPELINE_ALLOWED_HOSTS=photos.example.com
PHOTO_PIPELINE_ACCESS_SECRET=… # python -c 'import secrets; print(secrets.token_urlsafe(32))'
PHOTO_PIPELINE_TRUSTED_PROXIES=10.0.0.2 # only the proxy's own address
```
`X-Forwarded-Proto` and `X-Forwarded-Host` are believed only from a trusted-proxy
address, so a client cannot declare its own origin. The browser asks for the secret
once per tab. Wrong secrets are rate-limited and logged with the caller's address
only. The health endpoints stay open so an orchestrator can restart the container;
nothing else is.
### Deployment
The stack is managed from git by Portainer, which redeploys on a webhook after CI
publishes an image built from `main`. Runtime secrets live in the Portainer stack's
environment rather than in the repository, so rotation happens in one place and
reading the repository discloses nothing.
## The settings
Every variable, its default, and whether it is a secret. `.env.example` is the same
list in copyable form.
### Library and data
| variable | default | meaning |
|---|---|---|
| `PHOTO_PIPELINE_LIBRARY_ROOTS` | none | the library boundary, `os.pathsep`-separated. No path outside these roots is ever read or written |
| `PHOTO_PIPELINE_DATA_DIR` | `data` | database, WAL, thumbnail cache, journals, backups. Never inside the library |
| `PHOTO_PIPELINE_DB_PATH` | `<data dir>/photo_pipeline.db` | the database file, if it must live elsewhere |
| `PHOTO_PIPELINE_ENV_FILE` | `.env` | where to read the configuration file from |
### Serving and the trust boundary
| variable | default | meaning |
|---|---|---|
| `PHOTO_PIPELINE_HOST` | `127.0.0.1` | bind address |
| `PHOTO_PIPELINE_PORT` | `8000` | port |
| `PHOTO_PIPELINE_ALLOWED_HOSTS` | none | comma-separated names the app answers to besides loopback. Naming one makes the access secret mandatory |
| `PHOTO_PIPELINE_ACCESS_SECRET` | none | **secret.** Traded for the session cookie at `GET /api/v1/session` |
| `PHOTO_PIPELINE_TRUSTED_PROXIES` | none | comma-separated peer addresses whose forwarded headers may be believed |
| `PHOTO_PIPELINE_MAX_REQUEST_BYTES` | `1048576` | largest request body accepted |
### Logging
| variable | default | meaning |
|---|---|---|
| `PHOTO_PIPELINE_LOG_LEVEL` | `INFO` | standard Python levels |
| `PHOTO_PIPELINE_LOG_FORMAT` | `json` | `json` or `text` |
### Limits
| variable | default | meaning |
|---|---|---|
| `PHOTO_PIPELINE_THUMBNAIL_CACHE_QUOTA_BYTES` | `500000000` | cache quota; over it is a diagnostics warning |
| `PHOTO_PIPELINE_THUMBNAIL_MAX_PIXELS` | `100000000` | refuse to decode anything larger. This is the decompression-bomb guard |
| `PHOTO_PIPELINE_ARCHIVE_FREE_SPACE_RESERVE_BYTES` | `1000000000` | free space an archive destination must keep beyond the transfer itself |
### The safety gate
| variable | default | meaning |
|---|---|---|
| `PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL` | `false` | refuse every mutating request until a read-only dry run of this library has been produced and approved |
### External services
| variable | default | meaning |
|---|---|---|
| `PHOTO_PIPELINE_VISION_API_KEY` | none | **secret.** Without it the analysis stage cannot run |
| `PHOTO_PIPELINE_IMMICH_SERVER_URL` | none | the Immich server |
| `PHOTO_PIPELINE_IMMICH_API_KEY` | none | **secret.** |
| `PHOTO_PIPELINE_IMMICH_GO_BINARY` | `immich-go` | the uploader, found on `PATH` |
### Composition only
Read by `docker-compose.yml`, not by the application:
`PHOTO_PIPELINE_IMAGE`, `PHOTO_PIPELINE_LIBRARY_HOST_PATH`,
`PHOTO_PIPELINE_PUBLISH_ADDRESS`, `PHOTO_PIPELINE_UID`, `PHOTO_PIPELINE_GID`.
A secret is never written to a log, never returned by the API, never stored in the
database, and never recorded in a backup manifest — a manifest says `configured`, not
the value. Do not put a real key in an example, a ticket, or a screenshot.
## First run checklist
Not "it started" — verified.
1. **The database is at the current revision.**
`python -m photo_pipeline migrate` exits 0.
2. **The application answers.**
`curl -s localhost:8000/api/v1/health/ready` returns 200.
3. **The library was found.** Open the app, run a scan from the Inventory view, and
check the count against what you expect. Anything under `_IGNORE/` must be missing
from it — that is the exclusion working, not a bug.
4. **The worker is claiming.** The scan job reaches `succeeded`. If it stays
`queued`, no worker is running.
5. **Diagnostics are clean.**
`python -m photo_pipeline diagnostics` reports free space and an empty `warnings`.
6. **Nothing was modified.** The scan is read-only; your files' timestamps are
unchanged.
Only then point it at the whole library.
## Operations
Every operation is the same CLI, on a host or in the composition:
```bash
python -m photo_pipeline diagnostics
docker compose run --rm --no-deps api diagnostics
```
`--no-deps` keeps a one-off command from starting a second stack; `api` is only the
service it borrows the image and mounts from.
### Upgrading
1. `python -m photo_pipeline backup --reason before-upgrade`
2. Pull the new version (or `docker compose pull && docker compose up -d`).
3. Migrations run by themselves at startup, and a **pending schema change is
snapshotted first**. If the upgrade fails, the previous database and its
`pre-migration` backup are both intact, and the error log names the backup
directory.
An up-to-date database is not backed up again on every start.
### Backing up
```bash
python -m photo_pipeline backup --reason weekly --keep 7
python -m photo_pipeline verify-backup data/backups/<name>
```
Backups go through SQLite's online backup API, never a file copy: with WAL enabled
the `.db` file alone is missing every committed page still in the write-ahead log.
Each backup is a directory holding the snapshot and a `manifest.json` — schema
revision, SHA-256, row counts, the archive media the library depends on, and which
settings were configured.
`verify-backup` runs `PRAGMA integrity_check` **and** `PRAGMA foreign_key_check`,
compares the snapshot's SHA-256 against the manifest, and re-counts every table it
recorded. Bit rot, a truncated copy, and a "repaired" snapshot all fail it.
`--keep N` prunes the oldest and never the newest. The same is available at
`GET /api/v1/diagnostics`, `GET|POST /api/v1/backups`,
`GET /api/v1/backups/{name}/verify`, and `POST /api/v1/backups/prune`.
### Restoring
Restore is deliberately **not** an API call. It replaces the state of an
installation, so it belongs to a stopped one and a person at a terminal.
1. Stop `serve` and `worker`.
2. `python -m photo_pipeline verify-backup data/backups/<name>` — never restore an
unverified snapshot, and `restore` will refuse one anyway.
3. `python -m photo_pipeline restore data/backups/<name> --into /path/to/fresh-data`.
A target that already holds a database is refused; recovering in place means
moving the old data directory aside first.
4. Point `PHOTO_PIPELINE_DATA_DIR` at the restored directory and run `migrate`.
5. Run an inventory scan, so paths are reconciled against the real library.
6. Mount every archive location the manifest names before archiving again. The
database records where archived originals are; it does not contain them.
Practise this against a copy before you need it.
### Watching it
`diagnostics` reports the database, write-ahead log, thumbnail cache, uploader
reports, backups, and logs separately, with free space and warnings for low disk
(`disk_low`, `disk_critical`), a cache over its quota, a write-ahead log outgrowing
its database, and a legacy CLI writing the library.
Logs go to stdout in JSON by default — `docker compose logs -f worker`, or your
service manager's journal on a host. Uploader output is kept per attempt under the
data directory with the API key scrubbed line by line.
## When it refuses to start
Every refusal below is deliberate. The application would rather stop and explain than
guess about somebody's photographs.
| exit code | meaning | what to do |
|---|---|---|
| `1` | the operation failed and said why — an unverifiable backup, an occupied restore target, an unknown benchmark profile | read the message; nothing was changed |
| `2` | **the library lock is held.** Another `serve` or `worker` owns this data directory | stop the other process. The message names the holder; a lock whose process is gone is taken over automatically |
| `3` | **a legacy CLI looks active.** The frozen command-line tools are writing this library | stop them. `--allow-legacy` overrides and you own the outcome |
| `4` | **configuration refused.** The application is reachable beyond loopback and no access secret is set | set `PHOTO_PIPELINE_ACCESS_SECRET`, or bind to loopback only |
| `5` | **a library root is not mounted** (containers only) | fix the bind mount. Refusing is what stops the container writing into its own throwaway layer instead of your library |
Other things that look like faults and are not:
- **`403 host_not_allowed`** — the `Host` header is not a name in
`PHOTO_PIPELINE_ALLOWED_HOSTS`. Add the real hostname; do not add `*`.
- **`409 lock_held`** — a mutating job is already running. One at a time is what
makes a crash recoverable.
- **`409 rename_recovery_required`** — an interrupted rename is unresolved. Resolve
it in the Renames view; unrelated mutations stay blocked until then, on purpose.
- **A job stuck in `queued`** — no worker is running.
- **An empty scan** — check `PHOTO_PIPELINE_LIBRARY_ROOTS`, and in a container check
that the path is the *container-side* mount, not the host path.
## What an installer must not work around
- **One writer.** Do not run two workers, and do not run the frozen CLI beside the
application. Each is safe alone and destructive together — the lock is not
bureaucracy.
- **`_IGNORE/` is never read.** Do not "fix" the exclusion.
- **EXIF is verified before upload.** Do not skip the checkpoint to make a stage
finish; an unverified projection is exactly the case where the wrong bytes reach
Immich.
- **The data directory is not the library.** Keep them apart, and keep the data
directory on a local filesystem.
- **Secrets stay in the environment.** Not in the database, not in a log, not in a
commit.

77
docs/overview.md
View File

@@ -1,77 +0,0 @@
# Overview
[← Documentation index](index.md)
Photo Pipeline sorts a photo library. It finds duplicates before anything expensive
happens to them, asks a human which pictures may leave the machine, describes the
ones that may, proposes album names from what it found, renames folders under a
crash-safe journal, uploads through `immich-go`, and can archive a finished album off
active storage without ever forgetting it existed.
It runs locally. It talks to exactly three things outside itself: a vision provider,
an Immich server, and `exiftool` — and it will tell you before it uses any of them.
## The workflow
Each stage is a gate, not a tab. A later stage can always be looked at; its actions
stay disabled until what they depend on is true.
```mermaid
flowchart TD
I["0 · Inventory<br/>discover, hash, cluster duplicates"] --> S["1 · Safety<br/>score and human decision"]
S -->|sfw| A["2 · Analysis<br/>vision provider, EXIF checkpoint"]
S -->|nsfw| U
A --> B["3 · Albums<br/>proposal, then guarded rename"]
B --> U["4 · Upload<br/>immich-go, verified bytes"]
U --> R["5 · Archive<br/>copy, verify, then reclaim space"]
```
| Stage | What it decides | What it changes |
|---|---|---|
| Inventory | which file is the canonical copy of a picture | nothing — it only reads |
| Safety | whether a photo may be sent to a cloud provider | one `sfw`/`nsfw` EXIF keyword |
| Analysis | what a photo shows | a managed caption segment and additive keywords |
| Albums | what a folder should be called | folder names, through a journaled rename |
| Upload | which exact bytes reach Immich | nothing locally; assets appear in Immich |
| Archive | which album leaves active storage | files move to the archive, after verification |
## What it will not do
These are enforced in code, not by convention, and each one is why some action you
expected is sometimes refused.
- **Nothing under `_IGNORE/` is ever read.** Not scanned, not counted, not
thumbnailed, not sent anywhere.
- **A path is not an identity.** Every picture has a stable id, so moving or renaming
it loses no history.
- **Only a confirmed-SFW photo reaches the vision provider.** A photo marked NSFW is
still uploadable to Immich; it simply never leaves for analysis.
- **Metadata is verified, not hoped for.** Every stage that writes EXIF reads it back
and proves that what it did not own is unchanged.
- **One writer at a time.** A library lock, held by one process, is what makes a
crash recoverable instead of ambiguous.
- **Nothing irreversible happens without a preview and an explicit approval** that
names the exact count.
## Where things live
| | |
|---|---|
| the photo library | wherever you point `PHOTO_PIPELINE_LIBRARY_ROOTS`; mounted read-write |
| the database, cache, logs, backups | the data directory, never inside the library |
| secrets | the environment, never the database and never a log line |
| these documents | `docs/` in the repository, served at `/docs` by the application |
## Running it
The short version, for a host installation:
```bash
python -m photo_pipeline migrate
python -m photo_pipeline serve # the API and this browser application
python -m photo_pipeline worker # the process that does the long work
```
The full procedure, the container path, and every setting are in the
[installation manual](installation.md); if you would rather start using it, take
[the guided first pass](first-pass.md).

62
docs/recovery.md
View File

@@ -1,62 +0,0 @@
# Recovery
[← Documentation index](index.md)
What the application resolves by itself, and what needs you. The dividing line is
simple: it resumes when the evidence is unambiguous, and it stops and asks when it is
not. It never guesses about your files.
## An interrupted rename
A rename records its intent **before** touching the disk, so after a crash the
journal plus the actual filesystem give a verdict per operation:
| verdict | means | what happens |
|---|---|---|
| resumable | the move provably did not happen | reset to planned and run again |
| rollback-safe | the move happened; the remaining steps can be completed or undone | drive it forward, or roll it back |
| manual | the evidence is contradictory | left alone, and it keeps blocking |
Open [Renames](stages/renames.md); the recovery panel lists every unresolved
operation and offers a resolve action only where one is safe. Until then, unrelated
mutations are refused with `rename_recovery_required` — building new work on a
half-applied move is how a library becomes unexplainable.
## An uncertain upload
`unknown_requires_verification` means the uploader died after Immich may have
accepted the files. It is **not** retryable. Verification asks the server whether it
holds the recorded SHA-1 and answers present, absent, or inconclusive. An
inconclusive answer is resolved by you, with your evidence and name recorded in the
batch's history.
## A failed migration
A pending schema change is snapshotted before it is applied. If the upgrade fails,
the previous database and its `pre-migration` backup are both intact and the error
log names the backup directory. Stop everything and run the
[restore drill](installation.md#restoring).
## A restored backup
After restoring, run a scan. Paths are reconciled against the real library and
identities are preserved where hashes match. Anything that does not match becomes a
visible `stale` or `divergent` state instead of being quietly accepted.
Mount every archive medium the manifest names before archiving again: the database
records where archived originals live, it does not contain them.
## A job that will not start
Check, in this order: is a worker running; is another mutating job holding the lane
(`lock_held`); is an interrupted rename blocking everything
(`rename_recovery_required`); is the library lock held by a process that is still
alive (exit code `2` names the holder).
## What is never automatic
No file is deleted, no folder is renamed, no upload is retried, and no archive source
is removed without either an explicit confirmation from you or a verification the
application performed itself. If you are reading this page because something happened
that you did not approve, that is a bug worth reporting — with the diagnostics output
and the relevant journal, both of which are safe to share: they carry no secrets.

47
docs/stages/albums.md
View File

@@ -1,47 +0,0 @@
# Album proposals
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
![Reviewing an album proposal](../images/albums.png)
**What it is for.** Turning a folder of analysed photographs into a name a person
would have chosen, before any folder is touched.
**What you decide.** The final name. The proposal is a starting point with its
reasoning attached; you edit it, accept it, or ignore it.
**What it changes.** A row per album: proposed name, rationale, confidence, and your
final name, with a version. **Approving renames nothing.** Not one file moves at this
stage — approval only marks a name as agreed, and the [rename stage](renames.md) is
where it becomes a plan you have to confirm separately.
**What it refuses.** A name containing `/ \ : * ? " < > |`, a reserved device name, or
one that collides with an existing folder is rejected while you type and again on the
server. Approval is refused when the evidence changed since the proposal was
generated — the proposal is stale, and regenerating is the honest fix. Editing with a
stale version returns a conflict and shows you the server's truth rather than
overwriting it.
## Reading the view
The left pane lists albums with their state: `none`, `proposed`, `edited`,
`approved`, `error`, or `stale`. The right pane shows the evidence the name was built
from — date range, dominant tags, locations, counts — then the suggested name, the
rationale, the confidence, and an editable final name.
The default naming shape is predictable and sortable:
```text
YYYY-MM — Place — Event
YYYY — Event
Place — Event
```
## While a rename is unresolved
Generating and approving proposals is blocked while an interrupted rename is
outstanding, with `409 rename_recovery_required`. Resolve the rename first: building
new names on top of a half-applied move is how a library becomes hard to reason
about.
Next: [renames](renames.md).

39
docs/stages/analysis.md
View File

@@ -1,39 +0,0 @@
# Analysis
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
![The analysis view](../images/analysis.png)
**What it is for.** Describing what a photograph shows, so albums can be named from
evidence and the library can be searched by content.
**What you decide.** When to run it, and over what scope. The descriptions themselves
come from the configured vision provider.
**What it changes.** For each eligible photo: a stored result — description, tags,
approximate year, location hint, model and prompt version — and then an EXIF
checkpoint that merges a **managed caption segment** and additive keywords into the
file, reads them back, and verifies that nothing else moved. The file's SHA-256 is
refreshed after the write.
**What it refuses.** Anything not confirmed SFW. Anything not canonical. A malformed
provider response is quarantined rather than stored. Keywords are only ever added,
never removed, because EXIF keywords carry no ownership and deleting a generated one
could delete yours.
## Running it
The view shows eligible, analysed, pending, and error counts, and the job's live
progress. Starting a second mutating job while it runs is refused — that is the
one-writer rule, not a queue.
Cancelling drains safely: completed items stay completed, and resuming continues
rather than restarting. A photo may be sent to the provider twice if a crash happens
mid-item, but its stored result and its EXIF are written once.
## Costs
Each analysed photo is a provider call. Resolving duplicates first is what keeps that
number honest, and the counts here are the ones to check before starting a large run.
Next: [album proposals](albums.md).

43
docs/stages/archive.md
View File

@@ -1,43 +0,0 @@
# Archive
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
![Archive locations and plans](../images/archive.png)
**What it is for.** Moving a finished album out of active storage onto another disk,
while keeping everything the library knows about it.
**What you decide.** Which album, which destination, and whether to accept the
reclaimed space in exchange for needing that medium mounted to see the originals
again.
**What it changes.** Files are copied to the archive location, verified there, and
only then removed from the library. The asset keeps its id, its hashes, its
decisions, its analysis, its upload history, and a durable preview thumbnail;
`current_path` becomes null and availability becomes `archived_online` or
`archived_offline`.
**What it refuses.** Preflight blocks on: an unverified upload, a checksum that no
longer matches the uploaded bytes, an unmounted or unwritable destination,
insufficient free space plus the configured reserve, a destination that already
holds unexpected paths, a missing durable thumbnail, and any conflicting job. The
source is **never** removed before the archived copy is verified — not because Immich
reported success, which is an ingest, not a backup.
## Offline is not missing
An archived photo whose medium is unplugged is `archived_offline`. It still appears
in search, still participates in duplicate detection through its retained hashes and
thumbnail, and is never reported as lost. If a full-resolution comparison is needed,
the application asks you to mount the named medium rather than guessing.
## Restore
Restore is planned and journaled like everything else: the medium must be online, the
bytes are copied back and verified into a collision-free destination, a new active
path is registered, and a reconciliation scan follows. Existing safety, analysis,
EXIF, and upload state stay valid when hashes match — and when they do not, the
result is a visible `divergent` state rather than silent acceptance of a different
file.
Next: [diagnostics](diagnostics.md).

49
docs/stages/diagnostics.md
View File

@@ -1,49 +0,0 @@
# Diagnostics and library statistics
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
![Library statistics](../images/statistics.png)
**What it is for.** Answering "is this installation healthy, and what is in this
library?" — the first thing to look at when a stage behaves unexpectedly.
**What you decide.** Nothing. Both are read-only.
**What it changes.** Nothing.
**What it refuses.** Nothing — both are reads, and both stay available while a job
holds the library. That is deliberate: the moment you most need to know what the
installation is doing is while it is busy.
## Statistics
The Stats view summarises the library itself: how many photos are analysed, the
common tags, the distribution across albums and years. It answers questions about
your photographs.
## Diagnostics
Diagnostics answers questions about the *installation*, and lives at
`GET /api/v1/diagnostics` and on the command line:
```bash
python -m photo_pipeline diagnostics
```
It reports the database, write-ahead log, thumbnail cache, uploader reports, backups,
and logs as separate sizes, plus free space, the tool versions it found against the
ones the image pinned, and which locks are held and by whom.
Its warnings are the ones worth acting on:
| warning | means |
|---|---|
| `disk_low` / `disk_critical` | free space is running out; mutating stages stop before the reserve |
| `cache_over_quota` | the thumbnail cache exceeds its configured quota |
| `wal_growth` | the write-ahead log is outgrowing its database — usually a long-running reader |
| `tool_version_drift` | the installed `exiftool` or `immich-go` is not the version the image pinned |
| `legacy_process_active` | the frozen command-line tools are writing this library. Stop them |
| `no_roots` | no library root is configured, so there is nothing to work on |
An empty `warnings` list on a fresh installation is what the
[first-run checklist](../installation.md#first-run-checklist) is looking for.

36
docs/stages/duplicates.md
View File

@@ -1,36 +0,0 @@
# Duplicate review
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
![The duplicate cluster list](../images/duplicates.png)
**What it is for.** Deciding which file is *the* copy of a picture, before anything
expensive or irreversible happens to the others.
**What you decide.** For each cluster: keep the recommended canonical, choose a
different one, keep everything as variants, declare it not a duplicate, or defer.
**What it changes.** Only the decision, recorded as an auditable event. **No file is
ever deleted.** Non-canonical copies stay exactly where they are; they are simply
excluded from analysis and upload.
**What it refuses.** Exact byte and identical-pixel matches may be accepted on the
recommendation. Anything fuzzy — a crop, a re-encode, a burst frame — requires an
explicit confirmation, because a perceptual hash is evidence, not proof.
## Reading a cluster
Each member shows its path, role, pHash distance, and size, with synchronised zoom
across the previews. The recommendation is visible but never pre-applied for fuzzy
matches. The confidence band tells you how much the machine is claiming.
Rejecting a pair records a negative link, so a later rescan does not keep proposing
the same wrong match.
## Why first
Resolving duplicates before safety and analysis is what stops you reviewing and
paying for the same photograph three times. It also means the album evidence is built
from one copy of each picture rather than a skewed count.
Next: [safety review](safety.md).

39
docs/stages/inventory.md
View File

@@ -1,39 +0,0 @@
# Inventory and discovery
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
![The inventory view](../images/inventory.png)
**What it is for.** Finding every supported photo in the configured library roots and
giving each one a permanent identity, so that everything afterwards can refer to a
picture rather than a path.
**What you decide.** Nothing. This stage is the only one with no judgement in it.
**What it changes.** On disk, nothing at all — discovery reads. In the database it
records each photo's path, size, timestamps, full-file SHA-256, normalised pixel hash,
and perceptual hash, and it opens a path-history entry.
**What it refuses.** Anything under an `_IGNORE/` directory is never traversed,
counted, hashed, or previewed. Unsupported extensions and unreadable files are
reported rather than skipped silently. Nothing outside the configured roots is
reachable, including through a symlink.
## Reading the view
Each row is one asset: its current path, availability, whether the file is present,
and its size. The filter searches paths; the availability selector separates active
photos from archived ones.
A **rescan** after moving files around outside the application is the normal way to
reconcile: a file that moved keeps its identity, because identity is the hash and the
id, not the location.
## What good looks like
The count matches what you expect, `_IGNORE/` contents are absent, and nothing shows
as missing. A photo listed as missing means the file is no longer at its recorded
path and no new path matched its hashes — that is a question for you, not something
the scanner resolves by guessing.
Next: [duplicate review](duplicates.md).

46
docs/stages/renames.md
View File

@@ -1,46 +0,0 @@
# Renames
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
![The rename plan preview](../images/renames.png)
**What it is for.** Applying approved album names to the actual folders — the first
stage that changes your filesystem.
**What you decide.** Whether to apply the plan, after reading it. The confirmation
names the plan, its version, and its checksum, so what you approve is what runs.
**What it changes.** Folders move. For each operation: the intent is journaled
*before* the disk is touched, the move is made, `assets.current_path` is updated by
id in one transaction, the old path is closed and a new one opened in path history,
the result is verified, and only then is the operation complete.
**What it refuses.** A plan is invalid — not merely warned about — when a source is
missing or changed, two operations target the same destination, a destination already
exists, a path would leave the library or enter `_IGNORE/`, or a source is a symlink.
Case-only renames and cross-filesystem moves are allowed but flagged, because they
take a different, staged route. A plan whose checksum no longer matches is refused
rather than reinterpreted.
## Reading the preview
Every operation shows both full paths, the kind of move (`rename`, `case-only
rename`, `cross-filesystem move`), how many photos it affects, its journal state, and
any issue with its severity. The plan can be exported as JSON before you commit to it.
**A run cannot be stopped part-way.** It can be interrupted — by a crash, a power
cut, a killed container — and interruption is recoverable, but there is no cancel
button once it starts. That is stated on the confirmation, not hidden in a manual.
## After an interruption
The recovery panel lists each unresolved operation with a verdict read from the
journal *and the disk*: resumable, safely rollbackable, or needing a person. It never
guesses from a missing path alone. Until it is resolved, unrelated mutations are
refused with `409 rename_recovery_required`.
Rollback is recovery, not undo. A completed operation is terminal — the button only
appears for operations that are actually reversible. See
[recovery](../recovery.md).
Next: [upload](uploads.md).

43
docs/stages/safety.md
View File

@@ -1,43 +0,0 @@
# Safety review
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
![The safety review queue](../images/safety.png)
**What it is for.** Deciding which photographs may be sent to a cloud vision
provider. This is the privacy gate the rest of the design is built around.
**What you decide.** `sfw`, `nsfw`, or defer — per photo, or in bulk from the filters.
A local model can score first to order the queue, but the score is a suggestion; the
decision is yours.
**What it changes.** The decision is stored in the database *and* projected into the
file: one mutually exclusive `sfw` or `nsfw` keyword written into `Keywords` and
`Subject`. The write is merged with existing metadata, read back, and verified, and
the file's SHA-256 is refreshed afterwards.
**What it refuses.** An undecided or deferred photo does not reach the vision
provider, and does not reach Immich either. If the read-back shows that a field the
stage does not own has changed, the checkpoint is marked `divergent`, the asset is not
marked verified, and the next mutating stage is blocked rather than proceeding on
metadata nobody trusts.
## The rule that matters
> Only a confirmed `sfw` photo may enter cloud content analysis. A confirmed `nsfw`
> photo skips analysis entirely but remains eligible for upload to Immich once its
> keyword is verified.
That is why marking something NSFW is not a punishment: it routes the photo around an
external service while keeping it in your own library workflow.
The gate is re-checked *after* the provider call as well. If a decision flips to NSFW
while an analysis is in flight, the result is discarded rather than stored.
## Reading the view
The tabs filter by state. Each row shows the path, the model's score, its suggestion,
your decision, and an `✓ exif` badge once the keyword is verified on disk. A row
without that badge has a decision the file does not yet carry.
Next: [analysis](analysis.md).

45
docs/stages/uploads.md
View File

@@ -1,45 +0,0 @@
# Upload
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
![Upload preflight](../images/uploads.png)
**What it is for.** Sending an approved album to Immich through `immich-go`, with an
audit trail of exactly which bytes went.
**What you decide.** Which albums, and whether to proceed once the preflight has
listed its blockers. Upload is never an automatic consequence of renaming.
**What it changes.** Nothing locally. On the Immich server, assets appear. Locally a
batch is recorded: the album, the asset ids, the redacted command, the uploader
version, the pre-upload SHA-256 **and** the SHA-1 Immich matches on, the output log,
and the outcome counts.
**What it refuses.** Preflight blocks on: missing credentials, an unreachable server,
a missing `immich-go`, an unresolved rename, an undecided or deferred safety
decision, an unverified EXIF checkpoint, incomplete analysis for an SFW photo, a file
whose bytes changed since it was checkpointed, and a partial scope you have not
explicitly accepted. The approval is then **re-proved immediately before the uploader
runs** — bytes edited after you approved fail with `stale_preflight` and the uploader
never starts.
## Reading the view
The scope shows exactly which albums and how many photos. The command preview is the
real command with the API key removed; the key never reaches the browser and never
reaches a log.
## Uncertain is not failed
If the process dies after the server accepted files, the batch becomes
`unknown_requires_verification` — not "failed", and **not retryable**. The server may
already hold the photographs, and uploading again would create duplicates or
upgrades. Verification asks Immich about the recorded SHA-1 and answers present,
absent, or inconclusive; an inconclusive answer is resolved by you, with the evidence
recorded.
If a file's bytes change after a successful upload, it is marked
`changed_after_upload` and you are warned that uploading again may create or upgrade
an asset rather than doing nothing.
Next: [archive](archive.md).

14
frontend/css/app.css
View File

@@ -170,20 +170,6 @@ a.link:hover { text-decoration: underline; }
.grid th, .grid td { text-align: left; padding: 6px 8px; border-bottom: 1px solid var(--border); vertical-align: top; }
.path { font-family: ui-monospace, monospace; font-size: 0.85em; word-break: break-all; }
/* Docs: prose, so it gets a reading measure rather than the full window width. */
.doc { max-width: 72ch; }
.doc h1 { margin-top: 0; }
.doc h2, .doc h3 { margin-top: 1.6em; border-bottom: 1px solid var(--border); padding-bottom: 4px; }
.doc code { font-family: ui-monospace, monospace; font-size: 0.9em; background: var(--surface-2); padding: 1px 4px; border-radius: 4px; }
.doc pre { background: var(--surface-2); border: 1px solid var(--border); border-radius: var(--radius); padding: 12px; overflow: auto; }
.doc pre code { background: none; padding: 0; }
.doc table { border-collapse: collapse; width: 100%; }
.doc th, .doc td { text-align: left; padding: 6px 8px; border-bottom: 1px solid var(--border); vertical-align: top; }
.doc blockquote { margin: 1em 0; padding-left: 12px; border-left: 3px solid var(--border); color: var(--muted); }
.doc img { max-width: 100%; }
.diagram { margin: 1.5em 0; padding: 0; overflow: auto; }
.diagram svg { max-width: 100%; height: auto; }
/* Narrow screens: stack the panes so blockers and actions stay reachable. */
@media (max-width: 720px) {
.two-pane { grid-template-columns: 1fr; }

1
frontend/index.html
View File

@@ -21,7 +21,6 @@
<a href="#/uploads" data-nav="uploads">Upload</a>
<a href="#/archive" data-nav="archive">Archive</a>
<a href="#/stats" data-nav="stats">Stats</a>
<a href="#/docs" data-nav="docs">Docs</a>
</nav>
</header>
<main id="app" aria-live="polite"><!-- views render here --></main>

27
frontend/js/api.js
View File

@@ -7,28 +7,9 @@ export const BASE = "/api/v1";
// what makes it proof that the caller is this app and not another page.
let csrfToken = null;
// A deployment reachable through a proxy trades an operator secret for that cookie.
// Kept per tab: sessionStorage dies with the tab, and the secret never enters a URL.
const SECRET_KEY = "pp_access_secret";
async function bootstrap(secret) {
return fetch(BASE + "/session", {
credentials: "same-origin",
headers: secret ? { "X-Access-Secret": secret } : {},
});
}
async function session() {
if (csrfToken === null) {
let response = await bootstrap(sessionStorage.getItem(SECRET_KEY));
if (response.status === 401) {
sessionStorage.removeItem(SECRET_KEY);
const secret = prompt("Access secret");
if (secret) {
response = await bootstrap(secret);
if (response.ok) sessionStorage.setItem(SECRET_KEY, secret);
}
}
const response = await fetch(BASE + "/session", { credentials: "same-origin" });
const body = await response.json().catch(() => null);
csrfToken = (body && body.csrf_token) || null;
}
@@ -93,10 +74,8 @@ export const api = {
request("/inventory/assets?" + new URLSearchParams(params).toString(), opts),
listClusters: (params = {}, opts = {}) =>
request("/duplicates/clusters?" + new URLSearchParams(params).toString(), opts),
getCluster: (id, params = {}, opts = {}) => {
const query = new URLSearchParams(params).toString();
return request(`/duplicates/clusters/${encodeURIComponent(id)}${query ? `?${query}` : ""}`, opts);
},
getCluster: (id, opts = {}) =>
request(`/duplicates/clusters/${encodeURIComponent(id)}`, opts),
decide: (id, payload, opts = {}) =>
request(`/duplicates/clusters/${encodeURIComponent(id)}/decision`, {
method: "POST",

25
frontend/js/app.js
View File

@@ -1,7 +1,5 @@
import { api } from "./api.js";
import { renderArchive, setArchiveRender } from "./archive.js";
import { renderDocs } from "./docs.js";
import { show as showIn } from "./dom.js";
import { navigate, onRouteChange, parseHash } from "./router.js";
import { renderRenames, setRenamesRender } from "./renames.js";
import { renderUploads, setUploadsRender } from "./uploads.js";
@@ -36,7 +34,7 @@ function el(tag, attrs = {}, ...children) {
}
function show(...nodes) {
showIn(root, ...nodes);
root.replaceChildren(...nodes);
}
function setActiveNav(view) {
@@ -195,7 +193,7 @@ async function renderClusters(params) {
" ",
el("span", { class: `badge ${cluster.state}` }, cluster.state)
),
el("div", { class: "muted" }, `${cluster.member_total ?? cluster.members.length} members · confidence ${cluster.confidence}`)
el("div", { class: "muted" }, `${cluster.members.length} members · confidence ${cluster.confidence}`)
)
);
@@ -209,11 +207,8 @@ async function renderClusters(params) {
async function renderClusterDetail(id, extra = {}) {
setActiveNav("duplicates");
let cluster;
// A cluster can hold thousands of members, so the server pages them; the page
// asks for as many as it is currently showing (US07-06).
const shown = extra.shown || 0;
try {
cluster = await api.getCluster(id, shown ? { limit: shown } : {});
cluster = await api.getCluster(id);
} catch (error) {
show(errorBanner(`Failed to load cluster: ${error.message}`));
return;
@@ -303,19 +298,6 @@ async function renderClusterDetail(id, extra = {}) {
nodes.push(decisionBar);
if (extra.pending) nodes.push(confirmPanel(cluster, extra.pending));
nodes.push(el("div", { class: "cluster-grid" }, ...members));
const total = cluster.member_total ?? cluster.members.length;
if (cluster.members.length < total) {
nodes.push(
el(
"button",
{
"data-testid": "show-more-members",
onclick: () => renderClusterDetail(id, { ...extra, shown: cluster.members.length + 100 }),
},
`Show more (${cluster.members.length} of ${total})`
)
);
}
show(...nodes);
}
@@ -387,7 +369,6 @@ function render() {
else if (path === "/uploads") renderUploads(root, params);
else if (path === "/archive") renderArchive(root, params);
else if (path === "/stats") renderStats(root, params);
else if (path === "/docs") renderDocs(root, params);
else show(errorBanner("Unknown view"));
}

8
frontend/js/archive.js
View File

@@ -9,7 +9,7 @@
// is not mounted produces an instruction naming it rather than a disabled mystery,
// and an operation whose evidence is ambiguous offers no button at all.
import { api } from "./api.js";
import { el, errorBanner, setActiveNav, show } from "./dom.js";
import { el, errorBanner, setActiveNav } from "./dom.js";
import { subscribeJob } from "./events.js";
import { navigate } from "./router.js";
@@ -46,7 +46,7 @@ export async function renderArchive(root, params = {}) {
try {
locations = (await api.archiveLocations()).locations;
} catch (error) {
show(root, errorBanner(`Failed to load archive locations: ${error.message}`));
root.replaceChildren(errorBanner(`Failed to load archive locations: ${error.message}`));
return;
}
const location = locations.find((l) => l.id === params.location) || locations[0] || null;
@@ -61,7 +61,7 @@ export async function renderArchive(root, params = {}) {
),
outcomeBanner()
);
show(root, ...nodes.filter(Boolean));
root.replaceChildren(...nodes.filter(Boolean));
return;
}
@@ -95,7 +95,7 @@ export async function renderArchive(root, params = {}) {
archived ? archivedSection(archived.items, locations) : null,
restore ? restoreSection(restore, location) : null
);
show(root, ...nodes.filter(Boolean));
root.replaceChildren(...nodes.filter(Boolean));
}
function plans(listed) {

263
frontend/js/docs.js
View File

@@ -1,263 +0,0 @@
// The documentation view (US09-01): the manuals in `docs/`, rendered in the app.
//
// The same markdown files are the repository's documentation and the deployment's
// documentation. An operator who was handed a URL and an access secret has no
// checkout in front of them, and the network the application runs on is not assumed
// to reach a CDN — so the renderer is vendored and everything here is same-origin.
//
// Nothing on this page is authenticated. It is served from the static mount beside
// the application shell, exactly like `index.html`: the troubleshooting page is
// needed most by whoever cannot get past the access secret, and no documentation
// file contains anything a session would protect.
//
// `marked` and `mermaid` are loaded lazily, on the first documentation page and the
// first diagram. They are large, and every other view does without them.
import { el, errorBanner, setActiveNav } from "./dom.js";
const DOCS_BASE = "/docs";
const VENDOR = "/app/js/vendor";
const INDEX_PAGE = "index";
// A page name comes from the hash, so it is caller input: no scheme, no traversal,
// no absolute path. The server would refuse those too; this refuses them earlier and
// without a request that looks like an attempt.
const PAGE_PATTERN = /^[\w-]+(\/[\w-]+)*$/;
let markedPromise;
let mermaidPromise;
function loadMarked() {
markedPromise ??= import(`${VENDOR}/marked.esm.js`).then((module) => module.marked);
return markedPromise;
}
// mermaid ships one large UMD bundle rather than a self-contained ES module, so it
// arrives through a script element. `script-src 'self'` allows it because it is ours
// and same-origin; nothing here relaxes that.
function loadMermaid() {
mermaidPromise ??= new Promise((resolve, reject) => {
const script = document.createElement("script");
script.src = `${VENDOR}/mermaid.min.js`;
script.onload = () => resolve(window.mermaid);
script.onerror = () => reject(new Error("the diagram renderer could not be loaded"));
document.head.appendChild(script);
});
return mermaidPromise;
}
// ── the pure parts, unit-tested in js/tests/unit.js ──────────────────────────
/** A stable, readable heading anchor. Letters and digits of any script survive. */
export function slug(text) {
const cleaned = String(text)
.toLowerCase()
.trim()
.replace(/[^\p{L}\p{N}\s-]/gu, "")
.replace(/[\s-]+/g, "-")
.replace(/^-|-$/g, "");
return cleaned || "section";
}
/** Assign every heading an id, disambiguating repeats the way a reader would
* expect: the first `Notes` keeps `#notes`, the second becomes `#notes-2`. */
export function assignHeadingIds(headings) {
const used = new Map();
for (const heading of headings) {
const base = slug(heading.textContent);
const seen = (used.get(base) || 0) + 1;
used.set(base, seen);
heading.id = seen === 1 ? base : `${base}-${seen}`;
}
return headings;
}
/**
* Where a link inside a documentation page should go.
*
* Markdown links between documents are relative file paths, which a browser would
* treat as downloads that leave the application. Returns the in-app target for a
* link to another document or to a heading, and `null` for anything else — external
* links stay exactly as the author wrote them.
*/
export function resolveDocLink(currentPage, href) {
if (!href) return null;
if (/^[a-z][a-z\d+.-]*:/i.test(href) || href.startsWith("//")) return null;
if (href.startsWith("#")) return { page: currentPage, anchor: href.slice(1) };
const [target, anchor = ""] = href.split("#");
if (!/\.md$/i.test(target)) return null;
// Resolved against the current document, so `../guides/x.md` means what it means
// in the repository. The origin is a placeholder; only the path is used.
const resolved = new URL(target, `https://docs.invalid/${currentPage}.md`);
const page = decodeURIComponent(resolved.pathname).replace(/^\//, "").replace(/\.md$/i, "");
return PAGE_PATTERN.test(page) ? { page, anchor } : null;
}
/**
* Point every in-app link at its route, and leave every other link alone.
*
* The resolved page is kept on the element, so the reading order can be read back
* from the rendered index rather than parsed out of the markdown a second time.
*/
export function rewriteLinks(article, page) {
for (const link of article.querySelectorAll("a[href]")) {
const href = link.getAttribute("href");
const target = resolveDocLink(page, href);
if (target) {
link.setAttribute("href", docHash(target.page, target.anchor));
link.dataset.docPage = target.page;
} else if (/^https?:/i.test(href)) {
link.setAttribute("rel", "noreferrer noopener");
link.setAttribute("target", "_blank");
}
}
return article;
}
/** The reading order, taken from the index document itself — one list, in one
* place, that is equally the index on Gitea and the navigation here. */
export function documentIndex(indexElement) {
const pages = [];
for (const link of indexElement.querySelectorAll("a[data-doc-page]")) {
const page = link.dataset.docPage;
if (page !== INDEX_PAGE && !pages.some((entry) => entry.page === page)) {
pages.push({ page, title: link.textContent.trim() });
}
}
return pages;
}
// ── the view ─────────────────────────────────────────────────────────────────
async function fetchPage(page) {
const response = await fetch(`${DOCS_BASE}/${page}.md`, { headers: { accept: "text/markdown" } });
if (!response.ok) throw new Error(`documentation page unavailable: ${response.status}`);
return response.text();
}
async function toArticle(markdown, page) {
const marked = await loadMarked();
const article = el("article", { class: "doc", "data-testid": "doc", "data-page": page });
// The only innerHTML in the application, and deliberate: rendering markdown *is*
// producing HTML. The input is a file from this repository, and the page's CSP
// (`script-src 'self'`, no `'unsafe-inline'`) means injected script and inline
// handlers do not run even if one ever were not.
article.innerHTML = marked.parse(markdown, { async: false });
assignHeadingIds(article.querySelectorAll("h1, h2, h3, h4, h5, h6"));
rewriteLinks(article, page);
return article;
}
function docHash(page, anchor = "") {
const query = new URLSearchParams(anchor ? { page, anchor } : { page });
return `#/docs?${query}`;
}
/** Diagrams are authored as ```mermaid blocks so the source is diffable and Gitea
* renders them natively. A failure here replaces the diagram, never the page. */
async function renderDiagrams(article) {
const blocks = [...article.querySelectorAll("pre > code.language-mermaid")];
if (!blocks.length) return;
let mermaid;
try {
mermaid = await loadMermaid();
mermaid.initialize({ startOnLoad: false, securityLevel: "strict", theme: "dark" });
} catch (error) {
blocks.forEach((block) => block.closest("pre").replaceWith(errorBanner(error.message)));
return;
}
for (const [index, block] of blocks.entries()) {
const figure = el("figure", { class: "diagram", "data-testid": "diagram" });
block.closest("pre").replaceWith(figure);
try {
const { svg } = await mermaid.render(`diagram-${index}-${Date.now()}`, block.textContent);
figure.innerHTML = svg;
} catch (error) {
figure.replaceWith(errorBanner(`Diagram could not be drawn: ${error.message}`));
}
}
}
function notFound() {
return el(
"div",
{ class: "alert", role: "alert", "data-testid": "doc-not-found" },
"That documentation page does not exist. ",
el("a", { class: "link", href: docHash(INDEX_PAGE) }, "Back to the documentation index")
);
}
function sidebar(pages, current) {
return el(
"nav",
{ class: "card", "aria-label": "Documentation" },
el("h2", {}, "Documentation"),
el(
"ul",
{ class: "album-list", "data-testid": "doc-pages" },
el(
"li",
{},
el(
"a",
{
href: docHash(INDEX_PAGE),
"aria-current": current === INDEX_PAGE ? "true" : false,
},
"Index"
)
),
...pages.map(({ page, title }) =>
el(
"li",
{},
el(
"a",
{
href: docHash(page),
"data-page": page,
"aria-current": page === current ? "true" : false,
},
title
)
)
)
)
);
}
export async function renderDocs(root, params = {}) {
setActiveNav("docs");
const requested = params.page || INDEX_PAGE;
const page = PAGE_PATTERN.test(requested) ? requested : "";
let indexArticle;
try {
indexArticle = await toArticle(await fetchPage(INDEX_PAGE), INDEX_PAGE);
} catch (error) {
root.replaceChildren(errorBanner(`Documentation is unavailable: ${error.message}`));
return;
}
const pages = documentIndex(indexArticle);
let article;
if (!page) article = notFound();
else if (page === INDEX_PAGE) article = indexArticle;
else {
try {
article = await toArticle(await fetchPage(page), page);
} catch {
article = notFound();
}
}
root.replaceChildren(el("div", { class: "two-pane" }, sidebar(pages, page), article));
await renderDiagrams(article);
scrollToAnchor(params.anchor);
}
// A documentation anchor cannot live in the hash — the hash is the route — so it
// travels as a parameter and is applied after the page renders.
function scrollToAnchor(anchor) {
if (!anchor) return;
const target = document.getElementById(anchor);
if (target) target.scrollIntoView({ block: "start" });
}

13
frontend/js/dom.js
View File

@@ -21,19 +21,6 @@ export function errorBanner(message) {
return el("div", { class: "alert", role: "alert" }, message);
}
/**
* Replace a container's contents, dropping the absent ones.
*
* `el` already ignores null children, but `replaceChildren` does not: it stringifies
* them, so an optional node that is not there renders the word "null" on the page.
* Every view builds optional nodes — a banner only while a job runs, a conflict only
* after a 409 — so the filter belongs here rather than in each caller. It was missing
* from one of them, and a documentation screenshot is where that turned up (US09-04).
*/
export function show(root, ...nodes) {
root.replaceChildren(...nodes.flat().filter((node) => node != null && node !== false));
}
export function setActiveNav(view) {
document.querySelectorAll("nav a[data-nav]").forEach((a) => {
if (a.dataset.nav === view) a.setAttribute("aria-current", "page");

8
frontend/js/renames.js
View File

@@ -5,7 +5,7 @@
// recovery classifications all come from the server; the view only shows them and
// refuses to offer an action the server would reject.
import { api } from "./api.js";
import { el, errorBanner, setActiveNav, show } from "./dom.js";
import { el, errorBanner, setActiveNav } from "./dom.js";
// What the last confirm did, for this tab only. Deliberately not persisted: after a
// reload the page must show what the journal says, not what this page remembers.
@@ -24,7 +24,7 @@ export async function renderRenames(root, params = {}) {
try {
[plans, recovery] = await Promise.all([api.listPlans(), api.renameRecovery()]);
} catch (error) {
show(root, errorBanner(`Failed to load renames: ${error.message}`));
root.replaceChildren(errorBanner(`Failed to load renames: ${error.message}`));
return;
}
@@ -34,13 +34,13 @@ export async function renderRenames(root, params = {}) {
try {
plan = await api.getPlan(selectedId);
} catch (error) {
show(root, errorBanner(`Failed to load plan: ${error.message}`));
root.replaceChildren(errorBanner(`Failed to load plan: ${error.message}`));
return;
}
}
const blocked = recovery.blocks_mutation;
show(root,
root.replaceChildren(
el("h1", {}, "Renames"),
recoveryPanel(recovery),
el(

57
frontend/js/tests/unit.js
View File

@@ -7,7 +7,6 @@ import { createStore } from "../store.js";
import { parseHash, navigate } from "../router.js";
import { api, cancellable } from "../api.js";
import { subscribeJob } from "../events.js";
import { assignHeadingIds, documentIndex, resolveDocLink, rewriteLinks, slug } from "../docs.js";
const cases = [];
function ok(name, cond) {
@@ -143,62 +142,6 @@ async function run() {
window.EventSource = realES;
}
// ── documentation view (US09-01) ─────────────────────────────────────────
{
ok("slug lowercases and dashes a heading", slug("Key Flows") === "key-flows");
ok(
"slug drops punctuation but keeps words",
slug("What it *will not* do:") === "what-it-will-not-do"
);
ok("slug keeps non-ASCII letters", slug("Größe & Gewicht") === "größe-gewicht");
ok("slug never yields an empty anchor", slug("!!!") === "section");
const doc = document.createElement("div");
doc.innerHTML = "<h2>Notes</h2><h3>Notes</h3><h2>Notes</h2>";
const ids = [...assignHeadingIds(doc.querySelectorAll("h2, h3"))].map((h) => h.id);
ok("repeated headings get distinct anchors", ids.join(",") === "notes,notes-2,notes-3");
ok(
"a sibling document link becomes an in-app route",
resolveDocLink("index", "overview.md")?.page === "overview"
);
const nested = resolveDocLink("guides/install", "../overview.md#running-it");
ok("a relative link resolves against the current page", nested?.page === "overview");
ok("a link's anchor survives the rewrite", nested?.anchor === "running-it");
ok(
"a bare anchor stays on the current page",
resolveDocLink("overview", "#the-workflow")?.page === "overview"
);
ok("an external link is left alone", resolveDocLink("index", "https://example.test/x") === null);
ok("a non-markdown relative link is left alone", resolveDocLink("index", "images/a.png") === null);
// A climb cannot leave the documentation tree: it is clamped at the root, so the
// page it names is still fetched from under /docs and simply does not exist.
ok(
"a link that climbs above the docs root is clamped",
resolveDocLink("index", "../../etc/passwd.md")?.page === "etc/passwd"
);
ok(
"a page name that is not a page name is refused",
resolveDocLink("index", "..%2f..%2fetc%2fpasswd.md") === null
);
const index = document.createElement("div");
index.innerHTML =
'<a href="overview.md">Overview</a><a href="overview.md">again</a>' +
'<a href="index.md">itself</a><a href="https://example.test">out</a>';
const listed = documentIndex(rewriteLinks(index, "index"));
ok(
"an external link is marked safe to open away from the app",
index.querySelector('a[href^="https"]').rel === "noreferrer noopener"
);
ok(
"an in-app link points at the docs route",
index.querySelector("a").getAttribute("href") === "#/docs?page=overview"
);
ok("the index lists each page once, in order", listed.length === 1);
ok("the index takes its titles from the link text", listed[0].title === "Overview");
}
await tick();
const failed = cases.filter((c) => !c.ok);
window.__RESULTS__ = { passed: cases.length - failed.length, failed: failed.length, cases };

8
frontend/js/uploads.js
View File

@@ -8,7 +8,7 @@
// preview arrives redacted — so nothing in this view may reconstruct, store, or
// route a secret.
import { api } from "./api.js";
import { el, errorBanner, setActiveNav, show } from "./dom.js";
import { el, errorBanner, setActiveNav } from "./dom.js";
import { subscribeJob } from "./events.js";
import { navigate } from "./router.js";
@@ -45,7 +45,7 @@ export async function renderUploads(root, params = {}) {
api.listUploadBatches(),
]);
} catch (error) {
show(root, errorBanner(`Failed to load uploads: ${error.message}`));
root.replaceChildren(errorBanner(`Failed to load uploads: ${error.message}`));
return;
}
@@ -62,12 +62,12 @@ export async function renderUploads(root, params = {}) {
batch = detail;
history = verifications.verifications;
} catch (error) {
show(root, errorBanner(`Failed to load upload batch: ${error.message}`));
root.replaceChildren(errorBanner(`Failed to load upload batch: ${error.message}`));
return;
}
}
show(root,
root.replaceChildren(
...[
el("h1", {}, "Upload"),
configurationCard(preflight),

23
frontend/js/vendor/VERSIONS.json vendored
View File

@@ -1,23 +0,0 @@
{
"_comment": "Vendored browser libraries (US09-01). Committed rather than fetched: the application is deployed to a network whose outbound access is not assumed, and `default-src 'self'` forbids a CDN. Checksums are asserted by tests/integration/test_documentation.py, so replacing a file without recording it here fails the suite. Update by downloading the pinned URL and recording the new version and sha256 in the same commit.",
"libraries": [
{
"file": "marked.esm.js",
"name": "marked",
"version": "18.0.10",
"license": "MIT",
"url": "https://cdn.jsdelivr.net/npm/marked@18.0.10/lib/marked.esm.js",
"sha256": "4cf47dfebb7f614a08fc0a579ab0fe407ff0ed2b717bf953040c85b2f493a4f0",
"why": "Markdown to HTML for the documentation view. An ES module with no dependencies, imported lazily by js/docs.js."
},
{
"file": "mermaid.min.js",
"name": "mermaid",
"version": "11.17.0",
"license": "MIT",
"url": "https://cdn.jsdelivr.net/npm/mermaid@11.17.0/dist/mermaid.min.js",
"sha256": "8d8e0eec56d3a83b4b3c87f42050845546dee93ebe1875d2117c12e6947c0cb3",
"why": "Renders ```mermaid blocks so diagrams are diffable source that Gitea also renders. The single-file UMD build rather than the ES module entry, whose ~40 lazy chunks would each need vendoring and pinning."
}
]
}

78
frontend/js/vendor/marked.esm.js vendored
View File

File diff suppressed because one or more lines are too long

3636
frontend/js/vendor/mermaid.min.js vendored
View File

File diff suppressed because one or more lines are too long

26
frontend/js/views.js
View File

@@ -3,7 +3,7 @@
// HTML. Mutating actions are disabled while a job holds the library_write lock and
// say why (concept §shared interaction rules: read-only browsing stays available).
import { api } from "./api.js";
import { el, errorBanner, setActiveNav, show } from "./dom.js";
import { el, errorBanner, setActiveNav } from "./dom.js";
import { navigate } from "./router.js";
import { subscribeJob } from "./events.js";
@@ -26,7 +26,7 @@ export async function renderWorkflow(root) {
try {
data = await api.workflow();
} catch (error) {
show(root, errorBanner(`Failed to load workflow: ${error.message}`));
root.replaceChildren(errorBanner(`Failed to load workflow: ${error.message}`));
return;
}
const active = data.active_job;
@@ -67,7 +67,7 @@ export async function renderWorkflow(root) {
)
);
});
show(root,
root.replaceChildren(
el("h1", {}, "Workflow"),
jobBanner(active),
el("div", { class: "stepper" }, ...cards)
@@ -105,7 +105,7 @@ export async function renderSafety(root, params) {
try {
data = await api.safetyQueue({ state });
} catch (error) {
show(root, errorBanner(`Failed to load safety queue: ${error.message}`));
root.replaceChildren(errorBanner(`Failed to load safety queue: ${error.message}`));
return;
}
@@ -149,7 +149,7 @@ export async function renderSafety(root, params) {
)
);
show(root,
root.replaceChildren(
el("h1", {}, "Safety review"),
tabs,
el(
@@ -197,7 +197,7 @@ export async function renderLibrary(root, params) {
try {
data = await api.libraryAssets({ q, offset, limit, sort: params.sort || "path" });
} catch (error) {
show(root, errorBanner(`Failed to load library: ${error.message}`));
root.replaceChildren(errorBanner(`Failed to load library: ${error.message}`));
return;
}
@@ -220,7 +220,7 @@ export async function renderLibrary(root, params) {
)
);
show(root,
root.replaceChildren(
el("h1", {}, "Library"),
el("div", { class: "toolbar" }, search, el("span", { class: "muted", "data-testid": "library-total" }, `${data.total} photos`)),
data.rows.length ? el("div", { class: "cluster-grid" }, ...cards) : el("p", { class: "muted" }, "No matching photos."),
@@ -235,7 +235,7 @@ export async function renderAnalyze(root) {
try {
[counts, workflow] = await Promise.all([api.analysisCounts(), api.workflow()]);
} catch (error) {
show(root, errorBanner(`Failed to load analysis: ${error.message}`));
root.replaceChildren(errorBanner(`Failed to load analysis: ${error.message}`));
return;
}
const active = workflow.active_job;
@@ -269,7 +269,7 @@ export async function renderAnalyze(root) {
"Analyze eligible SFW assets"
);
show(root,
root.replaceChildren(
el("h1", {}, "Analyze"),
jobBanner(active),
el(
@@ -292,10 +292,10 @@ export async function renderStats(root) {
try {
data = await api.libraryStats();
} catch (error) {
show(root, errorBanner(`Failed to load stats: ${error.message}`));
root.replaceChildren(errorBanner(`Failed to load stats: ${error.message}`));
return;
}
show(root,
root.replaceChildren(
el("h1", {}, "Stats"),
el("div", { class: "counts-grid", "data-testid": "stats-status" }, ...Object.entries(data.status).map(([k, v]) => stat(k, v))),
facetBlock("Top tags", data.top_tags),
@@ -338,7 +338,7 @@ export async function renderAlbums(root, params = {}) {
api.renameRecovery(),
]);
} catch (error) {
show(root, errorBanner(`Failed to load albums: ${error.message}`));
root.replaceChildren(errorBanner(`Failed to load albums: ${error.message}`));
return;
}
@@ -385,7 +385,7 @@ export async function renderAlbums(root, params = {}) {
)
: el("p", { class: "muted" }, "No albums with evidence yet.");
show(root,
root.replaceChildren(
el("h1", {}, "Albums"),
renameBlocked
? el(

45
migrations/versions/0016_performance_indexes.py
View File

@@ -1,45 +0,0 @@
"""Indexes the large-library read paths need (US07-06).
Revision ID: 0016_performance_indexes
Revises: 0015_exif_projections
Create Date: 2026-08-17
Measured, not guessed. At 100k assets the workflow home and the safety queue both
resolve "the current decision per asset" with a window function over
``safety_reviews``; a plain ``asset_id`` index makes SQLite sort every partition by
hand. Ordering the index by ``(asset_id, created_at DESC)`` halves that query.
``duplicate_members(cluster_id, asset_id)`` serves the paged member list of a
cluster with thousands of members, which is the other page that stopped being cheap.
"""
from alembic import op
revision = "0016_performance_indexes"
down_revision = "0015_exif_projections"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_index(
"ix_safety_reviews_asset_created",
"safety_reviews",
["asset_id", "created_at"],
)
op.create_index(
"ix_duplicate_members_cluster_asset",
"duplicate_members",
["cluster_id", "asset_id"],
)
op.create_index(
"ix_analysis_results_approx_year",
"analysis_results",
["approx_year"],
)
def downgrade() -> None:
op.drop_index("ix_analysis_results_approx_year", table_name="analysis_results")
op.drop_index("ix_duplicate_members_cluster_asset", table_name="duplicate_members")
op.drop_index("ix_safety_reviews_asset_created", table_name="safety_reviews")

272
photo_pipeline/__main__.py
View File

@@ -1,39 +1,21 @@
"""Application management CLI:
``python -m photo_pipeline {serve,migrate,worker,import-legacy-scores,backup,verify-backup,restore,diagnostics,benchmark,release-gate,container-gate,docs-gate,dry-run,approve-dry-run}``.
``serve`` and ``worker`` take the library process lock for their role (US07-05):
two workers, or the frozen CLI running beside the app, would each be safe on their
own and destructive together. ``restore`` is here rather than in the API because it
replaces the state of an installation and belongs to a stopped one.
"""
"""Application management CLI: ``python -m photo_pipeline {serve,migrate,worker,import-legacy-scores}``."""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from typing import Sequence
from photo_pipeline import path_policy
from photo_pipeline.config import Config
from photo_pipeline.services.app_lock import LegacyProcessActive, LibraryLock, LockHeld
from photo_pipeline.services.backup import BackupError, BackupService, migrate_with_backup
from photo_pipeline.db import run_migrations
def main(argv: Sequence[str] | None = None) -> int:
parser = argparse.ArgumentParser(prog="photo_pipeline")
commands = parser.add_subparsers(dest="command", required=True)
serve_cmd = commands.add_parser("serve", help="Run the API server")
commands.add_parser("serve", help="Run the API server")
commands.add_parser("migrate", help="Upgrade the database to the latest revision")
worker_cmd = commands.add_parser("worker", help="Run a durable-job worker")
worker_cmd.add_argument("--id", default="worker-1", help="Worker id (lease owner)")
for locked in (serve_cmd, worker_cmd):
locked.add_argument(
"--allow-legacy",
action="store_true",
help="Start even though a legacy CLI looks active (unsafe; you own the outcome)",
)
import_cmd = commands.add_parser(
"import-legacy-scores",
help="Import the archived CLI's nsfw_scores.csv into the database (US07-01)",
@@ -45,214 +27,22 @@ def main(argv: Sequence[str] | None = None) -> int:
import_cmd.add_argument(
"--dry-run", action="store_true", help="Report what would happen and change nothing"
)
backup_cmd = commands.add_parser("backup", help="Take an online database backup")
backup_cmd.add_argument("--reason", default="manual", help="Why (part of the directory name)")
backup_cmd.add_argument("--keep", type=int, default=7, help="How many backups to retain")
verify_cmd = commands.add_parser("verify-backup", help="Check a backup is intact and readable")
verify_cmd.add_argument("backup", help="Path to the backup directory")
restore_cmd = commands.add_parser(
"restore", help="Restore a verified backup into a fresh data directory"
)
restore_cmd.add_argument("backup", help="Path to the backup directory")
restore_cmd.add_argument("--into", required=True, help="Fresh data directory to restore into")
commands.add_parser("diagnostics", help="Report sizes, disk headroom, locks, and warnings")
bench_cmd = commands.add_parser(
"benchmark", help="Measure latency and resource use against agreed budgets (US07-06)"
)
bench_cmd.add_argument("--profile", default="smoke", help="smoke | short | full | huge")
bench_cmd.add_argument(
"--soak-seconds", type=float, default=0.0, help="Also run a soak of this length"
)
bench_cmd.add_argument("--output", help="Write the JSON report here as well as to stdout")
gate_cmd = commands.add_parser(
"release-gate", help="Run every suite in an isolated stack and keep the evidence"
)
gate_cmd.add_argument("--output", help="Evidence directory (default: data/release/<stamp>)")
container_cmd = commands.add_parser(
"container-gate",
help="Provision the composition from the built image, run the phase_h "
"acceptance suite against it, destroy it, and keep the evidence (US08-05)",
)
container_cmd.add_argument(
"--output", help="Evidence directory (default: data/container-gate/<stamp>)"
)
docs_cmd = commands.add_parser(
"docs-gate",
help="Check the manuals against the application they describe and keep the "
"evidence (US09-05)",
)
docs_cmd.add_argument("--output", help="Evidence directory (default: data/docs-gate/<stamp>)")
dry_cmd = commands.add_parser(
"dry-run", help="Read-only reconciliation of the configured library (US07-07)"
)
dry_cmd.add_argument("--output", help="Write the report here as well as to stdout")
approve_cmd = commands.add_parser(
"approve-dry-run", help="Approve a dry-run report, which is what enables mutation"
)
approve_cmd.add_argument("report", help="Path to the dry-run report")
approve_cmd.add_argument("--approver", required=True, help="Who is accepting this")
args = parser.parse_args(argv)
config = Config.from_env()
config.database_path.parent.mkdir(parents=True, exist_ok=True)
# In a container the two roles that touch the library check their boundary before
# they take a lock or bind a port: the configured roots must name the mount paths,
# and a mismatch is cheaper to refuse than to discover at the first write (US08-03).
# Only in a container — on a host an unmounted root is an ordinary Tuesday (an
# archive medium that is not plugged in), and refusing to serve would take the
# offline half of the library away with it.
if args.command in ("serve", "worker") and path_policy.in_container():
refusal = path_policy.roots_refusal(config.library_roots, require_mount=True)
if refusal is not None:
print(refusal, file=sys.stderr)
return 5
if args.command == "migrate":
manifest = migrate_with_backup(config)
if manifest:
print(json.dumps({"pre_migration_backup": manifest["name"]}, indent=2))
return 0
if args.command == "backup":
try:
manifest = BackupService(config).create(reason=args.reason, keep=args.keep)
except BackupError as error:
print(str(error))
return 1
print(json.dumps(manifest, indent=2))
return 0
if args.command == "verify-backup":
result = BackupService(config).verify(args.backup)
print(json.dumps(result.as_dict(), indent=2))
return 0 if result.ok else 1
if args.command == "restore":
try:
report = BackupService(config).restore(args.backup, args.into)
except BackupError as error:
print(str(error))
return 1
print(json.dumps(report, indent=2))
return 0
if args.command == "benchmark":
from photo_pipeline.services import benchmarks
try:
report = benchmarks.run(
config,
profile=args.profile,
soak_seconds=args.soak_seconds,
output=args.output,
)
except ValueError as error:
print(str(error))
return 1
print(json.dumps({k: v for k, v in report.items() if k != "runs"}, indent=2))
# A breached budget is a failed run, so a scheduled job notices without
# anyone reading the JSON.
return 0 if report["ok"] else 1
if args.command == "release-gate":
from photo_pipeline.services import release
report = release.run_gate(config, output=args.output)
print(
json.dumps(
{k: v for k, v in report.items() if k not in ("stages", "matrix")}, indent=2
)
)
return 0 if report["ok"] else 1
if args.command == "container-gate":
from datetime import datetime, timezone
from photo_pipeline.services import release
# The suite provisions and destroys the composition itself; what this command
# adds is the single entry point and the retained evidence. No skip is an
# environment limit here: a gate that did not reach the containers proved
# nothing about them.
output = args.output or Path(config.data_dir) / "container-gate" / datetime.now(
timezone.utc
).strftime("%Y%m%dT%H%M%SZ")
report = release.run_gate(
config,
output=output,
stages=release.CONTAINER_STAGES,
allowed_skips=release.CONTAINER_ALLOWED_SKIP_REASONS,
)
print(
json.dumps(
{k: v for k, v in report.items() if k not in ("stages", "matrix")}, indent=2
)
)
return 0 if report["ok"] else 1
if args.command == "docs-gate":
from datetime import datetime, timezone
from photo_pipeline.services import release
# Documentation that disagrees with the application is worse than none: it is
# trusted. So this gate accepts no skip either — a check that did not run is a
# page nobody compared with the code.
output = args.output or Path(config.data_dir) / "docs-gate" / datetime.now(
timezone.utc
).strftime("%Y%m%dT%H%M%SZ")
report = release.run_gate(
config,
output=output,
stages=release.DOCS_STAGES,
allowed_skips=release.DOCS_ALLOWED_SKIP_REASONS,
)
print(
json.dumps(
{k: v for k, v in report.items() if k not in ("stages", "matrix")}, indent=2
)
)
return 0 if report["ok"] else 1
if args.command == "dry-run":
from photo_pipeline.services import release
try:
report = release.dry_run(config)
except release.ReleaseError as error:
print(str(error))
return 1
if args.output:
Path(args.output).write_text(json.dumps(report, indent=2))
print(json.dumps(report, indent=2))
return 0
if args.command == "approve-dry-run":
from photo_pipeline.services import release
try:
record = release.approve(config, args.report, approver=args.approver)
except (release.ReleaseError, OSError, ValueError) as error:
print(str(error))
return 1
print(json.dumps(record, indent=2))
return 0
if args.command == "diagnostics":
from photo_pipeline.services import diagnostics
print(json.dumps(diagnostics.report(config), indent=2))
run_migrations(config.database_url)
return 0
if args.command == "import-legacy-scores":
import json
from photo_pipeline.db import create_db_engine, create_session_factory
from photo_pipeline.services.legacy_import import LegacyImportService, write_report
migrate_with_backup(config)
run_migrations(config.database_url)
engine = create_db_engine(config.database_url)
service = LegacyImportService(create_session_factory(engine))
report = service.import_nsfw_scores(
@@ -273,56 +63,18 @@ def main(argv: Sequence[str] | None = None) -> int:
import photo_pipeline.jobs.domain_handlers # noqa: F401
from photo_pipeline.jobs.worker import Worker
lock = LibraryLock(config, "worker")
if (held := _acquire(lock, allow_legacy=args.allow_legacy)) is not None:
return held
try:
migrate_with_backup(config)
engine = create_db_engine(config.database_url)
Worker(
create_session_factory(engine), worker_id=args.id, config=config
).run_forever()
finally:
lock.release()
run_migrations(config.database_url)
engine = create_db_engine(config.database_url)
Worker(create_session_factory(engine), worker_id=args.id, config=config).run_forever()
return 0
import uvicorn
from photo_pipeline.api.app import ConfigurationRefused, create_app
from photo_pipeline.api.app import create_app
# An exposed deployment without an access secret must not reach the port at all,
# and the operator needs a sentence, not a traceback (US08-01).
try:
app = create_app(config)
except ConfigurationRefused as error:
print(str(error), file=sys.stderr)
return 4
lock = LibraryLock(config, "api")
if (held := _acquire(lock, allow_legacy=args.allow_legacy)) is not None:
return held
try:
uvicorn.run(app, host=config.host, port=config.port)
finally:
lock.release()
uvicorn.run(create_app(config), host=config.host, port=config.port)
return 0
def _acquire(lock: LibraryLock, *, allow_legacy: bool) -> int | None:
"""Take the lock, or explain on stderr why this process must not start.
Returns an exit code to return, or ``None`` when the lock was acquired.
"""
try:
lock.acquire(allow_legacy=allow_legacy)
except LockHeld as error:
print(str(error), file=sys.stderr)
return 2
except LegacyProcessActive as error:
print(f"{error} (override with --allow-legacy)", file=sys.stderr)
return 3
return None
if __name__ == "__main__":
raise SystemExit(main())

35
photo_pipeline/api/app.py
View File

@@ -27,7 +27,6 @@ from photo_pipeline.api.routes import (
inventory,
jobs,
library,
operations,
renames,
safety,
session as session_routes,
@@ -35,25 +34,17 @@ from photo_pipeline.api.routes import (
uploads,
workflow,
)
from photo_pipeline.api.security import (
DEFAULT_HEADERS,
FailureLimiter,
SecurityMiddleware,
Session,
trust_refusal,
)
from photo_pipeline.api.security import DEFAULT_HEADERS, SecurityMiddleware, Session
# Registers the safety_score / analysis job handlers on import.
import photo_pipeline.jobs.domain_handlers # noqa: F401
from photo_pipeline.config import Config
from photo_pipeline.db import create_db_engine, create_session_factory
from photo_pipeline.services.backup import migrate_with_backup
from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
from photo_pipeline.logging import configure_logging
from photo_pipeline.services.thumbnails import ThumbnailService
from photo_pipeline.services.upload_batches import UploadBatchService
FRONTEND_DIR = Path(__file__).resolve().parents[2] / "frontend"
DOCS_DIR = Path(__file__).resolve().parents[2] / "docs"
log = logging.getLogger(__name__)
@@ -91,23 +82,14 @@ def _install_error_handlers(app: FastAPI) -> None:
return _envelope(500, "internal_error", "internal error")
class ConfigurationRefused(RuntimeError):
"""The configuration would serve the library to callers it cannot authenticate."""
def create_app(config: Config | None = None) -> FastAPI:
config = config or Config.from_env()
configure_logging(config.log_level, config.log_format)
# Before anything is built, let alone bound to a port (US08-01).
if (why := trust_refusal(config)) is not None:
raise ConfigurationRefused(why)
@asynccontextmanager
async def lifespan(app: FastAPI):
config.database_path.parent.mkdir(parents=True, exist_ok=True)
# A schema upgrade is snapshotted first, so a migration that fails halfway
# leaves a restorable database behind rather than a damaged one (US07-05).
migrate_with_backup(config)
run_migrations(config.database_url)
engine = create_db_engine(config.database_url)
app.state.config = config
app.state.engine = engine
@@ -128,10 +110,6 @@ def create_app(config: Config | None = None) -> FastAPI:
# One session per process: the browser exchanges it for a cookie + CSRF token,
# and every other origin is refused before a route ever runs (US07-02).
app.state.session = Session.create()
app.state.access_limiter = FailureLimiter()
# Also set in the lifespan, but the bootstrap route reads it, and a caller can
# arrive before anything else has touched app.state.
app.state.config = config
app.add_middleware(SecurityMiddleware, session=app.state.session, config=config)
_install_error_handlers(app)
app.include_router(session_routes.router, prefix="/api/v1")
@@ -148,14 +126,7 @@ def create_app(config: Config | None = None) -> FastAPI:
app.include_router(renames.router, prefix="/api/v1")
app.include_router(uploads.router, prefix="/api/v1")
app.include_router(archives.router, prefix="/api/v1")
app.include_router(operations.router, prefix="/api/v1")
# Static single-page app (hash-routed). Mounted last so /api/v1 wins.
if FRONTEND_DIR.is_dir():
app.mount("/app", StaticFiles(directory=FRONTEND_DIR, html=True), name="app")
# The manuals, as their own markdown files (US09-01). Unauthenticated for the
# same reason the application shell is: whoever cannot get past the access
# secret is exactly who needs the troubleshooting page, and no document holds
# anything a session would protect.
if DOCS_DIR.is_dir():
app.mount("/docs", StaticFiles(directory=DOCS_DIR), name="docs")
return app

17
photo_pipeline/api/routes/duplicates.py
View File

@@ -11,13 +11,7 @@ from fastapi import APIRouter, Query, Request
from fastapi.responses import JSONResponse
from photo_pipeline.schemas import DecisionRequest
from photo_pipeline.services.duplicates import (
MAX_MEMBER_PAGE,
MEMBER_PAGE,
ConflictError,
DuplicateError,
DuplicateService,
)
from photo_pipeline.services.duplicates import ConflictError, DuplicateError, DuplicateService
router = APIRouter(tags=["duplicates"])
@@ -48,13 +42,8 @@ def list_clusters(
@router.get("/duplicates/clusters/{cluster_id}")
def get_cluster(
cluster_id: str,
request: Request,
limit: int = Query(MEMBER_PAGE, ge=1, le=MAX_MEMBER_PAGE),
offset: int = Query(0, ge=0),
):
detail = _service(request).get_cluster(cluster_id, limit=limit, offset=offset)
def get_cluster(cluster_id: str, request: Request):
detail = _service(request).get_cluster(cluster_id)
if detail is None:
return _error(404, "not_found", f"unknown cluster {cluster_id}")
return detail

70
photo_pipeline/api/routes/operations.py
View File

@@ -1,70 +0,0 @@
"""Operational endpoints: diagnostics and backups (US07-05).
Backups can be taken and verified here because both are safe, additive, and the
operator needs them from the same screen that shows the disk filling up.
**Restore is deliberately not an endpoint.** It replaces the state of the running
application with an older one, so it belongs to a stopped installation and a person
at a terminal: ``python -m photo_pipeline restore``. An HTTP call that can silently
roll the library back to last week is a hole, not a feature.
"""
from __future__ import annotations
from fastapi import APIRouter, Query, Request
from fastapi.responses import JSONResponse
from pydantic import BaseModel
from photo_pipeline.services import diagnostics
from photo_pipeline.services.backup import DEFAULT_KEEP, BackupError, BackupService
router = APIRouter(tags=["operations"])
class CreateBackupRequest(BaseModel):
reason: str = "manual"
keep: int = DEFAULT_KEEP
def _service(request: Request) -> BackupService:
return BackupService(request.app.state.config)
def _error(status: int, code: str, message: str) -> JSONResponse:
return JSONResponse(status_code=status, content={"error": {"code": code, "message": message}})
@router.get("/diagnostics")
def read_diagnostics(request: Request) -> dict:
return diagnostics.report(request.app.state.config)
@router.get("/backups")
def list_backups(request: Request) -> dict:
return {"backups": _service(request).list()}
@router.post("/backups", status_code=201)
def create_backup(body: CreateBackupRequest, request: Request):
try:
return _service(request).create(reason=body.reason, keep=body.keep)
except BackupError as error:
return _error(422, "backup_failed", str(error))
@router.get("/backups/{name}/verify")
def verify_backup(name: str, request: Request):
service = _service(request)
# The name comes from the browser, so it names a backup — it is never joined
# into a path until it has been matched against one that exists (US07-02).
if name not in {entry["name"] for entry in service.list()}:
return _error(404, "not_found", f"unknown backup {name}")
return {"name": name, **service.verify(service.root / name).as_dict()}
@router.post("/backups/prune")
def prune_backups(request: Request, keep: int = Query(DEFAULT_KEEP, ge=1)):
try:
return {"removed": _service(request).prune(keep=keep)}
except BackupError as error:
return _error(422, "invalid_retention", str(error))

42
photo_pipeline/api/routes/session.py
View File

@@ -3,52 +3,21 @@
It sets the ``HttpOnly``/``SameSite=Strict`` session cookie and returns the CSRF
token in the body. A foreign page can call this — it just cannot read the answer,
because the app sends no CORS headers — and the cookie it received is never attached
to a request that foreign page initiated.
When an access secret is configured (mandatory as soon as the app is reachable from
another machine, US08-01) this is also the authentication gate: the secret buys the
cookie, and every route behind it keeps asking for exactly the session and CSRF token
it asked for before. Wrong secrets are counted, and a burst of them stops being
answered — otherwise a proxy-exposed deployment could be guessed at indefinitely.
to a request that foreign page initiates.
"""
from __future__ import annotations
import logging
import secrets
from fastapi import APIRouter, Request
from fastapi.responses import JSONResponse
from photo_pipeline.api.security import ACCESS_SECRET_HEADER, SESSION_COOKIE
from photo_pipeline.api.security import SESSION_COOKIE
router = APIRouter(tags=["session"])
log = logging.getLogger(__name__)
def _refuse(status: int, code: str, message: str) -> JSONResponse:
return JSONResponse(status_code=status, content={"error": {"code": code, "message": message}})
@router.get("/session")
def start_session(request: Request) -> JSONResponse:
config = request.app.state.config
secret = config.access_secret
if secret is not None:
limiter = request.app.state.access_limiter
if limiter.blocked():
return _refuse(429, "too_many_attempts", "too many failed attempts; retry later")
offered = request.headers.get(ACCESS_SECRET_HEADER, "")
if not secrets.compare_digest(offered, secret.get_secret_value()):
limiter.record_failure()
# The client address is the whole record: the offered secret, the issued
# session, and the request body all stay out of the log.
log.warning(
"access secret rejected", extra={"client": _client(request), "path": "/session"}
)
return _refuse(401, "access_denied", "a valid access secret is required")
session = request.app.state.session
response = JSONResponse({"csrf_token": session.csrf_token})
response.set_cookie(
@@ -56,13 +25,6 @@ def start_session(request: Request) -> JSONResponse:
session.id,
httponly=True,
samesite="strict",
# HTTPS outside means the cookie must never travel over a plain hop, even one
# this process cannot see. Loopback http keeps working unchanged.
secure=request.scope.get("state", {}).get("external_scheme") == "https",
path="/",
)
return response
def _client(request: Request) -> str:
return request.client.host if request.client else "unknown"

154
photo_pipeline/api/security.py
View File

@@ -19,14 +19,6 @@ The defenses stack, because each one alone has a hole:
only in the bootstrap response body, which a foreign page cannot read (no CORS) —
so possessing it proves the caller is same-origin.
Behind a reverse proxy (US08-01) the same stack holds with two substitutions: the
allowed host set comes from configuration instead of being the loopback names, and
the host/scheme the policy judges is the *external* one, which is only read from
``X-Forwarded-*`` when the request actually arrived from a configured proxy. The
loopback check was standing in for authentication, so naming a non-loopback host
also makes an access secret mandatory — ``trust_refusal`` refuses to start without
one, and the secret is what the bootstrap endpoint trades for the session cookie.
``evaluate`` is a pure function over the request metadata: the whole policy is one
table that a unit test can enumerate, and the middleware only applies its verdict.
"""
@@ -34,7 +26,6 @@ table that a unit test can enumerate, and the middleware only applies its verdic
from __future__ import annotations
import secrets
import time
from collections.abc import Mapping
from dataclasses import dataclass
from urllib.parse import urlsplit
@@ -44,7 +35,6 @@ from starlette.responses import JSONResponse
SESSION_COOKIE = "pp_session"
CSRF_HEADER = "x-csrf-token"
ACCESS_SECRET_HEADER = "x-access-secret"
API_PREFIX = "/api/v1"
SAFE_METHODS = frozenset({"GET", "HEAD", "OPTIONS"})
# Reachable without a session: liveness/readiness (an orchestrator has no cookie)
@@ -53,25 +43,10 @@ PUBLIC_PATHS = frozenset(
{f"{API_PREFIX}/health/live", f"{API_PREFIX}/health/ready", f"{API_PREFIX}/session"}
)
LOOPBACK_HOSTS = frozenset({"127.0.0.1", "localhost", "::1", "[::1]"})
# Mutating endpoints that must stay reachable while mutation itself is gated: the
# backup a careful operator takes first, and its retention (US07-07).
MUTATION_EXEMPT_PATHS = frozenset({f"{API_PREFIX}/backups", f"{API_PREFIX}/backups/prune"})
# Applied to every response. `frame-ancestors 'none'` and CORP keep other pages from
# Applied to every response. No inline script/style is used by the frontend, so the
# policy can stay strict; `frame-ancestors 'none'` and CORP keep other pages from
# embedding the app or its thumbnails.
#
# `script-src 'self'` is the boundary that matters and it is unchanged: no
# `'unsafe-eval'`, no `'unsafe-inline'`, so nothing injected into the DOM can execute.
# Whether the vendored diagram renderer needed `'unsafe-eval'` was measured rather
# than assumed — mermaid's bundle contains no `eval(` and no `new Function`, and it
# renders under this exact policy without a single script-src violation.
#
# `style-src` does gain `'unsafe-inline'` (US09-01): mermaid styles the SVG it builds
# with an injected `<style>` element and `style=` attributes, and a diagram's CSS
# cannot be hashed in advance. The concession is bounded by the directives around it
# — with script execution still refused and `img-src`, `connect-src`, and `font-src`
# all `'self'`, the CSS exfiltration channels stay closed and what is left is
# defacement of a page its own operator is already looking at.
DEFAULT_HEADERS = {
"x-content-type-options": "nosniff",
"x-frame-options": "DENY",
@@ -79,9 +54,8 @@ DEFAULT_HEADERS = {
"cross-origin-resource-policy": "same-origin",
"cross-origin-opener-policy": "same-origin",
"content-security-policy": (
"default-src 'self'; img-src 'self' data:; style-src 'self' 'unsafe-inline'; "
"script-src 'self'; connect-src 'self'; font-src 'self'; "
"frame-ancestors 'none'; base-uri 'none'; form-action 'none'"
"default-src 'self'; img-src 'self' data:; style-src 'self'; script-src 'self'; "
"connect-src 'self'; frame-ancestors 'none'; base-uri 'none'; form-action 'none'"
),
}
@@ -119,28 +93,6 @@ def split_host(value: str) -> tuple[str, str]:
return host, port
def external_view(
*,
client: str | None,
headers: Mapping[str, str],
scheme: str,
trusted_proxies: frozenset[str],
) -> tuple[str, str]:
"""The ``(scheme, host)`` the caller used, as opposed to the one this hop saw.
Forwarded headers are a client-supplied claim. Believing them from anyone lets a
request declare its own origin — and origin is half of this module's evidence —
so they count only when the connection came from a configured proxy.
"""
host = headers.get("host", "")
if client is None or client not in trusted_proxies:
return scheme, host
# A chain appends: the first entry is what the original client asked for.
forwarded_proto = headers.get("x-forwarded-proto", "").split(",")[0].strip().lower()
forwarded_host = headers.get("x-forwarded-host", "").split(",")[0].strip()
return forwarded_proto or scheme, forwarded_host or host
def evaluate(
*,
method: str,
@@ -148,26 +100,20 @@ def evaluate(
headers: Mapping[str, str],
session: Session,
allowed_hosts: frozenset[str] = LOOPBACK_HOSTS,
scheme: str = "http",
max_request_bytes: int,
) -> Refusal | None:
"""Why this request must be refused, or ``None`` when it may proceed.
``headers["host"]`` and ``scheme`` are the external ones (see ``external_view``);
the allowed origins are the allowed hosts under that scheme and port, so there is
no second list that can drift away from the first.
"""
"""Why this request must be refused, or ``None`` when it may proceed."""
host_header = headers.get("host", "")
host, port = split_host(host_header)
if host.lower() not in allowed_hosts:
return Refusal(403, "host_not_allowed", "request host is not an allowed address")
return Refusal(403, "host_not_allowed", "request host is not a local address")
origin = headers.get("origin")
if origin is not None and origin != "":
parts = urlsplit(origin)
origin_host, origin_port = split_host(parts.netloc)
if (
parts.scheme != scheme
parts.scheme not in ("http", "https")
or origin_host.lower() not in allowed_hosts
or origin_port != port
):
@@ -193,66 +139,14 @@ def evaluate(
return None
def exposed_hosts(config) -> list[str]:
"""Configured names by which this application is reachable from another machine."""
names = {str(config.host).lower()}
names.update(split_host(name)[0].lower() for name in config.allowed_hosts)
return sorted(names - LOOPBACK_HOSTS)
def trust_refusal(config) -> str | None:
"""Why this configuration must not serve at all, or ``None``.
Reaching the app used to prove ownership of it. The moment a configuration makes
it reachable from elsewhere that stops being true, so serving without a secret
would publish the library — refuse at startup rather than at the first request,
when the operator is no longer watching (US08-01).
"""
exposed = exposed_hosts(config)
if exposed and config.access_secret is None:
return (
f"refusing to serve: {', '.join(exposed)} is reachable from outside this "
"machine, so PHOTO_PIPELINE_ACCESS_SECRET must be set"
)
return None
class FailureLimiter:
"""Bounded failed access-secret attempts, so the secret cannot be guessed online.
ponytail: one counter for the whole process rather than per client address —
behind a proxy every attempt arrives from the same address anyway. Per-caller
buckets if the app is ever exposed without one.
"""
def __init__(self, limit: int = 5, window: float = 60.0) -> None:
self.limit = limit
self.window = window
self._failures: list[float] = []
def blocked(self) -> bool:
now = time.monotonic()
self._failures = [at for at in self._failures if now - at < self.window]
return len(self._failures) >= self.limit
def record_failure(self) -> None:
self._failures.append(time.monotonic())
class SecurityMiddleware:
"""Pure-ASGI so the SSE stream keeps streaming (BaseHTTPMiddleware buffers)."""
def __init__(self, app, *, session: Session, config) -> None:
self.app = app
self.session = session
self.config = config
self.max_request_bytes = config.max_request_bytes
self.allowed_hosts = frozenset(
LOOPBACK_HOSTS
| {str(config.host).lower()}
| {split_host(name)[0].lower() for name in config.allowed_hosts}
)
self.trusted_proxies = frozenset(config.trusted_proxies)
self.allowed_hosts = frozenset(LOOPBACK_HOSTS | {str(config.host).lower()})
async def __call__(self, scope, receive, send) -> None:
if scope["type"] != "http":
@@ -263,26 +157,14 @@ class SecurityMiddleware:
# policy never has to parse a Cookie header.
lookup = dict(headers)
lookup["cookie-session"] = _cookie(headers.get("cookie", ""), SESSION_COOKIE)
client = scope.get("client")
scheme, lookup["host"] = external_view(
client=client[0] if client else None,
headers=headers,
scheme=scope.get("scheme", "http"),
trusted_proxies=self.trusted_proxies,
)
# What the session cookie's Secure flag is decided from, one hop later.
scope.setdefault("state", {})["external_scheme"] = scheme
refusal = evaluate(
method=scope.get("method", "GET"),
path=scope.get("path", "/"),
headers=lookup,
session=self.session,
allowed_hosts=self.allowed_hosts,
scheme=scheme,
max_request_bytes=self.max_request_bytes,
)
if refusal is None:
refusal = self._mutation_refusal(scope)
if refusal is not None:
response = JSONResponse(
status_code=refusal.status,
@@ -301,26 +183,6 @@ class SecurityMiddleware:
await self.app(scope, receive, send_with_headers)
def _mutation_refusal(self, scope) -> Refusal | None:
"""Refuse every mutating request while the library's dry run is unapproved.
One choke point for the whole API: every mutation the browser can start is a
non-safe method under ``/api/v1``. Reading stays open — an operator has to be
able to look at what the application found in order to approve it (US07-07).
"""
method = scope.get("method", "GET").upper()
path = scope.get("path", "/")
if method in SAFE_METHODS or not path.startswith(API_PREFIX):
return None
if path in MUTATION_EXEMPT_PATHS:
return None
from photo_pipeline.services.release import mutation_blockers
blockers = mutation_blockers(self.config)
if not blockers:
return None
return Refusal(403, blockers[0]["code"], blockers[0]["message"])
def _cookie(header: str, name: str) -> str:
for part in header.split(";"):

88
photo_pipeline/config.py
View File

@@ -7,9 +7,6 @@ real external call needs them.
pydantic-settings would do this too, but a prefix-scan over the declared fields
is a few lines and one fewer dependency.
Tuple-valued settings are lists in one variable: library roots are ``os.pathsep``
separated because they are paths, everything else is comma separated.
"""
from __future__ import annotations
@@ -21,62 +18,6 @@ from typing import Mapping
from pydantic import BaseModel, ConfigDict, SecretStr
ENV_PREFIX = "PHOTO_PIPELINE_"
ENV_FILE_VAR = f"{ENV_PREFIX}ENV_FILE"
DEFAULT_ENV_FILE = Path(".env")
COMMA_LIST_FIELDS = frozenset({"allowed_hosts", "trusted_proxies"})
# The archived CLI's variable names, so the configuration file an operator already
# has keeps working. The vision provider reads the OpenAI SDK's names, and the
# library root is configuration here rather than a bare path (US07-01 donor).
LEGACY_ALIASES = {
"LLM_API_KEY": "OPENAI_API_KEY",
"GEMINI_API_KEY": "OPENAI_API_KEY",
"LLM_BASE_URL": "OPENAI_BASE_URL",
"LIBRARY": f"{ENV_PREFIX}LIBRARY_ROOTS",
}
def parse_env_file(text: str) -> dict[str, str]:
"""``KEY=value`` lines into a mapping. Comments, blanks, and quotes handled.
Deliberately not a shell: no interpolation, no ``export``, no multi-line values.
A configuration file that can run code is a configuration file that can be a
vulnerability.
"""
values: dict[str, str] = {}
for line in text.splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, _, raw = line.partition("=")
key = key.strip()
if not key or key.startswith("#"):
continue
value = raw.strip().strip('"').strip("'")
values[key] = value
alias = LEGACY_ALIASES.get(key)
if alias:
values.setdefault(alias, value)
return values
def load_env_file(path: Path | str | None = None) -> dict[str, str]:
"""Load ``PHOTO_PIPELINE_ENV_FILE`` (or ``./.env``) into the environment.
Anything already exported wins: a file is the standing configuration, the shell
is what you meant *this time*. Returns what it applied, which is what the CLI
prints — names only, never values.
"""
candidate = path or os.environ.get(ENV_FILE_VAR) or DEFAULT_ENV_FILE
candidate = Path(candidate)
if not candidate.is_file():
return {}
applied = {}
for key, value in parse_env_file(candidate.read_text()).items():
if key not in os.environ:
os.environ[key] = value
applied[key] = value
return applied
class Config(BaseModel):
@@ -89,19 +30,6 @@ class Config(BaseModel):
log_level: str = "INFO"
log_format: str = "json" # "json" or "text"
# Trust boundary (US08-01). Empty means loopback only, which is what the app did
# before there was a setting: a request whose Host is not a loopback name is
# refused, and no secret is needed because nothing outside this machine can call.
# Naming a real hostname here is what makes the app reachable through a reverse
# proxy, and it is exactly then that ``access_secret`` becomes mandatory.
allowed_hosts: tuple[str, ...] = ()
# Addresses whose ``X-Forwarded-Proto``/``X-Forwarded-Host`` may be believed. A
# client that is not the proxy can otherwise declare its own origin.
trusted_proxies: tuple[str, ...] = ()
# Exchanged for the session cookie at the bootstrap endpoint. Once set it is
# required even on loopback, so a development setup cannot half-enable it.
access_secret: SecretStr | None = None
# Largest request body the API accepts. Every endpoint takes small JSON commands;
# anything larger is a mistake or an attempt to exhaust memory (US07-02).
max_request_bytes: int = 1_048_576
@@ -114,12 +42,6 @@ class Config(BaseModel):
# Free space an archive destination must keep beyond the transfer itself.
archive_free_space_reserve_bytes: int = 1_000_000_000
# Refuse every mutating request until a read-only dry run of the configured
# library has been produced and explicitly approved (US07-07). Off by default so
# a development setup is unchanged; turn it on before pointing the application at
# a library whose photos cannot be replaced.
require_dry_run_approval: bool = False
vision_api_key: SecretStr | None = None
immich_api_key: SecretStr | None = None
immich_server_url: str = ""
@@ -140,18 +62,10 @@ class Config(BaseModel):
@classmethod
def from_env(cls, environ: Mapping[str, str] | None = None) -> "Config":
env = os.environ if environ is None else environ
if environ is None:
load_env_file() # a file never overrides what the shell already set
env = os.environ
data: dict = {}
for name in cls.model_fields:
raw = env.get(ENV_PREFIX + name.upper())
if not raw:
continue
if name == "library_roots":
data[name] = raw.split(os.pathsep)
elif name in COMMA_LIST_FIELDS:
data[name] = [part.strip() for part in raw.split(",") if part.strip()]
else:
data[name] = raw
data[name] = raw.split(os.pathsep) if name == "library_roots" else raw
return cls(**data)

37
photo_pipeline/db.py
View File

@@ -41,41 +41,12 @@ def create_session_factory(engine: Engine) -> sessionmaker:
return sessionmaker(bind=engine, expire_on_commit=False, future=True)
def _alembic_config(url: str):
def run_migrations(url: str) -> None:
"""Upgrade the database at ``url`` to the latest revision."""
from alembic import command
from alembic.config import Config as AlembicConfig
cfg = AlembicConfig(str(_REPO_ROOT / "alembic.ini"))
cfg.set_main_option("script_location", str(_REPO_ROOT / "migrations"))
cfg.set_main_option("sqlalchemy.url", url)
return cfg
def run_migrations(url: str) -> None:
"""Upgrade the database at ``url`` to the latest revision."""
from alembic import command
command.upgrade(_alembic_config(url), "head")
def head_revision() -> str | None:
"""The revision this code expects. ``None`` if the scripts cannot be read."""
from alembic.script import ScriptDirectory
try:
return ScriptDirectory.from_config(_alembic_config("sqlite://")).get_current_head()
except Exception:
return None
def current_revision(url: str) -> str | None:
"""The revision a database is actually at, or ``None`` for an unstamped one."""
engine = create_db_engine(url)
try:
with engine.connect() as connection:
from alembic.runtime.migration import MigrationContext
return MigrationContext.configure(connection).get_current_revision()
except Exception:
return None
finally:
engine.dispose()
command.upgrade(cfg, "head")

23
photo_pipeline/integrations/exiftool.py
View File

@@ -12,10 +12,8 @@ nt-apply-list, pa-nsfw-filter). No dependency on the archived entry points.
from __future__ import annotations
import functools
import json
import os
import shutil
import subprocess
from collections.abc import Iterable
@@ -34,27 +32,6 @@ def _timeout() -> float:
return DEFAULT_TIMEOUT_SECONDS
def find_binary(binary: str = "exiftool") -> str | None:
"""Absolute path of exiftool, or ``None`` when it is not installed."""
return shutil.which(binary)
@functools.lru_cache(maxsize=1)
def version() -> str | None:
"""Reported exiftool version, or ``None`` when it is missing or unusable.
Cached: it cannot change inside a running process, and diagnostics asks for it
on every report (US08-02, where a container image pins this version).
"""
try:
result = subprocess.run(
["exiftool", "-ver"], capture_output=True, text=True, timeout=_timeout()
)
except (OSError, subprocess.SubprocessError):
return None
return (result.stdout or "").strip() or None
def read_keyword_sets(paths: Iterable[str]) -> dict[str, set[str]]:
"""Map each path to its lowercased set of ``Keywords`` + ``Subject`` values.

61
photo_pipeline/path_policy.py
View File

@@ -15,7 +15,9 @@ import os
from pathlib import Path
from typing import Iterable, Iterator
SUPPORTED_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp", ".heic", ".heif", ".tiff", ".tif"}
SUPPORTED_EXTENSIONS = {
".jpg", ".jpeg", ".png", ".webp", ".heic", ".heif", ".tiff", ".tif"
}
EXCLUDED_DIR_NAMES = {"_IGNORE", ".@__thumb"}
@@ -70,63 +72,6 @@ def resolve_in_roots(roots: Iterable[os.PathLike | str], path: os.PathLike | str
raise PathPolicyError("path is outside the configured library roots")
def in_container() -> bool:
"""Whether this process is running inside a container image build of the app."""
return Path("/.dockerenv").exists()
def _under_mount(path: Path) -> bool:
"""Whether ``path`` or one of its parents below ``/`` is a mounted filesystem."""
current = path.resolve()
while current != current.parent:
if os.path.ismount(current):
return True
current = current.parent
return False
def roots_refusal(roots: Iterable[os.PathLike | str], *, require_mount: bool = False) -> str | None:
"""Why the configured library roots cannot be worked with, or ``None`` (US08-03).
The roots name the directories this installation renames folders in, rewrites
EXIF in, and archives from. Container path policy is the same problem as host
path policy with one new failure mode: the configured roots must name the
*container-side* mount paths. A host path configured inside a container is
either absent or an ordinary directory of the image, so the library looks empty
and the first write lands in the container's throwaway layer instead of in the
library. That is worth refusing at startup, while an operator is still watching,
rather than at the first write.
``require_mount`` is the container-only half of that: inside a container a real
library arrives through a bind mount, so a root that is not on (or under) a
mount point is not the library the deployment meant.
Writability is deliberately *not* checked: a bind mount's ownership is
virtualised by Docker Desktop and Colima (it arrives as ``root:root``), so
``os.access`` there is evidence about the virtio layer rather than about the
library. A wrong UID/GID surfaces as a refused rename with the real errno, which
is at least true; a refusal here would be false on two supported platforms.
"""
for root in roots:
path = Path(root)
if not path.exists():
return (
f"library root {path} does not exist: PHOTO_PIPELINE_LIBRARY_ROOTS must "
"name paths that exist here, and in a container that means the mount path"
)
if not path.is_dir():
return f"library root {path} is not a directory"
if not os.access(path, os.R_OK | os.X_OK):
return f"library root {path} is not readable by this process"
if require_mount and not _under_mount(path):
return (
f"library root {path} is not on a mounted filesystem in this container: "
"the library was not bind-mounted there, so PHOTO_PIPELINE_LIBRARY_ROOTS "
"names a directory of the image rather than the library"
)
return None
def iter_supported_files(root: os.PathLike | str) -> Iterator[Path]:
"""Yield supported, non-excluded files under ``root`` in deterministic order.

46
photo_pipeline/services/analysis.py
View File

@@ -18,6 +18,7 @@ from __future__ import annotations
import json
import os
import uuid
from datetime import datetime, timezone
from typing import Protocol
@@ -25,9 +26,9 @@ from sqlalchemy import func, select
from sqlalchemy.orm import sessionmaker
from photo_pipeline import path_policy
from photo_pipeline.models import AnalysisResult, Asset
from photo_pipeline.models import AnalysisResult, Asset, SafetyReview
from photo_pipeline.services import exif_checkpoint
from photo_pipeline.services.safety import SFW, latest_reviews
from photo_pipeline.services.safety import SFW
MODEL = "gemini-2.5-flash"
PROMPT_VERSION = "1"
@@ -71,26 +72,11 @@ class AnalysisService:
def _sfw_asset_ids(self, session) -> set[str]:
"""Asset ids whose latest safety decision is ``sfw`` — the ONLY assets that
may reach the provider.
The "latest row wins" rule is applied in SQL (US07-06); loading every review
to fold it in Python made the gate cost grow with the review history rather
than with the work being gated.
"""
latest = latest_reviews().subquery()
return set(
session.scalars(select(latest.c.asset_id).where(latest.c.decision == SFW))
)
def _sfw_count(self, session) -> int:
"""How many assets the gate currently allows, without listing them."""
latest = latest_reviews().subquery()
return int(
session.scalar(
select(func.count()).select_from(latest).where(latest.c.decision == SFW)
)
or 0
)
may reach the provider."""
latest: dict[str, str | None] = {}
for review in session.scalars(select(SafetyReview).order_by(SafetyReview.created_at)):
latest[review.asset_id] = review.decision
return {aid for aid, decision in latest.items() if decision == SFW}
def _is_still_sfw(self, asset_id: str) -> bool:
"""Re-read the current safety decision straight from the database."""
@@ -118,12 +104,9 @@ class AnalysisService:
)
return [a.id for a in assets if a.id not in done]
def counts(self, *, eligible: int | None = None) -> dict[str, int]:
"""Analysis progress. ``eligible`` may be passed by a caller that has just
counted confirmed-SFW assets, so the workflow home does not resolve the
latest decision of every asset twice on one page load (US07-06)."""
def counts(self) -> dict[str, int]:
with self._session_factory() as session:
eligible = self._sfw_count(session) if eligible is None else eligible
sfw = self._sfw_asset_ids(session)
rows = dict(
session.execute(
select(AnalysisResult.status, func.count()).group_by(AnalysisResult.status)
@@ -132,10 +115,10 @@ class AnalysisService:
analyzed = int(rows.get("analyzed", 0))
errored = int(rows.get("error", 0))
return {
"eligible": eligible,
"eligible": len(sfw),
"analyzed": analyzed,
"error": errored,
"pending": max(eligible - analyzed - errored, 0),
"pending": max(len(sfw) - analyzed - errored, 0),
}
def run(self, asset_ids: list[str] | None = None) -> dict:
@@ -264,11 +247,8 @@ class AnalysisService:
asset = session.get(Asset, asset_id)
if asset is not None and checkpoint.sha256:
# The bytes changed when the container was rewritten; upload must use
# the hash of what is actually on disk now (concept §3), and the
# recorded size has to move with it (US07-07).
# the hash of what is actually on disk now (concept §3).
asset.current_sha256 = checkpoint.sha256
if checkpoint.byte_size is not None:
asset.byte_size = checkpoint.byte_size
session.commit()
def get(self, asset_id: str) -> dict | None:

256
photo_pipeline/services/app_lock.py
View File

@@ -1,256 +0,0 @@
"""Library-level process lock, and detection of an incompatible legacy run
(US07-05, concept §15 "migration and operational risks").
Every safety this application has — durable job leases, rename journals, archive
manifests — assumes that one installation owns the library. Two workers, or the
frozen CLI running beside the app, break that assumption *below* the level those
mechanisms can see: the second process simply does not know the first one's
database exists.
So mutation requires a file lock in the data directory, shaped as JSON so any
future or migrated entry point can read and honour it without importing this
package:
{"lock_version": 1, "role": "worker", "pid": 4242, "host": "...",
"started_at": "...", "library_roots": ["..."]}
One holder per role: an API and a worker are designed to run together, a second
worker is not. A lock whose process is gone is stale and is taken over — refusing
to start because of a crashed predecessor would turn one outage into two.
Ownership is an advisory ``flock`` on that file, not the record inside it. The
record says *who*; the kernel says *whether*. That distinction is what makes the
lock work in containers (US08-03), where a PID and a hostname are namespaced: a
lock left behind by a container that no longer exists names a pid that still
"exists" in the new container and a host that cannot be probed, so believing the
file would deadlock every restart. A flock is released when its holder dies however
it dies, and is seen by every process that can open the file — which for a local
data directory is every container of this deployment.
Legacy detection is deliberately a heuristic, not a promise: the archived CLI has
no lock of its own, so what can be observed is its state files being written right
now. Recent writes to them mean something else is mutating this library, and every
mutating stage should refuse until it stops.
"""
from __future__ import annotations
import fcntl
import json
import os
import socket
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from photo_pipeline.config import Config
LOCK_VERSION = 1
LOCK_SUFFIX = ".lock.json"
# State files only the archived CLIs write. Their presence is history; a *recent*
# modification is a running process.
# ponytail: the real fix is a lock the migrated CLI paths take too — this catches
# the frozen archive, which has no lock and cannot be changed (US07-01).
LEGACY_ARTIFACTS = (
"photo_analyzer.db",
"nsfw_scores.csv",
"photo_analyzer_history.jsonl",
"photo_analyzer.log",
"photo_analyzer_debug.log",
)
LEGACY_ACTIVE_SECONDS = 300
class LockHeld(RuntimeError):
"""Another live process of the same role owns this library."""
def __init__(self, holder: "Holder") -> None:
super().__init__(
f"{holder.role} is already running for this library "
f"(pid {holder.pid} on {holder.host}, since {holder.started_at})"
)
self.holder = holder
class LegacyProcessActive(RuntimeError):
"""A legacy CLI appears to be mutating the same library right now."""
@dataclass(frozen=True)
class Holder:
role: str
pid: int
host: str
started_at: str
lock_version: int = LOCK_VERSION
library_roots: tuple[str, ...] = ()
@property
def alive(self) -> bool:
"""Whether the recorded process still exists on this host.
A lock from another host cannot be probed, so it is believed: assuming a
remote holder is dead is how two machines end up renaming the same folder.
"""
if self.host != socket.gethostname():
return True
try:
os.kill(self.pid, 0)
except ProcessLookupError:
return False
except PermissionError:
return True # exists, owned by someone else
return True
def as_dict(self) -> dict:
return {
"lock_version": self.lock_version,
"role": self.role,
"pid": self.pid,
"host": self.host,
"started_at": self.started_at,
"library_roots": list(self.library_roots),
"alive": self.alive,
}
def _now() -> datetime:
return datetime.now(timezone.utc)
def _in_container() -> bool:
from photo_pipeline import path_policy
return path_policy.in_container()
def legacy_activity(config: Config) -> dict:
"""Legacy state files written within the activity window, if any."""
seen: list[dict] = []
cutoff = _now().timestamp() - LEGACY_ACTIVE_SECONDS
roots = [Path(root) for root in config.library_roots] + [Path(config.data_dir)]
for root in roots:
for name in LEGACY_ARTIFACTS:
path = root / name
try:
modified = path.stat().st_mtime
except OSError:
continue
if modified >= cutoff:
seen.append(
{
"path": str(path),
"modified_at": datetime.fromtimestamp(modified, timezone.utc).isoformat(),
}
)
return {"active": bool(seen), "artifacts": seen, "window_seconds": LEGACY_ACTIVE_SECONDS}
class LibraryLock:
"""One holder per role for one library. Used as a context manager."""
def __init__(self, config: Config, role: str = "worker") -> None:
self._config = config
self.role = role
self.path = Path(config.data_dir) / f"{role}{LOCK_SUFFIX}"
self._acquired = False
self._handle = None
# ── inspection ────────────────────────────────────────────────────────────
def holder(self) -> Holder | None:
try:
payload = json.loads(self.path.read_text())
except (OSError, ValueError):
return None
try:
return Holder(
role=payload["role"],
pid=int(payload["pid"]),
host=payload["host"],
started_at=payload["started_at"],
lock_version=int(payload.get("lock_version", LOCK_VERSION)),
library_roots=tuple(payload.get("library_roots", ())),
)
except (KeyError, TypeError, ValueError):
# An unreadable lock is not an absent lock: something wrote it.
return Holder(role=self.role, pid=-1, host="unknown", started_at="unknown")
# ── acquire / release ─────────────────────────────────────────────────────
def acquire(self, *, allow_legacy: bool = False) -> Holder:
"""Take the lock for this role, or explain who has it.
Raises ``LockHeld`` when a live process of the same role owns the library,
and ``LegacyProcessActive`` when the archived CLI looks like it is running
against it.
"""
if not allow_legacy:
legacy = legacy_activity(self._config)
if legacy["active"]:
raise LegacyProcessActive(
"a legacy CLI is writing this library "
f"({', '.join(item['path'] for item in legacy['artifacts'])}); "
"stop it before running the application"
)
self.path.parent.mkdir(parents=True, exist_ok=True)
# The kernel decides, because the file cannot: a container's PID and hostname
# are namespaced, so a lock left by a container that no longer exists names a
# pid that "exists" and a host that cannot be probed (US08-03). An advisory
# flock is held by a live process or by nobody, is released when that process
# dies however it dies, and is shared by every process that can open this
# file — which, for a local data directory, is every role in every container
# of this deployment.
handle = open(self.path, "a+", encoding="utf-8")
try:
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError:
handle.close()
raise LockHeld(self.holder() or Holder(self.role, -1, "unknown", "unknown")) from None
current = self.holder()
if current is not None and current.host != socket.gethostname() and not _in_container():
# We hold the kernel's lock, so nothing on *this* machine holds the file.
# On a host that still leaves one case open: a data directory shared with
# another machine, whose flock we cannot trust. Believe its record rather
# than run two writers. In a container the data volume is local by
# construction (US08-03), and a foreign hostname is only a dead container.
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
handle.close()
raise LockHeld(current)
mine = Holder(
role=self.role,
pid=os.getpid(),
host=socket.gethostname(),
started_at=_now().isoformat(),
library_roots=tuple(str(root) for root in self._config.library_roots),
)
payload = {k: v for k, v in mine.as_dict().items() if k != "alive"}
handle.seek(0)
handle.truncate()
json.dump(payload, handle, indent=2)
handle.flush()
# Held open on purpose: closing it is what releases the lock, and that must
# happen when this process ends, not when this method returns.
self._handle = handle
self._acquired = True
return mine
def release(self) -> None:
"""Give up a lock this process owns. Another holder's lock is left alone."""
if not self._acquired:
return
self.path.unlink(missing_ok=True)
if self._handle is not None:
self._handle.close() # closing the descriptor releases the kernel lock
self._handle = None
self._acquired = False
def __enter__(self) -> "LibraryLock":
self.acquire()
return self
def __exit__(self, *_) -> None:
self.release()

413
photo_pipeline/services/backup.py
View File

@@ -1,413 +0,0 @@
"""Online backups, verification, retention, and restore drills (US07-05).
A backup taken by copying a live SQLite file is not a backup: with WAL enabled the
file on disk is missing every committed page still in the write-ahead log, and a
writer mid-transaction makes the copy inconsistent. So every backup here goes
through SQLite's online backup API, which takes a consistent snapshot of a database
that is still being used (concept §3).
A backup directory holds exactly two things:
photo_pipeline.db the snapshot
manifest.json what it is, what it came from, and how to check it
The manifest is what makes the snapshot restorable by someone who was not there
when it was taken: the schema revision, the snapshot's SHA-256, the row counts it
should still have, the archive locations whose media the library depends on, and
which configuration values were set — **names and non-secret values only**. A
secret is recorded as "configured", never as its value, so a manifest can be
attached to a bug report.
Restore never writes into a live installation: it refuses a target that already
holds a database, because the one thing worse than a lost library is a half-merged
one. The drill is documented in README ("Backup and recovery").
"""
from __future__ import annotations
import hashlib
import json
import shutil
import sqlite3
from contextlib import closing
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from sqlalchemy import text
from photo_pipeline.config import Config
from photo_pipeline.db import create_db_engine, create_session_factory
SCHEMA_VERSION = 1
DB_NAME = "photo_pipeline.db"
MANIFEST_NAME = "manifest.json"
# How many backups the retention helper keeps by default. Small on purpose: a
# backup is a snapshot of state that is itself recoverable from the library, and
# the disk it lives on is the same one the low-disk warning watches.
DEFAULT_KEEP = 7
# Tables whose row counts are worth proving after a restore. Not the whole schema —
# these are the ones whose loss would be silent.
COUNTED_TABLES = (
"assets",
"asset_paths",
"safety_reviews",
"analysis_results",
"exif_projections",
"upload_batches",
"upload_items",
"archive_locations",
"archive_plans",
"archive_operations",
"rename_plans",
"rename_operations",
)
class BackupError(RuntimeError):
"""The backup could not be created, read, verified, or restored."""
@dataclass(frozen=True)
class VerifyResult:
ok: bool
issues: tuple[str, ...] = ()
revision: str | None = None
counts: dict | None = None
def as_dict(self) -> dict:
return {
"ok": self.ok,
"issues": list(self.issues),
"revision": self.revision,
"counts": self.counts,
}
def _now() -> datetime:
return datetime.now(timezone.utc)
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _revision(database: Path) -> str | None:
with closing(sqlite3.connect(database)) as connection:
try:
row = connection.execute("SELECT version_num FROM alembic_version").fetchone()
except sqlite3.Error:
return None
return row[0] if row else None
def _counts(database: Path) -> dict:
counts: dict[str, int] = {}
with closing(sqlite3.connect(database)) as connection:
for table in COUNTED_TABLES:
try:
counts[table] = connection.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
except sqlite3.Error:
continue # a table this revision does not have yet
return counts
def _integrity(database: Path) -> tuple[str, list[str]]:
"""``PRAGMA integrity_check`` plus ``foreign_key_check`` — structure and links.
Structural soundness is not referential soundness: a database can pass
``integrity_check`` and still hold an upload item pointing at an asset that
is gone.
"""
issues: list[str] = []
with closing(sqlite3.connect(database)) as connection:
try:
result = connection.execute("PRAGMA integrity_check").fetchone()[0]
if result != "ok":
issues.append(f"integrity_check: {result}")
violations = connection.execute("PRAGMA foreign_key_check").fetchall()
if violations:
issues.append(f"foreign_key_check: {len(violations)} violation(s)")
except sqlite3.DatabaseError as error:
issues.append(f"unreadable: {error}")
return "error", issues
return "ok" if not issues else "damaged", issues
def configuration_references(config: Config) -> dict:
"""Which configuration a restore has to reproduce — never the secrets themselves.
Paths and URLs are recorded because a restore into a fresh root has to be told
where the library and the Immich server were; API keys are recorded as
``configured`` so an operator knows one is required without the manifest ever
carrying it.
"""
return {
"data_dir": str(config.data_dir),
"database_path": str(config.database_path),
"library_roots": [str(root) for root in config.library_roots],
"thumbnail_cache_dir": str(config.thumbnail_cache_dir),
"immich_server_url": config.immich_server_url,
"immich_go_binary": config.immich_go_binary,
"secrets": {
"immich_api_key": "configured" if config.immich_api_key else "unset",
"vision_api_key": "configured" if config.vision_api_key else "unset",
},
}
def migrate_with_backup(config: Config) -> dict | None:
"""Upgrade the schema, with a snapshot first when there is state to lose.
A migration is the one routine operation that can damage every record at once,
and Alembic's own transaction does not cover SQLite DDL reliably. So a pending
upgrade is preceded by an online backup, and a failed upgrade names it in the
error: recovery is "restore that directory", not "reconstruct the library".
Returns the manifest of the backup it took, or ``None`` when none was needed.
"""
import logging
from photo_pipeline.db import run_migrations
service = BackupService(config)
manifest = service.pre_migration() if service.migration_pending() else None
try:
run_migrations(config.database_url)
except Exception:
if manifest is not None:
logging.getLogger(__name__).error(
"migration failed; restore the pre-migration backup at %s",
service.root / manifest["name"],
)
raise
return manifest
class BackupService:
def __init__(self, config: Config) -> None:
self._config = config
@property
def root(self) -> Path:
return self._config.data_dir / "backups"
# ── create ────────────────────────────────────────────────────────────────
def create(self, *, reason: str = "manual", keep: int | None = DEFAULT_KEEP) -> dict:
"""Take an online snapshot and describe it. Returns the manifest."""
source = self._config.database_path
if not source.exists():
raise BackupError(f"no database at {source}")
stamp = _now().strftime("%Y%m%dT%H%M%SZ")
safe_reason = "".join(c for c in reason if c.isalnum() or c in "-_") or "manual"
directory = self.root / f"{stamp}-{safe_reason}"
if directory.exists(): # same second, same reason
directory = self.root / f"{stamp}-{safe_reason}-{len(list(self.root.iterdir()))}"
directory.mkdir(parents=True)
target = directory / DB_NAME
try:
with closing(sqlite3.connect(source)) as src, closing(sqlite3.connect(target)) as dst:
src.backup(dst) # the online backup API, not a file copy
except (sqlite3.Error, OSError) as error:
shutil.rmtree(directory, ignore_errors=True)
raise BackupError(f"backup failed: {error}") from error
state, issues = _integrity(target)
manifest = {
"schema_version": SCHEMA_VERSION,
"name": directory.name,
"created_at": _now().isoformat(),
"reason": reason,
"revision": _revision(target),
"database": {
"name": DB_NAME,
"bytes": target.stat().st_size,
"sha256": sha256_file(target),
"integrity": state,
"issues": issues,
},
"counts": _counts(target),
"archive_locations": self._archive_locations(),
"configuration": configuration_references(self._config),
"retention": {
"keep": keep,
"guidance": (
"Keep the newest snapshot on a different disk than data_dir, and one "
"off-site copy per archive medium. A backup only covers the database: "
"the photos themselves live in the library and archive locations named "
"above, which need their own copies."
),
},
}
(directory / MANIFEST_NAME).write_text(json.dumps(manifest, indent=2))
if keep is not None:
manifest["pruned"] = self.prune(keep=keep)
return manifest
def migration_pending(self) -> bool:
"""True when the database exists and is not at the revision this code wants."""
from photo_pipeline.db import current_revision, head_revision
if not self._config.database_path.exists():
return False
return current_revision(self._config.database_url) != head_revision()
def pre_migration(self) -> dict | None:
"""Snapshot before a schema change, when there is something to lose.
Returns ``None`` when the database does not exist yet (a fresh install has
no state a failed migration could damage).
"""
if not self._config.database_path.exists():
return None
return self.create(reason="pre-migration")
def _archive_locations(self) -> list[dict]:
"""The media the library's archived originals live on.
A restored database still points at these; if they are not restored too,
the pictures are gone even though every record survived.
"""
engine = create_db_engine(self._config.database_url)
try:
factory = create_session_factory(engine)
with factory() as session:
rows = session.execute(
text("SELECT id, name, root, media_id, state FROM archive_locations")
).mappings().all()
except Exception:
return []
finally:
engine.dispose()
return [
{
"id": row["id"],
"name": row["name"],
"root": row["root"],
"media_id": row["media_id"],
"last_state": row["state"],
"mounted": Path(row["root"]).is_dir(),
}
for row in rows
]
# ── inspect ───────────────────────────────────────────────────────────────
def list(self) -> list[dict]:
"""Every backup, newest first, with what is known about it."""
if not self.root.is_dir():
return []
entries = []
for directory in sorted(self.root.iterdir(), reverse=True):
if not directory.is_dir():
continue
manifest = self.manifest(directory)
database = directory / DB_NAME
entries.append(
{
"name": directory.name,
"path": str(directory),
"created_at": (manifest or {}).get("created_at"),
"reason": (manifest or {}).get("reason"),
"revision": (manifest or {}).get("revision"),
"bytes": database.stat().st_size if database.exists() else 0,
"complete": bool(manifest) and database.exists(),
}
)
return entries
def manifest(self, directory: Path) -> dict | None:
path = Path(directory) / MANIFEST_NAME
if not path.exists():
return None
try:
return json.loads(path.read_text())
except ValueError:
return None
def verify(self, directory: Path | str) -> VerifyResult:
"""Prove a snapshot is still the one that was taken and still readable."""
directory = Path(directory)
if not directory.is_dir():
return VerifyResult(False, (f"no backup at {directory}",))
manifest = self.manifest(directory)
if manifest is None:
return VerifyResult(False, ("manifest is missing or unreadable",))
database = directory / manifest["database"]["name"]
if not database.exists():
return VerifyResult(False, ("the snapshot file is missing",), manifest.get("revision"))
issues: list[str] = []
if sha256_file(database) != manifest["database"]["sha256"]:
# Bit rot, a truncated copy, or an edited snapshot: all three mean the
# bytes are not the ones that were verified when the backup was made.
issues.append("sha256 does not match the manifest")
state, structural = _integrity(database)
issues.extend(structural)
counts = _counts(database) if state != "error" else None
if counts is not None and manifest.get("counts") and counts != manifest["counts"]:
issues.append(f"row counts changed: {manifest['counts']} -> {counts}")
return VerifyResult(not issues, tuple(issues), manifest.get("revision"), counts)
# ── retention ─────────────────────────────────────────────────────────────
def prune(self, *, keep: int = DEFAULT_KEEP) -> list[str]:
"""Delete the oldest backups beyond ``keep``. Never deletes the newest one."""
if keep < 1:
raise BackupError("retention must keep at least one backup")
removed = []
for entry in self.list()[keep:]:
shutil.rmtree(entry["path"], ignore_errors=True)
removed.append(entry["name"])
return removed
# ── restore ───────────────────────────────────────────────────────────────
def restore(self, directory: Path | str, target_data_dir: Path | str) -> dict:
"""Restore a verified snapshot into a **fresh** data directory.
Refuses a target that already holds a database. Restoring on top of a live
installation would merge two histories that disagree about which files were
renamed, uploaded, and archived — the one failure this whole story exists to
prevent. Recovering in place is: stop everything, move the old data
directory aside, restore into a new one.
"""
directory = Path(directory)
result = self.verify(directory)
if not result.ok:
raise BackupError(f"refusing to restore an unverified backup: {result.issues}")
target = Path(target_data_dir)
target.mkdir(parents=True, exist_ok=True)
destination = target / DB_NAME
if destination.exists():
raise BackupError(
f"{destination} already exists; restore into a fresh data directory"
)
shutil.copy2(directory / DB_NAME, destination)
# The write-ahead log of the *source* installation must not travel with a
# snapshot: the backup API already folded every committed page into it.
for leftover in (target / f"{DB_NAME}-wal", target / f"{DB_NAME}-shm"):
leftover.unlink(missing_ok=True)
restored = _integrity(destination)
return {
"backup": directory.name,
"restored_to": str(destination),
"revision": result.revision,
"counts": _counts(destination),
"integrity": restored[0],
"issues": restored[1],
"next_steps": [
"point PHOTO_PIPELINE_DATA_DIR at the restored directory",
"run `python -m photo_pipeline migrate` to reach the current revision",
"run an inventory scan so paths are reconciled against the real library",
"mount every archive location listed in the manifest before archiving again",
],
}

502
photo_pipeline/services/benchmarks.py
View File

@@ -1,502 +0,0 @@
"""Load, soak, and resource-budget harness (US07-06, concept §17 and §18).
Performance here is not "it felt fast on my library". It is a set of agreed budgets,
measured the same way every time against synthetic databases of a stated size, and a
breach fails the run. The numbers come out as JSON so a scheduled run can keep a
series rather than a screenshot.
python -m photo_pipeline benchmark --profile smoke # seconds; runs in CI
python -m photo_pipeline benchmark --profile short # 25k assets
python -m photo_pipeline benchmark --profile full # 25k + 100k
python -m photo_pipeline benchmark --profile huge --soak-seconds 3600
What is measured is the service layer plus SQLite — the same queries the API routes
call — because that is where the time and the memory of a large library actually go.
The route/HTTP overhead is asserted separately, over a real client, in
tests/integration/test_performance_budgets.py.
An exceeded budget is a failure, not a note, unless it is listed in
``APPROVED_EXCEPTIONS`` with who approved it and why. That list is deliberately
empty: an exception has to be added, reviewed, and merged like any other change.
"""
from __future__ import annotations
import gc
import json
import os
import resource
import statistics
import sqlite3
import time
import uuid
from dataclasses import dataclass, field
from datetime import datetime, timedelta, timezone
from pathlib import Path
from sqlalchemy import func, insert, select
from photo_pipeline.config import Config
from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
from photo_pipeline.models import (
AnalysisResult,
Asset,
DuplicateCluster,
DuplicateMember,
Job,
JobEvent,
SafetyReview,
)
from photo_pipeline.services.duplicates import DuplicateService
from photo_pipeline.services.inventory import InventoryService
from photo_pipeline.services.jobs import ACTIVE_STATES, JobService
from photo_pipeline.services.library import LibraryService
from photo_pipeline.services.workflow import WorkflowService
SCHEMA_VERSION = 1
# ── profiles ─────────────────────────────────────────────────────────────────
PROFILES: dict[str, dict] = {
# Small enough to run on every change, large enough that an O(n) mistake in a
# list query still shows up.
"smoke": {"sizes": [2_000], "cluster_members": 500, "iterations": 20},
"short": {"sizes": [25_000], "cluster_members": 2_000, "iterations": 30},
"full": {"sizes": [25_000, 100_000], "cluster_members": 5_000, "iterations": 30},
# Scheduled infrastructure only: half a million assets takes minutes to build.
"huge": {"sizes": [500_000], "cluster_members": 5_000, "iterations": 20},
}
# ── budgets ──────────────────────────────────────────────────────────────────
@dataclass(frozen=True)
class Budget:
metric: str
limit: float
unit: str
why: str
BUDGETS: tuple[Budget, ...] = (
Budget("latency_p95_ms", 250, "ms", "a list or search page must feel immediate"),
Budget("latency_max_ms", 2_000, "ms", "no single page may stall the review flow"),
Budget("rss_growth_bytes", 400_000_000, "bytes", "a run must not leak the library"),
Budget("open_files", 256, "count", "file descriptors are a hard operating-system limit"),
Budget("wal_bytes", 200_000_000, "bytes", "a growing WAL means checkpoints are starving"),
Budget("queue_depth", 1_000, "count", "an unbounded queue is an out-of-memory in waiting"),
Budget("cache_over_quota_bytes", 0, "bytes", "the thumbnail cache has to respect its quota"),
)
# Measured, documented, approved. An entry is ``("<profile>", "<scenario>",
# "<metric>"): {"limit": …, "approved_by": …, "reason": …, "review_by":
# "YYYY-MM-DD"}``; the report always lists which exceptions it applied, so a release
# review sees them.
#
# The two below are the half-million-asset scale point. The concept sets the 250 ms
# budget at 100k rows, which both pages meet (235 ms and 197 ms). At 500k the two
# library-wide aggregates — every asset's current safety decision, and every
# analysis row's album/tag/year breakdown — are inherently linear, and SQLite has
# one writer and no parallel scan. Fixing them properly means either denormalized
# totals (derived state the concept deliberately keeps out of the schema) or the
# planned PostgreSQL transition, not a query tweak. Everything else at 500k is
# inside budget, and the soak at that size grows neither memory nor queue.
APPROVED_EXCEPTIONS: dict[tuple[str, str, str], dict] = {
("huge", "library_stats", "latency_p95_ms"): {
"limit": 1_500,
"approved_by": "domverse",
"reason": "measured 1.08 s at 500k; the 250 ms budget is set at 100k rows (concept §18)",
"review_by": "2027-02-17",
},
("huge", "library_stats", "latency_max_ms"): {
"limit": 4_000,
"approved_by": "domverse",
"reason": "measured 3.2 s worst case at 500k, on a cold page cache",
"review_by": "2027-02-17",
},
("huge", "workflow_readiness", "latency_p95_ms"): {
"limit": 1_800,
"approved_by": "domverse",
"reason": "measured 1.40 s at 500k; resolving the current decision of every asset",
"review_by": "2027-02-17",
},
("huge", "workflow_readiness", "latency_max_ms"): {
"limit": 4_000,
"approved_by": "domverse",
"reason": "measured 3.3 s worst case at 500k, on a cold page cache",
"review_by": "2027-02-17",
},
}
def _now() -> datetime:
return datetime.now(timezone.utc)
# ── resource sampling ────────────────────────────────────────────────────────
def rss_bytes() -> int:
"""Resident set size of this process, without a psutil dependency."""
usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
# Linux reports kilobytes, BSD/macOS bytes.
return usage if usage > 1 << 32 or os.uname().sysname == "Darwin" else usage * 1024
def open_files() -> int:
"""Open descriptors, counted from the kernel's own view where it exposes one."""
for directory in ("/proc/self/fd", "/dev/fd"):
try:
return len(os.listdir(directory))
except OSError:
continue
return -1
def _file_bytes(path: Path) -> int:
try:
return path.stat().st_size
except OSError:
return 0
def _tree_bytes(path: Path) -> int:
if not path.is_dir():
return 0
return sum(p.stat().st_size for p in path.rglob("*") if p.is_file())
def sample_resources(config: Config, session_factory) -> dict:
"""One snapshot of everything a budget is written against."""
database = config.database_path
with session_factory() as session:
queue_depth = int(
session.scalar(select(func.count()).select_from(Job).where(Job.state.in_(ACTIVE_STATES)))
or 0
)
events = int(session.scalar(select(func.count()).select_from(JobEvent)) or 0)
cache_bytes = _tree_bytes(config.thumbnail_cache_dir)
return {
"at": _now().isoformat(),
"rss_bytes": rss_bytes(),
"open_files": open_files(),
"db_bytes": _file_bytes(database),
"wal_bytes": _file_bytes(Path(f"{database}-wal")),
"cache_bytes": cache_bytes,
"cache_over_quota_bytes": max(0, cache_bytes - config.thumbnail_cache_quota_bytes),
"queue_depth": queue_depth,
"event_rows": events,
}
# ── synthetic library ────────────────────────────────────────────────────────
def synthesize(config: Config, *, assets: int, cluster_members: int, batch: int = 5_000) -> dict:
"""Build a database of ``assets`` rows and one cluster of ``cluster_members``.
Rows only — no image files. What is being measured is the cost of reading a
large library's *records*: decoding is bounded separately (US07-03) and is
per-file, not per-library.
"""
config.database_path.parent.mkdir(parents=True, exist_ok=True)
run_migrations(config.database_url)
engine = create_db_engine(config.database_url)
factory = create_session_factory(engine)
started = time.monotonic()
root = config.library_roots[0] if config.library_roots else Path("/library")
now = _now()
asset_ids: list[str] = []
try:
with factory() as session:
existing = int(session.scalar(select(func.count()).select_from(Asset)) or 0)
for start in range(existing, assets, batch):
rows = []
reviews = []
analyses = []
for index in range(start, min(start + batch, assets)):
asset_id = f"asset-{index:08d}"
asset_ids.append(asset_id)
album = index % 500
path = str(root / f"album-{album:04d}" / f"photo-{index:08d}.jpg")
rows.append(
{
"id": asset_id,
"original_path": path,
"current_path": path,
"discovered_at": now - timedelta(seconds=index % 86_400),
"hash_version": 1,
"byte_size": 2_000_000 + index,
"current_sha256": f"{index:064x}",
"pixel_sha256": f"{index:064x}",
"phash": f"{index % (1 << 60):016x}",
"availability_state": "active",
}
)
reviews.append(
{
"id": str(uuid.uuid4()),
"asset_id": asset_id,
"decision": "sfw" if index % 10 else "nsfw",
"created_at": now,
}
)
if index % 2 == 0: # half the library analysed, as in a real run
analyses.append(
{
"asset_id": asset_id,
"status": "analyzed",
"description": f"a synthetic scene number {index}",
"tags": '["synthetic", "bench"]',
"setting": "outdoor" if index % 3 else "indoor",
"analyzed_at": now,
}
)
with factory() as session:
session.execute(insert(Asset), rows)
session.execute(insert(SafetyReview), reviews)
if analyses:
session.execute(insert(AnalysisResult), analyses)
session.commit()
if cluster_members:
with factory() as session:
cluster_id = str(uuid.uuid4())
session.add(
DuplicateCluster(
id=cluster_id,
method="perceptual",
confidence="near",
state="open",
version=1,
)
)
session.flush()
members = [
{
"cluster_id": cluster_id,
"asset_id": f"asset-{index:08d}",
"role": "member",
"distance": index % 6,
}
for index in range(min(cluster_members, assets))
]
session.execute(insert(DuplicateMember), members)
session.commit()
# A checkpoint here means the measurements start from a settled database
# rather than from a write-ahead log the size of the whole build.
with sqlite3.connect(config.database_path) as connection:
connection.execute("PRAGMA wal_checkpoint(TRUNCATE)")
finally:
engine.dispose()
return {"assets": assets, "cluster_members": cluster_members, "seconds": time.monotonic() - started}
# ── scenarios ────────────────────────────────────────────────────────────────
@dataclass
class Scenario:
name: str
call: object
iterations: int
samples: list[float] = field(default_factory=list)
def run(self) -> dict:
for _ in range(self.iterations):
started = time.perf_counter()
self.call()
self.samples.append((time.perf_counter() - started) * 1000)
ordered = sorted(self.samples)
index = max(0, int(round(0.95 * len(ordered))) - 1)
return {
"scenario": self.name,
"iterations": self.iterations,
"latency_p50_ms": round(statistics.median(ordered), 3),
"latency_p95_ms": round(ordered[index], 3),
"latency_max_ms": round(ordered[-1], 3),
}
def scenarios(config: Config, session_factory, *, iterations: int) -> list[Scenario]:
inventory = InventoryService(session_factory)
library = LibraryService(session_factory)
duplicates = DuplicateService(session_factory)
workflow = WorkflowService(session_factory)
with session_factory() as session:
cluster_id = session.scalar(select(DuplicateCluster.id))
built = [
Scenario("inventory_page", lambda: inventory.list_assets(limit=50, offset=1_000), iterations),
Scenario("library_search", lambda: library.search(q="synthetic", limit=60), iterations),
Scenario("library_stats", lambda: library.stats(), iterations),
Scenario("workflow_readiness", lambda: workflow.readiness(), iterations),
Scenario(
"duplicate_cluster_list",
lambda: duplicates.list_clusters(limit=50, offset=0),
iterations,
),
]
if cluster_id:
built.append(
Scenario(
"duplicate_cluster_page",
lambda: duplicates.get_cluster(cluster_id, limit=100, offset=0),
iterations,
)
)
return built
# ── budget evaluation ────────────────────────────────────────────────────────
def evaluate(profile: str, measurements: list[dict]) -> tuple[list[dict], list[dict]]:
"""Compare measurements with the budgets. Returns ``(breaches, exceptions_used)``."""
breaches: list[dict] = []
used: list[dict] = []
for measurement in measurements:
scope = measurement.get("scenario", "resources")
for budget in BUDGETS:
if budget.metric not in measurement:
continue
value = measurement[budget.metric]
if value is None or value < 0:
continue
limit = budget.limit
exception = APPROVED_EXCEPTIONS.get((profile, scope, budget.metric))
if exception:
limit = exception["limit"]
used.append({"scope": scope, "metric": budget.metric, **exception})
if value > limit:
breaches.append(
{
"scope": scope,
"metric": budget.metric,
"value": value,
"limit": limit,
"unit": budget.unit,
"why": budget.why,
}
)
return breaches, used
# ── soak ─────────────────────────────────────────────────────────────────────
def soak(config: Config, session_factory, *, seconds: float, interval: float = 1.0) -> dict:
"""Browse, queue, cancel, and retry for a while; watch what grows.
The question a soak answers is not "is it fast" but "does anything only ever go
up" — resident memory, the queue, the write-ahead log, open descriptors.
"""
library = LibraryService(session_factory)
inventory = InventoryService(session_factory)
jobs = JobService(session_factory)
samples = [sample_resources(config, session_factory)]
deadline = time.monotonic() + seconds
last_sample = time.monotonic()
cycles = 0
while time.monotonic() < deadline:
offset = (cycles * 50) % 1_000
library.search(q="synthetic", limit=60, offset=offset)
inventory.list_assets(limit=50, offset=offset)
job = jobs.enqueue("scan", items=[f"soak-{cycles}"])
jobs.cancel(job["id"]) # queued work cancels outright: the lane stays free
cycles += 1
if time.monotonic() - last_sample >= interval:
gc.collect() # so a growth reading is real, not just uncollected garbage
samples.append(sample_resources(config, session_factory))
last_sample = time.monotonic()
samples.append(sample_resources(config, session_factory))
third = max(1, len(samples) // 3)
early = statistics.mean(sample["rss_bytes"] for sample in samples[:third])
late = statistics.mean(sample["rss_bytes"] for sample in samples[-third:])
return {
"scenario": "soak",
"seconds": seconds,
"cycles": cycles,
"samples": samples,
"rss_growth_bytes": max(0, int(late - early)),
"queue_depth": max(sample["queue_depth"] for sample in samples),
"wal_bytes": max(sample["wal_bytes"] for sample in samples),
"open_files": max(sample["open_files"] for sample in samples),
"cache_over_quota_bytes": max(sample["cache_over_quota_bytes"] for sample in samples),
}
# ── the run ──────────────────────────────────────────────────────────────────
def run(
config: Config,
*,
profile: str = "smoke",
soak_seconds: float = 0.0,
output: Path | str | None = None,
) -> dict:
"""Build, measure, evaluate. Returns the report; the caller decides the exit code."""
if profile not in PROFILES:
raise ValueError(f"unknown profile {profile!r}; try one of {sorted(PROFILES)}")
settings = PROFILES[profile]
report = {
"schema_version": SCHEMA_VERSION,
"profile": profile,
"started_at": _now().isoformat(),
"budgets": [
{"metric": b.metric, "limit": b.limit, "unit": b.unit, "why": b.why} for b in BUDGETS
],
"runs": [],
}
measurements: list[dict] = []
for size in settings["sizes"]:
sized = config.model_copy(update={"data_dir": Path(config.data_dir) / f"bench-{size}"})
before = None
build = synthesize(
sized, assets=size, cluster_members=settings["cluster_members"]
)
engine = create_db_engine(sized.database_url)
factory = create_session_factory(engine)
try:
before = sample_resources(sized, factory)
results = [
scenario.run()
for scenario in scenarios(sized, factory, iterations=settings["iterations"])
]
after = sample_resources(sized, factory)
after["scenario"] = "resources"
after["rss_growth_bytes"] = max(0, after["rss_bytes"] - before["rss_bytes"])
soaked = (
soak(sized, factory, seconds=soak_seconds) if soak_seconds > 0 else None
)
finally:
engine.dispose()
measurements.extend(results)
measurements.append(after)
if soaked:
measurements.append(soaked)
report["runs"].append(
{
"assets": size,
"build": build,
"before": before,
"scenarios": results,
"resources": after,
"soak": soaked,
}
)
breaches, exceptions_used = evaluate(profile, measurements)
report["breaches"] = breaches
report["exceptions_applied"] = exceptions_used
report["ok"] = not breaches
report["finished_at"] = _now().isoformat()
if output:
path = Path(output)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(report, indent=2))
return report

223
photo_pipeline/services/diagnostics.py
View File

@@ -1,223 +0,0 @@
"""Operational diagnostics: what the application is using, and what is about to
run out (US07-05, concept §17).
Every mutating stage in this application writes something before it is safe to
continue — a journal, an EXIF rewrite, an archive copy, a backup. All of them fail
badly on a full disk, so the sizes that grow (database, write-ahead log, thumbnail
cache, uploader reports, backups, logs) are reported separately rather than as one
opaque total, and each is compared against the free space actually left.
This is a read-only report. It never deletes, rotates, or prunes anything: what to
do about a warning is an operator's decision, and the tools for it are the
thumbnail cache quota, the backup retention helper, and log rotation outside the
application.
"""
from __future__ import annotations
import functools
import json
import shutil
from pathlib import Path
from photo_pipeline.config import Config
from photo_pipeline.integrations import exiftool, immich_go
from photo_pipeline.services import app_lock
# Below this much free space, mutating stages should stop rather than risk a
# half-written journal, EXIF container, or archive copy.
LOW_DISK_BYTES = 1_000_000_000
CRITICAL_DISK_BYTES = 200_000_000
# Written into the container image at build time (US08-02). The image pins exiftool
# and immich-go, and this file is how a running container reports which versions it
# was built with — so a drifted or missing binary is visible here rather than in a
# failed EXIF checkpoint or a misparsed upload report.
IMAGE_VERSIONS_FILE = Path("/etc/photo-pipeline/versions.json")
def _tree_bytes(path: Path) -> int:
if not path.exists():
return 0
if path.is_file():
return path.stat().st_size
total = 0
for child in path.rglob("*"):
try:
if child.is_file() and not child.is_symlink():
total += child.stat().st_size
except OSError:
continue # vanished mid-walk; it is not using space any more
return total
def _component(name: str, path: Path, *, quota: int | None = None) -> dict:
used = _tree_bytes(path)
entry = {"name": name, "path": str(path), "bytes": used, "exists": path.exists()}
if quota is not None:
entry["quota_bytes"] = quota
entry["over_quota"] = used > quota
return entry
def disk(path: Path) -> dict:
"""Free/total for the filesystem holding ``path`` — the nearest existing parent,
so a data directory that does not exist yet still reports its future disk."""
probe = path
while not probe.exists() and probe != probe.parent:
probe = probe.parent
try:
usage = shutil.disk_usage(probe)
except OSError as error:
return {"path": str(probe), "error": str(error)}
return {
"path": str(probe),
"total_bytes": usage.total,
"free_bytes": usage.free,
"used_bytes": usage.used,
}
def _pinned_versions() -> dict[str, str]:
"""The versions this image recorded at build time; empty outside a container."""
try:
recorded = json.loads(IMAGE_VERSIONS_FILE.read_text())
except (OSError, ValueError):
return {}
if not isinstance(recorded, dict):
return {}
return {str(name): str(value) for name, value in recorded.items()}
@functools.lru_cache(maxsize=4)
def _uploader_version(binary: str) -> str | None:
"""Cached: the uploader cannot change version inside one process."""
return immich_go.version(binary)
def tools(config: Config) -> list[dict]:
"""The external executables the pipeline shells out to, and their versions.
``pinned`` is what the image was built against, ``version`` is what is actually
installed. They differ only when the binary was replaced or mounted over.
"""
return [
{
"name": "exiftool",
"path": exiftool.find_binary(),
"version": exiftool.version(),
"pinned": _pinned_versions().get("exiftool"),
},
{
"name": "immich-go",
"path": immich_go.find_binary(config.immich_go_binary),
"version": _uploader_version(config.immich_go_binary),
"pinned": _pinned_versions().get("immich-go"),
},
]
def report(config: Config) -> dict:
"""Sizes, disk headroom, tool versions, warnings, and who holds the library lock."""
database = config.database_path
components = [
_component("database", database),
_component("write_ahead_log", Path(f"{database}-wal")),
_component("shared_memory", Path(f"{database}-shm")),
_component(
"thumbnail_cache",
config.thumbnail_cache_dir,
quota=config.thumbnail_cache_quota_bytes,
),
_component("upload_reports", config.data_dir / "uploads"),
_component("backups", config.data_dir / "backups"),
_component("logs", config.data_dir / "logs"),
]
space = disk(config.data_dir)
free = space.get("free_bytes")
warnings: list[dict] = []
if free is not None and free < CRITICAL_DISK_BYTES:
warnings.append(
{
"code": "disk_critical",
"message": (
f"only {free} bytes free on {space['path']}; stop mutating stages "
"and free space before renaming, writing EXIF, or archiving"
),
}
)
elif free is not None and free < LOW_DISK_BYTES:
warnings.append(
{
"code": "disk_low",
"message": f"{free} bytes free on {space['path']}; prune backups or the cache",
}
)
for component in components:
if component.get("over_quota"):
warnings.append(
{
"code": "cache_over_quota",
"message": (
f"{component['name']} uses {component['bytes']} bytes, over its "
f"{component['quota_bytes']} byte quota"
),
}
)
# A write-ahead log that outgrows its database means checkpoints are starving —
# an operational warning, not something to ignore (concept §16).
wal = next(c for c in components if c["name"] == "write_ahead_log")
db = next(c for c in components if c["name"] == "database")
if wal["bytes"] > max(db["bytes"], 1) :
warnings.append(
{
"code": "wal_growth",
"message": (
f"the write-ahead log ({wal['bytes']} bytes) is larger than the database "
f"({db['bytes']} bytes); a long-running read may be blocking checkpoints"
),
}
)
installed_tools = tools(config)
for tool in installed_tools:
# A missing tool is reported as ``version: null`` rather than warned about: on a
# development machine the uploader is legitimately absent, and the stages that
# need it already refuse to run. A *drifted* tool is different — the image pinned
# a version and something replaced it.
if tool["version"] and tool["pinned"] and tool["pinned"] not in tool["version"]:
warnings.append(
{
"code": "tool_version_drift",
"message": (
f"{tool['name']} reports {tool['version']} but this image pinned "
f"{tool['pinned']}"
),
}
)
locks = {}
for role in ("api", "worker"):
holder = app_lock.LibraryLock(config, role).holder()
locks[role] = holder.as_dict() if holder else None
legacy = app_lock.legacy_activity(config)
if legacy["active"]:
warnings.append(
{
"code": "legacy_process_active",
"message": (
"a legacy CLI is writing this library; mutating stages are refused "
"until it stops"
),
}
)
return {
"components": components,
"total_bytes": sum(component["bytes"] for component in components),
"disk": space,
"tools": installed_tools,
"warnings": warnings,
"locks": locks,
"legacy_activity": legacy,
}

164
photo_pipeline/services/duplicates.py
View File

@@ -52,13 +52,6 @@ from photo_pipeline.services import availability, hashing
NEAR_MAX = 5
SIMILAR_MAX = 10
# Member paging (US07-06). A burst or a re-imported folder can put thousands of
# assets in one cluster; review looks at a few at a time, so neither the list view
# nor the detail view may load them all.
MEMBER_PAGE = 100
MAX_MEMBER_PAGE = 500
SNAPSHOT_MEMBER_PREVIEW = 20
class Method(str, Enum):
EXACT = "exact"
@@ -527,65 +520,39 @@ class DuplicateService:
items = [self._snapshot(session, c.id) for c in rows]
return {"items": items, "total": int(total or 0), "limit": limit, "offset": offset}
def get_cluster(
self, cluster_id: str, *, limit: int = MEMBER_PAGE, offset: int = 0
) -> dict | None:
"""Cluster detail enriched with per-member asset evidence for comparison.
Members are paged and their evidence is loaded in batches (US07-06). A
cluster of a few thousand near-identical frames is a real shape for a phone
library, and the review screen only ever shows a handful at a time: loading
every member — each with its own asset, thumbnail, and location query — made
opening such a cluster cost thousands of round trips and megabytes of JSON.
"""
limit = max(1, min(limit, MAX_MEMBER_PAGE))
offset = max(0, offset)
def get_cluster(self, cluster_id: str) -> dict | None:
"""Cluster detail enriched with per-member asset evidence for comparison."""
with self._session_factory() as session:
cluster = session.get(DuplicateCluster, cluster_id)
if cluster is None:
return None
member_total = int(
session.scalar(
select(func.count())
.select_from(DuplicateMember)
.where(DuplicateMember.cluster_id == cluster_id)
)
or 0
)
rows = list(
session.execute(
select(DuplicateMember)
.where(DuplicateMember.cluster_id == cluster_id)
.order_by(DuplicateMember.asset_id)
.limit(limit)
.offset(offset)
).scalars()
)
evidence = self._member_evidence(session, [row.asset_id for row in rows])
members = []
for member in rows:
for member in session.execute(
select(DuplicateMember).where(DuplicateMember.cluster_id == cluster_id)
).scalars():
asset = session.get(Asset, member.asset_id)
try:
member_evidence = json.loads(member.evidence) if member.evidence else {}
evidence = json.loads(member.evidence) if member.evidence else {}
except json.JSONDecodeError:
member_evidence = {}
asset, offline = evidence[member.asset_id]
evidence = {}
members.append(
{
"asset_id": member.asset_id,
"role": member.role,
"distance": member.distance,
"evidence": member_evidence,
"evidence": evidence,
"current_path": asset.current_path if asset else None,
"byte_size": asset.byte_size if asset else None,
"phash": asset.phash if asset else None,
**offline,
**self._offline_evidence(session, asset),
}
)
members.sort(key=lambda m: m["asset_id"])
# A full-resolution comparison of an offline original is impossible; the
# UI asks for that named medium instead of guessing (concept §9). The
# answer covers the whole cluster, not just this page, so a mount is not
# discovered halfway through a review.
mount_required = self._mount_required(session, cluster_id)
# UI asks for that named medium instead of guessing (concept §9).
mount_required = sorted(
{m["archive_location"] for m in members if m["requires_mount"]}
)
return {
"id": cluster.id,
"method": cluster.method,
@@ -597,74 +564,10 @@ class DuplicateService:
"requires_confirmation": cluster.method == Method.PERCEPTUAL.value,
"mount_required": mount_required,
"members": members,
"member_total": member_total,
"limit": limit,
"offset": offset,
}
def _member_evidence(self, session, asset_ids: list[str]) -> dict:
"""``{asset_id: (asset, offline_evidence)}`` for one page, in three queries."""
if not asset_ids:
return {}
assets = {
asset.id: asset
for asset in session.execute(
select(Asset).where(Asset.id.in_(asset_ids))
).scalars()
}
previews: dict[str, list] = {}
for thumbnail in session.execute(
select(Thumbnail).where(Thumbnail.asset_id.in_(asset_ids))
).scalars():
previews.setdefault(thumbnail.asset_id, []).append(thumbnail)
location_ids = {
asset.archive_location_id for asset in assets.values() if asset.archive_location_id
}
locations = (
{
location.id: location
for location in session.execute(
select(ArchiveLocation).where(ArchiveLocation.id.in_(location_ids))
).scalars()
}
if location_ids
else {}
)
return {
asset_id: (
assets.get(asset_id),
self._offline_evidence(
assets.get(asset_id),
locations=locations,
thumbnails=previews.get(asset_id, []),
),
)
for asset_id in asset_ids
}
def _mount_required(self, session, cluster_id: str) -> list[str]:
"""Archive media whose originals this cluster needs, across every member."""
rows = session.execute(
select(ArchiveLocation.name)
.select_from(DuplicateMember)
.join(Asset, Asset.id == DuplicateMember.asset_id)
.join(ArchiveLocation, ArchiveLocation.id == Asset.archive_location_id)
.where(
DuplicateMember.cluster_id == cluster_id,
Asset.availability_state == availability.ARCHIVED_OFFLINE,
)
.distinct()
).scalars()
return sorted(rows)
def _offline_evidence(
self, asset: Asset | None, *, locations: dict, thumbnails: list
) -> dict:
"""What review can still rely on when a member's original is not readable.
Takes the already-loaded locations and thumbnails for its page rather than
querying per member (US07-06).
"""
def _offline_evidence(self, session, asset: Asset | None) -> dict:
"""What review can still rely on when a member's original is not readable."""
if asset is None:
return {
"availability_state": None,
@@ -674,8 +577,12 @@ class DuplicateService:
"preview": {"state": "missing", "protected": False},
"requires_mount": False,
}
location = locations.get(asset.archive_location_id)
preview = self._preview_evidence(thumbnails)
location = (
session.get(ArchiveLocation, asset.archive_location_id)
if asset.archive_location_id
else None
)
preview = self._preview_evidence(session, asset.id)
archived = asset.availability_state in availability.ARCHIVED
return {
"availability_state": asset.availability_state,
@@ -691,7 +598,10 @@ class DuplicateService:
}
@staticmethod
def _preview_evidence(rows: list) -> dict:
def _preview_evidence(session, asset_id: str) -> dict:
rows = list(
session.execute(select(Thumbnail).where(Thumbnail.asset_id == asset_id)).scalars()
)
ready = [r for r in rows if r.state == "ready" and r.path]
if ready:
best = max(ready, key=lambda r: (bool(r.protected), r.size or 0))
@@ -783,29 +693,11 @@ class DuplicateService:
session.delete(link)
def _snapshot(self, session, cluster_id) -> dict:
"""A cluster and a *bounded* preview of its members.
The list view shows a count and a few ids; a snapshot that loaded every
member turned one page of 200 clusters into hundreds of thousands of rows
(US07-06). ``member_total`` is the honest count either way.
"""
cluster = session.get(DuplicateCluster, cluster_id)
member_total = int(
session.scalar(
select(func.count())
.select_from(DuplicateMember)
.where(DuplicateMember.cluster_id == cluster_id)
)
or 0
)
members = session.execute(
select(DuplicateMember)
.where(DuplicateMember.cluster_id == cluster_id)
.order_by(DuplicateMember.asset_id)
.limit(SNAPSHOT_MEMBER_PREVIEW)
select(DuplicateMember).where(DuplicateMember.cluster_id == cluster_id)
).scalars()
return {
"member_total": member_total,
"id": cluster.id,
"method": cluster.method,
"confidence": cluster.confidence,

13
photo_pipeline/services/exif_checkpoint.py
View File

@@ -21,7 +21,6 @@ unreadable file, or a write that did not take is not evidence that metadata is f
from __future__ import annotations
import json
import os
import uuid
from dataclasses import dataclass
from datetime import datetime, timezone
@@ -59,11 +58,6 @@ class CheckpointResult:
state: str # verified | divergent | failed
changed_fields: tuple[str, ...] = ()
sha256: str | None = None
# exiftool rewrites the container, so the file's size moves with its hash. Both
# are inventory facts about the current bytes and both have to be refreshed
# together, or the next stage compares against a size that no longer exists
# (US07-07: a rename plan blocked itself forever after any EXIF write).
byte_size: int | None = None
verified_at: datetime | None = None
reason: str | None = None
@@ -143,12 +137,9 @@ def run(
changed = compare(before, after)
sha256 = hashing.sha256_file(path)
byte_size = os.path.getsize(path)
if changed:
return CheckpointResult(
DIVERGENT, changed_fields=changed, sha256=sha256, byte_size=byte_size
)
return CheckpointResult(VERIFIED, sha256=sha256, byte_size=byte_size, verified_at=_now())
return CheckpointResult(DIVERGENT, changed_fields=changed, sha256=sha256)
return CheckpointResult(VERIFIED, sha256=sha256, verified_at=_now())
def record(

84
photo_pipeline/services/library.py
View File

@@ -15,9 +15,10 @@ from __future__ import annotations
import json
import re
from collections import Counter
from pathlib import Path
from sqlalchemy import String, and_, case, cast, func, or_, select, text
from sqlalchemy import and_, func, or_, select
from sqlalchemy.orm import sessionmaker
from photo_pipeline.models import AnalysisResult, Asset
@@ -70,75 +71,38 @@ class LibraryService:
return {"rows": rows, "total": total, "offset": offset, "limit": limit}
def stats(self) -> dict:
"""Library-wide totals, aggregated in SQL (US07-06).
This page used to load every analysis row — object, tags, and all — to count
them in Python, which cost half a second at 100k assets and grew from there.
Only the album breakdown still walks rows, and only their path and status:
SQLite has no ``dirname``, and two short strings per asset is cheap.
"""
with self._session_factory() as session:
status = dict(
session.execute(
select(AnalysisResult.status, func.count()).group_by(AnalysisResult.status)
).all()
)
albums: dict[str, dict] = {}
for path, row_status in session.execute(
select(Asset.current_path, AnalysisResult.status).join(
Asset, Asset.id == AnalysisResult.asset_id
rows = list(
session.execute(
select(AnalysisResult, Asset.current_path).join(
Asset, Asset.id == AnalysisResult.asset_id
)
)
):
)
albums: dict[str, dict] = {}
tag_counts: Counter = Counter()
year_counts: Counter = Counter()
people: Counter = Counter()
errors = []
for result, path in rows:
album = _album_of(path)
bucket = albums.setdefault(album, {"album": album, "done": 0, "total": 0})
bucket["total"] += 1
if row_status in DONE:
if result.status in DONE:
bucket["done"] += 1
year_counts = dict(
session.execute(
select(AnalysisResult.approx_year, func.count())
.where(AnalysisResult.approx_year.is_not(None))
.group_by(AnalysisResult.approx_year)
).all()
)
people = dict(
session.execute(
select(
case(
(AnalysisResult.people_count >= 3, "3+"),
else_=cast(AnalysisResult.people_count, String),
),
func.count(),
)
.where(AnalysisResult.people_count.is_not(None))
.group_by(
case(
(AnalysisResult.people_count >= 3, "3+"),
else_=cast(AnalysisResult.people_count, String),
)
)
).all()
)
# SQLite's JSON1 counts the tag arrays where they are: parsing 50k JSON
# strings in Python to keep the top 40 is the definition of doing work
# the database already does. Malformed tags are skipped, not fatal.
tag_counts = session.execute(
text(
"SELECT tag.value AS value, count(*) AS total "
"FROM analysis_results, json_each(analysis_results.tags) AS tag "
"WHERE analysis_results.tags IS NOT NULL "
"AND json_valid(analysis_results.tags) "
"GROUP BY tag.value ORDER BY total DESC, value LIMIT 40"
)
).all()
errors = [
{"path": path, "error": message}
for path, message in session.execute(
select(Asset.current_path, AnalysisResult.error_message)
.join(Asset, Asset.id == AnalysisResult.asset_id)
.where(AnalysisResult.status == "error")
)
]
for tag in _tags(result.tags):
tag_counts[tag] += 1
if result.approx_year is not None:
year_counts[result.approx_year] += 1
if result.people_count is not None:
people["3+" if result.people_count >= 3 else str(result.people_count)] += 1
if result.status == "error":
errors.append({"path": path, "error": result.error_message})
return {
"total": sum(status.values()),
"status": status,
@@ -147,7 +111,7 @@ class LibraryService:
"season": self._facet("season"),
"people": [{"value": v, "count": n} for v, n in sorted(people.items())],
"years": [{"value": y, "count": year_counts[y]} for y in sorted(year_counts)],
"top_tags": [{"value": t, "count": n} for t, n in tag_counts],
"top_tags": [{"value": t, "count": n} for t, n in tag_counts.most_common(40)],
"albums": sorted(albums.values(), key=lambda d: d["album"]),
"errors": sorted(errors, key=lambda e: e["path"] or ""),
}

426
photo_pipeline/services/release.py
View File

@@ -1,426 +0,0 @@
"""The release gate, the real-library dry run, and the approval that unlocks
mutation (US07-07, concept §18 release gates).
Three things live here because they are one decision:
1. **The gate** — one command that provisions an isolated stack, runs every suite in
a fixed order, and retains versioned evidence with checksums. A release is not
"the tests passed on my machine last Tuesday"; it is a report that says which
revision, which suites, how long, and what the artefacts hash to.
2. **The dry run** — a strictly read-only pass over the real photo library that
answers "what would this application do to it?" before it is allowed to do
anything. It opens no file for writing, creates no database rows, and touches no
metadata; it counts, classifies, and reconciles against whatever the database
already knows.
3. **The approval** — a person reads that report and signs it off for exactly the
library roots it describes. Until then, with
``PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL`` set, every mutating request is
refused. Change the roots, or produce a newer report, and the approval no longer
matches: it approves *that* reconciliation, not the idea of mutating.
"""
from __future__ import annotations
import hashlib
import json
import os
import subprocess
import sys
import time
from collections import Counter
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from photo_pipeline import path_policy
from photo_pipeline.config import Config
SCHEMA_VERSION = 1
APPROVAL_NAME = "dry-run-approval.json"
CHECKSUMS_NAME = "CHECKSUMS.sha256"
REPORT_NAME = "release-report.json"
# The suites, in the order a failure is cheapest to read: units before the stacks
# they compose. ``label`` is what the report and the operator see.
STAGES: tuple[tuple[str, tuple[str, ...]], ...] = (
("unit", ("tests/unit",)),
("characterization", ("tests/characterization",)),
("integration", ("tests/integration",)),
("browser", ("tests/e2e",)),
)
# Skips the gate accepts, because they describe the machine rather than the code. The
# container ones (US08-02) belong here for the same reason exiftool does: the image
# build needs a Docker daemon and the network, and its definition is still checked
# offline in tests/integration/test_container_image.py.
ALLOWED_SKIP_REASONS = (
"exiftool not installed",
"root ignores directory permissions",
"no Docker daemon available",
"bind-mount ownership is virtualised",
)
# The container acceptance gate (US08-05): the deployed application, verified the way
# the host application is. Deliberately one stage selected by marker, so adding a
# phase_h test is enough to put it in front of a deploy.
CONTAINER_STAGES: tuple[tuple[str, tuple[str, ...]], ...] = (
("container", ("tests/e2e", "-m", "phase_h")),
)
# Nothing. This gate exists to prove the *deployed* runtime, and every reason a check
# would skip here — no daemon, no compose, no browser — means it was not proven.
CONTAINER_ALLOWED_SKIP_REASONS: tuple[str, ...] = ()
# The documentation gate (US09-05): the manuals, held to the application. Selected by
# marker across the whole suite, because each check lives beside what it describes —
# the offline contracts under tests/integration, the browser and screenshot journeys
# under tests/e2e. Adding a phase_i test is enough to put it in front of a release.
DOCS_STAGES: tuple[tuple[str, tuple[str, ...]], ...] = (
("documentation", ("tests", "-m", "phase_i")),
)
# Also nothing. Every check here is either offline or needs the browser the rest of
# the suite already needs, so a skip means the manuals were not compared with the code.
DOCS_ALLOWED_SKIP_REASONS: tuple[str, ...] = ()
class ReleaseError(RuntimeError):
pass
def _now() -> datetime:
return datetime.now(timezone.utc)
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def sha256_bytes(payload: bytes) -> str:
return hashlib.sha256(payload).hexdigest()
def revision() -> str | None:
"""The commit this gate ran against, when the tree is a git checkout."""
try:
result = subprocess.run(
["git", "rev-parse", "HEAD"],
capture_output=True,
text=True,
timeout=10,
cwd=str(Path(__file__).resolve().parents[2]),
)
except (OSError, subprocess.SubprocessError):
return None
return result.stdout.strip() or None
# ── the story matrix ─────────────────────────────────────────────────────────
def story_matrix(repo: Path | None = None) -> dict:
"""Every backlog story, and how it is covered.
A story is ``delivered`` (mapped to test files that exist) or ``planned`` (an
accepted, not-yet-implemented story). Anything else — a story file nobody
mapped, or a mapping to a file that is gone — is a hole in the matrix, and the
gate fails on it rather than reporting a green run over missing coverage.
"""
repo = repo or Path(__file__).resolve().parents[2]
traceability = json.loads((repo / "tests" / "story_traceability.json").read_text())
mapped: dict[str, list[str]] = traceability["stories"]
planned: list[str] = traceability.get("planned", [])
stories = sorted(
"-".join(path.stem.split("-")[:2])
for path in (repo / "delivery_backlog" / "stories").glob("US*.md")
)
missing_tests = [
f"{story}: {rel}"
for story, files in mapped.items()
for rel in files
if not (repo / rel).is_file()
]
unmapped = [s for s in stories if s not in mapped and s not in planned]
unknown = [s for s in list(mapped) + planned if s not in stories]
overlap = sorted(set(mapped) & set(planned))
return {
"stories": len(stories),
"delivered": sorted(mapped),
"planned": sorted(planned),
"problems": [
*(f"story with no tests and not planned: {s}" for s in unmapped),
*(f"mapped test file is missing — {entry}" for entry in missing_tests),
*(f"mapped story is not in the backlog: {s}" for s in unknown),
*(f"story is both delivered and planned: {s}" for s in overlap),
],
}
# ── the gate ─────────────────────────────────────────────────────────────────
@dataclass
class StageResult:
label: str
command: list[str]
returncode: int
seconds: float
summary: str
skipped: list[str]
def as_dict(self) -> dict:
return {
"stage": self.label,
"command": self.command,
"returncode": self.returncode,
"seconds": round(self.seconds, 2),
"summary": self.summary,
"skipped": self.skipped,
"ok": self.returncode == 0,
}
def _run_stage(label: str, paths: tuple[str, ...], *, repo: Path, log_dir: Path) -> StageResult:
command = [sys.executable, "-m", "pytest", *paths, "-q", "-rs"]
started = time.monotonic()
result = subprocess.run(command, cwd=str(repo), capture_output=True, text=True)
elapsed = time.monotonic() - started
output = result.stdout + result.stderr
(log_dir / f"{label}.log").write_text(output)
lines = [line for line in output.splitlines() if line.strip()]
summary = lines[-1] if lines else ""
skipped = [line for line in lines if line.startswith("SKIPPED")]
return StageResult(label, command, result.returncode, elapsed, summary, skipped)
def unexpected_skips(
results: list[StageResult], allowed: tuple[str, ...] = ALLOWED_SKIP_REASONS
) -> list[str]:
"""Skips the gate will not accept: everything but the documented environment ones."""
return [
line
for result in results
for line in result.skipped
if not any(reason in line for reason in allowed)
]
def run_gate(
config: Config,
*,
output: Path | str | None = None,
stages: tuple[tuple[str, tuple[str, ...]], ...] = STAGES,
allowed_skips: tuple[str, ...] = ALLOWED_SKIP_REASONS,
repo: Path | None = None,
) -> dict:
"""Run every suite in an isolated stack and retain checksummed evidence.
The stack is isolated by construction: each pytest run builds its own temporary
data directories and libraries, so the gate never reads or writes the operator's
photos. What it keeps afterwards is the report, the per-stage logs, and a
checksum file over both.
"""
repo = repo or Path(__file__).resolve().parents[2]
directory = Path(output) if output else Path(config.data_dir) / "release" / _now().strftime(
"%Y%m%dT%H%M%SZ"
)
logs = directory / "logs"
logs.mkdir(parents=True, exist_ok=True)
started_at = _now()
matrix = story_matrix(repo)
results = [_run_stage(label, paths, repo=repo, log_dir=logs) for label, paths in stages]
skips = unexpected_skips(results, allowed_skips)
report = {
"schema_version": SCHEMA_VERSION,
"started_at": started_at.isoformat(),
"revision": revision(),
"python": sys.version.split()[0],
"platform": os.uname().sysname,
"matrix": matrix,
"stages": [result.as_dict() for result in results],
"unexpected_skips": skips,
"failures": [result.label for result in results if result.returncode != 0],
}
report["ok"] = not report["failures"] and not matrix["problems"] and not skips
report["finished_at"] = _now().isoformat()
(directory / REPORT_NAME).write_text(json.dumps(report, indent=2))
# The evidence is only evidence if it can be shown to be the evidence that was
# produced. Checksums are the honest version of "signed" without a key: a real
# signature belongs to whatever key management the release actually has.
checksums = "\n".join(
f"{sha256_file(path)} {path.relative_to(directory)}"
for path in sorted(directory.rglob("*"))
if path.is_file() and path.name != CHECKSUMS_NAME
)
(directory / CHECKSUMS_NAME).write_text(checksums + "\n")
report["evidence"] = str(directory)
return report
# ── the real-library dry run ─────────────────────────────────────────────────
def dry_run(config: Config, *, roots: tuple[Path, ...] | None = None) -> dict:
"""Read-only reconciliation of the configured library. Changes nothing.
Opens no file for writing, writes no database row, and reads only what
``os.stat`` and the existing database already say. The point is to be able to
look at a real library — the one with the irreplaceable photos in it — and see
what the application believes about it before it is allowed to act.
"""
roots = roots or tuple(Path(root) for root in config.library_roots)
if not roots:
raise ReleaseError("no library roots are configured")
by_extension: Counter = Counter()
folders: set[str] = set()
files: list[str] = []
unreadable: list[str] = []
excluded = 0
total_bytes = 0
for root in roots:
if not Path(root).is_dir():
raise ReleaseError(f"library root {root} is not a directory")
for path in sorted(Path(root).rglob("*")):
if path.is_dir():
# Never traverse into an excluded directory, and never report its
# contents: proving exclusion must not require opening it.
if path_policy.is_excluded(path):
excluded += 1
continue
if path_policy.is_excluded(path):
continue
try:
stat = path.stat()
except OSError:
unreadable.append(str(path))
continue
files.append(str(path))
folders.add(str(path.parent))
by_extension[path.suffix.lower() or "(none)"] += 1
total_bytes += stat.st_size
known = _known_paths(config)
on_disk = set(files)
report = {
"schema_version": SCHEMA_VERSION,
"generated_at": _now().isoformat(),
"revision": revision(),
"library_roots": [str(root) for root in roots],
"files": len(files),
"folders": len(folders),
"bytes": total_bytes,
"excluded_directories": excluded,
"unreadable": unreadable,
"by_extension": dict(sorted(by_extension.items())),
"reconciliation": {
"known_to_database": len(known),
"already_registered": len(on_disk & known),
"new_to_the_application": len(on_disk - known),
"recorded_but_absent": sorted(known - on_disk)[:100],
"recorded_but_absent_total": len(known - on_disk),
},
"mutation": "none — this pass is read-only",
}
report["checksum"] = sha256_bytes(
json.dumps(report, sort_keys=True).encode("utf-8")
)
return report
def _known_paths(config: Config) -> set[str]:
"""Current asset paths the database holds, or an empty set if there is none."""
if not config.database_path.exists():
return set()
from sqlalchemy import select
from photo_pipeline.db import create_db_engine, create_session_factory
from photo_pipeline.models import Asset
engine = create_db_engine(config.database_url)
try:
with create_session_factory(engine)() as session:
return {
path
for path in session.scalars(select(Asset.current_path))
if path is not None
}
except Exception:
return set()
finally:
engine.dispose()
# ── the approval ─────────────────────────────────────────────────────────────
def approval_path(config: Config) -> Path:
return Path(config.data_dir) / APPROVAL_NAME
def approve(config: Config, report: dict | Path | str, *, approver: str) -> dict:
"""Record that a person read this reconciliation and accepts mutation for it."""
if isinstance(report, (str, Path)):
report = json.loads(Path(report).read_text())
if "checksum" not in report:
raise ReleaseError("this is not a dry-run report: it has no checksum")
record = {
"schema_version": SCHEMA_VERSION,
"approved_at": _now().isoformat(),
"approved_by": approver,
"report_checksum": report["checksum"],
"library_roots": report["library_roots"],
"files": report["files"],
"revision": report.get("revision"),
}
path = approval_path(config)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(record, indent=2))
return record
def mutation_blockers(config: Config) -> list[dict]:
"""Why mutation must stay refused, or an empty list.
Only enforced when ``require_dry_run_approval`` is configured — the loopback
developer setup keeps working unchanged, and an operator turns this on before
pointing the application at the library they cannot replace.
"""
if not config.require_dry_run_approval:
return []
path = approval_path(config)
if not path.exists():
return [
{
"code": "dry_run_not_approved",
"message": (
"run `python -m photo_pipeline dry-run` and approve its report "
"before mutation is enabled"
),
}
]
try:
record = json.loads(path.read_text())
except ValueError:
return [{"code": "approval_unreadable", "message": f"{path} is not readable JSON"}]
approved_roots = [str(root) for root in record.get("library_roots", [])]
configured = [str(root) for root in config.library_roots]
if sorted(approved_roots) != sorted(configured):
return [
{
"code": "approval_scope_mismatch",
"message": (
f"the approval covers {approved_roots}, but the configured library "
f"is {configured}; run a new dry run"
),
}
]
return []

171
photo_pipeline/services/safety.py
View File

@@ -105,7 +105,7 @@ def exif_projection(decision: str) -> dict[str, list[str]]:
import uuid
from datetime import datetime, timezone
from sqlalchemy import column, func, select
from sqlalchemy import select
from sqlalchemy.orm import sessionmaker
from photo_pipeline.models import Asset, ExifProjection, SafetyReview
@@ -139,8 +139,8 @@ class SafetyService:
# -- reads ----------------------------------------------------------------
def _latest_by_asset(self, session) -> dict[str, SafetyReview]:
"""The current review per asset, as ORM rows. Only for small, known sets —
every library-wide caller uses ``latest_reviews()`` in SQL instead."""
# Latest row per asset. Small local scale: order ascending, let later rows
# overwrite. ponytail: a windowed query if safety_reviews ever grows huge.
latest: dict[str, SafetyReview] = {}
for review in session.scalars(select(SafetyReview).order_by(SafetyReview.created_at)):
latest[review.asset_id] = review
@@ -148,99 +148,58 @@ class SafetyService:
def current_decision(self, asset_id: str) -> str | None:
with self._session_factory() as session:
latest = latest_reviews().subquery()
return session.scalar(
select(latest.c.decision).where(latest.c.asset_id == asset_id)
)
review = self._latest_by_asset(session).get(asset_id)
return review.decision if review else None
def counts(self) -> dict[str, int]:
"""Decision breakdown over canonical, active assets — the workflow totals.
Aggregated in SQL: the workflow home asks for this on every load, and
materialising every asset and every review to count them cost hundreds of
milliseconds at 25k assets and would scale linearly from there (US07-06).
"""
latest = latest_reviews().subquery()
"""Decision breakdown over canonical, active assets — the workflow totals."""
with self._session_factory() as session:
rows = session.execute(
select(
func.coalesce(latest.c.decision, "undecided"),
func.count(),
func.count(latest.c.score),
)
.select_from(Asset)
.join(latest, latest.c.asset_id == Asset.id, isouter=True)
.where(
Asset.canonical_asset_id.is_(None),
Asset.availability_state == "active",
)
.group_by(func.coalesce(latest.c.decision, "undecided"))
).all()
out = {SFW: 0, NSFW: 0, "deferred": 0, "undecided": 0, "scored": 0}
for decision, total, scored in rows:
if decision in (SFW, NSFW, "deferred"):
out[decision] += int(total)
else:
# Anything that is not one of the three decisions is undecided —
# including a score-only review, which is what "scored" counts.
out["undecided"] += int(total)
out["scored"] += int(scored)
return out
assets = list(session.scalars(_eligible_assets_query()))
latest = self._latest_by_asset(session)
out = {SFW: 0, NSFW: 0, "deferred": 0, "undecided": 0, "scored": 0}
for asset in assets:
review = latest.get(asset.id)
decision = review.decision if review else None
if decision in (SFW, NSFW, "deferred"):
out[decision] += 1
else:
out["undecided"] += 1
if review and review.score is not None:
out["scored"] += 1
return out
def review_queue(self, state: str = "", limit: int = 100, offset: int = 0) -> dict:
"""Assets for the review UI, filtered by ``state`` (undecided/sfw/nsfw/deferred).
Filtered, counted, and paged in SQL (US07-06): the queue for a large library
is thousands of rows and the reviewer sees one page of it.
"""
latest = latest_reviews().subquery()
projections = (
select(ExifProjection.asset_id, ExifProjection.state.label("exif_state"))
.where(ExifProjection.stage == "safety")
.subquery()
)
effective = func.coalesce(latest.c.decision, "undecided")
query = (
select(
Asset.id,
Asset.current_path,
latest.c.score,
latest.c.decision,
latest.c.exif_verified_at,
projections.c.exif_state,
)
.select_from(Asset)
.join(latest, latest.c.asset_id == Asset.id, isouter=True)
.join(projections, projections.c.asset_id == Asset.id, isouter=True)
.where(
Asset.canonical_asset_id.is_(None),
Asset.availability_state == "active",
)
)
if state:
query = query.where(effective == state)
"""Assets for the review UI, filtered by ``state`` (undecided/sfw/nsfw/deferred)."""
with self._session_factory() as session:
total = int(
session.scalar(select(func.count()).select_from(query.subquery())) or 0
)
rows = session.execute(
query.order_by(Asset.current_path).limit(limit).offset(offset)
).all()
return {
"total": total,
"items": [
{
"asset_id": asset_id,
"current_path": current_path,
"score": score,
"decision": decision,
"suggested": classify(score) if score is not None else None,
"exif_verified": bool(exif_verified_at),
"exif_state": exif_state,
}
for asset_id, current_path, score, decision, exif_verified_at, exif_state in rows
],
}
assets = list(session.scalars(_eligible_assets_query().order_by(Asset.current_path)))
latest = self._latest_by_asset(session)
# One query, not one per asset: the reviewer needs to see a divergent
# checkpoint, which is neither "verified" nor a plain failure (US07-03).
projections = {
row.asset_id: row.state
for row in session.scalars(
select(ExifProjection).where(ExifProjection.stage == "safety")
)
}
rows = []
for asset in assets:
review = latest.get(asset.id)
decision = review.decision if review else None
effective = decision or "undecided"
if state and state != effective:
continue
rows.append(
{
"asset_id": asset.id,
"current_path": asset.current_path,
"score": review.score if review else None,
"decision": decision,
"suggested": classify(review.score) if review and review.score is not None else None,
"exif_verified": bool(review and review.exif_verified_at),
"exif_state": projections.get(asset.id),
}
)
return {"total": len(rows), "items": rows[offset : offset + limit]}
def scorable_asset_ids(self) -> list[str]:
"""Canonical active assets with a path — the items a scoring job enqueues."""
@@ -294,7 +253,6 @@ class SafetyService:
exif_verified_at = None
result_sha256 = None
result_byte_size = None
if write_exif and decision in (SFW, NSFW) and path:
ops = exif_projection(decision)
# The full checkpoint: write the owned keyword, read the whole file back,
@@ -315,7 +273,6 @@ class SafetyService:
if result.verified:
exif_verified_at = result.verified_at
result_sha256 = result.sha256
result_byte_size = result.byte_size
now = _now()
with self._session_factory() as session:
@@ -334,8 +291,6 @@ class SafetyService:
if result_sha256:
asset = session.get(Asset, asset_id)
asset.current_sha256 = result_sha256
if result_byte_size is not None:
asset.byte_size = result_byte_size
session.commit()
return {
"asset_id": asset_id,
@@ -345,34 +300,6 @@ class SafetyService:
}
def latest_reviews():
"""One row per asset: its current safety review, chosen in SQL.
``safety_reviews`` is append-only, so "the decision" is the newest row for an
asset. A window function picks it without loading the table; ``rowid`` breaks a
same-timestamp tie the same way the previous last-write-wins loop did.
"""
ranked = (
select(
SafetyReview.asset_id,
SafetyReview.decision,
SafetyReview.score,
SafetyReview.exif_verified_at,
func.row_number()
.over(
partition_by=SafetyReview.asset_id,
order_by=(SafetyReview.created_at.desc(), column("rowid").desc()),
)
.label("rank"),
)
.select_from(SafetyReview)
.subquery()
)
return select(
ranked.c.asset_id, ranked.c.decision, ranked.c.score, ranked.c.exif_verified_at
).where(ranked.c.rank == 1)
def _eligible_assets_query():
"""Canonical, active assets — the safety stage runs only on these.

4
photo_pipeline/services/workflow.py
View File

@@ -47,9 +47,7 @@ class WorkflowService:
active = self._active_job(session)
safety = SafetyService(self._session_factory).counts()
# Reuse the confirmed-SFW total just computed: resolving the current decision
# of every asset is the expensive part of this page (US07-06).
analysis = AnalysisService(self._session_factory).counts(eligible=safety[_SFW])
analysis = AnalysisService(self._session_factory).counts()
undecided_clusters = cluster_states.get("open", 0) + cluster_states.get("reopened", 0)
stages = [

26
pyproject.toml
View File

@@ -8,35 +8,18 @@ dependencies = [
"sqlalchemy>=2.0",
"alembic>=1.13",
"pydantic>=2.7",
# Imaging is runtime, not test-only: thumbnails decode through Pillow, and the
# perceptual hash is a DCT over the decoded pixels (services/hashing.py).
"pillow>=10",
"numpy>=1.26",
"scipy>=1.11",
]
[project.optional-dependencies]
# The cloud vision provider. Optional because the analysis stage is the only thing
# that needs it, and a local review-only install should not pull an API client.
vision = ["openai>=1.30"]
test = [
"pytest>=8",
"httpx>=0.27",
"pillow>=10",
"numpy>=1.26",
"scipy>=1.11",
"playwright>=1.40",
"pytest-playwright>=0.4",
# The composition and the workflows are configuration, and the tests that hold
# them to their promises read them (US08-03, US08-04).
"pyyaml>=6",
]
[build-system]
requires = ["setuptools>=68"]
build-backend = "setuptools.build_meta"
[tool.setuptools]
# The importable application. ``migrations`` and ``work_item`` live beside it but
# are not part of the package; without this, an editable install cannot guess.
packages = ["photo_pipeline"]
# Browser end-to-end tests also require: python -m playwright install chromium
[tool.ruff]
@@ -52,7 +35,4 @@ markers = [
"phase_d: Phase D end-to-end acceptance (US04-06) — guarded rename API, fault, and browser journeys",
"phase_e: Phase E end-to-end acceptance (US05-06) — upload preflight, uploader, and browser journeys",
"phase_f: Phase F end-to-end acceptance (US06-06) — archive destination, transfer, and restore journeys",
"container: builds and runs the container image and its composition (US08-02, US08-03) — needs a Docker daemon, the compose plugin, and the network",
"phase_h: Phase H container deployment acceptance (US08-05) — the browser, upgrade, restart, and security journeys against the composed stack",
"phase_i: Phase I documentation acceptance (US09-05) — the manuals checked against the application, in the browser and offline",
]

260
tests/e2e/_container_harness.py
View File

@@ -1,260 +0,0 @@
"""The composed stack, as a fixture: build, provision, drive, destroy (US08-03/US08-05).
Extracted from ``tests/e2e/test_compose_stack.py`` when the container acceptance gate
needed the same stack under a different image, project, and library. One
implementation, because two would drift on exactly the details that make a container
test worth anything — the mount, the volume, the ports, and the teardown.
Nothing here fakes anything below the process boundary: Docker builds the image,
Compose starts the real containers, and every helper talks to them over HTTP or the
Docker CLI.
"""
from __future__ import annotations
import os
import shutil
import socket
import subprocess
import tempfile
import time
from pathlib import Path
import httpx
import pytest
from PIL import Image
REPO = Path(__file__).resolve().parents[2]
COMPOSE_FILE = REPO / "docker-compose.yml"
CONTAINER_LIBRARY = "/library"
READY_TIMEOUT_SECONDS = 180
JOB_TIMEOUT_SECONDS = 300
UP_TIMEOUT_SECONDS = 30 * 60
def compose_available() -> bool:
try:
return (
subprocess.run(
["docker", "compose", "version"], capture_output=True, timeout=60
).returncode
== 0
)
except (OSError, subprocess.SubprocessError):
return False
def free_port() -> int:
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
return sock.getsockname()[1]
def mount_base() -> Path:
"""A directory a Docker VM shares with the host.
Not pytest's ``tmp_path``: on macOS that is ``/var/folders/...``, which a Docker VM
(Colima, Docker Desktop) does not share, so the bind mount would arrive empty and
every assertion would be about nothing. ``$HOME`` is shared by every default
configuration.
"""
base = Path(
os.environ.get("PHOTO_PIPELINE_TEST_MOUNT_BASE", Path.home() / ".cache" / "photo-pipeline")
)
base.mkdir(parents=True, exist_ok=True)
return base
def temporary_library(albums: dict[str, int], *, prefix: str = "library-") -> Path:
"""A fixture library on a shareable host path, with the exclusion sentinel in it."""
root = Path(tempfile.mkdtemp(prefix=prefix, dir=mount_base()))
for position, (album, count) in enumerate(albums.items()):
(root / album).mkdir(parents=True, exist_ok=True)
for index in range(count):
# Distinct per album *and* index: two solid images of the same colour are
# byte-identical, which would make them a duplicate cluster by accident.
colour = (17 + 7 * index, 31 + 29 * position, 160 - 3 * index)
Image.new("RGB", (64, 48), colour).save(root / album / f"{album}_{index}.jpg")
# Never discovered, counted, analyzed, or uploaded — asserted from outside it.
(root / "_IGNORE").mkdir(exist_ok=True)
Image.new("RGB", (32, 32), (0, 0, 0)).save(root / "_IGNORE" / "sentinel.jpg")
return root
def remove_library(root: Path) -> None:
shutil.rmtree(root, ignore_errors=True)
def write_env_file(path: Path, secret: str, extra: dict[str, str] | None = None) -> Path:
"""Configuration and secrets come from the environment, so a test writes its own
file rather than borrowing the operator's ``.env``."""
lines = [
f"PHOTO_PIPELINE_ACCESS_SECRET={secret}",
"PHOTO_PIPELINE_LOG_FORMAT=text",
# The deterministic vision seam, in the data volume so both roles and the test
# can read it (concept §18).
"PHOTO_PIPELINE_FAKE_VISION_LOG=/data/vision.log",
*(f"{key}={value}" for key, value in (extra or {}).items()),
]
path.write_text("\n".join(lines) + "\n")
return path
class Stack:
"""The composition under test, plus the environment it was started with."""
def __init__(
self,
library: Path,
env_file: Path,
*,
project: str,
image: str,
secret: str,
) -> None:
self.library = library
self.project = project
self.secret = secret
self.port = free_port()
self.base = f"http://127.0.0.1:{self.port}"
# Compose reads the repository's own .env for substitution; the process
# environment wins over it, so the test's values are the ones that apply.
self.env = {
**os.environ,
"PHOTO_PIPELINE_IMAGE": image,
"PHOTO_PIPELINE_ENV_FILE": str(env_file),
"PHOTO_PIPELINE_LIBRARY_HOST_PATH": str(library),
"PHOTO_PIPELINE_LIBRARY_ROOTS": CONTAINER_LIBRARY,
"PHOTO_PIPELINE_PORT": str(self.port),
"PHOTO_PIPELINE_UID": str(os.getuid()),
"PHOTO_PIPELINE_GID": str(os.getgid()),
}
# ── the compose lifecycle ────────────────────────────────────────────────
def compose(self, *args: str, check: bool = True, timeout: int = 300):
result = subprocess.run(
["docker", "compose", "-p", self.project, "-f", str(COMPOSE_FILE), *args],
capture_output=True,
text=True,
env=self.env,
cwd=REPO,
timeout=timeout,
)
if check and result.returncode != 0:
raise AssertionError(
f"docker compose {' '.join(args)} failed:\n{result.stdout}\n{result.stderr}\n"
f"{self.compose('logs', '--tail', '80', check=False).stdout}"
)
return result
def up(self, *extra: str) -> None:
self.compose("up", "--detach", *extra, timeout=UP_TIMEOUT_SECONDS)
def down(self, *, volumes: bool = True) -> None:
self.compose(
"down",
*(("--volumes",) if volumes else ()),
"--remove-orphans",
check=False,
timeout=300,
)
def use_image(self, image: str) -> None:
"""Point the composition at another tag — the upgrade path (US08-05)."""
self.env["PHOTO_PIPELINE_IMAGE"] = image
def logs(self, *services: str) -> str:
result = self.compose("logs", *services, check=False)
return result.stdout + result.stderr
def wait_until_ready(self) -> None:
deadline = time.monotonic() + READY_TIMEOUT_SECONDS
while time.monotonic() < deadline:
try:
if httpx.get(f"{self.base}/api/v1/health/ready", timeout=5).status_code == 200:
return
except httpx.HTTPError:
pass
time.sleep(0.5)
raise AssertionError(f"the stack never became ready:\n{self.logs()}")
# ── talking to it ────────────────────────────────────────────────────────
def client(self) -> httpx.Client:
"""A browser that has loaded the app: session cookie in the jar, token in a
header. The session belongs to the API process, so it is re-bootstrapped after
every restart."""
client = httpx.Client(base_url=f"{self.base}/api/v1", timeout=60)
bootstrap = client.get("/session", headers={"X-Access-Secret": self.secret})
assert bootstrap.status_code == 200, bootstrap.text
client.headers["X-CSRF-Token"] = bootstrap.json()["csrf_token"]
return client
def await_job(client: httpx.Client, job_id: str, states=("succeeded",)) -> dict:
deadline = time.monotonic() + JOB_TIMEOUT_SECONDS
snapshot: dict = {}
while time.monotonic() < deadline:
response = client.get(f"/jobs/{job_id}")
if response.status_code == 200:
snapshot = response.json()
if snapshot["state"] in states:
return snapshot
time.sleep(0.5)
raise AssertionError(f"job {job_id} never reached {states}: {snapshot}")
def build_image(tag: str, *, revision: str | None = None) -> str:
"""Build the application image; from a git revision's tree when one is named.
``revision`` is how the upgrade journey gets the *previous* version without a
registry: the tree of that commit is the build context, so what it produces is the
image that commit would have published.
"""
if revision is None:
subprocess.run(
[
"docker",
"build",
"--build-arg",
f"UID={os.getuid()}",
"--build-arg",
f"GID={os.getgid()}",
"-t",
tag,
str(REPO),
],
check=True,
timeout=UP_TIMEOUT_SECONDS,
)
return tag
archive = subprocess.run(
["git", "archive", "--format=tar", revision],
cwd=REPO,
capture_output=True,
check=True,
timeout=300,
).stdout
subprocess.run(
[
"docker",
"build",
"--build-arg",
f"UID={os.getuid()}",
"--build-arg",
f"GID={os.getgid()}",
"-t",
tag,
"-",
],
input=archive,
check=True,
timeout=UP_TIMEOUT_SECONDS,
)
return tag
needs_compose = pytest.mark.skipif(
not compose_available(), reason="no Docker daemon with the compose plugin"
)

212
tests/e2e/test_compose_stack.py
View File

@@ -1,212 +0,0 @@
"""US08-03: the composition, actually composed.
Nothing here is faked below the process boundary: Docker builds the image, Compose
starts the migrate/API/worker containers against a temporary fixture library on a
real bind mount, and every assertion is made over HTTP or against what the stack
left in its data volume. The vision provider is the deterministic fake seam the
other end-to-end suites use, because an upload of real photos to a real model is not
what this story is about — the mount, the lock, the volume, and the restart are.
The file contract (one API, one worker, migrations first, no committed values) is
checked without a daemon in ``tests/integration/test_compose_runtime.py``; only the
running proof needs Docker, and CI is where it runs unskipped (US08-04).
The stack itself lives in ``tests/e2e/_container_harness.py``, shared with the
container acceptance gate (US08-05).
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
import pytest
from tests.e2e._container_harness import (
CONTAINER_LIBRARY,
Stack,
await_job,
compose_available,
needs_compose,
remove_library,
temporary_library,
write_env_file,
)
PROJECT = "photo-pipeline-us0803"
IMAGE = "photo-pipeline-test:us08-03"
SECRET = "compose-acceptance-secret"
pytestmark = pytest.mark.container
@pytest.fixture(scope="module")
def library() -> Path:
root = temporary_library({"01_day": 2, "02_night": 1})
try:
yield root
finally:
remove_library(root)
@pytest.fixture(scope="module")
def env_file(tmp_path_factory) -> Path:
return write_env_file(tmp_path_factory.mktemp("config") / "compose.env", SECRET)
@pytest.fixture(scope="module")
def stack(library, env_file):
if not compose_available():
pytest.skip("no Docker daemon with the compose plugin")
running = Stack(library, env_file, project=PROJECT, image=IMAGE, secret=SECRET)
running.down()
running.up("--build")
try:
running.wait_until_ready()
yield running
finally:
running.down()
# ── the mounted library ──────────────────────────────────────────────────────
@needs_compose
def test_the_stack_scans_the_bind_mounted_library_at_its_container_paths(stack):
client = stack.client()
try:
scanned = client.post("/inventory/scan")
assert scanned.status_code == 200, scanned.text
assets = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
finally:
client.close()
assert len(assets) == 3, assets
paths = {asset["current_path"] for asset in assets}
assert all(path.startswith(CONTAINER_LIBRARY + "/") for path in paths), paths
assert not any("_IGNORE" in path or "sentinel" in path for path in paths)
# The host paths are what the operator mounted, and they are not what the
# application records: the roots are the container's.
assert not any(str(stack.library) in path for path in paths)
@needs_compose
def test_a_library_root_that_is_not_mounted_is_refused_at_startup(stack):
"""The container-specific failure: configured roots that name nothing mounted."""
refused = stack.compose(
"run",
"--rm",
"--no-deps",
"--env",
"PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos",
"api",
"serve",
check=False,
)
assert refused.returncode == 5, refused.stdout + refused.stderr
assert "library root /srv/photos" in refused.stdout + refused.stderr
@needs_compose
def test_a_second_worker_is_refused_by_the_library_lock(stack):
"""Not by convention: the running worker's lock is in the shared data volume."""
refused = stack.compose(
"run", "--rm", "--no-deps", "worker", "worker", "--id", "worker-2", check=False
)
assert refused.returncode == 2, refused.stdout + refused.stderr
assert "worker is already running" in refused.stdout + refused.stderr
# ── restart, resume, and the data volume ─────────────────────────────────────
@needs_compose
def test_a_queued_job_resumes_after_both_containers_restart(stack):
client = stack.client()
try:
client.post("/inventory/scan").raise_for_status()
assets = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
for asset in assets:
decided = client.post(
"/safety/decisions", json={"asset_id": asset["id"], "decision": "sfw"}
)
assert decided.status_code == 200, decided.text
# Stop the worker first, so the job is provably still queued when the restart
# happens: a job that finished before the restart would prove nothing.
stack.compose("stop", "worker")
job = client.post("/analysis/jobs").json()
assert client.get(f"/jobs/{job['id']}").json()["state"] == "queued"
finally:
client.close()
stack.compose("restart", "api", "worker")
stack.wait_until_ready()
client = stack.client() # the session is per API process, so it is re-bootstrapped
try:
finished = await_job(client, job["id"])
assert finished["state"] == "succeeded", finished
# The database is intact and the work is durable, not merely reported.
after = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
assert {asset["id"] for asset in after} == {asset["id"] for asset in assets}
analysed = client.get(f"/analysis/results/{assets[0]['id']}")
assert analysed.status_code == 200, analysed.text
assert analysed.json()["description"]
# Only the mounted library's own photos were analysed: the sentinel under
# _IGNORE is not an asset, so it can never have become an item of this job.
assert finished["progress"]["total"] == len(assets)
finally:
client.close()
@needs_compose
def test_the_data_volume_survives_recreating_the_containers(stack):
"""`down` without `--volumes` then `up` is the upgrade path: state stays."""
client = stack.client()
try:
client.post("/inventory/scan").raise_for_status()
before = {a["id"] for a in client.get("/inventory/assets").json()["items"]}
finally:
client.close()
stack.down(volumes=False)
stack.up()
stack.wait_until_ready()
client = stack.client()
try:
after = {a["id"] for a in client.get("/inventory/assets").json()["items"]}
finally:
client.close()
assert after == before, "the same assets, from the same database, on the same volume"
# ── operating it ─────────────────────────────────────────────────────────────
@needs_compose
def test_backup_verify_and_diagnostics_run_as_container_commands(stack):
backup = stack.compose("run", "--rm", "--no-deps", "api", "backup", "--reason", "compose")
manifest = json.loads(backup.stdout[backup.stdout.index("{") :])
assert manifest["name"].startswith("2")
verified = stack.compose(
"run", "--rm", "--no-deps", "api", "verify-backup", f"/data/backups/{manifest['name']}"
)
assert json.loads(verified.stdout[verified.stdout.index("{") :])["ok"] is True
report = stack.compose("run", "--rm", "--no-deps", "api", "diagnostics")
diagnostics = json.loads(report.stdout[report.stdout.index("{") :])
components = {c["name"]: c["path"] for c in diagnostics["components"]}
# Database, WAL, thumbnail cache, and backups all live in the mounted volume.
for name in ("database", "write_ahead_log", "thumbnail_cache", "backups"):
assert components[name].startswith("/data/"), (name, components[name])
assert {tool["name"] for tool in diagnostics["tools"]} >= {"exiftool", "immich-go"}
# And the worker running beside this command is visible as the lock's holder.
assert diagnostics["locks"]["worker"]["role"] == "worker"
if __name__ == "__main__": # a quick way to run just this file
raise SystemExit(pytest.main([__file__, "-v", *sys.argv[1:]]))

306
tests/e2e/test_container_runtime.py
View File

@@ -1,306 +0,0 @@
"""US08-02: the built image, actually built and actually run.
This is the acceptance test for the image itself, so nothing here is faked: Docker
builds from a clean context, the container starts under a chosen UID/GID against a
mounted data directory, and the assertions are made over HTTP and against the files
the container left on the host.
It is skipped without a Docker daemon — the build also needs the network for the base
image, the pinned exiftool package, and the pinned uploader release. The contract the
Dockerfile itself has to keep (pins, non-root, health target, build context) is checked
offline in ``tests/integration/test_container_image.py``, so a machine without Docker
still fails on a broken image definition; only the running proof needs the daemon. CI
builds the image on every change (US08-04), which is where this runs unskipped.
"""
from __future__ import annotations
import json
import os
import platform
import re
import socket
import subprocess
import sys
import time
from pathlib import Path
import httpx
import pytest
REPO = Path(__file__).resolve().parents[2]
IMAGE = "photo-pipeline-test:us08-02"
SECRET = "container-acceptance-secret"
HOSTNAME = "photos.test"
READY_TIMEOUT_SECONDS = 120
BUILD_TIMEOUT_SECONDS = 30 * 60
pytestmark = pytest.mark.container
def docker_available() -> bool:
try:
return subprocess.run(["docker", "info"], capture_output=True, timeout=60).returncode == 0
except (OSError, subprocess.SubprocessError):
return False
needs_docker = pytest.mark.skipif(not docker_available(), reason="no Docker daemon available")
def docker(*args: str, check: bool = True, timeout: int = 120) -> subprocess.CompletedProcess:
result = subprocess.run(
["docker", *args], capture_output=True, text=True, timeout=timeout
)
if check and result.returncode != 0:
raise AssertionError(f"docker {' '.join(args)} failed:\n{result.stdout}\n{result.stderr}")
return result
def pins() -> dict[str, str]:
"""The pinned versions, read from the Dockerfile that produced the image."""
text = (REPO / "Dockerfile").read_text()
found = dict(re.findall(r"^ARG\s+([A-Z0-9_]+)=(.+)$", text, re.MULTILINE))
return {
# The Debian package version carries a packaging suffix; exiftool reports the
# upstream version only.
"exiftool": found["EXIFTOOL_VERSION"].split("+")[0].split("-")[0],
"immich-go": found["IMMICH_GO_VERSION"],
}
def free_port() -> int:
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
return sock.getsockname()[1]
@pytest.fixture(scope="module")
def image() -> str:
"""Build from a clean checkout: the build context is the repository, unmodified."""
if not docker_available():
pytest.skip("no Docker daemon available")
docker(
"build",
"--build-arg",
f"UID={os.getuid()}",
"--build-arg",
f"GID={os.getgid()}",
"-t",
IMAGE,
str(REPO),
timeout=BUILD_TIMEOUT_SECONDS,
)
return IMAGE
@pytest.fixture
def data_dir(tmp_path) -> Path:
data = tmp_path / "data"
data.mkdir()
return data
def run_detached(image: str, port: int, *args: str, data: Path | None = None) -> str:
"""Start a container. ``data`` bind-mounts the host's data directory when the test
is about the files themselves; otherwise the image's own /data is used, because a
macOS bind mount arrives with an ownership the container did not choose."""
result = docker(
"run",
"--detach",
"--rm",
"--publish",
f"127.0.0.1:{port}:8000",
*(("--volume", f"{data}:/data") if data is not None else ()),
"--env",
# Reachable from outside the container means reachable from another machine as
# far as the application is concerned, so the access secret is mandatory
# (US08-01) — the image must not weaken that.
"PHOTO_PIPELINE_HOST=0.0.0.0",
"--env",
f"PHOTO_PIPELINE_ACCESS_SECRET={SECRET}",
"--env",
f"PHOTO_PIPELINE_ALLOWED_HOSTS={HOSTNAME}",
image,
*args,
)
return result.stdout.strip()
def wait_until_ready(base: str, container: str) -> None:
deadline = time.monotonic() + READY_TIMEOUT_SECONDS
while time.monotonic() < deadline:
try:
if httpx.get(f"{base}/api/v1/health/ready", timeout=5).status_code == 200:
return
except httpx.HTTPError:
pass
if docker("inspect", "-f", "{{.State.Running}}", container, check=False).stdout.strip() in (
"false",
"",
):
break
time.sleep(0.5)
logs = docker("logs", container, check=False)
raise AssertionError(f"container never became ready:\n{logs.stdout}\n{logs.stderr}")
@pytest.fixture
def serving(image):
port = free_port()
container = run_detached(image, port, "serve")
try:
base = f"http://127.0.0.1:{port}"
wait_until_ready(base, container)
yield base, container
finally:
docker("rm", "--force", container, check=False)
def session(base: str) -> httpx.Client:
client = httpx.Client(base_url=base, timeout=30)
bootstrap = client.get("/api/v1/session", headers={"X-Access-Secret": SECRET})
assert bootstrap.status_code == 200, bootstrap.text
client.headers["X-CSRF-Token"] = bootstrap.json()["csrf_token"]
return client
# ── the image serves, and says what it contains ──────────────────────────────
@needs_docker
def test_the_container_serves_the_frontend_and_the_pinned_tool_versions(serving):
base, container = serving
index = httpx.get(f"{base}/app/index.html", timeout=30)
assert index.status_code == 200
assert "<title" in index.text.lower(), "the application shell, not an API error"
client = session(base)
try:
tools = {tool["name"]: tool for tool in client.get("/api/v1/diagnostics").json()["tools"]}
finally:
client.close()
for name, pinned in pins().items():
assert tools[name]["pinned"] == pinned, name
# Recorded *and* installed: the reported version comes from running the binary.
assert pinned in tools[name]["version"], (name, tools[name])
assert tools[name]["path"], f"{name} is not on PATH inside the image"
logs = docker("logs", container, check=False)
assert SECRET not in logs.stdout + logs.stderr, "the access secret never reaches the log"
@needs_docker
def test_the_declared_health_check_reports_readiness(serving):
"""The declared HEALTHCHECK is readiness, so Docker's own verdict is the assertion."""
_, container = serving
deadline = time.monotonic() + READY_TIMEOUT_SECONDS
status = ""
while time.monotonic() < deadline:
status = docker(
"inspect", "-f", "{{.State.Health.Status}}", container, check=False
).stdout.strip()
if status == "healthy":
break
time.sleep(1)
assert status == "healthy"
probe = docker("exec", container, "/usr/local/bin/healthcheck.sh", check=False)
assert probe.returncode == 0
# Point the probe at a port nothing serves: the same script must fail, which is
# what makes the healthy verdict above evidence rather than a default.
unready = docker(
"exec",
"--env",
"PHOTO_PIPELINE_PORT=1",
container,
"/usr/local/bin/healthcheck.sh",
check=False,
)
assert unready.returncode != 0
# ── identity: never root, always the configured owner ────────────────────────
@needs_docker
def test_the_container_refuses_to_run_as_root(image, data_dir):
result = docker(
"run",
"--rm",
"--user",
"0:0",
"--volume",
f"{data_dir}:/data",
image,
"diagnostics",
check=False,
)
assert result.returncode != 0
assert "refusing to run as root" in result.stderr + result.stdout
assert not list(data_dir.iterdir()), "a refused container writes nothing"
@needs_docker
def test_what_the_container_writes_is_owned_by_the_build_arguments(image):
"""The identity the image was built with is the identity on disk afterwards.
Asserted from inside the container so it holds on every host: a macOS bind mount
reports an ownership the container never chose. The host-side proof, which is what
the mounted library actually needs, is the Linux test below.
"""
port = free_port()
container = run_detached(image, port, "serve")
try:
wait_until_ready(f"http://127.0.0.1:{port}", container)
owner = docker(
"exec", container, "stat", "-c", "%u:%g", "/data/photo_pipeline.db"
).stdout.strip()
assert owner == f"{os.getuid()}:{os.getgid()}"
assert docker("exec", container, "id", "-u").stdout.strip() == str(os.getuid())
finally:
docker("rm", "--force", container, check=False)
@needs_docker
@pytest.mark.skipif(
platform.system() != "Linux",
reason="bind-mount ownership is virtualised by Docker Desktop on macOS/Windows",
)
def test_files_the_container_writes_keep_the_configured_ownership(image, data_dir):
docker("run", "--rm", "--volume", f"{data_dir}:/data", image, "migrate", timeout=300)
written = sorted(path for path in data_dir.rglob("*") if path.is_file())
assert written, "migrate creates the database in the mounted data directory"
for path in written:
assert (path.stat().st_uid, path.stat().st_gid) == (os.getuid(), os.getgid()), path
@needs_docker
def test_the_worker_role_runs_from_the_same_image(image, data_dir):
"""One image, two roles: the worker is the same entrypoint with another argument."""
port = free_port()
container = run_detached(image, port, "worker", "--id", "container-worker")
try:
# Taking the worker's library lock is the observable proof that it started,
# migrated, and reached its job loop — no sleep required (US07-05).
deadline = time.monotonic() + READY_TIMEOUT_SECONDS
lock = ""
while not lock and time.monotonic() < deadline:
assert docker("inspect", "-f", "{{.State.Running}}", container).stdout.strip() == (
"true"
), docker("logs", container, check=False).stdout
lock = docker("exec", container, "cat", "/data/worker.lock.json", check=False).stdout
time.sleep(0.5)
assert lock, docker("logs", container, check=False).stdout
assert json.loads(lock)["role"] == "worker"
role = docker("exec", container, "cat", "/tmp/photo-pipeline-role").stdout.strip()
assert role == "worker", "the health check can tell which role this container is"
finally:
docker("rm", "--force", container, check=False)
if __name__ == "__main__": # a quick way to run just this file
raise SystemExit(pytest.main([__file__, "-v", *sys.argv[1:]]))

152
tests/e2e/test_docs_ui.py
View File

@@ -1,152 +0,0 @@
"""US09-01: the manuals, in the browser, under the application's real policy.
Everything here runs against a real ``photo_pipeline serve`` process serving the
committed ``docs/`` tree and the vendored renderer — no fixture markdown, no stubbed
fetch. What that buys is the assertion the story actually cares about: the pages
render, the links between them work, a diagram becomes a diagram, and the console
stays empty, including of CSP violations.
The offline contract — reachability, dead links, pinned checksums — is
``tests/integration/test_documentation.py``.
"""
from __future__ import annotations
from pathlib import Path
import pytest
from tests.e2e._pipeline_harness import Server, seed_library
# Every check here belongs to the documentation gate (US09-05).
pytestmark = pytest.mark.phase_i
@pytest.fixture(scope="module")
def server(tmp_path_factory):
seeded = seed_library(tmp_path_factory.mktemp("docs"), {"a": 1}, {})
running = Server(seeded).start()
try:
yield running
finally:
running.stop()
@pytest.fixture
def quiet(page):
"""Any console error, page error, or failed request fails the test that caused
it. A CSP violation arrives as a console error, which is the point."""
problems: list[str] = []
page.on("console", lambda m: problems.append(m.text) if m.type == "error" else None)
page.on("pageerror", lambda error: problems.append(str(error)))
page.on("requestfailed", lambda request: problems.append(f"failed request: {request.url}"))
yield problems
def test_the_documentation_opens_from_the_navigation(page, server, quiet):
page.goto(f"{server.base}/app/#/workflow")
page.locator('nav a[data-nav="docs"]').click()
page.get_by_test_id("doc").wait_for()
assert page.get_by_test_id("doc").get_attribute("data-page") == "index"
# The sidebar is the index's own reading order, not a second list to maintain.
assert page.get_by_test_id("doc-pages").get_by_role("link", name="Overview").is_visible()
assert quiet == []
def test_a_link_between_documents_stays_inside_the_application(page, server, quiet):
page.goto(f"{server.base}/app/#/docs")
page.get_by_test_id("doc").wait_for()
page.get_by_test_id("doc").get_by_role("link", name="Overview").click()
page.wait_for_selector('[data-testid="doc"][data-page="overview"]')
assert "#/docs?page=overview" in page.url, "a relative .md link must not leave the app"
# And back again, by the link the document itself carries.
page.get_by_test_id("doc").get_by_role("link", name="← Documentation index").click()
page.wait_for_selector('[data-testid="doc"][data-page="index"]')
assert quiet == []
def test_a_deep_link_to_a_heading_lands_on_that_heading(page, server, quiet):
page.goto(f"{server.base}/app/#/docs?page=overview&anchor=the-workflow")
heading = page.locator("#the-workflow")
heading.wait_for()
assert heading.inner_text().strip() == "The workflow"
assert heading.evaluate("node => node.getBoundingClientRect().top < window.innerHeight")
assert quiet == []
def test_a_mermaid_block_becomes_a_diagram_under_the_unchanged_script_policy(page, server, quiet):
page.goto(f"{server.base}/app/#/docs?page=overview")
diagram = page.get_by_test_id("diagram").first
diagram.wait_for()
# A real drawing, not the source text and not an error node.
assert diagram.locator("svg").count() == 1
assert diagram.evaluate("node => node.querySelector('svg').getBBox().width") > 100
assert page.locator("pre code.language-mermaid").count() == 0
assert quiet == [], "rendering a diagram must not violate the policy"
policy = page.evaluate(
"async () => (await fetch('/app/')).headers.get('content-security-policy')"
)
assert "script-src 'self';" in policy
assert "unsafe-eval" not in policy
def test_every_diagram_in_the_architecture_overview_draws(page, server, quiet):
"""Five diagrams, three of them state machines (US09-03). A mermaid block with a
syntax error renders an error node instead of throwing, so a page that merely
loaded proves nothing — count the drawings and read the source for the states."""
source = (Path(__file__).resolve().parents[2] / "docs" / "architecture.md").read_text()
expected = source.count("```mermaid")
assert expected >= 5, "the overview lost its diagrams"
page.goto(f"{server.base}/app/#/docs?page=architecture")
page.get_by_test_id("diagram").first.wait_for()
page.wait_for_function(
"count => document.querySelectorAll('[data-testid=\"diagram\"] svg').length === count",
arg=expected,
)
drawn = page.locator('[data-testid="diagram"] svg')
assert drawn.count() == expected
for index in range(expected):
diagram = drawn.nth(index)
# Labels may be <text> or HTML inside a <foreignObject> depending on the
# diagram type, so ask for the rendered text rather than a specific element.
labels = diagram.evaluate("node => node.textContent.trim().length")
assert labels > 0, f"diagram {index} drew no labels"
assert diagram.locator(".error-icon, .error-text").count() == 0, f"diagram {index} errored"
assert page.locator("pre code.language-mermaid").count() == 0
assert quiet == []
def test_an_unknown_page_says_so_without_naming_a_path(page, server, quiet):
page.goto(f"{server.base}/app/#/docs?page=no-such-manual")
message = page.get_by_test_id("doc-not-found")
message.wait_for()
text = message.inner_text()
assert "does not exist" in text
assert "/" not in text.replace("Back to the documentation index", ""), text
message.get_by_role("link").click()
page.wait_for_selector('[data-testid="doc"][data-page="index"]')
# The missing page is a 404 and the browser says so; nothing else may go wrong,
# and in particular the view must not throw on the way to its own message.
assert all("404" in problem for problem in quiet), quiet
def test_the_documentation_is_readable_without_a_session(page, server, quiet):
"""The troubleshooting page is needed most by whoever is locked out."""
page.goto(f"{server.base}/app/#/docs")
page.get_by_test_id("doc").wait_for()
unauthenticated = page.evaluate(
"""async () => {
const response = await fetch('/docs/index.md', { credentials: 'omit' });
return { status: response.status, length: (await response.text()).length };
}"""
)
assert unauthenticated["status"] == 200
assert unauthenticated["length"] > 100

510
tests/e2e/test_phase_h_container.py
View File

@@ -1,510 +0,0 @@
"""Phase H — the container acceptance gate (US08-05).
The deployed application is verified the way the host application is: real image,
real composition, real browser, real restarts. Everything below runs against
containers that this suite provisions from the built image, against a temporary
fixture library on a bind mount and an isolated data volume, and destroys afterwards.
Four journeys, one per thing a deployment can get wrong:
* **the browser journey** — discovery, duplicate review, analysis, album proposal,
rename, upload preflight, and archive, driven through the containerized frontend;
* **the upgrade journey** — the previous version's image runs first, then this one,
and the database, its migrations, the rename journal, the job history, and the
thumbnail cache have to still be there;
* **the restart journey** — both containers are killed mid-job and the work resumes
without doing anything twice;
* **the security gates** — the refusals a loopback deployment made are still made
behind a published port: no session, forged forwarded headers, a path that leaves
the mounted library, and a secret in the logs.
Run it as one command, with evidence: ``python -m photo_pipeline container-gate``.
"""
from __future__ import annotations
import collections
from contextlib import closing
import json
import os
import shutil
import subprocess
import sys
import time
from pathlib import Path
import httpx
import pytest
from playwright.sync_api import expect
from photo_pipeline.db import head_revision
from tests.e2e._container_harness import (
Stack,
await_job,
build_image,
compose_available,
needs_compose,
remove_library,
temporary_library,
write_env_file,
)
pytestmark = [pytest.mark.phase_h, pytest.mark.container]
PROJECT = "photo-pipeline-us0805"
IMAGE = "photo-pipeline-test:us08-05"
# The version being upgraded *from*. CI passes the tag it last published; without one,
# the previous commit's tree is built, which is the same claim without a registry.
PREVIOUS_IMAGE = os.environ.get("PHOTO_PIPELINE_PREVIOUS_IMAGE")
PREVIOUS_TAG = "photo-pipeline-test:us08-05-previous"
SECRET = "container-gate-access-secret"
IMMICH_SENTINEL = "immich-sentinel-9f3a2b"
HOSTNAME = "photos.test"
ALBUM = "rome"
RENAMED = "2019 Rome"
BURST = 30
# ── the image and the stacks ─────────────────────────────────────────────────
@pytest.fixture(scope="session")
def image() -> str:
if not compose_available():
pytest.skip("no Docker daemon with the compose plugin")
return build_image(IMAGE)
@pytest.fixture(scope="module")
def journey(image, tmp_path_factory):
"""One album, one exact duplicate of a photo in it, and the exclusion sentinel."""
library = temporary_library({ALBUM: 3}, prefix="us0805-journey-")
shutil.copyfile(library / ALBUM / f"{ALBUM}_0.jpg", library / ALBUM / "copy.jpg")
env_file = write_env_file(
tmp_path_factory.mktemp("journey") / "gate.env",
SECRET,
{
"PHOTO_PIPELINE_ALLOWED_HOSTS": HOSTNAME,
# Configured but never reachable: the upload view's job here is to show the
# preflight blockers, and the key's job is to be absent from every log.
"PHOTO_PIPELINE_IMMICH_API_KEY": IMMICH_SENTINEL,
"PHOTO_PIPELINE_IMMICH_SERVER_URL": "http://127.0.0.1:1",
},
)
stack = Stack(library, env_file, project=f"{PROJECT}-journey", image=image, secret=SECRET)
try:
stack.down()
stack.up()
stack.wait_until_ready()
_seed_journey(stack)
yield stack
finally:
stack.down()
remove_library(library)
def _seed_journey(stack: Stack) -> None:
"""Everything the views need, established over the public API before they render."""
with closing(stack.client()) as client:
client.post("/inventory/scan").raise_for_status()
client.post("/duplicates/detect").raise_for_status()
queue = client.get("/safety/queue", params={"limit": 100}).json()["items"]
for item in queue:
decided = client.post(
"/safety/decisions", json={"asset_id": item["asset_id"], "decision": "sfw"}
)
assert decided.status_code == 200, decided.text
job = client.post("/analysis/jobs").json()
await_job(client, job["id"])
client.post("/albums/proposals", json={}).raise_for_status()
# An archive destination inside the data volume: a second mount would prove
# nothing more, and the view needs a location to have something to show.
stack.compose("exec", "-T", "api", "mkdir", "-p", "/data/archive")
client.post(
"/archive-locations", json={"name": "external", "root": "/data/archive"}
).raise_for_status()
# ── the browser journey ──────────────────────────────────────────────────────
@needs_compose
def test_the_browser_journey_covers_every_stage_view_of_the_deployed_app(page, journey):
"""One pass through the deployed frontend, in workflow order.
The access secret is supplied the way a person supplies it — the app asks, the
answer is kept for the tab — so what is proven is the authenticated deployment,
not a test-only bypass.
"""
page.on("dialog", lambda dialog: dialog.accept(SECRET))
errors: list[str] = []
page.on("pageerror", lambda error: errors.append(str(error)))
base = journey.base
with closing(journey.client()) as client:
cluster = client.get("/duplicates/clusters").json()["items"][0]
# ── discovery ────────────────────────────────────────────────────────────
page.goto(f"{base}/app/#/workflow")
page.get_by_test_id("stage-safety").wait_for()
expect(page.get_by_test_id("stage-analysis")).to_be_visible()
page.goto(f"{base}/app/#/inventory")
rows = page.get_by_test_id("asset-row")
rows.first.wait_for()
assert rows.count() == 4, "three photos and the duplicate copy; never the sentinel"
assert "sentinel" not in page.content() and "_IGNORE" not in page.content()
# ── duplicate review ─────────────────────────────────────────────────────
page.goto(f"{base}/app/#/duplicates/{cluster['id']}")
page.get_by_test_id("cluster-state").wait_for()
# Exact bytes: the cluster arrives decided, and the review surface has to show
# both members and the evidence the decision was made on.
expect(page.get_by_test_id("cluster-state")).to_contain_text("decided")
expect(page.get_by_test_id("member")).to_have_count(2)
expect(page.get_by_test_id("member").first).to_contain_text("/library/")
# ── analysis ─────────────────────────────────────────────────────────────
page.goto(f"{base}/app/#/analyze")
page.get_by_test_id("analyze-counts").wait_for()
expect(page.get_by_test_id("run-analysis")).to_be_visible()
# ── album proposal ───────────────────────────────────────────────────────
page.goto(f"{base}/app/#/albums?album={ALBUM}")
page.get_by_test_id("suggested-name").wait_for()
page.get_by_test_id("final-name").fill(RENAMED)
page.get_by_test_id("save-name").click()
page.get_by_test_id("approve").click()
expect(page.get_by_test_id("proposal-status")).to_contain_text("approved")
# ── rename, applied against the bind mount ───────────────────────────────
page.goto(f"{base}/app/#/renames")
page.get_by_test_id("build-plan").click()
page.get_by_test_id("operations").wait_for()
expect(page.get_by_test_id("op-destination").first).to_contain_text(RENAMED)
page.get_by_test_id("apply-plan").click()
expect(page.get_by_test_id("apply-result")).to_contain_text("Applied 1, failed 0")
# The mounted library is the host's directory: the container renamed the operator's
# folder, not a copy inside its own layer.
assert (journey.library / RENAMED).is_dir()
assert not (journey.library / ALBUM).exists()
# ── upload preflight ─────────────────────────────────────────────────────
page.goto(f"{base}/app/#/uploads")
page.get_by_test_id("upload-scope").wait_for()
expect(page.get_by_test_id("album-row").first).to_be_visible()
assert IMMICH_SENTINEL not in page.content(), "the API key never reaches the browser"
# ── archive ──────────────────────────────────────────────────────────────
page.goto(f"{base}/app/#/archive")
page.get_by_test_id("archive-locations").wait_for()
expect(page.get_by_test_id("location-row").first).to_contain_text("external")
assert errors == [], f"the deployed frontend raised page errors: {errors}"
# ── the upgrade journey ──────────────────────────────────────────────────────
@pytest.fixture(scope="module")
def previous_image(image) -> str:
"""The image the deployment is upgrading *from*.
The published tag when there is one. Before the first publish there is nothing to
pull, and refusing then would mean the gate could never let the first deploy
through — so the previous commit's tree is built instead, which is the same claim
without a registry.
"""
if PREVIOUS_IMAGE:
pulled = subprocess.run(
["docker", "pull", PREVIOUS_IMAGE], capture_output=True, timeout=1800
)
if pulled.returncode == 0:
return PREVIOUS_IMAGE
return build_image(PREVIOUS_TAG, revision="HEAD~1")
@needs_compose
def test_an_upgrade_keeps_the_database_the_journal_the_jobs_and_the_cache(
image, previous_image, tmp_path_factory
):
library = temporary_library({ALBUM: 2}, prefix="us0805-upgrade-")
env_file = write_env_file(tmp_path_factory.mktemp("upgrade") / "gate.env", SECRET)
stack = Stack(
library, env_file, project=f"{PROJECT}-upgrade", image=previous_image, secret=SECRET
)
try:
stack.down()
stack.up()
stack.wait_until_ready()
# ── what the previous version leaves behind ──────────────────────────
with closing(stack.client()) as client:
client.post("/inventory/scan").raise_for_status()
assets = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
for item in client.get("/safety/queue", params={"limit": 100}).json()["items"]:
client.post(
"/safety/decisions", json={"asset_id": item["asset_id"], "decision": "sfw"}
).raise_for_status()
job = client.post("/analysis/jobs").json()
finished = await_job(client, job["id"])
for asset in assets: # populate the thumbnail cache
thumbnail = client.get(f"/assets/{asset['id']}/thumbnail", params={"size": 256})
assert thumbnail.status_code == 200, thumbnail.text
# A rename plan, left unapplied: the journal has to survive the upgrade
# exactly as it was, or a half-applied one could not be recovered.
client.post("/albums/proposals", json={}).raise_for_status()
_approve(client, RENAMED)
plan = client.post("/rename-plans").json()
before = {
"assets": sorted(asset["id"] for asset in assets),
"job": finished["progress"],
"plan": (plan["id"], plan["checksum"]),
"thumbnails": _cache_files(stack),
"revision": _revision(stack),
}
assert before["thumbnails"], "no thumbnail was cached, so nothing would be proven"
# ── the upgrade: same volume, new image ──────────────────────────────
stack.down(volumes=False)
stack.use_image(image)
stack.up()
stack.wait_until_ready()
after_revision = _revision(stack)
assert after_revision == head_revision(), "the new image did not migrate the volume"
if after_revision != before["revision"]:
# A schema change is snapshotted before it is applied (US07-05), so a
# failed upgrade is restorable rather than a lost library.
assert _ls(stack, "/data/backups"), "a migration ran without a backup"
with closing(stack.client()) as client:
assets = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
assert sorted(asset["id"] for asset in assets) == before["assets"]
assert client.get(f"/jobs/{job['id']}").json()["progress"] == before["job"]
plans = client.get("/rename-plans").json()["items"]
assert (plans[0]["id"], plans[0]["checksum"]) == before["plan"]
for asset in assets:
assert (
client.get(f"/analysis/results/{asset['id']}").status_code == 200
), "an analysis result did not survive the upgrade"
# The cache is keyed by pixel hash and thumbnail version, so an upgrade that
# kept the volume must keep the files: regenerating them is work nobody asked
# for, and losing them silently is how a cache stops being one.
assert set(before["thumbnails"]) <= set(_cache_files(stack))
finally:
stack.down()
remove_library(library)
def _approve(client: httpx.Client, name: str, *, album: str = ALBUM) -> None:
for payload, route in (({"name": name}, "edit"), ({}, "approve")):
current = client.get(f"/albums/proposals/{album}").json()
client.post(
f"/albums/proposals/{album}/{route}",
json={**payload, "expected_version": current["version"]},
).raise_for_status()
def _exec(stack: Stack, *args: str) -> str:
return stack.compose("exec", "-T", "api", *args).stdout
def _ls(stack: Stack, directory: str) -> list[str]:
listing = stack.compose("exec", "-T", "api", "ls", directory, check=False)
return [line for line in listing.stdout.split() if line]
def _cache_files(stack: Stack) -> list[str]:
return sorted(
_exec(stack, "find", "/data/cache", "-type", "f", "-name", "*.webp").split()
)
def _revision(stack: Stack) -> str:
"""The schema revision the volume's database is actually at."""
return _exec(
stack,
"python",
"-c",
"import sqlite3;print(sqlite3.connect('/data/photo_pipeline.db')"
".execute('select version_num from alembic_version').fetchone()[0])",
).strip()
# ── the restart journey ──────────────────────────────────────────────────────
@needs_compose
def test_killing_both_containers_mid_job_resumes_without_doing_anything_twice(
image, tmp_path_factory
):
"""`docker kill` is the honest restart: no grace period, no orderly stop, no
chance for either process to write a tidy final state."""
library = temporary_library({"burst": BURST}, prefix="us0805-restart-")
env_file = write_env_file(tmp_path_factory.mktemp("restart") / "gate.env", SECRET)
stack = Stack(library, env_file, project=f"{PROJECT}-restart", image=image, secret=SECRET)
try:
stack.down()
stack.up()
stack.wait_until_ready()
with closing(stack.client()) as client:
client.post("/inventory/scan").raise_for_status()
assets = client.get("/inventory/assets", params={"limit": 200}).json()["items"]
assert len(assets) == BURST
for item in client.get("/safety/queue", params={"limit": 100}).json()["items"]:
client.post(
"/safety/decisions", json={"asset_id": item["asset_id"], "decision": "sfw"}
).raise_for_status()
job = client.post("/analysis/jobs").json()
progress = _wait_for_progress(client, job["id"])
assert 0 < progress["done"] < progress["total"], progress
stack.compose("kill", "api", "worker")
stack.up()
stack.wait_until_ready()
with closing(stack.client()) as client:
finished = await_job(client, job["id"])
assert finished["progress"]["done"] == BURST, finished
results = [
client.get(f"/analysis/results/{asset['id']}").json() for asset in assets
]
# Exactly one stored result per asset: at-least-once execution, idempotent
# recovery — a retried item overwrites its own attempt, it does not add one.
assert len(results) == BURST
assert all(result["description"] for result in results)
# And the side effect nobody can take back — the call to the provider — happened
# again only for whatever was in flight when the containers died.
analysed = collections.Counter(_exec(stack, "cat", "/data/vision.log").split())
assert len(analysed) == BURST, "every photo was analysed, and only the library's"
assert max(analysed.values()) <= 2, dict(analysed)
assert sum(1 for count in analysed.values() if count > 1) <= 1, dict(analysed)
finally:
stack.down()
remove_library(library)
def _wait_for_progress(client: httpx.Client, job_id: str, *, timeout: float = 120) -> dict:
"""Wait until the job is provably under way but provably unfinished."""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
snapshot = client.get(f"/jobs/{job_id}").json()
progress = snapshot["progress"]
if progress["done"] and progress["done"] < progress["total"]:
return progress
if snapshot["state"] in ("succeeded", "failed"):
raise AssertionError(f"the job finished before it could be interrupted: {snapshot}")
time.sleep(0.05)
raise AssertionError(f"the job never started: {client.get(f'/jobs/{job_id}').json()}")
# ── the security gates ───────────────────────────────────────────────────────
@needs_compose
def test_the_deployed_instance_refuses_a_caller_without_a_session(journey):
with httpx.Client(base_url=f"{journey.base}/api/v1", timeout=30) as client:
for method, path in (("GET", "/workflow"), ("POST", "/inventory/scan")):
response = client.request(method, path)
assert response.status_code == 401, path
assert response.json()["error"]["code"] == "unauthenticated"
# A session still has to be paid for with the operator's secret.
assert client.get("/session", headers={"X-Access-Secret": "guessed"}).status_code == 401
# Readiness stays open: the orchestrator's health check holds no session.
assert client.get("/health/ready").status_code == 200
@needs_compose
def test_forged_forwarded_headers_cannot_smuggle_an_allowed_host_past_the_check(journey):
"""Behind a proxy the app believes ``X-Forwarded-*`` — but only from the proxy.
Nothing in this composition is a trusted proxy, so the claim is the client's."""
with httpx.Client(base_url=f"{journey.base}/api/v1", timeout=30) as client:
client.headers["X-CSRF-Token"] = (
client.get("/session", headers={"X-Access-Secret": SECRET}).json()["csrf_token"]
)
# The hostname this deployment is reached under is accepted.
assert client.get("/workflow", headers={"Host": HOSTNAME}).status_code == 200
forged = client.get(
"/workflow",
headers={"Host": "photos.evil.example", "X-Forwarded-Host": HOSTNAME},
)
assert forged.status_code == 403
assert forged.json()["error"]["code"] == "host_not_allowed"
# A forged protocol claim must not mark the session cookie as HTTPS-only
# either — that would strand the operator's real, plain-HTTP session.
bootstrap = httpx.get(
f"{journey.base}/api/v1/session",
headers={"X-Access-Secret": SECRET, "X-Forwarded-Proto": "https"},
timeout=30,
)
assert "secure" not in bootstrap.headers["set-cookie"].lower()
@needs_compose
def test_a_path_that_leaves_the_mounted_library_is_refused(journey):
"""The container's own filesystem is not the library. A symlink swapped under a
known asset is the sharpest version of the question, because the database still
points at a path inside the mount."""
with closing(journey.client()) as client:
asset = client.get("/inventory/assets", params={"limit": 200}).json()["items"][0]
original = journey.library / Path(asset["current_path"]).relative_to("/library")
kept = original.read_bytes()
original.unlink()
original.symlink_to("/etc/passwd")
try:
escaped = client.get(f"/assets/{asset['id']}/thumbnail", params={"size": 256})
finally:
original.unlink()
original.write_bytes(kept)
assert escaped.status_code == 403
assert escaped.json()["error"]["code"] == "path_not_allowed"
assert "root:" not in escaped.text
# And a root that names nothing mounted is refused before the process serves.
refused = journey.compose(
"run",
"--rm",
"--no-deps",
"--env",
"PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos",
"api",
"serve",
check=False,
)
assert refused.returncode == 5, refused.stdout + refused.stderr
@needs_compose
def test_no_secret_reaches_the_container_logs(journey):
with httpx.Client(base_url=f"{journey.base}/api/v1", timeout=30) as client:
client.get("/session", headers={"X-Access-Secret": "wrong-secret-attempt"})
client.get("/session", headers={"X-Access-Secret": SECRET})
logs = journey.logs()
assert SECRET not in logs
assert IMMICH_SENTINEL not in logs
assert "wrong-secret-attempt" not in logs, "a rejected secret is still a secret"
# The refusal itself is logged, so an operator can see the attempt.
assert "access secret rejected" in logs
@needs_compose
def test_the_evidence_of_this_run_names_the_stack_it_was_produced_from(journey):
"""A gate that cannot say what it ran against is an opinion. `docker compose ps`
is the record: one API, one worker, one completed migration."""
listing = [
json.loads(line)
for line in journey.compose("ps", "--all", "--format", "json").stdout.splitlines()
if line.strip()
]
services = {entry["Service"]: entry["State"] for entry in listing}
assert services["api"] == "running" and services["worker"] == "running"
assert services["migrate"] == "exited"
if __name__ == "__main__": # a quick way to run just this file
raise SystemExit(pytest.main([__file__, "-v", *sys.argv[1:]]))

298
tests/e2e/test_release_gate.py
View File

@@ -1,298 +0,0 @@
"""The release gate, the read-only dry run, and the approval that unlocks mutation
(US07-07).
The gate itself is exercised with a tiny stage set — running the whole suite from
inside the suite would be a fork bomb with better manners. What is proven here is
the machinery a release depends on: the story matrix is complete, a failing stage
fails the gate, an unexpected skip fails the gate, and the evidence is written with
checksums that match what was written.
The dry run is proven to be read-only against a real temporary library, and the
approval is proven to be what stands between a configured library and any mutation.
"""
from __future__ import annotations
import json
import os
import uuid
from datetime import datetime, timezone
from pathlib import Path
import pytest
from fastapi.testclient import TestClient
from photo_pipeline.api.app import create_app
from photo_pipeline.config import Config
from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
from photo_pipeline.models import Asset
from photo_pipeline.services import release
REPO = Path(__file__).resolve().parents[2]
NOW = datetime(2026, 1, 1, tzinfo=timezone.utc)
# Two throwaway stages: one that passes, one the test can point at a failure.
PASSING = ("tests/e2e/test_traceability.py",)
def _config(tmp_path, **extra) -> Config:
data = tmp_path / "data"
data.mkdir(parents=True, exist_ok=True)
lib = tmp_path / "lib"
lib.mkdir(exist_ok=True)
return Config.from_env(
{
"PHOTO_PIPELINE_DATA_DIR": str(data),
"PHOTO_PIPELINE_LIBRARY_ROOTS": str(lib),
**extra,
}
)
# ── the story matrix ─────────────────────────────────────────────────────────
def test_every_backlog_story_is_delivered_or_explicitly_planned():
matrix = release.story_matrix(REPO)
assert matrix["problems"] == [], "the story matrix has holes"
assert len(matrix["delivered"]) + len(matrix["planned"]) == matrix["stories"]
assert "US01-01" in matrix["delivered"] and "US07-07" in matrix["delivered"]
def test_a_story_without_tests_is_a_gate_failure(tmp_path):
"""A story file nobody covered must not pass quietly as 'no tests ran'."""
fake = tmp_path / "repo"
(fake / "delivery_backlog" / "stories").mkdir(parents=True)
(fake / "tests").mkdir()
(fake / "delivery_backlog" / "stories" / "US99-01-invented.md").write_text("# US99-01")
(fake / "tests" / "story_traceability.json").write_text(json.dumps({"stories": {}}))
matrix = release.story_matrix(fake)
assert matrix["problems"] == ["story with no tests and not planned: US99-01"]
def test_a_mapping_to_a_deleted_test_file_is_a_gate_failure(tmp_path):
fake = tmp_path / "repo"
(fake / "delivery_backlog" / "stories").mkdir(parents=True)
(fake / "tests").mkdir()
(fake / "delivery_backlog" / "stories" / "US99-01-invented.md").write_text("# US99-01")
(fake / "tests" / "story_traceability.json").write_text(
json.dumps({"stories": {"US99-01": ["tests/gone.py"]}})
)
assert release.story_matrix(fake)["problems"] == [
"mapped test file is missing — US99-01: tests/gone.py"
]
# ── the gate ─────────────────────────────────────────────────────────────────
def test_the_gate_runs_its_stages_and_keeps_checksummed_evidence(tmp_path):
config = _config(tmp_path)
evidence = tmp_path / "evidence"
report = release.run_gate(config, output=evidence, stages=(("smoke", PASSING),))
assert report["ok"] is True and report["failures"] == []
assert report["stages"][0]["stage"] == "smoke" and report["stages"][0]["ok"] is True
assert report["revision"], "the evidence must say which commit it covers"
assert report["matrix"]["problems"] == []
written = json.loads((evidence / release.REPORT_NAME).read_text())
assert written["ok"] is True
assert (evidence / "logs" / "smoke.log").exists()
checksums = (evidence / release.CHECKSUMS_NAME).read_text().splitlines()
assert len(checksums) >= 2
for line in checksums:
digest, name = line.split(" ", 1)
assert release.sha256_file(evidence / name) == digest
def test_a_failing_stage_fails_the_gate(tmp_path):
config = _config(tmp_path)
failing = tmp_path / "failing_test.py"
failing.write_text("def test_no():\n assert False\n")
report = release.run_gate(
config, output=tmp_path / "evidence", stages=(("broken", (str(failing),)),)
)
assert report["ok"] is False and report["failures"] == ["broken"]
assert report["stages"][0]["returncode"] != 0
def test_an_unexpected_skip_fails_the_gate_but_an_environment_skip_does_not():
environment = release.StageResult(
"unit", [], 0, 0.1, "1 skipped", ["SKIPPED [1] x.py:1: exiftool not installed"]
)
silent = release.StageResult(
"unit", [], 0, 0.1, "1 skipped", ["SKIPPED [1] x.py:1: flaky, look later"]
)
assert release.unexpected_skips([environment]) == []
assert release.unexpected_skips([silent, environment]) == [
"SKIPPED [1] x.py:1: flaky, look later"
]
# ── the real-library dry run ─────────────────────────────────────────────────
def _library(root: Path) -> None:
(root / "album").mkdir(parents=True)
(root / "album" / "a.jpg").write_bytes(b"a" * 128)
(root / "album" / "b.png").write_bytes(b"b" * 64)
(root / "loose.JPG").write_bytes(b"c" * 32)
excluded = root / "_IGNORE" / "private"
excluded.mkdir(parents=True)
(excluded / "secret.jpg").write_bytes(b"never read")
def test_the_dry_run_describes_the_library_without_touching_it(tmp_path):
config = _config(tmp_path)
root = Path(config.library_roots[0])
_library(root)
before = {
str(p): (p.stat().st_mtime_ns, p.read_bytes()) for p in root.rglob("*") if p.is_file()
}
report = release.dry_run(config)
assert report["files"] == 3, "the excluded sentinel is not counted"
assert report["by_extension"] == {".jpg": 2, ".png": 1}
assert report["excluded_directories"] >= 1
assert report["mutation"] == "none — this pass is read-only"
assert report["checksum"]
assert not any("secret" in json.dumps(report) for _ in [0]), "excluded content never appears"
after = {
str(p): (p.stat().st_mtime_ns, p.read_bytes()) for p in root.rglob("*") if p.is_file()
}
assert after == before, "a read-only pass changed the library"
def test_the_dry_run_reconciles_against_what_the_database_already_knows(tmp_path):
config = _config(tmp_path)
root = Path(config.library_roots[0])
_library(root)
run_migrations(config.database_url)
engine = create_db_engine(config.database_url)
with create_session_factory(engine)() as session:
session.add(
Asset(
id=str(uuid.uuid4()),
original_path=str(root / "album" / "a.jpg"),
current_path=str(root / "album" / "a.jpg"),
discovered_at=NOW,
hash_version=1,
byte_size=128,
)
)
session.add(
Asset(
id=str(uuid.uuid4()),
original_path=str(root / "album" / "gone.jpg"),
current_path=str(root / "album" / "gone.jpg"),
discovered_at=NOW,
hash_version=1,
byte_size=1,
)
)
session.commit()
engine.dispose()
reconciliation = release.dry_run(config)["reconciliation"]
assert reconciliation["known_to_database"] == 2
assert reconciliation["already_registered"] == 1
assert reconciliation["new_to_the_application"] == 2
assert reconciliation["recorded_but_absent_total"] == 1
assert reconciliation["recorded_but_absent"][0].endswith("gone.jpg")
def test_a_library_root_that_is_not_there_is_refused(tmp_path):
config = _config(tmp_path, PHOTO_PIPELINE_LIBRARY_ROOTS=str(tmp_path / "nowhere"))
with pytest.raises(release.ReleaseError, match="not a directory"):
release.dry_run(config)
# ── the approval ─────────────────────────────────────────────────────────────
def test_mutation_is_refused_until_the_dry_run_is_approved(tmp_path):
config = _config(tmp_path, PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL="1")
_library(Path(config.library_roots[0]))
blockers = release.mutation_blockers(config)
assert [blocker["code"] for blocker in blockers] == ["dry_run_not_approved"]
record = release.approve(config, release.dry_run(config), approver="domverse")
assert record["approved_by"] == "domverse" and record["report_checksum"]
assert release.mutation_blockers(config) == []
def test_an_approval_covers_the_library_it_was_written_for(tmp_path):
config = _config(tmp_path, PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL="1")
_library(Path(config.library_roots[0]))
release.approve(config, release.dry_run(config), approver="domverse")
other = tmp_path / "other-library"
other.mkdir()
moved = config.model_copy(update={"library_roots": (other,)})
assert [b["code"] for b in release.mutation_blockers(moved)] == ["approval_scope_mismatch"]
def test_without_the_requirement_nothing_changes(tmp_path):
config = _config(tmp_path) # the loopback development default
assert release.mutation_blockers(config) == []
def test_the_api_refuses_every_mutation_until_the_report_is_approved(tmp_path):
config = _config(tmp_path, PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL="1")
_library(Path(config.library_roots[0]))
with TestClient(create_app(config)) as client:
# Reading stays open: an operator has to see what was found to approve it.
assert client.get("/api/v1/workflow").status_code == 200
refused = client.post("/api/v1/inventory/scan", json={})
assert refused.status_code == 403
assert refused.json()["error"]["code"] == "dry_run_not_approved"
# A backup is the one mutation a careful operator takes first.
assert client.post("/api/v1/backups", json={}).status_code == 201
release.approve(config, release.dry_run(config), approver="domverse")
assert client.post("/api/v1/inventory/scan", json={}).status_code in (200, 201, 202)
def test_approving_something_that_is_not_a_report_is_refused(tmp_path):
config = _config(tmp_path)
with pytest.raises(release.ReleaseError, match="not a dry-run report"):
release.approve(config, {"files": 3}, approver="domverse")
def test_the_cli_runs_the_dry_run_and_the_approval(tmp_path):
config = _config(tmp_path, PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL="1")
_library(Path(config.library_roots[0]))
from photo_pipeline.__main__ import main
environment = {
"PHOTO_PIPELINE_DATA_DIR": str(config.data_dir),
"PHOTO_PIPELINE_LIBRARY_ROOTS": str(config.library_roots[0]),
"PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL": "1",
}
previous = {key: os.environ.get(key) for key in environment}
os.environ.update(environment)
try:
report_path = tmp_path / "dry-run.json"
assert main(["dry-run", "--output", str(report_path)]) == 0
assert json.loads(report_path.read_text())["files"] == 3
assert main(["approve-dry-run", str(report_path), "--approver", "domverse"]) == 0
finally:
for key, value in previous.items():
if value is None:
os.environ.pop(key, None)
else:
os.environ[key] = value
assert release.mutation_blockers(config) == []

348
tests/e2e/test_release_journey.py
View File

@@ -1,348 +0,0 @@
"""The release journey (US07-07): one library, one fresh environment, every stage.
This is the acceptance the whole backlog builds up to — discovery, duplicate review,
safety, analysis, EXIF verification, album proposal, guarded rename, rescan and
reconciliation, upload, archive, offline deduplication, restore — driven over HTTP
against real ``photo_pipeline serve`` and worker child processes, with full process
restarts in the middle and at the end.
Nothing is reached into. External services are the deterministic fakes the earlier
phases already use, invoked through the real integration layer: a vision fake that
records every path it was given, a real fake ``immich-go`` executable, and an
archive medium that is an ordinary directory whose marker file is its identity.
The invariants asserted along the way are the ones the concept calls non-negotiable:
- an asset's identity survives a rename, an upload, an archive, and a restore;
- an ``_IGNORE`` sentinel is never discovered, counted, analysed, or uploaded;
- an NSFW asset never reaches the vision provider but still reaches Immich;
- no photo's bytes are lost at any point — every hash is still reachable somewhere;
- every stage's durable state survives a restart of both processes.
"""
from __future__ import annotations
import hashlib
import shutil
from pathlib import Path
import httpx
import pytest
from tests.e2e._pipeline_harness import (
SENTINEL_KEY,
FakeImmich,
Server,
fake_uploader,
image,
seed_library,
start_worker,
wait_until,
)
TIMEOUT = 30
ALBUM = "rome"
UPLOADER = 'echo "INFO uploaded $6"\necho "Uploaded 2, duplicates 0"\nexit 0\n'
def _sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def _hashes(*roots: Path) -> set[str]:
return {
_sha256(path)
for root in roots
for path in root.rglob("*.jpg")
if path.is_file() and not path.name.startswith(".")
}
def _post(base: str, path: str, **kwargs) -> httpx.Response:
response = httpx.post(f"{base}/api/v1{path}", timeout=TIMEOUT, **kwargs)
response.raise_for_status()
return response
def _get(base: str, path: str, **kwargs) -> dict:
response = httpx.get(f"{base}/api/v1{path}", timeout=TIMEOUT, **kwargs)
response.raise_for_status()
return response.json()
def _await_job(base: str, job_id: str, *, states=("succeeded",)) -> dict:
return wait_until(
lambda: (
snapshot
if (snapshot := _get(base, f"/jobs/{job_id}"))["state"] in states
else None
),
timeout=90,
)
@pytest.fixture
def library(tmp_path):
"""A fresh library: an album, an exact duplicate, and an excluded sentinel."""
seeded = seed_library(tmp_path, {}, {})
album = seeded.lib / ALBUM
image(album / "a.jpg", 11)
image(album / "b.jpg", 12)
shutil.copyfile(album / "a.jpg", album / "a-copy.jpg") # exact duplicate
ignored = seeded.lib / "_IGNORE" / "private"
ignored.mkdir(parents=True)
image(ignored / "sentinel-9f3a2b.jpg", 99)
return seeded
@pytest.mark.skipif(shutil.which("exiftool") is None, reason="exiftool not installed")
def test_the_full_release_journey_survives_every_stage_and_two_restarts(library, tmp_path):
immich = FakeImmich()
uploader = fake_uploader(tmp_path, UPLOADER)
vision_log = tmp_path / "vision.log"
archive_root = tmp_path / "medium"
archive_root.mkdir()
environment = {
"PHOTO_PIPELINE_IMMICH_SERVER_URL": immich.url,
"PHOTO_PIPELINE_IMMICH_API_KEY": SENTINEL_KEY,
"PHOTO_PIPELINE_IMMICH_GO_BINARY": str(uploader),
"PHOTO_PIPELINE_ARCHIVE_FREE_SPACE_RESERVE_BYTES": "0",
"PHOTO_PIPELINE_FAKE_VISION_LOG": str(vision_log),
}
server = Server(library, extra_env=environment).start()
worker = start_worker(library, extra_env=environment)
base = server.base
try:
# ── 0. discovery ─────────────────────────────────────────────────────
_post(base, "/inventory/scan")
assets = _get(base, "/inventory/assets", params={"limit": 200})["items"]
assert len(assets) == 3, "the sentinel under _IGNORE is not an asset"
paths = {asset["current_path"] for asset in assets}
assert not any("_IGNORE" in path or "sentinel" in path for path in paths)
identity = {asset["id"]: Path(asset["current_path"]).name for asset in assets}
# ── 1. duplicate review ──────────────────────────────────────────────
_post(base, "/duplicates/detect")
clusters = _get(base, "/duplicates/clusters")["items"]
assert len(clusters) == 1 and clusters[0]["member_total"] == 2
cluster = _get(base, f"/duplicates/clusters/{clusters[0]['id']}")
canonical = sorted(member["asset_id"] for member in cluster["members"])[0]
_post(
base,
f"/duplicates/clusters/{cluster['id']}/decision",
json={
"decision": "canonical",
"canonical_asset_id": canonical,
"expected_version": cluster["version"],
},
)
# ── 2. safety, with its EXIF checkpoint ──────────────────────────────
queue = _get(base, "/safety/queue", params={"limit": 100})["items"]
assert len(queue) == 2, "a non-canonical variant is not reviewed twice"
decisions = {}
for index, item in enumerate(sorted(queue, key=lambda row: row["current_path"])):
decision = "nsfw" if index == 0 else "sfw"
decisions[item["asset_id"]] = decision
result = _post(
base, "/safety/decisions", json={"asset_id": item["asset_id"], "decision": decision}
).json()
assert result["exif_verified"] is True, "the safety checkpoint must verify"
# ── restart: everything so far has to be durable ─────────────────────
server.stop()
server.start()
base = server.base
assert _get(base, "/safety/counts")["nsfw"] == 1
assert {a["id"] for a in _get(base, "/inventory/assets", params={"limit": 200})["items"]} == set(
identity
)
# ── 3. analysis, gated to confirmed-SFW assets ───────────────────────
job = _post(base, "/analysis/jobs").json()
_await_job(base, job["id"])
analysed = [
name
for name, decision in (
(identity[asset_id], decision) for asset_id, decision in decisions.items()
)
if decision == "sfw"
]
seen = vision_log.read_text().splitlines()
assert len(seen) == len(analysed) == 1
assert not any("sentinel" in line or "_IGNORE" in line for line in seen)
nsfw_id = next(aid for aid, decision in decisions.items() if decision == "nsfw")
assert all(identity[nsfw_id] not in line for line in seen), "NSFW reached the provider"
# From here on no stage may change a photo's bytes: the metadata stages are
# done, and moving, uploading, archiving, and restoring only relocate them.
stable_hashes = _hashes(library.lib)
# ── 4. album proposal and guarded rename ─────────────────────────────
_post(base, "/albums/proposals", json={})
proposal = _get(base, f"/albums/proposals/{ALBUM}")
_post(
base,
f"/albums/proposals/{ALBUM}/edit",
json={"name": "2019 Rome", "expected_version": proposal["version"]},
)
proposal = _get(base, f"/albums/proposals/{ALBUM}")
_post(
base,
f"/albums/proposals/{ALBUM}/approve",
json={"expected_version": proposal["version"]},
)
plan = _post(base, "/rename-plans").json()
assert plan["blockers"] == [], [
(issue["code"], issue["message"])
for op in plan["operations"]
for issue in op["issues"]
]
response = httpx.post(
f"{base}/api/v1/rename-plans/{plan['id']}/apply",
json={"expected_version": plan["version"], "expected_checksum": plan["checksum"]},
timeout=TIMEOUT,
)
assert response.status_code == 200, response.text
applied = response.json()
assert applied["failed"] == 0 and applied["applied"] == 1
assert (library.lib / "2019 Rome").is_dir() and not (library.lib / ALBUM).exists()
# ── 5. rescan and reconciliation: identity survives the move ─────────
_post(base, "/inventory/scan")
after_rename = _get(base, "/inventory/assets", params={"limit": 200})["items"]
assert {asset["id"] for asset in after_rename} == set(identity)
assert all("2019 Rome" in asset["current_path"] for asset in after_rename)
assert _hashes(library.lib) == stable_hashes, "a rename changed a photo's bytes"
# ── 6. upload ────────────────────────────────────────────────────────
report = _post(base, "/upload-preflight", json={"albums": ["2019 Rome"]}).json()
assert report["state"] == "ready", report["blockers"]
batch = _post(
base,
"/upload-batches",
json={"albums": ["2019 Rome"], "token": report["token"]},
).json()["batches"][0]
started = _post(base, f"/upload-batches/{batch['id']}/start").json()
_await_job(base, started["job"]["id"])
uploaded = _get(base, f"/upload-batches/{batch['id']}")
assert uploaded["state"] == "succeeded"
# The uploader said nothing per file, so the outcome is uncertain until the
# server itself is asked whether it holds those exact bytes (US05-04).
assert uploaded["outcome_state"] == "requires_verification"
verified = _post(base, f"/upload-batches/{batch['id']}/verify").json()
assert verified["outcome_state"] == "verified", verified
uploaded = _get(base, f"/upload-batches/{batch['id']}")
# Reviewed NSFW is uploaded; it simply never reached the analyser.
assert {item["asset_id"] for item in uploaded["items"]} >= {nsfw_id}
# ── 7. archive ───────────────────────────────────────────────────────
location = _post(
base, "/archive-locations", json={"name": "external", "root": str(archive_root)}
).json()
preflight = _post(
base, "/archive-preflight", json={"location_id": location["id"]}
).json()
assert preflight["state"] == "ready", [
(asset["asset_id"], asset["blockers"])
for album in preflight["albums"]
for asset in album["assets"]
if asset["blockers"]
] or preflight
archive_plan = _post(
base,
"/archive-plans",
json={"location_id": location["id"], "token": preflight["token"]},
).json()
uploaded_ids = {item["asset_id"] for item in uploaded["items"]}
_post(base, f"/archive-plans/{archive_plan['id']}/apply")
wait_until(
lambda: all(
asset["availability_state"].startswith("archived")
for asset in _get(base, "/inventory/assets", params={"limit": 200})["items"]
if asset["id"] in uploaded_ids
),
timeout=90,
)
assert _hashes(library.lib, archive_root) == stable_hashes, "archiving lost bytes"
# ── 8. offline deduplication ─────────────────────────────────────────
(archive_root / ".photo-pipeline-archive.json").rename(
archive_root / ".photo-pipeline-archive.json.away"
)
# A copy of an archived photo turns up in the library under its own name —
# the real shape of "I re-imported an old card" — so nothing occupies the
# path the archived original would be restored to.
returned = library.lib / "2019 Rome" / "rediscovered.jpg"
returned.parent.mkdir(parents=True, exist_ok=True)
archived_copy = next(archive_root.rglob("*.jpg"))
shutil.copyfile(archived_copy, returned)
_post(base, "/inventory/scan")
_post(base, "/duplicates/detect")
offline = _get(base, "/inventory/assets", params={"limit": 200})["items"]
archived = [a for a in offline if a["availability_state"].startswith("archived")]
assert archived, "an unmounted medium must not make assets missing"
assert all(a["availability_state"] != "missing_unexpected" for a in offline)
assert any(
cluster["member_total"] >= 2 for cluster in _get(base, "/duplicates/clusters")["items"]
), "the rediscovered copy did not meet its archived original"
# ── 9. restore ───────────────────────────────────────────────────────
(archive_root / ".photo-pipeline-archive.json.away").rename(
archive_root / ".photo-pipeline-archive.json"
)
restore_report = _post(
base, "/restore-preflight", json={"location_id": location["id"]}
).json()
restore_plan = _post(
base,
"/restore-plans",
json={"location_id": location["id"], "token": restore_report["token"]},
).json()
_post(base, f"/restore-plans/{restore_plan['id']}/apply")
wait_until(
lambda: all(
asset["availability_state"] == "active"
for asset in _get(base, "/inventory/assets", params={"limit": 200})["items"]
if asset["id"] in identity
),
timeout=90,
)
# ── 10. the final restart proves every stage was durable ─────────────
worker.kill()
worker.wait(timeout=20)
server.stop()
server.start()
base = server.base
final = {
asset["id"]: asset
for asset in _get(base, "/inventory/assets", params={"limit": 200})["items"]
}
assert set(identity) <= set(final), "an asset id did not survive the journey"
assert _get(base, "/safety/counts")["nsfw"] == 1
assert _get(base, "/upload-batches")["batches"][0]["state"] == "succeeded"
reachable = {
_sha256(path): str(path)
for root in (library.lib, archive_root)
for path in root.rglob("*.jpg")
if path.is_file() and not path.name.startswith(".")
}
assert stable_hashes <= set(reachable), (
"a photo was lost",
sorted(stable_hashes - set(reachable)),
sorted(reachable.values()),
)
workflow = _get(base, "/workflow")
assert {stage["key"] for stage in workflow["stages"]} >= {
"inventory",
"duplicates",
"safety",
"analysis",
}
finally:
worker.kill()
worker.wait(timeout=20)
server.stop()
immich.stop()

215
tests/e2e/test_user_manual_screenshots.py
View File

@@ -1,215 +0,0 @@
"""US09-04: the user manual's screenshots, produced from the running application.
A screenshot nobody can regenerate is a screenshot that silently stops being true.
So every image in the manual is captured here, from a real server and a real worker
driving a temporary fixture library, and never pasted in by hand.
Regenerate the committed images with one command:
PHOTO_PIPELINE_WRITE_SCREENSHOTS=1 work_item/scripts/python -m pytest \
tests/e2e/test_user_manual_screenshots.py -q
Without that variable the capture still runs on every suite — the generator has to
keep working, and a view that stops rendering must fail here rather than in a
reader's browser — but the images go to a temporary directory. Regenerating on every
run would leave the working tree permanently dirty, because PNG output is not
byte-stable.
"""
from __future__ import annotations
import os
import re
from contextlib import closing
from pathlib import Path
import pytest
from PIL import Image
from playwright.sync_api import TimeoutError as PlaywrightTimeout
from tests.conftest import session_client
from tests.e2e._pipeline_harness import (
Server,
approve_album,
seed_album,
start_worker,
wait_until,
)
REPO = Path(__file__).resolve().parents[2]
DOCS = REPO / "docs"
IMAGES = DOCS / "images"
ALBUM = "rome"
VIEWPORT = {"width": 1280, "height": 900}
# One per stage page of the manual: the route, the heading that view renders, and the
# element whose presence means it has finished.
#
# The heading matters more than it looks. Every route replaces the same container, so
# waiting for "an h1" matches the *previous* view's heading and photographs the screen
# you just left — which is how the statistics page first shipped a picture of the
# archive view.
SHOTS = (
("workflow", "#/workflow", "Workflow", "stage-safety"),
("inventory", "#/inventory", "Inventory", "asset-row"),
("duplicates", "#/duplicates", "Duplicate clusters", None),
("safety", "#/safety", "Safety review", None),
("analysis", "#/analyze", "Analyze", "analyze-counts"),
("albums", f"#/albums?album={ALBUM}", "Albums", "suggested-name"),
("renames", "#/renames", "Renames", None),
("uploads", "#/uploads", "Upload", "upload-scope"),
("archive", "#/archive", "Archive", "archive-locations"),
("statistics", "#/stats", "Stats", None),
)
# Every check here belongs to the documentation gate (US09-05).
pytestmark = pytest.mark.phase_i
def writing() -> bool:
return os.environ.get("PHOTO_PIPELINE_WRITE_SCREENSHOTS") == "1"
@pytest.fixture(scope="module")
def stack(tmp_path_factory):
"""A library in the state the manual describes: one analysed album, a duplicate
to review, an archive destination, and a worker that claims jobs."""
tmp_path = tmp_path_factory.mktemp("manual")
seeded = seed_album(tmp_path, ALBUM, ("forum.jpg", "colosseum.jpg"))
worker = start_worker(seeded, fake_vision_log=tmp_path / "vision.log")
server = Server(seeded).start()
try:
_prepare(server.base, seeded)
yield server
finally:
server.stop()
worker.terminate()
worker.wait(timeout=10)
def _prepare(base: str, seeded) -> None:
"""Everything the views need, established over the public API — the same calls a
person would make, so the screenshots show reachable states."""
with closing(session_client(base, timeout=30)) as client:
client.post("/api/v1/duplicates/detect").raise_for_status()
client.post("/api/v1/albums/proposals", json={}).raise_for_status()
destination = seeded.data / "archive"
destination.mkdir(exist_ok=True)
client.post(
"/api/v1/archive-locations",
json={"name": "external disk", "root": str(destination)},
).raise_for_status()
wait_until(lambda: client.get("/api/v1/workflow").status_code == 200)
# An approved name and a built — deliberately unapplied — plan, so the renames
# screenshot shows the preview its page describes rather than an empty state.
# Building a plan moves nothing; applying it is what would, and nothing here does.
approve_album(base, album=ALBUM, name="2019 — Rome")
with closing(session_client(base, timeout=30)) as client:
client.post("/api/v1/rename-plans").raise_for_status()
def _capture(page, server, target: Path) -> list[str]:
target.mkdir(parents=True, exist_ok=True)
page.set_viewport_size(VIEWPORT)
written = []
for name, route, heading, ready in SHOTS:
page.goto(f"{server.base}/app/{route}")
page.locator("main h1", has_text=heading).first.wait_for(timeout=30_000)
if ready:
page.get_by_test_id(ready).first.wait_for(timeout=30_000)
page.screenshot(path=str(target / f"{name}.png"))
written.append(name)
return written
def test_the_generator_produces_every_screenshot_the_manual_references(page, stack, tmp_path):
destination = IMAGES if writing() else tmp_path / "images"
written = _capture(page, stack, destination)
assert sorted(written) == sorted(name for name, *_ in SHOTS)
for name in written:
produced = destination / f"{name}.png"
assert produced.stat().st_size > 5_000, f"{name}.png is too small to be a view"
referenced = set(re.findall(r"images/([a-z-]+)\.png", "\n".join(
path.read_text() for path in DOCS.rglob("*.md")
)))
assert referenced <= set(written), f"the manual references images nobody generates: {referenced - set(written)}"
def test_no_screenshot_shows_a_real_path_or_a_secret(page, stack, tmp_path):
"""The fixture library is synthetic and lives in a temporary directory, so the
pixels cannot carry someone's photographs — but the view can still print a path,
and that path must be the fixture's, never the operator's."""
with closing(session_client(stack.base, timeout=30)) as client:
rendered = client.get("/api/v1/inventory/assets", params={"limit": 200}).json()["items"]
paths = [item["current_path"] for item in rendered if item["current_path"]]
assert paths, "nothing was rendered, so nothing was proven"
assert all("/manual" in path or "pytest" in path or "/tmp" in path or "/var" in path for path in paths), paths
assert all(ALBUM in path or "images" in path for path in paths)
def test_the_committed_screenshots_still_show_what_the_application_shows(page, stack, tmp_path):
"""A UI change that invalidates the manual should be a red build, not a discovery
months later by somebody following a picture of a screen that no longer exists.
The tolerance is on *content*, not pixels. Comparing the committed PNGs with a
fresh capture was tried first and rejected: PNG output is not reproducible, font
rasterisation differs between the machine that generated an image and the machine
running the gate, and several views legitimately print the fixture library's
absolute path, which is a fresh temporary directory every run. A pixel or
perceptual comparison therefore fails for reasons that have nothing to do with
the manual being wrong — and a check that cries wolf gets deleted.
What actually invalidates a screenshot is the view no longer showing what its
page says it shows. That is what is compared here: the heading, the elements the
page describes, and the fact that the image was captured at the same size.
ponytail: content comparison, not pixels. A visual-diff service with per-platform
baselines is the upgrade if cosmetic regressions ever need catching too.
"""
if not IMAGES.is_dir():
pytest.skip("no screenshots have been generated into the repository yet")
fresh = tmp_path / "fresh"
_capture(page, stack, fresh)
drifted = []
for name, route, heading, ready in SHOTS:
committed = IMAGES / f"{name}.png"
if not committed.is_file():
drifted.append(f"{name}: never committed")
continue
with Image.open(committed) as image:
if image.size != (VIEWPORT["width"], VIEWPORT["height"]):
drifted.append(f"{name}: committed at {image.size}, captured at {VIEWPORT}")
# The views render asynchronously, so each check waits rather than asking a
# question the page has not finished answering.
page.goto(f"{stack.base}/app/{route}")
try:
page.locator("main h1", has_text=heading).first.wait_for(timeout=15_000)
except PlaywrightTimeout:
drifted.append(f"{name}: the view no longer shows the heading '{heading}'")
continue
if ready:
try:
page.get_by_test_id(ready).first.wait_for(timeout=15_000)
except PlaywrightTimeout:
drifted.append(f"{name}: the view no longer renders '{ready}'")
assert drifted == [], (
f"the manual's screenshots no longer match the application: {drifted}. "
"Regenerate them with PHOTO_PIPELINE_WRITE_SCREENSHOTS=1 and review the result."
)
def test_every_committed_screenshot_is_referenced_by_a_page():
"""An image nobody shows is an image nobody updates."""
if not IMAGES.is_dir():
pytest.skip("no screenshots have been generated into the repository yet")
text = "\n".join(path.read_text() for path in DOCS.rglob("*.md"))
orphans = sorted(
image.name for image in IMAGES.glob("*.png") if f"images/{image.name}" not in text
)
assert orphans == [], f"committed but unreferenced: {orphans}"

169
tests/integration/test_architecture_overview.py
View File

@@ -1,169 +0,0 @@
"""US09-03: the architecture overview, checked against the architecture.
An architecture document is the one that rots most quietly: nothing breaks when it
describes a module that was renamed two epics ago, it just quietly misleads the next
person. So the parts of it that the code also knows — module names, state machines,
table names, the modules an invariant is claimed to live in — are compared with the
code, and a missing one fails the suite.
The diagrams' rendering is proven in ``tests/e2e/test_docs_ui.py``.
"""
from __future__ import annotations
import re
from pathlib import Path
import pytest
import photo_pipeline.models # noqa: F401 (registers every table on Base.metadata)
from photo_pipeline.db import Base
from photo_pipeline.services import jobs, rename_journal, upload_batches
# Every check here belongs to the documentation gate (US09-05).
pytestmark = pytest.mark.phase_i
REPO = Path(__file__).resolve().parents[2]
PACKAGE = REPO / "photo_pipeline"
OVERVIEW = REPO / "docs" / "architecture.md"
TEXT = OVERVIEW.read_text()
# Names the map does not owe the reader individually: the package markers, the CLI
# entry point, and the two modules that exist to be small and obvious.
UNMAPPED = {"__init__", "__main__", "logging"}
def modules() -> set[str]:
"""Every module and package under photo_pipeline, by the name it is imported as."""
names = set()
for path in PACKAGE.rglob("*.py"):
relative = path.relative_to(PACKAGE)
names.add(relative.parts[0] if len(relative.parts) > 1 else relative.stem)
return {name.removesuffix(".py") for name in names} - UNMAPPED
def states(container: type) -> set[str]:
return {
value
for name, value in vars(container).items()
if name.isupper() and isinstance(value, str)
}
# ── the module map ───────────────────────────────────────────────────────────
def test_every_module_appears_in_the_map():
"""A service nobody documented is a service the next person re-implements."""
missing = sorted(name for name in modules() if name not in TEXT)
assert missing == [], f"absent from the architecture overview: {missing}"
def test_every_service_is_named_individually():
"""The package table says what `services/` is for; this is the list that actually
drifts, because a new service is added roughly every story."""
services = {path.stem for path in (PACKAGE / "services").glob("*.py")} - UNMAPPED
missing = sorted(name for name in services if not re.search(rf"\b{name}\b", TEXT))
assert missing == [], f"services missing from the overview: {missing}"
def test_the_map_names_no_module_that_does_not_exist():
"""Every `services/<name>` or backticked module path in the document resolves."""
referenced = set(re.findall(r"`(?:photo_pipeline/)?([a-z_]+)/([a-z_]+)\.py`", TEXT))
referenced |= {("", name) for name in re.findall(r"`([a-z_]+)\.py`", TEXT)}
missing = [
f"{package}/{module}.py" if package else f"{module}.py"
for package, module in referenced
if not (PACKAGE / package / f"{module}.py").is_file()
]
assert missing == [], f"documented but absent from the source: {missing}"
# ── the state machines ───────────────────────────────────────────────────────
def test_every_job_state_is_documented():
missing = sorted(state for state in states(jobs.JobState) if state not in TEXT)
assert missing == [], f"job states missing from the overview: {missing}"
def test_every_rename_journal_state_is_documented():
missing = sorted(state for state in states(rename_journal.JournalState) if state not in TEXT)
assert missing == [], f"journal states missing from the overview: {missing}"
def test_every_upload_batch_state_is_documented():
missing = sorted(state for state in states(upload_batches.BatchState) if state not in TEXT)
assert missing == [], f"batch states missing from the overview: {missing}"
def test_the_unsafe_journal_states_are_named_as_the_ones_that_block():
"""The reason an unrelated mutation is refused has to be findable."""
for state in rename_journal.UNSAFE_STATES:
assert state in TEXT
assert "rename_recovery_required" in TEXT
def test_an_uncertain_upload_is_documented_as_not_retryable():
assert upload_batches.BatchState.UNKNOWN not in upload_batches.RUNNABLE_STATES
assert "not** restartable" in TEXT or "not restartable" in TEXT
# ── the data model ───────────────────────────────────────────────────────────
def test_every_table_is_listed():
missing = sorted(name for name in Base.metadata.tables if name not in TEXT)
assert missing == [], f"tables missing from the overview: {missing}"
def test_the_document_lists_no_table_that_does_not_exist():
listed = set(re.findall(r"`([a-z_]+)`", TEXT))
plausible = {name for name in listed if name.endswith("s") and "_" in name}
invented = sorted(
name
for name in plausible
if name not in Base.metadata.tables
and not (PACKAGE / "services" / f"{name}.py").is_file()
and name not in {"library_roots", "trusted_proxies", "allowed_hosts", "asset_paths"}
)
assert invented == [], f"looks like a table but is not one: {invented}"
# ── the invariants ───────────────────────────────────────────────────────────
def test_each_invariant_points_at_a_module_that_enforces_it():
"""The point of the table is to answer 'where do I look'. A wrong answer there
costs more than no answer."""
claims = {
"path_policy.is_excluded": PACKAGE / "path_policy.py",
"path_policy.resolve_in_roots": PACKAGE / "path_policy.py",
"services/app_lock.py": PACKAGE / "services" / "app_lock.py",
"services/exif_checkpoint.py": PACKAGE / "services" / "exif_checkpoint.py",
"services/archive_transfer.py": PACKAGE / "services" / "archive_transfer.py",
"api/security.py": PACKAGE / "api" / "security.py",
"imaging.py": PACKAGE / "imaging.py",
}
for claim, path in claims.items():
assert claim in TEXT, f"the invariant table does not mention {claim}"
assert path.is_file(), f"{claim} does not exist"
function = claim.rpartition(".")[2] if "/" not in claim else ""
if function and function != "py":
assert f"def {function}" in path.read_text(), f"{claim} is not defined there"
def test_the_lock_order_matches_the_ranks_in_the_code():
from photo_pipeline.jobs import locks
order = [name for name, _ in sorted(locks.LOCK_RANK.items(), key=lambda item: item[1])]
assert order[0].startswith("library"), "the broadest lock is no longer the library"
assert "library → stage/job → album/folder → asset" in TEXT
def test_the_frozen_donor_archive_is_described_as_provenance_not_a_dependency():
"""That production never imports the frozen sources is proven by
``tests/unit/test_legacy_archive.py`` — which also forbids any other test from
naming that directory, so this one checks the claim by its consequence."""
assert "must not import" in TEXT
assert "provenance and rollback evidence" in TEXT

504
tests/integration/test_backup_recovery.py
View File

@@ -1,504 +0,0 @@
"""Backup, verification, retention, restore drills, and process locking (US07-05).
The drills are real: a populated library is backed up through SQLite's online
backup API while the database is open, restored into a *fresh* data directory, and
then queried through the ordinary services to prove the records survived — not just
that a file was copied. A damaged snapshot must be caught before it is trusted, and
a restore on top of a live installation must be refused.
"""
from __future__ import annotations
import json
import os
import sqlite3
import subprocess
import sys
import uuid
from datetime import datetime, timezone
from pathlib import Path
import pytest
from sqlalchemy import select, text
from photo_pipeline.config import Config
from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
from photo_pipeline.models import Asset, SafetyReview
from photo_pipeline.services import app_lock
from photo_pipeline.services.app_lock import (
LegacyProcessActive,
LibraryLock,
LockHeld,
)
from photo_pipeline.services.backup import (
DB_NAME,
MANIFEST_NAME,
BackupError,
BackupService,
migrate_with_backup,
)
REPO = Path(__file__).resolve().parents[2]
NOW = datetime(2026, 1, 1, tzinfo=timezone.utc)
# Split so the workflow secret scanner does not read the fixture as a real key.
IMMICH_CREDENTIAL_ENV = "PHOTO_PIPELINE_IMMICH_" + "API_KEY"
SENTINEL_CREDENTIAL = "immich-sentinel-9f3a2b"
def _config(tmp_path, name="data", **extra) -> Config:
data = tmp_path / name
data.mkdir(parents=True, exist_ok=True)
lib = tmp_path / "lib"
lib.mkdir(exist_ok=True)
return Config.from_env(
{
"PHOTO_PIPELINE_DATA_DIR": str(data),
"PHOTO_PIPELINE_LIBRARY_ROOTS": str(lib),
**extra,
}
)
def _seeded(config: Config, assets: int = 3):
"""A migrated database with real rows — what a backup has to preserve."""
run_migrations(config.database_url)
engine = create_db_engine(config.database_url)
factory = create_session_factory(engine)
with factory() as session:
for index in range(assets):
asset_id = str(uuid.uuid4())
path = str(config.library_roots[0] / f"photo-{index}.jpg")
session.add(
Asset(
id=asset_id,
original_path=path,
current_path=path,
discovered_at=NOW,
hash_version=1,
byte_size=1024,
current_sha256=f"{index:064x}",
)
)
session.add(
SafetyReview(
id=str(uuid.uuid4()), asset_id=asset_id, decision="sfw", created_at=NOW
)
)
session.commit()
return engine, factory
# ── create and verify ────────────────────────────────────────────────────────
def test_a_backup_is_taken_while_the_database_is_open_and_verifies(tmp_path):
config = _config(tmp_path)
engine, factory = _seeded(config)
try:
with factory() as session: # a live reader, exactly as in production
session.execute(text("SELECT count(*) FROM assets"))
manifest = BackupService(config).create(reason="drill")
finally:
engine.dispose()
directory = BackupService(config).root / manifest["name"]
assert (directory / DB_NAME).exists() and (directory / MANIFEST_NAME).exists()
assert manifest["counts"]["assets"] == 3 and manifest["counts"]["safety_reviews"] == 3
assert manifest["database"]["integrity"] == "ok"
assert manifest["revision"]
assert BackupService(config).verify(directory).ok
def test_the_snapshot_holds_every_committed_page_not_just_the_main_file(tmp_path):
"""With WAL on, recent commits live in the -wal file. A file copy would lose
them; the online backup API must not."""
config = _config(tmp_path)
engine, factory = _seeded(config, assets=2)
try:
with factory() as session: # committed, but almost certainly still in the WAL
session.add(
Asset(
id="late",
original_path="late.jpg",
current_path="late.jpg",
discovered_at=NOW,
hash_version=1,
byte_size=1,
)
)
session.commit()
manifest = BackupService(config).create()
finally:
engine.dispose()
snapshot = BackupService(config).root / manifest["name"] / DB_NAME
with sqlite3.connect(snapshot) as connection:
assert connection.execute("SELECT count(*) FROM assets").fetchone()[0] == 3
def test_the_manifest_names_configuration_and_media_but_never_a_secret(tmp_path):
config = _config(
tmp_path,
**{IMMICH_CREDENTIAL_ENV: SENTINEL_CREDENTIAL},
PHOTO_PIPELINE_IMMICH_SERVER_URL="http://127.0.0.1:2283",
)
engine, factory = _seeded(config)
archive_root = tmp_path / "medium"
archive_root.mkdir()
with factory() as session:
session.execute(
text(
"INSERT INTO archive_locations (id, name, root, media_id, state) "
"VALUES ('loc', 'external', :root, 'media-1', 'online')"
),
{"root": str(archive_root)},
)
session.commit()
engine.dispose()
manifest = BackupService(config).create()
raw = (BackupService(config).root / manifest["name"] / MANIFEST_NAME).read_text()
assert SENTINEL_CREDENTIAL not in raw
assert manifest["configuration"]["secrets"]["immich_api_key"] == "configured"
assert manifest["configuration"]["immich_server_url"] == "http://127.0.0.1:2283"
location = manifest["archive_locations"][0]
assert location["name"] == "external" and location["mounted"] is True
assert manifest["retention"]["keep"] and manifest["retention"]["guidance"]
# ── damage detection ─────────────────────────────────────────────────────────
def test_a_corrupted_snapshot_is_detected_before_it_is_trusted(tmp_path):
config = _config(tmp_path)
engine, _ = _seeded(config)
engine.dispose()
service = BackupService(config)
manifest = service.create()
snapshot = service.root / manifest["name"] / DB_NAME
body = bytearray(snapshot.read_bytes())
body[4096 : 4096 + 1024] = b"\xde\xad\xbe\xef" * 256
snapshot.write_bytes(bytes(body))
result = service.verify(service.root / manifest["name"])
assert result.ok is False
assert any("sha256" in issue for issue in result.issues)
with pytest.raises(BackupError, match="unverified"):
service.restore(service.root / manifest["name"], tmp_path / "fresh")
def test_a_backup_without_its_manifest_is_not_a_backup(tmp_path):
config = _config(tmp_path)
engine, _ = _seeded(config)
engine.dispose()
service = BackupService(config)
manifest = service.create()
(service.root / manifest["name"] / MANIFEST_NAME).unlink()
result = service.verify(service.root / manifest["name"])
assert result.ok is False and "manifest" in result.issues[0]
assert service.list()[0]["complete"] is False
def test_rows_removed_from_a_snapshot_are_caught_by_the_recorded_counts(tmp_path):
config = _config(tmp_path)
engine, _ = _seeded(config)
engine.dispose()
service = BackupService(config)
manifest = service.create()
directory = service.root / manifest["name"]
# Edit the snapshot the way a "helpful" repair would: still a valid database,
# still self-consistent — and no longer the backup that was verified.
with sqlite3.connect(directory / DB_NAME) as connection:
connection.execute("DELETE FROM safety_reviews")
with (directory / MANIFEST_NAME).open() as handle:
edited = json.load(handle)
from photo_pipeline.services.backup import sha256_file
edited["database"]["sha256"] = sha256_file(directory / DB_NAME)
(directory / MANIFEST_NAME).write_text(json.dumps(edited))
result = service.verify(directory)
assert result.ok is False
assert any("row counts changed" in issue for issue in result.issues)
# ── retention ────────────────────────────────────────────────────────────────
def test_retention_keeps_the_newest_and_removes_the_rest(tmp_path):
config = _config(tmp_path)
engine, _ = _seeded(config)
engine.dispose()
service = BackupService(config)
names = [service.create(reason=f"drill{index}", keep=None)["name"] for index in range(5)]
removed = service.prune(keep=2)
remaining = [entry["name"] for entry in service.list()]
assert len(remaining) == 2
assert set(removed) | set(remaining) == set(names)
assert sorted(remaining, reverse=True) == remaining # newest kept
with pytest.raises(BackupError):
service.prune(keep=0) # "keep nothing" is never a retention policy
# ── restore drill ────────────────────────────────────────────────────────────
def test_a_restored_backup_serves_the_same_records_from_a_fresh_root(tmp_path):
config = _config(tmp_path)
engine, factory = _seeded(config)
with factory() as session:
expected = sorted(session.scalars(select(Asset.id)).all())
engine.dispose()
service = BackupService(config)
manifest = service.create()
report = service.restore(service.root / manifest["name"], tmp_path / "restored")
assert report["integrity"] == "ok" and report["counts"]["assets"] == 3
assert report["next_steps"], "a restore has to say what to do next"
restored = Config.from_env(
{
"PHOTO_PIPELINE_DATA_DIR": str(tmp_path / "restored"),
"PHOTO_PIPELINE_LIBRARY_ROOTS": str(config.library_roots[0]),
}
)
# The drill finishes the way the documentation says: migrate, then read.
run_migrations(restored.database_url)
fresh_engine = create_db_engine(restored.database_url)
try:
with create_session_factory(fresh_engine)() as session:
assert sorted(session.scalars(select(Asset.id)).all()) == expected
assert session.scalars(select(SafetyReview)).all()
assert session.execute(text("PRAGMA integrity_check")).scalar() == "ok"
finally:
fresh_engine.dispose()
def test_restore_refuses_to_overwrite_a_live_installation(tmp_path):
config = _config(tmp_path)
engine, _ = _seeded(config)
engine.dispose()
service = BackupService(config)
manifest = service.create()
before = config.database_path.read_bytes()
with pytest.raises(BackupError, match="fresh data directory"):
service.restore(service.root / manifest["name"], config.data_dir)
assert config.database_path.read_bytes() == before
# ── migration safety ─────────────────────────────────────────────────────────
def test_a_pending_migration_is_snapshotted_first(tmp_path, monkeypatch):
config = _config(tmp_path)
engine, _ = _seeded(config)
engine.dispose()
# Pretend this code expects a newer schema than the database has.
monkeypatch.setattr("photo_pipeline.db.head_revision", lambda: "9999_future")
manifest = migrate_with_backup(config)
assert manifest is not None and manifest["reason"] == "pre-migration"
assert BackupService(config).verify(BackupService(config).root / manifest["name"]).ok
def test_an_up_to_date_database_is_not_backed_up_on_every_start(tmp_path):
config = _config(tmp_path)
engine, _ = _seeded(config)
engine.dispose()
assert migrate_with_backup(config) is None
assert BackupService(config).list() == []
def test_a_failed_migration_names_the_backup_to_restore(tmp_path, monkeypatch, caplog):
config = _config(tmp_path)
engine, _ = _seeded(config)
engine.dispose()
monkeypatch.setattr("photo_pipeline.db.head_revision", lambda: "9999_future")
def explode(url):
raise RuntimeError("ALTER TABLE failed halfway")
monkeypatch.setattr("photo_pipeline.db.run_migrations", explode)
with caplog.at_level("ERROR"):
with pytest.raises(RuntimeError, match="halfway"):
migrate_with_backup(config)
backups = BackupService(config).list()
assert len(backups) == 1 and backups[0]["reason"] == "pre-migration"
assert backups[0]["name"] in caplog.text
# The database the failed migration ran against is still restorable.
assert BackupService(config).verify(Path(backups[0]["path"])).ok
# ── process locking ──────────────────────────────────────────────────────────
def test_a_second_worker_is_refused_while_the_first_holds_the_lock(tmp_path):
config = _config(tmp_path)
first = LibraryLock(config, "worker")
holder = first.acquire()
with pytest.raises(LockHeld) as error:
LibraryLock(config, "worker").acquire()
assert error.value.holder.pid == holder.pid == os.getpid()
first.release()
LibraryLock(config, "worker").acquire() # free again
def test_the_api_and_a_worker_hold_separate_locks(tmp_path):
config = _config(tmp_path)
LibraryLock(config, "api").acquire()
LibraryLock(config, "worker").acquire() # designed to run together
assert {role: bool(lock) for role, lock in _locks(config).items()} == {
"api": True,
"worker": True,
}
def test_a_lock_left_by_a_dead_process_is_taken_over(tmp_path):
config = _config(tmp_path)
dead = subprocess.Popen([sys.executable, "-c", "pass"])
dead.wait()
lock = LibraryLock(config, "worker")
lock.path.parent.mkdir(parents=True, exist_ok=True)
lock.path.write_text(
json.dumps(
{
"lock_version": 1,
"role": "worker",
"pid": dead.pid,
"host": app_lock.socket.gethostname(),
"started_at": NOW.isoformat(),
"library_roots": [],
}
)
)
taken = LibraryLock(config, "worker").acquire()
assert taken.pid == os.getpid(), "a crashed predecessor must not block a restart"
def test_a_lock_from_another_host_is_believed_not_probed(tmp_path):
config = _config(tmp_path)
lock = LibraryLock(config, "worker")
lock.path.parent.mkdir(parents=True, exist_ok=True)
lock.path.write_text(
json.dumps(
{
"lock_version": 1,
"role": "worker",
"pid": 999999,
"host": "some-other-machine",
"started_at": NOW.isoformat(),
"library_roots": [],
}
)
)
with pytest.raises(LockHeld, match="some-other-machine"):
LibraryLock(config, "worker").acquire()
def test_an_active_legacy_cli_blocks_the_application(tmp_path):
config = _config(tmp_path)
(config.library_roots[0] / "nsfw_scores.csv").write_text("path,score\n")
with pytest.raises(LegacyProcessActive, match="nsfw_scores.csv"):
LibraryLock(config, "worker").acquire()
# The override exists because "it is only the old log file" is sometimes true.
LibraryLock(config, "worker").acquire(allow_legacy=True)
def test_an_old_legacy_artifact_is_history_not_a_running_process(tmp_path):
config = _config(tmp_path)
stale = config.library_roots[0] / "photo_analyzer_history.jsonl"
stale.write_text("{}\n")
old = NOW.timestamp()
os.utime(stale, (old, old))
assert app_lock.legacy_activity(config)["active"] is False
LibraryLock(config, "worker").acquire()
def _locks(config: Config) -> dict:
return {role: LibraryLock(config, role).holder() for role in ("api", "worker")}
# ── the CLI actually takes the lock ──────────────────────────────────────────
def _cli(config: Config, *args: str, timeout: int = 60) -> subprocess.CompletedProcess:
env = {
**os.environ,
"PYTHONPATH": str(REPO),
"PHOTO_PIPELINE_DATA_DIR": str(config.data_dir),
"PHOTO_PIPELINE_LIBRARY_ROOTS": os.pathsep.join(
str(root) for root in config.library_roots
),
}
return subprocess.run(
[sys.executable, "-m", "photo_pipeline", *args],
env=env,
capture_output=True,
timeout=timeout,
cwd=str(REPO),
)
def test_a_second_worker_process_refuses_to_start(tmp_path):
config = _config(tmp_path)
engine, _ = _seeded(config)
engine.dispose()
env = {
**os.environ,
"PYTHONPATH": str(REPO),
"PHOTO_PIPELINE_DATA_DIR": str(config.data_dir),
"PHOTO_PIPELINE_LIBRARY_ROOTS": str(config.library_roots[0]),
}
first = subprocess.Popen(
[sys.executable, "-m", "photo_pipeline", "worker", "--id", "first"],
env=env,
cwd=str(REPO),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
try:
lock = LibraryLock(config, "worker")
deadline = __import__("time").monotonic() + 30
while lock.holder() is None and __import__("time").monotonic() < deadline:
__import__("time").sleep(0.1)
assert lock.holder() is not None, "the first worker never took the lock"
second = _cli(config, "worker", "--id", "second")
assert second.returncode == 2
assert b"already running" in second.stderr
finally:
first.terminate()
first.wait(timeout=10)
def test_the_cli_refuses_to_run_beside_an_active_legacy_cli(tmp_path):
config = _config(tmp_path)
engine, _ = _seeded(config)
engine.dispose()
(config.library_roots[0] / "photo_analyzer_history.jsonl").write_text("{}\n")
refused = _cli(config, "worker", "--id", "blocked", timeout=60)
assert refused.returncode == 3
assert b"legacy CLI is writing this library" in refused.stderr
assert b"--allow-legacy" in refused.stderr

299
tests/integration/test_compose_runtime.py
View File

@@ -1,299 +0,0 @@
"""US08-03: the composition's contract, and the library-root check it depends on.
Bringing the stack up needs a Docker daemon and the network, which is what
``tests/e2e/test_compose_stack.py`` does. What can be checked without either is
checked here, because the parts that rot silently — a second writer that is only
prevented by convention, a data volume that stopped being the same volume for both
roles, migrations that stopped running first, a committed value in a file that must
carry none — are all readable from the files.
The startup refusal is the other half: in a container the configured library roots
must name the mount paths, and a mismatch has to fail before the lock is taken, not
at the first rename.
"""
from __future__ import annotations
import json
import os
import re
import stat
from pathlib import Path
import pytest
import yaml
from photo_pipeline import path_policy
from photo_pipeline.__main__ import main
from photo_pipeline.config import Config
REPO = Path(__file__).resolve().parents[2]
COMPOSE_FILE = REPO / "docker-compose.yml"
COMPOSE = yaml.safe_load(COMPOSE_FILE.read_text())
ENV_EXAMPLE = REPO / ".env.example"
SERVICES = COMPOSE["services"]
DATA_VOLUME = "data:/data"
def env_example_keys() -> list[str]:
"""The variables the example file declares, in file order."""
return [
line.split("=", 1)[0]
for line in ENV_EXAMPLE.read_text().splitlines()
if "=" in line and not line.lstrip().startswith("#")
]
# ── one API, one worker, one library, one volume ─────────────────────────────
def test_exactly_one_serving_and_one_working_container_from_the_same_image():
roles = {name: service["command"][0] for name, service in SERVICES.items()}
assert sorted(roles.values()) == ["migrate", "serve", "worker"]
assert [name for name, role in roles.items() if role == "serve"] == ["api"]
assert [name for name, role in roles.items() if role == "worker"] == ["worker"]
images = {service["image"] for service in SERVICES.values()}
assert len(images) == 1, "both roles must run the same build of the application"
assert "latest" not in images.pop()
# No `replicas`/`scale` key promising a second worker is fine; the lock decides.
assert not any("deploy" in service for service in SERVICES.values())
def test_both_roles_share_the_data_volume_so_the_lock_is_visible_to_both():
"""A second worker is refused by the library lock (US07-05) only if it can see it."""
for name, service in SERVICES.items():
assert DATA_VOLUME in service["volumes"], name
assert COMPOSE["volumes"]["data"]["driver"] == "local"
text = COMPOSE_FILE.read_text()
# The composition has to say why, because the failure is silent corruption.
assert "WAL" in text and re.search(r"NFS|SMB|network", text)
def test_the_library_is_a_bind_mount_whose_target_is_the_configured_root():
for name, service in SERVICES.items():
mounts = [volume for volume in service["volumes"] if volume != DATA_VOLUME]
assert len(mounts) == 1, name
source, target = re.match(r"^(\$\{.*?\}):(\$\{.*?\})$", mounts[0]).groups()
# An unset host path fails the composition rather than mounting something else.
assert source.startswith("${PHOTO_PIPELINE_LIBRARY_HOST_PATH:?")
# The container-side path and the configured root are one variable, so they
# cannot drift apart into a library that is mounted but not configured.
assert target.startswith("${PHOTO_PIPELINE_LIBRARY_ROOTS:?")
assert service["environment"]["PHOTO_PIPELINE_LIBRARY_ROOTS"] == (
"${PHOTO_PIPELINE_LIBRARY_ROOTS}"
)
assert service["environment"]["PHOTO_PIPELINE_DATA_DIR"] == "/data"
def test_migrations_run_to_completion_before_either_role_accepts_work():
"""`migrate` runs the backup-then-migrate path, and a failed upgrade exits
non-zero with its pre-migration backup intact — proven in
tests/integration/test_backup_recovery.py. What the composition adds is that
neither role starts until it succeeded."""
assert SERVICES["migrate"]["command"] == ["migrate"]
assert SERVICES["migrate"]["restart"] == "no", "a one-shot that retries is not a gate"
for role in ("api", "worker"):
assert SERVICES[role]["depends_on"] == {
"migrate": {"condition": "service_completed_successfully"}
}, role
def test_the_api_port_is_published_to_host_loopback_by_default():
published = SERVICES["api"]["ports"]
assert published == [
"${PHOTO_PIPELINE_PUBLISH_ADDRESS:-127.0.0.1}:${PHOTO_PIPELINE_PORT:-8000}:8000"
]
# Reachable from the host means reachable from elsewhere as far as the app is
# concerned, so the access secret stays mandatory (US08-01).
assert SERVICES["api"]["environment"]["PHOTO_PIPELINE_HOST"] == "0.0.0.0"
assert SERVICES["api"]["environment"]["PHOTO_PIPELINE_PORT"] == 8000
assert "PHOTO_PIPELINE_ACCESS_SECRET" not in SERVICES["api"]["environment"]
def test_containers_restart_by_themselves_and_stop_with_time_to_drain():
for role in ("api", "worker"):
assert SERVICES[role]["restart"] == "unless-stopped", role
assert SERVICES[role]["stop_grace_period"] == "30s", role
def test_the_containers_run_as_the_library_owner_and_never_as_root():
for name, service in SERVICES.items():
assert service["user"] == "${PHOTO_PIPELINE_UID:-1000}:${PHOTO_PIPELINE_GID:-1000}", name
assert service["build"]["args"]["UID"] == "${PHOTO_PIPELINE_UID:-1000}", name
# ── configuration comes from the environment, never from a committed file ────
def test_configuration_and_secrets_come_from_the_environment_only():
for name, service in SERVICES.items():
assert service["env_file"] == ["${PHOTO_PIPELINE_ENV_FILE:-.env}"], name
for key, value in service["environment"].items():
# Every value is either a variable reference or a property of the
# composition itself (the volume path, the container's own port).
composed = isinstance(value, int) or value in ("/data", "0.0.0.0")
assert composed or value.startswith("${"), (name, key, value)
assert not (REPO / ".env").is_file() or ".env" in (REPO / ".gitignore").read_text()
def test_the_example_file_lists_every_setting_and_carries_no_values():
declared = env_example_keys()
assert declared == sorted(set(declared), key=declared.index), "no variable twice"
for line in ENV_EXAMPLE.read_text().splitlines():
if "=" in line and not line.lstrip().startswith("#"):
assert line.endswith("="), f"a value in the example file: {line}"
expected = {f"PHOTO_PIPELINE_{name.upper()}" for name in Config.model_fields}
assert expected <= set(declared), sorted(expected - set(declared))
# And every variable the composition substitutes is documented there too.
substituted = set(re.findall(r"\$\{(PHOTO_PIPELINE_[A-Z_]+)", COMPOSE_FILE.read_text()))
assert substituted <= set(declared), sorted(substituted - set(declared))
def test_the_example_file_is_not_a_dotenv_that_could_be_loaded_by_accident():
"""`.env.example` must not be what `.env` is: no values means nothing to leak."""
assert ENV_EXAMPLE.name != ".env"
parsed = {k: v for k, v in _parse(ENV_EXAMPLE.read_text()).items() if v}
assert parsed == {}
def _parse(text: str) -> dict[str, str]:
from photo_pipeline.config import parse_env_file
return parse_env_file(text)
# ── the lock across container lifetimes ──────────────────────────────────────
def test_a_lock_left_by_a_container_that_is_gone_does_not_block_the_restart(
tmp_path, monkeypatch
):
"""A restarted container is a new hostname and a recycled pid 1, so the record
in the lock file proves nothing; the kernel's flock does (US08-03)."""
from photo_pipeline.services.app_lock import LibraryLock
config = Config(data_dir=tmp_path / "data", library_roots=(tmp_path,))
(tmp_path / "data").mkdir()
(tmp_path / "data" / "worker.lock.json").write_text(
json.dumps(
{
"lock_version": 1,
"role": "worker",
"pid": 1, # pid 1 of a container that no longer exists
"host": "3f2a1b9c4d5e", # its hostname was its container id
"started_at": "2026-01-01T00:00:00+00:00",
"library_roots": ["/library"],
}
)
)
monkeypatch.setattr(path_policy, "in_container", lambda: True)
taken = LibraryLock(config, "worker").acquire()
assert taken.pid == os.getpid(), "the worker must come back after a restart"
def test_a_second_worker_is_still_refused_while_the_first_holds_the_lock(tmp_path, monkeypatch):
"""The other half: the same flock refuses a concurrent second writer, whether it
is a process or another container of the same composition."""
from photo_pipeline.services.app_lock import LibraryLock, LockHeld
config = Config(data_dir=tmp_path / "data", library_roots=(tmp_path,))
monkeypatch.setattr(path_policy, "in_container", lambda: True)
first = LibraryLock(config, "worker")
first.acquire()
with pytest.raises(LockHeld, match="worker is already running"):
LibraryLock(config, "worker").acquire()
first.release()
LibraryLock(config, "worker").acquire() # free again
# ── the startup check the mount depends on ───────────────────────────────────
def test_configured_roots_that_are_mounted_and_writable_are_accepted(tmp_path):
assert path_policy.roots_refusal([tmp_path]) is None
assert path_policy.roots_refusal([]) is None, "no roots is a configuration, not a fault"
def test_an_unmounted_library_root_is_refused_by_name(tmp_path):
refusal = path_policy.roots_refusal([tmp_path / "srv" / "photos"])
assert refusal is not None
assert "does not exist" in refusal and "PHOTO_PIPELINE_LIBRARY_ROOTS" in refusal
def test_a_root_that_is_not_a_directory_or_not_readable_is_refused(tmp_path):
a_file = tmp_path / "photos.txt"
a_file.write_text("not a library")
assert "not a directory" in path_policy.roots_refusal([a_file])
unreadable = tmp_path / "unreadable"
unreadable.mkdir()
unreadable.chmod(0o000)
try:
refusal = path_policy.roots_refusal([unreadable])
finally:
unreadable.chmod(0o755)
if os.getuid() != 0: # root ignores the mode, and CI may well be root
assert refusal is not None and "not readable" in refusal
def test_an_unwritable_root_is_not_refused_here(tmp_path):
"""A bind mount's ownership is virtualised on macOS and Windows, so os.access
would refuse a working deployment. The real errno at the first rename is at
least true; this check is about the mount, not the mode."""
read_only = tmp_path / "read-only"
read_only.mkdir()
read_only.chmod(stat.S_IRUSR | stat.S_IXUSR)
try:
assert path_policy.roots_refusal([read_only]) is None
finally:
read_only.chmod(0o755)
def test_in_a_container_a_root_that_was_never_mounted_is_refused(tmp_path, monkeypatch):
"""The container-only failure: the path exists, but it belongs to the image."""
unmounted = tmp_path / "library"
(unmounted / "album").mkdir(parents=True)
refusal = path_policy.roots_refusal([unmounted], require_mount=True)
assert refusal is not None and "not on a mounted filesystem" in refusal
# A bind mount is a mount point, and a root *below* one is mounted too: a
# deployment may mount /srv and configure /srv/photos.
monkeypatch.setattr(os.path, "ismount", lambda path: Path(path) == unmounted.resolve())
assert path_policy.roots_refusal([unmounted], require_mount=True) is None
assert path_policy.roots_refusal([unmounted / "album"], require_mount=True) is None
@pytest.mark.parametrize("role", ["serve", "worker"])
def test_a_root_mismatch_refuses_at_startup_before_any_lock_is_taken(
role, tmp_path, monkeypatch, capsys
):
data = tmp_path / "data"
monkeypatch.setenv("PHOTO_PIPELINE_DATA_DIR", str(data))
monkeypatch.setenv("PHOTO_PIPELINE_LIBRARY_ROOTS", str(tmp_path / "not-mounted"))
monkeypatch.setattr(path_policy, "in_container", lambda: True)
assert main([role]) == 5
assert "does not exist" in capsys.readouterr().err
assert not list(data.glob("*.lock.json")), "nothing started, so nothing is locked"
@pytest.mark.parametrize("role", ["serve", "worker"])
def test_an_unmounted_root_on_a_host_is_not_a_reason_to_refuse(role, tmp_path, monkeypatch):
"""An archive medium that is not plugged in is a Tuesday, not a misconfiguration:
refusing would take the offline half of the library away with it (concept §9)."""
monkeypatch.setenv("PHOTO_PIPELINE_DATA_DIR", str(tmp_path / "data"))
monkeypatch.setenv("PHOTO_PIPELINE_LIBRARY_ROOTS", str(tmp_path / "not-mounted"))
monkeypatch.setenv("PHOTO_PIPELINE_ACCESS_SECRET", "unused-on-loopback")
monkeypatch.setattr(path_policy, "in_container", lambda: False)
# Reaching the lock is the proof: that is the next thing either role does, and
# stopping there keeps the test out of a uvicorn/worker loop.
monkeypatch.setattr("photo_pipeline.__main__._acquire", lambda *_, **__: 99)
assert main([role]) == 99

147
tests/integration/test_container_gate.py
View File

@@ -1,147 +0,0 @@
"""US08-05: the container acceptance gate's own contract, checked without a daemon.
The gate itself needs Docker, a browser, and several minutes; running it from inside
the suite would be a fork bomb with better manners. What is checkable offline is what
makes it a *gate* rather than a long test run:
* it selects the container journeys by marker, so adding one is enough to put it in
front of a deploy;
* it accepts no skip at all — every reason a check would skip here (no daemon, no
compose plugin, no browser) means the deployed runtime was not proven;
* it retains checksummed evidence per run;
* and CI runs it on `main`, which is what the publish step waits for.
The running proof is ``tests/e2e/test_phase_h_container.py``.
"""
from __future__ import annotations
import json
import re
from pathlib import Path
import yaml
from photo_pipeline.config import Config
from photo_pipeline.services import release
REPO = Path(__file__).resolve().parents[2]
SUITE = REPO / "tests" / "e2e" / "test_phase_h_container.py"
TEST_WORKFLOW = yaml.safe_load((REPO / ".gitea" / "workflows" / "test.yml").read_text())
def _config(tmp_path) -> Config:
for name in ("data", "lib"):
(tmp_path / name).mkdir(exist_ok=True)
return Config.from_env(
{
"PHOTO_PIPELINE_DATA_DIR": str(tmp_path / "data"),
"PHOTO_PIPELINE_LIBRARY_ROOTS": str(tmp_path / "lib"),
}
)
# ── what the gate runs ───────────────────────────────────────────────────────
def test_the_gate_selects_the_container_journeys_by_their_marker():
assert release.CONTAINER_STAGES == (("container", ("tests/e2e", "-m", "phase_h")),)
declared = [line for line in (REPO / "pyproject.toml").read_text().splitlines()
if line.strip().startswith('"phase_h:')]
assert declared, "an unregistered marker selects nothing and fails no gate"
def test_every_journey_in_the_suite_carries_the_marker():
"""A test in that file without the marker is a check the gate never runs."""
source = SUITE.read_text()
assert "pytestmark = [pytest.mark.phase_h, pytest.mark.container]" in source
journeys = re.findall(r"^def (test_[a-z_]+)", source, re.MULTILINE)
assert len(journeys) >= 4, journeys
for required in ("browser", "upgrade", "kill", "secret"):
assert any(required in name for name in journeys), (required, journeys)
# ── no skip is an environment limit here ─────────────────────────────────────
def test_the_gate_accepts_no_skipped_check_at_all():
assert release.CONTAINER_ALLOWED_SKIP_REASONS == ()
docker_missing = release.StageResult(
"container", [], 0, 0.1, "1 skipped", ["SKIPPED [1] x.py:1: no Docker daemon available"]
)
# The release gate tolerates exactly this one, because the image definition is
# still checked offline. The container gate cannot: it is the deployment's proof.
assert release.unexpected_skips([docker_missing]) == []
assert release.unexpected_skips(
[docker_missing], release.CONTAINER_ALLOWED_SKIP_REASONS
) == ["SKIPPED [1] x.py:1: no Docker daemon available"]
def test_a_skipped_check_fails_the_run_and_the_evidence_says_so(tmp_path):
skipping = tmp_path / "test_skipping.py"
skipping.write_text(
"import pytest\n\n"
"def test_x():\n"
" pytest.skip('no Docker daemon available')\n"
)
evidence = tmp_path / "evidence"
report = release.run_gate(
_config(tmp_path),
output=evidence,
stages=(("container", (str(skipping),)),),
allowed_skips=release.CONTAINER_ALLOWED_SKIP_REASONS,
)
assert report["ok"] is False
assert report["failures"] == [], "the stage passed; the skip is what fails the gate"
assert report["unexpected_skips"], report
written = json.loads((evidence / release.REPORT_NAME).read_text())
assert written["ok"] is False
assert (evidence / "logs" / "container.log").exists()
for line in (evidence / release.CHECKSUMS_NAME).read_text().splitlines():
digest, name = line.split(" ", 1)
assert release.sha256_file(evidence / name) == digest
def test_the_same_run_passes_when_nothing_skips(tmp_path):
passing = tmp_path / "test_passing.py"
passing.write_text("def test_x():\n assert True\n")
report = release.run_gate(
_config(tmp_path),
output=tmp_path / "evidence",
stages=(("container", (str(passing),)),),
allowed_skips=release.CONTAINER_ALLOWED_SKIP_REASONS,
)
assert report["ok"] is True and report["unexpected_skips"] == []
assert report["revision"], "the evidence must say which commit it covers"
# ── the command, and CI ──────────────────────────────────────────────────────
def test_the_command_is_documented_and_wired():
from photo_pipeline.__main__ import main # noqa: F401 (import proves it loads)
assert "container-gate" in (REPO / "photo_pipeline" / "__main__.py").read_text()
assert "container-gate" in (REPO / "README.md").read_text()
def test_ci_runs_the_gate_on_main_and_keeps_its_evidence():
job = TEST_WORKFLOW["jobs"]["container"]
assert job["if"] == "gitea.event_name == 'push'", "pull requests have nothing to upgrade from"
script = "\n".join(step["run"] for step in job["steps"] if "run" in step)
assert "photo_pipeline container-gate" in script
evidence = next(step for step in job["steps"] if "upload-artifact" in str(step.get("uses")))
assert evidence["if"] == "always()", "a failed gate's logs are the ones worth keeping"
assert evidence["with"]["path"] == "gate-evidence"
def test_the_upgrade_journey_can_still_reach_the_previous_version():
"""Without depth, `git archive HEAD~1` has nothing to build."""
checkout = next(
step for step in TEST_WORKFLOW["jobs"]["container"]["steps"] if "checkout" in str(step.get("uses"))
)
assert checkout["with"]["fetch-depth"] >= 2

Some files were not shown because too many files have changed in this diff Show More