domverse/photoanalyzer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Compare commits

base: domverse:us/US09-05-automate-documentation-acceptance
Branches Tags
domverse:main domverse:us/US09-05-automate-documentation-acceptance domverse:us/US09-04-write-the-user-manual-with-generated-screenshots domverse:us/US09-03-write-the-architecture-overview domverse:us/US09-02-write-the-installation-and-operations-manual domverse:us/US09-01-serve-the-documentation-inside-the-application domverse:chore/E09-documentation-backlog domverse:us/US08-05-automate-container-deployment-acceptance domverse:us/US08-04-publish-and-deploy-from-gitea-actions domverse:us/US08-03-compose-the-runtime-and-mount-the-library-safely domverse:us/US08-02-build-a-reproducible-application-image domverse:us/US08-01-make-the-trust-boundary-configurable-and-authent domverse:chore/local-run-and-env-file domverse:us/US07-07-automate-full-release-acceptance domverse:us/US07-06-validate-performance-and-resource-bounds domverse:us/US07-05-deliver-backup-and-operational-recovery domverse:us/US07-04-prove-concurrency-and-crash-recovery domverse:us/US07-03-harden-media-and-metadata-edge-cases domverse:us/US07-02-harden-api-authorization-and-path-boundaries domverse:us/US07-01-complete-donor-migration-and-freeze-the-cli-arch domverse:us/US06-06-automate-phase-f-end-to-end-acceptance domverse:us/US06-05-operate-archive-and-restore-in-the-browser domverse:us/US06-04-plan-and-execute-safe-restores domverse:us/US06-03-preserve-offline-identity-and-review-evidence domverse:us/US06-02-transfer-verify-and-remove-active-sources domverse:us/US06-01-configure-and-preflight-archive-destinations domverse:us/US05-06-automate-phase-e-end-to-end-acceptance domverse:us/US05-05-operate-uploads-in-the-browser domverse:us/US05-04-verify-retry-and-resolve-uncertain-uploads domverse:us/US05-03-parse-and-persist-uploader-outcomes domverse:us/US05-02-orchestrate-album-upload-batches domverse:us/US05-01-validate-credentials-and-upload-readiness domverse:us/US04-06-automate-phase-d-end-to-end-acceptance domverse:us/US04-05-operate-rename-plans-in-the-browser domverse:us/US04-03-apply-and-verify-guarded-renames
..
compare: domverse:us/US05-05-operate-uploads-in-the-browser
Branches Tags
domverse:main domverse:us/US09-05-automate-documentation-acceptance domverse:us/US09-04-write-the-user-manual-with-generated-screenshots domverse:us/US09-03-write-the-architecture-overview domverse:us/US09-02-write-the-installation-and-operations-manual domverse:us/US09-01-serve-the-documentation-inside-the-application domverse:chore/E09-documentation-backlog domverse:us/US08-05-automate-container-deployment-acceptance domverse:us/US08-04-publish-and-deploy-from-gitea-actions domverse:us/US08-03-compose-the-runtime-and-mount-the-library-safely domverse:us/US08-02-build-a-reproducible-application-image domverse:us/US08-01-make-the-trust-boundary-configurable-and-authent domverse:chore/local-run-and-env-file domverse:us/US07-07-automate-full-release-acceptance domverse:us/US07-06-validate-performance-and-resource-bounds domverse:us/US07-05-deliver-backup-and-operational-recovery domverse:us/US07-04-prove-concurrency-and-crash-recovery domverse:us/US07-03-harden-media-and-metadata-edge-cases domverse:us/US07-02-harden-api-authorization-and-path-boundaries domverse:us/US07-01-complete-donor-migration-and-freeze-the-cli-arch domverse:us/US06-06-automate-phase-f-end-to-end-acceptance domverse:us/US06-05-operate-archive-and-restore-in-the-browser domverse:us/US06-04-plan-and-execute-safe-restores domverse:us/US06-03-preserve-offline-identity-and-review-evidence domverse:us/US06-02-transfer-verify-and-remove-active-sources domverse:us/US06-01-configure-and-preflight-archive-destinations domverse:us/US05-06-automate-phase-e-end-to-end-acceptance domverse:us/US05-05-operate-uploads-in-the-browser domverse:us/US05-04-verify-retry-and-resolve-uncertain-uploads domverse:us/US05-03-parse-and-persist-uploader-outcomes domverse:us/US05-02-orchestrate-album-upload-batches domverse:us/US05-01-validate-credentials-and-upload-readiness domverse:us/US04-06-automate-phase-d-end-to-end-acceptance domverse:us/US04-05-operate-rename-plans-in-the-browser domverse:us/US04-03-apply-and-verify-guarded-renames

1 Commits
us/US09-05 ... us/US05-05

Author SHA1 Message Date
domverse
c860c6d412 US05-05: Operate Uploads in the Browser 2026-08-16 16:07:55 +02:00
210 changed files with 520 additions and 29565 deletions
Expand all files Collapse all files

23
.dockerignore
View File

@@ -1,23 +0,0 @@
# Deny-by-default build context (US08-02): the image must contain no secrets, no
# photos, no database, no logs, and no .git. An allow list is the only version of this
# rule that stays true when a new file appears in the working copy.
*
!pyproject.toml
!alembic.ini
!README.md
!photo_pipeline
!migrations
!frontend
!docs
!docker
# Nothing generated, even under an allowed directory.
**/__pycache__
**/*.py[cod]
**/.DS_Store
**/*.env
**/*.log
**/*.db
**/*.db-*
**/*.sqlite*

85
.env.example
View File

@@ -1,85 +0,0 @@
# Every setting the application and its composition read, with no values in it.
# Copy to `.env`, fill in what you need, and keep that copy out of git (it is
# gitignored, and the work-item safety checks refuse to stage it).
#
# cp .env.example .env
#
# Empty means "use the default noted beside it". Anything already exported in the
# shell wins over this file, both for the app and for `docker compose`.
# ── the composition (host side; read by docker-compose.yml only) ─────────────
# The photo library on this host. Bind-mounted at PHOTO_PIPELINE_LIBRARY_ROOTS.
PHOTO_PIPELINE_LIBRARY_HOST_PATH=
# Which image to run. Default: photo-pipeline:dev (what `up --build` builds).
PHOTO_PIPELINE_IMAGE=
# Must be the owner of the library above: what the containers rename and rewrite
# keeps this ownership. Default: 1000 / 1000.
PHOTO_PIPELINE_UID=
PHOTO_PIPELINE_GID=
# Host address the API port is published on. Default: 127.0.0.1. Anything else
# exposes the app beyond this machine — then ALLOWED_HOSTS and ACCESS_SECRET below
# are what stand in for the loopback boundary (US08-01).
PHOTO_PIPELINE_PUBLISH_ADDRESS=
# This file's own path, if it is not ./.env. Default: .env.
PHOTO_PIPELINE_ENV_FILE=
# ── library and data ────────────────────────────────────────────────────────
# os.pathsep-separated. In a container these are the *container-side* mount paths,
# and `serve`/`worker` refuse to start when they are not mounted.
PHOTO_PIPELINE_LIBRARY_ROOTS=
# Database, WAL, thumbnail cache, journals, backups. Default: data (the container
# sets /data, which is the persistent volume; a network mount is unsupported).
PHOTO_PIPELINE_DATA_DIR=
# Database file, if it should not live in the data directory. Default:
# <data dir>/photo_pipeline.db.
PHOTO_PIPELINE_DB_PATH=
# ── serving ─────────────────────────────────────────────────────────────────
# Bind address. Default: 127.0.0.1. The composition sets 0.0.0.0 inside the
# container and publishes to loopback on the host instead.
PHOTO_PIPELINE_HOST=
# Under the composition this is the *published* host port; the container serves
# 8000. Default: 8000.
PHOTO_PIPELINE_PORT=
# Comma-separated hostnames the app answers to besides loopback. Empty means
# loopback only. Naming one makes the access secret mandatory.
PHOTO_PIPELINE_ALLOWED_HOSTS=
# Traded for the session cookie at GET /api/v1/session via X-Access-Secret.
# Required as soon as the app is reachable from anywhere but loopback. Generate
# one with: python -c 'import secrets; print(secrets.token_urlsafe(32))'
PHOTO_PIPELINE_ACCESS_SECRET=
# Comma-separated peer addresses whose X-Forwarded-Proto/-Host may be believed.
# Only the reverse proxy's address belongs here. Default: none.
PHOTO_PIPELINE_TRUSTED_PROXIES=
# Largest request body accepted, in bytes. Default: 1048576.
PHOTO_PIPELINE_MAX_REQUEST_BYTES=
# ── logging ─────────────────────────────────────────────────────────────────
# Default: INFO.
PHOTO_PIPELINE_LOG_LEVEL=
# json or text. Default: json.
PHOTO_PIPELINE_LOG_FORMAT=
# ── limits ──────────────────────────────────────────────────────────────────
# Thumbnail cache quota in bytes. Default: 500000000.
PHOTO_PIPELINE_THUMBNAIL_CACHE_QUOTA_BYTES=
# Refuse to decode images larger than this many pixels. Default: 100000000.
PHOTO_PIPELINE_THUMBNAIL_MAX_PIXELS=
# Free space an archive destination must keep beyond the transfer. Default:
# 1000000000.
PHOTO_PIPELINE_ARCHIVE_FREE_SPACE_RESERVE_BYTES=
# ── safety gate ─────────────────────────────────────────────────────────────
# true refuses every mutating request until a read-only dry run of this library has
# been produced and approved (US07-07). Default: false. Turn it on before pointing
# the app at photos that cannot be replaced.
PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL=
# ── external services ───────────────────────────────────────────────────────
# Vision provider key for the analysis stage. Without it, analysis cannot run.
PHOTO_PIPELINE_VISION_API_KEY=
# Immich server and its API key, for the upload stage.
PHOTO_PIPELINE_IMMICH_SERVER_URL=
PHOTO_PIPELINE_IMMICH_API_KEY=
# Uploader binary. Default: immich-go (on PATH; pinned inside the image).
PHOTO_PIPELINE_IMMICH_GO_BINARY=

9
.gitattributes vendored
View File

@@ -1,9 +0,0 @@
# Vendored third-party browser bundles (US09-01). Minified upstream files carry
# trailing whitespace and very long lines; they are not ours to reformat, and the
# submit gate's `git diff --check` would refuse them forever. Their integrity is
# controlled where it belongs instead: a pinned version and a recorded sha256 in
# frontend/js/vendor/VERSIONS.json, asserted by tests/integration/test_documentation.py.
#
# Only the whitespace check is turned off. They stay text, so a credential scan or a
# search still reads them.
frontend/js/vendor/*.js -whitespace linguist-vendored

99
.gitea/workflows/deploy.yml
View File

@@ -1,99 +0,0 @@
# Publish and redeploy (US08-04). Adapted from the crowdsec-admin deploy workflow:
# build, log in to the Gitea registry, push, trigger the Portainer webhook, prune.
# The difference is the gate — this project has a required suite that must not be
# skipped, so publishing happens only after `Test` (`.gitea/workflows/test.yml`)
# succeeded on `main`, never on the push itself.
#
# The stack is managed by Portainer from git (`docker-compose.yml`), and the runtime
# secrets it needs — vision key, Immich key, access secret — live in the Portainer
# stack's environment. They are deliberately not repository secrets and are not in the
# image: rotation stays in one place, and a repository read never discloses them.
#
# Repository secrets required:
# REGISTRY_USER user with write:package on the registry
# REGISTRY_TOKEN that user's token
# PORTAINER_WEBHOOK_URL POST URL from the stack's auto-update setting
name: Deploy
on:
workflow_run:
workflows:
- Test
types:
- completed
branches:
- main
workflow_dispatch:
inputs:
dry_run:
description: Build and push a scratch tag only — leave `latest` and the running stack alone
type: boolean
default: true
concurrency:
# Deliberately not keyed by commit: the point is that two deploys of *different*
# commits cannot overlap. Queued, not cancelled — a half-pushed tag set is worse
# than a late one.
group: deploy
cancel-in-progress: false
env:
IMAGE: git.domverse-berlin.eu/domverse/photoanalyzer
# The commit that was tested, not whatever `main` points at by the time this starts.
SHA: ${{ gitea.event.workflow_run.head_sha || gitea.sha }}
jobs:
publish:
# A completed `Test` run is not a passing one.
if: >-
(gitea.event_name == 'workflow_run' && gitea.event.workflow_run.conclusion == 'success')
|| (gitea.event_name == 'workflow_dispatch' && inputs.dry_run == false)
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
ref: ${{ env.SHA }}
- name: Log in to the Gitea registry
run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login git.domverse-berlin.eu -u "${{ secrets.REGISTRY_USER }}" --password-stdin
- name: Build and push
# The commit tag is pushed first, so a `latest` that exists is always a tag
# that also exists under its own commit — which is what makes a rollback a
# tag change rather than a rebuild.
run: |
docker build -t "$IMAGE:$SHA" -t "$IMAGE:latest" .
docker push "$IMAGE:$SHA"
docker push "$IMAGE:latest"
- name: Trigger the Portainer redeploy
# --fail turns an HTTP error into a non-zero exit: a redeploy that did not
# happen must not read as a green deploy.
run: curl -sS --fail -X POST "${{ secrets.PORTAINER_WEBHOOK_URL }}"
- name: Prune dangling images
# Untagged layers only. Published tags are the rollback history; `-a` would
# delete exactly the images this workflow exists to keep.
run: docker image prune -f
dry-run:
# Manual only, and the default: prove the image still builds and the registry
# still accepts it without moving `latest` or touching the running stack.
if: gitea.event_name == 'workflow_dispatch' && inputs.dry_run
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Log in to the Gitea registry
run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login git.domverse-berlin.eu -u "${{ secrets.REGISTRY_USER }}" --password-stdin
- name: Build and push a scratch tag
run: |
docker build -t "$IMAGE:scratch-$SHA" .
docker push "$IMAGE:scratch-$SHA"
- name: Prune dangling images
run: docker image prune -f

139
.gitea/workflows/test.yml
View File

@@ -1,139 +0,0 @@
# The test gate. Deploy waits for this workflow by name (`.gitea/workflows/deploy.yml`
# triggers on `workflow_run: [Test]`), so renaming it here without renaming it there
# would leave `main` publishing without a suite. `tests/integration/test_deploy_workflows.py`
# asserts both halves of that link, and that the commands below are still the ones
# configured in `work_item/.work-item.yml`.
name: Test
on:
pull_request:
push:
branches:
- main
concurrency:
# One run per branch; a newer push makes the older run's answer irrelevant.
group: test-${{ gitea.ref }}
cancel-in-progress: true
jobs:
suites:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install exiftool
# The EXIF checkpoints are the safety invariant of every metadata stage; a run
# without exiftool would skip them for a reason the release gate accepts.
run: |
SUDO=
if [ "$(id -u)" -ne 0 ]; then SUDO=sudo; fi
$SUDO apt-get update
$SUDO apt-get install -y --no-install-recommends libimage-exiftool-perl
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install the application and its test dependencies
run: |
python -m pip install --upgrade pip
python -m pip install -e '.[test]'
python -m playwright install --with-deps chromium
- name: Helper suite
run: work_item/scripts/python -m unittest discover -s work_item/tests -v
- name: Application suite
run: work_item/scripts/python -m pytest tests -q
# The deployed container, verified the way the host application is (US08-05). It
# runs on `main` only — a pull request has nothing published to upgrade *from*, and
# building two images per push would pay for that on every commit. Deploy waits for
# this whole workflow, so a red container gate is a deploy that does not happen.
container:
if: gitea.event_name == 'push'
runs-on: ubuntu-latest
steps:
- name: Checkout
# The upgrade journey builds the previous commit's tree when no published
# image is named, so the history has to be there.
uses: actions/checkout@v4
with:
fetch-depth: 2
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install the application and its test dependencies
run: |
python -m pip install --upgrade pip
python -m pip install -e '.[test]'
python -m playwright install --with-deps chromium
- name: Container acceptance gate
# One command: it builds the image, provisions the composition against a
# temporary fixture library and an isolated volume, runs the phase_h journeys,
# destroys the stack, and writes the evidence. Any skipped check fails it.
env:
# The fixture libraries are bind-mounted into the containers, so this path
# has to be one the Docker daemon can see. On a runner that talks to a
# sibling daemon, point it at a shared host path instead of the workspace —
# an unshared path arrives as an empty mount and the journeys fail on the
# scan, which is the symptom to recognise.
PHOTO_PIPELINE_DATA_DIR: ${{ gitea.workspace }}/gate-data
PHOTO_PIPELINE_LIBRARY_ROOTS: ${{ gitea.workspace }}/gate-library
PHOTO_PIPELINE_TEST_MOUNT_BASE: ${{ gitea.workspace }}/gate-mounts
run: |
mkdir -p "$PHOTO_PIPELINE_LIBRARY_ROOTS" "$PHOTO_PIPELINE_TEST_MOUNT_BASE"
python -m photo_pipeline container-gate --output gate-evidence
- name: Keep the evidence
# Retained per run, and retained on failure especially: the logs are the only
# account of what the containers did.
if: always()
uses: actions/upload-artifact@v3
with:
name: container-gate-${{ gitea.sha }}
path: gate-evidence
# The manuals, checked against the application they describe (US09-05). Runs on
# every pull request as well as `main`: documentation drifts by the same commits
# that change behaviour, and telling the author while the change is still open is
# the only time the fix is cheap.
documentation:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install the application and its test dependencies
run: |
python -m pip install --upgrade pip
python -m pip install -e '.[test]'
python -m playwright install --with-deps chromium
- name: Documentation gate
# One command: every phase_i check, offline and in the browser, with the
# evidence retained. It accepts no skip — a check that did not run is a page
# nobody compared with the code.
env:
PHOTO_PIPELINE_DATA_DIR: ${{ gitea.workspace }}/docs-gate-data
run: python -m photo_pipeline docs-gate --output docs-evidence
- name: Keep the evidence
if: always()
uses: actions/upload-artifact@v3
with:
name: docs-gate-${{ gitea.sha }}
path: docs-evidence

12
.gitignore vendored
View File

@@ -17,15 +17,3 @@ _todo/
pictures/
photos/
_IGNORE/
# Test failure evidence (US07-04)
.artifacts/
# Any dotenv, not only the default name.
*.env
# Local virtualenv for running the app.
.venv/
# setuptools editable-install metadata.
*.egg-info/

3
AGENTS.md
View File

@@ -80,8 +80,7 @@ are the authoritative backlog.
uncertain. Work only on the claimed story and its generated feature branch.
4. Read the entire issue, linked specification, dependencies, and acceptance criteria.
Reconcile them with the concept before designing or changing code.
5. Inspect the legacy CLI donors — frozen in `legacy_cli_archive/` since US07-01,
with their ledger — before replacing applicable behavior. Update the donor
5. Inspect the legacy CLI donors before replacing applicable behavior. Update the donor
ledger and characterization tests required by the story.
6. Implement every acceptance criterion and its automated tests.
7. Run story-specific tests and the accumulated regression suite required by the epic.

117
Dockerfile
View File

@@ -1,117 +0,0 @@
# One image, two roles (US08-02).
#
# The application is not self-contained Python: it shells out to `exiftool` for every
# EXIF checkpoint and to `immich-go` for every upload, and it serves the static
# frontend from `frontend/`. All three are installed here at pinned versions, because
# an image whose external tools drift is an image whose metadata checkpoints and
# upload reports drift with them (concept §15, "External integration risks").
#
# Everything is pinned:
# * the base image by tag *and* digest, so a moved tag cannot change the runtime;
# * exiftool by its Debian package version, verified against `exiftool -ver`;
# * immich-go by release version and per-architecture SHA-256 of the release asset.
# The verified versions become image labels and /etc/photo-pipeline/versions.json,
# which `python -m photo_pipeline diagnostics` reports — so a running container can
# prove what it contains instead of being trusted about it.
#
# The project is installed editable on purpose: `photo_pipeline.db` resolves
# `alembic.ini` and `migrations/`, and the API resolves `frontend/`, relative to the
# repository root. An editable install keeps that one layout instead of scattering the
# same files across site-packages and a source tree.
ARG PYTHON_IMAGE=python:3.12.14-slim-trixie@sha256:2c941e860699f878900b0edc2403613c234d4b32eda3cc9fa7036991a2a63c4a
# ── the uploader, fetched and verified outside the final layer ────────────────
FROM ${PYTHON_IMAGE} AS uploader
ARG IMMICH_GO_VERSION=0.32.0
ARG IMMICH_GO_SHA256_AMD64=6e2ad86bafdadb9466d6515de7cb882726c0aea1a21d51164dff361d7d480a97
ARG IMMICH_GO_SHA256_ARM64=2c35d9284baae407ef9540bdac5f488971b0bdc7be758a4d7c05ab270af09fdb
COPY docker/fetch-immich-go.py /tmp/fetch-immich-go.py
RUN python /tmp/fetch-immich-go.py \
--version "${IMMICH_GO_VERSION}" \
--sha256-amd64 "${IMMICH_GO_SHA256_AMD64}" \
--sha256-arm64 "${IMMICH_GO_SHA256_ARM64}" \
--into /usr/local/bin \
&& /usr/local/bin/immich-go version
# ── the application ──────────────────────────────────────────────────────────
FROM ${PYTHON_IMAGE} AS runtime
ARG EXIFTOOL_VERSION=13.25+dfsg-1
ARG IMMICH_GO_VERSION=0.32.0
# The library is mounted from the host, so the container's identity must match the
# ownership that library already has: everything this application renames, writes
# EXIF into, or archives has to stay owned by the host user afterwards.
ARG UID=1000
ARG GID=1000
LABEL org.opencontainers.image.title="photo_pipeline" \
org.opencontainers.image.source="https://github.com/domverse/photoanalyzer" \
io.photoanalyzer.exiftool.version="${EXIFTOOL_VERSION}" \
io.photoanalyzer.immich-go.version="${IMMICH_GO_VERSION}"
ENV PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
PATH=/opt/venv/bin:$PATH \
PHOTO_PIPELINE_DATA_DIR=/data
RUN set -eu; \
apt-get update; \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
"libimage-exiftool-perl=${EXIFTOOL_VERSION}"; \
rm -rf /var/lib/apt/lists/*
COPY --from=uploader /usr/local/bin/immich-go /usr/local/bin/immich-go
WORKDIR /app
COPY pyproject.toml alembic.ini README.md ./
COPY photo_pipeline ./photo_pipeline
COPY migrations ./migrations
COPY frontend ./frontend
# The manuals are served by the application itself (US09-01), so they ship with it.
COPY docs ./docs
COPY docker/entrypoint.sh docker/healthcheck.sh /usr/local/bin/
# Runtime dependencies only: the `test` extra (pytest, playwright) and the `vision`
# extra stay out, and pip's build isolation leaves no build tooling behind.
RUN set -eu; \
python -m venv /opt/venv; \
/opt/venv/bin/pip install --no-cache-dir -e .
# What is installed must be what was pinned, or the labels and the version record
# would be a claim rather than a fact.
RUN set -eu; \
mkdir -p /etc/photo-pipeline; \
exiftool_version="$(exiftool -ver)"; \
immich_go_version="$(immich-go version | head -n 1 | tr -d '\r')"; \
expected_exiftool="$(printf '%s' "${EXIFTOOL_VERSION}" | cut -d+ -f1 | cut -d- -f1)"; \
[ "${exiftool_version}" = "${expected_exiftool}" ] \
|| { echo "exiftool ${exiftool_version} is not the pinned ${expected_exiftool}" >&2; exit 1; }; \
case "${immich_go_version}" in \
*"${IMMICH_GO_VERSION}"*) ;; \
*) echo "immich-go '${immich_go_version}' is not pinned ${IMMICH_GO_VERSION}" >&2; exit 1 ;; \
esac; \
printf '{\n "exiftool": "%s",\n "immich-go": "%s"\n}\n' \
"${exiftool_version}" "${IMMICH_GO_VERSION}" > /etc/photo-pipeline/versions.json
# Non-root, with the host library's ownership. /data is the persistent volume; the
# photo library itself is mounted by the deployment (US08-03), never baked in.
RUN set -eu; \
groupadd --gid "${GID}" --non-unique app; \
useradd --uid "${UID}" --gid "${GID}" --non-unique --no-create-home --home-dir /app app; \
mkdir -p /data; \
chown "${UID}:${GID}" /data
USER ${UID}:${GID}
EXPOSE 8000
# Readiness, not liveness: an unmigrated or misconfigured database answers
# /api/v1/health/ready with 503, and a container that cannot serve must not be
# reported healthy. The worker role has no endpoint, so its check is a no-op here.
HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
CMD ["/usr/local/bin/healthcheck.sh"]
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
CMD ["serve"]

618
README.md
View File

@@ -4,295 +4,19 @@ Integrated, restart-safe photo analysis, duplicate review, metadata, upload, and
archive workflow. Planning lives in `INTEGRATED_PIPELINE_CONCEPT.md` and
`delivery_backlog/`.
## Documentation
The manuals live in [`docs/`](docs/index.md) and are the same files the running
application serves at `/app/#/docs`:
| | |
|---|---|
| [Overview](docs/overview.md) | what it does and the rules it will not break |
| [Installation and operations](docs/installation.md) | host and container install, every setting, backup and restore, every refusal |
| [A guided first pass](docs/first-pass.md) | one library from scan to verified upload |
| [The stages](docs/index.md) | one illustrated page each, from inventory to archive |
| [Errors and refusals](docs/errors.md) | every error code, cause, and remedy |
| [Architecture](docs/architecture.md) | diagrams, module map, journals, invariants |
This README stays the repository's own notes — how the code is built, tested, and
released. Anything an operator or a user needs belongs in `docs/`, and one gate
(`python -m photo_pipeline docs-gate`) keeps those pages honest against the code.
## Application (`photo_pipeline`)
The target application lives in `photo_pipeline/` (FastAPI + SQLAlchemy + Alembic).
Install it into a virtualenv once:
Run it with:
```bash
python3.12 -m venv .venv
.venv/bin/pip install -e ".[vision]" # drop [vision] for a review-only install
python -m photo_pipeline migrate # apply database migrations
python -m photo_pipeline serve # start the API + static review UI (127.0.0.1:8000)
```
Then run the two processes:
```bash
.venv/bin/python -m photo_pipeline migrate # apply database migrations
.venv/bin/python -m photo_pipeline serve # API + review UI at 127.0.0.1:8000/app/
.venv/bin/python -m photo_pipeline worker # second terminal: runs the jobs
```
The server enqueues work and serves the UI; nothing actually scans, scores,
analyses, uploads, or archives without a worker. `work_item/scripts/python` is the
*helper's* launcher — it prefers Conda base and falls back to a bare system
interpreter, so it is not how the application is run.
Configuration comes from `PHOTO_PIPELINE_*` environment variables (see
`photo_pipeline/config.py`); secrets are referenced, never logged.
### Configuration file
`.env` in the working directory is read at startup, or any path named by
`PHOTO_PIPELINE_ENV_FILE`. It is parsed, never executed: `KEY=value` lines,
`#` comments, optional quotes — no interpolation and no `export`. **Anything already
exported wins**, so the file is the standing configuration and the shell is the
override for one run.
The archived CLI's variable names still work, so an existing `photo_analyzer.env`
can be used as-is:
| in the file | applied as |
|---|---|
| `LLM_API_KEY` / `GEMINI_API_KEY` | `OPENAI_API_KEY` |
| `LLM_BASE_URL` | `OPENAI_BASE_URL` |
| `LIBRARY` | `PHOTO_PIPELINE_LIBRARY_ROOTS` |
`.env` and `*.env` are gitignored and denied by the work-item safety checks: the
file holds a real key and must never be committed.
### API access (US07-02)
The app listens on loopback, so its attacker is another page in the same browser.
Every `/api/v1` route except `health/live`, `health/ready`, and `session` requires
the application session, and every mutation requires its CSRF token as well:
```bash
BASE=http://127.0.0.1:8000
TOKEN=$(curl -sc /tmp/pp.jar $BASE/api/v1/session | python -c 'import json,sys; print(json.load(sys.stdin)["csrf_token"])')
curl -sb /tmp/pp.jar -H "X-CSRF-Token: $TOKEN" -X POST $BASE/api/v1/albums/proposals -d '{}' -H 'Content-Type: application/json'
```
The session is per server process — restarting `serve` invalidates it, and the
browser client re-bootstraps by itself. Requests are also refused when the `Host` is
not a loopback name (DNS rebinding), when `Origin` is any other origin, when
`Sec-Fetch-Site` says the request came from another site (an `<img>` pointed at a
thumbnail), or when the body exceeds `PHOTO_PIPELINE_MAX_REQUEST_BYTES`. There is no
CORS middleware at all, so no other origin can read a response.
### Reaching it through a hostname or proxy (US08-01)
| variable | meaning |
|---|---|
| `PHOTO_PIPELINE_ALLOWED_HOSTS` | comma-separated extra names the app answers to; empty means loopback only |
| `PHOTO_PIPELINE_ACCESS_SECRET` | traded for the session cookie at `GET /api/v1/session` via `X-Access-Secret` |
| `PHOTO_PIPELINE_TRUSTED_PROXIES` | comma-separated peer addresses whose `X-Forwarded-Proto`/`X-Forwarded-Host` are believed |
Being reachable *was* the authentication: whoever could open `127.0.0.1:8000` owned
the library. So naming any non-loopback host — or binding to one, `0.0.0.0` included
— makes the access secret mandatory, and `serve` refuses to start without it rather
than publishing the library. Loopback-only deployments need no secret and behave
exactly as before.
```bash
curl -sc /tmp/pp.jar -H "X-Access-Secret: $PHOTO_PIPELINE_ACCESS_SECRET" \
https://photos.example.com/api/v1/session
```
The browser asks for the secret once per tab and keeps it in `sessionStorage`.
Wrong secrets are rate-limited (5 per minute) and logged with the caller's address
only. `Host` and `Origin` are judged against the configured names; the *external*
scheme and host come from the forwarded headers only when the request arrived from a
`PHOTO_PIPELINE_TRUSTED_PROXIES` address, so a client cannot declare its own origin,
and the session cookie is marked `Secure` when that external scheme is HTTPS. Health
endpoints stay reachable without the secret so an orchestrator can restart the
container; nothing else does.
## Container image (US08-02)
One image runs either role. It is built from a clean checkout with no arguments:
```bash
docker build -t photo-pipeline:dev .
```
Everything external is pinned, and the build fails rather than drifting: the Python
base image by tag *and* digest, `exiftool` by its Debian package version (verified
against `exiftool -ver`), and `immich-go` by release version and per-architecture
SHA-256 of the release asset. The verified versions become image labels and
`/etc/photo-pipeline/versions.json`, which `diagnostics` reports as `tools[].pinned`
beside the version actually installed — so a replaced binary shows up as a
`tool_version_drift` warning instead of as a misparsed upload report.
| build argument | default | why change it |
|---|---|---|
| `UID` / `GID` | `1000` | must match the owner of the mounted photo library |
| `PYTHON_IMAGE` | pinned digest | upgrading the base image |
| `EXIFTOOL_VERSION` | Debian package version | upgrading exiftool |
| `IMMICH_GO_VERSION` + `IMMICH_GO_SHA256_AMD64`/`_ARM64` | pinned release | upgrading the uploader (take the digests from that release's `checksums.txt`) |
The first argument is the role, and every other management command still works:
```bash
docker run --rm -v /srv/photos:/srv/photos -v pp-data:/data \
-e PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos photo-pipeline:dev migrate
docker run -d -p 127.0.0.1:8000:8000 -v /srv/photos:/srv/photos -v pp-data:/data \
-e PHOTO_PIPELINE_HOST=0.0.0.0 -e PHOTO_PIPELINE_ACCESS_SECRET=... \
-e PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos photo-pipeline:dev serve
docker run -d -v /srv/photos:/srv/photos -v pp-data:/data \
-e PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos photo-pipeline:dev worker
```
One role per container: `serve` and `worker` each take the library process lock for
their role (US07-05), so no supervisor starts both. The container refuses to run as
UID 0 — files it renames or writes must keep the ownership the host library expects —
and `/data` is the persistent volume holding the database, journals, backups, and
thumbnail cache. Binding to `0.0.0.0` makes the access secret mandatory
([above](#reaching-it-through-a-hostname-or-proxy-us08-01)); `serve` refuses to start
without it. The declared `HEALTHCHECK` polls `/api/v1/health/ready`, so a container
whose database is unmigrated or misconfigured is never reported healthy.
## Composed runtime (US08-03)
`docker-compose.yml` is the deployment: one `serve` container, one `worker`
container, one bind-mounted library, one data volume, and a one-shot `migrate` that
both roles wait for.
```bash
cp .env.example .env && $EDITOR .env # nothing has a value in it; fill in yours
docker compose up -d --build
```
`.env.example` lists every `PHOTO_PIPELINE_*` variable with its default in a comment
and no values at all. The three the composition cannot start without are
`PHOTO_PIPELINE_LIBRARY_HOST_PATH` (the library on this host),
`PHOTO_PIPELINE_LIBRARY_ROOTS` (where it is mounted *inside* the container), and
`PHOTO_PIPELINE_ACCESS_SECRET` — publishing the port means the app is reachable from
outside the container, so US08-01 makes the secret mandatory. Set
`PHOTO_PIPELINE_UID`/`GID` to the owner of the library: what the containers rename
and rewrite keeps that ownership.
| invariant | how the composition keeps it |
|---|---|
| one writer | `serve` and `worker` take their role's library lock (US07-05) in the shared `/data` volume, so `--scale worker=2` is refused by the lock, not by convention |
| migrations first | `migrate` runs the backup-then-migrate path and must exit 0 before `api` and `worker` start; a failed upgrade leaves the previous database and its pre-migration backup intact |
| container paths | `PHOTO_PIPELINE_LIBRARY_ROOTS` is both the mount target and the configured root; a root that is not mounted makes `serve`/`worker` exit 5 at startup instead of writing into the container's throwaway layer |
| loopback by default | the port is published to `127.0.0.1` unless `PHOTO_PIPELINE_PUBLISH_ADDRESS` says otherwise, and exposing it needs the hostname in `PHOTO_PIPELINE_ALLOWED_HOSTS` plus the access secret |
| restart safety | both roles are `restart: unless-stopped` with a 30 s stop grace period, and a job interrupted by a restart resumes exactly as it does on a host restart |
The `data` volume holds the database, its write-ahead log, the thumbnail cache,
journals, and backups. **It must stay on a local filesystem** — SQLite in WAL mode
needs real local locking, so NFS, SMB, and network volume drivers are unsupported
there and corrupt the database rather than slow it down. The library bind mount has
no such restriction.
Operating the deployment is operating the same CLI:
```bash
docker compose run --rm --no-deps api diagnostics
docker compose run --rm --no-deps api backup --reason pre-upgrade
docker compose run --rm --no-deps api verify-backup /data/backups/<name>
docker compose run --rm --no-deps api restore /data/backups/<name> --into /data/restored
docker compose logs -f worker
```
`--no-deps` keeps a one-off command from starting a second stack; `api` is only the
service the command borrows the image and mounts from. `restore` is deliberately not
an API call: it replaces the state of an installation and belongs to a stopped one,
so stop `api` and `worker` first and restart them against the restored directory.
## Publishing and deploying (US08-04)
Two workflows in `.gitea/workflows/` make `main` the only path to the running stack:
| workflow | runs on | does |
|---|---|---|
| `test.yml` (**Test**) | every pull request, and every push to `main` | installs exiftool, the package, and Chromium, then runs the suites configured in `work_item/.work-item.yml` |
| `deploy.yml` (**Deploy**) | a **successful** `Test` run on `main`, or manual dispatch | builds the image, pushes it, triggers the Portainer webhook, prunes dangling layers |
Deploy waits for `Test` through `workflow_run`, so publishing is downstream of a green
suite rather than of a push. `completed` is not `success`: the publish job runs only on
`conclusion == 'success'`. Because an unmatched `workflow_run` filter does not fail but
simply never fires, `tests/integration/test_deploy_workflows.py` asserts the link from
both ends — renaming either workflow, or dropping one of the configured suites out of
the test job, fails that test rather than silently unhooking the gate.
The image is `git.domverse-berlin.eu/domverse/photoanalyzer`, tagged with the commit
SHA and `latest`. The commit tag is pushed first, so a `latest` that exists always has
a commit tag beside it and **a rollback is a tag change, not a rebuild**:
```bash
docker pull git.domverse-berlin.eu/domverse/photoanalyzer:<known-good-sha>
docker tag git.domverse-berlin.eu/domverse/photoanalyzer:<known-good-sha> \
git.domverse-berlin.eu/domverse/photoanalyzer:latest
docker push git.domverse-berlin.eu/domverse/photoanalyzer:latest
# then trigger the Portainer webhook, or redeploy the stack from Portainer
```
Manual dispatch defaults to **dry run**: it builds and pushes `scratch-<sha>` only, and
touches neither `latest` nor the running stack. Clear the `dry_run` input to publish by
hand. A workflow-level `concurrency: deploy` group with `cancel-in-progress: false`
queues deploys instead of overlapping them, so two commits can never race to `latest`.
### Secrets
Repository secrets — used only by the pipeline:
| secret | purpose |
|---|---|
| `REGISTRY_USER` | registry user with `write:package` |
| `REGISTRY_TOKEN` | that user's token, passed on stdin, never as an argument |
| `PORTAINER_WEBHOOK_URL` | the stack's auto-update POST URL; `curl --fail` makes a refused redeploy a failed workflow |
Runtime secrets — `PHOTO_PIPELINE_VISION_API_KEY`, `PHOTO_PIPELINE_IMMICH_API_KEY`,
`PHOTO_PIPELINE_ACCESS_SECRET` — are **not** repository secrets and are not in the
image. The stack is managed by Portainer from git (`docker-compose.yml`), and those
values live in the Portainer stack's environment, so rotation is one place and a
repository read discloses nothing. A test asserts the workflows never name them.
## Container acceptance gate (US08-05)
The deployed container is verified the way the host application is, by one command:
```bash
work_item/scripts/python -m photo_pipeline container-gate --output gate-evidence
```
It builds the image, provisions the composition against a **temporary fixture library
on a bind mount and an isolated data volume**, runs the `phase_h` journeys against it,
destroys every stack afterwards, and writes `release-report.json`, `logs/container.log`,
and `CHECKSUMS.sha256` into the evidence directory. It exits non-zero when a journey
fails, when the story matrix has a hole, **or when any check skipped at all** — unlike
the release gate, this one accepts no environment excuse: a run that never reached the
containers proved nothing about them.
The journeys (`tests/e2e/test_phase_h_container.py`) are:
| journey | what it proves |
|---|---|
| browser | discovery, duplicate review, analysis, album proposal, rename, upload preflight, and archive views, driven through the containerized frontend — including a rename that really moves the operator's folder on the bind mount |
| upgrade | the previous version's image runs first, then this one against the same volume: schema at the new head, assets, analysis results, job progress, the unapplied rename plan, and the thumbnail cache all survive |
| restart | `docker kill` on both containers mid-job; the job resumes, every photo ends with exactly one stored result, and only the in-flight item ever reaches the provider twice |
| security | no session refused, a forged `X-Forwarded-Host` cannot smuggle an allowed hostname past the check, a symlink out of the mounted library is refused, an unmounted library root refuses startup, and no secret appears in `docker compose logs` |
The upgrade journey builds the previous commit's tree when no published image is named;
point it at the real one with `PHOTO_PIPELINE_PREVIOUS_IMAGE`. On a Docker VM (Colima,
Docker Desktop) the fixture library must live on a shared path — it defaults to
`~/.cache/photo-pipeline`, overridable with `PHOTO_PIPELINE_TEST_MOUNT_BASE`.
CI runs this gate as the `container` job of **Test** on pushes to `main` and keeps its
evidence as a run artefact. Deploy waits for the whole `Test` workflow, so a red
container gate is a publish that does not happen.
## Testing
One offline command runs the whole suite (unit, integration, and browser
@@ -412,339 +136,3 @@ work_item/scripts/python -m pytest tests/e2e -m phase_d -q
The fault barrier is test-only configuration; without `PHOTO_PIPELINE_FAULT_AFTER`
the apply path has no crash points. Phases A–C remain green in the full run above.
### Phase E acceptance gate
Phase E (Epic E05: Immich upload) is the one stage the application cannot take back,
so its gate runs the fake-uploader suites, the black-box upload API journeys, and the
browser suite as a single command:
```bash
work_item/scripts/python -m pytest -m phase_e -q
```
- `tests/integration/test_upload_*.py` drive a **real executable** standing in for
`immich-go` through the real adapter and `subprocess` — argument construction,
output bounding, report parsing, verification, and killing a running process.
- `tests/e2e/test_phase_e_pipeline.py` drives a real server and a real durable worker
over HTTP: credential failure and an unreachable server, preflight blockers and the
explicitly approved partial scope, a new album, an exact duplicate, an upgrade, a
retryable failure and its successful retry, a lost acceptance response, verification
against Immich, an inconclusive answer resolved by an operator with evidence, bytes
edited after upload, cancellation and resume, and an interrupted attempt recovered
across a restart.
- **EXIF precedes upload** is asserted, not assumed: an album without its verified
safety and analysis checkpoints cannot be approved, and the uploader's own argv log
proves it was never executed. Each finished upload re-hashes the files in the folder
the uploader was handed and requires the persisted SHA-256/SHA-1 to match.
- **No secret is retained.** The API key is a sentinel string; after a full upload and
verification it must appear in the uploader's argv and nowhere else — not in the
database, the retained report, or any response the browser can read.
- `tests/e2e/test_uploads_ui.py` covers the browser journeys (preflight preview,
confirmation, progress, stopping a run, verification, manual resolution, stale
bytes, and recovery after a restart).
Phases A–D remain green in the full run above.
### Phase F acceptance gate
Phase F (Epic E06: archive lifecycle) is the only stage that *removes* originals
from the library, and the only one whose storage can walk away in someone's bag.
One command runs the archive fault-injection suites, the black-box archive and
restore API journeys, and the browser suite:
```bash
work_item/scripts/python -m pytest -m phase_f -q
```
- `tests/integration/test_archive_*.py` and `tests/integration/test_restore.py`
drive real files on real filesystems: preflight against a mounted, missing,
swapped, read-only, or full medium; copy-verify-remove and the same-filesystem
move path; and a crash at **every** persisted journal transition in both transfer
modes, asserting that no source is ever removed without a durable, byte-identical
archive copy.
- `tests/e2e/test_phase_f_pipeline.py` drives a real server and a real durable
worker over HTTP: preflight blockers (offline medium, wrong volume, insufficient
capacity, bytes changed after upload), a verified archive whose manifest, hashes,
and path history are checked on the medium itself, a worker killed at each of
`transferring`, `verified`, `removing`, `source_removed`, and `complete`, the
evidence-based recovery that follows, offline deduplication of an exact and a
fuzzy copy while the medium is away, mount return, restore, and a collision that
restores beside its occupant.
- **Archived is not missing.** An unmounted medium leaves its photos
`archived_offline` — still hashed, still in the duplicate indexes, still
previewable through their protected thumbnails — and a rescan neither prunes nor
flags them.
- **Ambiguity is never guessed.** A journal state the medium contradicts stays
`manual`, offers no automatic action, and keeps blocking further archiving until
a human decides.
- `tests/e2e/test_archive_ui.py` covers the browser journeys (preview with
destination identity and reclaimable bytes, blockers and mount instructions,
progress split into transfer/verification/removal, interruption and recovery,
offline browsing, restore, collision, keyboard confirmation, and reload).
Phases A–E remain green in the full run above.
## Media and metadata hardening (US07-03)
Every pixel the application reads goes through `photo_pipeline/imaging.py`: the
declared dimensions are checked before anything is decoded, Pillow's
decompression-bomb warning is treated as a refusal, JPEG decodes near the requested
size, and each decoder failure becomes one of two typed errors. A damaged file is a
per-item error with a persisted code, never a failed scan or a dead worker.
Every metadata stage ends with an EXIF checkpoint (`services/exif_checkpoint.py`):
snapshot, write the owned keywords, read back, prove the owned fields landed and that
nothing else moved, refresh the file hash. A field the stage does not own that
changed anyway makes the checkpoint `divergent` — recorded in `exif_projections`,
shown in the review queue, never repaired behind the user's back, and not counted as
verified, so upload stays blocked.
The golden corpus that proves all of it is generated, not committed:
`tests/fixtures/media_corpus.py` declares every format, orientation, profile,
damage, and metadata case with its expected outcome, and the suite regenerates it
twice to prove it does not drift.
```bash
work_item/scripts/python -m pytest tests/integration/test_media_hardening.py tests/integration/test_exif_checkpoints.py -q
```
## Concurrency and crash recovery (US07-04)
Crash safety is proven by crashing. `photo_pipeline/faults.py` defines the control
points — the persisted transitions of the rename, archive, EXIF, upload, and job
lanes — and arms one only when `PHOTO_PIPELINE_FAULT_AFTER` names it, at which
point the process dies the way a `SIGKILL` does. There is no endpoint and no
configuration field that can reach a barrier; a deployment that never sets the
variable can never hit one.
The race suite runs each scenario several times with a seed recorded on the test
result (`race_seed`) and asserts invariants rather than schedules: work is never
claimed or executed twice, a stale fencing token never commits, no file body is
lost or overwritten, and the database still passes `PRAGMA integrity_check`.
```bash
work_item/scripts/python -m pytest tests/integration/test_concurrency_races.py \
tests/integration/test_fault_matrix.py tests/e2e/test_crash_recovery.py -q
# replay a failure, or soak for new interleavings
PHOTO_PIPELINE_RACE_SEED=1234 PHOTO_PIPELINE_RACE_REPEATS=50 \
work_item/scripts/python -m pytest tests/integration/test_concurrency_races.py -q
```
Any failing test keeps its evidence: the temporary database (with its write-ahead
log), the journals, the logs, the recorded seed, and a SHA-256 manifest of every
file in the temporary library are copied to `.artifacts/<test id>/` before pytest
deletes the directory. Point `PHOTO_PIPELINE_TEST_ARTIFACTS` elsewhere to collect
them from CI.
## Release gate (US07-07)
One command runs every suite in an isolated stack and keeps the evidence:
```bash
work_item/scripts/python -m photo_pipeline release-gate --output data/release/$(date -u +%Y%m%dT%H%M%SZ)
```
It fails — and exits non-zero — when any stage fails, when a suite skips a test for
a reason that is not a documented environment limit (`exiftool not installed`,
`root ignores directory permissions`), or when the story matrix has a hole. The
evidence directory holds `release-report.json` (revision, per-stage result, timings,
summaries), `logs/<stage>.log`, and `CHECKSUMS.sha256` over both.
**The story matrix** lives in `tests/story_traceability.json`: every story under
`delivery_backlog/stories/` is either mapped to test files that exist, or listed in
`planned` as an accepted but unimplemented story. A story that is neither, or a
mapping to a file that has been deleted, fails the gate.
**The journey** (`tests/e2e/test_release_journey.py`) takes one fresh library through
discovery, duplicate review, safety, analysis, EXIF verification, album proposal,
guarded rename, rescan, upload with server-side verification, archive, offline
deduplication, and restore — over HTTP against real server and worker processes,
with a full restart in the middle and at the end.
### Real-library dry run and approval
Before the application is pointed at photos that cannot be replaced:
```bash
work_item/scripts/python -m photo_pipeline dry-run --output dry-run.json
work_item/scripts/python -m photo_pipeline approve-dry-run dry-run.json --approver "$(whoami)"
```
The dry run is strictly read-only: it opens no file for writing, writes no database
row, and reports what it found — file counts by extension, folders, bytes, unreadable
files, excluded directories, and a reconciliation against what the database already
knows (already registered, new, recorded but absent). Set
`PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL=1` and **every mutating API request is
refused with `403 dry_run_not_approved`** until a report for exactly those library
roots has been approved. Reading stays open — you have to be able to see what was
found in order to approve it — and so does taking a backup. Change the library roots
and the approval no longer applies: it approves that reconciliation, not the idea of
mutating.
## Performance budgets (US07-06)
Budgets are measured, not asserted in prose. `python -m photo_pipeline benchmark`
builds a synthetic library of a stated size, runs the same scenarios every time,
writes a machine-readable report, and **exits non-zero when a budget is breached**.
```bash
work_item/scripts/python -m photo_pipeline benchmark --profile smoke # ~2 s, runs in CI
work_item/scripts/python -m photo_pipeline benchmark --profile short # 25k assets
work_item/scripts/python -m photo_pipeline benchmark --profile full \
--output data/benchmarks/full.json # 25k + 100k
work_item/scripts/python -m photo_pipeline benchmark --profile huge \
--soak-seconds 3600 --output data/benchmarks/soak.json # 500k + soak
```
| Metric | Budget | Why |
|---|---|---|
| `latency_p95_ms` | 250 ms | a list or search page must feel immediate |
| `latency_max_ms` | 2 000 ms | no single page may stall the review flow |
| `rss_growth_bytes` | 400 MB | a run must not leak the library |
| `open_files` | 256 | file descriptors are a hard operating-system limit |
| `wal_bytes` | 200 MB | a growing write-ahead log means checkpoints are starving |
| `queue_depth` | 1 000 | an unbounded queue is an out-of-memory in waiting |
| `cache_over_quota_bytes` | 0 | the thumbnail cache has to respect its quota |
Measured on the reference machine (Apple Silicon, SQLite WAL), p95 per scenario:
| Scenario | 25k | 100k |
|---|---|---|
| `inventory_page` | 0.5 ms | 0.6 ms |
| `library_search` | 4.8 ms | 17.1 ms |
| `library_stats` | 56.8 ms | 197.4 ms |
| `workflow_readiness` | 52.6 ms | 235.4 ms |
| `duplicate_cluster_list` | 0.6 ms | 0.5 ms |
| `duplicate_cluster_page` | 1.7 ms | 1.7 ms |
CI runs the `smoke` profile through `tests/integration/test_performance_budgets.py`;
the 25k/100k/500k matrix and the multi-hour soak belong to scheduled infrastructure,
because minutes of build time do not belong in the suite that runs on every change.
**Exceptions.** A budget that cannot be met is not a warning to ignore: it goes into
`APPROVED_EXCEPTIONS` in `photo_pipeline/services/benchmarks.py` with its raised
limit, who approved it, why, and a review date. Every report lists the exceptions it
applied, so a release review sees them.
Approved today, both for the 500k `huge` profile only, review by 2027-02-17:
| Scenario | Measured at 500k | Raised limit |
|---|---|---|
| `library_stats` | 1.08 s p95 · 3.2 s max | 1.5 s p95 · 4 s max |
| `workflow_readiness` | 1.40 s p95 · 3.3 s max | 1.8 s p95 · 4 s max |
Both are library-wide aggregates — the current safety decision of every asset, and
the album/tag/year breakdown of every analysis row — and both meet the 250 ms budget
at the 100k rows the concept sets it for. Beyond that they are linear against one
SQLite writer; the fix is denormalized totals or the planned PostgreSQL transition,
not a query tweak. Everything else at 500k is inside budget, and a soak at that size
grows neither resident memory nor the job queue.
## Backup and recovery (US07-05)
Backups go through SQLite's online backup API, never a file copy: with WAL enabled
the `.db` file alone is missing every committed page still in the write-ahead log.
Each backup is a directory under `data/backups/` holding the snapshot and a
`manifest.json` describing it — schema revision, SHA-256, row counts, the archive
media the library depends on, and which configuration was set. Secrets are recorded
as `configured`, never as values, so a manifest is safe to attach to a bug report.
```bash
work_item/scripts/python -m photo_pipeline backup --reason before-upgrade --keep 7
work_item/scripts/python -m photo_pipeline verify-backup data/backups/<name>
work_item/scripts/python -m photo_pipeline diagnostics
```
The same is available at `GET /api/v1/diagnostics`, `GET|POST /api/v1/backups`,
`GET /api/v1/backups/{name}/verify`, and `POST /api/v1/backups/prune`. **Restore is
not an endpoint** — it replaces the state of an installation, so it belongs to a
stopped one and a person at a terminal.
### Integrity check
`verify-backup` runs `PRAGMA integrity_check` (structure) *and*
`PRAGMA foreign_key_check` (references), compares the snapshot's SHA-256 with the
manifest, and re-counts every table the manifest recorded. Any mismatch — bit rot, a
truncated copy, a "repaired" snapshot — fails the check, and `restore` refuses a
backup that does not verify.
### Restore drill
1. Stop the server and the worker.
2. `python -m photo_pipeline verify-backup data/backups/<name>` — never restore an
unverified snapshot.
3. `python -m photo_pipeline restore data/backups/<name> --into /path/to/fresh-data`
(a target that already holds a database is refused; recovering in place means
moving the old data directory aside first).
4. Point `PHOTO_PIPELINE_DATA_DIR` at the restored directory and run
`python -m photo_pipeline migrate`.
5. Run an inventory scan so paths are reconciled against the real library.
6. Mount every archive location named in the manifest before archiving again — the
database records where archived originals are, but it does not contain them.
Practise this against a copy before you need it; the drill is exercised
automatically by `tests/integration/test_backup_recovery.py`.
### Failed migration
A pending schema upgrade is snapshotted first (`reason: pre-migration`), by both the
API startup and `python -m photo_pipeline migrate`. If a migration fails, the error
log names the backup directory: stop everything and run the restore drill against
it. An up-to-date database is not backed up again on every start.
### Archive media
Archived originals live on their medium, not in the backup. The manifest lists every
archive location with its `media_id` and whether it was mounted when the backup was
taken. Keep one copy of each medium off-site, and remount a location before
restoring assets from it.
### Retention and disk
`--keep N` (default 7) prunes the oldest backups and never the newest.
`diagnostics` reports the database, write-ahead log, thumbnail cache, uploader
reports, backups, and logs separately, with free space and warnings for low disk
(`disk_low`, `disk_critical`), a cache over its quota, a write-ahead log outgrowing
its database, and a legacy CLI writing the library.
### Process locking
`serve` and `worker` take a JSON lock in the data directory (`api.lock.json`,
`worker.lock.json`). A second worker exits `2` and names the holder; a lock whose
process is gone is taken over. If the frozen CLI's state files are being written,
both refuse with exit `3` — `--allow-legacy` overrides, and you own the outcome.
## Legacy CLI archive
The command-line tools this application was extracted from are frozen in
`legacy_cli_archive/` (US07-01): the original sources, their docs, the dependency
lock they were last verified against, schema notes, a redacted sample
configuration, the donor ledger, and a checksum for every file.
```bash
cd legacy_cli_archive && shasum -a 256 -c CHECKSUMS.sha256 # verify the archive
work_item/scripts/python -m pytest tests/unit/test_legacy_archive.py -q # lint it
```
They are reference material and rollback evidence only. No module under
`photo_pipeline/` imports or executes them, the archive is not on the application's
import path, and `tests/unit/test_legacy_archive.py` enforces that along with the
checksums and the redaction. Only the two suites that compare *against* the donors —
`tests/characterization/` and `tests/integration/test_safety_parity.py` — put the
archived sources on `sys.path`.
The last path-keyed state they owned, `nsfw_scores.csv`, is imported once and then
left alone:
```bash
work_item/scripts/python -m photo_pipeline import-legacy-scores /path/to/nsfw_scores.csv --dry-run
```
The import writes scored-but-unreviewed `safety_reviews` rows onto stable asset ids,
never invents an asset for an unknown path, never overwrites a human decision, and
writes a reconciliation report to the data directory saying exactly what it did.
`legacy_cli_archive/donor_ledger.yaml` records every migrated behavior with its
target, the tests that pin the donor, the tests that prove the replacement, and each
intentional delta; rows still marked `pending` name the story that will resolve them.

0
legacy_cli_archive/src/compare_models.py → compare_models.py
View File

32
delivery_backlog/E08-container-deployment.md
View File

@@ -1,32 +0,0 @@
# E08 — Container Deployment
Concept phase: none. This epic is a delivery-format addition on top of the concept:
the same application, same safety invariants, packaged as a Docker image and deployed
continuously from Gitea Actions instead of being started by hand from a working copy.
It does not change the product scope in
[`INTEGRATED_PIPELINE_CONCEPT.md`](../INTEGRATED_PIPELINE_CONCEPT.md). SQLite stays the
store, one worker stays the writer, the library process lock stays authoritative, and
no path outside the configured library roots becomes reachable because the process now
runs in a container.
One decision does extend the concept and is made here explicitly: the application may
be reached through a reverse proxy under a real hostname, not only over loopback. That
requires a configurable trust boundary and an authentication gate, because the
loopback-only checks of US07-02 are what currently stand in for authentication.
## Stories
1. [US08-01 — Make the trust boundary configurable and authenticated](stories/US08-01-trusted-hosts-auth.md)
2. [US08-02 — Build a reproducible application image](stories/US08-02-container-image.md)
3. [US08-03 — Compose the runtime and mount the library safely](stories/US08-03-compose-runtime.md)
4. [US08-04 — Publish and deploy from Gitea Actions](stories/US08-04-gitea-cicd.md)
5. [US08-05 — Automate container deployment acceptance](stories/US08-05-container-e2e.md)
## Epic outcome
A tagged image built from `main` runs the API and the worker as separate containers
against a mounted library and a persistent data volume, is published to the Gitea
registry, is redeployed by webhook, survives restart and upgrade with its database and
journals intact, and refuses every request that a loopback deployment would have
refused.

77
delivery_backlog/E09-documentation.md
View File

@@ -1,77 +0,0 @@
# E09 — Product Documentation
Concept phase: none. Like [E08](E08-container-deployment.md), this epic is a delivery
addition rather than a product-scope change: the same application, documented well
enough that somebody who did not build it can install it, understand it, and operate it
without reading the source.
It does not change the product scope in
[`INTEGRATED_PIPELINE_CONCEPT.md`](../INTEGRATED_PIPELINE_CONCEPT.md). No safety
invariant moves, no schema changes, no new runtime capability. The one change to the
running application is a documentation view and the static assets it needs.
## Why the application serves its own documentation
The manuals describe an application that is reached over HTTP behind an access secret.
Documentation that lives only in the repository is unreachable from the deployment it
describes: an operator who has just been handed a URL and a secret has no Gitea account
in front of them. So the same markdown files are both the repository's documentation and
the deployment's `/docs` view, and neither is a copy of the other.
## Decisions made in this epic
**Markdown is the source.** Everything is written as markdown under `docs/`, so it is
reviewable in a diff, readable on Gitea, and renderable in the app. No documentation
format that only a tool can read.
**The renderer is vendored, not written and not fetched.** `marked` (MIT, no
dependencies, ships an ES module) is pinned and committed under
`frontend/js/vendor/`. A CDN is not an option: the application is deployed to a network
whose outbound access is not assumed, and `default-src 'self'` forbids it.
**Diagrams are mermaid, rendered client-side, with the script boundary intact.** The
claim that mermaid requires `'unsafe-eval'` was tested rather than believed: mermaid 11's
bundle contains no `eval(` and no `new Function` — only a lodash `Function("return this")`
global-detection fallback that short-circuits on `globalThis` and never executes.
Rendered under this application's exact CSP, a flowchart produced a 15.8 KB SVG and
raised **no `script-src` violation**. What it does raise is `style-src`: mermaid styles
its output with an injected `<style>` element and `style=` attributes.
So `script-src 'self'` stays exactly as US07-02 and US08-01 left it, and `style-src`
gains `'unsafe-inline'`. That relaxation is bounded on purpose: with `script-src` intact
no injected markup can execute, and with `img-src`, `connect-src`, and `font-src` all
still `'self'`, the CSS-based exfiltration channels stay closed. What remains is
defacement of a page the operator is already authenticated on.
The rejected alternative is recorded because it may become the better trade later:
pre-rendering each diagram to a committed SVG with the already-installed Playwright
(no Node toolchain needed) and serving that, which would leave CSP untouched entirely at
the cost of a generator and a freshness check. If `style-src 'unsafe-inline'` is ever
judged too much, that is the migration, and it does not change a single markdown file.
**Screenshots are produced, not pasted.** Every screenshot in the user manual is captured
by Playwright from the real application against a temporary fixture library, by the same
kind of code that already drives the browser suites. A screenshot nobody can regenerate
is a screenshot that silently stops being true.
**The documentation is held to the code by tests.** Configuration keys, CLI commands,
exit codes, API error codes, job and journal states, and module names all appear in both
the code and the manuals. Each of those is cross-checked, so the failure mode of stale
documentation is a red test rather than a misled operator.
## Stories
1. [US09-01 — Serve the documentation inside the application](stories/US09-01-docs-in-app.md)
2. [US09-02 — Write the installation and operations manual](stories/US09-02-installation-manual.md)
3. [US09-03 — Write the architecture overview](stories/US09-03-architecture-overview.md)
4. [US09-04 — Write the user manual with generated screenshots](stories/US09-04-user-manual.md)
5. [US09-05 — Automate documentation acceptance](stories/US09-05-docs-gate.md)
## Epic outcome
A deployed instance serves its own installation manual, architecture overview, and
illustrated user manual at `/app/#/docs`, rendered from the same markdown files that are
readable in the repository. Screenshots are regenerated from the running application,
every documented command, setting, state, and error code is cross-checked against the
code that implements it, and one gate fails the build when documentation and application
disagree.

9
delivery_backlog/README.md
View File

@@ -2,14 +2,11 @@
This backlog decomposes the phases in
[`INTEGRATED_PIPELINE_CONCEPT.md`](../INTEGRATED_PIPELINE_CONCEPT.md) into seven
epics and small, independently verifiable user stories, plus two delivery-format
epics: E08 packages the released application as a deployable container, and E09
documents it for the people who install, operate, and use it.
epics and small, independently verifiable user stories.
## Numbering and file naming
- Epics: `E01` through `E07`, matching concept Phases A through G; `E08` and `E09`
have no concept phase and must not change product scope.
- Epics: `E01` through `E07`, matching concept Phases A through G.
- Stories: `US<epic>-<sequence>`, for example `US03-02`.
- Epic files: `E01-<slug>.md`.
- Story files: `stories/US01-01-<slug>.md`.
@@ -39,8 +36,6 @@ documents it for the people who install, operate, and use it.
5. [E05 — Immich upload](E05-immich-upload.md)
6. [E06 — Archive lifecycle](E06-archive-lifecycle.md)
7. [E07 — Hardening and release](E07-hardening-release.md)
8. [E08 — Container deployment](E08-container-deployment.md)
9. [E09 — Product documentation](E09-documentation.md)
## Shared definition of done

42
delivery_backlog/stories/US08-01-trusted-hosts-auth.md
View File

@@ -1,42 +0,0 @@
# US08-01 — Make the Trust Boundary Configurable and Authenticated
Epic: [E08](../E08-container-deployment.md)
As an operator, I want to reach the application through my own hostname without
weakening it, so a container behind a reverse proxy is as safe as the loopback
deployment it replaces.
## Context
`photo_pipeline/api/security.py` refuses any request whose `Host` or `Origin` is not
loopback. That check is the current stand-in for authentication: whoever can reach
`127.0.0.1:8000` is the owner. Behind a proxy the hostname is no longer loopback, so
relaxing the check without adding an authentication gate would publish the library.
## Acceptance criteria
- Allowed hosts and origins come from configuration (`PHOTO_PIPELINE_*`), default to
the current loopback set, and an unset configuration behaves exactly as today.
- Whenever a non-loopback host is configured, startup requires an access secret and
refuses to serve without one; loopback-only deployments keep working with no secret.
- The secret is exchanged for the existing session cookie and CSRF token through the
bootstrap endpoint; every protected route keeps its current session and CSRF
requirements unchanged.
- Forwarded headers (`X-Forwarded-Proto`, `X-Forwarded-Host`) are honored only from a
configured trusted proxy and ignored otherwise, so a client cannot forge its origin.
- Cookies are marked `Secure` when the effective external scheme is HTTPS.
- Failed authentication is rate-limited and logged without the secret, the session id,
or any request body.
- Health endpoints stay reachable without the secret; nothing else does.
## Automated tests
- Unit tests for host/origin evaluation across loopback default, configured host,
unconfigured host, forged forwarded headers, and trusted-proxy forwarded headers.
- Integration tests: startup refusal without a secret, successful exchange, wrong
secret, replay of an old session, cross-site request, and unauthenticated access to
every route class.
## Dependencies
- US07-02

41
delivery_backlog/stories/US08-02-container-image.md
View File

@@ -1,41 +0,0 @@
# US08-02 — Build a Reproducible Application Image
Epic: [E08](../E08-container-deployment.md)
As an operator, I want one image that can run either application role, so deployment is
a pull instead of a Python environment I have to reproduce by hand.
## Context
The application shells out to `exiftool` and `immich-go`, writes into the library as a
normal filesystem user, and serves a static frontend from `frontend/`. All three have to
be true inside the image, or the container starts and then fails on the first real
operation.
## Acceptance criteria
- A `Dockerfile` builds from a pinned Python base, installs the project and its runtime
dependencies, and contains no test, playwright, or build-only tooling in the final
layer.
- `exiftool` and `immich-go` are present at pinned versions, and their versions are
recorded in the image and reported by `python -m photo_pipeline diagnostics`.
- The image runs as a non-root user whose UID/GID are build-time arguments, so files
the application renames or writes keep the ownership the host library expects.
- One entrypoint selects the role: `serve` or `worker`, passing through the existing
CLI arguments; no supervisor runs two roles in one container.
- `serve` containers declare a `HEALTHCHECK` against `/api/v1/health/ready`, so an
unmigrated or misconfigured database is not reported healthy.
- The image contains no secrets, no library data, no database, and no `.git`; the build
context is constrained by `.dockerignore`.
- Image build is reproducible from a clean checkout and documented in `README.md`.
## Automated tests
- A build-and-run test asserts the image starts, reports ready, serves the frontend
index, and returns the pinned `exiftool` and `immich-go` versions.
- A test asserts the container refuses to run as UID 0 and that a file created by the
container is owned by the configured UID/GID.
## Dependencies
- US07-05

47
delivery_backlog/stories/US08-03-compose-runtime.md
View File

@@ -1,47 +0,0 @@
# US08-03 — Compose the Runtime and Mount the Library Safely
Epic: [E08](../E08-container-deployment.md)
As an operator, I want a single compose file that runs the API and the worker against my
real library, so a deployment is one command and the safety invariants survive it.
## Context
The library process lock (US07-05) assumes both roles see the same lock file, and SQLite
in WAL mode assumes a real local filesystem. Container path policy is the same problem
as host path policy with a new failure mode: the configured library roots must name the
in-container mount paths, not the host paths.
## Acceptance criteria
- `docker-compose.yml` runs exactly one `serve` and one `worker` container from the same
image and the same data volume, and a second worker is refused by the existing lock
rather than by convention.
- The library is a bind mount; `PHOTO_PIPELINE_LIBRARY_ROOTS` names the container-side
paths, and a mismatch between mounted and configured roots fails at startup with a
clear message instead of at the first write.
- The data volume holds the database, WAL, thumbnail cache, and backups on a local
filesystem; the composition documents that a network mount is unsupported for it.
- Migrations run before `serve` and `worker` accept work, using the existing backup-then-
migrate path, and an upgrade that fails leaves the previous database intact.
- Configuration and secrets come from the environment, never from the image or a
committed file; a `.env.example` lists every `PHOTO_PIPELINE_*` variable with safe
defaults and no values.
- The API port is published to host loopback by default; exposing it publicly requires
the configured hostname and access secret from US08-01.
- Containers restart automatically, and a restart mid-job resumes exactly as a host
restart does today.
- Backup, verify-backup, restore, and diagnostics are documented as container commands
and work against the mounted volumes.
## Automated tests
- An integration test brings the composition up against a temporary fixture library,
runs a job, restarts both containers, and asserts the job resumes and the database is
intact.
- Tests for: second worker refused, library-root mismatch refused at startup, failed
migration leaving the previous database restorable.
## Dependencies
- US08-01, US08-02

40
delivery_backlog/stories/US08-04-gitea-cicd.md
View File

@@ -1,40 +0,0 @@
# US08-04 — Publish and Deploy from Gitea Actions
Epic: [E08](../E08-container-deployment.md)
As a release owner, I want `main` to build, publish, and redeploy the image
automatically, so deployment is the same reproducible path every time.
## Context
The workflow is adapted from the `crowdsec-admin` deployment workflow
(`.gitea/workflows/deploy.yml` in that repository): build, log in to the Gitea registry,
push, trigger a Portainer webhook, prune. This project needs the same shape plus a test
gate, because unlike that project it has a required suite that must not be skipped.
## Acceptance criteria
- `.gitea/workflows/` contains a test workflow that runs on pull requests and on `main`,
executing the configured required suites, and a deploy workflow that runs only after
the tests pass on `main` and on manual dispatch.
- The deploy workflow publishes to `git.domverse-berlin.eu` under this project's own
image path, tagged `latest` and the commit SHA, so a rollback is a tag change.
- Registry credentials and the Portainer webhook come from repository secrets; runtime
secrets (vision key, Immich key, access secret) stay in the Portainer stack and never
enter the repository or the image.
- Redeploy is triggered by webhook and the workflow fails when the webhook call fails.
- Dangling images are pruned; published tags are not.
- A concurrency guard prevents two deploys of different commits overlapping.
- `README.md` documents the required secrets, the image path, the rollback procedure,
and that the stack is managed by Portainer from git.
## Automated tests
- Workflow files are validated (syntax and required job/step names) by a repository test
so a rename cannot silently disable the test gate.
- A dry-run job builds and pushes to a scratch tag on manual dispatch without touching
`latest` or triggering a redeploy.
## Dependencies
- US08-02, US08-03

30
delivery_backlog/stories/US08-05-container-e2e.md
View File

@@ -1,30 +0,0 @@
# US08-05 — Automate Container Deployment Acceptance
Epic: [E08](../E08-container-deployment.md)
As a release owner, I want one automated gate that proves the deployed container, so the
packaged application is verified the same way the host application is.
## Acceptance criteria
- One documented command provisions the composition from the built image against a
temporary fixture library and an isolated data volume, and destroys it afterwards.
- A browser journey against the containerized application covers discovery, duplicate
review, analysis, album proposal, rename, upload preflight, and archive views.
- An upgrade journey runs the previous published image, then the new one, and asserts
migrations, journals, jobs, and the thumbnail cache survive.
- A restart journey kills both containers mid-job and asserts resume without duplicate
side effects.
- Security gates run against the deployed instance: unauthenticated access refused,
forged forwarded headers refused, paths outside the mounted library roots refused, and
no secret in container logs.
- Evidence is retained per run and the gate fails on any skipped required check.
## Automated tests
- The container acceptance suite runs on a `phase_h` marker in CI on `main` and before a
published deploy; earlier epic suites keep running unchanged.
## Dependencies
- US08-01 through US08-04

49
delivery_backlog/stories/US09-01-docs-in-app.md
View File

@@ -1,49 +0,0 @@
# US09-01 — Serve the Documentation Inside the Application
Epic: [E09](../E09-documentation.md)
As an operator who was handed a URL and an access secret, I want the manuals inside the
application I am looking at, so that understanding it does not require a repository
checkout or an internet connection.
## Acceptance criteria
- Documentation lives as markdown under `docs/`, with an index that names every page and
the order it is meant to be read in. The same files render on Gitea without
modification.
- The application serves a `/docs` view reachable from the main navigation, listing the
pages and rendering the selected one.
- Rendering uses a vendored, version-pinned markdown library committed under
`frontend/js/vendor/`. It is not fetched from a CDN, not bundled by a build step, and
not written by hand. The build records the version and a checksum of the vendored file.
- Diagrams written as ```mermaid``` fenced blocks render as diagrams. Mermaid is vendored
the same way.
- `script-src` in the Content-Security-Policy is unchanged: no `'unsafe-eval'`, no
`'unsafe-inline'`. Only `style-src` gains `'unsafe-inline'`, and the reason is recorded
where the policy is defined.
- Links between documents work in the app: a relative `../foo.md#section` link navigates
to that page and heading rather than downloading a file or leaving the application.
Every heading has a stable anchor, and a deep link to an anchor scrolls to it.
- An unknown page renders a documentation-specific not-found message with a link back to
the index, and never exposes a filesystem path.
- Documentation is readable without an access secret **or** behind the same session as
the rest of the application — whichever is chosen is stated explicitly in the story's
implementation notes and covered by a test, because an operator locked out by a
configuration mistake is exactly who needs the troubleshooting page.
- The view works with no outbound network access at all.
## Automated tests
- Unit: heading-anchor slugs (duplicates, punctuation, non-ASCII), and the rewriting of
relative markdown links into in-app routes.
- Integration: every markdown file under `docs/` is reachable from the index; every
internal link in every document resolves to a file and, where an anchor is given, to a
heading that exists; the vendored library files match their recorded checksums.
- Browser: the documentation view renders a page, follows an internal link, deep-links to
an anchor, renders a mermaid diagram to an SVG, and shows the not-found message for an
unknown page — all with **zero console errors and zero CSP violations**, asserted, not
eyeballed.
## Dependencies
- None beyond the delivered application.

46
delivery_backlog/stories/US09-02-installation-manual.md
View File

@@ -1,46 +0,0 @@
# US09-02 — Write the Installation and Operations Manual
Epic: [E09](../E09-documentation.md)
As somebody installing this application for the first time, I want one manual that takes
me from nothing to a running instance pointed at my photo library, so that I do not have
to reconstruct the procedure from the README, the compose file, and the test suite.
## Acceptance criteria
- A **host installation** path: prerequisites and their versions (Python, exiftool,
immich-go, Playwright's browser for the test suite), virtual environment, dependency
install, `.env`, database migration, starting `serve` and `worker`, and how to verify
the install succeeded.
- A **container installation** path: the published image, the compose file, the data
volume, the library bind mount and why it is mounted the way it is, published ports,
running behind a reverse proxy with `PHOTO_PIPELINE_ALLOWED_HOSTS` and
`PHOTO_PIPELINE_TRUSTED_PROXIES`, and the Portainer/webhook deployment already in use.
- A **configuration reference**: every `PHOTO_PIPELINE_*` setting with its meaning,
default, accepted values, and whether it is a secret. Secrets are described, never
exemplified with a real-looking value.
- A **first run checklist** that ends in a verified state: library discovered, worker
claiming jobs, readiness endpoint green, diagnostics clean.
- **Operations**: upgrading (including what the migration backup does), backup, verify,
restore, pruning, diagnostics, and the log locations for each role.
- **Troubleshooting**: each refusal an operator can hit at startup — every non-zero exit
code of the CLI, the trust-boundary refusal, the unmounted library root refusal, the
library lock being held, and a legacy CLI still running — with the cause and the fix.
- The manual states the safety invariants an installer must not work around: one worker
writes, `_IGNORE/` is never read, EXIF is verified before upload, and the library lock
is authoritative.
## Automated tests
- Every `PHOTO_PIPELINE_*` setting documented exists as a field on `Config`, and every
field on `Config` is documented — both directions, so a new setting cannot ship
undocumented.
- Every CLI command and flag named in the manual exists in the argument parser, and every
subcommand the parser accepts appears in the manual.
- Every exit code documented is one the CLI can actually return.
- No documented example value collides with a real secret pattern (the repository's own
credential scanner runs over `docs/`).
## Dependencies
- US09-01

47
delivery_backlog/stories/US09-03-architecture-overview.md
View File

@@ -1,47 +0,0 @@
# US09-03 — Write the Architecture Overview
Epic: [E09](../E09-documentation.md)
As a developer or reviewer new to this repository, I want an architecture overview that
explains what the pieces are and which rules they enforce, so that I can find the right
module and not violate an invariant I never knew existed.
## Acceptance criteria
- A **context diagram**: the operator, the browser application, the API and worker, the
photo library, the Immich server, the vision provider, and the archive destination —
with the direction and nature of every interaction, including which ones leave the
machine.
- A **runtime diagram**: the migrate/api/worker composition, the data volume, the library
bind mount, the SQLite database, and the process lock — showing what is shared and what
is exclusive.
- A **module map**: every package under `photo_pipeline/` with its responsibility and the
boundary it must not cross (which modules may touch the filesystem, which may call an
external provider, which own schema).
- **Key flows**, each as a diagram plus prose: the durable job lifecycle from enqueue to
recovery; the rename journal state machine including the rollback and manual-resolution
paths; the upload lifecycle through preflight, run, report ingestion, and verification.
- **The invariants and where they are enforced**, each pointing at the module that owns
it: one writer at a time, the library process lock, the path policy and library roots,
`_IGNORE/` exclusion, verified EXIF before upload, safety decisions gating the vision
provider, and restart-safety of every mutation.
- **The data model**: the tables, what identity means for an asset, and how a moved file
keeps its identity.
- A short **history and donor** section: what was migrated out of the legacy CLIs, where
the archive and its ledger live, and why they are kept.
- Diagrams are ```mermaid``` blocks so the source is diffable and Gitea renders them
natively.
## Automated tests
- Every package and top-level module under `photo_pipeline/` appears in the module map,
and every module named in the map exists — a new service cannot appear on the map's
blind side.
- Every job state, journal state, and upload batch state named in the document exists in
the code, and every state the code defines is named in the document.
- Every mermaid block parses (rendered in the browser test without an error node).
- Every code path referenced by file name exists.
## Dependencies
- US09-01

52
delivery_backlog/stories/US09-04-user-manual.md
View File

@@ -1,52 +0,0 @@
# US09-04 — Write the User Manual with Generated Screenshots
Epic: [E09](../E09-documentation.md)
As the person actually sorting a photo library, I want a manual that walks the workflow
screen by screen and tells me what each refusal means, so that I can use the application
confidently and know what it is about to do to my files before it does it.
## Acceptance criteria
- One page per workflow stage, in the order the application presents them: discovery and
inventory, duplicate review, safety review, analysis, album proposals, renames, upload,
archive, and diagnostics. Each page answers the same four questions: what this stage is
for, what I have to decide, **what it changes on disk or on the server**, and what it
refuses to do.
- A **guided first pass** that takes a new library from scan to a verified upload, naming
the point of no return in each stage and what is reversible after it.
- Every stage page carries at least one screenshot of the real application showing the
state being described.
- Screenshots are **generated** by a committed Playwright script that seeds a temporary
fixture library, drives the application, and writes the images. Regenerating them is one
documented command. No screenshot is captured by hand.
- Screenshots contain no real library paths, no personal photos, and no secrets — the
fixture library is synthetic, and this is asserted rather than assumed.
- An **error and refusal catalogue**: every `error.code` the API can return, with what
causes it, what the application did or refused to do, and what the operator should do
next. Refusals that protect data (`lock_held`, `rename_recovery_required`,
`stale_preflight`, `path_not_allowed`, `host_not_allowed`, `dry_run_not_approved`,
conflict codes) are explained as intentional, not as faults.
- A **recovery** page: an interrupted rename, an uncertain upload, a failed migration, a
restored backup — what the application does by itself and what needs a decision.
- The work-item safety allow list is extended to permit `docs/images/**`, since the
repository denies image files by default. This is an explicit, reviewed change, not a
quiet one, and it stays narrow enough that a real photo still cannot be committed.
## Automated tests
- Every `error.code` the application can emit is documented, and every code documented
exists in the code — both directions.
- Every image referenced by a documentation page exists, and every image under
`docs/images/` is referenced by a page.
- The screenshot generator runs end to end in the test environment and produces every
image the manual references, against a temporary fixture library that is destroyed
afterwards.
- Generated screenshots are checked for library paths outside the fixture root and for
the configured secrets.
- Browser: each stage page renders in the documentation view with its screenshot loaded
and no console error.
## Dependencies
- US09-01

38
delivery_backlog/stories/US09-05-docs-gate.md
View File

@@ -1,38 +0,0 @@
# US09-05 — Automate Documentation Acceptance
Epic: [E09](../E09-documentation.md)
As a release owner, I want one gate that proves the documentation still describes the
application, so that a change to the code cannot quietly make the manuals wrong.
## Acceptance criteria
- One documented command runs every documentation check and retains its evidence, in the
shape the release and container gates already use.
- The gate fails when: an internal link or anchor is dead; a page is unreachable from the
index; an image is referenced but missing or present but unreferenced; a documented
setting, command, exit code, error code, or state does not exist in the code; a code
path exists that the documentation is required to cover and does not.
- The gate regenerates the screenshots and fails when a regenerated image no longer
matches the committed one beyond a stated tolerance, so a UI change that invalidates the
manual is a red build rather than a discovery months later.
- The gate renders every documentation page in a real browser and fails on any console
error or CSP violation, and asserts that `script-src` contains neither `'unsafe-eval'`
nor `'unsafe-inline'`.
- The checks run on a `phase_i` marker; CI runs the gate, and the earlier epic suites keep
running unchanged.
- The README and the application's documentation index point at each other, so neither is
the forgotten copy.
- Documentation stories are mapped in the story traceability matrix like every other
story.
## Automated tests
- The gate's own contract is testable without a browser: a seeded broken link, a missing
image, an undocumented error code, and a stale screenshot each fail it, and a clean tree
passes.
- The full documentation suite runs on `phase_i` in CI.
## Dependencies
- US09-01 through US09-04

105
docker-compose.yml
View File

@@ -1,105 +0,0 @@
# The deployed runtime (US08-03): one API, one worker, one library, one volume.
#
# The same image (US08-02) runs both roles, so what is composed here is process
# topology, not a second application. Three invariants shape it:
#
# * one writer — `serve` and `worker` each take the library process lock for their
# role (US07-05), and both containers mount the *same* data volume, which is what
# makes the lock file visible to both. Scaling `worker` past 1 is refused by that
# lock rather than by anyone remembering not to;
# * one local filesystem — the database, its write-ahead log, the thumbnail cache,
# and the backups live in the `data` volume, and SQLite in WAL mode requires real
# local-filesystem locking. A network mount (NFS, SMB, or a cloud volume driver)
# is unsupported for it; that is a corrupted database, not a slow one;
# * one library path vocabulary — `PHOTO_PIPELINE_LIBRARY_ROOTS` names the path
# *inside* the container, which is also the bind mount's target below. A root
# that is not mounted there makes `serve` and `worker` refuse at startup instead
# of writing into the container's throwaway layer.
#
# Configuration and secrets come from the environment only: copy `.env.example` to
# `.env` and fill it in. Nothing is baked into the image and nothing with a value in
# it is committed.
#
# cp .env.example .env && $EDITOR .env
# docker compose up -d --build
#
# Operating it is operating the same CLI — `docker compose run --rm --no-deps api
# <command>` — see README, "Composed runtime".
name: photo-pipeline
x-runtime: &runtime
image: ${PHOTO_PIPELINE_IMAGE:-photo-pipeline:dev}
build:
context: .
args:
# Everything the app renames or rewrites has to stay owned by the host user
# the library already belongs to.
UID: ${PHOTO_PIPELINE_UID:-1000}
GID: ${PHOTO_PIPELINE_GID:-1000}
user: "${PHOTO_PIPELINE_UID:-1000}:${PHOTO_PIPELINE_GID:-1000}"
env_file:
- ${PHOTO_PIPELINE_ENV_FILE:-.env}
volumes:
- data:/data
- "${PHOTO_PIPELINE_LIBRARY_HOST_PATH:?set PHOTO_PIPELINE_LIBRARY_HOST_PATH to the photo library on this host}:${PHOTO_PIPELINE_LIBRARY_ROOTS:?set PHOTO_PIPELINE_LIBRARY_ROOTS to the container-side library path}"
# Jobs check for cancellation between items and leave a resumable record; a
# too-short grace period turns an orderly stop into a recovery on next start.
stop_grace_period: 30s
x-environment: &environment
# Set here rather than left to the file: these two are what the composition itself
# promises, and an `.env` that disagreed would move the database off the volume or
# the library off its mount.
PHOTO_PIPELINE_DATA_DIR: /data
PHOTO_PIPELINE_LIBRARY_ROOTS: ${PHOTO_PIPELINE_LIBRARY_ROOTS}
services:
# Migrations run to completion before either role accepts work, through the same
# backup-then-migrate path the roles use (US07-05): a pending upgrade is snapshotted
# first, and a failed one exits non-zero with the backup named — so `api` and
# `worker` never start, and the previous database is left intact and restorable.
migrate:
<<: *runtime
command: ["migrate"]
environment: *environment
restart: "no"
api:
<<: *runtime
command: ["serve"]
environment:
<<: *environment
# Published to host loopback below. Inside the container the server must bind
# the container's own interface for that publish to reach it, which is exactly
# what makes the access secret mandatory (US08-01) — `serve` refuses to start
# without one. Exposing the port beyond loopback additionally needs
# PHOTO_PIPELINE_ALLOWED_HOSTS to name the hostname it is reached under.
PHOTO_PIPELINE_HOST: 0.0.0.0
PHOTO_PIPELINE_PORT: 8000
ports:
# Host side only: PHOTO_PIPELINE_PORT in `.env` moves the *published* port, and
# the container always serves 8000, which is what the image's health check probes.
- "${PHOTO_PIPELINE_PUBLISH_ADDRESS:-127.0.0.1}:${PHOTO_PIPELINE_PORT:-8000}:8000"
depends_on:
migrate:
condition: service_completed_successfully
restart: unless-stopped
# One worker. A second one is refused by the library lock in the shared data
# volume, which is the point: `docker compose up --scale worker=2` fails loudly
# instead of running two writers against one library.
worker:
<<: *runtime
command: ["worker", "--id", "worker-1"]
environment: *environment
depends_on:
migrate:
condition: service_completed_successfully
restart: unless-stopped
volumes:
# Local driver on purpose: the database, WAL, thumbnail cache, and backups need a
# real local filesystem. Do not point this at NFS, SMB, or a network volume driver.
data:
driver: local

22
docker/entrypoint.sh
View File

@@ -1,22 +0,0 @@
#!/bin/sh
# One entrypoint, one role per container (US08-02).
#
# The first argument is the management command the image runs — `serve` and `worker`
# are the two roles, and every other `python -m photo_pipeline` command (migrate,
# diagnostics, backup, restore, dry-run) is passed through unchanged so operating the
# container is operating the same CLI. No supervisor: two roles in one container would
# share a process lock they are each meant to hold alone (US07-05).
set -eu
if [ "$(id -u)" = "0" ]; then
echo "refusing to run as root: start this image with a non-root UID/GID so files" \
"it renames or writes keep the ownership the mounted library expects" >&2
exit 1
fi
role="${1:-serve}"
# The health check has to know which role it is checking, and only the API has an
# endpoint to check. /tmp is writable for the unprivileged user; /run may not be.
printf '%s' "${role}" > "${PHOTO_PIPELINE_ROLE_FILE:-/tmp/photo-pipeline-role}" 2>/dev/null || true
exec python -m photo_pipeline "$@"

72
docker/fetch-immich-go.py
View File

@@ -1,72 +0,0 @@
"""Download one pinned immich-go release and verify it before unpacking (US08-02).
Run at image build time by the `uploader` stage, with the interpreter that is already
in the base image: no curl, no wget, and no download tooling in the layer that ships.
The checksum is not advisory — a release asset that does not match the pinned digest
is a failed build, not a warning, because the uploader's flags and report format are
what the upload parser is written against (concept §15).
"""
from __future__ import annotations
import argparse
import hashlib
import platform
import tarfile
import tempfile
import urllib.request
from pathlib import Path
RELEASE_URL = "https://github.com/simulot/immich-go/releases/download/v{version}/{asset}"
# Debian/BuildKit architecture as the interpreter sees it → release asset name.
ASSETS = {
"x86_64": ("immich-go_Linux_x86_64.tar.gz", "amd64"),
"amd64": ("immich-go_Linux_x86_64.tar.gz", "amd64"),
"aarch64": ("immich-go_Linux_arm64.tar.gz", "arm64"),
"arm64": ("immich-go_Linux_arm64.tar.gz", "arm64"),
}
TIMEOUT_SECONDS = 300
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--version", required=True, help="immich-go release, without the v")
parser.add_argument("--sha256-amd64", required=True)
parser.add_argument("--sha256-arm64", required=True)
parser.add_argument("--into", default="/usr/local/bin")
args = parser.parse_args()
machine = platform.machine().lower()
if machine not in ASSETS:
raise SystemExit(f"unsupported architecture: {machine}")
asset, arch = ASSETS[machine]
expected = {"amd64": args.sha256_amd64, "arm64": args.sha256_arm64}[arch]
url = RELEASE_URL.format(version=args.version, asset=asset)
with urllib.request.urlopen(url, timeout=TIMEOUT_SECONDS) as response: # noqa: S310
payload = response.read()
digest = hashlib.sha256(payload).hexdigest()
if digest != expected:
raise SystemExit(f"checksum mismatch for {url}: {digest} != {expected}")
target = Path(args.into)
target.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory() as work:
archive = Path(work) / asset
archive.write_bytes(payload)
with tarfile.open(archive) as tar:
member = tar.getmember("immich-go")
# Extract exactly the one file this pin is about, by name, so nothing
# else in the archive can decide where it lands.
extracted = tar.extractfile(member)
if extracted is None:
raise SystemExit("release archive contains no immich-go binary")
binary = target / "immich-go"
binary.write_bytes(extracted.read())
binary.chmod(0o755)
print(f"immich-go {args.version} ({arch}) verified {digest}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

34
docker/healthcheck.sh
View File

@@ -1,34 +0,0 @@
#!/bin/sh
# Container health for the `serve` role: readiness, not liveness (US08-02).
#
# /api/v1/health/ready is 503 until the database is reachable, migrated, and in WAL
# mode with foreign keys on, so an unmigrated or misconfigured container never reports
# healthy. Health endpoints need no session and no access secret, which is what lets an
# orchestrator restart a container it holds no credentials for (US08-01).
set -eu
role="$(cat "${PHOTO_PIPELINE_ROLE_FILE:-/tmp/photo-pipeline-role}" 2>/dev/null || echo unknown)"
if [ "${role}" != "serve" ]; then
# ponytail: the worker has no endpoint to probe; its liveness is its lease and job
# heartbeat in the database. Add a `worker --health` command if a restart policy
# ever needs to act on it.
exit 0
fi
port="${PHOTO_PIPELINE_PORT:-8000}"
exec python - "${port}" <<'PY'
import sys
import urllib.error
import urllib.request
url = f"http://127.0.0.1:{sys.argv[1]}/api/v1/health/ready"
try:
with urllib.request.urlopen(url, timeout=5) as response: # noqa: S310 — loopback
sys.exit(0 if response.status == 200 else 1)
except urllib.error.HTTPError as error:
print(f"not ready: HTTP {error.code}", file=sys.stderr)
sys.exit(1)
except OSError as error:
print(f"not ready: {error}", file=sys.stderr)
sys.exit(1)
PY

224
docs/architecture.md
View File

@@ -1,224 +0,0 @@
# Architecture
[← Documentation index](index.md)
For whoever has to change this code without breaking somebody's photo library. It
explains what the pieces are, which rules each one keeps, and where to look when a
stage refuses.
The product decisions behind all of it live in `INTEGRATED_PIPELINE_CONCEPT.md`; this
page describes what was built.
## Context
Five things outside the application, and what actually crosses each boundary.
```mermaid
flowchart LR
operator([Operator]):::person -->|browser, one session| app
app[Photo Pipeline]:::system -->|read, rename, EXIF write| library[(Photo library<br/>bind mount)]
app -->|database, cache, journals, backups| data[(Data directory<br/>local filesystem)]
app -->|confirmed-SFW images only| vision[Vision provider]:::ext
app -->|immich-go, verified bytes| immich[Immich server]:::ext
app -->|copy, verify, then remove| archive[(Archive medium)]
classDef person fill:#1f6feb,stroke:#58a6ff,color:#fff
classDef system fill:#238636,stroke:#3fb950,color:#fff
classDef ext fill:#6e40c9,stroke:#a371f7,color:#fff
```
| boundary | leaves the machine? | carries |
|---|---|---|
| operator → app | no (loopback, or a proxy you configured) | commands against stable ids, never paths |
| app → library | no | reads, folder renames, EXIF merges |
| app → data directory | no | SQLite in WAL mode, thumbnails, journals, backups |
| app → vision provider | **yes** | image bytes of confirmed-SFW canonical assets only |
| app → Immich | **yes** | the exact verified bytes of an approved album |
| app → archive medium | no | copies, verified before the source is removed |
## Runtime
Two long-lived processes and a one-shot migration, sharing one database and one
library.
```mermaid
flowchart TD
browser([Browser]) -->|JSON + SSE| api
migrate["migrate<br/>backup, then upgrade"] -->|must exit 0| api
migrate --> worker
api["api · serve<br/>enqueues, serves, reads"] -->|jobs table| db[(SQLite WAL)]
worker["worker<br/>claims and does the work"] -->|jobs table| db
worker --> tools["exiftool · vision API · immich-go"]
api -.->|api.lock.json| lock{{library lock}}
worker -.->|worker.lock.json| lock
api --> lib[(library)]
worker --> lib
```
`serve` enqueues and renders; **it does not do the work**. Everything that scans,
scores, analyses, renames, uploads, or archives happens in the worker, which claims
a queued job atomically with a fencing token. One mutating job runs at a time, and
the lock file in the data directory is what makes a second worker impossible rather
than merely discouraged.
## The modules
| package | owns | must not |
|---|---|---|
| `api/` | HTTP surface, error envelope, security policy | contain SQL or business logic |
| `api/routes/` | one module per resource group, all under `/api/v1` | accept a filesystem path from the browser |
| `api/security.py` | host/origin/CSRF/session/size policy as one pure `evaluate`, plus the ASGI middleware | be bypassed per-route |
| `schemas/` | Pydantic request and response contracts | reach the database |
| `services/` | all domain logic; the only place a decision is made | be imported by the frozen CLI archive |
| `models/` | SQLAlchemy tables | hold behaviour |
| `jobs/` | durable job lifecycle, worker loop, lock ranks, handler registry | run work in the API process |
| `integrations/` | the real outside world: `exiftool`, vision, NSFW model, `immich-go` and its report grammars | be called without a version recorded |
| `path_policy.py` | the library boundary, `_IGNORE/` exclusion, symlink-escape refusal | be duplicated anywhere |
| `imaging.py` | the single bounded-decode door | let a caller open an image directly |
| `faults.py` | the crash barriers the tests fire | read anything but its one env var |
| `config.py` | typed settings, secrets as `SecretStr` | log or return a value |
| `db.py` | engine, sessions, WAL and foreign keys, migration entry | be used to bypass a repository |
Services worth knowing by name: `inventory`, `duplicates`, `safety`, `analysis`,
`albums`/`proposals`/`naming`, `renames`/`rename_apply`/`rename_journal`,
`uploads`/`upload_batches`/`upload_reports`/`upload_verification`,
`archives`/`archive_transfer`/`archive_journal`/`restores`/`availability`,
`thumbnails`, `hashing`, `exif_checkpoint`, `jobs`, `workflow`, `backup`,
`diagnostics`, `app_lock`, `release`, `benchmarks`, `library`, `legacy_import`.
## The state machines
Three journals decide what a restart is allowed to assume. All three are read from
the database plus the real world — never guessed from a missing file.
### Durable jobs
```mermaid
stateDiagram-v2
[*] --> queued
queued --> running: claimed with a fencing token
queued --> cancelled
queued --> cancelling
running --> succeeded
running --> failed
running --> cancelling
cancelling --> cancelled
failed --> retry_queued
retry_queued --> running
succeeded --> [*]
cancelled --> [*]
```
`succeeded` and `cancelled` are terminal, so a duplicate delivery cannot move a
finished job. Claiming is compare-and-set on the row version; a worker whose lease
expired cannot commit after another has taken over.
### The rename journal
The only state machine that moves somebody's folders.
```mermaid
stateDiagram-v2
[*] --> planned
planned --> moving
moving --> moved
moving --> planned: proven untouched
moving --> rollback_required
moved --> database_updated
database_updated --> verified
verified --> complete
moved --> rollback_required
database_updated --> rollback_required
verified --> rollback_required
planned --> failed
failed --> planned
rollback_required --> rolled_back
complete --> [*]
rolled_back --> [*]
```
Intent is written **before** the disk is touched, which is why `moving` can resolve
backwards: the evidence decides. `moving`, `moved`, `database_updated`, and
`rollback_required` are the unsafe states — while any operation sits in one, the
library may be half-renamed, so unrelated mutations are refused with
`409 rename_recovery_required` until a person resolves it.
### Upload batches
```mermaid
stateDiagram-v2
[*] --> planned
planned --> running
running --> succeeded
running --> failed
running --> cancelling
running --> unknown_requires_verification: process died after acceptance
cancelling --> cancelled
failed --> running: retry
cancelled --> running: retry
```
`unknown_requires_verification` is deliberately **not** restartable. The server may
already hold the files; the answer is to ask Immich for the recorded SHA-1, not to
upload again and hope.
Archive transfers use the same shape — `planned → transferring → verified →
removing → complete` — and the source is removed only after the archived bytes are
verified.
## Identity and the data model
A path is metadata. The identity is `assets.id`, a UUID that never changes, and
`asset_paths` records every path an asset has ever had with the reason it changed.
That is what makes a rename cheap: nothing else in the database has to move.
Tables: `assets`, `asset_paths`, `thumbnails`, `jobs`, `job_items`, `job_events`,
`safety_reviews`, `analysis_results`, `exif_projections`, `duplicate_clusters`,
`duplicate_members`, `duplicate_negative_links`, `album_proposals`, `rename_plans`,
`rename_operations`, `upload_batches`, `upload_items`, `upload_verifications`,
`archive_locations`, `archive_plans`, `archive_operations`.
Availability is independent of workflow progress: `active`, `archiving`,
`archived_online`, `archived_offline`, `restoring`, `missing_unexpected`. An
unmounted archive disk is `archived_offline`, never `missing` — the scanner is not
allowed to conclude that a photo is gone because a disk is unplugged.
EXIF is a projection with its own verified state (`verified`, `divergent`,
`failed`): the desired values are written, read back, and compared, and anything the
stage does not own having changed makes the asset `divergent` and blocks the next
mutating stage.
## Where each invariant lives
| invariant | enforced in |
|---|---|
| `_IGNORE/` is never traversed, counted, or opened | `path_policy.is_excluded`, used by every discovery path |
| no path outside the library roots is reachable | `path_policy.resolve_in_roots` — it returns the resolved path, because validating one name and opening another is the symlink race |
| one writer per library | `services/app_lock.py` (an `flock` on a JSON lock file, per role) |
| one mutating job at a time | `services/jobs.py` lock keys plus `jobs/locks.py` rank ordering |
| only confirmed-SFW assets reach the vision provider | `services/analysis.py`, re-checked *after* the provider call so a decision that flipped mid-flight discards the result |
| EXIF is verified, and other fields preserved | `services/exif_checkpoint.py` |
| uploads carry the exact verified bytes | `services/uploads.py` preflight, re-proved immediately before the uploader runs |
| an uncertain upload is not a failure | `services/upload_batches.py`, `services/upload_verification.py` |
| the archive source outlives its copy until verified | `services/archive_transfer.py` |
| decoding is bounded | `imaging.py` |
| the trust boundary and CSRF | `api/security.py` |
## Concurrency
Locks are taken broad to narrow — `library → stage/job → album/folder → asset` — and
never the other way, which is what makes deadlock structural rather than lucky.
Ranks live in `jobs/locks.py`.
Exactly-once execution is not achievable across SQLite, a filesystem, subprocesses,
and a remote server. The design promises **at-least-once with idempotent recovery**:
every handler may run twice, and running twice must not produce two side effects.
That is why the rename journal records intent before moving, why EXIF writes merge
and verify, and why upload retries consult both local history and Immich.
## History
This application was extracted from two command-line tools rather than written from
nothing. Their sources are frozen in `legacy_cli_archive/` with the ledger mapping
each donated behaviour to the service that now owns it, the characterization tests
that pinned it, and every intentional difference. Production code must not import
them; they are provenance and rollback evidence.

95
docs/errors.md
View File

@@ -1,95 +0,0 @@
# Errors and refusals
[← Documentation index](index.md)
Every error the API returns carries a code:
```json
{"error": {"code": "lock_held", "message": "a library_write job is already running"}}
```
Most of them are **refusals, not faults**. This application would rather stop and
explain than guess about somebody's photographs, so a code below usually means it
protected something. Each row says what caused it and what to do.
## Access and the trust boundary
| code | cause | what to do |
|---|---|---|
| `unauthenticated` | no application session | reload the page; the browser bootstraps one |
| `access_denied` | wrong or missing access secret | check `PHOTO_PIPELINE_ACCESS_SECRET`. Attempts are logged with the caller's address only |
| `too_many_attempts` | more than five failed secret attempts in a minute | wait. This is the rate limit, not a lockout |
| `csrf_failed` | a mutation without the session's token | reload; a stale tab has an old token |
| `host_not_allowed` | the `Host` header is not a configured name | add the real hostname to `PHOTO_PIPELINE_ALLOWED_HOSTS` |
| `origin_not_allowed` | the request came from another origin | not something a browser tab of this app produces |
| `cross_site_blocked` | another site triggered the request | expected — this is the protection working |
| `payload_too_large` | the body exceeds `PHOTO_PIPELINE_MAX_REQUEST_BYTES` | send less; every endpoint takes small commands |
| `path_not_allowed` | a path resolved outside the library roots | usually a symlink. Nothing outside the roots is reachable, by design |
## The safety gate
| code | cause | what to do |
|---|---|---|
| `dry_run_not_approved` | mutation is gated until a dry run is approved | run `dry-run`, read it, then `approve-dry-run` |
| `approval_scope_mismatch` | the approval covers different library roots | approve a report for the roots actually configured |
| `approval_unreadable` | the approval record cannot be read | re-approve; do not edit it by hand |
## Concurrency and staleness
| code | cause | what to do |
|---|---|---|
| `lock_held` | a mutating job already holds the lane | wait for it. One at a time is what makes a crash recoverable |
| `version_conflict` | you acted on a version that changed underneath you | the view re-renders with the server's truth; decide again |
| `invalid_transition` | a state change that the machine does not allow | usually a stale tab; reload |
| `job_error` | the job service refused the request | the message says why |
| `rename_recovery_required` | an interrupted rename is unresolved | resolve it in [Renames](stages/renames.md). Unrelated mutations stay blocked on purpose |
## Stage refusals
| code | cause | what to do |
|---|---|---|
| `nothing_to_score` | no photo is eligible for safety scoring | the queue is already decided |
| `nothing_eligible` | no photo is eligible for analysis | resolve safety decisions first |
| `nothing_to_plan` | no approved album name to build a plan from | approve a proposal first |
| `invalid_decision` | the decision is not one this cluster accepts | reload the cluster |
| `invalid_proposal` | the name is empty, reserved, or has forbidden characters | fix the name; the rules are in [Album proposals](stages/albums.md) |
| `unknown_album` | the album is not a folder the library knows | rescan |
| `cannot_apply` | the plan is invalid, stale, or another plan is unresolved | read the blockers in the preview |
| `cannot_rollback` | the operation is complete or its preconditions no longer hold | rollback is recovery, not undo |
## Upload
| code | cause | what to do |
|---|---|---|
| `stale_preflight` | bytes changed between approval and running | re-run preflight; this is the check that stops the wrong bytes being uploaded |
| `not_runnable` | the batch is in a state that must not be re-run | an uncertain batch is verified, never retried |
| `requires_verification` | the outcome is unknown; the server may hold the files | verify against Immich, then resolve |
| `changed_after_upload` | the file changed after a successful upload | uploading again may create or upgrade an asset |
## Operations
| code | cause | what to do |
|---|---|---|
| `backup_failed` | the snapshot could not be taken or verified | check free space and the message |
| `invalid_retention` | a `--keep` value that is not a positive count | the newest backup is never pruned |
| `not_found` | no such id | usually a stale link |
## Generic
| code | cause |
|---|---|
| `invalid_request` | the request body failed validation. Field names only — never the values, which end up in logs and screenshots |
| `http_error` | a plain HTTP-level refusal, with its status |
| `internal_error` | an unhandled error. The code is all you get; the traceback is in the server log, because it can carry paths and credentials |
## Diagnostics warnings
Not errors — reported by `diagnostics` and worth acting on. They are listed with
their meanings in [Diagnostics](stages/diagnostics.md): `disk_low`, `disk_critical`,
`cache_over_quota`, `wal_growth`, `tool_version_drift`, `legacy_process_active`, and
`no_roots`.
## Startup refusals
Exit codes 2 to 5 are covered by the installation manual under
[when it refuses to start](installation.md#when-it-refuses-to-start).

66
docs/first-pass.md
View File

@@ -1,66 +0,0 @@
# A guided first pass
[← Documentation index](index.md)
One library, from nothing to a verified upload. Each stage links to its own page; this
is the order, and — more importantly — where the points of no return are.
![The workflow home, with every stage and its counts](images/workflow.png)
The workflow page is the home view and the honest summary: each stage shows its state,
its counts, why it is blocked if it is, and the one action that moves it forward.
| # | stage | reversible afterwards? |
|---|---|---|
| 0 | [Inventory](stages/inventory.md) | nothing was changed |
| 1 | [Duplicate review](stages/duplicates.md) | yes — decisions can be changed, no file is deleted |
| 2 | [Safety review](stages/safety.md) | the decision, yes; the EXIF keyword is a file write |
| 3 | [Analysis](stages/analysis.md) | the result, yes; the caption and keywords are a file write |
| 4 | [Album proposals](stages/albums.md) | yes — approving renames nothing |
| 5 | [Renames](stages/renames.md) | **your folders move.** Recoverable, journaled, but real |
| 6 | [Upload](stages/uploads.md) | **assets exist on the Immich server.** Not undone from here |
| 7 | [Archive](stages/archive.md) | **originals leave active storage.** Restorable while the medium is reachable |
| — | [Diagnostics](stages/diagnostics.md) | read-only, any time |
## Before the first run
Take a backup of the photo library itself. This application is careful — it previews,
journals, and verifies — but it is the first time you are pointing it at your
photographs, and a backup is cheaper than confidence.
If the library is irreplaceable, turn on `PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL`
before anything else. Every mutating request is then refused until you have produced
a read-only reconciliation report and approved it by name.
## The pass
1. **Scan.** Inventory → *Rescan*. Reads only. Check the count, and check that
nothing under `_IGNORE/` appears.
2. **Resolve duplicates.** Exact byte matches can be accepted as recommended; anything
fuzzy gets looked at. Resolving first is what stops you paying to analyse the same
photo twice.
3. **Decide safety.** Every canonical photo becomes `sfw` or `nsfw`. Nothing reaches
the vision provider until it is confirmed SFW — this is the gate the whole design
exists around.
4. **Analyse.** Only confirmed-SFW photos. Each result is written to the database and
projected into EXIF, then read back and verified.
5. **Propose album names.** Evidence from what was analysed. Edit anything you disagree
with. Approving changes no file.
6. **Build the rename plan, read it, apply it.** The preview shows every old → new
path. This is where folders move.
7. **Rescan.** Paths are reconciled; identities are unchanged.
8. **Upload.** Preflight lists every blocker. The command is previewed without the API
key. One album at a time.
9. **Verify.** An uncertain upload is not a failure and not a success — ask the server.
10. **Archive** (optional). Copy, verify, and only then reclaim the space.
Stop anywhere. Every stage is resumable, and closing the browser cancels nothing.
## Two rules worth internalising
**The application refuses more than it warns.** A refusal is not a fault; it is the
design working. [Errors and refusals](errors.md) explains each one.
**One long job at a time.** Browsing stays available while a job runs, but a second
mutating job is refused — that is what makes a crash recoverable rather than
ambiguous.

BIN
docs/images/albums.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 64 KiB

BIN
docs/images/analysis.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 31 KiB

BIN
docs/images/archive.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 128 KiB

BIN
docs/images/duplicates.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 24 KiB

BIN
docs/images/inventory.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 55 KiB

BIN
docs/images/renames.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 78 KiB

BIN
docs/images/safety.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 31 KiB

BIN
docs/images/statistics.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 33 KiB

BIN
docs/images/uploads.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 101 KiB

BIN
docs/images/workflow.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 63 KiB

68
docs/index.md
View File

@@ -1,68 +0,0 @@
# Photo Pipeline documentation
One local application that takes a photo library from discovery to a verified Immich
upload: duplicate detection, safety review, content analysis, album naming, guarded
renaming, upload, and archive — one visible, resume-safe workflow.
These pages are readable three ways, and they are the same files each time: in the
repository under `docs/`, on Gitea, and inside the running application under
**Docs**. There is no separate copy to fall out of date.
## Read in this order
1. [Overview](overview.md) — what the application does, the stages it moves a photo
through, and the rules it will not break.
2. [Installation and operations](installation.md) — host and container installation,
every setting, the first-run checklist, upgrades, backup and restore, and what
each refusal at startup means.
3. [A guided first pass](first-pass.md) — one library from scan to verified upload,
with the point of no return named in each stage.
4. [Architecture](architecture.md) — the context and runtime diagrams, what each
module owns, the three journals a restart reads, and where every invariant is
enforced.
## The stages, one page each
Every page answers the same four questions: what the stage is for, what you decide,
what it changes on disk or on the server, and what it refuses.
1. [Inventory and discovery](stages/inventory.md)
2. [Duplicate review](stages/duplicates.md)
3. [Safety review](stages/safety.md)
4. [Analysis](stages/analysis.md)
5. [Album proposals](stages/albums.md)
6. [Renames](stages/renames.md)
7. [Upload](stages/uploads.md)
8. [Archive](stages/archive.md)
9. [Diagnostics and library statistics](stages/diagnostics.md)
## When something goes wrong
- [Errors and refusals](errors.md) — every error code, what caused it, what to do.
- [Recovery](recovery.md) — what the application resolves by itself, and what needs
you.
## Keeping these pages true
Documentation that disagrees with the application is worse than none, because it is
trusted. So one command compares them:
```bash
python -m photo_pipeline docs-gate
```
It runs every `phase_i` check — dead links and anchors, pages nobody links to, images
nobody shows, settings, commands, exit codes, error codes, and states that no longer
exist in the code, the pages rendering in a real browser without a console or policy
error, and the screenshots still showing what the application shows. It retains its
evidence and **accepts no skipped check**: a check that did not run is a page nobody
compared. CI runs it on every pull request.
The repository's own build, test, and release notes live in the
[README](../README.md).
## Conventions
A page tells you what a stage **changes on disk or on the server** before it tells
you how to run it. Refusals are documented as intentional: this application would
rather stop and explain than guess about somebody's photographs.

336
docs/installation.md
View File

@@ -1,336 +0,0 @@
# Installation and operations
[← Documentation index](index.md)
From nothing to a running instance pointed at your photo library, and everything you
need afterwards: upgrading, backing up, restoring, and understanding a refusal.
Two ways to install. **Container** is the deployment this project builds and ships;
**host** is what you want for development or a single machine you already manage.
Both run the same two processes against the same database.
> Read [what it will not do](overview.md#what-it-will-not-do) first. Several of the
> steps below only make sense once you know which rules the application is keeping.
## Before you start
| you need | version | why |
|---|---|---|
| Python | 3.12 or newer | the application |
| `exiftool` | 13.x (the image pins `13.25+dfsg-1`) | every EXIF read and write |
| `immich-go` | 0.32.0 (pinned in the image) | the upload stage only |
| Docker + the compose plugin | any current | the container installation only |
| a vision provider key | — | the analysis stage only |
You also need a photo library you can afford to be wrong about. Take a backup of it
before pointing anything at it for the first time, and consider running with
[`PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL`](#the-settings) on.
## Host installation
```bash
python3.12 -m venv .venv
.venv/bin/pip install -e ".[vision]" # drop [vision] for a review-only install
```
Configure it. Every setting is an environment variable; a `.env` file in the working
directory is read at startup:
```bash
cp .env.example .env && $EDITOR .env
```
`.env.example` lists every variable with its default and no values at all. For a
loopback installation you need exactly one:
```bash
PHOTO_PIPELINE_LIBRARY_ROOTS=/home/you/Pictures
```
Then migrate and start the two processes:
```bash
.venv/bin/python -m photo_pipeline migrate # create or upgrade the database
.venv/bin/python -m photo_pipeline serve # API and browser app on 127.0.0.1:8000
.venv/bin/python -m photo_pipeline worker # second terminal
```
**Both processes are required.** `serve` enqueues work and renders the application;
nothing is scanned, scored, analysed, renamed, uploaded, or archived without a
worker. Open <http://127.0.0.1:8000/app/> and confirm the workflow page loads.
### The configuration file
`.env`, or any path named by `PHOTO_PIPELINE_ENV_FILE`. It is parsed, never executed:
`KEY=value` lines, `#` comments, optional quotes, no interpolation and no `export`. A
configuration file that can run code is a configuration file that can be a
vulnerability.
**Anything already exported in the shell wins.** The file is your standing
configuration; the environment is the override for one run.
The archived CLI's names still work, so an existing `photo_analyzer.env` can be used
as it is:
| in the file | applied as |
|---|---|
| `LLM_API_KEY` / `GEMINI_API_KEY` | `OPENAI_API_KEY` |
| `LLM_BASE_URL` | `OPENAI_BASE_URL` |
| `LIBRARY` | `PHOTO_PIPELINE_LIBRARY_ROOTS` |
`.env` and `*.env` are gitignored and refused by the repository's safety checks. The
file holds a real key; it must never be committed.
## Container installation
```bash
cp .env.example .env && $EDITOR .env
docker compose up -d --build
```
The composition is one `serve` container, one `worker` container, a one-shot
`migrate` that both wait for, one bind-mounted library, and one named data volume.
Three variables it cannot start without:
| variable | is |
|---|---|
| `PHOTO_PIPELINE_LIBRARY_HOST_PATH` | the library on this host |
| `PHOTO_PIPELINE_LIBRARY_ROOTS` | where that library is mounted **inside** the container |
| `PHOTO_PIPELINE_ACCESS_SECRET` | required, because publishing a port means the app is reachable from outside the container |
Set `PHOTO_PIPELINE_UID` and `PHOTO_PIPELINE_GID` to the owner of the library: what
the containers rename and rewrite keeps that ownership.
### The data volume
The `data` volume holds the database, its write-ahead log, the thumbnail cache, the
operation journals, and the backups. **It must stay on a local filesystem.** SQLite
in WAL mode needs real local locking, so NFS, SMB, and network volume drivers do not
slow it down — they corrupt it. The library bind mount has no such restriction.
### Ports and reaching it
The API port is published to `127.0.0.1` unless `PHOTO_PIPELINE_PUBLISH_ADDRESS` says
otherwise. Being reachable *was* the authentication in earlier versions: whoever could
open the port owned the library. So the moment the application answers to anything but
loopback — a hostname, a proxy, `0.0.0.0` — the access secret becomes mandatory and
`serve` refuses to start without it rather than publishing your photographs.
Behind a reverse proxy:
```bash
PHOTO_PIPELINE_ALLOWED_HOSTS=photos.example.com
PHOTO_PIPELINE_ACCESS_SECRET=… # python -c 'import secrets; print(secrets.token_urlsafe(32))'
PHOTO_PIPELINE_TRUSTED_PROXIES=10.0.0.2 # only the proxy's own address
```
`X-Forwarded-Proto` and `X-Forwarded-Host` are believed only from a trusted-proxy
address, so a client cannot declare its own origin. The browser asks for the secret
once per tab. Wrong secrets are rate-limited and logged with the caller's address
only. The health endpoints stay open so an orchestrator can restart the container;
nothing else is.
### Deployment
The stack is managed from git by Portainer, which redeploys on a webhook after CI
publishes an image built from `main`. Runtime secrets live in the Portainer stack's
environment rather than in the repository, so rotation happens in one place and
reading the repository discloses nothing.
## The settings
Every variable, its default, and whether it is a secret. `.env.example` is the same
list in copyable form.
### Library and data
| variable | default | meaning |
|---|---|---|
| `PHOTO_PIPELINE_LIBRARY_ROOTS` | none | the library boundary, `os.pathsep`-separated. No path outside these roots is ever read or written |
| `PHOTO_PIPELINE_DATA_DIR` | `data` | database, WAL, thumbnail cache, journals, backups. Never inside the library |
| `PHOTO_PIPELINE_DB_PATH` | `<data dir>/photo_pipeline.db` | the database file, if it must live elsewhere |
| `PHOTO_PIPELINE_ENV_FILE` | `.env` | where to read the configuration file from |
### Serving and the trust boundary
| variable | default | meaning |
|---|---|---|
| `PHOTO_PIPELINE_HOST` | `127.0.0.1` | bind address |
| `PHOTO_PIPELINE_PORT` | `8000` | port |
| `PHOTO_PIPELINE_ALLOWED_HOSTS` | none | comma-separated names the app answers to besides loopback. Naming one makes the access secret mandatory |
| `PHOTO_PIPELINE_ACCESS_SECRET` | none | **secret.** Traded for the session cookie at `GET /api/v1/session` |
| `PHOTO_PIPELINE_TRUSTED_PROXIES` | none | comma-separated peer addresses whose forwarded headers may be believed |
| `PHOTO_PIPELINE_MAX_REQUEST_BYTES` | `1048576` | largest request body accepted |
### Logging
| variable | default | meaning |
|---|---|---|
| `PHOTO_PIPELINE_LOG_LEVEL` | `INFO` | standard Python levels |
| `PHOTO_PIPELINE_LOG_FORMAT` | `json` | `json` or `text` |
### Limits
| variable | default | meaning |
|---|---|---|
| `PHOTO_PIPELINE_THUMBNAIL_CACHE_QUOTA_BYTES` | `500000000` | cache quota; over it is a diagnostics warning |
| `PHOTO_PIPELINE_THUMBNAIL_MAX_PIXELS` | `100000000` | refuse to decode anything larger. This is the decompression-bomb guard |
| `PHOTO_PIPELINE_ARCHIVE_FREE_SPACE_RESERVE_BYTES` | `1000000000` | free space an archive destination must keep beyond the transfer itself |
### The safety gate
| variable | default | meaning |
|---|---|---|
| `PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL` | `false` | refuse every mutating request until a read-only dry run of this library has been produced and approved |
### External services
| variable | default | meaning |
|---|---|---|
| `PHOTO_PIPELINE_VISION_API_KEY` | none | **secret.** Without it the analysis stage cannot run |
| `PHOTO_PIPELINE_IMMICH_SERVER_URL` | none | the Immich server |
| `PHOTO_PIPELINE_IMMICH_API_KEY` | none | **secret.** |
| `PHOTO_PIPELINE_IMMICH_GO_BINARY` | `immich-go` | the uploader, found on `PATH` |
### Composition only
Read by `docker-compose.yml`, not by the application:
`PHOTO_PIPELINE_IMAGE`, `PHOTO_PIPELINE_LIBRARY_HOST_PATH`,
`PHOTO_PIPELINE_PUBLISH_ADDRESS`, `PHOTO_PIPELINE_UID`, `PHOTO_PIPELINE_GID`.
A secret is never written to a log, never returned by the API, never stored in the
database, and never recorded in a backup manifest — a manifest says `configured`, not
the value. Do not put a real key in an example, a ticket, or a screenshot.
## First run checklist
Not "it started" — verified.
1. **The database is at the current revision.**
`python -m photo_pipeline migrate` exits 0.
2. **The application answers.**
`curl -s localhost:8000/api/v1/health/ready` returns 200.
3. **The library was found.** Open the app, run a scan from the Inventory view, and
check the count against what you expect. Anything under `_IGNORE/` must be missing
from it — that is the exclusion working, not a bug.
4. **The worker is claiming.** The scan job reaches `succeeded`. If it stays
`queued`, no worker is running.
5. **Diagnostics are clean.**
`python -m photo_pipeline diagnostics` reports free space and an empty `warnings`.
6. **Nothing was modified.** The scan is read-only; your files' timestamps are
unchanged.
Only then point it at the whole library.
## Operations
Every operation is the same CLI, on a host or in the composition:
```bash
python -m photo_pipeline diagnostics
docker compose run --rm --no-deps api diagnostics
```
`--no-deps` keeps a one-off command from starting a second stack; `api` is only the
service it borrows the image and mounts from.
### Upgrading
1. `python -m photo_pipeline backup --reason before-upgrade`
2. Pull the new version (or `docker compose pull && docker compose up -d`).
3. Migrations run by themselves at startup, and a **pending schema change is
snapshotted first**. If the upgrade fails, the previous database and its
`pre-migration` backup are both intact, and the error log names the backup
directory.
An up-to-date database is not backed up again on every start.
### Backing up
```bash
python -m photo_pipeline backup --reason weekly --keep 7
python -m photo_pipeline verify-backup data/backups/<name>
```
Backups go through SQLite's online backup API, never a file copy: with WAL enabled
the `.db` file alone is missing every committed page still in the write-ahead log.
Each backup is a directory holding the snapshot and a `manifest.json` — schema
revision, SHA-256, row counts, the archive media the library depends on, and which
settings were configured.
`verify-backup` runs `PRAGMA integrity_check` **and** `PRAGMA foreign_key_check`,
compares the snapshot's SHA-256 against the manifest, and re-counts every table it
recorded. Bit rot, a truncated copy, and a "repaired" snapshot all fail it.
`--keep N` prunes the oldest and never the newest. The same is available at
`GET /api/v1/diagnostics`, `GET|POST /api/v1/backups`,
`GET /api/v1/backups/{name}/verify`, and `POST /api/v1/backups/prune`.
### Restoring
Restore is deliberately **not** an API call. It replaces the state of an
installation, so it belongs to a stopped one and a person at a terminal.
1. Stop `serve` and `worker`.
2. `python -m photo_pipeline verify-backup data/backups/<name>` — never restore an
unverified snapshot, and `restore` will refuse one anyway.
3. `python -m photo_pipeline restore data/backups/<name> --into /path/to/fresh-data`.
A target that already holds a database is refused; recovering in place means
moving the old data directory aside first.
4. Point `PHOTO_PIPELINE_DATA_DIR` at the restored directory and run `migrate`.
5. Run an inventory scan, so paths are reconciled against the real library.
6. Mount every archive location the manifest names before archiving again. The
database records where archived originals are; it does not contain them.
Practise this against a copy before you need it.
### Watching it
`diagnostics` reports the database, write-ahead log, thumbnail cache, uploader
reports, backups, and logs separately, with free space and warnings for low disk
(`disk_low`, `disk_critical`), a cache over its quota, a write-ahead log outgrowing
its database, and a legacy CLI writing the library.
Logs go to stdout in JSON by default — `docker compose logs -f worker`, or your
service manager's journal on a host. Uploader output is kept per attempt under the
data directory with the API key scrubbed line by line.
## When it refuses to start
Every refusal below is deliberate. The application would rather stop and explain than
guess about somebody's photographs.
| exit code | meaning | what to do |
|---|---|---|
| `1` | the operation failed and said why — an unverifiable backup, an occupied restore target, an unknown benchmark profile | read the message; nothing was changed |
| `2` | **the library lock is held.** Another `serve` or `worker` owns this data directory | stop the other process. The message names the holder; a lock whose process is gone is taken over automatically |
| `3` | **a legacy CLI looks active.** The frozen command-line tools are writing this library | stop them. `--allow-legacy` overrides and you own the outcome |
| `4` | **configuration refused.** The application is reachable beyond loopback and no access secret is set | set `PHOTO_PIPELINE_ACCESS_SECRET`, or bind to loopback only |
| `5` | **a library root is not mounted** (containers only) | fix the bind mount. Refusing is what stops the container writing into its own throwaway layer instead of your library |
Other things that look like faults and are not:
- **`403 host_not_allowed`** — the `Host` header is not a name in
`PHOTO_PIPELINE_ALLOWED_HOSTS`. Add the real hostname; do not add `*`.
- **`409 lock_held`** — a mutating job is already running. One at a time is what
makes a crash recoverable.
- **`409 rename_recovery_required`** — an interrupted rename is unresolved. Resolve
it in the Renames view; unrelated mutations stay blocked until then, on purpose.
- **A job stuck in `queued`** — no worker is running.
- **An empty scan** — check `PHOTO_PIPELINE_LIBRARY_ROOTS`, and in a container check
that the path is the *container-side* mount, not the host path.
## What an installer must not work around
- **One writer.** Do not run two workers, and do not run the frozen CLI beside the
application. Each is safe alone and destructive together — the lock is not
bureaucracy.
- **`_IGNORE/` is never read.** Do not "fix" the exclusion.
- **EXIF is verified before upload.** Do not skip the checkpoint to make a stage
finish; an unverified projection is exactly the case where the wrong bytes reach
Immich.
- **The data directory is not the library.** Keep them apart, and keep the data
directory on a local filesystem.
- **Secrets stay in the environment.** Not in the database, not in a log, not in a
commit.

77
docs/overview.md
View File

@@ -1,77 +0,0 @@
# Overview
[← Documentation index](index.md)
Photo Pipeline sorts a photo library. It finds duplicates before anything expensive
happens to them, asks a human which pictures may leave the machine, describes the
ones that may, proposes album names from what it found, renames folders under a
crash-safe journal, uploads through `immich-go`, and can archive a finished album off
active storage without ever forgetting it existed.
It runs locally. It talks to exactly three things outside itself: a vision provider,
an Immich server, and `exiftool` — and it will tell you before it uses any of them.
## The workflow
Each stage is a gate, not a tab. A later stage can always be looked at; its actions
stay disabled until what they depend on is true.
```mermaid
flowchart TD
I["0 · Inventory<br/>discover, hash, cluster duplicates"] --> S["1 · Safety<br/>score and human decision"]
S -->|sfw| A["2 · Analysis<br/>vision provider, EXIF checkpoint"]
S -->|nsfw| U
A --> B["3 · Albums<br/>proposal, then guarded rename"]
B --> U["4 · Upload<br/>immich-go, verified bytes"]
U --> R["5 · Archive<br/>copy, verify, then reclaim space"]
```
| Stage | What it decides | What it changes |
|---|---|---|
| Inventory | which file is the canonical copy of a picture | nothing — it only reads |
| Safety | whether a photo may be sent to a cloud provider | one `sfw`/`nsfw` EXIF keyword |
| Analysis | what a photo shows | a managed caption segment and additive keywords |
| Albums | what a folder should be called | folder names, through a journaled rename |
| Upload | which exact bytes reach Immich | nothing locally; assets appear in Immich |
| Archive | which album leaves active storage | files move to the archive, after verification |
## What it will not do
These are enforced in code, not by convention, and each one is why some action you
expected is sometimes refused.
- **Nothing under `_IGNORE/` is ever read.** Not scanned, not counted, not
thumbnailed, not sent anywhere.
- **A path is not an identity.** Every picture has a stable id, so moving or renaming
it loses no history.
- **Only a confirmed-SFW photo reaches the vision provider.** A photo marked NSFW is
still uploadable to Immich; it simply never leaves for analysis.
- **Metadata is verified, not hoped for.** Every stage that writes EXIF reads it back
and proves that what it did not own is unchanged.
- **One writer at a time.** A library lock, held by one process, is what makes a
crash recoverable instead of ambiguous.
- **Nothing irreversible happens without a preview and an explicit approval** that
names the exact count.
## Where things live
| | |
|---|---|
| the photo library | wherever you point `PHOTO_PIPELINE_LIBRARY_ROOTS`; mounted read-write |
| the database, cache, logs, backups | the data directory, never inside the library |
| secrets | the environment, never the database and never a log line |
| these documents | `docs/` in the repository, served at `/docs` by the application |
## Running it
The short version, for a host installation:
```bash
python -m photo_pipeline migrate
python -m photo_pipeline serve # the API and this browser application
python -m photo_pipeline worker # the process that does the long work
```
The full procedure, the container path, and every setting are in the
[installation manual](installation.md); if you would rather start using it, take
[the guided first pass](first-pass.md).

62
docs/recovery.md
View File

@@ -1,62 +0,0 @@
# Recovery
[← Documentation index](index.md)
What the application resolves by itself, and what needs you. The dividing line is
simple: it resumes when the evidence is unambiguous, and it stops and asks when it is
not. It never guesses about your files.
## An interrupted rename
A rename records its intent **before** touching the disk, so after a crash the
journal plus the actual filesystem give a verdict per operation:
| verdict | means | what happens |
|---|---|---|
| resumable | the move provably did not happen | reset to planned and run again |
| rollback-safe | the move happened; the remaining steps can be completed or undone | drive it forward, or roll it back |
| manual | the evidence is contradictory | left alone, and it keeps blocking |
Open [Renames](stages/renames.md); the recovery panel lists every unresolved
operation and offers a resolve action only where one is safe. Until then, unrelated
mutations are refused with `rename_recovery_required` — building new work on a
half-applied move is how a library becomes unexplainable.
## An uncertain upload
`unknown_requires_verification` means the uploader died after Immich may have
accepted the files. It is **not** retryable. Verification asks the server whether it
holds the recorded SHA-1 and answers present, absent, or inconclusive. An
inconclusive answer is resolved by you, with your evidence and name recorded in the
batch's history.
## A failed migration
A pending schema change is snapshotted before it is applied. If the upgrade fails,
the previous database and its `pre-migration` backup are both intact and the error
log names the backup directory. Stop everything and run the
[restore drill](installation.md#restoring).
## A restored backup
After restoring, run a scan. Paths are reconciled against the real library and
identities are preserved where hashes match. Anything that does not match becomes a
visible `stale` or `divergent` state instead of being quietly accepted.
Mount every archive medium the manifest names before archiving again: the database
records where archived originals live, it does not contain them.
## A job that will not start
Check, in this order: is a worker running; is another mutating job holding the lane
(`lock_held`); is an interrupted rename blocking everything
(`rename_recovery_required`); is the library lock held by a process that is still
alive (exit code `2` names the holder).
## What is never automatic
No file is deleted, no folder is renamed, no upload is retried, and no archive source
is removed without either an explicit confirmation from you or a verification the
application performed itself. If you are reading this page because something happened
that you did not approve, that is a bug worth reporting — with the diagnostics output
and the relevant journal, both of which are safe to share: they carry no secrets.

47
docs/stages/albums.md
View File

@@ -1,47 +0,0 @@
# Album proposals
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
![Reviewing an album proposal](../images/albums.png)
**What it is for.** Turning a folder of analysed photographs into a name a person
would have chosen, before any folder is touched.
**What you decide.** The final name. The proposal is a starting point with its
reasoning attached; you edit it, accept it, or ignore it.
**What it changes.** A row per album: proposed name, rationale, confidence, and your
final name, with a version. **Approving renames nothing.** Not one file moves at this
stage — approval only marks a name as agreed, and the [rename stage](renames.md) is
where it becomes a plan you have to confirm separately.
**What it refuses.** A name containing `/ \ : * ? " < > |`, a reserved device name, or
one that collides with an existing folder is rejected while you type and again on the
server. Approval is refused when the evidence changed since the proposal was
generated — the proposal is stale, and regenerating is the honest fix. Editing with a
stale version returns a conflict and shows you the server's truth rather than
overwriting it.
## Reading the view
The left pane lists albums with their state: `none`, `proposed`, `edited`,
`approved`, `error`, or `stale`. The right pane shows the evidence the name was built
from — date range, dominant tags, locations, counts — then the suggested name, the
rationale, the confidence, and an editable final name.
The default naming shape is predictable and sortable:
```text
YYYY-MM — Place — Event
YYYY — Event
Place — Event
```
## While a rename is unresolved
Generating and approving proposals is blocked while an interrupted rename is
outstanding, with `409 rename_recovery_required`. Resolve the rename first: building
new names on top of a half-applied move is how a library becomes hard to reason
about.
Next: [renames](renames.md).

39
docs/stages/analysis.md
View File

@@ -1,39 +0,0 @@
# Analysis
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
![The analysis view](../images/analysis.png)
**What it is for.** Describing what a photograph shows, so albums can be named from
evidence and the library can be searched by content.
**What you decide.** When to run it, and over what scope. The descriptions themselves
come from the configured vision provider.
**What it changes.** For each eligible photo: a stored result — description, tags,
approximate year, location hint, model and prompt version — and then an EXIF
checkpoint that merges a **managed caption segment** and additive keywords into the
file, reads them back, and verifies that nothing else moved. The file's SHA-256 is
refreshed after the write.
**What it refuses.** Anything not confirmed SFW. Anything not canonical. A malformed
provider response is quarantined rather than stored. Keywords are only ever added,
never removed, because EXIF keywords carry no ownership and deleting a generated one
could delete yours.
## Running it
The view shows eligible, analysed, pending, and error counts, and the job's live
progress. Starting a second mutating job while it runs is refused — that is the
one-writer rule, not a queue.
Cancelling drains safely: completed items stay completed, and resuming continues
rather than restarting. A photo may be sent to the provider twice if a crash happens
mid-item, but its stored result and its EXIF are written once.
## Costs
Each analysed photo is a provider call. Resolving duplicates first is what keeps that
number honest, and the counts here are the ones to check before starting a large run.
Next: [album proposals](albums.md).

43
docs/stages/archive.md
View File

@@ -1,43 +0,0 @@
# Archive
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
![Archive locations and plans](../images/archive.png)
**What it is for.** Moving a finished album out of active storage onto another disk,
while keeping everything the library knows about it.
**What you decide.** Which album, which destination, and whether to accept the
reclaimed space in exchange for needing that medium mounted to see the originals
again.
**What it changes.** Files are copied to the archive location, verified there, and
only then removed from the library. The asset keeps its id, its hashes, its
decisions, its analysis, its upload history, and a durable preview thumbnail;
`current_path` becomes null and availability becomes `archived_online` or
`archived_offline`.
**What it refuses.** Preflight blocks on: an unverified upload, a checksum that no
longer matches the uploaded bytes, an unmounted or unwritable destination,
insufficient free space plus the configured reserve, a destination that already
holds unexpected paths, a missing durable thumbnail, and any conflicting job. The
source is **never** removed before the archived copy is verified — not because Immich
reported success, which is an ingest, not a backup.
## Offline is not missing
An archived photo whose medium is unplugged is `archived_offline`. It still appears
in search, still participates in duplicate detection through its retained hashes and
thumbnail, and is never reported as lost. If a full-resolution comparison is needed,
the application asks you to mount the named medium rather than guessing.
## Restore
Restore is planned and journaled like everything else: the medium must be online, the
bytes are copied back and verified into a collision-free destination, a new active
path is registered, and a reconciliation scan follows. Existing safety, analysis,
EXIF, and upload state stay valid when hashes match — and when they do not, the
result is a visible `divergent` state rather than silent acceptance of a different
file.
Next: [diagnostics](diagnostics.md).

49
docs/stages/diagnostics.md
View File

@@ -1,49 +0,0 @@
# Diagnostics and library statistics
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
![Library statistics](../images/statistics.png)
**What it is for.** Answering "is this installation healthy, and what is in this
library?" — the first thing to look at when a stage behaves unexpectedly.
**What you decide.** Nothing. Both are read-only.
**What it changes.** Nothing.
**What it refuses.** Nothing — both are reads, and both stay available while a job
holds the library. That is deliberate: the moment you most need to know what the
installation is doing is while it is busy.
## Statistics
The Stats view summarises the library itself: how many photos are analysed, the
common tags, the distribution across albums and years. It answers questions about
your photographs.
## Diagnostics
Diagnostics answers questions about the *installation*, and lives at
`GET /api/v1/diagnostics` and on the command line:
```bash
python -m photo_pipeline diagnostics
```
It reports the database, write-ahead log, thumbnail cache, uploader reports, backups,
and logs as separate sizes, plus free space, the tool versions it found against the
ones the image pinned, and which locks are held and by whom.
Its warnings are the ones worth acting on:
| warning | means |
|---|---|
| `disk_low` / `disk_critical` | free space is running out; mutating stages stop before the reserve |
| `cache_over_quota` | the thumbnail cache exceeds its configured quota |
| `wal_growth` | the write-ahead log is outgrowing its database — usually a long-running reader |
| `tool_version_drift` | the installed `exiftool` or `immich-go` is not the version the image pinned |
| `legacy_process_active` | the frozen command-line tools are writing this library. Stop them |
| `no_roots` | no library root is configured, so there is nothing to work on |
An empty `warnings` list on a fresh installation is what the
[first-run checklist](../installation.md#first-run-checklist) is looking for.

36
docs/stages/duplicates.md
View File

@@ -1,36 +0,0 @@
# Duplicate review
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
![The duplicate cluster list](../images/duplicates.png)
**What it is for.** Deciding which file is *the* copy of a picture, before anything
expensive or irreversible happens to the others.
**What you decide.** For each cluster: keep the recommended canonical, choose a
different one, keep everything as variants, declare it not a duplicate, or defer.
**What it changes.** Only the decision, recorded as an auditable event. **No file is
ever deleted.** Non-canonical copies stay exactly where they are; they are simply
excluded from analysis and upload.
**What it refuses.** Exact byte and identical-pixel matches may be accepted on the
recommendation. Anything fuzzy — a crop, a re-encode, a burst frame — requires an
explicit confirmation, because a perceptual hash is evidence, not proof.
## Reading a cluster
Each member shows its path, role, pHash distance, and size, with synchronised zoom
across the previews. The recommendation is visible but never pre-applied for fuzzy
matches. The confidence band tells you how much the machine is claiming.
Rejecting a pair records a negative link, so a later rescan does not keep proposing
the same wrong match.
## Why first
Resolving duplicates before safety and analysis is what stops you reviewing and
paying for the same photograph three times. It also means the album evidence is built
from one copy of each picture rather than a skewed count.
Next: [safety review](safety.md).

39
docs/stages/inventory.md
View File

@@ -1,39 +0,0 @@
# Inventory and discovery
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
![The inventory view](../images/inventory.png)
**What it is for.** Finding every supported photo in the configured library roots and
giving each one a permanent identity, so that everything afterwards can refer to a
picture rather than a path.
**What you decide.** Nothing. This stage is the only one with no judgement in it.
**What it changes.** On disk, nothing at all — discovery reads. In the database it
records each photo's path, size, timestamps, full-file SHA-256, normalised pixel hash,
and perceptual hash, and it opens a path-history entry.
**What it refuses.** Anything under an `_IGNORE/` directory is never traversed,
counted, hashed, or previewed. Unsupported extensions and unreadable files are
reported rather than skipped silently. Nothing outside the configured roots is
reachable, including through a symlink.
## Reading the view
Each row is one asset: its current path, availability, whether the file is present,
and its size. The filter searches paths; the availability selector separates active
photos from archived ones.
A **rescan** after moving files around outside the application is the normal way to
reconcile: a file that moved keeps its identity, because identity is the hash and the
id, not the location.
## What good looks like
The count matches what you expect, `_IGNORE/` contents are absent, and nothing shows
as missing. A photo listed as missing means the file is no longer at its recorded
path and no new path matched its hashes — that is a question for you, not something
the scanner resolves by guessing.
Next: [duplicate review](duplicates.md).

46
docs/stages/renames.md
View File

@@ -1,46 +0,0 @@
# Renames
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
![The rename plan preview](../images/renames.png)
**What it is for.** Applying approved album names to the actual folders — the first
stage that changes your filesystem.
**What you decide.** Whether to apply the plan, after reading it. The confirmation
names the plan, its version, and its checksum, so what you approve is what runs.
**What it changes.** Folders move. For each operation: the intent is journaled
*before* the disk is touched, the move is made, `assets.current_path` is updated by
id in one transaction, the old path is closed and a new one opened in path history,
the result is verified, and only then is the operation complete.
**What it refuses.** A plan is invalid — not merely warned about — when a source is
missing or changed, two operations target the same destination, a destination already
exists, a path would leave the library or enter `_IGNORE/`, or a source is a symlink.
Case-only renames and cross-filesystem moves are allowed but flagged, because they
take a different, staged route. A plan whose checksum no longer matches is refused
rather than reinterpreted.
## Reading the preview
Every operation shows both full paths, the kind of move (`rename`, `case-only
rename`, `cross-filesystem move`), how many photos it affects, its journal state, and
any issue with its severity. The plan can be exported as JSON before you commit to it.
**A run cannot be stopped part-way.** It can be interrupted — by a crash, a power
cut, a killed container — and interruption is recoverable, but there is no cancel
button once it starts. That is stated on the confirmation, not hidden in a manual.
## After an interruption
The recovery panel lists each unresolved operation with a verdict read from the
journal *and the disk*: resumable, safely rollbackable, or needing a person. It never
guesses from a missing path alone. Until it is resolved, unrelated mutations are
refused with `409 rename_recovery_required`.
Rollback is recovery, not undo. A completed operation is terminal — the button only
appears for operations that are actually reversible. See
[recovery](../recovery.md).
Next: [upload](uploads.md).

43
docs/stages/safety.md
View File

@@ -1,43 +0,0 @@
# Safety review
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
![The safety review queue](../images/safety.png)
**What it is for.** Deciding which photographs may be sent to a cloud vision
provider. This is the privacy gate the rest of the design is built around.
**What you decide.** `sfw`, `nsfw`, or defer — per photo, or in bulk from the filters.
A local model can score first to order the queue, but the score is a suggestion; the
decision is yours.
**What it changes.** The decision is stored in the database *and* projected into the
file: one mutually exclusive `sfw` or `nsfw` keyword written into `Keywords` and
`Subject`. The write is merged with existing metadata, read back, and verified, and
the file's SHA-256 is refreshed afterwards.
**What it refuses.** An undecided or deferred photo does not reach the vision
provider, and does not reach Immich either. If the read-back shows that a field the
stage does not own has changed, the checkpoint is marked `divergent`, the asset is not
marked verified, and the next mutating stage is blocked rather than proceeding on
metadata nobody trusts.
## The rule that matters
> Only a confirmed `sfw` photo may enter cloud content analysis. A confirmed `nsfw`
> photo skips analysis entirely but remains eligible for upload to Immich once its
> keyword is verified.
That is why marking something NSFW is not a punishment: it routes the photo around an
external service while keeping it in your own library workflow.
The gate is re-checked *after* the provider call as well. If a decision flips to NSFW
while an analysis is in flight, the result is discarded rather than stored.
## Reading the view
The tabs filter by state. Each row shows the path, the model's score, its suggestion,
your decision, and an `✓ exif` badge once the keyword is verified on disk. A row
without that badge has a decision the file does not yet carry.
Next: [analysis](analysis.md).

45
docs/stages/uploads.md
View File

@@ -1,45 +0,0 @@
# Upload
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
![Upload preflight](../images/uploads.png)
**What it is for.** Sending an approved album to Immich through `immich-go`, with an
audit trail of exactly which bytes went.
**What you decide.** Which albums, and whether to proceed once the preflight has
listed its blockers. Upload is never an automatic consequence of renaming.
**What it changes.** Nothing locally. On the Immich server, assets appear. Locally a
batch is recorded: the album, the asset ids, the redacted command, the uploader
version, the pre-upload SHA-256 **and** the SHA-1 Immich matches on, the output log,
and the outcome counts.
**What it refuses.** Preflight blocks on: missing credentials, an unreachable server,
a missing `immich-go`, an unresolved rename, an undecided or deferred safety
decision, an unverified EXIF checkpoint, incomplete analysis for an SFW photo, a file
whose bytes changed since it was checkpointed, and a partial scope you have not
explicitly accepted. The approval is then **re-proved immediately before the uploader
runs** — bytes edited after you approved fail with `stale_preflight` and the uploader
never starts.
## Reading the view
The scope shows exactly which albums and how many photos. The command preview is the
real command with the API key removed; the key never reaches the browser and never
reaches a log.
## Uncertain is not failed
If the process dies after the server accepted files, the batch becomes
`unknown_requires_verification` — not "failed", and **not retryable**. The server may
already hold the photographs, and uploading again would create duplicates or
upgrades. Verification asks Immich about the recorded SHA-1 and answers present,
absent, or inconclusive; an inconclusive answer is resolved by you, with the evidence
recorded.
If a file's bytes change after a successful upload, it is marked
`changed_after_upload` and you are warned that uploading again may create or upgrade
an asset rather than doing nothing.
Next: [archive](archive.md).

132
legacy_cli_archive/donor_ledger.yaml → donor_ledger.yaml
View File

@@ -12,15 +12,7 @@
# (if anything) carries over
# Every row needs either `tests` (existing test IDs, module::function) or
# `pending_story` (the backlog story that will characterize/deliver it).
# status: characterized — donor behavior pinned by characterization tests
# resolved — replacement shipped; `parity` names the tests that prove
# it, and `delta` states every intentional difference
# pending — not migrated yet; `pending_story` says which story will
# parity: test ids (path::function) in any suite, proving the replacement
# delta: what the replacement deliberately does differently, or not at all
#
# Archived by US07-01: the sources referenced below now live beside this file in
# src/ and are frozen (see README.md). Nothing in photo_pipeline imports them.
# status: characterized | pending
rows:
# ── photo_analyzer.py ──────────────────────────────────────────────────────
@@ -88,12 +80,8 @@ rows:
Copying a primary's analysis into variant rows survives, but keyed by
asset_id and recorded as stage state instead of raw row copies.
target: photo_pipeline/services/duplicates.py
parity:
- tests/integration/test_duplicate_engine.py::test_exact_copies_form_auto_decided_cluster
- tests/integration/test_duplicate_engine.py::test_perceptual_variant_is_review_only
delta: >
The donor propagated variant links implicitly while writing rows; the replacement makes the canonical link a reviewable, reversible cluster decision, so a propagated link can always be undone.
status: resolved
pending_story: US01-04
status: pending
- id: pa-hashing
area: hashing
@@ -153,12 +141,8 @@ rows:
classification: replace
rationale: Console report; superseded by the duplicate-review API/UI (US01-06).
target: photo_pipeline/api/routes + frontend duplicate review
parity:
- tests/integration/test_review_api.py::test_clusters_list_and_detail
- tests/e2e/test_review_ui.py::test_fuzzy_decision_requires_confirmation
delta: >
The text listing became the paged cluster API and the comparison UI; no textual report is produced.
status: resolved
pending_story: US01-06
status: pending
- id: pa-reconcile
area: database
@@ -218,7 +202,7 @@ rows:
RGB-normalize (drops alpha, converts HEIC), LANCZOS resize to 2048px
long-edge, JPEG q85 base64 — the provider-input contract. Truncated-image
tolerance (ImageFile.LOAD_TRUNCATED_IMAGES) carries with it.
target: photo_pipeline/services/analysis.py
target: photo_pipeline/integrations/vision.py
tests:
- test_pa_imaging::test_prepare_image_small_passthrough_jpeg
- test_pa_imaging::test_prepare_image_resizes_to_max_long_edge
@@ -246,12 +230,9 @@ rows:
response validation, 429/503 retry with exponential backoff. Prompt and
model/config version must be persisted per analysis_runs. Characterized
against a deterministic fake provider when the analysis service is ported.
target: photo_pipeline/services/analysis.py
parity:
- tests/integration/test_safety_analysis.py::test_provider_called_only_for_confirmed_sfw
delta: >
The prompt and response schema carry over; the provider is an injected adapter so the privacy gate is testable, and results are keyed to asset ids rather than paths.
status: resolved
target: photo_pipeline/integrations/vision.py
pending_story: US02-06
status: pending
- id: pa-throttle
area: logging
@@ -260,8 +241,8 @@ rows:
rationale: >
Rolling throttle window + persistent throttle_events.jsonl + RPD day
counter become job metrics/events on the durable job model.
target: photo_pipeline/services/jobs.py + photo_pipeline/services/analysis.py
pending_story: US07-04
target: photo_pipeline/jobs/coordinator.py
pending_story: US02-02
status: pending
- id: pa-nsfw-filter
@@ -331,12 +312,8 @@ rows:
checked between items, double-SIGINT force quit — becomes the durable
JobRunner worker loop with the same drain-and-resume semantics.
target: photo_pipeline/jobs/worker.py
parity:
- tests/integration/test_worker.py::test_worker_processes_all_items
- tests/integration/test_worker.py::test_cooperative_cancellation_leaves_items_resumable
delta: >
The in-process folder loop with SIGINT handling became durable jobs claimed by a worker: cancellation is a persisted request, not a signal, and an interrupted run resumes from the database instead of restarting.
status: resolved
pending_story: US02-02
status: pending
- id: pa-ui-terminal
area: ui
@@ -394,12 +371,8 @@ rows:
the JSONL history logger become structured JSON logging with job_id/
asset_id and job_events rows; per-photo history maps to job events.
target: photo_pipeline structured logging + jobs/job_events
parity:
- tests/integration/test_jobs.py::test_enqueue_persists_items_and_event
- tests/integration/test_jobs_sse.py::test_sse_streams_all_events_then_closes
delta: >
The JSONL history file and rich console handler are replaced by structured JSON logs plus durable job_events; the browser reads events over SSE rather than tailing a file.
status: resolved
pending_story: US02-02
status: pending
- id: pa-balance
area: vision
@@ -409,11 +382,8 @@ rows:
Provider balance/quota probes (report 'unsupported' on providers without
the endpoint). Network-bound; characterized against the fake provider.
target: photo_pipeline/services/analysis.py
parity:
- tests/integration/test_safety_analysis.py::test_provider_called_only_for_confirmed_sfw
delta: >
Not carried over: balance/quota polling was provider-specific (Gemini/OpenAI billing endpoints) and key-scoped. Cost reporting, when a story asks for it, comes from the per-run usage recorded with each analysis result rather than from a vendor endpoint.
status: resolved
pending_story: US02-06
status: pending
- id: pa-cli
area: configuration
@@ -423,13 +393,9 @@ rows:
argparse surface is superseded by the API; flags map to job configs
(documented in WEBAPP_CONCEPT.md §8 parity table). Transitional CLI calls
the shared services until archival (E07).
target: photo_pipeline/__main__.py (serve | worker | migrate | import-legacy-scores) + /api/v1
parity:
- tests/integration/test_app_lifecycle.py::test_restart_preserves_data_and_reruns_migrations
- tests/unit/test_legacy_archive.py::test_production_code_never_imports_an_archived_module
delta: >
The argparse surface is not reproduced. Every flag that drove work became an API command or a job configuration; the CLI keeps only what an application needs to be operated (serve, worker, migrate) plus the one-off legacy CSV import.
status: resolved
target: photo_pipeline/api + transitional CLI
pending_story: US07-01
status: pending
# ── nsfwtag/ ───────────────────────────────────────────────────────────────
- id: nt-discovery
@@ -456,17 +422,9 @@ rows:
nsfw_scores.csv stops being the source of truth (concept: DB state).
Format characterized (4-decimal scores, bad rows dropped) because the
existing CSV must migrate into assets.safety_score.
target: photo_pipeline/services/legacy_import.py (one-off import into safety_reviews)
target: photo_pipeline/repositories (safety), CSV import in US01-02 migration
tests: [test_nsfwtag::test_score_cache_roundtrip_and_tolerance]
parity:
- tests/integration/test_legacy_import.py::test_scores_are_imported_onto_asset_identity
- tests/integration/test_legacy_import.py::test_a_reviewed_asset_is_never_overwritten_by_the_csv
delta: >
The CSV is no longer read at runtime at all: it is imported once into
scored-but-unreviewed safety_reviews rows and left on disk untouched. A path
that matches nothing is reported, never turned into an asset, and a human
decision always outranks an imported score.
status: resolved
status: characterized
- id: nt-score-model
area: nsfw
@@ -480,13 +438,6 @@ rows:
inference path needs the local model + deterministic fake.
target: photo_pipeline/integrations/nsfw_model.py
tests: [test_nsfwtag::test_score_images_cache_hit_skips_model]
delta: >
The donor set Pillow's process-global ImageFile.LOAD_TRUNCATED_IMAGES so a
partially downloaded file still scored. Here the same process also hashes
files and renders previews, and those must keep refusing a truncated file
rather than silently working on half of one; scoring opens images through
the bounded photo_pipeline.imaging door instead and skips the ones it cannot
read, leaving them unscored and visibly undecided (US07-03).
status: characterized
- id: nt-exif-keyword
@@ -534,11 +485,8 @@ rows:
Newline-list bulk tagging (nsfw_confirmed.txt flow) is superseded by DB
review decisions; the existing list is a one-time migration input.
target: photo_pipeline/services/safety.py (decision import in US01-02)
parity:
- tests/integration/test_safety_parity.py::test_extracted_marks_and_partition_match_donor
delta: >
Bulk keyword application from a file list is replaced by decisions against asset ids; the EXIF write itself is the extracted, read-back-verified one.
status: resolved
pending_story: US01-02
status: pending
- id: nt-ui
area: ui
@@ -550,11 +498,8 @@ rows:
threshold/score review flow, lightbox and keyboard model are the frontend
donor for the Safety view (preserved per concept §10; ported in US02-01).
target: photo_pipeline/api + frontend Safety view
parity:
- tests/e2e/test_workflow_views.py::test_safety_review_decide_persists_across_reload
delta: >
The stdlib review server is replaced by the API plus the Safety view; the donor's layout, thresholds, and keyboard flow carry over, its embedded HTML generation does not.
status: resolved
pending_story: US02-01
status: pending
- id: nt-bench
area: nsfw
@@ -564,11 +509,8 @@ rows:
Dev-only model benchmark; archived without webapp replacement (recorded
basis of the AdamCodd model choice). No production caller.
target: none (archive as reference)
parity:
- tests/unit/test_legacy_archive.py::test_every_archived_source_matches_its_checksum
delta: >
No replacement: a dev-only benchmark whose result (the AdamCodd model choice) is already recorded. Kept in the archive as the basis of that choice.
status: resolved
pending_story: US07-01
status: pending
# ── webapp/ ────────────────────────────────────────────────────────────────
- id: wa-query-search
@@ -637,13 +579,9 @@ rows:
Subprocess-driving-the-CLI job control is superseded by durable DB jobs
with a worker process. Two ideas carry over: progress derived from DB
counts (not job-private state) and single-mutating-job enforcement.
target: photo_pipeline/services/jobs.py + photo_pipeline/jobs/worker.py
parity:
- tests/integration/test_jobs.py::test_idempotency_key_returns_same_job
- tests/integration/test_worker.py::test_handler_failure_fails_the_job
delta: >
Subprocess supervision of a CLI is replaced by durable jobs in the same process family: there is no subprocess to supervise, and progress is persisted rather than scraped from stdout.
status: resolved
target: photo_pipeline/jobs/coordinator.py
pending_story: US02-02
status: pending
- id: wa-server
area: ui
@@ -655,9 +593,5 @@ rows:
browser. analyzer.html + page.py design (dark OLED tokens, Library/
Analyze/Stats views) is frontend donor material per concept §10.
target: photo_pipeline/api/app.py + frontend
parity:
- tests/e2e/test_frontend_shell.py::test_shell_loads_assets_without_console_or_network_errors
- tests/e2e/test_frontend_shell.py::test_deep_link_and_reload_restore_view_and_filters
delta: >
The stdlib HTTP routes become the versioned FastAPI API and the static frontend shell; Python no longer interpolates HTML.
status: resolved
pending_story: US02-05
status: pending

14
frontend/css/app.css
View File

@@ -170,20 +170,6 @@ a.link:hover { text-decoration: underline; }
.grid th, .grid td { text-align: left; padding: 6px 8px; border-bottom: 1px solid var(--border); vertical-align: top; }
.path { font-family: ui-monospace, monospace; font-size: 0.85em; word-break: break-all; }
/* Docs: prose, so it gets a reading measure rather than the full window width. */
.doc { max-width: 72ch; }
.doc h1 { margin-top: 0; }
.doc h2, .doc h3 { margin-top: 1.6em; border-bottom: 1px solid var(--border); padding-bottom: 4px; }
.doc code { font-family: ui-monospace, monospace; font-size: 0.9em; background: var(--surface-2); padding: 1px 4px; border-radius: 4px; }
.doc pre { background: var(--surface-2); border: 1px solid var(--border); border-radius: var(--radius); padding: 12px; overflow: auto; }
.doc pre code { background: none; padding: 0; }
.doc table { border-collapse: collapse; width: 100%; }
.doc th, .doc td { text-align: left; padding: 6px 8px; border-bottom: 1px solid var(--border); vertical-align: top; }
.doc blockquote { margin: 1em 0; padding-left: 12px; border-left: 3px solid var(--border); color: var(--muted); }
.doc img { max-width: 100%; }
.diagram { margin: 1.5em 0; padding: 0; overflow: auto; }
.diagram svg { max-width: 100%; height: auto; }
/* Narrow screens: stack the panes so blockers and actions stay reachable. */
@media (max-width: 720px) {
.two-pane { grid-template-columns: 1fr; }

2
frontend/index.html
View File

@@ -19,9 +19,7 @@
<a href="#/albums" data-nav="albums">Albums</a>
<a href="#/renames" data-nav="renames">Renames</a>
<a href="#/uploads" data-nav="uploads">Upload</a>
<a href="#/archive" data-nav="archive">Archive</a>
<a href="#/stats" data-nav="stats">Stats</a>
<a href="#/docs" data-nav="docs">Docs</a>
</nav>
</header>
<main id="app" aria-live="polite"><!-- views render here --></main>

91
frontend/js/api.js
View File

@@ -2,62 +2,14 @@
// cancellation. Every method accepts an optional { signal } from cancellable().
export const BASE = "/api/v1";
// The API refuses every request without the session cookie, and every mutation
// without this token echoed back. The token is readable only same-origin, which is
// what makes it proof that the caller is this app and not another page.
let csrfToken = null;
// A deployment reachable through a proxy trades an operator secret for that cookie.
// Kept per tab: sessionStorage dies with the tab, and the secret never enters a URL.
const SECRET_KEY = "pp_access_secret";
async function bootstrap(secret) {
return fetch(BASE + "/session", {
credentials: "same-origin",
headers: secret ? { "X-Access-Secret": secret } : {},
});
}
async function session() {
if (csrfToken === null) {
let response = await bootstrap(sessionStorage.getItem(SECRET_KEY));
if (response.status === 401) {
sessionStorage.removeItem(SECRET_KEY);
const secret = prompt("Access secret");
if (secret) {
response = await bootstrap(secret);
if (response.ok) sessionStorage.setItem(SECRET_KEY, secret);
}
}
const body = await response.json().catch(() => null);
csrfToken = (body && body.csrf_token) || null;
}
return csrfToken || "";
}
async function send(path, { signal, ...options }) {
return fetch(BASE + path, {
credentials: "same-origin",
signal,
...options,
headers: {
"Content-Type": "application/json",
"X-CSRF-Token": await session(),
...(options.headers || {}),
},
});
}
async function request(path, { signal, ...options } = {}) {
let response;
try {
response = await send(path, { signal, ...options });
// A restarted server issues a new session; re-bootstrap once rather than
// stranding an open tab on 401.
if (response.status === 401) {
csrfToken = null;
response = await send(path, { signal, ...options });
}
response = await fetch(BASE + path, {
headers: { "Content-Type": "application/json" },
signal,
...options,
});
} catch (error) {
// A caller-cancelled fetch is not a failure; tag it so views can ignore it.
if (error.name === "AbortError") {
@@ -93,10 +45,8 @@ export const api = {
request("/inventory/assets?" + new URLSearchParams(params).toString(), opts),
listClusters: (params = {}, opts = {}) =>
request("/duplicates/clusters?" + new URLSearchParams(params).toString(), opts),
getCluster: (id, params = {}, opts = {}) => {
const query = new URLSearchParams(params).toString();
return request(`/duplicates/clusters/${encodeURIComponent(id)}${query ? `?${query}` : ""}`, opts);
},
getCluster: (id, opts = {}) =>
request(`/duplicates/clusters/${encodeURIComponent(id)}`, opts),
decide: (id, payload, opts = {}) =>
request(`/duplicates/clusters/${encodeURIComponent(id)}/decision`, {
method: "POST",
@@ -188,31 +138,4 @@ export const api = {
}),
uploadVerifications: (id, opts = {}) =>
request(`/upload-batches/${encodeURIComponent(id)}/verifications`, opts),
// ── Archive and restore: destinations, preflight, plans, recovery ────────
archiveLocations: (opts = {}) => request("/archive-locations", opts),
registerArchiveLocation: (payload, opts = {}) =>
request("/archive-locations", { method: "POST", body: JSON.stringify(payload), ...opts }),
archivePreflight: (payload, opts = {}) =>
request("/archive-preflight", { method: "POST", body: JSON.stringify(payload), ...opts }),
createArchivePlan: (payload, opts = {}) =>
request("/archive-plans", { method: "POST", body: JSON.stringify(payload), ...opts }),
listArchivePlans: (opts = {}) => request("/archive-plans", opts),
getArchivePlan: (id, opts = {}) => request(`/archive-plans/${encodeURIComponent(id)}`, opts),
applyArchivePlan: (id, opts = {}) =>
request(`/archive-plans/${encodeURIComponent(id)}/apply`, { method: "POST", ...opts }),
archiveRecovery: (opts = {}) => request("/archive-recovery", opts),
resolveArchiveRecovery: (opts = {}) =>
request("/archive-recovery/resolve", { method: "POST", ...opts }),
restorePreflight: (payload, opts = {}) =>
request("/restore-preflight", { method: "POST", body: JSON.stringify(payload), ...opts }),
createRestorePlan: (payload, opts = {}) =>
request("/restore-plans", { method: "POST", body: JSON.stringify(payload), ...opts }),
listRestorePlans: (opts = {}) => request("/restore-plans", opts),
getRestorePlan: (id, opts = {}) => request(`/restore-plans/${encodeURIComponent(id)}`, opts),
applyRestorePlan: (id, opts = {}) =>
request(`/restore-plans/${encodeURIComponent(id)}/apply`, { method: "POST", ...opts }),
restoreRecovery: (opts = {}) => request("/restore-recovery", opts),
resolveRestoreRecovery: (opts = {}) =>
request("/restore-recovery/resolve", { method: "POST", ...opts }),
};

28
frontend/js/app.js
View File

@@ -1,7 +1,4 @@
import { api } from "./api.js";
import { renderArchive, setArchiveRender } from "./archive.js";
import { renderDocs } from "./docs.js";
import { show as showIn } from "./dom.js";
import { navigate, onRouteChange, parseHash } from "./router.js";
import { renderRenames, setRenamesRender } from "./renames.js";
import { renderUploads, setUploadsRender } from "./uploads.js";
@@ -36,7 +33,7 @@ function el(tag, attrs = {}, ...children) {
}
function show(...nodes) {
showIn(root, ...nodes);
root.replaceChildren(...nodes);
}
function setActiveNav(view) {
@@ -195,7 +192,7 @@ async function renderClusters(params) {
" ",
el("span", { class: `badge ${cluster.state}` }, cluster.state)
),
el("div", { class: "muted" }, `${cluster.member_total ?? cluster.members.length} members · confidence ${cluster.confidence}`)
el("div", { class: "muted" }, `${cluster.members.length} members · confidence ${cluster.confidence}`)
)
);
@@ -209,11 +206,8 @@ async function renderClusters(params) {
async function renderClusterDetail(id, extra = {}) {
setActiveNav("duplicates");
let cluster;
// A cluster can hold thousands of members, so the server pages them; the page
// asks for as many as it is currently showing (US07-06).
const shown = extra.shown || 0;
try {
cluster = await api.getCluster(id, shown ? { limit: shown } : {});
cluster = await api.getCluster(id);
} catch (error) {
show(errorBanner(`Failed to load cluster: ${error.message}`));
return;
@@ -303,19 +297,6 @@ async function renderClusterDetail(id, extra = {}) {
nodes.push(decisionBar);
if (extra.pending) nodes.push(confirmPanel(cluster, extra.pending));
nodes.push(el("div", { class: "cluster-grid" }, ...members));
const total = cluster.member_total ?? cluster.members.length;
if (cluster.members.length < total) {
nodes.push(
el(
"button",
{
"data-testid": "show-more-members",
onclick: () => renderClusterDetail(id, { ...extra, shown: cluster.members.length + 100 }),
},
`Show more (${cluster.members.length} of ${total})`
)
);
}
show(...nodes);
}
@@ -385,9 +366,7 @@ function render() {
else if (path === "/albums") renderAlbums(root, params);
else if (path === "/renames") renderRenames(root, params);
else if (path === "/uploads") renderUploads(root, params);
else if (path === "/archive") renderArchive(root, params);
else if (path === "/stats") renderStats(root, params);
else if (path === "/docs") renderDocs(root, params);
else show(errorBanner("Unknown view"));
}
@@ -395,6 +374,5 @@ function render() {
setRender(render);
setRenamesRender(render);
setUploadsRender(render);
setArchiveRender(render);
onRouteChange(render);
render();

870
frontend/js/archive.js
View File

@@ -1,870 +0,0 @@
// Archive view (US06-05): preview what would leave active storage, confirm it
// exactly, watch the transfer, recover an interrupted one, browse what is already
// archived, and bring it back.
//
// Archiving is the only stage that removes originals, so this view never decides
// anything itself: the destination's identity, every blocker, the confirmation
// token, and what recovery may do all come from the server, and an action the
// server would refuse is not offered. Two things follow from that. A medium that
// is not mounted produces an instruction naming it rather than a disabled mystery,
// and an operation whose evidence is ambiguous offers no button at all.
import { api } from "./api.js";
import { el, errorBanner, setActiveNav, show } from "./dom.js";
import { subscribeJob } from "./events.js";
import { navigate } from "./router.js";
let outcome = null;
let activity = [];
let render = () => {};
export function setArchiveRender(fn) {
render = fn;
}
// The per-file journal states, split into the three things an operator actually
// wants told apart: bytes moving, bytes proven, original removed (concept §9).
const PHASES = [
["planned", "planned", "waiting"],
["transferring", "transfer", "copying to the medium"],
["verified", "verified", "archive copy hashed and manifested"],
["removing", "removing", "removing the active original"],
["complete", "complete", "archived and removed"],
["failed", "failed", "left alone for a decision"],
];
const AVAILABILITY_LABEL = {
active: "in the library",
archived_online: "archived · medium mounted",
archived_offline: "archived · medium away",
missing_unexpected: "missing — unexplained",
};
export async function renderArchive(root, params = {}) {
setActiveNav("archive");
let locations;
try {
locations = (await api.archiveLocations()).locations;
} catch (error) {
show(root, errorBanner(`Failed to load archive locations: ${error.message}`));
return;
}
const location = locations.find((l) => l.id === params.location) || locations[0] || null;
const nodes = [el("h1", {}, "Archive"), locationsCard(locations, location, params)];
if (!location) {
nodes.push(
el(
"p",
{ class: "muted", "data-testid": "no-locations" },
"Register the disk, NAS share, or removable medium that will hold archived originals."
),
outcomeBanner()
);
show(root, ...nodes.filter(Boolean));
return;
}
const [preflight, archivePlans, restorePlans, recovery, restoreRecovery, archived, restore] =
await Promise.all([
load(() => api.archivePreflight({ location_id: location.id })),
load(() => api.listArchivePlans()),
load(() => api.listRestorePlans()),
load(() => api.archiveRecovery()),
load(() => api.restoreRecovery()),
load(() => api.listAssets({ limit: 200 })),
load(() => api.restorePreflight({ location_id: location.id })),
]);
// Both directions share the runs table: one lane moves these originals, so one
// history is what an operator has to reason about.
const runs = [...plans(archivePlans), ...plans(restorePlans)].sort((a, b) =>
(a.created_at || "").localeCompare(b.created_at || "")
);
const chosen = runs.find((run) => run.id === params.plan) || runs[runs.length - 1] || null;
const selectedPlan = chosen ? await load(() => planDetailOf(chosen)) : null;
nodes.push(
preflight ? previewSection(preflight, location) : null,
preflight ? confirmBlock(preflight, location) : null,
outcomeBanner(),
activityLog(),
recoverySection(mergeRecovery(recovery, restoreRecovery)),
planList(runs, chosen && chosen.id),
selectedPlan ? planDetail(selectedPlan) : null,
archived ? archivedSection(archived.items, locations) : null,
restore ? restoreSection(restore, location) : null
);
show(root, ...nodes.filter(Boolean));
}
function plans(listed) {
return listed ? listed.plans : [];
}
function planDetailOf(run) {
return run.direction === "restore" ? api.getRestorePlan(run.id) : api.getArchivePlan(run.id);
}
// Archive and restore recovery answer the same question about the same lane, so
// they are one list; an unresolved item of either kind blocks the other.
function mergeRecovery(archive, restore) {
if (!archive && !restore) return null;
return {
operations: [...((archive || {}).operations || []), ...((restore || {}).operations || [])],
manual: [...((archive || {}).manual || []), ...((restore || {}).manual || [])],
};
}
// A section whose data failed to load must not take the rest of the view with it:
// the medium being away is exactly when the archived-asset list matters most.
async function load(call) {
try {
return await call();
} catch (_) {
return null;
}
}
// ── locations ────────────────────────────────────────────────────────────────
// A location is a medium, not a path: the marker's ``media_id`` is what proves the
// right disk is mounted, so it is shown next to the state it produced.
function locationsCard(locations, selected, params) {
const name = el("input", {
type: "text",
"data-testid": "location-name",
"aria-label": "Archive location name",
placeholder: "External disk",
});
const root = el("input", {
type: "text",
"data-testid": "location-root",
"aria-label": "Archive location path",
placeholder: "/Volumes/archive",
});
return el(
"div",
{ class: "card", "data-testid": "archive-locations" },
el("h2", {}, "Destinations"),
locations.length
? el(
"table",
{ class: "grid", "data-testid": "locations" },
el(
"thead",
{},
el(
"tr",
{},
...["", "Name", "Root", "Medium", "State", "Last seen"].map((label) =>
el("th", { scope: "col" }, label)
)
)
),
el(
"tbody",
{},
...locations.map((location) =>
el(
"tr",
{
"data-testid": "location-row",
"data-name": location.name,
"aria-current": selected && location.id === selected.id ? "true" : false,
},
el(
"td",
{},
el("input", {
type: "radio",
name: "archive-location",
"data-testid": "select-location",
"aria-label": `Use ${location.name}`,
checked: selected && location.id === selected.id ? "checked" : false,
onchange: () => navigate("/archive", { ...params, location: location.id }),
})
),
el("td", { "data-testid": "location-label" }, location.name),
el("td", { class: "path", "data-testid": "location-path" }, location.root),
el("td", { class: "path", "data-testid": "location-media" }, location.media_id),
el(
"td",
{},
el(
"span",
{
class: `badge ${location.state === "online" ? "complete" : "attention"}`,
"data-testid": "location-state",
},
location.state
)
),
el("td", { class: "muted" }, location.last_seen_at || "never")
)
)
)
)
: null,
selected && selected.state !== "online" ? mountInstruction(selected) : null,
el(
"div",
{ class: "toolbar" },
name,
root,
el(
"button",
{
"data-testid": "register-location",
onclick: () =>
run(() => api.registerArchiveLocation({ name: name.value, root: root.value })),
},
"Register destination"
)
)
);
}
// The one thing the app cannot do for the user: name the medium to connect.
function mountInstruction(location) {
const detail =
location.state === "wrong_volume"
? `A different medium is mounted at ${location.root}.`
: `Nothing is mounted at ${location.root}.`;
return el(
"div",
{ class: "confirm", role: "status", "data-testid": "mount-instruction" },
`${detail} Connect “${location.name}” (medium ${location.media_id}) and mount it there, ` +
"then reload this view. Archived photos stay listed and searchable meanwhile."
);
}
// ── preview ──────────────────────────────────────────────────────────────────
function previewSection(preflight, location) {
const totals = preflight.totals;
const capacity = preflight.capacity;
const rows = preflight.albums.map((album) =>
el(
"tr",
{ "data-testid": "archive-album-row", "data-album": album.album },
el("td", { "data-testid": "album-name" }, album.album),
el("td", { class: "path", "data-testid": "album-folder" }, album.folder),
el("td", { class: "path", "data-testid": "album-destination" }, album.destination),
el(
"td",
{ "data-testid": "album-method" },
album.transfer_method === "move" ? "move (same filesystem)" : "copy · verify · remove"
),
el("td", { "data-testid": "album-assets" }, String(album.asset_count)),
el("td", { "data-testid": "album-reclaim" }, bytes(album.reclaimable_bytes)),
el(
"td",
{},
el("span", { class: `badge ${album.state}`, "data-testid": "album-state" }, album.state),
...album.blockers.map((blocker) =>
el(
"div",
{ class: "blocker", "data-testid": "album-blocker", "data-code": blocker.code },
blocker.message
)
)
)
)
);
return el(
"div",
{ "data-testid": "archive-preview" },
el("h2", {}, "Preview"),
el(
"div",
{ class: "decision-bar" },
el(
"span",
{ class: "badge", "data-testid": "destination-identity" },
`${location.name} · ${location.media_id}`
),
el("span", { class: "badge", "data-testid": "total-albums" }, `${totals.albums} album(s)`),
el("span", { class: "badge", "data-testid": "total-assets" }, `${totals.assets} photo(s)`),
el(
"span",
{ class: "badge", "data-testid": "total-reclaim" },
`${bytes(totals.bytes)} reclaimable`
),
el(
"span",
{
class: `badge ${capacity.sufficient ? "complete" : "blocked"}`,
"data-testid": "capacity",
},
`free ${bytes(capacity.free_bytes)} · reserve ${bytes(capacity.reserve_bytes)}`
)
),
blockerList(preflight.blockers, "preflight-blockers", "preflight-blocker", "This scope cannot be archived yet"),
rows.length
? el(
"table",
{ class: "grid", "data-testid": "archive-albums" },
el(
"thead",
{},
el(
"tr",
{},
...["Album", "Folder", "Destination", "Transfer", "Photos", "Reclaims", "State"].map(
(label) => el("th", { scope: "col" }, label)
)
)
),
el("tbody", {}, ...rows)
)
: el(
"p",
{ class: "muted", "data-testid": "no-albums" },
"No album has a verified upload whose bytes are still unchanged, so nothing may be archived."
)
);
}
function blockerList(blockers, containerId, itemId, title) {
if (!blockers || !blockers.length) return null;
return el(
"div",
{ class: "alert", role: "alert", "data-testid": containerId },
el("strong", {}, title),
el(
"ul",
{},
...blockers.map((blocker) =>
el("li", { "data-testid": itemId, "data-code": blocker.code }, `${blocker.code}: ${blocker.message}`)
)
)
);
}
// ── confirmation ─────────────────────────────────────────────────────────────
function confirmBlock(preflight, location) {
const ready = preflight.state === "ready";
const totals = preflight.totals;
return el(
"div",
{ class: "card", "data-testid": "confirm" },
el("h2", {}, "Confirm"),
el(
"p",
{ class: "muted", "data-testid": "confirm-token" },
`Preflight ${preflight.token.slice(0, 20)}… · ${location.name}`
),
el(
"p",
{ "data-testid": "archive-note" },
"Archiving removes each original from the library — but only after its copy on " +
"the medium has been written, hashed, and recorded in the manifest. The photos " +
"stay searchable and deduplicable while the medium is away, and can be restored " +
"from this view."
),
el(
"div",
{ class: "toolbar" },
el(
"button",
{
class: "primary",
"data-testid": "start-archive",
disabled: ready ? false : "disabled",
title: ready ? false : "resolve the blockers above first",
onclick: () =>
run(async () => {
const plan = await api.createArchivePlan({
location_id: location.id,
token: preflight.token,
});
const started = await api.applyArchivePlan(plan.id);
watch(started.job.id, plan.id, "archive");
return { archiving: plan.asset_count };
}),
},
`Archive ${totals.ready_albums} album(s) · reclaim ${bytes(totals.bytes)}`
)
)
);
}
// ── plans and progress ───────────────────────────────────────────────────────
function planList(runs, selectedId) {
if (!runs.length) {
return el("p", { class: "muted", "data-testid": "no-plans" }, "Nothing has been archived yet.");
}
return el(
"div",
{ "data-testid": "archive-plans" },
el("h2", {}, "Runs"),
el(
"table",
{ class: "grid", "data-testid": "plans" },
el(
"thead",
{},
el(
"tr",
{},
...["Created", "Direction", "State", "Photos", "Bytes"].map((l) =>
el("th", { scope: "col" }, l)
)
)
),
el(
"tbody",
{},
...runs.map((plan) =>
el(
"tr",
{
"data-testid": "plan-row",
"data-plan": plan.id,
"data-direction": plan.direction,
"aria-current": plan.id === selectedId ? "true" : false,
},
el(
"td",
{},
el("a", { class: "link", href: `#/archive?plan=${encodeURIComponent(plan.id)}` }, plan.created_at || plan.id)
),
el("td", { "data-testid": "plan-direction" }, plan.direction),
el("td", {}, el("span", { class: `badge ${plan.state}`, "data-testid": "plan-state" }, plan.state)),
el("td", {}, String(plan.asset_count)),
el("td", {}, bytes(plan.byte_size))
)
)
)
)
);
}
function planDetail(plan) {
const operations = plan.operations || [];
const counts = {};
for (const operation of operations) {
counts[operation.journal_state] = (counts[operation.journal_state] || 0) + 1;
}
return el(
"div",
{ class: "card", "data-testid": "plan-detail", "data-plan": plan.id },
el("h2", {}, `${plan.direction === "restore" ? "Restore" : "Archive"} run ${plan.created_at || plan.id}`),
// Transfer, verification, and removal are separate answers to separate
// questions: what has moved, what is proven, and what is already gone.
el(
"div",
{ class: "decision-bar", "data-testid": "plan-progress" },
el("span", { class: `badge ${plan.state}`, "data-testid": "detail-state" }, plan.state),
...PHASES.map(([key, label, title]) =>
el(
"span",
{ class: `badge ${key}`, "data-testid": `count-${label}`, title },
`${label}: ${counts[key] ?? 0}`
)
)
),
el(
"table",
{ class: "grid", "data-testid": "operations" },
el(
"thead",
{},
el(
"tr",
{},
...["Photo", "Destination", "Phase", "Attempts", "Problem"].map((l) =>
el("th", { scope: "col" }, l)
)
)
),
el(
"tbody",
{},
...operations.map((operation) =>
el(
"tr",
{ "data-testid": "operation-row", "data-asset-id": operation.asset_id },
el("td", { class: "path", "data-testid": "operation-source" }, operation.source_path),
el("td", { class: "path", "data-testid": "operation-destination" }, operation.destination_path),
el(
"td",
{},
el(
"span",
{ class: `badge ${operation.journal_state}`, "data-testid": "operation-phase" },
phaseLabel(operation.journal_state)
)
),
el("td", {}, String(operation.attempt_count)),
el(
"td",
{ class: "muted", "data-testid": "operation-error", "data-code": operation.error_code || "" },
operation.error_code ? `${operation.error_code}: ${operation.error_message || ""}` : "—"
)
)
)
)
)
);
}
function phaseLabel(state) {
const found = PHASES.find(([key]) => key === state);
return found ? found[1] : state;
}
// ── recovery ─────────────────────────────────────────────────────────────────
// What an interrupted run left behind, straight from the journal plus the files on
// disk. Only the operations the server itself classified as resolvable get an
// action; ambiguous ones are shown with their evidence and no button.
function recoverySection(recovery) {
const operations = recovery ? recovery.operations : [];
if (!operations.length) return null;
const manual = recovery.manual || [];
const resolvable = operations.length - manual.length;
return el(
"div",
{ class: "card", "data-testid": "archive-recovery" },
el("h2", {}, "Interrupted work"),
el(
"p",
{ "data-testid": "recovery-summary" },
`${operations.length} operation(s) did not finish · ${resolvable} resolvable · ` +
`${manual.length} need a decision`
),
el(
"table",
{ class: "grid", "data-testid": "recovery-operations" },
el(
"thead",
{},
el(
"tr",
{},
...["Photo", "Phase", "Verdict", "Evidence"].map((l) => el("th", { scope: "col" }, l))
)
),
el(
"tbody",
{},
...operations.map((verdict) =>
el(
"tr",
{
"data-testid": "recovery-row",
"data-classification": verdict.classification,
"data-direction": verdict.direction,
},
el("td", { class: "path", "data-testid": "recovery-source" }, verdict.source_path),
el("td", {}, el("span", { class: "badge" }, phaseLabel(verdict.journal_state))),
el(
"td",
{},
el(
"span",
{
class: `badge ${verdict.classification === "manual" ? "blocked" : "attention"}`,
"data-testid": "recovery-verdict",
},
verdict.classification
)
),
el(
"td",
{ class: "muted", "data-testid": "recovery-reason" },
`${verdict.reason} (source ${verdict.source_exists ? "present" : "absent"}, ` +
`archive copy ${verdict.destination_matches ? "verified" : verdict.destination_exists ? "different bytes" : "absent"})`
)
)
)
)
),
manual.length
? el(
"div",
{ class: "alert", role: "alert", "data-testid": "recovery-manual" },
`${manual.length} operation(s) cannot be resolved from the evidence. Nothing will be ` +
"removed or retried for them here: inspect the medium and the library, then decide."
)
: null,
el(
"div",
{ class: "toolbar" },
resolvable
? el(
"button",
{
class: "primary",
"data-testid": "resolve-recovery",
onclick: () => run(() => api.resolveArchiveRecovery()),
},
`Finish ${resolvable} recoverable operation(s)`
)
: el(
"span",
{ class: "muted", "data-testid": "no-safe-recovery" },
"No operation can be finished safely from here."
)
)
);
}
// ── archived assets ──────────────────────────────────────────────────────────
// Browsing what is already archived, including while the medium is away: the
// retained protected preview and the recorded hashes are the evidence, so the row
// stays complete and honest instead of turning into a missing file.
function archivedSection(assets, locations) {
const archived = assets.filter((asset) => asset.availability_state !== "active");
if (!archived.length) return null;
const names = Object.fromEntries(locations.map((location) => [location.id, location.name]));
return el(
"div",
{ "data-testid": "archived-assets" },
el("h2", {}, "Archived photos"),
el(
"table",
{ class: "grid", "data-testid": "archived" },
el(
"thead",
{},
el(
"tr",
{},
...["Preview", "Archived as", "Medium", "Availability", "Size"].map((l) =>
el("th", { scope: "col" }, l)
)
)
),
el(
"tbody",
{},
...archived.map((asset) =>
el(
"tr",
{ "data-testid": "archived-row", "data-asset-id": asset.id },
el(
"td",
{},
el("img", {
"data-testid": "archived-preview",
width: 96,
loading: "lazy",
src: api.thumbnailUrl(asset.id, 256),
alt: `Preview of ${asset.archive_path || asset.id}`,
})
),
el("td", { class: "path", "data-testid": "archived-path" }, asset.archive_path || "—"),
el(
"td",
{ "data-testid": "archived-medium" },
names[asset.archive_location_id] || asset.archive_location_id || "—"
),
el(
"td",
{},
el(
"span",
{
class: `badge ${asset.availability_state === "archived_online" ? "complete" : "attention"}`,
"data-testid": "archived-availability",
},
AVAILABILITY_LABEL[asset.availability_state] || asset.availability_state
)
),
el("td", {}, bytes(asset.byte_size))
)
)
)
)
);
}
// ── restore ──────────────────────────────────────────────────────────────────
function restoreSection(restore, location) {
const ready = restore.state === "ready";
const items = restore.items || [];
if (!items.length && !restore.blockers.length) return null;
return el(
"div",
{ class: "card", "data-testid": "restore" },
el("h2", {}, "Restore"),
el(
"p",
{ "data-testid": "restore-note" },
"Restoring copies the archived bytes back into the library and leaves the archive " +
"copy where it is. A name that is already taken is never overwritten: the photo " +
"comes back beside it under a visibly different name."
),
blockerList(restore.blockers, "restore-blockers", "restore-blocker", "This restore cannot run yet"),
items.length
? el(
"table",
{ class: "grid", "data-testid": "restore-items" },
el(
"thead",
{},
el(
"tr",
{},
...["Archived as", "Comes back as", "Size", "State"].map((l) =>
el("th", { scope: "col" }, l)
)
)
),
el(
"tbody",
{},
...items.map((item) =>
el(
"tr",
{ "data-testid": "restore-row", "data-asset-id": item.asset_id },
el("td", { class: "path", "data-testid": "restore-source" }, item.archive_path),
el("td", { class: "path", "data-testid": "restore-destination" }, item.destination_path || "—"),
el("td", {}, bytes(item.byte_size)),
el(
"td",
{},
item.blockers.length
? el(
"span",
{
class: "badge blocked",
"data-testid": "restore-item-blocker",
"data-code": item.blockers[0].code,
},
item.blockers[0].code
)
: el("span", { class: "badge ready", "data-testid": "restore-item-state" }, "ready")
)
)
)
)
)
: null,
el(
"div",
{ class: "toolbar" },
el(
"button",
{
class: "primary",
"data-testid": "start-restore",
disabled: ready ? false : "disabled",
title: ready ? false : "the medium and every archived copy must check out first",
onclick: () =>
run(async () => {
const plan = await api.createRestorePlan({
location_id: location.id,
token: restore.token,
});
const started = await api.applyRestorePlan(plan.id);
watch(started.job.id, plan.id, "restore");
return { restoring: plan.asset_count };
}),
},
`Restore ${items.length} photo(s) from ${location.name}`
)
)
);
}
// ── running commands ─────────────────────────────────────────────────────────
async function run(action) {
try {
outcome = { kind: "ok", result: await action() };
} catch (error) {
outcome = error.status === 409 ? { kind: "conflict", error } : { kind: "error", error };
}
render();
}
// Live job activity. The plan panel is refreshed on its own tick because the
// journal advances per file, not per job event; a full re-render would re-run
// preflight (which re-hashes the library), so that happens once when the job ends.
const REFRESH_MS = 500;
function watch(jobId, planId, kind) {
activity = [`Started ${kind} job ${jobId}`];
const tick = setInterval(() => refreshPlan(planId, kind), REFRESH_MS);
subscribeJob(jobId, {
onEvent: (event) => {
activity.push(`${event.type}${event.message ? ": " + event.message : ""}`);
const log = document.querySelector('[data-testid="archive-activity"]');
if (log) log.textContent = activity.join("\n");
},
onDone: () => {
clearInterval(tick);
activity.push("done");
render();
},
});
}
async function refreshPlan(planId, kind) {
const node = document.querySelector(`[data-testid="plan-detail"][data-plan="${planId}"]`);
if (!node) return; // the user navigated away from the running plan
try {
const plan = await (kind === "restore" ? api.getRestorePlan(planId) : api.getArchivePlan(planId));
node.replaceWith(planDetail(plan));
} catch (_) {
// Transient; the next tick tries again and the job's end re-renders anyway.
}
}
function activityLog() {
return el(
"pre",
{
class: "activity-log",
role: "status",
"aria-live": "polite",
"data-testid": "archive-activity",
},
activity.join("\n")
);
}
function outcomeBanner() {
if (!outcome) return null;
if (outcome.kind === "conflict") {
return el(
"div",
{ class: "alert", role: "alert", "data-testid": "conflict" },
`The server refused this: ${outcome.error.message}. Nothing was moved or removed; ` +
"the state below is the server's current one — review it and decide again."
);
}
if (outcome.kind === "error") {
return el(
"div",
{ class: "alert", role: "alert", "data-testid": "archive-error" },
`Failed: ${outcome.error.message}`
);
}
const result = outcome.result || {};
const message =
result.archiving !== undefined
? `Archiving ${result.archiving} photo(s). Originals are removed only after their copies verify.`
: result.restoring !== undefined
? `Restoring ${result.restoring} photo(s) into the library.`
: "Done — the state below is the server's.";
return el("div", { class: "alert", role: "status", "data-testid": "archive-result" }, message);
}
// ── formatting ───────────────────────────────────────────────────────────────
function bytes(value) {
if (value == null) return "unknown";
const units = ["B", "kB", "MB", "GB", "TB"];
let size = value;
let unit = 0;
while (size >= 1000 && unit < units.length - 1) {
size /= 1000;
unit += 1;
}
return `${unit === 0 ? size : size.toFixed(1)} ${units[unit]}`;
}

263
frontend/js/docs.js
View File

@@ -1,263 +0,0 @@
// The documentation view (US09-01): the manuals in `docs/`, rendered in the app.
//
// The same markdown files are the repository's documentation and the deployment's
// documentation. An operator who was handed a URL and an access secret has no
// checkout in front of them, and the network the application runs on is not assumed
// to reach a CDN — so the renderer is vendored and everything here is same-origin.
//
// Nothing on this page is authenticated. It is served from the static mount beside
// the application shell, exactly like `index.html`: the troubleshooting page is
// needed most by whoever cannot get past the access secret, and no documentation
// file contains anything a session would protect.
//
// `marked` and `mermaid` are loaded lazily, on the first documentation page and the
// first diagram. They are large, and every other view does without them.
import { el, errorBanner, setActiveNav } from "./dom.js";
const DOCS_BASE = "/docs";
const VENDOR = "/app/js/vendor";
const INDEX_PAGE = "index";
// A page name comes from the hash, so it is caller input: no scheme, no traversal,
// no absolute path. The server would refuse those too; this refuses them earlier and
// without a request that looks like an attempt.
const PAGE_PATTERN = /^[\w-]+(\/[\w-]+)*$/;
let markedPromise;
let mermaidPromise;
function loadMarked() {
markedPromise ??= import(`${VENDOR}/marked.esm.js`).then((module) => module.marked);
return markedPromise;
}
// mermaid ships one large UMD bundle rather than a self-contained ES module, so it
// arrives through a script element. `script-src 'self'` allows it because it is ours
// and same-origin; nothing here relaxes that.
function loadMermaid() {
mermaidPromise ??= new Promise((resolve, reject) => {
const script = document.createElement("script");
script.src = `${VENDOR}/mermaid.min.js`;
script.onload = () => resolve(window.mermaid);
script.onerror = () => reject(new Error("the diagram renderer could not be loaded"));
document.head.appendChild(script);
});
return mermaidPromise;
}
// ── the pure parts, unit-tested in js/tests/unit.js ──────────────────────────
/** A stable, readable heading anchor. Letters and digits of any script survive. */
export function slug(text) {
const cleaned = String(text)
.toLowerCase()
.trim()
.replace(/[^\p{L}\p{N}\s-]/gu, "")
.replace(/[\s-]+/g, "-")
.replace(/^-|-$/g, "");
return cleaned || "section";
}
/** Assign every heading an id, disambiguating repeats the way a reader would
* expect: the first `Notes` keeps `#notes`, the second becomes `#notes-2`. */
export function assignHeadingIds(headings) {
const used = new Map();
for (const heading of headings) {
const base = slug(heading.textContent);
const seen = (used.get(base) || 0) + 1;
used.set(base, seen);
heading.id = seen === 1 ? base : `${base}-${seen}`;
}
return headings;
}
/**
* Where a link inside a documentation page should go.
*
* Markdown links between documents are relative file paths, which a browser would
* treat as downloads that leave the application. Returns the in-app target for a
* link to another document or to a heading, and `null` for anything else — external
* links stay exactly as the author wrote them.
*/
export function resolveDocLink(currentPage, href) {
if (!href) return null;
if (/^[a-z][a-z\d+.-]*:/i.test(href) || href.startsWith("//")) return null;
if (href.startsWith("#")) return { page: currentPage, anchor: href.slice(1) };
const [target, anchor = ""] = href.split("#");
if (!/\.md$/i.test(target)) return null;
// Resolved against the current document, so `../guides/x.md` means what it means
// in the repository. The origin is a placeholder; only the path is used.
const resolved = new URL(target, `https://docs.invalid/${currentPage}.md`);
const page = decodeURIComponent(resolved.pathname).replace(/^\//, "").replace(/\.md$/i, "");
return PAGE_PATTERN.test(page) ? { page, anchor } : null;
}
/**
* Point every in-app link at its route, and leave every other link alone.
*
* The resolved page is kept on the element, so the reading order can be read back
* from the rendered index rather than parsed out of the markdown a second time.
*/
export function rewriteLinks(article, page) {
for (const link of article.querySelectorAll("a[href]")) {
const href = link.getAttribute("href");
const target = resolveDocLink(page, href);
if (target) {
link.setAttribute("href", docHash(target.page, target.anchor));
link.dataset.docPage = target.page;
} else if (/^https?:/i.test(href)) {
link.setAttribute("rel", "noreferrer noopener");
link.setAttribute("target", "_blank");
}
}
return article;
}
/** The reading order, taken from the index document itself — one list, in one
* place, that is equally the index on Gitea and the navigation here. */
export function documentIndex(indexElement) {
const pages = [];
for (const link of indexElement.querySelectorAll("a[data-doc-page]")) {
const page = link.dataset.docPage;
if (page !== INDEX_PAGE && !pages.some((entry) => entry.page === page)) {
pages.push({ page, title: link.textContent.trim() });
}
}
return pages;
}
// ── the view ─────────────────────────────────────────────────────────────────
async function fetchPage(page) {
const response = await fetch(`${DOCS_BASE}/${page}.md`, { headers: { accept: "text/markdown" } });
if (!response.ok) throw new Error(`documentation page unavailable: ${response.status}`);
return response.text();
}
async function toArticle(markdown, page) {
const marked = await loadMarked();
const article = el("article", { class: "doc", "data-testid": "doc", "data-page": page });
// The only innerHTML in the application, and deliberate: rendering markdown *is*
// producing HTML. The input is a file from this repository, and the page's CSP
// (`script-src 'self'`, no `'unsafe-inline'`) means injected script and inline
// handlers do not run even if one ever were not.
article.innerHTML = marked.parse(markdown, { async: false });
assignHeadingIds(article.querySelectorAll("h1, h2, h3, h4, h5, h6"));
rewriteLinks(article, page);
return article;
}
function docHash(page, anchor = "") {
const query = new URLSearchParams(anchor ? { page, anchor } : { page });
return `#/docs?${query}`;
}
/** Diagrams are authored as ```mermaid blocks so the source is diffable and Gitea
* renders them natively. A failure here replaces the diagram, never the page. */
async function renderDiagrams(article) {
const blocks = [...article.querySelectorAll("pre > code.language-mermaid")];
if (!blocks.length) return;
let mermaid;
try {
mermaid = await loadMermaid();
mermaid.initialize({ startOnLoad: false, securityLevel: "strict", theme: "dark" });
} catch (error) {
blocks.forEach((block) => block.closest("pre").replaceWith(errorBanner(error.message)));
return;
}
for (const [index, block] of blocks.entries()) {
const figure = el("figure", { class: "diagram", "data-testid": "diagram" });
block.closest("pre").replaceWith(figure);
try {
const { svg } = await mermaid.render(`diagram-${index}-${Date.now()}`, block.textContent);
figure.innerHTML = svg;
} catch (error) {
figure.replaceWith(errorBanner(`Diagram could not be drawn: ${error.message}`));
}
}
}
function notFound() {
return el(
"div",
{ class: "alert", role: "alert", "data-testid": "doc-not-found" },
"That documentation page does not exist. ",
el("a", { class: "link", href: docHash(INDEX_PAGE) }, "Back to the documentation index")
);
}
function sidebar(pages, current) {
return el(
"nav",
{ class: "card", "aria-label": "Documentation" },
el("h2", {}, "Documentation"),
el(
"ul",
{ class: "album-list", "data-testid": "doc-pages" },
el(
"li",
{},
el(
"a",
{
href: docHash(INDEX_PAGE),
"aria-current": current === INDEX_PAGE ? "true" : false,
},
"Index"
)
),
...pages.map(({ page, title }) =>
el(
"li",
{},
el(
"a",
{
href: docHash(page),
"data-page": page,
"aria-current": page === current ? "true" : false,
},
title
)
)
)
)
);
}
export async function renderDocs(root, params = {}) {
setActiveNav("docs");
const requested = params.page || INDEX_PAGE;
const page = PAGE_PATTERN.test(requested) ? requested : "";
let indexArticle;
try {
indexArticle = await toArticle(await fetchPage(INDEX_PAGE), INDEX_PAGE);
} catch (error) {
root.replaceChildren(errorBanner(`Documentation is unavailable: ${error.message}`));
return;
}
const pages = documentIndex(indexArticle);
let article;
if (!page) article = notFound();
else if (page === INDEX_PAGE) article = indexArticle;
else {
try {
article = await toArticle(await fetchPage(page), page);
} catch {
article = notFound();
}
}
root.replaceChildren(el("div", { class: "two-pane" }, sidebar(pages, page), article));
await renderDiagrams(article);
scrollToAnchor(params.anchor);
}
// A documentation anchor cannot live in the hash — the hash is the route — so it
// travels as a parameter and is applied after the page renders.
function scrollToAnchor(anchor) {
if (!anchor) return;
const target = document.getElementById(anchor);
if (target) target.scrollIntoView({ block: "start" });
}

13
frontend/js/dom.js
View File

@@ -21,19 +21,6 @@ export function errorBanner(message) {
return el("div", { class: "alert", role: "alert" }, message);
}
/**
* Replace a container's contents, dropping the absent ones.
*
* `el` already ignores null children, but `replaceChildren` does not: it stringifies
* them, so an optional node that is not there renders the word "null" on the page.
* Every view builds optional nodes — a banner only while a job runs, a conflict only
* after a 409 — so the filter belongs here rather than in each caller. It was missing
* from one of them, and a documentation screenshot is where that turned up (US09-04).
*/
export function show(root, ...nodes) {
root.replaceChildren(...nodes.flat().filter((node) => node != null && node !== false));
}
export function setActiveNav(view) {
document.querySelectorAll("nav a[data-nav]").forEach((a) => {
if (a.dataset.nav === view) a.setAttribute("aria-current", "page");

8
frontend/js/renames.js
View File

@@ -5,7 +5,7 @@
// recovery classifications all come from the server; the view only shows them and
// refuses to offer an action the server would reject.
import { api } from "./api.js";
import { el, errorBanner, setActiveNav, show } from "./dom.js";
import { el, errorBanner, setActiveNav } from "./dom.js";
// What the last confirm did, for this tab only. Deliberately not persisted: after a
// reload the page must show what the journal says, not what this page remembers.
@@ -24,7 +24,7 @@ export async function renderRenames(root, params = {}) {
try {
[plans, recovery] = await Promise.all([api.listPlans(), api.renameRecovery()]);
} catch (error) {
show(root, errorBanner(`Failed to load renames: ${error.message}`));
root.replaceChildren(errorBanner(`Failed to load renames: ${error.message}`));
return;
}
@@ -34,13 +34,13 @@ export async function renderRenames(root, params = {}) {
try {
plan = await api.getPlan(selectedId);
} catch (error) {
show(root, errorBanner(`Failed to load plan: ${error.message}`));
root.replaceChildren(errorBanner(`Failed to load plan: ${error.message}`));
return;
}
}
const blocked = recovery.blocks_mutation;
show(root,
root.replaceChildren(
el("h1", {}, "Renames"),
recoveryPanel(recovery),
el(

61
frontend/js/tests/unit.js
View File

@@ -7,7 +7,6 @@ import { createStore } from "../store.js";
import { parseHash, navigate } from "../router.js";
import { api, cancellable } from "../api.js";
import { subscribeJob } from "../events.js";
import { assignHeadingIds, documentIndex, resolveDocLink, rewriteLinks, slug } from "../docs.js";
const cases = [];
function ok(name, cond) {
@@ -30,10 +29,6 @@ function jsonResponse(status, body) {
const tick = (ms = 0) => new Promise((r) => setTimeout(r, ms));
async function run() {
// The client fetches its CSRF token once, lazily (US07-02). Do that against the
// real server first, so the stubbed fetch below only ever sees the call under test.
await api.workflow().catch(() => {});
// ── store ────────────────────────────────────────────────────────────────
{
const store = createStore({ n: 0 });
@@ -143,62 +138,6 @@ async function run() {
window.EventSource = realES;
}
// ── documentation view (US09-01) ─────────────────────────────────────────
{
ok("slug lowercases and dashes a heading", slug("Key Flows") === "key-flows");
ok(
"slug drops punctuation but keeps words",
slug("What it *will not* do:") === "what-it-will-not-do"
);
ok("slug keeps non-ASCII letters", slug("Größe & Gewicht") === "größe-gewicht");
ok("slug never yields an empty anchor", slug("!!!") === "section");
const doc = document.createElement("div");
doc.innerHTML = "<h2>Notes</h2><h3>Notes</h3><h2>Notes</h2>";
const ids = [...assignHeadingIds(doc.querySelectorAll("h2, h3"))].map((h) => h.id);
ok("repeated headings get distinct anchors", ids.join(",") === "notes,notes-2,notes-3");
ok(
"a sibling document link becomes an in-app route",
resolveDocLink("index", "overview.md")?.page === "overview"
);
const nested = resolveDocLink("guides/install", "../overview.md#running-it");
ok("a relative link resolves against the current page", nested?.page === "overview");
ok("a link's anchor survives the rewrite", nested?.anchor === "running-it");
ok(
"a bare anchor stays on the current page",
resolveDocLink("overview", "#the-workflow")?.page === "overview"
);
ok("an external link is left alone", resolveDocLink("index", "https://example.test/x") === null);
ok("a non-markdown relative link is left alone", resolveDocLink("index", "images/a.png") === null);
// A climb cannot leave the documentation tree: it is clamped at the root, so the
// page it names is still fetched from under /docs and simply does not exist.
ok(
"a link that climbs above the docs root is clamped",
resolveDocLink("index", "../../etc/passwd.md")?.page === "etc/passwd"
);
ok(
"a page name that is not a page name is refused",
resolveDocLink("index", "..%2f..%2fetc%2fpasswd.md") === null
);
const index = document.createElement("div");
index.innerHTML =
'<a href="overview.md">Overview</a><a href="overview.md">again</a>' +
'<a href="index.md">itself</a><a href="https://example.test">out</a>';
const listed = documentIndex(rewriteLinks(index, "index"));
ok(
"an external link is marked safe to open away from the app",
index.querySelector('a[href^="https"]').rel === "noreferrer noopener"
);
ok(
"an in-app link points at the docs route",
index.querySelector("a").getAttribute("href") === "#/docs?page=overview"
);
ok("the index lists each page once, in order", listed.length === 1);
ok("the index takes its titles from the link text", listed[0].title === "Overview");
}
await tick();
const failed = cases.filter((c) => !c.ok);
window.__RESULTS__ = { passed: cases.length - failed.length, failed: failed.length, cases };

8
frontend/js/uploads.js
View File

@@ -8,7 +8,7 @@
// preview arrives redacted — so nothing in this view may reconstruct, store, or
// route a secret.
import { api } from "./api.js";
import { el, errorBanner, setActiveNav, show } from "./dom.js";
import { el, errorBanner, setActiveNav } from "./dom.js";
import { subscribeJob } from "./events.js";
import { navigate } from "./router.js";
@@ -45,7 +45,7 @@ export async function renderUploads(root, params = {}) {
api.listUploadBatches(),
]);
} catch (error) {
show(root, errorBanner(`Failed to load uploads: ${error.message}`));
root.replaceChildren(errorBanner(`Failed to load uploads: ${error.message}`));
return;
}
@@ -62,12 +62,12 @@ export async function renderUploads(root, params = {}) {
batch = detail;
history = verifications.verifications;
} catch (error) {
show(root, errorBanner(`Failed to load upload batch: ${error.message}`));
root.replaceChildren(errorBanner(`Failed to load upload batch: ${error.message}`));
return;
}
}
show(root,
root.replaceChildren(
...[
el("h1", {}, "Upload"),
configurationCard(preflight),

23
frontend/js/vendor/VERSIONS.json vendored
View File

@@ -1,23 +0,0 @@
{
"_comment": "Vendored browser libraries (US09-01). Committed rather than fetched: the application is deployed to a network whose outbound access is not assumed, and `default-src 'self'` forbids a CDN. Checksums are asserted by tests/integration/test_documentation.py, so replacing a file without recording it here fails the suite. Update by downloading the pinned URL and recording the new version and sha256 in the same commit.",
"libraries": [
{
"file": "marked.esm.js",
"name": "marked",
"version": "18.0.10",
"license": "MIT",
"url": "https://cdn.jsdelivr.net/npm/marked@18.0.10/lib/marked.esm.js",
"sha256": "4cf47dfebb7f614a08fc0a579ab0fe407ff0ed2b717bf953040c85b2f493a4f0",
"why": "Markdown to HTML for the documentation view. An ES module with no dependencies, imported lazily by js/docs.js."
},
{
"file": "mermaid.min.js",
"name": "mermaid",
"version": "11.17.0",
"license": "MIT",
"url": "https://cdn.jsdelivr.net/npm/mermaid@11.17.0/dist/mermaid.min.js",
"sha256": "8d8e0eec56d3a83b4b3c87f42050845546dee93ebe1875d2117c12e6947c0cb3",
"why": "Renders ```mermaid blocks so diagrams are diffable source that Gitea also renders. The single-file UMD build rather than the ES module entry, whose ~40 lazy chunks would each need vendoring and pinning."
}
]
}

78
frontend/js/vendor/marked.esm.js vendored
View File

File diff suppressed because one or more lines are too long

3636
frontend/js/vendor/mermaid.min.js vendored
View File

File diff suppressed because one or more lines are too long

26
frontend/js/views.js
View File

@@ -3,7 +3,7 @@
// HTML. Mutating actions are disabled while a job holds the library_write lock and
// say why (concept §shared interaction rules: read-only browsing stays available).
import { api } from "./api.js";
import { el, errorBanner, setActiveNav, show } from "./dom.js";
import { el, errorBanner, setActiveNav } from "./dom.js";
import { navigate } from "./router.js";
import { subscribeJob } from "./events.js";
@@ -26,7 +26,7 @@ export async function renderWorkflow(root) {
try {
data = await api.workflow();
} catch (error) {
show(root, errorBanner(`Failed to load workflow: ${error.message}`));
root.replaceChildren(errorBanner(`Failed to load workflow: ${error.message}`));
return;
}
const active = data.active_job;
@@ -67,7 +67,7 @@ export async function renderWorkflow(root) {
)
);
});
show(root,
root.replaceChildren(
el("h1", {}, "Workflow"),
jobBanner(active),
el("div", { class: "stepper" }, ...cards)
@@ -105,7 +105,7 @@ export async function renderSafety(root, params) {
try {
data = await api.safetyQueue({ state });
} catch (error) {
show(root, errorBanner(`Failed to load safety queue: ${error.message}`));
root.replaceChildren(errorBanner(`Failed to load safety queue: ${error.message}`));
return;
}
@@ -149,7 +149,7 @@ export async function renderSafety(root, params) {
)
);
show(root,
root.replaceChildren(
el("h1", {}, "Safety review"),
tabs,
el(
@@ -197,7 +197,7 @@ export async function renderLibrary(root, params) {
try {
data = await api.libraryAssets({ q, offset, limit, sort: params.sort || "path" });
} catch (error) {
show(root, errorBanner(`Failed to load library: ${error.message}`));
root.replaceChildren(errorBanner(`Failed to load library: ${error.message}`));
return;
}
@@ -220,7 +220,7 @@ export async function renderLibrary(root, params) {
)
);
show(root,
root.replaceChildren(
el("h1", {}, "Library"),
el("div", { class: "toolbar" }, search, el("span", { class: "muted", "data-testid": "library-total" }, `${data.total} photos`)),
data.rows.length ? el("div", { class: "cluster-grid" }, ...cards) : el("p", { class: "muted" }, "No matching photos."),
@@ -235,7 +235,7 @@ export async function renderAnalyze(root) {
try {
[counts, workflow] = await Promise.all([api.analysisCounts(), api.workflow()]);
} catch (error) {
show(root, errorBanner(`Failed to load analysis: ${error.message}`));
root.replaceChildren(errorBanner(`Failed to load analysis: ${error.message}`));
return;
}
const active = workflow.active_job;
@@ -269,7 +269,7 @@ export async function renderAnalyze(root) {
"Analyze eligible SFW assets"
);
show(root,
root.replaceChildren(
el("h1", {}, "Analyze"),
jobBanner(active),
el(
@@ -292,10 +292,10 @@ export async function renderStats(root) {
try {
data = await api.libraryStats();
} catch (error) {
show(root, errorBanner(`Failed to load stats: ${error.message}`));
root.replaceChildren(errorBanner(`Failed to load stats: ${error.message}`));
return;
}
show(root,
root.replaceChildren(
el("h1", {}, "Stats"),
el("div", { class: "counts-grid", "data-testid": "stats-status" }, ...Object.entries(data.status).map(([k, v]) => stat(k, v))),
facetBlock("Top tags", data.top_tags),
@@ -338,7 +338,7 @@ export async function renderAlbums(root, params = {}) {
api.renameRecovery(),
]);
} catch (error) {
show(root, errorBanner(`Failed to load albums: ${error.message}`));
root.replaceChildren(errorBanner(`Failed to load albums: ${error.message}`));
return;
}
@@ -385,7 +385,7 @@ export async function renderAlbums(root, params = {}) {
)
: el("p", { class: "muted" }, "No albums with evidence yet.");
show(root,
root.replaceChildren(
el("h1", {}, "Albums"),
renameBlocked
? el(

22
legacy_cli_archive/CHECKSUMS.sha256
View File

@@ -1,22 +0,0 @@
d098bbde13d2ebc872ca781e244cc1e48b97cc551fedaca3251d6a5278a15234 src/compare_models.py
7d0f68cb95fbd6603e4c558620098b46929ebdb6fd91a598bbf84f26eea35e6c src/nsfw_tag.py
35329c53570e215cef15f59429c1a5251a0448b90522b0910ab9d68efe1bc307 src/nsfwtag/__init__.py
7f52b55e4f3b77eda3657d7cd2272c7cca8422100e241671de495603c1126ee2 src/nsfwtag/__main__.py
6f5a97114e0d87d272ce22d065dc31fa0dd72ff8590d11ad14cb1c1f486d339c src/nsfwtag/bench.py
29498fac1d73ba2b7420ffe1ffe49a87f684ec299e2c3b9c6e14e63e95643ba4 src/nsfwtag/exif.py
934e82c402813ebf503e5a20eb03d5103f84df13117bc4dd1fd95deaa01ab263 src/nsfwtag/README.md
68125e6184c7e4d2a5b0153f2155675753ab0b1329d1dbc4933d4769492ec4be src/nsfwtag/review.html
a597eab74803dd31452b5d70abf7d6d6320eb17a61596c891b009da15d624e98 src/nsfwtag/scoring.py
674969a18e58ee511a2abf574a875db92f2f524769b1b614d0d3662ab879a709 src/nsfwtag/server.py
24a7c8d029da6e97d46b110a9fe9dbb3900127f8ae3c142f634fdd91d6c45243 src/nsfwtag/webapp.py
2c2ea558f1b9095c1150f078ea29c8e0f180cc045a9d1097b83f61bfb145295b src/photo_analyzer.py
5fb7ce2f977a17da1f501c45d7985328318ae14298a3f347eab46a6e3f02aab3 src/test_dedup.py
1673dc76cc60aba56a5b065d9bc9f5dda00342cf2a79d000a3f74686fb7cffec src/test_nsfw_skip.py
55aa82f348e0e90be0163ba6aa278e5fc8e445996e5335774917e5ef59ad3563 src/webapp/__init__.py
ba4342bd0175591a2121f063f14a78a602d5679bcfcf24aa14c8d4b8cd5da1a2 src/webapp/__main__.py
a60f24035a989909467778f888d62854c1afc2702f47b8d3cca32bf3cf04d5cf src/webapp/analyzer.html
018bcc2f770d716444b456db58d6b4a41800138a98598b3c27ba503ab22d939e src/webapp/page.py
5f599b107b2b117ca118b6fbdec5e5786a9ab4eea424aa9af1bfea7bf87cebab src/webapp/query.py
875cce697caa02717c85a2707ba935c46dd9ace18428fd4abb13105b7a8e24d9 src/webapp/README.md
af7d0d72d245b4bbb1e0e30f9708239697543212d4171defd1bfe26dd24a4f05 src/webapp/runner.py
cddf0555b06fe7e4daa14309baaaf91130f8506922cb0ffb6507d76cbe39add4 src/webapp/server.py

80
legacy_cli_archive/README.md
View File

@@ -1,80 +0,0 @@
# Legacy CLI archive (US07-01)
Frozen, read-only sources of the command-line tools this application was extracted
from. They are **reference material and rollback evidence** — provenance for
behavior that now lives in `photo_pipeline/`, and the only way to answer "what did
the original actually do?" once the replacement has drifted.
> **Nothing here is production code.** No module under `photo_pipeline/` imports or
> executes anything in this directory, and this directory is not on the application's
> import path. `tests/unit/test_legacy_archive.py` enforces both, along with the
> checksums and the redaction below.
## What is here
| Path | Role |
|---|---|
| `src/photo_analyzer.py` | the analysis CLI: discovery, hashing/dedup, vision analysis, EXIF writing, SQLite schema, album naming |
| `src/nsfwtag/` | NSFW scoring, EXIF safety keywords, and the review server (`__init__` 1.1.0) |
| `src/webapp/` | the stdlib review web app: FTS search, stats, subprocess runner, HTML shell (`__init__` 0.1.0) |
| `src/nsfw_tag.py` | thin backwards-compatible entry point for `nsfwtag` |
| `src/compare_models.py` | dev-only model comparison script |
| `src/test_dedup.py`, `src/test_nsfw_skip.py` | the CLIs' own standalone self-checks (never pytest suites) |
| `donor_ledger.yaml` | the donor ledger: every migrated behavior, its target, its tests, and every intentional delta |
| `requirements-lock.txt` | the dependency versions the frozen sources were last verified against |
| `photo_analyzer.env.sample` | the CLI's configuration surface, with every value replaced by a placeholder |
| `CHECKSUMS.sha256` | SHA-256 of every archived source file |
`photo_analyzer.py` carries no `__version__`; its identity is its checksum, recorded
in `CHECKSUMS.sha256` and taken at commit `9b7ee6b` (the merge of US06-06, the last
commit before archival).
## Verifying the archive
```bash
cd legacy_cli_archive && shasum -a 256 -c CHECKSUMS.sha256
```
Any edit to an archived source must be accompanied by a regenerated checksum file
and a note here explaining why a *frozen* archive changed — the normal answer being
that it should not.
## Schema notes
`photo_analyzer.py` owned a path-keyed SQLite database (`SCHEMA`, near the top of the
file):
- `photos(id, path UNIQUE, status, phash, file_sha1, dup_of, description, tags,
people_count, setting, time_of_day, season, mood, location_hint, approx_year,
raw_response, error_message, analyzed_at, exif_written_at)`;
- `photos_fts` — an FTS5 external-content index over `path, description, tags, mood,
location_hint`, kept in sync by insert/update/delete triggers;
- late columns (`phash`, `file_sha1`, `dup_of`) were added by an in-code
`_migrate_schema()` rather than a migration tool, and their indexes are created
only after the `ALTER`.
The replacement keeps the same analysis fields but re-keys everything to a stable
`assets.id` (Alembic migrations `0001`…), because a path is not an identity: the
donor's `path UNIQUE` is exactly what broke on every move and rename.
`nsfwtag` kept its safety scores outside the database in `nsfw_scores.csv`
(`path,nsfw_score`, four decimals, unreadable rows dropped). That file is no longer a
source of truth; `photo_pipeline/services/legacy_import.py` imports it into
`assets.safety_score` and reports exactly what matched, what did not, and why.
## Redaction
The archive contains no credentials. `photo_analyzer.env.sample` documents the
configuration surface (`LLM_API_KEY`, `LLM_BASE_URL`, `LLM_MODEL`, and the tuning
variables) with placeholder values only; the CLI itself never contained a key, it
read one from `photo_analyzer.env` or the environment. No `.env`, database, log, CSV,
or photo from the author's library is archived.
## Why these tools were retired
Each behavior's fate is recorded per row in `donor_ledger.yaml`: `reuse`, `extract`,
`refactor`, or `replace`, with the target module, the characterization tests that
pinned the donor's behavior, the parity tests the replacement passes, and — where the
replacement deliberately does something else or nothing at all — a `delta` saying so.
Rows still marked `pending` name the backlog story that will resolve them; they are
the honest list of what has *not* been carried over yet.

18
legacy_cli_archive/photo_analyzer.env.sample
View File

@@ -1,18 +0,0 @@
# Sample configuration for the archived photo_analyzer.py CLI (US07-01).
#
# REDACTED: no value below is real. Each line names a variable the CLI read and
# what belongs there; the placeholders are deliberately not key-shaped, so this
# file can never be mistaken for — or scanned as — a credential.
LLM_API_KEY=<paste your provider key here>
LLM_BASE_URL=<provider base url, e.g. the OpenAI-compatible Gemini endpoint>
LLM_MODEL=<model id, e.g. a Gemini Flash release>
# Optional tuning the CLI read from the same file:
PHASH_THRESHOLD=8
MAX_WORKERS=4
RETRY_ATTEMPTS=3
RPD_LIMIT=0
# The CLI ignored its own shipped placeholder (a literal "sk-REPLACE..." string)
# until it was replaced, and a shell variable always won over this file.

20
legacy_cli_archive/requirements-lock.txt
View File

@@ -1,20 +0,0 @@
# Final dependency lock of the archived CLIs (US07-01).
#
# These are the versions present in the environment the archive was taken from —
# what the frozen sources were last verified against by the characterization
# suite. Restoring a donor for forensics means pinning these, not "latest".
#
# Python 3.14.6
openai==3.0.0 # photo_analyzer: OpenAI-compatible vision client
numpy==2.4.6 # photo_analyzer, nsfwtag: pixel work
Pillow==12.3.0 # photo_analyzer, nsfwtag: decode/resize
rich==15.0.0 # photo_analyzer: console output
scipy==1.18.0 # photo_analyzer: perceptual-hash DCT
PyYAML==6.0.3 # tooling that reads the donor ledger
# NSFW inference (nsfwtag/scoring.py, nsfwtag/bench.py) was never installed in the
# archiving environment; the model stack is recorded here from the sources so a
# forensic run can reproduce it, not from a resolved lock:
# torch, transformers, timm — AdamCodd/vit-base-nsfw-detector (see nsfwtag/README.md)
# exiftool is an external binary, not a Python package.

40
migrations/versions/0011_archive_locations.py
View File

@@ -1,40 +0,0 @@
"""Archive locations (US06-01).
Revision ID: 0011_archive_locations
Revises: 0010_upload_verification
Create Date: 2026-08-16
Configured archive destinations with their stable media identity, last probed
capabilities, and state.
"""
import sqlalchemy as sa
from alembic import op
revision = "0011_archive_locations"
down_revision = "0010_upload_verification"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"archive_locations",
sa.Column("id", sa.String(), primary_key=True),
sa.Column("name", sa.String(), nullable=False, unique=True),
sa.Column("root", sa.String(), nullable=False),
sa.Column("media_id", sa.String(), nullable=False, unique=True),
sa.Column("capabilities", sa.String(), nullable=True),
sa.Column("state", sa.String(), nullable=False, server_default="offline"),
sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=True),
sa.Column(
"created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
),
sa.Column(
"updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
),
)
def downgrade() -> None:
op.drop_table("archive_locations")

96
migrations/versions/0012_archive_plans.py
View File

@@ -1,96 +0,0 @@
"""Archive plans, per-item transfer journal, and archived asset location (US06-02).
Revision ID: 0012_archive_plans
Revises: 0011_archive_locations
Create Date: 2026-08-16
The journal is what makes removing an original recoverable: every item records its
source, destination, expected hash, and the state it had reached before the process
died. Assets gain the archive location and relative path so an offline original is
still explained rather than looking missing.
"""
import sqlalchemy as sa
from alembic import op
revision = "0012_archive_plans"
down_revision = "0011_archive_locations"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"archive_plans",
sa.Column("id", sa.String(), primary_key=True),
sa.Column(
"location_id",
sa.String(),
sa.ForeignKey("archive_locations.id"),
nullable=False,
index=True,
),
# The preflight token this plan was approved against.
sa.Column("token", sa.String(), nullable=False),
sa.Column("albums", sa.String(), nullable=True), # JSON array
# planned | applying | complete | failed
sa.Column("state", sa.String(), nullable=False, server_default="planned"),
sa.Column("schema_version", sa.Integer(), nullable=False, server_default="1"),
sa.Column("asset_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("byte_size", sa.Integer(), nullable=False, server_default="0"),
# Bumped on every claim and used as the fencing token.
sa.Column("version", sa.Integer(), nullable=False, server_default="1"),
sa.Column("worker_id", sa.String(), nullable=True),
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column(
"created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
),
sa.Column(
"updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
),
)
op.create_table(
"archive_operations",
sa.Column("id", sa.String(), primary_key=True),
sa.Column(
"plan_id",
sa.String(),
sa.ForeignKey("archive_plans.id", ondelete="CASCADE"),
nullable=False,
index=True,
),
sa.Column("sequence", sa.Integer(), nullable=False),
sa.Column("album", sa.String(), nullable=False),
sa.Column("asset_id", sa.String(), sa.ForeignKey("assets.id"), nullable=False, index=True),
sa.Column("source_path", sa.String(), nullable=False),
sa.Column("destination_path", sa.String(), nullable=False),
# Relative to the location root: the medium can be mounted anywhere later.
sa.Column("archive_path", sa.String(), nullable=False),
sa.Column("expected_sha256", sa.String(), nullable=False),
sa.Column("byte_size", sa.Integer(), nullable=True),
sa.Column("same_filesystem", sa.Boolean(), nullable=True),
# planned | transferring | verified | removing | complete | failed
sa.Column("journal_state", sa.String(), nullable=False, server_default="planned"),
sa.Column("attempt_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column("fencing_token", sa.Integer(), nullable=True),
sa.Column("worker_id", sa.String(), nullable=True),
sa.Column("verified_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("removed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("error_code", sa.String(), nullable=True),
sa.Column("error_message", sa.String(), nullable=True),
sa.Column(
"updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
),
sa.UniqueConstraint("plan_id", "sequence", name="uq_archive_operations_plan_sequence"),
)
# Plain columns: SQLite cannot ALTER a table to add a foreign key, and the
# relationship is enforced by the service that writes them.
op.add_column("assets", sa.Column("archive_location_id", sa.String(), nullable=True))
op.add_column("assets", sa.Column("archive_path", sa.String(), nullable=True))
def downgrade() -> None:
op.drop_column("assets", "archive_path")
op.drop_column("assets", "archive_location_id")
op.drop_table("archive_operations")
op.drop_table("archive_plans")

29
migrations/versions/0013_protected_thumbnails.py
View File

@@ -1,29 +0,0 @@
"""Protected thumbnails (US06-03).
Revision ID: 0013_protected_thumbnails
Revises: 0012_archive_plans
Create Date: 2026-08-16
A protected thumbnail is the durable comparison preview of an asset whose
original has left active storage. It is evidence rather than cache, so the LRU
quota must not evict it: the archive medium may be offline when it is needed.
"""
import sqlalchemy as sa
from alembic import op
revision = "0013_protected_thumbnails"
down_revision = "0012_archive_plans"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"thumbnails",
sa.Column("protected", sa.Boolean(), nullable=False, server_default=sa.false()),
)
def downgrade() -> None:
op.drop_column("thumbnails", "protected")

37
migrations/versions/0014_restore_plans.py
View File

@@ -1,37 +0,0 @@
"""Restore plans and archive divergence (US06-04).
Revision ID: 0014_restore_plans
Revises: 0013_protected_thumbnails
Create Date: 2026-08-16
Restore reuses the archive plan and journal tables: the crash-safe question is the
same one in the opposite direction (copy, verify, publish, register), so the rows
gain a ``direction`` instead of a parallel pair of tables. ``archive_divergent_at``
records the moment an archived copy was proven to hold bytes that are not the ones
the database recorded — a restore must never silently accept a different file.
"""
import sqlalchemy as sa
from alembic import op
revision = "0014_restore_plans"
down_revision = "0013_protected_thumbnails"
branch_labels = None
depends_on = None
def upgrade() -> None:
for table in ("archive_plans", "archive_operations"):
op.add_column(
table,
sa.Column("direction", sa.String(), nullable=False, server_default="archive"),
)
op.add_column(
"assets", sa.Column("archive_divergent_at", sa.DateTime(timezone=True), nullable=True)
)
def downgrade() -> None:
op.drop_column("assets", "archive_divergent_at")
for table in ("archive_plans", "archive_operations"):
op.drop_column(table, "direction")

42
migrations/versions/0015_exif_projections.py
View File

@@ -1,42 +0,0 @@
"""Durable EXIF projections per asset and stage (US07-03).
Revision ID: 0015_exif_projections
Revises: 0014_restore_plans
Create Date: 2026-08-16
The concept's ``exif_projections`` table, added at the point it earns its keep: a
checkpoint that finds a field it does not own changed must be able to say so after
a restart. ``state`` is verified | divergent | failed, and only ``verified`` counts
as a completed metadata stage.
"""
import sqlalchemy as sa
from alembic import op
revision = "0015_exif_projections"
down_revision = "0014_restore_plans"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"exif_projections",
sa.Column("asset_id", sa.String(), sa.ForeignKey("assets.id"), primary_key=True),
sa.Column("stage", sa.String(), primary_key=True), # safety | analysis
sa.Column("id", sa.String(), nullable=False),
sa.Column("projection_version", sa.Integer(), nullable=False, server_default="0"),
# What the stage asked for: {"add": [...], "remove": [...]}.
sa.Column("desired_json", sa.String(), nullable=True),
# Fields outside the stage's ownership that did not survive the write.
sa.Column("divergent_fields", sa.String(), nullable=True),
sa.Column("result_file_sha256", sa.String(), nullable=True),
sa.Column("state", sa.String(), nullable=False),
sa.Column("error_code", sa.String(), nullable=True),
sa.Column("verified_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True),
)
def downgrade() -> None:
op.drop_table("exif_projections")

45
migrations/versions/0016_performance_indexes.py
View File

@@ -1,45 +0,0 @@
"""Indexes the large-library read paths need (US07-06).
Revision ID: 0016_performance_indexes
Revises: 0015_exif_projections
Create Date: 2026-08-17
Measured, not guessed. At 100k assets the workflow home and the safety queue both
resolve "the current decision per asset" with a window function over
``safety_reviews``; a plain ``asset_id`` index makes SQLite sort every partition by
hand. Ordering the index by ``(asset_id, created_at DESC)`` halves that query.
``duplicate_members(cluster_id, asset_id)`` serves the paged member list of a
cluster with thousands of members, which is the other page that stopped being cheap.
"""
from alembic import op
revision = "0016_performance_indexes"
down_revision = "0015_exif_projections"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_index(
"ix_safety_reviews_asset_created",
"safety_reviews",
["asset_id", "created_at"],
)
op.create_index(
"ix_duplicate_members_cluster_asset",
"duplicate_members",
["cluster_id", "asset_id"],
)
op.create_index(
"ix_analysis_results_approx_year",
"analysis_results",
["approx_year"],
)
def downgrade() -> None:
op.drop_index("ix_analysis_results_approx_year", table_name="analysis_results")
op.drop_index("ix_duplicate_members_cluster_asset", table_name="duplicate_members")
op.drop_index("ix_safety_reviews_asset_created", table_name="safety_reviews")

0
legacy_cli_archive/src/nsfw_tag.py → nsfw_tag.py
View File

0
legacy_cli_archive/src/nsfwtag/README.md → nsfwtag/README.md
View File

0
legacy_cli_archive/src/nsfwtag/__init__.py → nsfwtag/__init__.py
View File

0
legacy_cli_archive/src/nsfwtag/__main__.py → nsfwtag/__main__.py
View File

0
legacy_cli_archive/src/nsfwtag/bench.py → nsfwtag/bench.py
View File

0
legacy_cli_archive/src/nsfwtag/exif.py → nsfwtag/exif.py
View File

0
legacy_cli_archive/src/nsfwtag/review.html → nsfwtag/review.html
View File

0
legacy_cli_archive/src/nsfwtag/scoring.py → nsfwtag/scoring.py
View File

0
legacy_cli_archive/src/nsfwtag/server.py → nsfwtag/server.py
View File

0
legacy_cli_archive/src/nsfwtag/webapp.py → nsfwtag/webapp.py
View File

0
legacy_cli_archive/src/photo_analyzer.py → photo_analyzer.py
View File

295
photo_pipeline/__main__.py
View File

@@ -1,267 +1,28 @@
"""Application management CLI:
``python -m photo_pipeline {serve,migrate,worker,import-legacy-scores,backup,verify-backup,restore,diagnostics,benchmark,release-gate,container-gate,docs-gate,dry-run,approve-dry-run}``.
``serve`` and ``worker`` take the library process lock for their role (US07-05):
two workers, or the frozen CLI running beside the app, would each be safe on their
own and destructive together. ``restore`` is here rather than in the API because it
replaces the state of an installation and belongs to a stopped one.
"""
"""Application management CLI: ``python -m photo_pipeline {serve,migrate}``."""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from typing import Sequence
from photo_pipeline import path_policy
from photo_pipeline.config import Config
from photo_pipeline.services.app_lock import LegacyProcessActive, LibraryLock, LockHeld
from photo_pipeline.services.backup import BackupError, BackupService, migrate_with_backup
from photo_pipeline.db import run_migrations
def main(argv: Sequence[str] | None = None) -> int:
parser = argparse.ArgumentParser(prog="photo_pipeline")
commands = parser.add_subparsers(dest="command", required=True)
serve_cmd = commands.add_parser("serve", help="Run the API server")
commands.add_parser("serve", help="Run the API server")
commands.add_parser("migrate", help="Upgrade the database to the latest revision")
worker_cmd = commands.add_parser("worker", help="Run a durable-job worker")
worker_cmd.add_argument("--id", default="worker-1", help="Worker id (lease owner)")
for locked in (serve_cmd, worker_cmd):
locked.add_argument(
"--allow-legacy",
action="store_true",
help="Start even though a legacy CLI looks active (unsafe; you own the outcome)",
)
import_cmd = commands.add_parser(
"import-legacy-scores",
help="Import the archived CLI's nsfw_scores.csv into the database (US07-01)",
)
import_cmd.add_argument("csv", help="Path to nsfw_scores.csv")
import_cmd.add_argument(
"--overwrite", action="store_true", help="Replace differing imported scores"
)
import_cmd.add_argument(
"--dry-run", action="store_true", help="Report what would happen and change nothing"
)
backup_cmd = commands.add_parser("backup", help="Take an online database backup")
backup_cmd.add_argument("--reason", default="manual", help="Why (part of the directory name)")
backup_cmd.add_argument("--keep", type=int, default=7, help="How many backups to retain")
verify_cmd = commands.add_parser("verify-backup", help="Check a backup is intact and readable")
verify_cmd.add_argument("backup", help="Path to the backup directory")
restore_cmd = commands.add_parser(
"restore", help="Restore a verified backup into a fresh data directory"
)
restore_cmd.add_argument("backup", help="Path to the backup directory")
restore_cmd.add_argument("--into", required=True, help="Fresh data directory to restore into")
commands.add_parser("diagnostics", help="Report sizes, disk headroom, locks, and warnings")
bench_cmd = commands.add_parser(
"benchmark", help="Measure latency and resource use against agreed budgets (US07-06)"
)
bench_cmd.add_argument("--profile", default="smoke", help="smoke | short | full | huge")
bench_cmd.add_argument(
"--soak-seconds", type=float, default=0.0, help="Also run a soak of this length"
)
bench_cmd.add_argument("--output", help="Write the JSON report here as well as to stdout")
gate_cmd = commands.add_parser(
"release-gate", help="Run every suite in an isolated stack and keep the evidence"
)
gate_cmd.add_argument("--output", help="Evidence directory (default: data/release/<stamp>)")
container_cmd = commands.add_parser(
"container-gate",
help="Provision the composition from the built image, run the phase_h "
"acceptance suite against it, destroy it, and keep the evidence (US08-05)",
)
container_cmd.add_argument(
"--output", help="Evidence directory (default: data/container-gate/<stamp>)"
)
docs_cmd = commands.add_parser(
"docs-gate",
help="Check the manuals against the application they describe and keep the "
"evidence (US09-05)",
)
docs_cmd.add_argument("--output", help="Evidence directory (default: data/docs-gate/<stamp>)")
dry_cmd = commands.add_parser(
"dry-run", help="Read-only reconciliation of the configured library (US07-07)"
)
dry_cmd.add_argument("--output", help="Write the report here as well as to stdout")
approve_cmd = commands.add_parser(
"approve-dry-run", help="Approve a dry-run report, which is what enables mutation"
)
approve_cmd.add_argument("report", help="Path to the dry-run report")
approve_cmd.add_argument("--approver", required=True, help="Who is accepting this")
args = parser.parse_args(argv)
config = Config.from_env()
config.database_path.parent.mkdir(parents=True, exist_ok=True)
# In a container the two roles that touch the library check their boundary before
# they take a lock or bind a port: the configured roots must name the mount paths,
# and a mismatch is cheaper to refuse than to discover at the first write (US08-03).
# Only in a container — on a host an unmounted root is an ordinary Tuesday (an
# archive medium that is not plugged in), and refusing to serve would take the
# offline half of the library away with it.
if args.command in ("serve", "worker") and path_policy.in_container():
refusal = path_policy.roots_refusal(config.library_roots, require_mount=True)
if refusal is not None:
print(refusal, file=sys.stderr)
return 5
if args.command == "migrate":
manifest = migrate_with_backup(config)
if manifest:
print(json.dumps({"pre_migration_backup": manifest["name"]}, indent=2))
return 0
if args.command == "backup":
try:
manifest = BackupService(config).create(reason=args.reason, keep=args.keep)
except BackupError as error:
print(str(error))
return 1
print(json.dumps(manifest, indent=2))
return 0
if args.command == "verify-backup":
result = BackupService(config).verify(args.backup)
print(json.dumps(result.as_dict(), indent=2))
return 0 if result.ok else 1
if args.command == "restore":
try:
report = BackupService(config).restore(args.backup, args.into)
except BackupError as error:
print(str(error))
return 1
print(json.dumps(report, indent=2))
return 0
if args.command == "benchmark":
from photo_pipeline.services import benchmarks
try:
report = benchmarks.run(
config,
profile=args.profile,
soak_seconds=args.soak_seconds,
output=args.output,
)
except ValueError as error:
print(str(error))
return 1
print(json.dumps({k: v for k, v in report.items() if k != "runs"}, indent=2))
# A breached budget is a failed run, so a scheduled job notices without
# anyone reading the JSON.
return 0 if report["ok"] else 1
if args.command == "release-gate":
from photo_pipeline.services import release
report = release.run_gate(config, output=args.output)
print(
json.dumps(
{k: v for k, v in report.items() if k not in ("stages", "matrix")}, indent=2
)
)
return 0 if report["ok"] else 1
if args.command == "container-gate":
from datetime import datetime, timezone
from photo_pipeline.services import release
# The suite provisions and destroys the composition itself; what this command
# adds is the single entry point and the retained evidence. No skip is an
# environment limit here: a gate that did not reach the containers proved
# nothing about them.
output = args.output or Path(config.data_dir) / "container-gate" / datetime.now(
timezone.utc
).strftime("%Y%m%dT%H%M%SZ")
report = release.run_gate(
config,
output=output,
stages=release.CONTAINER_STAGES,
allowed_skips=release.CONTAINER_ALLOWED_SKIP_REASONS,
)
print(
json.dumps(
{k: v for k, v in report.items() if k not in ("stages", "matrix")}, indent=2
)
)
return 0 if report["ok"] else 1
if args.command == "docs-gate":
from datetime import datetime, timezone
from photo_pipeline.services import release
# Documentation that disagrees with the application is worse than none: it is
# trusted. So this gate accepts no skip either — a check that did not run is a
# page nobody compared with the code.
output = args.output or Path(config.data_dir) / "docs-gate" / datetime.now(
timezone.utc
).strftime("%Y%m%dT%H%M%SZ")
report = release.run_gate(
config,
output=output,
stages=release.DOCS_STAGES,
allowed_skips=release.DOCS_ALLOWED_SKIP_REASONS,
)
print(
json.dumps(
{k: v for k, v in report.items() if k not in ("stages", "matrix")}, indent=2
)
)
return 0 if report["ok"] else 1
if args.command == "dry-run":
from photo_pipeline.services import release
try:
report = release.dry_run(config)
except release.ReleaseError as error:
print(str(error))
return 1
if args.output:
Path(args.output).write_text(json.dumps(report, indent=2))
print(json.dumps(report, indent=2))
return 0
if args.command == "approve-dry-run":
from photo_pipeline.services import release
try:
record = release.approve(config, args.report, approver=args.approver)
except (release.ReleaseError, OSError, ValueError) as error:
print(str(error))
return 1
print(json.dumps(record, indent=2))
return 0
if args.command == "diagnostics":
from photo_pipeline.services import diagnostics
print(json.dumps(diagnostics.report(config), indent=2))
return 0
if args.command == "import-legacy-scores":
from photo_pipeline.db import create_db_engine, create_session_factory
from photo_pipeline.services.legacy_import import LegacyImportService, write_report
migrate_with_backup(config)
engine = create_db_engine(config.database_url)
service = LegacyImportService(create_session_factory(engine))
report = service.import_nsfw_scores(
args.csv, overwrite=args.overwrite, dry_run=args.dry_run
)
# The report is the point: an import nobody can audit is not a migration.
if not args.dry_run:
write_report(report, config.data_dir)
print(json.dumps(report.counts, indent=2))
run_migrations(config.database_url)
return 0
if args.command == "worker":
@@ -273,56 +34,18 @@ def main(argv: Sequence[str] | None = None) -> int:
import photo_pipeline.jobs.domain_handlers # noqa: F401
from photo_pipeline.jobs.worker import Worker
lock = LibraryLock(config, "worker")
if (held := _acquire(lock, allow_legacy=args.allow_legacy)) is not None:
return held
try:
migrate_with_backup(config)
engine = create_db_engine(config.database_url)
Worker(
create_session_factory(engine), worker_id=args.id, config=config
).run_forever()
finally:
lock.release()
run_migrations(config.database_url)
engine = create_db_engine(config.database_url)
Worker(create_session_factory(engine), worker_id=args.id, config=config).run_forever()
return 0
import uvicorn
from photo_pipeline.api.app import ConfigurationRefused, create_app
from photo_pipeline.api.app import create_app
# An exposed deployment without an access secret must not reach the port at all,
# and the operator needs a sentence, not a traceback (US08-01).
try:
app = create_app(config)
except ConfigurationRefused as error:
print(str(error), file=sys.stderr)
return 4
lock = LibraryLock(config, "api")
if (held := _acquire(lock, allow_legacy=args.allow_legacy)) is not None:
return held
try:
uvicorn.run(app, host=config.host, port=config.port)
finally:
lock.release()
uvicorn.run(create_app(config), host=config.host, port=config.port)
return 0
def _acquire(lock: LibraryLock, *, allow_legacy: bool) -> int | None:
"""Take the lock, or explain on stderr why this process must not start.
Returns an exit code to return, or ``None`` when the lock was acquired.
"""
try:
lock.acquire(allow_legacy=allow_legacy)
except LockHeld as error:
print(str(error), file=sys.stderr)
return 2
except LegacyProcessActive as error:
print(f"{error} (override with --allow-legacy)", file=sys.stderr)
return 3
return None
if __name__ == "__main__":
raise SystemExit(main())

88
photo_pipeline/api/app.py
View File

@@ -8,106 +8,45 @@ and exposes the versioned ``/api/v1`` surface; US01-02 ships only health.
from __future__ import annotations
import logging
from contextlib import asynccontextmanager
from pathlib import Path
from fastapi import FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from fastapi import FastAPI
from fastapi.staticfiles import StaticFiles
from starlette.exceptions import HTTPException as StarletteHTTPException
from photo_pipeline.api.routes import (
albums,
analysis,
archives,
duplicates,
health,
inventory,
jobs,
library,
operations,
renames,
safety,
session as session_routes,
thumbnails,
uploads,
workflow,
)
from photo_pipeline.api.security import (
DEFAULT_HEADERS,
FailureLimiter,
SecurityMiddleware,
Session,
trust_refusal,
)
# Registers the safety_score / analysis job handlers on import.
import photo_pipeline.jobs.domain_handlers # noqa: F401
from photo_pipeline.config import Config
from photo_pipeline.db import create_db_engine, create_session_factory
from photo_pipeline.services.backup import migrate_with_backup
from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
from photo_pipeline.logging import configure_logging
from photo_pipeline.services.thumbnails import ThumbnailService
from photo_pipeline.services.upload_batches import UploadBatchService
FRONTEND_DIR = Path(__file__).resolve().parents[2] / "frontend"
DOCS_DIR = Path(__file__).resolve().parents[2] / "docs"
log = logging.getLogger(__name__)
def _envelope(status: int, code: str, message: str) -> JSONResponse:
return JSONResponse(
status_code=status,
content={"error": {"code": code, "message": message}},
headers=DEFAULT_HEADERS,
)
def _install_error_handlers(app: FastAPI) -> None:
"""One JSON error envelope everywhere, and nothing behind it.
An unhandled exception carries the library's absolute paths, SQL, and sometimes
a credential in its text; the client gets a code, the operator gets the traceback
in the server log (US07-02).
"""
@app.exception_handler(StarletteHTTPException)
async def _http_error(request: Request, exc: StarletteHTTPException):
return _envelope(exc.status_code, "http_error", str(exc.detail))
@app.exception_handler(RequestValidationError)
async def _validation_error(request: Request, exc: RequestValidationError):
# Field locations only: the echoed input can be the caller's own data, but it
# is also what ends up in shared logs and screenshots.
fields = sorted(".".join(str(part) for part in error["loc"]) for error in exc.errors())
return _envelope(422, "invalid_request", f"invalid request fields: {', '.join(fields)}")
@app.exception_handler(Exception)
async def _unhandled(request: Request, exc: Exception):
log.exception("unhandled error serving %s", request.url.path)
return _envelope(500, "internal_error", "internal error")
class ConfigurationRefused(RuntimeError):
"""The configuration would serve the library to callers it cannot authenticate."""
def create_app(config: Config | None = None) -> FastAPI:
config = config or Config.from_env()
configure_logging(config.log_level, config.log_format)
# Before anything is built, let alone bound to a port (US08-01).
if (why := trust_refusal(config)) is not None:
raise ConfigurationRefused(why)
@asynccontextmanager
async def lifespan(app: FastAPI):
config.database_path.parent.mkdir(parents=True, exist_ok=True)
# A schema upgrade is snapshotted first, so a migration that fails halfway
# leaves a restorable database behind rather than a damaged one (US07-05).
migrate_with_backup(config)
run_migrations(config.database_url)
engine = create_db_engine(config.database_url)
app.state.config = config
app.state.engine = engine
@@ -115,9 +54,6 @@ def create_app(config: Config | None = None) -> FastAPI:
# An upload whose process died left no outcome behind; resolve it now so the
# uploader lane is free and the uncertain batch is visible (US05-02).
UploadBatchService(app.state.session_factory, config=config).recover()
# A render killed mid-write leaves its temporary beside the cache entry;
# remove those recognized leftovers, and only those (US07-03).
ThumbnailService(app.state.session_factory, config).cleanup_temp_files()
try:
yield
finally:
@@ -125,16 +61,6 @@ def create_app(config: Config | None = None) -> FastAPI:
app.state.engine = None
app = FastAPI(title="Photo Pipeline", version="0.1.0", lifespan=lifespan)
# One session per process: the browser exchanges it for a cookie + CSRF token,
# and every other origin is refused before a route ever runs (US07-02).
app.state.session = Session.create()
app.state.access_limiter = FailureLimiter()
# Also set in the lifespan, but the bootstrap route reads it, and a caller can
# arrive before anything else has touched app.state.
app.state.config = config
app.add_middleware(SecurityMiddleware, session=app.state.session, config=config)
_install_error_handlers(app)
app.include_router(session_routes.router, prefix="/api/v1")
app.include_router(health.router, prefix="/api/v1")
app.include_router(inventory.router, prefix="/api/v1")
app.include_router(duplicates.router, prefix="/api/v1")
@@ -147,15 +73,7 @@ def create_app(config: Config | None = None) -> FastAPI:
app.include_router(albums.router, prefix="/api/v1")
app.include_router(renames.router, prefix="/api/v1")
app.include_router(uploads.router, prefix="/api/v1")
app.include_router(archives.router, prefix="/api/v1")
app.include_router(operations.router, prefix="/api/v1")
# Static single-page app (hash-routed). Mounted last so /api/v1 wins.
if FRONTEND_DIR.is_dir():
app.mount("/app", StaticFiles(directory=FRONTEND_DIR, html=True), name="app")
# The manuals, as their own markdown files (US09-01). Unauthenticated for the
# same reason the application shell is: whoever cannot get past the access
# secret is exactly who needs the troubleshooting page, and no document holds
# anything a session would protect.
if DOCS_DIR.is_dir():
app.mount("/docs", StaticFiles(directory=DOCS_DIR), name="docs")
return app

5
photo_pipeline/api/routes/analysis.py
View File

@@ -18,10 +18,7 @@ router = APIRouter(tags=["analysis"])
def _service(request: Request) -> AnalysisService:
return AnalysisService(
request.app.state.session_factory,
library_roots=tuple(request.app.state.config.library_roots),
)
return AnalysisService(request.app.state.session_factory)
def _error(status: int, code: str, message: str) -> JSONResponse:

234
photo_pipeline/api/routes/archives.py
View File

@@ -1,234 +0,0 @@
"""Archive location, preflight, plan, and restore API (US06-01, US06-02, US06-04).
Registering a location writes a marker onto the medium; preflight is a command
rather than a read, because it probes the destination, hashes the scope, and issues
the token an archive plan must present. Creating a plan writes only database rows —
the transfer itself runs on the durable ``archive`` lane, never in the request
thread, because it removes originals.
"""
from __future__ import annotations
from fastapi import APIRouter, Request
from fastapi.responses import JSONResponse
from pydantic import BaseModel
from photo_pipeline.jobs.domain_handlers import ARCHIVE_LOCK, ARCHIVE_PLAN, RESTORE_PLAN
from photo_pipeline.services.archives import ArchiveError, ArchiveService
from photo_pipeline.services.archive_transfer import ArchiveTransferService
from photo_pipeline.services.jobs import JobBlocked, JobService
from photo_pipeline.services.restores import RestoreService
router = APIRouter(tags=["archives"])
# Which failures are the caller's request (422), a missing thing (404), or state
# that changed under the caller (409).
NOT_FOUND_CODES = {"unknown_location", "unknown_plan"}
CONFLICT_CODES = {"stale_token", "stale_plan", "archive_pending"}
class RegisterLocationRequest(BaseModel):
name: str
root: str
class PreflightRequest(BaseModel):
location_id: str
# ``None`` means every album; an explicit list scopes the check.
albums: list[str] | None = None
class CreatePlanRequest(PreflightRequest):
# The token of the preflight the user approved; a stale one is refused.
token: str
class RestoreRequest(BaseModel):
location_id: str
# ``None`` means every asset archived at this location.
asset_ids: list[str] | None = None
class CreateRestoreRequest(RestoreRequest):
token: str
def _service(request: Request) -> ArchiveService:
return ArchiveService(request.app.state.session_factory, config=request.app.state.config)
def _restores(request: Request) -> RestoreService:
return RestoreService(request.app.state.session_factory, config=request.app.state.config)
def _transfers(request: Request) -> ArchiveTransferService:
return ArchiveTransferService(
request.app.state.session_factory, config=request.app.state.config
)
def _error(error: ArchiveError) -> JSONResponse:
if error.code in NOT_FOUND_CODES:
status = 404
elif error.code in CONFLICT_CODES:
status = 409
else:
status = 422
return JSONResponse(
status_code=status, content={"error": {"code": error.code, "message": str(error)}}
)
@router.post("/archive-locations", status_code=201)
def register_location(body: RegisterLocationRequest, request: Request):
try:
return _service(request).register(body.name, body.root)
except ArchiveError as error:
return _error(error)
@router.get("/archive-locations")
def list_locations(request: Request) -> dict:
return {"locations": _service(request).locations()}
@router.post("/archive-preflight")
def preflight(body: PreflightRequest, request: Request):
try:
return _service(request).preflight(body.location_id, body.albums)
except ArchiveError as error:
return _error(error)
@router.post("/archive-plans", status_code=201)
def create_plan(body: CreatePlanRequest, request: Request):
"""Turn an approved preflight into a durable, journaled plan. Nothing moves."""
try:
return _transfers(request).create(body.location_id, body.albums, token=body.token)
except ArchiveError as error:
return _error(error)
@router.get("/archive-plans")
def list_plans(request: Request) -> dict:
return {"plans": _transfers(request).list()}
@router.get("/archive-plans/{plan_id}")
def get_plan(plan_id: str, request: Request):
plan = _transfers(request).get(plan_id)
if plan is None:
return _error(ArchiveError("unknown_plan", f"unknown archive plan {plan_id}"))
return plan
@router.post("/archive-plans/{plan_id}/apply")
def apply_plan(plan_id: str, request: Request):
"""Queue the transfer on the archiver lane. The worker removes the sources."""
service = _transfers(request)
plan = service.get(plan_id)
if plan is None:
return _error(ArchiveError("unknown_plan", f"unknown archive plan {plan_id}"))
if service.journal.blocks_mutation():
unresolved = [row for row in service.journal.incomplete() if row["plan_id"] != plan_id]
if unresolved:
return _error(
ArchiveError(
"archive_pending",
f"an unresolved archive operation ({unresolved[0]['id']}) must be recovered",
)
)
try:
job = JobService(request.app.state.session_factory).enqueue(
ARCHIVE_PLAN,
lock=ARCHIVE_LOCK,
# One queued attempt per plan version: a double-clicked apply reuses it.
idempotency_key=f"archive:{plan_id}:{plan['version']}",
items=[plan_id],
)
except JobBlocked as error:
return JSONResponse(
status_code=409, content={"error": {"code": error.code, "message": str(error)}}
)
return {"plan_id": plan_id, "job": job}
@router.post("/restore-preflight")
def restore_preflight(body: RestoreRequest, request: Request):
"""Validate restoring archived assets back into the library. Nothing moves."""
try:
return _restores(request).preflight(body.location_id, body.asset_ids)
except ArchiveError as error:
return _error(error)
@router.post("/restore-plans", status_code=201)
def create_restore_plan(body: CreateRestoreRequest, request: Request):
try:
return _restores(request).create(body.location_id, body.asset_ids, token=body.token)
except ArchiveError as error:
return _error(error)
@router.get("/restore-plans")
def list_restore_plans(request: Request) -> dict:
return {"plans": _restores(request).list()}
@router.get("/restore-plans/{plan_id}")
def get_restore_plan(plan_id: str, request: Request):
plan = _restores(request).get(plan_id)
if plan is None:
return _error(ArchiveError("unknown_plan", f"unknown restore plan {plan_id}"))
return plan
@router.post("/restore-plans/{plan_id}/apply")
def apply_restore_plan(plan_id: str, request: Request):
"""Queue the restore on the archiver lane — the same single lane as archiving,
because both move the same originals."""
service = _restores(request)
plan = service.get(plan_id)
if plan is None:
return _error(ArchiveError("unknown_plan", f"unknown restore plan {plan_id}"))
unresolved = [row for row in service.journal.incomplete() if row["plan_id"] != plan_id]
if unresolved:
return _error(
ArchiveError(
"archive_pending",
f"an unresolved archive operation ({unresolved[0]['id']}) must be recovered",
)
)
try:
job = JobService(request.app.state.session_factory).enqueue(
RESTORE_PLAN,
lock=ARCHIVE_LOCK,
idempotency_key=f"restore:{plan_id}:{plan['version']}",
items=[plan_id],
)
except JobBlocked as error:
return JSONResponse(
status_code=409, content={"error": {"code": error.code, "message": str(error)}}
)
return {"plan_id": plan_id, "job": job}
@router.get("/restore-recovery")
def restore_recovery_status(request: Request) -> dict:
return _restores(request).recovery_status()
@router.post("/restore-recovery/resolve")
def resolve_restore_recovery(request: Request) -> dict:
return _restores(request).recover()
@router.get("/archive-recovery")
def recovery_status(request: Request) -> dict:
"""What an interrupted transfer left behind, straight from journal + disk."""
return _transfers(request).recovery_status()
@router.post("/archive-recovery/resolve")
def resolve_recovery(request: Request) -> dict:
return _transfers(request).recover()

17
photo_pipeline/api/routes/duplicates.py
View File

@@ -11,13 +11,7 @@ from fastapi import APIRouter, Query, Request
from fastapi.responses import JSONResponse
from photo_pipeline.schemas import DecisionRequest
from photo_pipeline.services.duplicates import (
MAX_MEMBER_PAGE,
MEMBER_PAGE,
ConflictError,
DuplicateError,
DuplicateService,
)
from photo_pipeline.services.duplicates import ConflictError, DuplicateError, DuplicateService
router = APIRouter(tags=["duplicates"])
@@ -48,13 +42,8 @@ def list_clusters(
@router.get("/duplicates/clusters/{cluster_id}")
def get_cluster(
cluster_id: str,
request: Request,
limit: int = Query(MEMBER_PAGE, ge=1, le=MAX_MEMBER_PAGE),
offset: int = Query(0, ge=0),
):
detail = _service(request).get_cluster(cluster_id, limit=limit, offset=offset)
def get_cluster(cluster_id: str, request: Request):
detail = _service(request).get_cluster(cluster_id)
if detail is None:
return _error(404, "not_found", f"unknown cluster {cluster_id}")
return detail

70
photo_pipeline/api/routes/operations.py
View File

@@ -1,70 +0,0 @@
"""Operational endpoints: diagnostics and backups (US07-05).
Backups can be taken and verified here because both are safe, additive, and the
operator needs them from the same screen that shows the disk filling up.
**Restore is deliberately not an endpoint.** It replaces the state of the running
application with an older one, so it belongs to a stopped installation and a person
at a terminal: ``python -m photo_pipeline restore``. An HTTP call that can silently
roll the library back to last week is a hole, not a feature.
"""
from __future__ import annotations
from fastapi import APIRouter, Query, Request
from fastapi.responses import JSONResponse
from pydantic import BaseModel
from photo_pipeline.services import diagnostics
from photo_pipeline.services.backup import DEFAULT_KEEP, BackupError, BackupService
router = APIRouter(tags=["operations"])
class CreateBackupRequest(BaseModel):
reason: str = "manual"
keep: int = DEFAULT_KEEP
def _service(request: Request) -> BackupService:
return BackupService(request.app.state.config)
def _error(status: int, code: str, message: str) -> JSONResponse:
return JSONResponse(status_code=status, content={"error": {"code": code, "message": message}})
@router.get("/diagnostics")
def read_diagnostics(request: Request) -> dict:
return diagnostics.report(request.app.state.config)
@router.get("/backups")
def list_backups(request: Request) -> dict:
return {"backups": _service(request).list()}
@router.post("/backups", status_code=201)
def create_backup(body: CreateBackupRequest, request: Request):
try:
return _service(request).create(reason=body.reason, keep=body.keep)
except BackupError as error:
return _error(422, "backup_failed", str(error))
@router.get("/backups/{name}/verify")
def verify_backup(name: str, request: Request):
service = _service(request)
# The name comes from the browser, so it names a backup — it is never joined
# into a path until it has been matched against one that exists (US07-02).
if name not in {entry["name"] for entry in service.list()}:
return _error(404, "not_found", f"unknown backup {name}")
return {"name": name, **service.verify(service.root / name).as_dict()}
@router.post("/backups/prune")
def prune_backups(request: Request, keep: int = Query(DEFAULT_KEEP, ge=1)):
try:
return {"removed": _service(request).prune(keep=keep)}
except BackupError as error:
return _error(422, "invalid_retention", str(error))

68
photo_pipeline/api/routes/session.py
View File

@@ -1,68 +0,0 @@
"""Session bootstrap: the one endpoint reachable without a session (US07-02).
It sets the ``HttpOnly``/``SameSite=Strict`` session cookie and returns the CSRF
token in the body. A foreign page can call this — it just cannot read the answer,
because the app sends no CORS headers — and the cookie it received is never attached
to a request that foreign page initiated.
When an access secret is configured (mandatory as soon as the app is reachable from
another machine, US08-01) this is also the authentication gate: the secret buys the
cookie, and every route behind it keeps asking for exactly the session and CSRF token
it asked for before. Wrong secrets are counted, and a burst of them stops being
answered — otherwise a proxy-exposed deployment could be guessed at indefinitely.
"""
from __future__ import annotations
import logging
import secrets
from fastapi import APIRouter, Request
from fastapi.responses import JSONResponse
from photo_pipeline.api.security import ACCESS_SECRET_HEADER, SESSION_COOKIE
router = APIRouter(tags=["session"])
log = logging.getLogger(__name__)
def _refuse(status: int, code: str, message: str) -> JSONResponse:
return JSONResponse(status_code=status, content={"error": {"code": code, "message": message}})
@router.get("/session")
def start_session(request: Request) -> JSONResponse:
config = request.app.state.config
secret = config.access_secret
if secret is not None:
limiter = request.app.state.access_limiter
if limiter.blocked():
return _refuse(429, "too_many_attempts", "too many failed attempts; retry later")
offered = request.headers.get(ACCESS_SECRET_HEADER, "")
if not secrets.compare_digest(offered, secret.get_secret_value()):
limiter.record_failure()
# The client address is the whole record: the offered secret, the issued
# session, and the request body all stay out of the log.
log.warning(
"access secret rejected", extra={"client": _client(request), "path": "/session"}
)
return _refuse(401, "access_denied", "a valid access secret is required")
session = request.app.state.session
response = JSONResponse({"csrf_token": session.csrf_token})
response.set_cookie(
SESSION_COOKIE,
session.id,
httponly=True,
samesite="strict",
# HTTPS outside means the cookie must never travel over a plain hop, even one
# this process cannot see. Loopback http keeps working unchanged.
secure=request.scope.get("state", {}).get("external_scheme") == "https",
path="/",
)
return response
def _client(request: Request) -> str:
return request.client.host if request.client else "unknown"

4
photo_pipeline/api/routes/thumbnails.py
View File

@@ -31,7 +31,5 @@ def get_thumbnail(asset_id: str, request: Request, size: int = Query(512)):
return FileResponse(
path,
media_type="image/webp",
# private: the URL is versioned and immutable, but these bytes are the user's
# photos and must never sit in a shared cache (US07-02).
headers={"Cache-Control": "private, max-age=31536000, immutable"},
headers={"Cache-Control": "public, max-age=31536000, immutable"},
)

330
photo_pipeline/api/security.py
View File

@@ -1,330 +0,0 @@
"""Local-web attack surface: session, CSRF, Origin/Host checks, default headers.
The app binds to 127.0.0.1, so the attacker is not a remote client but another page
in the user's browser (concept §15, "Local web attack"): any site can issue requests
to ``http://127.0.0.1:8000`` and can embed ``<img src=...>`` against media endpoints.
The defenses stack, because each one alone has a hole:
* **Host** must be a loopback name — a DNS rebinding host that resolves to 127.0.0.1
passes the browser's origin rules but not this check.
* **Origin**, when the browser sends one, must be this exact origin (scheme, host,
port). There is no CORS middleware at all, so a foreign page can never *read* a
response even if it manages to send a request.
* **Sec-Fetch-Site** rejects cross-site loads that carry no Origin, which is what an
``<img>`` or ``<script>`` against a media endpoint looks like.
* A **session cookie** (``SameSite=Strict``, ``HttpOnly``) is required by every
``/api/v1`` route except liveness/readiness and the bootstrap itself. Strict means
the browser never attaches it to a request another site initiated.
* A **CSRF token** must be echoed in a header on every mutation. It is handed out
only in the bootstrap response body, which a foreign page cannot read (no CORS) —
so possessing it proves the caller is same-origin.
Behind a reverse proxy (US08-01) the same stack holds with two substitutions: the
allowed host set comes from configuration instead of being the loopback names, and
the host/scheme the policy judges is the *external* one, which is only read from
``X-Forwarded-*`` when the request actually arrived from a configured proxy. The
loopback check was standing in for authentication, so naming a non-loopback host
also makes an access secret mandatory — ``trust_refusal`` refuses to start without
one, and the secret is what the bootstrap endpoint trades for the session cookie.
``evaluate`` is a pure function over the request metadata: the whole policy is one
table that a unit test can enumerate, and the middleware only applies its verdict.
"""
from __future__ import annotations
import secrets
import time
from collections.abc import Mapping
from dataclasses import dataclass
from urllib.parse import urlsplit
from starlette.datastructures import Headers, MutableHeaders
from starlette.responses import JSONResponse
SESSION_COOKIE = "pp_session"
CSRF_HEADER = "x-csrf-token"
ACCESS_SECRET_HEADER = "x-access-secret"
API_PREFIX = "/api/v1"
SAFE_METHODS = frozenset({"GET", "HEAD", "OPTIONS"})
# Reachable without a session: liveness/readiness (an orchestrator has no cookie)
# and the bootstrap that issues the session in the first place.
PUBLIC_PATHS = frozenset(
{f"{API_PREFIX}/health/live", f"{API_PREFIX}/health/ready", f"{API_PREFIX}/session"}
)
LOOPBACK_HOSTS = frozenset({"127.0.0.1", "localhost", "::1", "[::1]"})
# Mutating endpoints that must stay reachable while mutation itself is gated: the
# backup a careful operator takes first, and its retention (US07-07).
MUTATION_EXEMPT_PATHS = frozenset({f"{API_PREFIX}/backups", f"{API_PREFIX}/backups/prune"})
# Applied to every response. `frame-ancestors 'none'` and CORP keep other pages from
# embedding the app or its thumbnails.
#
# `script-src 'self'` is the boundary that matters and it is unchanged: no
# `'unsafe-eval'`, no `'unsafe-inline'`, so nothing injected into the DOM can execute.
# Whether the vendored diagram renderer needed `'unsafe-eval'` was measured rather
# than assumed — mermaid's bundle contains no `eval(` and no `new Function`, and it
# renders under this exact policy without a single script-src violation.
#
# `style-src` does gain `'unsafe-inline'` (US09-01): mermaid styles the SVG it builds
# with an injected `<style>` element and `style=` attributes, and a diagram's CSS
# cannot be hashed in advance. The concession is bounded by the directives around it
# — with script execution still refused and `img-src`, `connect-src`, and `font-src`
# all `'self'`, the CSS exfiltration channels stay closed and what is left is
# defacement of a page its own operator is already looking at.
DEFAULT_HEADERS = {
"x-content-type-options": "nosniff",
"x-frame-options": "DENY",
"referrer-policy": "no-referrer",
"cross-origin-resource-policy": "same-origin",
"cross-origin-opener-policy": "same-origin",
"content-security-policy": (
"default-src 'self'; img-src 'self' data:; style-src 'self' 'unsafe-inline'; "
"script-src 'self'; connect-src 'self'; font-src 'self'; "
"frame-ancestors 'none'; base-uri 'none'; form-action 'none'"
),
}
@dataclass(frozen=True)
class Session:
"""One process, one session. A local app has exactly one user; a session store
would be bookkeeping without a second subject to distinguish.
ponytail: per-session rows if the app ever serves more than one operator.
"""
id: str
csrf_token: str
@classmethod
def create(cls) -> Session:
return cls(secrets.token_urlsafe(32), secrets.token_urlsafe(32))
@dataclass(frozen=True)
class Refusal:
status: int
code: str
message: str
def split_host(value: str) -> tuple[str, str]:
"""``"127.0.0.1:8000"`` -> ``("127.0.0.1", "8000")``; bracketed IPv6 aware."""
value = value.strip()
if value.startswith("["):
host, _, port = value.partition("]")
return host + "]", port.lstrip(":")
host, _, port = value.partition(":")
return host, port
def external_view(
*,
client: str | None,
headers: Mapping[str, str],
scheme: str,
trusted_proxies: frozenset[str],
) -> tuple[str, str]:
"""The ``(scheme, host)`` the caller used, as opposed to the one this hop saw.
Forwarded headers are a client-supplied claim. Believing them from anyone lets a
request declare its own origin — and origin is half of this module's evidence —
so they count only when the connection came from a configured proxy.
"""
host = headers.get("host", "")
if client is None or client not in trusted_proxies:
return scheme, host
# A chain appends: the first entry is what the original client asked for.
forwarded_proto = headers.get("x-forwarded-proto", "").split(",")[0].strip().lower()
forwarded_host = headers.get("x-forwarded-host", "").split(",")[0].strip()
return forwarded_proto or scheme, forwarded_host or host
def evaluate(
*,
method: str,
path: str,
headers: Mapping[str, str],
session: Session,
allowed_hosts: frozenset[str] = LOOPBACK_HOSTS,
scheme: str = "http",
max_request_bytes: int,
) -> Refusal | None:
"""Why this request must be refused, or ``None`` when it may proceed.
``headers["host"]`` and ``scheme`` are the external ones (see ``external_view``);
the allowed origins are the allowed hosts under that scheme and port, so there is
no second list that can drift away from the first.
"""
host_header = headers.get("host", "")
host, port = split_host(host_header)
if host.lower() not in allowed_hosts:
return Refusal(403, "host_not_allowed", "request host is not an allowed address")
origin = headers.get("origin")
if origin is not None and origin != "":
parts = urlsplit(origin)
origin_host, origin_port = split_host(parts.netloc)
if (
parts.scheme != scheme
or origin_host.lower() not in allowed_hosts
or origin_port != port
):
return Refusal(403, "origin_not_allowed", "request origin is not this application")
# Absent means a non-browser client; "none" is a user-initiated navigation.
fetch_site = headers.get("sec-fetch-site")
if fetch_site is not None and fetch_site not in ("same-origin", "none"):
return Refusal(403, "cross_site_blocked", "cross-site requests are not accepted")
length = headers.get("content-length")
if length and length.isdigit() and int(length) > max_request_bytes:
return Refusal(413, "payload_too_large", "request body exceeds the configured limit")
protected = path.startswith(API_PREFIX) and path not in PUBLIC_PATHS
if not protected:
return None
if headers.get("cookie-session") != session.id:
return Refusal(401, "unauthenticated", "a valid application session is required")
if method.upper() not in SAFE_METHODS and headers.get(CSRF_HEADER) != session.csrf_token:
return Refusal(403, "csrf_failed", "missing or invalid CSRF token")
return None
def exposed_hosts(config) -> list[str]:
"""Configured names by which this application is reachable from another machine."""
names = {str(config.host).lower()}
names.update(split_host(name)[0].lower() for name in config.allowed_hosts)
return sorted(names - LOOPBACK_HOSTS)
def trust_refusal(config) -> str | None:
"""Why this configuration must not serve at all, or ``None``.
Reaching the app used to prove ownership of it. The moment a configuration makes
it reachable from elsewhere that stops being true, so serving without a secret
would publish the library — refuse at startup rather than at the first request,
when the operator is no longer watching (US08-01).
"""
exposed = exposed_hosts(config)
if exposed and config.access_secret is None:
return (
f"refusing to serve: {', '.join(exposed)} is reachable from outside this "
"machine, so PHOTO_PIPELINE_ACCESS_SECRET must be set"
)
return None
class FailureLimiter:
"""Bounded failed access-secret attempts, so the secret cannot be guessed online.
ponytail: one counter for the whole process rather than per client address —
behind a proxy every attempt arrives from the same address anyway. Per-caller
buckets if the app is ever exposed without one.
"""
def __init__(self, limit: int = 5, window: float = 60.0) -> None:
self.limit = limit
self.window = window
self._failures: list[float] = []
def blocked(self) -> bool:
now = time.monotonic()
self._failures = [at for at in self._failures if now - at < self.window]
return len(self._failures) >= self.limit
def record_failure(self) -> None:
self._failures.append(time.monotonic())
class SecurityMiddleware:
"""Pure-ASGI so the SSE stream keeps streaming (BaseHTTPMiddleware buffers)."""
def __init__(self, app, *, session: Session, config) -> None:
self.app = app
self.session = session
self.config = config
self.max_request_bytes = config.max_request_bytes
self.allowed_hosts = frozenset(
LOOPBACK_HOSTS
| {str(config.host).lower()}
| {split_host(name)[0].lower() for name in config.allowed_hosts}
)
self.trusted_proxies = frozenset(config.trusted_proxies)
async def __call__(self, scope, receive, send) -> None:
if scope["type"] != "http":
await self.app(scope, receive, send)
return
headers = Headers(scope=scope)
# The cookie is read here and handed to the pure policy as one value, so the
# policy never has to parse a Cookie header.
lookup = dict(headers)
lookup["cookie-session"] = _cookie(headers.get("cookie", ""), SESSION_COOKIE)
client = scope.get("client")
scheme, lookup["host"] = external_view(
client=client[0] if client else None,
headers=headers,
scheme=scope.get("scheme", "http"),
trusted_proxies=self.trusted_proxies,
)
# What the session cookie's Secure flag is decided from, one hop later.
scope.setdefault("state", {})["external_scheme"] = scheme
refusal = evaluate(
method=scope.get("method", "GET"),
path=scope.get("path", "/"),
headers=lookup,
session=self.session,
allowed_hosts=self.allowed_hosts,
scheme=scheme,
max_request_bytes=self.max_request_bytes,
)
if refusal is None:
refusal = self._mutation_refusal(scope)
if refusal is not None:
response = JSONResponse(
status_code=refusal.status,
content={"error": {"code": refusal.code, "message": refusal.message}},
headers=DEFAULT_HEADERS,
)
await response(scope, receive, send)
return
async def send_with_headers(message):
if message["type"] == "http.response.start":
out = MutableHeaders(scope=message)
for name, value in DEFAULT_HEADERS.items():
out.setdefault(name, value)
await send(message)
await self.app(scope, receive, send_with_headers)
def _mutation_refusal(self, scope) -> Refusal | None:
"""Refuse every mutating request while the library's dry run is unapproved.
One choke point for the whole API: every mutation the browser can start is a
non-safe method under ``/api/v1``. Reading stays open — an operator has to be
able to look at what the application found in order to approve it (US07-07).
"""
method = scope.get("method", "GET").upper()
path = scope.get("path", "/")
if method in SAFE_METHODS or not path.startswith(API_PREFIX):
return None
if path in MUTATION_EXEMPT_PATHS:
return None
from photo_pipeline.services.release import mutation_blockers
blockers = mutation_blockers(self.config)
if not blockers:
return None
return Refusal(403, blockers[0]["code"], blockers[0]["message"])
def _cookie(header: str, name: str) -> str:
for part in header.split(";"):
key, _, value = part.strip().partition("=")
if key == name:
return value
return ""

95
photo_pipeline/config.py
View File

@@ -7,9 +7,6 @@ real external call needs them.
pydantic-settings would do this too, but a prefix-scan over the declared fields
is a few lines and one fewer dependency.
Tuple-valued settings are lists in one variable: library roots are ``os.pathsep``
separated because they are paths, everything else is comma separated.
"""
from __future__ import annotations
@@ -21,62 +18,6 @@ from typing import Mapping
from pydantic import BaseModel, ConfigDict, SecretStr
ENV_PREFIX = "PHOTO_PIPELINE_"
ENV_FILE_VAR = f"{ENV_PREFIX}ENV_FILE"
DEFAULT_ENV_FILE = Path(".env")
COMMA_LIST_FIELDS = frozenset({"allowed_hosts", "trusted_proxies"})
# The archived CLI's variable names, so the configuration file an operator already
# has keeps working. The vision provider reads the OpenAI SDK's names, and the
# library root is configuration here rather than a bare path (US07-01 donor).
LEGACY_ALIASES = {
"LLM_API_KEY": "OPENAI_API_KEY",
"GEMINI_API_KEY": "OPENAI_API_KEY",
"LLM_BASE_URL": "OPENAI_BASE_URL",
"LIBRARY": f"{ENV_PREFIX}LIBRARY_ROOTS",
}
def parse_env_file(text: str) -> dict[str, str]:
"""``KEY=value`` lines into a mapping. Comments, blanks, and quotes handled.
Deliberately not a shell: no interpolation, no ``export``, no multi-line values.
A configuration file that can run code is a configuration file that can be a
vulnerability.
"""
values: dict[str, str] = {}
for line in text.splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, _, raw = line.partition("=")
key = key.strip()
if not key or key.startswith("#"):
continue
value = raw.strip().strip('"').strip("'")
values[key] = value
alias = LEGACY_ALIASES.get(key)
if alias:
values.setdefault(alias, value)
return values
def load_env_file(path: Path | str | None = None) -> dict[str, str]:
"""Load ``PHOTO_PIPELINE_ENV_FILE`` (or ``./.env``) into the environment.
Anything already exported wins: a file is the standing configuration, the shell
is what you meant *this time*. Returns what it applied, which is what the CLI
prints — names only, never values.
"""
candidate = path or os.environ.get(ENV_FILE_VAR) or DEFAULT_ENV_FILE
candidate = Path(candidate)
if not candidate.is_file():
return {}
applied = {}
for key, value in parse_env_file(candidate.read_text()).items():
if key not in os.environ:
os.environ[key] = value
applied[key] = value
return applied
class Config(BaseModel):
@@ -89,37 +30,11 @@ class Config(BaseModel):
log_level: str = "INFO"
log_format: str = "json" # "json" or "text"
# Trust boundary (US08-01). Empty means loopback only, which is what the app did
# before there was a setting: a request whose Host is not a loopback name is
# refused, and no secret is needed because nothing outside this machine can call.
# Naming a real hostname here is what makes the app reachable through a reverse
# proxy, and it is exactly then that ``access_secret`` becomes mandatory.
allowed_hosts: tuple[str, ...] = ()
# Addresses whose ``X-Forwarded-Proto``/``X-Forwarded-Host`` may be believed. A
# client that is not the proxy can otherwise declare its own origin.
trusted_proxies: tuple[str, ...] = ()
# Exchanged for the session cookie at the bootstrap endpoint. Once set it is
# required even on loopback, so a development setup cannot half-enable it.
access_secret: SecretStr | None = None
# Largest request body the API accepts. Every endpoint takes small JSON commands;
# anything larger is a mistake or an attempt to exhaust memory (US07-02).
max_request_bytes: int = 1_048_576
# Library boundary for path validation (os.pathsep-separated in the env var).
library_roots: tuple[Path, ...] = ()
thumbnail_cache_quota_bytes: int = 500_000_000
thumbnail_max_pixels: int = 100_000_000
# Free space an archive destination must keep beyond the transfer itself.
archive_free_space_reserve_bytes: int = 1_000_000_000
# Refuse every mutating request until a read-only dry run of the configured
# library has been produced and explicitly approved (US07-07). Off by default so
# a development setup is unchanged; turn it on before pointing the application at
# a library whose photos cannot be replaced.
require_dry_run_approval: bool = False
vision_api_key: SecretStr | None = None
immich_api_key: SecretStr | None = None
immich_server_url: str = ""
@@ -140,18 +55,10 @@ class Config(BaseModel):
@classmethod
def from_env(cls, environ: Mapping[str, str] | None = None) -> "Config":
env = os.environ if environ is None else environ
if environ is None:
load_env_file() # a file never overrides what the shell already set
env = os.environ
data: dict = {}
for name in cls.model_fields:
raw = env.get(ENV_PREFIX + name.upper())
if not raw:
continue
if name == "library_roots":
data[name] = raw.split(os.pathsep)
elif name in COMMA_LIST_FIELDS:
data[name] = [part.strip() for part in raw.split(",") if part.strip()]
else:
data[name] = raw
data[name] = raw.split(os.pathsep) if name == "library_roots" else raw
return cls(**data)

37
photo_pipeline/db.py
View File

@@ -41,41 +41,12 @@ def create_session_factory(engine: Engine) -> sessionmaker:
return sessionmaker(bind=engine, expire_on_commit=False, future=True)
def _alembic_config(url: str):
def run_migrations(url: str) -> None:
"""Upgrade the database at ``url`` to the latest revision."""
from alembic import command
from alembic.config import Config as AlembicConfig
cfg = AlembicConfig(str(_REPO_ROOT / "alembic.ini"))
cfg.set_main_option("script_location", str(_REPO_ROOT / "migrations"))
cfg.set_main_option("sqlalchemy.url", url)
return cfg
def run_migrations(url: str) -> None:
"""Upgrade the database at ``url`` to the latest revision."""
from alembic import command
command.upgrade(_alembic_config(url), "head")
def head_revision() -> str | None:
"""The revision this code expects. ``None`` if the scripts cannot be read."""
from alembic.script import ScriptDirectory
try:
return ScriptDirectory.from_config(_alembic_config("sqlite://")).get_current_head()
except Exception:
return None
def current_revision(url: str) -> str | None:
"""The revision a database is actually at, or ``None`` for an unstamped one."""
engine = create_db_engine(url)
try:
with engine.connect() as connection:
from alembic.runtime.migration import MigrationContext
return MigrationContext.configure(connection).get_current_revision()
except Exception:
return None
finally:
engine.dispose()
command.upgrade(cfg, "head")

Some files were not shown because too many files have changed in this diff Show More