Compare commits
1 Commits
us/US09-04
...
us/US06-02
| Author | SHA1 | Date | |
|---|---|---|---|
| 9503fd1cfc |
@@ -1,23 +0,0 @@
|
||||
# Deny-by-default build context (US08-02): the image must contain no secrets, no
|
||||
# photos, no database, no logs, and no .git. An allow list is the only version of this
|
||||
# rule that stays true when a new file appears in the working copy.
|
||||
*
|
||||
|
||||
!pyproject.toml
|
||||
!alembic.ini
|
||||
!README.md
|
||||
!photo_pipeline
|
||||
!migrations
|
||||
!frontend
|
||||
!docs
|
||||
!docker
|
||||
|
||||
# Nothing generated, even under an allowed directory.
|
||||
**/__pycache__
|
||||
**/*.py[cod]
|
||||
**/.DS_Store
|
||||
**/*.env
|
||||
**/*.log
|
||||
**/*.db
|
||||
**/*.db-*
|
||||
**/*.sqlite*
|
||||
85
.env.example
85
.env.example
@@ -1,85 +0,0 @@
|
||||
# Every setting the application and its composition read, with no values in it.
|
||||
# Copy to `.env`, fill in what you need, and keep that copy out of git (it is
|
||||
# gitignored, and the work-item safety checks refuse to stage it).
|
||||
#
|
||||
# cp .env.example .env
|
||||
#
|
||||
# Empty means "use the default noted beside it". Anything already exported in the
|
||||
# shell wins over this file, both for the app and for `docker compose`.
|
||||
|
||||
# ── the composition (host side; read by docker-compose.yml only) ─────────────
|
||||
# The photo library on this host. Bind-mounted at PHOTO_PIPELINE_LIBRARY_ROOTS.
|
||||
PHOTO_PIPELINE_LIBRARY_HOST_PATH=
|
||||
# Which image to run. Default: photo-pipeline:dev (what `up --build` builds).
|
||||
PHOTO_PIPELINE_IMAGE=
|
||||
# Must be the owner of the library above: what the containers rename and rewrite
|
||||
# keeps this ownership. Default: 1000 / 1000.
|
||||
PHOTO_PIPELINE_UID=
|
||||
PHOTO_PIPELINE_GID=
|
||||
# Host address the API port is published on. Default: 127.0.0.1. Anything else
|
||||
# exposes the app beyond this machine — then ALLOWED_HOSTS and ACCESS_SECRET below
|
||||
# are what stand in for the loopback boundary (US08-01).
|
||||
PHOTO_PIPELINE_PUBLISH_ADDRESS=
|
||||
# This file's own path, if it is not ./.env. Default: .env.
|
||||
PHOTO_PIPELINE_ENV_FILE=
|
||||
|
||||
# ── library and data ────────────────────────────────────────────────────────
|
||||
# os.pathsep-separated. In a container these are the *container-side* mount paths,
|
||||
# and `serve`/`worker` refuse to start when they are not mounted.
|
||||
PHOTO_PIPELINE_LIBRARY_ROOTS=
|
||||
# Database, WAL, thumbnail cache, journals, backups. Default: data (the container
|
||||
# sets /data, which is the persistent volume; a network mount is unsupported).
|
||||
PHOTO_PIPELINE_DATA_DIR=
|
||||
# Database file, if it should not live in the data directory. Default:
|
||||
# <data dir>/photo_pipeline.db.
|
||||
PHOTO_PIPELINE_DB_PATH=
|
||||
|
||||
# ── serving ─────────────────────────────────────────────────────────────────
|
||||
# Bind address. Default: 127.0.0.1. The composition sets 0.0.0.0 inside the
|
||||
# container and publishes to loopback on the host instead.
|
||||
PHOTO_PIPELINE_HOST=
|
||||
# Under the composition this is the *published* host port; the container serves
|
||||
# 8000. Default: 8000.
|
||||
PHOTO_PIPELINE_PORT=
|
||||
# Comma-separated hostnames the app answers to besides loopback. Empty means
|
||||
# loopback only. Naming one makes the access secret mandatory.
|
||||
PHOTO_PIPELINE_ALLOWED_HOSTS=
|
||||
# Traded for the session cookie at GET /api/v1/session via X-Access-Secret.
|
||||
# Required as soon as the app is reachable from anywhere but loopback. Generate
|
||||
# one with: python -c 'import secrets; print(secrets.token_urlsafe(32))'
|
||||
PHOTO_PIPELINE_ACCESS_SECRET=
|
||||
# Comma-separated peer addresses whose X-Forwarded-Proto/-Host may be believed.
|
||||
# Only the reverse proxy's address belongs here. Default: none.
|
||||
PHOTO_PIPELINE_TRUSTED_PROXIES=
|
||||
# Largest request body accepted, in bytes. Default: 1048576.
|
||||
PHOTO_PIPELINE_MAX_REQUEST_BYTES=
|
||||
|
||||
# ── logging ─────────────────────────────────────────────────────────────────
|
||||
# Default: INFO.
|
||||
PHOTO_PIPELINE_LOG_LEVEL=
|
||||
# json or text. Default: json.
|
||||
PHOTO_PIPELINE_LOG_FORMAT=
|
||||
|
||||
# ── limits ──────────────────────────────────────────────────────────────────
|
||||
# Thumbnail cache quota in bytes. Default: 500000000.
|
||||
PHOTO_PIPELINE_THUMBNAIL_CACHE_QUOTA_BYTES=
|
||||
# Refuse to decode images larger than this many pixels. Default: 100000000.
|
||||
PHOTO_PIPELINE_THUMBNAIL_MAX_PIXELS=
|
||||
# Free space an archive destination must keep beyond the transfer. Default:
|
||||
# 1000000000.
|
||||
PHOTO_PIPELINE_ARCHIVE_FREE_SPACE_RESERVE_BYTES=
|
||||
|
||||
# ── safety gate ─────────────────────────────────────────────────────────────
|
||||
# true refuses every mutating request until a read-only dry run of this library has
|
||||
# been produced and approved (US07-07). Default: false. Turn it on before pointing
|
||||
# the app at photos that cannot be replaced.
|
||||
PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL=
|
||||
|
||||
# ── external services ───────────────────────────────────────────────────────
|
||||
# Vision provider key for the analysis stage. Without it, analysis cannot run.
|
||||
PHOTO_PIPELINE_VISION_API_KEY=
|
||||
# Immich server and its API key, for the upload stage.
|
||||
PHOTO_PIPELINE_IMMICH_SERVER_URL=
|
||||
PHOTO_PIPELINE_IMMICH_API_KEY=
|
||||
# Uploader binary. Default: immich-go (on PATH; pinned inside the image).
|
||||
PHOTO_PIPELINE_IMMICH_GO_BINARY=
|
||||
9
.gitattributes
vendored
9
.gitattributes
vendored
@@ -1,9 +0,0 @@
|
||||
# Vendored third-party browser bundles (US09-01). Minified upstream files carry
|
||||
# trailing whitespace and very long lines; they are not ours to reformat, and the
|
||||
# submit gate's `git diff --check` would refuse them forever. Their integrity is
|
||||
# controlled where it belongs instead: a pinned version and a recorded sha256 in
|
||||
# frontend/js/vendor/VERSIONS.json, asserted by tests/integration/test_documentation.py.
|
||||
#
|
||||
# Only the whitespace check is turned off. They stay text, so a credential scan or a
|
||||
# search still reads them.
|
||||
frontend/js/vendor/*.js -whitespace linguist-vendored
|
||||
@@ -1,99 +0,0 @@
|
||||
# Publish and redeploy (US08-04). Adapted from the crowdsec-admin deploy workflow:
|
||||
# build, log in to the Gitea registry, push, trigger the Portainer webhook, prune.
|
||||
# The difference is the gate — this project has a required suite that must not be
|
||||
# skipped, so publishing happens only after `Test` (`.gitea/workflows/test.yml`)
|
||||
# succeeded on `main`, never on the push itself.
|
||||
#
|
||||
# The stack is managed by Portainer from git (`docker-compose.yml`), and the runtime
|
||||
# secrets it needs — vision key, Immich key, access secret — live in the Portainer
|
||||
# stack's environment. They are deliberately not repository secrets and are not in the
|
||||
# image: rotation stays in one place, and a repository read never discloses them.
|
||||
#
|
||||
# Repository secrets required:
|
||||
# REGISTRY_USER user with write:package on the registry
|
||||
# REGISTRY_TOKEN that user's token
|
||||
# PORTAINER_WEBHOOK_URL POST URL from the stack's auto-update setting
|
||||
|
||||
name: Deploy
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows:
|
||||
- Test
|
||||
types:
|
||||
- completed
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: Build and push a scratch tag only — leave `latest` and the running stack alone
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
concurrency:
|
||||
# Deliberately not keyed by commit: the point is that two deploys of *different*
|
||||
# commits cannot overlap. Queued, not cancelled — a half-pushed tag set is worse
|
||||
# than a late one.
|
||||
group: deploy
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
IMAGE: git.domverse-berlin.eu/domverse/photoanalyzer
|
||||
# The commit that was tested, not whatever `main` points at by the time this starts.
|
||||
SHA: ${{ gitea.event.workflow_run.head_sha || gitea.sha }}
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
# A completed `Test` run is not a passing one.
|
||||
if: >-
|
||||
(gitea.event_name == 'workflow_run' && gitea.event.workflow_run.conclusion == 'success')
|
||||
|| (gitea.event_name == 'workflow_dispatch' && inputs.dry_run == false)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ env.SHA }}
|
||||
|
||||
- name: Log in to the Gitea registry
|
||||
run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login git.domverse-berlin.eu -u "${{ secrets.REGISTRY_USER }}" --password-stdin
|
||||
|
||||
- name: Build and push
|
||||
# The commit tag is pushed first, so a `latest` that exists is always a tag
|
||||
# that also exists under its own commit — which is what makes a rollback a
|
||||
# tag change rather than a rebuild.
|
||||
run: |
|
||||
docker build -t "$IMAGE:$SHA" -t "$IMAGE:latest" .
|
||||
docker push "$IMAGE:$SHA"
|
||||
docker push "$IMAGE:latest"
|
||||
|
||||
- name: Trigger the Portainer redeploy
|
||||
# --fail turns an HTTP error into a non-zero exit: a redeploy that did not
|
||||
# happen must not read as a green deploy.
|
||||
run: curl -sS --fail -X POST "${{ secrets.PORTAINER_WEBHOOK_URL }}"
|
||||
|
||||
- name: Prune dangling images
|
||||
# Untagged layers only. Published tags are the rollback history; `-a` would
|
||||
# delete exactly the images this workflow exists to keep.
|
||||
run: docker image prune -f
|
||||
|
||||
dry-run:
|
||||
# Manual only, and the default: prove the image still builds and the registry
|
||||
# still accepts it without moving `latest` or touching the running stack.
|
||||
if: gitea.event_name == 'workflow_dispatch' && inputs.dry_run
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to the Gitea registry
|
||||
run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login git.domverse-berlin.eu -u "${{ secrets.REGISTRY_USER }}" --password-stdin
|
||||
|
||||
- name: Build and push a scratch tag
|
||||
run: |
|
||||
docker build -t "$IMAGE:scratch-$SHA" .
|
||||
docker push "$IMAGE:scratch-$SHA"
|
||||
|
||||
- name: Prune dangling images
|
||||
run: docker image prune -f
|
||||
@@ -1,103 +0,0 @@
|
||||
# The test gate. Deploy waits for this workflow by name (`.gitea/workflows/deploy.yml`
|
||||
# triggers on `workflow_run: [Test]`), so renaming it here without renaming it there
|
||||
# would leave `main` publishing without a suite. `tests/integration/test_deploy_workflows.py`
|
||||
# asserts both halves of that link, and that the commands below are still the ones
|
||||
# configured in `work_item/.work-item.yml`.
|
||||
|
||||
name: Test
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
concurrency:
|
||||
# One run per branch; a newer push makes the older run's answer irrelevant.
|
||||
group: test-${{ gitea.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
suites:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install exiftool
|
||||
# The EXIF checkpoints are the safety invariant of every metadata stage; a run
|
||||
# without exiftool would skip them for a reason the release gate accepts.
|
||||
run: |
|
||||
SUDO=
|
||||
if [ "$(id -u)" -ne 0 ]; then SUDO=sudo; fi
|
||||
$SUDO apt-get update
|
||||
$SUDO apt-get install -y --no-install-recommends libimage-exiftool-perl
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install the application and its test dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install -e '.[test]'
|
||||
python -m playwright install --with-deps chromium
|
||||
|
||||
- name: Helper suite
|
||||
run: work_item/scripts/python -m unittest discover -s work_item/tests -v
|
||||
|
||||
- name: Application suite
|
||||
run: work_item/scripts/python -m pytest tests -q
|
||||
|
||||
# The deployed container, verified the way the host application is (US08-05). It
|
||||
# runs on `main` only — a pull request has nothing published to upgrade *from*, and
|
||||
# building two images per push would pay for that on every commit. Deploy waits for
|
||||
# this whole workflow, so a red container gate is a deploy that does not happen.
|
||||
container:
|
||||
if: gitea.event_name == 'push'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
# The upgrade journey builds the previous commit's tree when no published
|
||||
# image is named, so the history has to be there.
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 2
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install the application and its test dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install -e '.[test]'
|
||||
python -m playwright install --with-deps chromium
|
||||
|
||||
- name: Container acceptance gate
|
||||
# One command: it builds the image, provisions the composition against a
|
||||
# temporary fixture library and an isolated volume, runs the phase_h journeys,
|
||||
# destroys the stack, and writes the evidence. Any skipped check fails it.
|
||||
env:
|
||||
# The fixture libraries are bind-mounted into the containers, so this path
|
||||
# has to be one the Docker daemon can see. On a runner that talks to a
|
||||
# sibling daemon, point it at a shared host path instead of the workspace —
|
||||
# an unshared path arrives as an empty mount and the journeys fail on the
|
||||
# scan, which is the symptom to recognise.
|
||||
PHOTO_PIPELINE_DATA_DIR: ${{ gitea.workspace }}/gate-data
|
||||
PHOTO_PIPELINE_LIBRARY_ROOTS: ${{ gitea.workspace }}/gate-library
|
||||
PHOTO_PIPELINE_TEST_MOUNT_BASE: ${{ gitea.workspace }}/gate-mounts
|
||||
run: |
|
||||
mkdir -p "$PHOTO_PIPELINE_LIBRARY_ROOTS" "$PHOTO_PIPELINE_TEST_MOUNT_BASE"
|
||||
python -m photo_pipeline container-gate --output gate-evidence
|
||||
|
||||
- name: Keep the evidence
|
||||
# Retained per run, and retained on failure especially: the logs are the only
|
||||
# account of what the containers did.
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: container-gate-${{ gitea.sha }}
|
||||
path: gate-evidence
|
||||
12
.gitignore
vendored
12
.gitignore
vendored
@@ -17,15 +17,3 @@ _todo/
|
||||
pictures/
|
||||
photos/
|
||||
_IGNORE/
|
||||
|
||||
# Test failure evidence (US07-04)
|
||||
.artifacts/
|
||||
|
||||
# Any dotenv, not only the default name.
|
||||
*.env
|
||||
|
||||
# Local virtualenv for running the app.
|
||||
.venv/
|
||||
|
||||
# setuptools editable-install metadata.
|
||||
*.egg-info/
|
||||
|
||||
@@ -80,8 +80,7 @@ are the authoritative backlog.
|
||||
uncertain. Work only on the claimed story and its generated feature branch.
|
||||
4. Read the entire issue, linked specification, dependencies, and acceptance criteria.
|
||||
Reconcile them with the concept before designing or changing code.
|
||||
5. Inspect the legacy CLI donors — frozen in `legacy_cli_archive/` since US07-01,
|
||||
with their ledger — before replacing applicable behavior. Update the donor
|
||||
5. Inspect the legacy CLI donors before replacing applicable behavior. Update the donor
|
||||
ledger and characterization tests required by the story.
|
||||
6. Implement every acceptance criterion and its automated tests.
|
||||
7. Run story-specific tests and the accumulated regression suite required by the epic.
|
||||
|
||||
117
Dockerfile
117
Dockerfile
@@ -1,117 +0,0 @@
|
||||
# One image, two roles (US08-02).
|
||||
#
|
||||
# The application is not self-contained Python: it shells out to `exiftool` for every
|
||||
# EXIF checkpoint and to `immich-go` for every upload, and it serves the static
|
||||
# frontend from `frontend/`. All three are installed here at pinned versions, because
|
||||
# an image whose external tools drift is an image whose metadata checkpoints and
|
||||
# upload reports drift with them (concept §15, "External integration risks").
|
||||
#
|
||||
# Everything is pinned:
|
||||
# * the base image by tag *and* digest, so a moved tag cannot change the runtime;
|
||||
# * exiftool by its Debian package version, verified against `exiftool -ver`;
|
||||
# * immich-go by release version and per-architecture SHA-256 of the release asset.
|
||||
# The verified versions become image labels and /etc/photo-pipeline/versions.json,
|
||||
# which `python -m photo_pipeline diagnostics` reports — so a running container can
|
||||
# prove what it contains instead of being trusted about it.
|
||||
#
|
||||
# The project is installed editable on purpose: `photo_pipeline.db` resolves
|
||||
# `alembic.ini` and `migrations/`, and the API resolves `frontend/`, relative to the
|
||||
# repository root. An editable install keeps that one layout instead of scattering the
|
||||
# same files across site-packages and a source tree.
|
||||
|
||||
ARG PYTHON_IMAGE=python:3.12.14-slim-trixie@sha256:2c941e860699f878900b0edc2403613c234d4b32eda3cc9fa7036991a2a63c4a
|
||||
|
||||
# ── the uploader, fetched and verified outside the final layer ────────────────
|
||||
FROM ${PYTHON_IMAGE} AS uploader
|
||||
|
||||
ARG IMMICH_GO_VERSION=0.32.0
|
||||
ARG IMMICH_GO_SHA256_AMD64=6e2ad86bafdadb9466d6515de7cb882726c0aea1a21d51164dff361d7d480a97
|
||||
ARG IMMICH_GO_SHA256_ARM64=2c35d9284baae407ef9540bdac5f488971b0bdc7be758a4d7c05ab270af09fdb
|
||||
|
||||
COPY docker/fetch-immich-go.py /tmp/fetch-immich-go.py
|
||||
RUN python /tmp/fetch-immich-go.py \
|
||||
--version "${IMMICH_GO_VERSION}" \
|
||||
--sha256-amd64 "${IMMICH_GO_SHA256_AMD64}" \
|
||||
--sha256-arm64 "${IMMICH_GO_SHA256_ARM64}" \
|
||||
--into /usr/local/bin \
|
||||
&& /usr/local/bin/immich-go version
|
||||
|
||||
# ── the application ──────────────────────────────────────────────────────────
|
||||
FROM ${PYTHON_IMAGE} AS runtime
|
||||
|
||||
ARG EXIFTOOL_VERSION=13.25+dfsg-1
|
||||
ARG IMMICH_GO_VERSION=0.32.0
|
||||
# The library is mounted from the host, so the container's identity must match the
|
||||
# ownership that library already has: everything this application renames, writes
|
||||
# EXIF into, or archives has to stay owned by the host user afterwards.
|
||||
ARG UID=1000
|
||||
ARG GID=1000
|
||||
|
||||
LABEL org.opencontainers.image.title="photo_pipeline" \
|
||||
org.opencontainers.image.source="https://github.com/domverse/photoanalyzer" \
|
||||
io.photoanalyzer.exiftool.version="${EXIFTOOL_VERSION}" \
|
||||
io.photoanalyzer.immich-go.version="${IMMICH_GO_VERSION}"
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
PATH=/opt/venv/bin:$PATH \
|
||||
PHOTO_PIPELINE_DATA_DIR=/data
|
||||
|
||||
RUN set -eu; \
|
||||
apt-get update; \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||
"libimage-exiftool-perl=${EXIFTOOL_VERSION}"; \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=uploader /usr/local/bin/immich-go /usr/local/bin/immich-go
|
||||
|
||||
WORKDIR /app
|
||||
COPY pyproject.toml alembic.ini README.md ./
|
||||
COPY photo_pipeline ./photo_pipeline
|
||||
COPY migrations ./migrations
|
||||
COPY frontend ./frontend
|
||||
# The manuals are served by the application itself (US09-01), so they ship with it.
|
||||
COPY docs ./docs
|
||||
COPY docker/entrypoint.sh docker/healthcheck.sh /usr/local/bin/
|
||||
|
||||
# Runtime dependencies only: the `test` extra (pytest, playwright) and the `vision`
|
||||
# extra stay out, and pip's build isolation leaves no build tooling behind.
|
||||
RUN set -eu; \
|
||||
python -m venv /opt/venv; \
|
||||
/opt/venv/bin/pip install --no-cache-dir -e .
|
||||
|
||||
# What is installed must be what was pinned, or the labels and the version record
|
||||
# would be a claim rather than a fact.
|
||||
RUN set -eu; \
|
||||
mkdir -p /etc/photo-pipeline; \
|
||||
exiftool_version="$(exiftool -ver)"; \
|
||||
immich_go_version="$(immich-go version | head -n 1 | tr -d '\r')"; \
|
||||
expected_exiftool="$(printf '%s' "${EXIFTOOL_VERSION}" | cut -d+ -f1 | cut -d- -f1)"; \
|
||||
[ "${exiftool_version}" = "${expected_exiftool}" ] \
|
||||
|| { echo "exiftool ${exiftool_version} is not the pinned ${expected_exiftool}" >&2; exit 1; }; \
|
||||
case "${immich_go_version}" in \
|
||||
*"${IMMICH_GO_VERSION}"*) ;; \
|
||||
*) echo "immich-go '${immich_go_version}' is not pinned ${IMMICH_GO_VERSION}" >&2; exit 1 ;; \
|
||||
esac; \
|
||||
printf '{\n "exiftool": "%s",\n "immich-go": "%s"\n}\n' \
|
||||
"${exiftool_version}" "${IMMICH_GO_VERSION}" > /etc/photo-pipeline/versions.json
|
||||
|
||||
# Non-root, with the host library's ownership. /data is the persistent volume; the
|
||||
# photo library itself is mounted by the deployment (US08-03), never baked in.
|
||||
RUN set -eu; \
|
||||
groupadd --gid "${GID}" --non-unique app; \
|
||||
useradd --uid "${UID}" --gid "${GID}" --non-unique --no-create-home --home-dir /app app; \
|
||||
mkdir -p /data; \
|
||||
chown "${UID}:${GID}" /data
|
||||
USER ${UID}:${GID}
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
# Readiness, not liveness: an unmigrated or misconfigured database answers
|
||||
# /api/v1/health/ready with 503, and a container that cannot serve must not be
|
||||
# reported healthy. The worker role has no endpoint, so its check is a no-op here.
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
|
||||
CMD ["/usr/local/bin/healthcheck.sh"]
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
|
||||
CMD ["serve"]
|
||||
567
README.md
567
README.md
@@ -7,274 +7,16 @@ archive workflow. Planning lives in `INTEGRATED_PIPELINE_CONCEPT.md` and
|
||||
## Application (`photo_pipeline`)
|
||||
|
||||
The target application lives in `photo_pipeline/` (FastAPI + SQLAlchemy + Alembic).
|
||||
Install it into a virtualenv once:
|
||||
Run it with:
|
||||
|
||||
```bash
|
||||
python3.12 -m venv .venv
|
||||
.venv/bin/pip install -e ".[vision]" # drop [vision] for a review-only install
|
||||
python -m photo_pipeline migrate # apply database migrations
|
||||
python -m photo_pipeline serve # start the API + static review UI (127.0.0.1:8000)
|
||||
```
|
||||
|
||||
Then run the two processes:
|
||||
|
||||
```bash
|
||||
.venv/bin/python -m photo_pipeline migrate # apply database migrations
|
||||
.venv/bin/python -m photo_pipeline serve # API + review UI at 127.0.0.1:8000/app/
|
||||
.venv/bin/python -m photo_pipeline worker # second terminal: runs the jobs
|
||||
```
|
||||
|
||||
The server enqueues work and serves the UI; nothing actually scans, scores,
|
||||
analyses, uploads, or archives without a worker. `work_item/scripts/python` is the
|
||||
*helper's* launcher — it prefers Conda base and falls back to a bare system
|
||||
interpreter, so it is not how the application is run.
|
||||
|
||||
Configuration comes from `PHOTO_PIPELINE_*` environment variables (see
|
||||
`photo_pipeline/config.py`); secrets are referenced, never logged.
|
||||
|
||||
### Configuration file
|
||||
|
||||
`.env` in the working directory is read at startup, or any path named by
|
||||
`PHOTO_PIPELINE_ENV_FILE`. It is parsed, never executed: `KEY=value` lines,
|
||||
`#` comments, optional quotes — no interpolation and no `export`. **Anything already
|
||||
exported wins**, so the file is the standing configuration and the shell is the
|
||||
override for one run.
|
||||
|
||||
The archived CLI's variable names still work, so an existing `photo_analyzer.env`
|
||||
can be used as-is:
|
||||
|
||||
| in the file | applied as |
|
||||
|---|---|
|
||||
| `LLM_API_KEY` / `GEMINI_API_KEY` | `OPENAI_API_KEY` |
|
||||
| `LLM_BASE_URL` | `OPENAI_BASE_URL` |
|
||||
| `LIBRARY` | `PHOTO_PIPELINE_LIBRARY_ROOTS` |
|
||||
|
||||
`.env` and `*.env` are gitignored and denied by the work-item safety checks: the
|
||||
file holds a real key and must never be committed.
|
||||
|
||||
### API access (US07-02)
|
||||
|
||||
The app listens on loopback, so its attacker is another page in the same browser.
|
||||
Every `/api/v1` route except `health/live`, `health/ready`, and `session` requires
|
||||
the application session, and every mutation requires its CSRF token as well:
|
||||
|
||||
```bash
|
||||
BASE=http://127.0.0.1:8000
|
||||
TOKEN=$(curl -sc /tmp/pp.jar $BASE/api/v1/session | python -c 'import json,sys; print(json.load(sys.stdin)["csrf_token"])')
|
||||
curl -sb /tmp/pp.jar -H "X-CSRF-Token: $TOKEN" -X POST $BASE/api/v1/albums/proposals -d '{}' -H 'Content-Type: application/json'
|
||||
```
|
||||
|
||||
The session is per server process — restarting `serve` invalidates it, and the
|
||||
browser client re-bootstraps by itself. Requests are also refused when the `Host` is
|
||||
not a loopback name (DNS rebinding), when `Origin` is any other origin, when
|
||||
`Sec-Fetch-Site` says the request came from another site (an `<img>` pointed at a
|
||||
thumbnail), or when the body exceeds `PHOTO_PIPELINE_MAX_REQUEST_BYTES`. There is no
|
||||
CORS middleware at all, so no other origin can read a response.
|
||||
|
||||
### Reaching it through a hostname or proxy (US08-01)
|
||||
|
||||
| variable | meaning |
|
||||
|---|---|
|
||||
| `PHOTO_PIPELINE_ALLOWED_HOSTS` | comma-separated extra names the app answers to; empty means loopback only |
|
||||
| `PHOTO_PIPELINE_ACCESS_SECRET` | traded for the session cookie at `GET /api/v1/session` via `X-Access-Secret` |
|
||||
| `PHOTO_PIPELINE_TRUSTED_PROXIES` | comma-separated peer addresses whose `X-Forwarded-Proto`/`X-Forwarded-Host` are believed |
|
||||
|
||||
Being reachable *was* the authentication: whoever could open `127.0.0.1:8000` owned
|
||||
the library. So naming any non-loopback host — or binding to one, `0.0.0.0` included
|
||||
— makes the access secret mandatory, and `serve` refuses to start without it rather
|
||||
than publishing the library. Loopback-only deployments need no secret and behave
|
||||
exactly as before.
|
||||
|
||||
```bash
|
||||
curl -sc /tmp/pp.jar -H "X-Access-Secret: $PHOTO_PIPELINE_ACCESS_SECRET" \
|
||||
https://photos.example.com/api/v1/session
|
||||
```
|
||||
|
||||
The browser asks for the secret once per tab and keeps it in `sessionStorage`.
|
||||
Wrong secrets are rate-limited (5 per minute) and logged with the caller's address
|
||||
only. `Host` and `Origin` are judged against the configured names; the *external*
|
||||
scheme and host come from the forwarded headers only when the request arrived from a
|
||||
`PHOTO_PIPELINE_TRUSTED_PROXIES` address, so a client cannot declare its own origin,
|
||||
and the session cookie is marked `Secure` when that external scheme is HTTPS. Health
|
||||
endpoints stay reachable without the secret so an orchestrator can restart the
|
||||
container; nothing else does.
|
||||
|
||||
## Container image (US08-02)
|
||||
|
||||
One image runs either role. It is built from a clean checkout with no arguments:
|
||||
|
||||
```bash
|
||||
docker build -t photo-pipeline:dev .
|
||||
```
|
||||
|
||||
Everything external is pinned, and the build fails rather than drifting: the Python
|
||||
base image by tag *and* digest, `exiftool` by its Debian package version (verified
|
||||
against `exiftool -ver`), and `immich-go` by release version and per-architecture
|
||||
SHA-256 of the release asset. The verified versions become image labels and
|
||||
`/etc/photo-pipeline/versions.json`, which `diagnostics` reports as `tools[].pinned`
|
||||
beside the version actually installed — so a replaced binary shows up as a
|
||||
`tool_version_drift` warning instead of as a misparsed upload report.
|
||||
|
||||
| build argument | default | why change it |
|
||||
|---|---|---|
|
||||
| `UID` / `GID` | `1000` | must match the owner of the mounted photo library |
|
||||
| `PYTHON_IMAGE` | pinned digest | upgrading the base image |
|
||||
| `EXIFTOOL_VERSION` | Debian package version | upgrading exiftool |
|
||||
| `IMMICH_GO_VERSION` + `IMMICH_GO_SHA256_AMD64`/`_ARM64` | pinned release | upgrading the uploader (take the digests from that release's `checksums.txt`) |
|
||||
|
||||
The first argument is the role, and every other management command still works:
|
||||
|
||||
```bash
|
||||
docker run --rm -v /srv/photos:/srv/photos -v pp-data:/data \
|
||||
-e PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos photo-pipeline:dev migrate
|
||||
|
||||
docker run -d -p 127.0.0.1:8000:8000 -v /srv/photos:/srv/photos -v pp-data:/data \
|
||||
-e PHOTO_PIPELINE_HOST=0.0.0.0 -e PHOTO_PIPELINE_ACCESS_SECRET=... \
|
||||
-e PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos photo-pipeline:dev serve
|
||||
|
||||
docker run -d -v /srv/photos:/srv/photos -v pp-data:/data \
|
||||
-e PHOTO_PIPELINE_LIBRARY_ROOTS=/srv/photos photo-pipeline:dev worker
|
||||
```
|
||||
|
||||
One role per container: `serve` and `worker` each take the library process lock for
|
||||
their role (US07-05), so no supervisor starts both. The container refuses to run as
|
||||
UID 0 — files it renames or writes must keep the ownership the host library expects —
|
||||
and `/data` is the persistent volume holding the database, journals, backups, and
|
||||
thumbnail cache. Binding to `0.0.0.0` makes the access secret mandatory
|
||||
([above](#reaching-it-through-a-hostname-or-proxy-us08-01)); `serve` refuses to start
|
||||
without it. The declared `HEALTHCHECK` polls `/api/v1/health/ready`, so a container
|
||||
whose database is unmigrated or misconfigured is never reported healthy.
|
||||
|
||||
## Composed runtime (US08-03)
|
||||
|
||||
`docker-compose.yml` is the deployment: one `serve` container, one `worker`
|
||||
container, one bind-mounted library, one data volume, and a one-shot `migrate` that
|
||||
both roles wait for.
|
||||
|
||||
```bash
|
||||
cp .env.example .env && $EDITOR .env # nothing has a value in it; fill in yours
|
||||
docker compose up -d --build
|
||||
```
|
||||
|
||||
`.env.example` lists every `PHOTO_PIPELINE_*` variable with its default in a comment
|
||||
and no values at all. The three the composition cannot start without are
|
||||
`PHOTO_PIPELINE_LIBRARY_HOST_PATH` (the library on this host),
|
||||
`PHOTO_PIPELINE_LIBRARY_ROOTS` (where it is mounted *inside* the container), and
|
||||
`PHOTO_PIPELINE_ACCESS_SECRET` — publishing the port means the app is reachable from
|
||||
outside the container, so US08-01 makes the secret mandatory. Set
|
||||
`PHOTO_PIPELINE_UID`/`GID` to the owner of the library: what the containers rename
|
||||
and rewrite keeps that ownership.
|
||||
|
||||
| invariant | how the composition keeps it |
|
||||
|---|---|
|
||||
| one writer | `serve` and `worker` take their role's library lock (US07-05) in the shared `/data` volume, so `--scale worker=2` is refused by the lock, not by convention |
|
||||
| migrations first | `migrate` runs the backup-then-migrate path and must exit 0 before `api` and `worker` start; a failed upgrade leaves the previous database and its pre-migration backup intact |
|
||||
| container paths | `PHOTO_PIPELINE_LIBRARY_ROOTS` is both the mount target and the configured root; a root that is not mounted makes `serve`/`worker` exit 5 at startup instead of writing into the container's throwaway layer |
|
||||
| loopback by default | the port is published to `127.0.0.1` unless `PHOTO_PIPELINE_PUBLISH_ADDRESS` says otherwise, and exposing it needs the hostname in `PHOTO_PIPELINE_ALLOWED_HOSTS` plus the access secret |
|
||||
| restart safety | both roles are `restart: unless-stopped` with a 30 s stop grace period, and a job interrupted by a restart resumes exactly as it does on a host restart |
|
||||
|
||||
The `data` volume holds the database, its write-ahead log, the thumbnail cache,
|
||||
journals, and backups. **It must stay on a local filesystem** — SQLite in WAL mode
|
||||
needs real local locking, so NFS, SMB, and network volume drivers are unsupported
|
||||
there and corrupt the database rather than slow it down. The library bind mount has
|
||||
no such restriction.
|
||||
|
||||
Operating the deployment is operating the same CLI:
|
||||
|
||||
```bash
|
||||
docker compose run --rm --no-deps api diagnostics
|
||||
docker compose run --rm --no-deps api backup --reason pre-upgrade
|
||||
docker compose run --rm --no-deps api verify-backup /data/backups/<name>
|
||||
docker compose run --rm --no-deps api restore /data/backups/<name> --into /data/restored
|
||||
docker compose logs -f worker
|
||||
```
|
||||
|
||||
`--no-deps` keeps a one-off command from starting a second stack; `api` is only the
|
||||
service the command borrows the image and mounts from. `restore` is deliberately not
|
||||
an API call: it replaces the state of an installation and belongs to a stopped one,
|
||||
so stop `api` and `worker` first and restart them against the restored directory.
|
||||
|
||||
## Publishing and deploying (US08-04)
|
||||
|
||||
Two workflows in `.gitea/workflows/` make `main` the only path to the running stack:
|
||||
|
||||
| workflow | runs on | does |
|
||||
|---|---|---|
|
||||
| `test.yml` (**Test**) | every pull request, and every push to `main` | installs exiftool, the package, and Chromium, then runs the suites configured in `work_item/.work-item.yml` |
|
||||
| `deploy.yml` (**Deploy**) | a **successful** `Test` run on `main`, or manual dispatch | builds the image, pushes it, triggers the Portainer webhook, prunes dangling layers |
|
||||
|
||||
Deploy waits for `Test` through `workflow_run`, so publishing is downstream of a green
|
||||
suite rather than of a push. `completed` is not `success`: the publish job runs only on
|
||||
`conclusion == 'success'`. Because an unmatched `workflow_run` filter does not fail but
|
||||
simply never fires, `tests/integration/test_deploy_workflows.py` asserts the link from
|
||||
both ends — renaming either workflow, or dropping one of the configured suites out of
|
||||
the test job, fails that test rather than silently unhooking the gate.
|
||||
|
||||
The image is `git.domverse-berlin.eu/domverse/photoanalyzer`, tagged with the commit
|
||||
SHA and `latest`. The commit tag is pushed first, so a `latest` that exists always has
|
||||
a commit tag beside it and **a rollback is a tag change, not a rebuild**:
|
||||
|
||||
```bash
|
||||
docker pull git.domverse-berlin.eu/domverse/photoanalyzer:<known-good-sha>
|
||||
docker tag git.domverse-berlin.eu/domverse/photoanalyzer:<known-good-sha> \
|
||||
git.domverse-berlin.eu/domverse/photoanalyzer:latest
|
||||
docker push git.domverse-berlin.eu/domverse/photoanalyzer:latest
|
||||
# then trigger the Portainer webhook, or redeploy the stack from Portainer
|
||||
```
|
||||
|
||||
Manual dispatch defaults to **dry run**: it builds and pushes `scratch-<sha>` only, and
|
||||
touches neither `latest` nor the running stack. Clear the `dry_run` input to publish by
|
||||
hand. A workflow-level `concurrency: deploy` group with `cancel-in-progress: false`
|
||||
queues deploys instead of overlapping them, so two commits can never race to `latest`.
|
||||
|
||||
### Secrets
|
||||
|
||||
Repository secrets — used only by the pipeline:
|
||||
|
||||
| secret | purpose |
|
||||
|---|---|
|
||||
| `REGISTRY_USER` | registry user with `write:package` |
|
||||
| `REGISTRY_TOKEN` | that user's token, passed on stdin, never as an argument |
|
||||
| `PORTAINER_WEBHOOK_URL` | the stack's auto-update POST URL; `curl --fail` makes a refused redeploy a failed workflow |
|
||||
|
||||
Runtime secrets — `PHOTO_PIPELINE_VISION_API_KEY`, `PHOTO_PIPELINE_IMMICH_API_KEY`,
|
||||
`PHOTO_PIPELINE_ACCESS_SECRET` — are **not** repository secrets and are not in the
|
||||
image. The stack is managed by Portainer from git (`docker-compose.yml`), and those
|
||||
values live in the Portainer stack's environment, so rotation is one place and a
|
||||
repository read discloses nothing. A test asserts the workflows never name them.
|
||||
|
||||
## Container acceptance gate (US08-05)
|
||||
|
||||
The deployed container is verified the way the host application is, by one command:
|
||||
|
||||
```bash
|
||||
work_item/scripts/python -m photo_pipeline container-gate --output gate-evidence
|
||||
```
|
||||
|
||||
It builds the image, provisions the composition against a **temporary fixture library
|
||||
on a bind mount and an isolated data volume**, runs the `phase_h` journeys against it,
|
||||
destroys every stack afterwards, and writes `release-report.json`, `logs/container.log`,
|
||||
and `CHECKSUMS.sha256` into the evidence directory. It exits non-zero when a journey
|
||||
fails, when the story matrix has a hole, **or when any check skipped at all** — unlike
|
||||
the release gate, this one accepts no environment excuse: a run that never reached the
|
||||
containers proved nothing about them.
|
||||
|
||||
The journeys (`tests/e2e/test_phase_h_container.py`) are:
|
||||
|
||||
| journey | what it proves |
|
||||
|---|---|
|
||||
| browser | discovery, duplicate review, analysis, album proposal, rename, upload preflight, and archive views, driven through the containerized frontend — including a rename that really moves the operator's folder on the bind mount |
|
||||
| upgrade | the previous version's image runs first, then this one against the same volume: schema at the new head, assets, analysis results, job progress, the unapplied rename plan, and the thumbnail cache all survive |
|
||||
| restart | `docker kill` on both containers mid-job; the job resumes, every photo ends with exactly one stored result, and only the in-flight item ever reaches the provider twice |
|
||||
| security | no session refused, a forged `X-Forwarded-Host` cannot smuggle an allowed hostname past the check, a symlink out of the mounted library is refused, an unmounted library root refuses startup, and no secret appears in `docker compose logs` |
|
||||
|
||||
The upgrade journey builds the previous commit's tree when no published image is named;
|
||||
point it at the real one with `PHOTO_PIPELINE_PREVIOUS_IMAGE`. On a Docker VM (Colima,
|
||||
Docker Desktop) the fixture library must live on a shared path — it defaults to
|
||||
`~/.cache/photo-pipeline`, overridable with `PHOTO_PIPELINE_TEST_MOUNT_BASE`.
|
||||
|
||||
CI runs this gate as the `container` job of **Test** on pushes to `main` and keeps its
|
||||
evidence as a run artefact. Deploy waits for the whole `Test` workflow, so a red
|
||||
container gate is a publish that does not happen.
|
||||
|
||||
## Testing
|
||||
|
||||
One offline command runs the whole suite (unit, integration, and browser
|
||||
@@ -427,306 +169,3 @@ work_item/scripts/python -m pytest -m phase_e -q
|
||||
bytes, and recovery after a restart).
|
||||
|
||||
Phases A–D remain green in the full run above.
|
||||
|
||||
### Phase F acceptance gate
|
||||
|
||||
Phase F (Epic E06: archive lifecycle) is the only stage that *removes* originals
|
||||
from the library, and the only one whose storage can walk away in someone's bag.
|
||||
One command runs the archive fault-injection suites, the black-box archive and
|
||||
restore API journeys, and the browser suite:
|
||||
|
||||
```bash
|
||||
work_item/scripts/python -m pytest -m phase_f -q
|
||||
```
|
||||
|
||||
- `tests/integration/test_archive_*.py` and `tests/integration/test_restore.py`
|
||||
drive real files on real filesystems: preflight against a mounted, missing,
|
||||
swapped, read-only, or full medium; copy-verify-remove and the same-filesystem
|
||||
move path; and a crash at **every** persisted journal transition in both transfer
|
||||
modes, asserting that no source is ever removed without a durable, byte-identical
|
||||
archive copy.
|
||||
- `tests/e2e/test_phase_f_pipeline.py` drives a real server and a real durable
|
||||
worker over HTTP: preflight blockers (offline medium, wrong volume, insufficient
|
||||
capacity, bytes changed after upload), a verified archive whose manifest, hashes,
|
||||
and path history are checked on the medium itself, a worker killed at each of
|
||||
`transferring`, `verified`, `removing`, `source_removed`, and `complete`, the
|
||||
evidence-based recovery that follows, offline deduplication of an exact and a
|
||||
fuzzy copy while the medium is away, mount return, restore, and a collision that
|
||||
restores beside its occupant.
|
||||
- **Archived is not missing.** An unmounted medium leaves its photos
|
||||
`archived_offline` — still hashed, still in the duplicate indexes, still
|
||||
previewable through their protected thumbnails — and a rescan neither prunes nor
|
||||
flags them.
|
||||
- **Ambiguity is never guessed.** A journal state the medium contradicts stays
|
||||
`manual`, offers no automatic action, and keeps blocking further archiving until
|
||||
a human decides.
|
||||
- `tests/e2e/test_archive_ui.py` covers the browser journeys (preview with
|
||||
destination identity and reclaimable bytes, blockers and mount instructions,
|
||||
progress split into transfer/verification/removal, interruption and recovery,
|
||||
offline browsing, restore, collision, keyboard confirmation, and reload).
|
||||
|
||||
Phases A–E remain green in the full run above.
|
||||
|
||||
## Media and metadata hardening (US07-03)
|
||||
|
||||
Every pixel the application reads goes through `photo_pipeline/imaging.py`: the
|
||||
declared dimensions are checked before anything is decoded, Pillow's
|
||||
decompression-bomb warning is treated as a refusal, JPEG decodes near the requested
|
||||
size, and each decoder failure becomes one of two typed errors. A damaged file is a
|
||||
per-item error with a persisted code, never a failed scan or a dead worker.
|
||||
|
||||
Every metadata stage ends with an EXIF checkpoint (`services/exif_checkpoint.py`):
|
||||
snapshot, write the owned keywords, read back, prove the owned fields landed and that
|
||||
nothing else moved, refresh the file hash. A field the stage does not own that
|
||||
changed anyway makes the checkpoint `divergent` — recorded in `exif_projections`,
|
||||
shown in the review queue, never repaired behind the user's back, and not counted as
|
||||
verified, so upload stays blocked.
|
||||
|
||||
The golden corpus that proves all of it is generated, not committed:
|
||||
`tests/fixtures/media_corpus.py` declares every format, orientation, profile,
|
||||
damage, and metadata case with its expected outcome, and the suite regenerates it
|
||||
twice to prove it does not drift.
|
||||
|
||||
```bash
|
||||
work_item/scripts/python -m pytest tests/integration/test_media_hardening.py tests/integration/test_exif_checkpoints.py -q
|
||||
```
|
||||
|
||||
## Concurrency and crash recovery (US07-04)
|
||||
|
||||
Crash safety is proven by crashing. `photo_pipeline/faults.py` defines the control
|
||||
points — the persisted transitions of the rename, archive, EXIF, upload, and job
|
||||
lanes — and arms one only when `PHOTO_PIPELINE_FAULT_AFTER` names it, at which
|
||||
point the process dies the way a `SIGKILL` does. There is no endpoint and no
|
||||
configuration field that can reach a barrier; a deployment that never sets the
|
||||
variable can never hit one.
|
||||
|
||||
The race suite runs each scenario several times with a seed recorded on the test
|
||||
result (`race_seed`) and asserts invariants rather than schedules: work is never
|
||||
claimed or executed twice, a stale fencing token never commits, no file body is
|
||||
lost or overwritten, and the database still passes `PRAGMA integrity_check`.
|
||||
|
||||
```bash
|
||||
work_item/scripts/python -m pytest tests/integration/test_concurrency_races.py \
|
||||
tests/integration/test_fault_matrix.py tests/e2e/test_crash_recovery.py -q
|
||||
|
||||
# replay a failure, or soak for new interleavings
|
||||
PHOTO_PIPELINE_RACE_SEED=1234 PHOTO_PIPELINE_RACE_REPEATS=50 \
|
||||
work_item/scripts/python -m pytest tests/integration/test_concurrency_races.py -q
|
||||
```
|
||||
|
||||
Any failing test keeps its evidence: the temporary database (with its write-ahead
|
||||
log), the journals, the logs, the recorded seed, and a SHA-256 manifest of every
|
||||
file in the temporary library are copied to `.artifacts/<test id>/` before pytest
|
||||
deletes the directory. Point `PHOTO_PIPELINE_TEST_ARTIFACTS` elsewhere to collect
|
||||
them from CI.
|
||||
|
||||
## Release gate (US07-07)
|
||||
|
||||
One command runs every suite in an isolated stack and keeps the evidence:
|
||||
|
||||
```bash
|
||||
work_item/scripts/python -m photo_pipeline release-gate --output data/release/$(date -u +%Y%m%dT%H%M%SZ)
|
||||
```
|
||||
|
||||
It fails — and exits non-zero — when any stage fails, when a suite skips a test for
|
||||
a reason that is not a documented environment limit (`exiftool not installed`,
|
||||
`root ignores directory permissions`), or when the story matrix has a hole. The
|
||||
evidence directory holds `release-report.json` (revision, per-stage result, timings,
|
||||
summaries), `logs/<stage>.log`, and `CHECKSUMS.sha256` over both.
|
||||
|
||||
**The story matrix** lives in `tests/story_traceability.json`: every story under
|
||||
`delivery_backlog/stories/` is either mapped to test files that exist, or listed in
|
||||
`planned` as an accepted but unimplemented story. A story that is neither, or a
|
||||
mapping to a file that has been deleted, fails the gate.
|
||||
|
||||
**The journey** (`tests/e2e/test_release_journey.py`) takes one fresh library through
|
||||
discovery, duplicate review, safety, analysis, EXIF verification, album proposal,
|
||||
guarded rename, rescan, upload with server-side verification, archive, offline
|
||||
deduplication, and restore — over HTTP against real server and worker processes,
|
||||
with a full restart in the middle and at the end.
|
||||
|
||||
### Real-library dry run and approval
|
||||
|
||||
Before the application is pointed at photos that cannot be replaced:
|
||||
|
||||
```bash
|
||||
work_item/scripts/python -m photo_pipeline dry-run --output dry-run.json
|
||||
work_item/scripts/python -m photo_pipeline approve-dry-run dry-run.json --approver "$(whoami)"
|
||||
```
|
||||
|
||||
The dry run is strictly read-only: it opens no file for writing, writes no database
|
||||
row, and reports what it found — file counts by extension, folders, bytes, unreadable
|
||||
files, excluded directories, and a reconciliation against what the database already
|
||||
knows (already registered, new, recorded but absent). Set
|
||||
`PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL=1` and **every mutating API request is
|
||||
refused with `403 dry_run_not_approved`** until a report for exactly those library
|
||||
roots has been approved. Reading stays open — you have to be able to see what was
|
||||
found in order to approve it — and so does taking a backup. Change the library roots
|
||||
and the approval no longer applies: it approves that reconciliation, not the idea of
|
||||
mutating.
|
||||
|
||||
## Performance budgets (US07-06)
|
||||
|
||||
Budgets are measured, not asserted in prose. `python -m photo_pipeline benchmark`
|
||||
builds a synthetic library of a stated size, runs the same scenarios every time,
|
||||
writes a machine-readable report, and **exits non-zero when a budget is breached**.
|
||||
|
||||
```bash
|
||||
work_item/scripts/python -m photo_pipeline benchmark --profile smoke # ~2 s, runs in CI
|
||||
work_item/scripts/python -m photo_pipeline benchmark --profile short # 25k assets
|
||||
work_item/scripts/python -m photo_pipeline benchmark --profile full \
|
||||
--output data/benchmarks/full.json # 25k + 100k
|
||||
work_item/scripts/python -m photo_pipeline benchmark --profile huge \
|
||||
--soak-seconds 3600 --output data/benchmarks/soak.json # 500k + soak
|
||||
```
|
||||
|
||||
| Metric | Budget | Why |
|
||||
|---|---|---|
|
||||
| `latency_p95_ms` | 250 ms | a list or search page must feel immediate |
|
||||
| `latency_max_ms` | 2 000 ms | no single page may stall the review flow |
|
||||
| `rss_growth_bytes` | 400 MB | a run must not leak the library |
|
||||
| `open_files` | 256 | file descriptors are a hard operating-system limit |
|
||||
| `wal_bytes` | 200 MB | a growing write-ahead log means checkpoints are starving |
|
||||
| `queue_depth` | 1 000 | an unbounded queue is an out-of-memory in waiting |
|
||||
| `cache_over_quota_bytes` | 0 | the thumbnail cache has to respect its quota |
|
||||
|
||||
Measured on the reference machine (Apple Silicon, SQLite WAL), p95 per scenario:
|
||||
|
||||
| Scenario | 25k | 100k |
|
||||
|---|---|---|
|
||||
| `inventory_page` | 0.5 ms | 0.6 ms |
|
||||
| `library_search` | 4.8 ms | 17.1 ms |
|
||||
| `library_stats` | 56.8 ms | 197.4 ms |
|
||||
| `workflow_readiness` | 52.6 ms | 235.4 ms |
|
||||
| `duplicate_cluster_list` | 0.6 ms | 0.5 ms |
|
||||
| `duplicate_cluster_page` | 1.7 ms | 1.7 ms |
|
||||
|
||||
CI runs the `smoke` profile through `tests/integration/test_performance_budgets.py`;
|
||||
the 25k/100k/500k matrix and the multi-hour soak belong to scheduled infrastructure,
|
||||
because minutes of build time do not belong in the suite that runs on every change.
|
||||
|
||||
**Exceptions.** A budget that cannot be met is not a warning to ignore: it goes into
|
||||
`APPROVED_EXCEPTIONS` in `photo_pipeline/services/benchmarks.py` with its raised
|
||||
limit, who approved it, why, and a review date. Every report lists the exceptions it
|
||||
applied, so a release review sees them.
|
||||
|
||||
Approved today, both for the 500k `huge` profile only, review by 2027-02-17:
|
||||
|
||||
| Scenario | Measured at 500k | Raised limit |
|
||||
|---|---|---|
|
||||
| `library_stats` | 1.08 s p95 · 3.2 s max | 1.5 s p95 · 4 s max |
|
||||
| `workflow_readiness` | 1.40 s p95 · 3.3 s max | 1.8 s p95 · 4 s max |
|
||||
|
||||
Both are library-wide aggregates — the current safety decision of every asset, and
|
||||
the album/tag/year breakdown of every analysis row — and both meet the 250 ms budget
|
||||
at the 100k rows the concept sets it for. Beyond that they are linear against one
|
||||
SQLite writer; the fix is denormalized totals or the planned PostgreSQL transition,
|
||||
not a query tweak. Everything else at 500k is inside budget, and a soak at that size
|
||||
grows neither resident memory nor the job queue.
|
||||
|
||||
## Backup and recovery (US07-05)
|
||||
|
||||
Backups go through SQLite's online backup API, never a file copy: with WAL enabled
|
||||
the `.db` file alone is missing every committed page still in the write-ahead log.
|
||||
Each backup is a directory under `data/backups/` holding the snapshot and a
|
||||
`manifest.json` describing it — schema revision, SHA-256, row counts, the archive
|
||||
media the library depends on, and which configuration was set. Secrets are recorded
|
||||
as `configured`, never as values, so a manifest is safe to attach to a bug report.
|
||||
|
||||
```bash
|
||||
work_item/scripts/python -m photo_pipeline backup --reason before-upgrade --keep 7
|
||||
work_item/scripts/python -m photo_pipeline verify-backup data/backups/<name>
|
||||
work_item/scripts/python -m photo_pipeline diagnostics
|
||||
```
|
||||
|
||||
The same is available at `GET /api/v1/diagnostics`, `GET|POST /api/v1/backups`,
|
||||
`GET /api/v1/backups/{name}/verify`, and `POST /api/v1/backups/prune`. **Restore is
|
||||
not an endpoint** — it replaces the state of an installation, so it belongs to a
|
||||
stopped one and a person at a terminal.
|
||||
|
||||
### Integrity check
|
||||
|
||||
`verify-backup` runs `PRAGMA integrity_check` (structure) *and*
|
||||
`PRAGMA foreign_key_check` (references), compares the snapshot's SHA-256 with the
|
||||
manifest, and re-counts every table the manifest recorded. Any mismatch — bit rot, a
|
||||
truncated copy, a "repaired" snapshot — fails the check, and `restore` refuses a
|
||||
backup that does not verify.
|
||||
|
||||
### Restore drill
|
||||
|
||||
1. Stop the server and the worker.
|
||||
2. `python -m photo_pipeline verify-backup data/backups/<name>` — never restore an
|
||||
unverified snapshot.
|
||||
3. `python -m photo_pipeline restore data/backups/<name> --into /path/to/fresh-data`
|
||||
(a target that already holds a database is refused; recovering in place means
|
||||
moving the old data directory aside first).
|
||||
4. Point `PHOTO_PIPELINE_DATA_DIR` at the restored directory and run
|
||||
`python -m photo_pipeline migrate`.
|
||||
5. Run an inventory scan so paths are reconciled against the real library.
|
||||
6. Mount every archive location named in the manifest before archiving again — the
|
||||
database records where archived originals are, but it does not contain them.
|
||||
|
||||
Practise this against a copy before you need it; the drill is exercised
|
||||
automatically by `tests/integration/test_backup_recovery.py`.
|
||||
|
||||
### Failed migration
|
||||
|
||||
A pending schema upgrade is snapshotted first (`reason: pre-migration`), by both the
|
||||
API startup and `python -m photo_pipeline migrate`. If a migration fails, the error
|
||||
log names the backup directory: stop everything and run the restore drill against
|
||||
it. An up-to-date database is not backed up again on every start.
|
||||
|
||||
### Archive media
|
||||
|
||||
Archived originals live on their medium, not in the backup. The manifest lists every
|
||||
archive location with its `media_id` and whether it was mounted when the backup was
|
||||
taken. Keep one copy of each medium off-site, and remount a location before
|
||||
restoring assets from it.
|
||||
|
||||
### Retention and disk
|
||||
|
||||
`--keep N` (default 7) prunes the oldest backups and never the newest.
|
||||
`diagnostics` reports the database, write-ahead log, thumbnail cache, uploader
|
||||
reports, backups, and logs separately, with free space and warnings for low disk
|
||||
(`disk_low`, `disk_critical`), a cache over its quota, a write-ahead log outgrowing
|
||||
its database, and a legacy CLI writing the library.
|
||||
|
||||
### Process locking
|
||||
|
||||
`serve` and `worker` take a JSON lock in the data directory (`api.lock.json`,
|
||||
`worker.lock.json`). A second worker exits `2` and names the holder; a lock whose
|
||||
process is gone is taken over. If the frozen CLI's state files are being written,
|
||||
both refuse with exit `3` — `--allow-legacy` overrides, and you own the outcome.
|
||||
|
||||
## Legacy CLI archive
|
||||
|
||||
The command-line tools this application was extracted from are frozen in
|
||||
`legacy_cli_archive/` (US07-01): the original sources, their docs, the dependency
|
||||
lock they were last verified against, schema notes, a redacted sample
|
||||
configuration, the donor ledger, and a checksum for every file.
|
||||
|
||||
```bash
|
||||
cd legacy_cli_archive && shasum -a 256 -c CHECKSUMS.sha256 # verify the archive
|
||||
work_item/scripts/python -m pytest tests/unit/test_legacy_archive.py -q # lint it
|
||||
```
|
||||
|
||||
They are reference material and rollback evidence only. No module under
|
||||
`photo_pipeline/` imports or executes them, the archive is not on the application's
|
||||
import path, and `tests/unit/test_legacy_archive.py` enforces that along with the
|
||||
checksums and the redaction. Only the two suites that compare *against* the donors —
|
||||
`tests/characterization/` and `tests/integration/test_safety_parity.py` — put the
|
||||
archived sources on `sys.path`.
|
||||
|
||||
The last path-keyed state they owned, `nsfw_scores.csv`, is imported once and then
|
||||
left alone:
|
||||
|
||||
```bash
|
||||
work_item/scripts/python -m photo_pipeline import-legacy-scores /path/to/nsfw_scores.csv --dry-run
|
||||
```
|
||||
|
||||
The import writes scored-but-unreviewed `safety_reviews` rows onto stable asset ids,
|
||||
never invents an asset for an unknown path, never overwrites a human decision, and
|
||||
writes a reconciliation report to the data directory saying exactly what it did.
|
||||
`legacy_cli_archive/donor_ledger.yaml` records every migrated behavior with its
|
||||
target, the tests that pin the donor, the tests that prove the replacement, and each
|
||||
intentional delta; rows still marked `pending` name the story that will resolve them.
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
# E08 — Container Deployment
|
||||
|
||||
Concept phase: none. This epic is a delivery-format addition on top of the concept:
|
||||
the same application, same safety invariants, packaged as a Docker image and deployed
|
||||
continuously from Gitea Actions instead of being started by hand from a working copy.
|
||||
|
||||
It does not change the product scope in
|
||||
[`INTEGRATED_PIPELINE_CONCEPT.md`](../INTEGRATED_PIPELINE_CONCEPT.md). SQLite stays the
|
||||
store, one worker stays the writer, the library process lock stays authoritative, and
|
||||
no path outside the configured library roots becomes reachable because the process now
|
||||
runs in a container.
|
||||
|
||||
One decision does extend the concept and is made here explicitly: the application may
|
||||
be reached through a reverse proxy under a real hostname, not only over loopback. That
|
||||
requires a configurable trust boundary and an authentication gate, because the
|
||||
loopback-only checks of US07-02 are what currently stand in for authentication.
|
||||
|
||||
## Stories
|
||||
|
||||
1. [US08-01 — Make the trust boundary configurable and authenticated](stories/US08-01-trusted-hosts-auth.md)
|
||||
2. [US08-02 — Build a reproducible application image](stories/US08-02-container-image.md)
|
||||
3. [US08-03 — Compose the runtime and mount the library safely](stories/US08-03-compose-runtime.md)
|
||||
4. [US08-04 — Publish and deploy from Gitea Actions](stories/US08-04-gitea-cicd.md)
|
||||
5. [US08-05 — Automate container deployment acceptance](stories/US08-05-container-e2e.md)
|
||||
|
||||
## Epic outcome
|
||||
|
||||
A tagged image built from `main` runs the API and the worker as separate containers
|
||||
against a mounted library and a persistent data volume, is published to the Gitea
|
||||
registry, is redeployed by webhook, survives restart and upgrade with its database and
|
||||
journals intact, and refuses every request that a loopback deployment would have
|
||||
refused.
|
||||
@@ -1,77 +0,0 @@
|
||||
# E09 — Product Documentation
|
||||
|
||||
Concept phase: none. Like [E08](E08-container-deployment.md), this epic is a delivery
|
||||
addition rather than a product-scope change: the same application, documented well
|
||||
enough that somebody who did not build it can install it, understand it, and operate it
|
||||
without reading the source.
|
||||
|
||||
It does not change the product scope in
|
||||
[`INTEGRATED_PIPELINE_CONCEPT.md`](../INTEGRATED_PIPELINE_CONCEPT.md). No safety
|
||||
invariant moves, no schema changes, no new runtime capability. The one change to the
|
||||
running application is a documentation view and the static assets it needs.
|
||||
|
||||
## Why the application serves its own documentation
|
||||
|
||||
The manuals describe an application that is reached over HTTP behind an access secret.
|
||||
Documentation that lives only in the repository is unreachable from the deployment it
|
||||
describes: an operator who has just been handed a URL and a secret has no Gitea account
|
||||
in front of them. So the same markdown files are both the repository's documentation and
|
||||
the deployment's `/docs` view, and neither is a copy of the other.
|
||||
|
||||
## Decisions made in this epic
|
||||
|
||||
**Markdown is the source.** Everything is written as markdown under `docs/`, so it is
|
||||
reviewable in a diff, readable on Gitea, and renderable in the app. No documentation
|
||||
format that only a tool can read.
|
||||
|
||||
**The renderer is vendored, not written and not fetched.** `marked` (MIT, no
|
||||
dependencies, ships an ES module) is pinned and committed under
|
||||
`frontend/js/vendor/`. A CDN is not an option: the application is deployed to a network
|
||||
whose outbound access is not assumed, and `default-src 'self'` forbids it.
|
||||
|
||||
**Diagrams are mermaid, rendered client-side, with the script boundary intact.** The
|
||||
claim that mermaid requires `'unsafe-eval'` was tested rather than believed: mermaid 11's
|
||||
bundle contains no `eval(` and no `new Function` — only a lodash `Function("return this")`
|
||||
global-detection fallback that short-circuits on `globalThis` and never executes.
|
||||
Rendered under this application's exact CSP, a flowchart produced a 15.8 KB SVG and
|
||||
raised **no `script-src` violation**. What it does raise is `style-src`: mermaid styles
|
||||
its output with an injected `<style>` element and `style=` attributes.
|
||||
|
||||
So `script-src 'self'` stays exactly as US07-02 and US08-01 left it, and `style-src`
|
||||
gains `'unsafe-inline'`. That relaxation is bounded on purpose: with `script-src` intact
|
||||
no injected markup can execute, and with `img-src`, `connect-src`, and `font-src` all
|
||||
still `'self'`, the CSS-based exfiltration channels stay closed. What remains is
|
||||
defacement of a page the operator is already authenticated on.
|
||||
|
||||
The rejected alternative is recorded because it may become the better trade later:
|
||||
pre-rendering each diagram to a committed SVG with the already-installed Playwright
|
||||
(no Node toolchain needed) and serving that, which would leave CSP untouched entirely at
|
||||
the cost of a generator and a freshness check. If `style-src 'unsafe-inline'` is ever
|
||||
judged too much, that is the migration, and it does not change a single markdown file.
|
||||
|
||||
**Screenshots are produced, not pasted.** Every screenshot in the user manual is captured
|
||||
by Playwright from the real application against a temporary fixture library, by the same
|
||||
kind of code that already drives the browser suites. A screenshot nobody can regenerate
|
||||
is a screenshot that silently stops being true.
|
||||
|
||||
**The documentation is held to the code by tests.** Configuration keys, CLI commands,
|
||||
exit codes, API error codes, job and journal states, and module names all appear in both
|
||||
the code and the manuals. Each of those is cross-checked, so the failure mode of stale
|
||||
documentation is a red test rather than a misled operator.
|
||||
|
||||
## Stories
|
||||
|
||||
1. [US09-01 — Serve the documentation inside the application](stories/US09-01-docs-in-app.md)
|
||||
2. [US09-02 — Write the installation and operations manual](stories/US09-02-installation-manual.md)
|
||||
3. [US09-03 — Write the architecture overview](stories/US09-03-architecture-overview.md)
|
||||
4. [US09-04 — Write the user manual with generated screenshots](stories/US09-04-user-manual.md)
|
||||
5. [US09-05 — Automate documentation acceptance](stories/US09-05-docs-gate.md)
|
||||
|
||||
## Epic outcome
|
||||
|
||||
A deployed instance serves its own installation manual, architecture overview, and
|
||||
illustrated user manual at `/app/#/docs`, rendered from the same markdown files that are
|
||||
readable in the repository. Screenshots are regenerated from the running application,
|
||||
every documented command, setting, state, and error code is cross-checked against the
|
||||
code that implements it, and one gate fails the build when documentation and application
|
||||
disagree.
|
||||
@@ -2,14 +2,11 @@
|
||||
|
||||
This backlog decomposes the phases in
|
||||
[`INTEGRATED_PIPELINE_CONCEPT.md`](../INTEGRATED_PIPELINE_CONCEPT.md) into seven
|
||||
epics and small, independently verifiable user stories, plus two delivery-format
|
||||
epics: E08 packages the released application as a deployable container, and E09
|
||||
documents it for the people who install, operate, and use it.
|
||||
epics and small, independently verifiable user stories.
|
||||
|
||||
## Numbering and file naming
|
||||
|
||||
- Epics: `E01` through `E07`, matching concept Phases A through G; `E08` and `E09`
|
||||
have no concept phase and must not change product scope.
|
||||
- Epics: `E01` through `E07`, matching concept Phases A through G.
|
||||
- Stories: `US<epic>-<sequence>`, for example `US03-02`.
|
||||
- Epic files: `E01-<slug>.md`.
|
||||
- Story files: `stories/US01-01-<slug>.md`.
|
||||
@@ -39,8 +36,6 @@ documents it for the people who install, operate, and use it.
|
||||
5. [E05 — Immich upload](E05-immich-upload.md)
|
||||
6. [E06 — Archive lifecycle](E06-archive-lifecycle.md)
|
||||
7. [E07 — Hardening and release](E07-hardening-release.md)
|
||||
8. [E08 — Container deployment](E08-container-deployment.md)
|
||||
9. [E09 — Product documentation](E09-documentation.md)
|
||||
|
||||
## Shared definition of done
|
||||
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
# US08-01 — Make the Trust Boundary Configurable and Authenticated
|
||||
|
||||
Epic: [E08](../E08-container-deployment.md)
|
||||
|
||||
As an operator, I want to reach the application through my own hostname without
|
||||
weakening it, so a container behind a reverse proxy is as safe as the loopback
|
||||
deployment it replaces.
|
||||
|
||||
## Context
|
||||
|
||||
`photo_pipeline/api/security.py` refuses any request whose `Host` or `Origin` is not
|
||||
loopback. That check is the current stand-in for authentication: whoever can reach
|
||||
`127.0.0.1:8000` is the owner. Behind a proxy the hostname is no longer loopback, so
|
||||
relaxing the check without adding an authentication gate would publish the library.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- Allowed hosts and origins come from configuration (`PHOTO_PIPELINE_*`), default to
|
||||
the current loopback set, and an unset configuration behaves exactly as today.
|
||||
- Whenever a non-loopback host is configured, startup requires an access secret and
|
||||
refuses to serve without one; loopback-only deployments keep working with no secret.
|
||||
- The secret is exchanged for the existing session cookie and CSRF token through the
|
||||
bootstrap endpoint; every protected route keeps its current session and CSRF
|
||||
requirements unchanged.
|
||||
- Forwarded headers (`X-Forwarded-Proto`, `X-Forwarded-Host`) are honored only from a
|
||||
configured trusted proxy and ignored otherwise, so a client cannot forge its origin.
|
||||
- Cookies are marked `Secure` when the effective external scheme is HTTPS.
|
||||
- Failed authentication is rate-limited and logged without the secret, the session id,
|
||||
or any request body.
|
||||
- Health endpoints stay reachable without the secret; nothing else does.
|
||||
|
||||
## Automated tests
|
||||
|
||||
- Unit tests for host/origin evaluation across loopback default, configured host,
|
||||
unconfigured host, forged forwarded headers, and trusted-proxy forwarded headers.
|
||||
- Integration tests: startup refusal without a secret, successful exchange, wrong
|
||||
secret, replay of an old session, cross-site request, and unauthenticated access to
|
||||
every route class.
|
||||
|
||||
## Dependencies
|
||||
|
||||
- US07-02
|
||||
@@ -1,41 +0,0 @@
|
||||
# US08-02 — Build a Reproducible Application Image
|
||||
|
||||
Epic: [E08](../E08-container-deployment.md)
|
||||
|
||||
As an operator, I want one image that can run either application role, so deployment is
|
||||
a pull instead of a Python environment I have to reproduce by hand.
|
||||
|
||||
## Context
|
||||
|
||||
The application shells out to `exiftool` and `immich-go`, writes into the library as a
|
||||
normal filesystem user, and serves a static frontend from `frontend/`. All three have to
|
||||
be true inside the image, or the container starts and then fails on the first real
|
||||
operation.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- A `Dockerfile` builds from a pinned Python base, installs the project and its runtime
|
||||
dependencies, and contains no test, playwright, or build-only tooling in the final
|
||||
layer.
|
||||
- `exiftool` and `immich-go` are present at pinned versions, and their versions are
|
||||
recorded in the image and reported by `python -m photo_pipeline diagnostics`.
|
||||
- The image runs as a non-root user whose UID/GID are build-time arguments, so files
|
||||
the application renames or writes keep the ownership the host library expects.
|
||||
- One entrypoint selects the role: `serve` or `worker`, passing through the existing
|
||||
CLI arguments; no supervisor runs two roles in one container.
|
||||
- `serve` containers declare a `HEALTHCHECK` against `/api/v1/health/ready`, so an
|
||||
unmigrated or misconfigured database is not reported healthy.
|
||||
- The image contains no secrets, no library data, no database, and no `.git`; the build
|
||||
context is constrained by `.dockerignore`.
|
||||
- Image build is reproducible from a clean checkout and documented in `README.md`.
|
||||
|
||||
## Automated tests
|
||||
|
||||
- A build-and-run test asserts the image starts, reports ready, serves the frontend
|
||||
index, and returns the pinned `exiftool` and `immich-go` versions.
|
||||
- A test asserts the container refuses to run as UID 0 and that a file created by the
|
||||
container is owned by the configured UID/GID.
|
||||
|
||||
## Dependencies
|
||||
|
||||
- US07-05
|
||||
@@ -1,47 +0,0 @@
|
||||
# US08-03 — Compose the Runtime and Mount the Library Safely
|
||||
|
||||
Epic: [E08](../E08-container-deployment.md)
|
||||
|
||||
As an operator, I want a single compose file that runs the API and the worker against my
|
||||
real library, so a deployment is one command and the safety invariants survive it.
|
||||
|
||||
## Context
|
||||
|
||||
The library process lock (US07-05) assumes both roles see the same lock file, and SQLite
|
||||
in WAL mode assumes a real local filesystem. Container path policy is the same problem
|
||||
as host path policy with a new failure mode: the configured library roots must name the
|
||||
in-container mount paths, not the host paths.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- `docker-compose.yml` runs exactly one `serve` and one `worker` container from the same
|
||||
image and the same data volume, and a second worker is refused by the existing lock
|
||||
rather than by convention.
|
||||
- The library is a bind mount; `PHOTO_PIPELINE_LIBRARY_ROOTS` names the container-side
|
||||
paths, and a mismatch between mounted and configured roots fails at startup with a
|
||||
clear message instead of at the first write.
|
||||
- The data volume holds the database, WAL, thumbnail cache, and backups on a local
|
||||
filesystem; the composition documents that a network mount is unsupported for it.
|
||||
- Migrations run before `serve` and `worker` accept work, using the existing backup-then-
|
||||
migrate path, and an upgrade that fails leaves the previous database intact.
|
||||
- Configuration and secrets come from the environment, never from the image or a
|
||||
committed file; a `.env.example` lists every `PHOTO_PIPELINE_*` variable with safe
|
||||
defaults and no values.
|
||||
- The API port is published to host loopback by default; exposing it publicly requires
|
||||
the configured hostname and access secret from US08-01.
|
||||
- Containers restart automatically, and a restart mid-job resumes exactly as a host
|
||||
restart does today.
|
||||
- Backup, verify-backup, restore, and diagnostics are documented as container commands
|
||||
and work against the mounted volumes.
|
||||
|
||||
## Automated tests
|
||||
|
||||
- An integration test brings the composition up against a temporary fixture library,
|
||||
runs a job, restarts both containers, and asserts the job resumes and the database is
|
||||
intact.
|
||||
- Tests for: second worker refused, library-root mismatch refused at startup, failed
|
||||
migration leaving the previous database restorable.
|
||||
|
||||
## Dependencies
|
||||
|
||||
- US08-01, US08-02
|
||||
@@ -1,40 +0,0 @@
|
||||
# US08-04 — Publish and Deploy from Gitea Actions
|
||||
|
||||
Epic: [E08](../E08-container-deployment.md)
|
||||
|
||||
As a release owner, I want `main` to build, publish, and redeploy the image
|
||||
automatically, so deployment is the same reproducible path every time.
|
||||
|
||||
## Context
|
||||
|
||||
The workflow is adapted from the `crowdsec-admin` deployment workflow
|
||||
(`.gitea/workflows/deploy.yml` in that repository): build, log in to the Gitea registry,
|
||||
push, trigger a Portainer webhook, prune. This project needs the same shape plus a test
|
||||
gate, because unlike that project it has a required suite that must not be skipped.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- `.gitea/workflows/` contains a test workflow that runs on pull requests and on `main`,
|
||||
executing the configured required suites, and a deploy workflow that runs only after
|
||||
the tests pass on `main` and on manual dispatch.
|
||||
- The deploy workflow publishes to `git.domverse-berlin.eu` under this project's own
|
||||
image path, tagged `latest` and the commit SHA, so a rollback is a tag change.
|
||||
- Registry credentials and the Portainer webhook come from repository secrets; runtime
|
||||
secrets (vision key, Immich key, access secret) stay in the Portainer stack and never
|
||||
enter the repository or the image.
|
||||
- Redeploy is triggered by webhook and the workflow fails when the webhook call fails.
|
||||
- Dangling images are pruned; published tags are not.
|
||||
- A concurrency guard prevents two deploys of different commits overlapping.
|
||||
- `README.md` documents the required secrets, the image path, the rollback procedure,
|
||||
and that the stack is managed by Portainer from git.
|
||||
|
||||
## Automated tests
|
||||
|
||||
- Workflow files are validated (syntax and required job/step names) by a repository test
|
||||
so a rename cannot silently disable the test gate.
|
||||
- A dry-run job builds and pushes to a scratch tag on manual dispatch without touching
|
||||
`latest` or triggering a redeploy.
|
||||
|
||||
## Dependencies
|
||||
|
||||
- US08-02, US08-03
|
||||
@@ -1,30 +0,0 @@
|
||||
# US08-05 — Automate Container Deployment Acceptance
|
||||
|
||||
Epic: [E08](../E08-container-deployment.md)
|
||||
|
||||
As a release owner, I want one automated gate that proves the deployed container, so the
|
||||
packaged application is verified the same way the host application is.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- One documented command provisions the composition from the built image against a
|
||||
temporary fixture library and an isolated data volume, and destroys it afterwards.
|
||||
- A browser journey against the containerized application covers discovery, duplicate
|
||||
review, analysis, album proposal, rename, upload preflight, and archive views.
|
||||
- An upgrade journey runs the previous published image, then the new one, and asserts
|
||||
migrations, journals, jobs, and the thumbnail cache survive.
|
||||
- A restart journey kills both containers mid-job and asserts resume without duplicate
|
||||
side effects.
|
||||
- Security gates run against the deployed instance: unauthenticated access refused,
|
||||
forged forwarded headers refused, paths outside the mounted library roots refused, and
|
||||
no secret in container logs.
|
||||
- Evidence is retained per run and the gate fails on any skipped required check.
|
||||
|
||||
## Automated tests
|
||||
|
||||
- The container acceptance suite runs on a `phase_h` marker in CI on `main` and before a
|
||||
published deploy; earlier epic suites keep running unchanged.
|
||||
|
||||
## Dependencies
|
||||
|
||||
- US08-01 through US08-04
|
||||
@@ -1,49 +0,0 @@
|
||||
# US09-01 — Serve the Documentation Inside the Application
|
||||
|
||||
Epic: [E09](../E09-documentation.md)
|
||||
|
||||
As an operator who was handed a URL and an access secret, I want the manuals inside the
|
||||
application I am looking at, so that understanding it does not require a repository
|
||||
checkout or an internet connection.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- Documentation lives as markdown under `docs/`, with an index that names every page and
|
||||
the order it is meant to be read in. The same files render on Gitea without
|
||||
modification.
|
||||
- The application serves a `/docs` view reachable from the main navigation, listing the
|
||||
pages and rendering the selected one.
|
||||
- Rendering uses a vendored, version-pinned markdown library committed under
|
||||
`frontend/js/vendor/`. It is not fetched from a CDN, not bundled by a build step, and
|
||||
not written by hand. The build records the version and a checksum of the vendored file.
|
||||
- Diagrams written as ```mermaid``` fenced blocks render as diagrams. Mermaid is vendored
|
||||
the same way.
|
||||
- `script-src` in the Content-Security-Policy is unchanged: no `'unsafe-eval'`, no
|
||||
`'unsafe-inline'`. Only `style-src` gains `'unsafe-inline'`, and the reason is recorded
|
||||
where the policy is defined.
|
||||
- Links between documents work in the app: a relative `../foo.md#section` link navigates
|
||||
to that page and heading rather than downloading a file or leaving the application.
|
||||
Every heading has a stable anchor, and a deep link to an anchor scrolls to it.
|
||||
- An unknown page renders a documentation-specific not-found message with a link back to
|
||||
the index, and never exposes a filesystem path.
|
||||
- Documentation is readable without an access secret **or** behind the same session as
|
||||
the rest of the application — whichever is chosen is stated explicitly in the story's
|
||||
implementation notes and covered by a test, because an operator locked out by a
|
||||
configuration mistake is exactly who needs the troubleshooting page.
|
||||
- The view works with no outbound network access at all.
|
||||
|
||||
## Automated tests
|
||||
|
||||
- Unit: heading-anchor slugs (duplicates, punctuation, non-ASCII), and the rewriting of
|
||||
relative markdown links into in-app routes.
|
||||
- Integration: every markdown file under `docs/` is reachable from the index; every
|
||||
internal link in every document resolves to a file and, where an anchor is given, to a
|
||||
heading that exists; the vendored library files match their recorded checksums.
|
||||
- Browser: the documentation view renders a page, follows an internal link, deep-links to
|
||||
an anchor, renders a mermaid diagram to an SVG, and shows the not-found message for an
|
||||
unknown page — all with **zero console errors and zero CSP violations**, asserted, not
|
||||
eyeballed.
|
||||
|
||||
## Dependencies
|
||||
|
||||
- None beyond the delivered application.
|
||||
@@ -1,46 +0,0 @@
|
||||
# US09-02 — Write the Installation and Operations Manual
|
||||
|
||||
Epic: [E09](../E09-documentation.md)
|
||||
|
||||
As somebody installing this application for the first time, I want one manual that takes
|
||||
me from nothing to a running instance pointed at my photo library, so that I do not have
|
||||
to reconstruct the procedure from the README, the compose file, and the test suite.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- A **host installation** path: prerequisites and their versions (Python, exiftool,
|
||||
immich-go, Playwright's browser for the test suite), virtual environment, dependency
|
||||
install, `.env`, database migration, starting `serve` and `worker`, and how to verify
|
||||
the install succeeded.
|
||||
- A **container installation** path: the published image, the compose file, the data
|
||||
volume, the library bind mount and why it is mounted the way it is, published ports,
|
||||
running behind a reverse proxy with `PHOTO_PIPELINE_ALLOWED_HOSTS` and
|
||||
`PHOTO_PIPELINE_TRUSTED_PROXIES`, and the Portainer/webhook deployment already in use.
|
||||
- A **configuration reference**: every `PHOTO_PIPELINE_*` setting with its meaning,
|
||||
default, accepted values, and whether it is a secret. Secrets are described, never
|
||||
exemplified with a real-looking value.
|
||||
- A **first run checklist** that ends in a verified state: library discovered, worker
|
||||
claiming jobs, readiness endpoint green, diagnostics clean.
|
||||
- **Operations**: upgrading (including what the migration backup does), backup, verify,
|
||||
restore, pruning, diagnostics, and the log locations for each role.
|
||||
- **Troubleshooting**: each refusal an operator can hit at startup — every non-zero exit
|
||||
code of the CLI, the trust-boundary refusal, the unmounted library root refusal, the
|
||||
library lock being held, and a legacy CLI still running — with the cause and the fix.
|
||||
- The manual states the safety invariants an installer must not work around: one worker
|
||||
writes, `_IGNORE/` is never read, EXIF is verified before upload, and the library lock
|
||||
is authoritative.
|
||||
|
||||
## Automated tests
|
||||
|
||||
- Every `PHOTO_PIPELINE_*` setting documented exists as a field on `Config`, and every
|
||||
field on `Config` is documented — both directions, so a new setting cannot ship
|
||||
undocumented.
|
||||
- Every CLI command and flag named in the manual exists in the argument parser, and every
|
||||
subcommand the parser accepts appears in the manual.
|
||||
- Every exit code documented is one the CLI can actually return.
|
||||
- No documented example value collides with a real secret pattern (the repository's own
|
||||
credential scanner runs over `docs/`).
|
||||
|
||||
## Dependencies
|
||||
|
||||
- US09-01
|
||||
@@ -1,47 +0,0 @@
|
||||
# US09-03 — Write the Architecture Overview
|
||||
|
||||
Epic: [E09](../E09-documentation.md)
|
||||
|
||||
As a developer or reviewer new to this repository, I want an architecture overview that
|
||||
explains what the pieces are and which rules they enforce, so that I can find the right
|
||||
module and not violate an invariant I never knew existed.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- A **context diagram**: the operator, the browser application, the API and worker, the
|
||||
photo library, the Immich server, the vision provider, and the archive destination —
|
||||
with the direction and nature of every interaction, including which ones leave the
|
||||
machine.
|
||||
- A **runtime diagram**: the migrate/api/worker composition, the data volume, the library
|
||||
bind mount, the SQLite database, and the process lock — showing what is shared and what
|
||||
is exclusive.
|
||||
- A **module map**: every package under `photo_pipeline/` with its responsibility and the
|
||||
boundary it must not cross (which modules may touch the filesystem, which may call an
|
||||
external provider, which own schema).
|
||||
- **Key flows**, each as a diagram plus prose: the durable job lifecycle from enqueue to
|
||||
recovery; the rename journal state machine including the rollback and manual-resolution
|
||||
paths; the upload lifecycle through preflight, run, report ingestion, and verification.
|
||||
- **The invariants and where they are enforced**, each pointing at the module that owns
|
||||
it: one writer at a time, the library process lock, the path policy and library roots,
|
||||
`_IGNORE/` exclusion, verified EXIF before upload, safety decisions gating the vision
|
||||
provider, and restart-safety of every mutation.
|
||||
- **The data model**: the tables, what identity means for an asset, and how a moved file
|
||||
keeps its identity.
|
||||
- A short **history and donor** section: what was migrated out of the legacy CLIs, where
|
||||
the archive and its ledger live, and why they are kept.
|
||||
- Diagrams are ```mermaid``` blocks so the source is diffable and Gitea renders them
|
||||
natively.
|
||||
|
||||
## Automated tests
|
||||
|
||||
- Every package and top-level module under `photo_pipeline/` appears in the module map,
|
||||
and every module named in the map exists — a new service cannot appear on the map's
|
||||
blind side.
|
||||
- Every job state, journal state, and upload batch state named in the document exists in
|
||||
the code, and every state the code defines is named in the document.
|
||||
- Every mermaid block parses (rendered in the browser test without an error node).
|
||||
- Every code path referenced by file name exists.
|
||||
|
||||
## Dependencies
|
||||
|
||||
- US09-01
|
||||
@@ -1,52 +0,0 @@
|
||||
# US09-04 — Write the User Manual with Generated Screenshots
|
||||
|
||||
Epic: [E09](../E09-documentation.md)
|
||||
|
||||
As the person actually sorting a photo library, I want a manual that walks the workflow
|
||||
screen by screen and tells me what each refusal means, so that I can use the application
|
||||
confidently and know what it is about to do to my files before it does it.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- One page per workflow stage, in the order the application presents them: discovery and
|
||||
inventory, duplicate review, safety review, analysis, album proposals, renames, upload,
|
||||
archive, and diagnostics. Each page answers the same four questions: what this stage is
|
||||
for, what I have to decide, **what it changes on disk or on the server**, and what it
|
||||
refuses to do.
|
||||
- A **guided first pass** that takes a new library from scan to a verified upload, naming
|
||||
the point of no return in each stage and what is reversible after it.
|
||||
- Every stage page carries at least one screenshot of the real application showing the
|
||||
state being described.
|
||||
- Screenshots are **generated** by a committed Playwright script that seeds a temporary
|
||||
fixture library, drives the application, and writes the images. Regenerating them is one
|
||||
documented command. No screenshot is captured by hand.
|
||||
- Screenshots contain no real library paths, no personal photos, and no secrets — the
|
||||
fixture library is synthetic, and this is asserted rather than assumed.
|
||||
- An **error and refusal catalogue**: every `error.code` the API can return, with what
|
||||
causes it, what the application did or refused to do, and what the operator should do
|
||||
next. Refusals that protect data (`lock_held`, `rename_recovery_required`,
|
||||
`stale_preflight`, `path_not_allowed`, `host_not_allowed`, `dry_run_not_approved`,
|
||||
conflict codes) are explained as intentional, not as faults.
|
||||
- A **recovery** page: an interrupted rename, an uncertain upload, a failed migration, a
|
||||
restored backup — what the application does by itself and what needs a decision.
|
||||
- The work-item safety allow list is extended to permit `docs/images/**`, since the
|
||||
repository denies image files by default. This is an explicit, reviewed change, not a
|
||||
quiet one, and it stays narrow enough that a real photo still cannot be committed.
|
||||
|
||||
## Automated tests
|
||||
|
||||
- Every `error.code` the application can emit is documented, and every code documented
|
||||
exists in the code — both directions.
|
||||
- Every image referenced by a documentation page exists, and every image under
|
||||
`docs/images/` is referenced by a page.
|
||||
- The screenshot generator runs end to end in the test environment and produces every
|
||||
image the manual references, against a temporary fixture library that is destroyed
|
||||
afterwards.
|
||||
- Generated screenshots are checked for library paths outside the fixture root and for
|
||||
the configured secrets.
|
||||
- Browser: each stage page renders in the documentation view with its screenshot loaded
|
||||
and no console error.
|
||||
|
||||
## Dependencies
|
||||
|
||||
- US09-01
|
||||
@@ -1,38 +0,0 @@
|
||||
# US09-05 — Automate Documentation Acceptance
|
||||
|
||||
Epic: [E09](../E09-documentation.md)
|
||||
|
||||
As a release owner, I want one gate that proves the documentation still describes the
|
||||
application, so that a change to the code cannot quietly make the manuals wrong.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- One documented command runs every documentation check and retains its evidence, in the
|
||||
shape the release and container gates already use.
|
||||
- The gate fails when: an internal link or anchor is dead; a page is unreachable from the
|
||||
index; an image is referenced but missing or present but unreferenced; a documented
|
||||
setting, command, exit code, error code, or state does not exist in the code; a code
|
||||
path exists that the documentation is required to cover and does not.
|
||||
- The gate regenerates the screenshots and fails when a regenerated image no longer
|
||||
matches the committed one beyond a stated tolerance, so a UI change that invalidates the
|
||||
manual is a red build rather than a discovery months later.
|
||||
- The gate renders every documentation page in a real browser and fails on any console
|
||||
error or CSP violation, and asserts that `script-src` contains neither `'unsafe-eval'`
|
||||
nor `'unsafe-inline'`.
|
||||
- The checks run on a `phase_i` marker; CI runs the gate, and the earlier epic suites keep
|
||||
running unchanged.
|
||||
- The README and the application's documentation index point at each other, so neither is
|
||||
the forgotten copy.
|
||||
- Documentation stories are mapped in the story traceability matrix like every other
|
||||
story.
|
||||
|
||||
## Automated tests
|
||||
|
||||
- The gate's own contract is testable without a browser: a seeded broken link, a missing
|
||||
image, an undocumented error code, and a stale screenshot each fail it, and a clean tree
|
||||
passes.
|
||||
- The full documentation suite runs on `phase_i` in CI.
|
||||
|
||||
## Dependencies
|
||||
|
||||
- US09-01 through US09-04
|
||||
@@ -1,105 +0,0 @@
|
||||
# The deployed runtime (US08-03): one API, one worker, one library, one volume.
|
||||
#
|
||||
# The same image (US08-02) runs both roles, so what is composed here is process
|
||||
# topology, not a second application. Three invariants shape it:
|
||||
#
|
||||
# * one writer — `serve` and `worker` each take the library process lock for their
|
||||
# role (US07-05), and both containers mount the *same* data volume, which is what
|
||||
# makes the lock file visible to both. Scaling `worker` past 1 is refused by that
|
||||
# lock rather than by anyone remembering not to;
|
||||
# * one local filesystem — the database, its write-ahead log, the thumbnail cache,
|
||||
# and the backups live in the `data` volume, and SQLite in WAL mode requires real
|
||||
# local-filesystem locking. A network mount (NFS, SMB, or a cloud volume driver)
|
||||
# is unsupported for it; that is a corrupted database, not a slow one;
|
||||
# * one library path vocabulary — `PHOTO_PIPELINE_LIBRARY_ROOTS` names the path
|
||||
# *inside* the container, which is also the bind mount's target below. A root
|
||||
# that is not mounted there makes `serve` and `worker` refuse at startup instead
|
||||
# of writing into the container's throwaway layer.
|
||||
#
|
||||
# Configuration and secrets come from the environment only: copy `.env.example` to
|
||||
# `.env` and fill it in. Nothing is baked into the image and nothing with a value in
|
||||
# it is committed.
|
||||
#
|
||||
# cp .env.example .env && $EDITOR .env
|
||||
# docker compose up -d --build
|
||||
#
|
||||
# Operating it is operating the same CLI — `docker compose run --rm --no-deps api
|
||||
# <command>` — see README, "Composed runtime".
|
||||
|
||||
name: photo-pipeline
|
||||
|
||||
x-runtime: &runtime
|
||||
image: ${PHOTO_PIPELINE_IMAGE:-photo-pipeline:dev}
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
# Everything the app renames or rewrites has to stay owned by the host user
|
||||
# the library already belongs to.
|
||||
UID: ${PHOTO_PIPELINE_UID:-1000}
|
||||
GID: ${PHOTO_PIPELINE_GID:-1000}
|
||||
user: "${PHOTO_PIPELINE_UID:-1000}:${PHOTO_PIPELINE_GID:-1000}"
|
||||
env_file:
|
||||
- ${PHOTO_PIPELINE_ENV_FILE:-.env}
|
||||
volumes:
|
||||
- data:/data
|
||||
- "${PHOTO_PIPELINE_LIBRARY_HOST_PATH:?set PHOTO_PIPELINE_LIBRARY_HOST_PATH to the photo library on this host}:${PHOTO_PIPELINE_LIBRARY_ROOTS:?set PHOTO_PIPELINE_LIBRARY_ROOTS to the container-side library path}"
|
||||
# Jobs check for cancellation between items and leave a resumable record; a
|
||||
# too-short grace period turns an orderly stop into a recovery on next start.
|
||||
stop_grace_period: 30s
|
||||
|
||||
x-environment: &environment
|
||||
# Set here rather than left to the file: these two are what the composition itself
|
||||
# promises, and an `.env` that disagreed would move the database off the volume or
|
||||
# the library off its mount.
|
||||
PHOTO_PIPELINE_DATA_DIR: /data
|
||||
PHOTO_PIPELINE_LIBRARY_ROOTS: ${PHOTO_PIPELINE_LIBRARY_ROOTS}
|
||||
|
||||
services:
|
||||
# Migrations run to completion before either role accepts work, through the same
|
||||
# backup-then-migrate path the roles use (US07-05): a pending upgrade is snapshotted
|
||||
# first, and a failed one exits non-zero with the backup named — so `api` and
|
||||
# `worker` never start, and the previous database is left intact and restorable.
|
||||
migrate:
|
||||
<<: *runtime
|
||||
command: ["migrate"]
|
||||
environment: *environment
|
||||
restart: "no"
|
||||
|
||||
api:
|
||||
<<: *runtime
|
||||
command: ["serve"]
|
||||
environment:
|
||||
<<: *environment
|
||||
# Published to host loopback below. Inside the container the server must bind
|
||||
# the container's own interface for that publish to reach it, which is exactly
|
||||
# what makes the access secret mandatory (US08-01) — `serve` refuses to start
|
||||
# without one. Exposing the port beyond loopback additionally needs
|
||||
# PHOTO_PIPELINE_ALLOWED_HOSTS to name the hostname it is reached under.
|
||||
PHOTO_PIPELINE_HOST: 0.0.0.0
|
||||
PHOTO_PIPELINE_PORT: 8000
|
||||
ports:
|
||||
# Host side only: PHOTO_PIPELINE_PORT in `.env` moves the *published* port, and
|
||||
# the container always serves 8000, which is what the image's health check probes.
|
||||
- "${PHOTO_PIPELINE_PUBLISH_ADDRESS:-127.0.0.1}:${PHOTO_PIPELINE_PORT:-8000}:8000"
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
|
||||
# One worker. A second one is refused by the library lock in the shared data
|
||||
# volume, which is the point: `docker compose up --scale worker=2` fails loudly
|
||||
# instead of running two writers against one library.
|
||||
worker:
|
||||
<<: *runtime
|
||||
command: ["worker", "--id", "worker-1"]
|
||||
environment: *environment
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
# Local driver on purpose: the database, WAL, thumbnail cache, and backups need a
|
||||
# real local filesystem. Do not point this at NFS, SMB, or a network volume driver.
|
||||
data:
|
||||
driver: local
|
||||
@@ -1,22 +0,0 @@
|
||||
#!/bin/sh
|
||||
# One entrypoint, one role per container (US08-02).
|
||||
#
|
||||
# The first argument is the management command the image runs — `serve` and `worker`
|
||||
# are the two roles, and every other `python -m photo_pipeline` command (migrate,
|
||||
# diagnostics, backup, restore, dry-run) is passed through unchanged so operating the
|
||||
# container is operating the same CLI. No supervisor: two roles in one container would
|
||||
# share a process lock they are each meant to hold alone (US07-05).
|
||||
set -eu
|
||||
|
||||
if [ "$(id -u)" = "0" ]; then
|
||||
echo "refusing to run as root: start this image with a non-root UID/GID so files" \
|
||||
"it renames or writes keep the ownership the mounted library expects" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
role="${1:-serve}"
|
||||
# The health check has to know which role it is checking, and only the API has an
|
||||
# endpoint to check. /tmp is writable for the unprivileged user; /run may not be.
|
||||
printf '%s' "${role}" > "${PHOTO_PIPELINE_ROLE_FILE:-/tmp/photo-pipeline-role}" 2>/dev/null || true
|
||||
|
||||
exec python -m photo_pipeline "$@"
|
||||
@@ -1,72 +0,0 @@
|
||||
"""Download one pinned immich-go release and verify it before unpacking (US08-02).
|
||||
|
||||
Run at image build time by the `uploader` stage, with the interpreter that is already
|
||||
in the base image: no curl, no wget, and no download tooling in the layer that ships.
|
||||
The checksum is not advisory — a release asset that does not match the pinned digest
|
||||
is a failed build, not a warning, because the uploader's flags and report format are
|
||||
what the upload parser is written against (concept §15).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import platform
|
||||
import tarfile
|
||||
import tempfile
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
RELEASE_URL = "https://github.com/simulot/immich-go/releases/download/v{version}/{asset}"
|
||||
# Debian/BuildKit architecture as the interpreter sees it → release asset name.
|
||||
ASSETS = {
|
||||
"x86_64": ("immich-go_Linux_x86_64.tar.gz", "amd64"),
|
||||
"amd64": ("immich-go_Linux_x86_64.tar.gz", "amd64"),
|
||||
"aarch64": ("immich-go_Linux_arm64.tar.gz", "arm64"),
|
||||
"arm64": ("immich-go_Linux_arm64.tar.gz", "arm64"),
|
||||
}
|
||||
TIMEOUT_SECONDS = 300
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--version", required=True, help="immich-go release, without the v")
|
||||
parser.add_argument("--sha256-amd64", required=True)
|
||||
parser.add_argument("--sha256-arm64", required=True)
|
||||
parser.add_argument("--into", default="/usr/local/bin")
|
||||
args = parser.parse_args()
|
||||
|
||||
machine = platform.machine().lower()
|
||||
if machine not in ASSETS:
|
||||
raise SystemExit(f"unsupported architecture: {machine}")
|
||||
asset, arch = ASSETS[machine]
|
||||
expected = {"amd64": args.sha256_amd64, "arm64": args.sha256_arm64}[arch]
|
||||
url = RELEASE_URL.format(version=args.version, asset=asset)
|
||||
|
||||
with urllib.request.urlopen(url, timeout=TIMEOUT_SECONDS) as response: # noqa: S310
|
||||
payload = response.read()
|
||||
digest = hashlib.sha256(payload).hexdigest()
|
||||
if digest != expected:
|
||||
raise SystemExit(f"checksum mismatch for {url}: {digest} != {expected}")
|
||||
|
||||
target = Path(args.into)
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory() as work:
|
||||
archive = Path(work) / asset
|
||||
archive.write_bytes(payload)
|
||||
with tarfile.open(archive) as tar:
|
||||
member = tar.getmember("immich-go")
|
||||
# Extract exactly the one file this pin is about, by name, so nothing
|
||||
# else in the archive can decide where it lands.
|
||||
extracted = tar.extractfile(member)
|
||||
if extracted is None:
|
||||
raise SystemExit("release archive contains no immich-go binary")
|
||||
binary = target / "immich-go"
|
||||
binary.write_bytes(extracted.read())
|
||||
binary.chmod(0o755)
|
||||
print(f"immich-go {args.version} ({arch}) verified {digest}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,34 +0,0 @@
|
||||
#!/bin/sh
|
||||
# Container health for the `serve` role: readiness, not liveness (US08-02).
|
||||
#
|
||||
# /api/v1/health/ready is 503 until the database is reachable, migrated, and in WAL
|
||||
# mode with foreign keys on, so an unmigrated or misconfigured container never reports
|
||||
# healthy. Health endpoints need no session and no access secret, which is what lets an
|
||||
# orchestrator restart a container it holds no credentials for (US08-01).
|
||||
set -eu
|
||||
|
||||
role="$(cat "${PHOTO_PIPELINE_ROLE_FILE:-/tmp/photo-pipeline-role}" 2>/dev/null || echo unknown)"
|
||||
if [ "${role}" != "serve" ]; then
|
||||
# ponytail: the worker has no endpoint to probe; its liveness is its lease and job
|
||||
# heartbeat in the database. Add a `worker --health` command if a restart policy
|
||||
# ever needs to act on it.
|
||||
exit 0
|
||||
fi
|
||||
|
||||
port="${PHOTO_PIPELINE_PORT:-8000}"
|
||||
exec python - "${port}" <<'PY'
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
url = f"http://127.0.0.1:{sys.argv[1]}/api/v1/health/ready"
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=5) as response: # noqa: S310 — loopback
|
||||
sys.exit(0 if response.status == 200 else 1)
|
||||
except urllib.error.HTTPError as error:
|
||||
print(f"not ready: HTTP {error.code}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except OSError as error:
|
||||
print(f"not ready: {error}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PY
|
||||
@@ -1,224 +0,0 @@
|
||||
# Architecture
|
||||
|
||||
[← Documentation index](index.md)
|
||||
|
||||
For whoever has to change this code without breaking somebody's photo library. It
|
||||
explains what the pieces are, which rules each one keeps, and where to look when a
|
||||
stage refuses.
|
||||
|
||||
The product decisions behind all of it live in `INTEGRATED_PIPELINE_CONCEPT.md`; this
|
||||
page describes what was built.
|
||||
|
||||
## Context
|
||||
|
||||
Five things outside the application, and what actually crosses each boundary.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
operator([Operator]):::person -->|browser, one session| app
|
||||
app[Photo Pipeline]:::system -->|read, rename, EXIF write| library[(Photo library<br/>bind mount)]
|
||||
app -->|database, cache, journals, backups| data[(Data directory<br/>local filesystem)]
|
||||
app -->|confirmed-SFW images only| vision[Vision provider]:::ext
|
||||
app -->|immich-go, verified bytes| immich[Immich server]:::ext
|
||||
app -->|copy, verify, then remove| archive[(Archive medium)]
|
||||
classDef person fill:#1f6feb,stroke:#58a6ff,color:#fff
|
||||
classDef system fill:#238636,stroke:#3fb950,color:#fff
|
||||
classDef ext fill:#6e40c9,stroke:#a371f7,color:#fff
|
||||
```
|
||||
|
||||
| boundary | leaves the machine? | carries |
|
||||
|---|---|---|
|
||||
| operator → app | no (loopback, or a proxy you configured) | commands against stable ids, never paths |
|
||||
| app → library | no | reads, folder renames, EXIF merges |
|
||||
| app → data directory | no | SQLite in WAL mode, thumbnails, journals, backups |
|
||||
| app → vision provider | **yes** | image bytes of confirmed-SFW canonical assets only |
|
||||
| app → Immich | **yes** | the exact verified bytes of an approved album |
|
||||
| app → archive medium | no | copies, verified before the source is removed |
|
||||
|
||||
## Runtime
|
||||
|
||||
Two long-lived processes and a one-shot migration, sharing one database and one
|
||||
library.
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
browser([Browser]) -->|JSON + SSE| api
|
||||
migrate["migrate<br/>backup, then upgrade"] -->|must exit 0| api
|
||||
migrate --> worker
|
||||
api["api · serve<br/>enqueues, serves, reads"] -->|jobs table| db[(SQLite WAL)]
|
||||
worker["worker<br/>claims and does the work"] -->|jobs table| db
|
||||
worker --> tools["exiftool · vision API · immich-go"]
|
||||
api -.->|api.lock.json| lock{{library lock}}
|
||||
worker -.->|worker.lock.json| lock
|
||||
api --> lib[(library)]
|
||||
worker --> lib
|
||||
```
|
||||
|
||||
`serve` enqueues and renders; **it does not do the work**. Everything that scans,
|
||||
scores, analyses, renames, uploads, or archives happens in the worker, which claims
|
||||
a queued job atomically with a fencing token. One mutating job runs at a time, and
|
||||
the lock file in the data directory is what makes a second worker impossible rather
|
||||
than merely discouraged.
|
||||
|
||||
## The modules
|
||||
|
||||
| package | owns | must not |
|
||||
|---|---|---|
|
||||
| `api/` | HTTP surface, error envelope, security policy | contain SQL or business logic |
|
||||
| `api/routes/` | one module per resource group, all under `/api/v1` | accept a filesystem path from the browser |
|
||||
| `api/security.py` | host/origin/CSRF/session/size policy as one pure `evaluate`, plus the ASGI middleware | be bypassed per-route |
|
||||
| `schemas/` | Pydantic request and response contracts | reach the database |
|
||||
| `services/` | all domain logic; the only place a decision is made | be imported by the frozen CLI archive |
|
||||
| `models/` | SQLAlchemy tables | hold behaviour |
|
||||
| `jobs/` | durable job lifecycle, worker loop, lock ranks, handler registry | run work in the API process |
|
||||
| `integrations/` | the real outside world: `exiftool`, vision, NSFW model, `immich-go` and its report grammars | be called without a version recorded |
|
||||
| `path_policy.py` | the library boundary, `_IGNORE/` exclusion, symlink-escape refusal | be duplicated anywhere |
|
||||
| `imaging.py` | the single bounded-decode door | let a caller open an image directly |
|
||||
| `faults.py` | the crash barriers the tests fire | read anything but its one env var |
|
||||
| `config.py` | typed settings, secrets as `SecretStr` | log or return a value |
|
||||
| `db.py` | engine, sessions, WAL and foreign keys, migration entry | be used to bypass a repository |
|
||||
|
||||
Services worth knowing by name: `inventory`, `duplicates`, `safety`, `analysis`,
|
||||
`albums`/`proposals`/`naming`, `renames`/`rename_apply`/`rename_journal`,
|
||||
`uploads`/`upload_batches`/`upload_reports`/`upload_verification`,
|
||||
`archives`/`archive_transfer`/`archive_journal`/`restores`/`availability`,
|
||||
`thumbnails`, `hashing`, `exif_checkpoint`, `jobs`, `workflow`, `backup`,
|
||||
`diagnostics`, `app_lock`, `release`, `benchmarks`, `library`, `legacy_import`.
|
||||
|
||||
## The state machines
|
||||
|
||||
Three journals decide what a restart is allowed to assume. All three are read from
|
||||
the database plus the real world — never guessed from a missing file.
|
||||
|
||||
### Durable jobs
|
||||
|
||||
```mermaid
|
||||
stateDiagram-v2
|
||||
[*] --> queued
|
||||
queued --> running: claimed with a fencing token
|
||||
queued --> cancelled
|
||||
queued --> cancelling
|
||||
running --> succeeded
|
||||
running --> failed
|
||||
running --> cancelling
|
||||
cancelling --> cancelled
|
||||
failed --> retry_queued
|
||||
retry_queued --> running
|
||||
succeeded --> [*]
|
||||
cancelled --> [*]
|
||||
```
|
||||
|
||||
`succeeded` and `cancelled` are terminal, so a duplicate delivery cannot move a
|
||||
finished job. Claiming is compare-and-set on the row version; a worker whose lease
|
||||
expired cannot commit after another has taken over.
|
||||
|
||||
### The rename journal
|
||||
|
||||
The only state machine that moves somebody's folders.
|
||||
|
||||
```mermaid
|
||||
stateDiagram-v2
|
||||
[*] --> planned
|
||||
planned --> moving
|
||||
moving --> moved
|
||||
moving --> planned: proven untouched
|
||||
moving --> rollback_required
|
||||
moved --> database_updated
|
||||
database_updated --> verified
|
||||
verified --> complete
|
||||
moved --> rollback_required
|
||||
database_updated --> rollback_required
|
||||
verified --> rollback_required
|
||||
planned --> failed
|
||||
failed --> planned
|
||||
rollback_required --> rolled_back
|
||||
complete --> [*]
|
||||
rolled_back --> [*]
|
||||
```
|
||||
|
||||
Intent is written **before** the disk is touched, which is why `moving` can resolve
|
||||
backwards: the evidence decides. `moving`, `moved`, `database_updated`, and
|
||||
`rollback_required` are the unsafe states — while any operation sits in one, the
|
||||
library may be half-renamed, so unrelated mutations are refused with
|
||||
`409 rename_recovery_required` until a person resolves it.
|
||||
|
||||
### Upload batches
|
||||
|
||||
```mermaid
|
||||
stateDiagram-v2
|
||||
[*] --> planned
|
||||
planned --> running
|
||||
running --> succeeded
|
||||
running --> failed
|
||||
running --> cancelling
|
||||
running --> unknown_requires_verification: process died after acceptance
|
||||
cancelling --> cancelled
|
||||
failed --> running: retry
|
||||
cancelled --> running: retry
|
||||
```
|
||||
|
||||
`unknown_requires_verification` is deliberately **not** restartable. The server may
|
||||
already hold the files; the answer is to ask Immich for the recorded SHA-1, not to
|
||||
upload again and hope.
|
||||
|
||||
Archive transfers use the same shape — `planned → transferring → verified →
|
||||
removing → complete` — and the source is removed only after the archived bytes are
|
||||
verified.
|
||||
|
||||
## Identity and the data model
|
||||
|
||||
A path is metadata. The identity is `assets.id`, a UUID that never changes, and
|
||||
`asset_paths` records every path an asset has ever had with the reason it changed.
|
||||
That is what makes a rename cheap: nothing else in the database has to move.
|
||||
|
||||
Tables: `assets`, `asset_paths`, `thumbnails`, `jobs`, `job_items`, `job_events`,
|
||||
`safety_reviews`, `analysis_results`, `exif_projections`, `duplicate_clusters`,
|
||||
`duplicate_members`, `duplicate_negative_links`, `album_proposals`, `rename_plans`,
|
||||
`rename_operations`, `upload_batches`, `upload_items`, `upload_verifications`,
|
||||
`archive_locations`, `archive_plans`, `archive_operations`.
|
||||
|
||||
Availability is independent of workflow progress: `active`, `archiving`,
|
||||
`archived_online`, `archived_offline`, `restoring`, `missing_unexpected`. An
|
||||
unmounted archive disk is `archived_offline`, never `missing` — the scanner is not
|
||||
allowed to conclude that a photo is gone because a disk is unplugged.
|
||||
|
||||
EXIF is a projection with its own verified state (`verified`, `divergent`,
|
||||
`failed`): the desired values are written, read back, and compared, and anything the
|
||||
stage does not own having changed makes the asset `divergent` and blocks the next
|
||||
mutating stage.
|
||||
|
||||
## Where each invariant lives
|
||||
|
||||
| invariant | enforced in |
|
||||
|---|---|
|
||||
| `_IGNORE/` is never traversed, counted, or opened | `path_policy.is_excluded`, used by every discovery path |
|
||||
| no path outside the library roots is reachable | `path_policy.resolve_in_roots` — it returns the resolved path, because validating one name and opening another is the symlink race |
|
||||
| one writer per library | `services/app_lock.py` (an `flock` on a JSON lock file, per role) |
|
||||
| one mutating job at a time | `services/jobs.py` lock keys plus `jobs/locks.py` rank ordering |
|
||||
| only confirmed-SFW assets reach the vision provider | `services/analysis.py`, re-checked *after* the provider call so a decision that flipped mid-flight discards the result |
|
||||
| EXIF is verified, and other fields preserved | `services/exif_checkpoint.py` |
|
||||
| uploads carry the exact verified bytes | `services/uploads.py` preflight, re-proved immediately before the uploader runs |
|
||||
| an uncertain upload is not a failure | `services/upload_batches.py`, `services/upload_verification.py` |
|
||||
| the archive source outlives its copy until verified | `services/archive_transfer.py` |
|
||||
| decoding is bounded | `imaging.py` |
|
||||
| the trust boundary and CSRF | `api/security.py` |
|
||||
|
||||
## Concurrency
|
||||
|
||||
Locks are taken broad to narrow — `library → stage/job → album/folder → asset` — and
|
||||
never the other way, which is what makes deadlock structural rather than lucky.
|
||||
Ranks live in `jobs/locks.py`.
|
||||
|
||||
Exactly-once execution is not achievable across SQLite, a filesystem, subprocesses,
|
||||
and a remote server. The design promises **at-least-once with idempotent recovery**:
|
||||
every handler may run twice, and running twice must not produce two side effects.
|
||||
That is why the rename journal records intent before moving, why EXIF writes merge
|
||||
and verify, and why upload retries consult both local history and Immich.
|
||||
|
||||
## History
|
||||
|
||||
This application was extracted from two command-line tools rather than written from
|
||||
nothing. Their sources are frozen in `legacy_cli_archive/` with the ledger mapping
|
||||
each donated behaviour to the service that now owns it, the characterization tests
|
||||
that pinned it, and every intentional difference. Production code must not import
|
||||
them; they are provenance and rollback evidence.
|
||||
@@ -1,95 +0,0 @@
|
||||
# Errors and refusals
|
||||
|
||||
[← Documentation index](index.md)
|
||||
|
||||
Every error the API returns carries a code:
|
||||
|
||||
```json
|
||||
{"error": {"code": "lock_held", "message": "a library_write job is already running"}}
|
||||
```
|
||||
|
||||
Most of them are **refusals, not faults**. This application would rather stop and
|
||||
explain than guess about somebody's photographs, so a code below usually means it
|
||||
protected something. Each row says what caused it and what to do.
|
||||
|
||||
## Access and the trust boundary
|
||||
|
||||
| code | cause | what to do |
|
||||
|---|---|---|
|
||||
| `unauthenticated` | no application session | reload the page; the browser bootstraps one |
|
||||
| `access_denied` | wrong or missing access secret | check `PHOTO_PIPELINE_ACCESS_SECRET`. Attempts are logged with the caller's address only |
|
||||
| `too_many_attempts` | more than five failed secret attempts in a minute | wait. This is the rate limit, not a lockout |
|
||||
| `csrf_failed` | a mutation without the session's token | reload; a stale tab has an old token |
|
||||
| `host_not_allowed` | the `Host` header is not a configured name | add the real hostname to `PHOTO_PIPELINE_ALLOWED_HOSTS` |
|
||||
| `origin_not_allowed` | the request came from another origin | not something a browser tab of this app produces |
|
||||
| `cross_site_blocked` | another site triggered the request | expected — this is the protection working |
|
||||
| `payload_too_large` | the body exceeds `PHOTO_PIPELINE_MAX_REQUEST_BYTES` | send less; every endpoint takes small commands |
|
||||
| `path_not_allowed` | a path resolved outside the library roots | usually a symlink. Nothing outside the roots is reachable, by design |
|
||||
|
||||
## The safety gate
|
||||
|
||||
| code | cause | what to do |
|
||||
|---|---|---|
|
||||
| `dry_run_not_approved` | mutation is gated until a dry run is approved | run `dry-run`, read it, then `approve-dry-run` |
|
||||
| `approval_scope_mismatch` | the approval covers different library roots | approve a report for the roots actually configured |
|
||||
| `approval_unreadable` | the approval record cannot be read | re-approve; do not edit it by hand |
|
||||
|
||||
## Concurrency and staleness
|
||||
|
||||
| code | cause | what to do |
|
||||
|---|---|---|
|
||||
| `lock_held` | a mutating job already holds the lane | wait for it. One at a time is what makes a crash recoverable |
|
||||
| `version_conflict` | you acted on a version that changed underneath you | the view re-renders with the server's truth; decide again |
|
||||
| `invalid_transition` | a state change that the machine does not allow | usually a stale tab; reload |
|
||||
| `job_error` | the job service refused the request | the message says why |
|
||||
| `rename_recovery_required` | an interrupted rename is unresolved | resolve it in [Renames](stages/renames.md). Unrelated mutations stay blocked on purpose |
|
||||
|
||||
## Stage refusals
|
||||
|
||||
| code | cause | what to do |
|
||||
|---|---|---|
|
||||
| `nothing_to_score` | no photo is eligible for safety scoring | the queue is already decided |
|
||||
| `nothing_eligible` | no photo is eligible for analysis | resolve safety decisions first |
|
||||
| `nothing_to_plan` | no approved album name to build a plan from | approve a proposal first |
|
||||
| `invalid_decision` | the decision is not one this cluster accepts | reload the cluster |
|
||||
| `invalid_proposal` | the name is empty, reserved, or has forbidden characters | fix the name; the rules are in [Album proposals](stages/albums.md) |
|
||||
| `unknown_album` | the album is not a folder the library knows | rescan |
|
||||
| `cannot_apply` | the plan is invalid, stale, or another plan is unresolved | read the blockers in the preview |
|
||||
| `cannot_rollback` | the operation is complete or its preconditions no longer hold | rollback is recovery, not undo |
|
||||
|
||||
## Upload
|
||||
|
||||
| code | cause | what to do |
|
||||
|---|---|---|
|
||||
| `stale_preflight` | bytes changed between approval and running | re-run preflight; this is the check that stops the wrong bytes being uploaded |
|
||||
| `not_runnable` | the batch is in a state that must not be re-run | an uncertain batch is verified, never retried |
|
||||
| `requires_verification` | the outcome is unknown; the server may hold the files | verify against Immich, then resolve |
|
||||
| `changed_after_upload` | the file changed after a successful upload | uploading again may create or upgrade an asset |
|
||||
|
||||
## Operations
|
||||
|
||||
| code | cause | what to do |
|
||||
|---|---|---|
|
||||
| `backup_failed` | the snapshot could not be taken or verified | check free space and the message |
|
||||
| `invalid_retention` | a `--keep` value that is not a positive count | the newest backup is never pruned |
|
||||
| `not_found` | no such id | usually a stale link |
|
||||
|
||||
## Generic
|
||||
|
||||
| code | cause |
|
||||
|---|---|
|
||||
| `invalid_request` | the request body failed validation. Field names only — never the values, which end up in logs and screenshots |
|
||||
| `http_error` | a plain HTTP-level refusal, with its status |
|
||||
| `internal_error` | an unhandled error. The code is all you get; the traceback is in the server log, because it can carry paths and credentials |
|
||||
|
||||
## Diagnostics warnings
|
||||
|
||||
Not errors — reported by `diagnostics` and worth acting on. They are listed with
|
||||
their meanings in [Diagnostics](stages/diagnostics.md): `disk_low`, `disk_critical`,
|
||||
`cache_over_quota`, `wal_growth`, `tool_version_drift`, `legacy_process_active`, and
|
||||
`no_roots`.
|
||||
|
||||
## Startup refusals
|
||||
|
||||
Exit codes 2 to 5 are covered by the installation manual under
|
||||
[when it refuses to start](installation.md#when-it-refuses-to-start).
|
||||
@@ -1,66 +0,0 @@
|
||||
# A guided first pass
|
||||
|
||||
[← Documentation index](index.md)
|
||||
|
||||
One library, from nothing to a verified upload. Each stage links to its own page; this
|
||||
is the order, and — more importantly — where the points of no return are.
|
||||
|
||||
|
||||
|
||||
The workflow page is the home view and the honest summary: each stage shows its state,
|
||||
its counts, why it is blocked if it is, and the one action that moves it forward.
|
||||
|
||||
| # | stage | reversible afterwards? |
|
||||
|---|---|---|
|
||||
| 0 | [Inventory](stages/inventory.md) | nothing was changed |
|
||||
| 1 | [Duplicate review](stages/duplicates.md) | yes — decisions can be changed, no file is deleted |
|
||||
| 2 | [Safety review](stages/safety.md) | the decision, yes; the EXIF keyword is a file write |
|
||||
| 3 | [Analysis](stages/analysis.md) | the result, yes; the caption and keywords are a file write |
|
||||
| 4 | [Album proposals](stages/albums.md) | yes — approving renames nothing |
|
||||
| 5 | [Renames](stages/renames.md) | **your folders move.** Recoverable, journaled, but real |
|
||||
| 6 | [Upload](stages/uploads.md) | **assets exist on the Immich server.** Not undone from here |
|
||||
| 7 | [Archive](stages/archive.md) | **originals leave active storage.** Restorable while the medium is reachable |
|
||||
| — | [Diagnostics](stages/diagnostics.md) | read-only, any time |
|
||||
|
||||
## Before the first run
|
||||
|
||||
Take a backup of the photo library itself. This application is careful — it previews,
|
||||
journals, and verifies — but it is the first time you are pointing it at your
|
||||
photographs, and a backup is cheaper than confidence.
|
||||
|
||||
If the library is irreplaceable, turn on `PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL`
|
||||
before anything else. Every mutating request is then refused until you have produced
|
||||
a read-only reconciliation report and approved it by name.
|
||||
|
||||
## The pass
|
||||
|
||||
1. **Scan.** Inventory → *Rescan*. Reads only. Check the count, and check that
|
||||
nothing under `_IGNORE/` appears.
|
||||
2. **Resolve duplicates.** Exact byte matches can be accepted as recommended; anything
|
||||
fuzzy gets looked at. Resolving first is what stops you paying to analyse the same
|
||||
photo twice.
|
||||
3. **Decide safety.** Every canonical photo becomes `sfw` or `nsfw`. Nothing reaches
|
||||
the vision provider until it is confirmed SFW — this is the gate the whole design
|
||||
exists around.
|
||||
4. **Analyse.** Only confirmed-SFW photos. Each result is written to the database and
|
||||
projected into EXIF, then read back and verified.
|
||||
5. **Propose album names.** Evidence from what was analysed. Edit anything you disagree
|
||||
with. Approving changes no file.
|
||||
6. **Build the rename plan, read it, apply it.** The preview shows every old → new
|
||||
path. This is where folders move.
|
||||
7. **Rescan.** Paths are reconciled; identities are unchanged.
|
||||
8. **Upload.** Preflight lists every blocker. The command is previewed without the API
|
||||
key. One album at a time.
|
||||
9. **Verify.** An uncertain upload is not a failure and not a success — ask the server.
|
||||
10. **Archive** (optional). Copy, verify, and only then reclaim the space.
|
||||
|
||||
Stop anywhere. Every stage is resumable, and closing the browser cancels nothing.
|
||||
|
||||
## Two rules worth internalising
|
||||
|
||||
**The application refuses more than it warns.** A refusal is not a fault; it is the
|
||||
design working. [Errors and refusals](errors.md) explains each one.
|
||||
|
||||
**One long job at a time.** Browsing stays available while a job runs, but a second
|
||||
mutating job is refused — that is what makes a crash recoverable rather than
|
||||
ambiguous.
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 64 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 31 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 127 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 24 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 56 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 30 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 31 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 127 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 101 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 63 KiB |
@@ -1,49 +0,0 @@
|
||||
# Photo Pipeline documentation
|
||||
|
||||
One local application that takes a photo library from discovery to a verified Immich
|
||||
upload: duplicate detection, safety review, content analysis, album naming, guarded
|
||||
renaming, upload, and archive — one visible, resume-safe workflow.
|
||||
|
||||
These pages are readable three ways, and they are the same files each time: in the
|
||||
repository under `docs/`, on Gitea, and inside the running application under
|
||||
**Docs**. There is no separate copy to fall out of date.
|
||||
|
||||
## Read in this order
|
||||
|
||||
1. [Overview](overview.md) — what the application does, the stages it moves a photo
|
||||
through, and the rules it will not break.
|
||||
2. [Installation and operations](installation.md) — host and container installation,
|
||||
every setting, the first-run checklist, upgrades, backup and restore, and what
|
||||
each refusal at startup means.
|
||||
3. [A guided first pass](first-pass.md) — one library from scan to verified upload,
|
||||
with the point of no return named in each stage.
|
||||
4. [Architecture](architecture.md) — the context and runtime diagrams, what each
|
||||
module owns, the three journals a restart reads, and where every invariant is
|
||||
enforced.
|
||||
|
||||
## The stages, one page each
|
||||
|
||||
Every page answers the same four questions: what the stage is for, what you decide,
|
||||
what it changes on disk or on the server, and what it refuses.
|
||||
|
||||
1. [Inventory and discovery](stages/inventory.md)
|
||||
2. [Duplicate review](stages/duplicates.md)
|
||||
3. [Safety review](stages/safety.md)
|
||||
4. [Analysis](stages/analysis.md)
|
||||
5. [Album proposals](stages/albums.md)
|
||||
6. [Renames](stages/renames.md)
|
||||
7. [Upload](stages/uploads.md)
|
||||
8. [Archive](stages/archive.md)
|
||||
9. [Diagnostics and library statistics](stages/diagnostics.md)
|
||||
|
||||
## When something goes wrong
|
||||
|
||||
- [Errors and refusals](errors.md) — every error code, what caused it, what to do.
|
||||
- [Recovery](recovery.md) — what the application resolves by itself, and what needs
|
||||
you.
|
||||
|
||||
## Conventions
|
||||
|
||||
A page tells you what a stage **changes on disk or on the server** before it tells
|
||||
you how to run it. Refusals are documented as intentional: this application would
|
||||
rather stop and explain than guess about somebody's photographs.
|
||||
@@ -1,336 +0,0 @@
|
||||
# Installation and operations
|
||||
|
||||
[← Documentation index](index.md)
|
||||
|
||||
From nothing to a running instance pointed at your photo library, and everything you
|
||||
need afterwards: upgrading, backing up, restoring, and understanding a refusal.
|
||||
|
||||
Two ways to install. **Container** is the deployment this project builds and ships;
|
||||
**host** is what you want for development or a single machine you already manage.
|
||||
Both run the same two processes against the same database.
|
||||
|
||||
> Read [what it will not do](overview.md#what-it-will-not-do) first. Several of the
|
||||
> steps below only make sense once you know which rules the application is keeping.
|
||||
|
||||
## Before you start
|
||||
|
||||
| you need | version | why |
|
||||
|---|---|---|
|
||||
| Python | 3.12 or newer | the application |
|
||||
| `exiftool` | 13.x (the image pins `13.25+dfsg-1`) | every EXIF read and write |
|
||||
| `immich-go` | 0.32.0 (pinned in the image) | the upload stage only |
|
||||
| Docker + the compose plugin | any current | the container installation only |
|
||||
| a vision provider key | — | the analysis stage only |
|
||||
|
||||
You also need a photo library you can afford to be wrong about. Take a backup of it
|
||||
before pointing anything at it for the first time, and consider running with
|
||||
[`PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL`](#the-settings) on.
|
||||
|
||||
## Host installation
|
||||
|
||||
```bash
|
||||
python3.12 -m venv .venv
|
||||
.venv/bin/pip install -e ".[vision]" # drop [vision] for a review-only install
|
||||
```
|
||||
|
||||
Configure it. Every setting is an environment variable; a `.env` file in the working
|
||||
directory is read at startup:
|
||||
|
||||
```bash
|
||||
cp .env.example .env && $EDITOR .env
|
||||
```
|
||||
|
||||
`.env.example` lists every variable with its default and no values at all. For a
|
||||
loopback installation you need exactly one:
|
||||
|
||||
```bash
|
||||
PHOTO_PIPELINE_LIBRARY_ROOTS=/home/you/Pictures
|
||||
```
|
||||
|
||||
Then migrate and start the two processes:
|
||||
|
||||
```bash
|
||||
.venv/bin/python -m photo_pipeline migrate # create or upgrade the database
|
||||
.venv/bin/python -m photo_pipeline serve # API and browser app on 127.0.0.1:8000
|
||||
.venv/bin/python -m photo_pipeline worker # second terminal
|
||||
```
|
||||
|
||||
**Both processes are required.** `serve` enqueues work and renders the application;
|
||||
nothing is scanned, scored, analysed, renamed, uploaded, or archived without a
|
||||
worker. Open <http://127.0.0.1:8000/app/> and confirm the workflow page loads.
|
||||
|
||||
### The configuration file
|
||||
|
||||
`.env`, or any path named by `PHOTO_PIPELINE_ENV_FILE`. It is parsed, never executed:
|
||||
`KEY=value` lines, `#` comments, optional quotes, no interpolation and no `export`. A
|
||||
configuration file that can run code is a configuration file that can be a
|
||||
vulnerability.
|
||||
|
||||
**Anything already exported in the shell wins.** The file is your standing
|
||||
configuration; the environment is the override for one run.
|
||||
|
||||
The archived CLI's names still work, so an existing `photo_analyzer.env` can be used
|
||||
as it is:
|
||||
|
||||
| in the file | applied as |
|
||||
|---|---|
|
||||
| `LLM_API_KEY` / `GEMINI_API_KEY` | `OPENAI_API_KEY` |
|
||||
| `LLM_BASE_URL` | `OPENAI_BASE_URL` |
|
||||
| `LIBRARY` | `PHOTO_PIPELINE_LIBRARY_ROOTS` |
|
||||
|
||||
`.env` and `*.env` are gitignored and refused by the repository's safety checks. The
|
||||
file holds a real key; it must never be committed.
|
||||
|
||||
## Container installation
|
||||
|
||||
```bash
|
||||
cp .env.example .env && $EDITOR .env
|
||||
docker compose up -d --build
|
||||
```
|
||||
|
||||
The composition is one `serve` container, one `worker` container, a one-shot
|
||||
`migrate` that both wait for, one bind-mounted library, and one named data volume.
|
||||
|
||||
Three variables it cannot start without:
|
||||
|
||||
| variable | is |
|
||||
|---|---|
|
||||
| `PHOTO_PIPELINE_LIBRARY_HOST_PATH` | the library on this host |
|
||||
| `PHOTO_PIPELINE_LIBRARY_ROOTS` | where that library is mounted **inside** the container |
|
||||
| `PHOTO_PIPELINE_ACCESS_SECRET` | required, because publishing a port means the app is reachable from outside the container |
|
||||
|
||||
Set `PHOTO_PIPELINE_UID` and `PHOTO_PIPELINE_GID` to the owner of the library: what
|
||||
the containers rename and rewrite keeps that ownership.
|
||||
|
||||
### The data volume
|
||||
|
||||
The `data` volume holds the database, its write-ahead log, the thumbnail cache, the
|
||||
operation journals, and the backups. **It must stay on a local filesystem.** SQLite
|
||||
in WAL mode needs real local locking, so NFS, SMB, and network volume drivers do not
|
||||
slow it down — they corrupt it. The library bind mount has no such restriction.
|
||||
|
||||
### Ports and reaching it
|
||||
|
||||
The API port is published to `127.0.0.1` unless `PHOTO_PIPELINE_PUBLISH_ADDRESS` says
|
||||
otherwise. Being reachable *was* the authentication in earlier versions: whoever could
|
||||
open the port owned the library. So the moment the application answers to anything but
|
||||
loopback — a hostname, a proxy, `0.0.0.0` — the access secret becomes mandatory and
|
||||
`serve` refuses to start without it rather than publishing your photographs.
|
||||
|
||||
Behind a reverse proxy:
|
||||
|
||||
```bash
|
||||
PHOTO_PIPELINE_ALLOWED_HOSTS=photos.example.com
|
||||
PHOTO_PIPELINE_ACCESS_SECRET=… # python -c 'import secrets; print(secrets.token_urlsafe(32))'
|
||||
PHOTO_PIPELINE_TRUSTED_PROXIES=10.0.0.2 # only the proxy's own address
|
||||
```
|
||||
|
||||
`X-Forwarded-Proto` and `X-Forwarded-Host` are believed only from a trusted-proxy
|
||||
address, so a client cannot declare its own origin. The browser asks for the secret
|
||||
once per tab. Wrong secrets are rate-limited and logged with the caller's address
|
||||
only. The health endpoints stay open so an orchestrator can restart the container;
|
||||
nothing else is.
|
||||
|
||||
### Deployment
|
||||
|
||||
The stack is managed from git by Portainer, which redeploys on a webhook after CI
|
||||
publishes an image built from `main`. Runtime secrets live in the Portainer stack's
|
||||
environment rather than in the repository, so rotation happens in one place and
|
||||
reading the repository discloses nothing.
|
||||
|
||||
## The settings
|
||||
|
||||
Every variable, its default, and whether it is a secret. `.env.example` is the same
|
||||
list in copyable form.
|
||||
|
||||
### Library and data
|
||||
|
||||
| variable | default | meaning |
|
||||
|---|---|---|
|
||||
| `PHOTO_PIPELINE_LIBRARY_ROOTS` | none | the library boundary, `os.pathsep`-separated. No path outside these roots is ever read or written |
|
||||
| `PHOTO_PIPELINE_DATA_DIR` | `data` | database, WAL, thumbnail cache, journals, backups. Never inside the library |
|
||||
| `PHOTO_PIPELINE_DB_PATH` | `<data dir>/photo_pipeline.db` | the database file, if it must live elsewhere |
|
||||
| `PHOTO_PIPELINE_ENV_FILE` | `.env` | where to read the configuration file from |
|
||||
|
||||
### Serving and the trust boundary
|
||||
|
||||
| variable | default | meaning |
|
||||
|---|---|---|
|
||||
| `PHOTO_PIPELINE_HOST` | `127.0.0.1` | bind address |
|
||||
| `PHOTO_PIPELINE_PORT` | `8000` | port |
|
||||
| `PHOTO_PIPELINE_ALLOWED_HOSTS` | none | comma-separated names the app answers to besides loopback. Naming one makes the access secret mandatory |
|
||||
| `PHOTO_PIPELINE_ACCESS_SECRET` | none | **secret.** Traded for the session cookie at `GET /api/v1/session` |
|
||||
| `PHOTO_PIPELINE_TRUSTED_PROXIES` | none | comma-separated peer addresses whose forwarded headers may be believed |
|
||||
| `PHOTO_PIPELINE_MAX_REQUEST_BYTES` | `1048576` | largest request body accepted |
|
||||
|
||||
### Logging
|
||||
|
||||
| variable | default | meaning |
|
||||
|---|---|---|
|
||||
| `PHOTO_PIPELINE_LOG_LEVEL` | `INFO` | standard Python levels |
|
||||
| `PHOTO_PIPELINE_LOG_FORMAT` | `json` | `json` or `text` |
|
||||
|
||||
### Limits
|
||||
|
||||
| variable | default | meaning |
|
||||
|---|---|---|
|
||||
| `PHOTO_PIPELINE_THUMBNAIL_CACHE_QUOTA_BYTES` | `500000000` | cache quota; over it is a diagnostics warning |
|
||||
| `PHOTO_PIPELINE_THUMBNAIL_MAX_PIXELS` | `100000000` | refuse to decode anything larger. This is the decompression-bomb guard |
|
||||
| `PHOTO_PIPELINE_ARCHIVE_FREE_SPACE_RESERVE_BYTES` | `1000000000` | free space an archive destination must keep beyond the transfer itself |
|
||||
|
||||
### The safety gate
|
||||
|
||||
| variable | default | meaning |
|
||||
|---|---|---|
|
||||
| `PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL` | `false` | refuse every mutating request until a read-only dry run of this library has been produced and approved |
|
||||
|
||||
### External services
|
||||
|
||||
| variable | default | meaning |
|
||||
|---|---|---|
|
||||
| `PHOTO_PIPELINE_VISION_API_KEY` | none | **secret.** Without it the analysis stage cannot run |
|
||||
| `PHOTO_PIPELINE_IMMICH_SERVER_URL` | none | the Immich server |
|
||||
| `PHOTO_PIPELINE_IMMICH_API_KEY` | none | **secret.** |
|
||||
| `PHOTO_PIPELINE_IMMICH_GO_BINARY` | `immich-go` | the uploader, found on `PATH` |
|
||||
|
||||
### Composition only
|
||||
|
||||
Read by `docker-compose.yml`, not by the application:
|
||||
`PHOTO_PIPELINE_IMAGE`, `PHOTO_PIPELINE_LIBRARY_HOST_PATH`,
|
||||
`PHOTO_PIPELINE_PUBLISH_ADDRESS`, `PHOTO_PIPELINE_UID`, `PHOTO_PIPELINE_GID`.
|
||||
|
||||
A secret is never written to a log, never returned by the API, never stored in the
|
||||
database, and never recorded in a backup manifest — a manifest says `configured`, not
|
||||
the value. Do not put a real key in an example, a ticket, or a screenshot.
|
||||
|
||||
## First run checklist
|
||||
|
||||
Not "it started" — verified.
|
||||
|
||||
1. **The database is at the current revision.**
|
||||
`python -m photo_pipeline migrate` exits 0.
|
||||
2. **The application answers.**
|
||||
`curl -s localhost:8000/api/v1/health/ready` returns 200.
|
||||
3. **The library was found.** Open the app, run a scan from the Inventory view, and
|
||||
check the count against what you expect. Anything under `_IGNORE/` must be missing
|
||||
from it — that is the exclusion working, not a bug.
|
||||
4. **The worker is claiming.** The scan job reaches `succeeded`. If it stays
|
||||
`queued`, no worker is running.
|
||||
5. **Diagnostics are clean.**
|
||||
`python -m photo_pipeline diagnostics` reports free space and an empty `warnings`.
|
||||
6. **Nothing was modified.** The scan is read-only; your files' timestamps are
|
||||
unchanged.
|
||||
|
||||
Only then point it at the whole library.
|
||||
|
||||
## Operations
|
||||
|
||||
Every operation is the same CLI, on a host or in the composition:
|
||||
|
||||
```bash
|
||||
python -m photo_pipeline diagnostics
|
||||
docker compose run --rm --no-deps api diagnostics
|
||||
```
|
||||
|
||||
`--no-deps` keeps a one-off command from starting a second stack; `api` is only the
|
||||
service it borrows the image and mounts from.
|
||||
|
||||
### Upgrading
|
||||
|
||||
1. `python -m photo_pipeline backup --reason before-upgrade`
|
||||
2. Pull the new version (or `docker compose pull && docker compose up -d`).
|
||||
3. Migrations run by themselves at startup, and a **pending schema change is
|
||||
snapshotted first**. If the upgrade fails, the previous database and its
|
||||
`pre-migration` backup are both intact, and the error log names the backup
|
||||
directory.
|
||||
|
||||
An up-to-date database is not backed up again on every start.
|
||||
|
||||
### Backing up
|
||||
|
||||
```bash
|
||||
python -m photo_pipeline backup --reason weekly --keep 7
|
||||
python -m photo_pipeline verify-backup data/backups/<name>
|
||||
```
|
||||
|
||||
Backups go through SQLite's online backup API, never a file copy: with WAL enabled
|
||||
the `.db` file alone is missing every committed page still in the write-ahead log.
|
||||
Each backup is a directory holding the snapshot and a `manifest.json` — schema
|
||||
revision, SHA-256, row counts, the archive media the library depends on, and which
|
||||
settings were configured.
|
||||
|
||||
`verify-backup` runs `PRAGMA integrity_check` **and** `PRAGMA foreign_key_check`,
|
||||
compares the snapshot's SHA-256 against the manifest, and re-counts every table it
|
||||
recorded. Bit rot, a truncated copy, and a "repaired" snapshot all fail it.
|
||||
|
||||
`--keep N` prunes the oldest and never the newest. The same is available at
|
||||
`GET /api/v1/diagnostics`, `GET|POST /api/v1/backups`,
|
||||
`GET /api/v1/backups/{name}/verify`, and `POST /api/v1/backups/prune`.
|
||||
|
||||
### Restoring
|
||||
|
||||
Restore is deliberately **not** an API call. It replaces the state of an
|
||||
installation, so it belongs to a stopped one and a person at a terminal.
|
||||
|
||||
1. Stop `serve` and `worker`.
|
||||
2. `python -m photo_pipeline verify-backup data/backups/<name>` — never restore an
|
||||
unverified snapshot, and `restore` will refuse one anyway.
|
||||
3. `python -m photo_pipeline restore data/backups/<name> --into /path/to/fresh-data`.
|
||||
A target that already holds a database is refused; recovering in place means
|
||||
moving the old data directory aside first.
|
||||
4. Point `PHOTO_PIPELINE_DATA_DIR` at the restored directory and run `migrate`.
|
||||
5. Run an inventory scan, so paths are reconciled against the real library.
|
||||
6. Mount every archive location the manifest names before archiving again. The
|
||||
database records where archived originals are; it does not contain them.
|
||||
|
||||
Practise this against a copy before you need it.
|
||||
|
||||
### Watching it
|
||||
|
||||
`diagnostics` reports the database, write-ahead log, thumbnail cache, uploader
|
||||
reports, backups, and logs separately, with free space and warnings for low disk
|
||||
(`disk_low`, `disk_critical`), a cache over its quota, a write-ahead log outgrowing
|
||||
its database, and a legacy CLI writing the library.
|
||||
|
||||
Logs go to stdout in JSON by default — `docker compose logs -f worker`, or your
|
||||
service manager's journal on a host. Uploader output is kept per attempt under the
|
||||
data directory with the API key scrubbed line by line.
|
||||
|
||||
## When it refuses to start
|
||||
|
||||
Every refusal below is deliberate. The application would rather stop and explain than
|
||||
guess about somebody's photographs.
|
||||
|
||||
| exit code | meaning | what to do |
|
||||
|---|---|---|
|
||||
| `1` | the operation failed and said why — an unverifiable backup, an occupied restore target, an unknown benchmark profile | read the message; nothing was changed |
|
||||
| `2` | **the library lock is held.** Another `serve` or `worker` owns this data directory | stop the other process. The message names the holder; a lock whose process is gone is taken over automatically |
|
||||
| `3` | **a legacy CLI looks active.** The frozen command-line tools are writing this library | stop them. `--allow-legacy` overrides and you own the outcome |
|
||||
| `4` | **configuration refused.** The application is reachable beyond loopback and no access secret is set | set `PHOTO_PIPELINE_ACCESS_SECRET`, or bind to loopback only |
|
||||
| `5` | **a library root is not mounted** (containers only) | fix the bind mount. Refusing is what stops the container writing into its own throwaway layer instead of your library |
|
||||
|
||||
Other things that look like faults and are not:
|
||||
|
||||
- **`403 host_not_allowed`** — the `Host` header is not a name in
|
||||
`PHOTO_PIPELINE_ALLOWED_HOSTS`. Add the real hostname; do not add `*`.
|
||||
- **`409 lock_held`** — a mutating job is already running. One at a time is what
|
||||
makes a crash recoverable.
|
||||
- **`409 rename_recovery_required`** — an interrupted rename is unresolved. Resolve
|
||||
it in the Renames view; unrelated mutations stay blocked until then, on purpose.
|
||||
- **A job stuck in `queued`** — no worker is running.
|
||||
- **An empty scan** — check `PHOTO_PIPELINE_LIBRARY_ROOTS`, and in a container check
|
||||
that the path is the *container-side* mount, not the host path.
|
||||
|
||||
## What an installer must not work around
|
||||
|
||||
- **One writer.** Do not run two workers, and do not run the frozen CLI beside the
|
||||
application. Each is safe alone and destructive together — the lock is not
|
||||
bureaucracy.
|
||||
- **`_IGNORE/` is never read.** Do not "fix" the exclusion.
|
||||
- **EXIF is verified before upload.** Do not skip the checkpoint to make a stage
|
||||
finish; an unverified projection is exactly the case where the wrong bytes reach
|
||||
Immich.
|
||||
- **The data directory is not the library.** Keep them apart, and keep the data
|
||||
directory on a local filesystem.
|
||||
- **Secrets stay in the environment.** Not in the database, not in a log, not in a
|
||||
commit.
|
||||
@@ -1,77 +0,0 @@
|
||||
# Overview
|
||||
|
||||
[← Documentation index](index.md)
|
||||
|
||||
Photo Pipeline sorts a photo library. It finds duplicates before anything expensive
|
||||
happens to them, asks a human which pictures may leave the machine, describes the
|
||||
ones that may, proposes album names from what it found, renames folders under a
|
||||
crash-safe journal, uploads through `immich-go`, and can archive a finished album off
|
||||
active storage without ever forgetting it existed.
|
||||
|
||||
It runs locally. It talks to exactly three things outside itself: a vision provider,
|
||||
an Immich server, and `exiftool` — and it will tell you before it uses any of them.
|
||||
|
||||
## The workflow
|
||||
|
||||
Each stage is a gate, not a tab. A later stage can always be looked at; its actions
|
||||
stay disabled until what they depend on is true.
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
I["0 · Inventory<br/>discover, hash, cluster duplicates"] --> S["1 · Safety<br/>score and human decision"]
|
||||
S -->|sfw| A["2 · Analysis<br/>vision provider, EXIF checkpoint"]
|
||||
S -->|nsfw| U
|
||||
A --> B["3 · Albums<br/>proposal, then guarded rename"]
|
||||
B --> U["4 · Upload<br/>immich-go, verified bytes"]
|
||||
U --> R["5 · Archive<br/>copy, verify, then reclaim space"]
|
||||
```
|
||||
|
||||
| Stage | What it decides | What it changes |
|
||||
|---|---|---|
|
||||
| Inventory | which file is the canonical copy of a picture | nothing — it only reads |
|
||||
| Safety | whether a photo may be sent to a cloud provider | one `sfw`/`nsfw` EXIF keyword |
|
||||
| Analysis | what a photo shows | a managed caption segment and additive keywords |
|
||||
| Albums | what a folder should be called | folder names, through a journaled rename |
|
||||
| Upload | which exact bytes reach Immich | nothing locally; assets appear in Immich |
|
||||
| Archive | which album leaves active storage | files move to the archive, after verification |
|
||||
|
||||
## What it will not do
|
||||
|
||||
These are enforced in code, not by convention, and each one is why some action you
|
||||
expected is sometimes refused.
|
||||
|
||||
- **Nothing under `_IGNORE/` is ever read.** Not scanned, not counted, not
|
||||
thumbnailed, not sent anywhere.
|
||||
- **A path is not an identity.** Every picture has a stable id, so moving or renaming
|
||||
it loses no history.
|
||||
- **Only a confirmed-SFW photo reaches the vision provider.** A photo marked NSFW is
|
||||
still uploadable to Immich; it simply never leaves for analysis.
|
||||
- **Metadata is verified, not hoped for.** Every stage that writes EXIF reads it back
|
||||
and proves that what it did not own is unchanged.
|
||||
- **One writer at a time.** A library lock, held by one process, is what makes a
|
||||
crash recoverable instead of ambiguous.
|
||||
- **Nothing irreversible happens without a preview and an explicit approval** that
|
||||
names the exact count.
|
||||
|
||||
## Where things live
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| the photo library | wherever you point `PHOTO_PIPELINE_LIBRARY_ROOTS`; mounted read-write |
|
||||
| the database, cache, logs, backups | the data directory, never inside the library |
|
||||
| secrets | the environment, never the database and never a log line |
|
||||
| these documents | `docs/` in the repository, served at `/docs` by the application |
|
||||
|
||||
## Running it
|
||||
|
||||
The short version, for a host installation:
|
||||
|
||||
```bash
|
||||
python -m photo_pipeline migrate
|
||||
python -m photo_pipeline serve # the API and this browser application
|
||||
python -m photo_pipeline worker # the process that does the long work
|
||||
```
|
||||
|
||||
The full procedure, the container path, and every setting are in the
|
||||
[installation manual](installation.md); if you would rather start using it, take
|
||||
[the guided first pass](first-pass.md).
|
||||
@@ -1,62 +0,0 @@
|
||||
# Recovery
|
||||
|
||||
[← Documentation index](index.md)
|
||||
|
||||
What the application resolves by itself, and what needs you. The dividing line is
|
||||
simple: it resumes when the evidence is unambiguous, and it stops and asks when it is
|
||||
not. It never guesses about your files.
|
||||
|
||||
## An interrupted rename
|
||||
|
||||
A rename records its intent **before** touching the disk, so after a crash the
|
||||
journal plus the actual filesystem give a verdict per operation:
|
||||
|
||||
| verdict | means | what happens |
|
||||
|---|---|---|
|
||||
| resumable | the move provably did not happen | reset to planned and run again |
|
||||
| rollback-safe | the move happened; the remaining steps can be completed or undone | drive it forward, or roll it back |
|
||||
| manual | the evidence is contradictory | left alone, and it keeps blocking |
|
||||
|
||||
Open [Renames](stages/renames.md); the recovery panel lists every unresolved
|
||||
operation and offers a resolve action only where one is safe. Until then, unrelated
|
||||
mutations are refused with `rename_recovery_required` — building new work on a
|
||||
half-applied move is how a library becomes unexplainable.
|
||||
|
||||
## An uncertain upload
|
||||
|
||||
`unknown_requires_verification` means the uploader died after Immich may have
|
||||
accepted the files. It is **not** retryable. Verification asks the server whether it
|
||||
holds the recorded SHA-1 and answers present, absent, or inconclusive. An
|
||||
inconclusive answer is resolved by you, with your evidence and name recorded in the
|
||||
batch's history.
|
||||
|
||||
## A failed migration
|
||||
|
||||
A pending schema change is snapshotted before it is applied. If the upgrade fails,
|
||||
the previous database and its `pre-migration` backup are both intact and the error
|
||||
log names the backup directory. Stop everything and run the
|
||||
[restore drill](installation.md#restoring).
|
||||
|
||||
## A restored backup
|
||||
|
||||
After restoring, run a scan. Paths are reconciled against the real library and
|
||||
identities are preserved where hashes match. Anything that does not match becomes a
|
||||
visible `stale` or `divergent` state instead of being quietly accepted.
|
||||
|
||||
Mount every archive medium the manifest names before archiving again: the database
|
||||
records where archived originals live, it does not contain them.
|
||||
|
||||
## A job that will not start
|
||||
|
||||
Check, in this order: is a worker running; is another mutating job holding the lane
|
||||
(`lock_held`); is an interrupted rename blocking everything
|
||||
(`rename_recovery_required`); is the library lock held by a process that is still
|
||||
alive (exit code `2` names the holder).
|
||||
|
||||
## What is never automatic
|
||||
|
||||
No file is deleted, no folder is renamed, no upload is retried, and no archive source
|
||||
is removed without either an explicit confirmation from you or a verification the
|
||||
application performed itself. If you are reading this page because something happened
|
||||
that you did not approve, that is a bug worth reporting — with the diagnostics output
|
||||
and the relevant journal, both of which are safe to share: they carry no secrets.
|
||||
@@ -1,47 +0,0 @@
|
||||
# Album proposals
|
||||
|
||||
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
|
||||
|
||||
|
||||
|
||||
**What it is for.** Turning a folder of analysed photographs into a name a person
|
||||
would have chosen, before any folder is touched.
|
||||
|
||||
**What you decide.** The final name. The proposal is a starting point with its
|
||||
reasoning attached; you edit it, accept it, or ignore it.
|
||||
|
||||
**What it changes.** A row per album: proposed name, rationale, confidence, and your
|
||||
final name, with a version. **Approving renames nothing.** Not one file moves at this
|
||||
stage — approval only marks a name as agreed, and the [rename stage](renames.md) is
|
||||
where it becomes a plan you have to confirm separately.
|
||||
|
||||
**What it refuses.** A name containing `/ \ : * ? " < > |`, a reserved device name, or
|
||||
one that collides with an existing folder is rejected while you type and again on the
|
||||
server. Approval is refused when the evidence changed since the proposal was
|
||||
generated — the proposal is stale, and regenerating is the honest fix. Editing with a
|
||||
stale version returns a conflict and shows you the server's truth rather than
|
||||
overwriting it.
|
||||
|
||||
## Reading the view
|
||||
|
||||
The left pane lists albums with their state: `none`, `proposed`, `edited`,
|
||||
`approved`, `error`, or `stale`. The right pane shows the evidence the name was built
|
||||
from — date range, dominant tags, locations, counts — then the suggested name, the
|
||||
rationale, the confidence, and an editable final name.
|
||||
|
||||
The default naming shape is predictable and sortable:
|
||||
|
||||
```text
|
||||
YYYY-MM — Place — Event
|
||||
YYYY — Event
|
||||
Place — Event
|
||||
```
|
||||
|
||||
## While a rename is unresolved
|
||||
|
||||
Generating and approving proposals is blocked while an interrupted rename is
|
||||
outstanding, with `409 rename_recovery_required`. Resolve the rename first: building
|
||||
new names on top of a half-applied move is how a library becomes hard to reason
|
||||
about.
|
||||
|
||||
Next: [renames](renames.md).
|
||||
@@ -1,39 +0,0 @@
|
||||
# Analysis
|
||||
|
||||
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
|
||||
|
||||
|
||||
|
||||
**What it is for.** Describing what a photograph shows, so albums can be named from
|
||||
evidence and the library can be searched by content.
|
||||
|
||||
**What you decide.** When to run it, and over what scope. The descriptions themselves
|
||||
come from the configured vision provider.
|
||||
|
||||
**What it changes.** For each eligible photo: a stored result — description, tags,
|
||||
approximate year, location hint, model and prompt version — and then an EXIF
|
||||
checkpoint that merges a **managed caption segment** and additive keywords into the
|
||||
file, reads them back, and verifies that nothing else moved. The file's SHA-256 is
|
||||
refreshed after the write.
|
||||
|
||||
**What it refuses.** Anything not confirmed SFW. Anything not canonical. A malformed
|
||||
provider response is quarantined rather than stored. Keywords are only ever added,
|
||||
never removed, because EXIF keywords carry no ownership and deleting a generated one
|
||||
could delete yours.
|
||||
|
||||
## Running it
|
||||
|
||||
The view shows eligible, analysed, pending, and error counts, and the job's live
|
||||
progress. Starting a second mutating job while it runs is refused — that is the
|
||||
one-writer rule, not a queue.
|
||||
|
||||
Cancelling drains safely: completed items stay completed, and resuming continues
|
||||
rather than restarting. A photo may be sent to the provider twice if a crash happens
|
||||
mid-item, but its stored result and its EXIF are written once.
|
||||
|
||||
## Costs
|
||||
|
||||
Each analysed photo is a provider call. Resolving duplicates first is what keeps that
|
||||
number honest, and the counts here are the ones to check before starting a large run.
|
||||
|
||||
Next: [album proposals](albums.md).
|
||||
@@ -1,43 +0,0 @@
|
||||
# Archive
|
||||
|
||||
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
|
||||
|
||||
|
||||
|
||||
**What it is for.** Moving a finished album out of active storage onto another disk,
|
||||
while keeping everything the library knows about it.
|
||||
|
||||
**What you decide.** Which album, which destination, and whether to accept the
|
||||
reclaimed space in exchange for needing that medium mounted to see the originals
|
||||
again.
|
||||
|
||||
**What it changes.** Files are copied to the archive location, verified there, and
|
||||
only then removed from the library. The asset keeps its id, its hashes, its
|
||||
decisions, its analysis, its upload history, and a durable preview thumbnail;
|
||||
`current_path` becomes null and availability becomes `archived_online` or
|
||||
`archived_offline`.
|
||||
|
||||
**What it refuses.** Preflight blocks on: an unverified upload, a checksum that no
|
||||
longer matches the uploaded bytes, an unmounted or unwritable destination,
|
||||
insufficient free space plus the configured reserve, a destination that already
|
||||
holds unexpected paths, a missing durable thumbnail, and any conflicting job. The
|
||||
source is **never** removed before the archived copy is verified — not because Immich
|
||||
reported success, which is an ingest, not a backup.
|
||||
|
||||
## Offline is not missing
|
||||
|
||||
An archived photo whose medium is unplugged is `archived_offline`. It still appears
|
||||
in search, still participates in duplicate detection through its retained hashes and
|
||||
thumbnail, and is never reported as lost. If a full-resolution comparison is needed,
|
||||
the application asks you to mount the named medium rather than guessing.
|
||||
|
||||
## Restore
|
||||
|
||||
Restore is planned and journaled like everything else: the medium must be online, the
|
||||
bytes are copied back and verified into a collision-free destination, a new active
|
||||
path is registered, and a reconciliation scan follows. Existing safety, analysis,
|
||||
EXIF, and upload state stay valid when hashes match — and when they do not, the
|
||||
result is a visible `divergent` state rather than silent acceptance of a different
|
||||
file.
|
||||
|
||||
Next: [diagnostics](diagnostics.md).
|
||||
@@ -1,49 +0,0 @@
|
||||
# Diagnostics and library statistics
|
||||
|
||||
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
|
||||
|
||||
|
||||
|
||||
**What it is for.** Answering "is this installation healthy, and what is in this
|
||||
library?" — the first thing to look at when a stage behaves unexpectedly.
|
||||
|
||||
**What you decide.** Nothing. Both are read-only.
|
||||
|
||||
**What it changes.** Nothing.
|
||||
|
||||
**What it refuses.** Nothing — both are reads, and both stay available while a job
|
||||
holds the library. That is deliberate: the moment you most need to know what the
|
||||
installation is doing is while it is busy.
|
||||
|
||||
## Statistics
|
||||
|
||||
The Stats view summarises the library itself: how many photos are analysed, the
|
||||
common tags, the distribution across albums and years. It answers questions about
|
||||
your photographs.
|
||||
|
||||
## Diagnostics
|
||||
|
||||
Diagnostics answers questions about the *installation*, and lives at
|
||||
`GET /api/v1/diagnostics` and on the command line:
|
||||
|
||||
```bash
|
||||
python -m photo_pipeline diagnostics
|
||||
```
|
||||
|
||||
It reports the database, write-ahead log, thumbnail cache, uploader reports, backups,
|
||||
and logs as separate sizes, plus free space, the tool versions it found against the
|
||||
ones the image pinned, and which locks are held and by whom.
|
||||
|
||||
Its warnings are the ones worth acting on:
|
||||
|
||||
| warning | means |
|
||||
|---|---|
|
||||
| `disk_low` / `disk_critical` | free space is running out; mutating stages stop before the reserve |
|
||||
| `cache_over_quota` | the thumbnail cache exceeds its configured quota |
|
||||
| `wal_growth` | the write-ahead log is outgrowing its database — usually a long-running reader |
|
||||
| `tool_version_drift` | the installed `exiftool` or `immich-go` is not the version the image pinned |
|
||||
| `legacy_process_active` | the frozen command-line tools are writing this library. Stop them |
|
||||
| `no_roots` | no library root is configured, so there is nothing to work on |
|
||||
|
||||
An empty `warnings` list on a fresh installation is what the
|
||||
[first-run checklist](../installation.md#first-run-checklist) is looking for.
|
||||
@@ -1,36 +0,0 @@
|
||||
# Duplicate review
|
||||
|
||||
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
|
||||
|
||||
|
||||
|
||||
**What it is for.** Deciding which file is *the* copy of a picture, before anything
|
||||
expensive or irreversible happens to the others.
|
||||
|
||||
**What you decide.** For each cluster: keep the recommended canonical, choose a
|
||||
different one, keep everything as variants, declare it not a duplicate, or defer.
|
||||
|
||||
**What it changes.** Only the decision, recorded as an auditable event. **No file is
|
||||
ever deleted.** Non-canonical copies stay exactly where they are; they are simply
|
||||
excluded from analysis and upload.
|
||||
|
||||
**What it refuses.** Exact byte and identical-pixel matches may be accepted on the
|
||||
recommendation. Anything fuzzy — a crop, a re-encode, a burst frame — requires an
|
||||
explicit confirmation, because a perceptual hash is evidence, not proof.
|
||||
|
||||
## Reading a cluster
|
||||
|
||||
Each member shows its path, role, pHash distance, and size, with synchronised zoom
|
||||
across the previews. The recommendation is visible but never pre-applied for fuzzy
|
||||
matches. The confidence band tells you how much the machine is claiming.
|
||||
|
||||
Rejecting a pair records a negative link, so a later rescan does not keep proposing
|
||||
the same wrong match.
|
||||
|
||||
## Why first
|
||||
|
||||
Resolving duplicates before safety and analysis is what stops you reviewing and
|
||||
paying for the same photograph three times. It also means the album evidence is built
|
||||
from one copy of each picture rather than a skewed count.
|
||||
|
||||
Next: [safety review](safety.md).
|
||||
@@ -1,39 +0,0 @@
|
||||
# Inventory and discovery
|
||||
|
||||
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
|
||||
|
||||
|
||||
|
||||
**What it is for.** Finding every supported photo in the configured library roots and
|
||||
giving each one a permanent identity, so that everything afterwards can refer to a
|
||||
picture rather than a path.
|
||||
|
||||
**What you decide.** Nothing. This stage is the only one with no judgement in it.
|
||||
|
||||
**What it changes.** On disk, nothing at all — discovery reads. In the database it
|
||||
records each photo's path, size, timestamps, full-file SHA-256, normalised pixel hash,
|
||||
and perceptual hash, and it opens a path-history entry.
|
||||
|
||||
**What it refuses.** Anything under an `_IGNORE/` directory is never traversed,
|
||||
counted, hashed, or previewed. Unsupported extensions and unreadable files are
|
||||
reported rather than skipped silently. Nothing outside the configured roots is
|
||||
reachable, including through a symlink.
|
||||
|
||||
## Reading the view
|
||||
|
||||
Each row is one asset: its current path, availability, whether the file is present,
|
||||
and its size. The filter searches paths; the availability selector separates active
|
||||
photos from archived ones.
|
||||
|
||||
A **rescan** after moving files around outside the application is the normal way to
|
||||
reconcile: a file that moved keeps its identity, because identity is the hash and the
|
||||
id, not the location.
|
||||
|
||||
## What good looks like
|
||||
|
||||
The count matches what you expect, `_IGNORE/` contents are absent, and nothing shows
|
||||
as missing. A photo listed as missing means the file is no longer at its recorded
|
||||
path and no new path matched its hashes — that is a question for you, not something
|
||||
the scanner resolves by guessing.
|
||||
|
||||
Next: [duplicate review](duplicates.md).
|
||||
@@ -1,46 +0,0 @@
|
||||
# Renames
|
||||
|
||||
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
|
||||
|
||||
|
||||
|
||||
**What it is for.** Applying approved album names to the actual folders — the first
|
||||
stage that changes your filesystem.
|
||||
|
||||
**What you decide.** Whether to apply the plan, after reading it. The confirmation
|
||||
names the plan, its version, and its checksum, so what you approve is what runs.
|
||||
|
||||
**What it changes.** Folders move. For each operation: the intent is journaled
|
||||
*before* the disk is touched, the move is made, `assets.current_path` is updated by
|
||||
id in one transaction, the old path is closed and a new one opened in path history,
|
||||
the result is verified, and only then is the operation complete.
|
||||
|
||||
**What it refuses.** A plan is invalid — not merely warned about — when a source is
|
||||
missing or changed, two operations target the same destination, a destination already
|
||||
exists, a path would leave the library or enter `_IGNORE/`, or a source is a symlink.
|
||||
Case-only renames and cross-filesystem moves are allowed but flagged, because they
|
||||
take a different, staged route. A plan whose checksum no longer matches is refused
|
||||
rather than reinterpreted.
|
||||
|
||||
## Reading the preview
|
||||
|
||||
Every operation shows both full paths, the kind of move (`rename`, `case-only
|
||||
rename`, `cross-filesystem move`), how many photos it affects, its journal state, and
|
||||
any issue with its severity. The plan can be exported as JSON before you commit to it.
|
||||
|
||||
**A run cannot be stopped part-way.** It can be interrupted — by a crash, a power
|
||||
cut, a killed container — and interruption is recoverable, but there is no cancel
|
||||
button once it starts. That is stated on the confirmation, not hidden in a manual.
|
||||
|
||||
## After an interruption
|
||||
|
||||
The recovery panel lists each unresolved operation with a verdict read from the
|
||||
journal *and the disk*: resumable, safely rollbackable, or needing a person. It never
|
||||
guesses from a missing path alone. Until it is resolved, unrelated mutations are
|
||||
refused with `409 rename_recovery_required`.
|
||||
|
||||
Rollback is recovery, not undo. A completed operation is terminal — the button only
|
||||
appears for operations that are actually reversible. See
|
||||
[recovery](../recovery.md).
|
||||
|
||||
Next: [upload](uploads.md).
|
||||
@@ -1,43 +0,0 @@
|
||||
# Safety review
|
||||
|
||||
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
|
||||
|
||||
|
||||
|
||||
**What it is for.** Deciding which photographs may be sent to a cloud vision
|
||||
provider. This is the privacy gate the rest of the design is built around.
|
||||
|
||||
**What you decide.** `sfw`, `nsfw`, or defer — per photo, or in bulk from the filters.
|
||||
A local model can score first to order the queue, but the score is a suggestion; the
|
||||
decision is yours.
|
||||
|
||||
**What it changes.** The decision is stored in the database *and* projected into the
|
||||
file: one mutually exclusive `sfw` or `nsfw` keyword written into `Keywords` and
|
||||
`Subject`. The write is merged with existing metadata, read back, and verified, and
|
||||
the file's SHA-256 is refreshed afterwards.
|
||||
|
||||
**What it refuses.** An undecided or deferred photo does not reach the vision
|
||||
provider, and does not reach Immich either. If the read-back shows that a field the
|
||||
stage does not own has changed, the checkpoint is marked `divergent`, the asset is not
|
||||
marked verified, and the next mutating stage is blocked rather than proceeding on
|
||||
metadata nobody trusts.
|
||||
|
||||
## The rule that matters
|
||||
|
||||
> Only a confirmed `sfw` photo may enter cloud content analysis. A confirmed `nsfw`
|
||||
> photo skips analysis entirely but remains eligible for upload to Immich once its
|
||||
> keyword is verified.
|
||||
|
||||
That is why marking something NSFW is not a punishment: it routes the photo around an
|
||||
external service while keeping it in your own library workflow.
|
||||
|
||||
The gate is re-checked *after* the provider call as well. If a decision flips to NSFW
|
||||
while an analysis is in flight, the result is discarded rather than stored.
|
||||
|
||||
## Reading the view
|
||||
|
||||
The tabs filter by state. Each row shows the path, the model's score, its suggestion,
|
||||
your decision, and an `✓ exif` badge once the keyword is verified on disk. A row
|
||||
without that badge has a decision the file does not yet carry.
|
||||
|
||||
Next: [analysis](analysis.md).
|
||||
@@ -1,45 +0,0 @@
|
||||
# Upload
|
||||
|
||||
[← Documentation index](../index.md) · [Guided first pass](../first-pass.md)
|
||||
|
||||
|
||||
|
||||
**What it is for.** Sending an approved album to Immich through `immich-go`, with an
|
||||
audit trail of exactly which bytes went.
|
||||
|
||||
**What you decide.** Which albums, and whether to proceed once the preflight has
|
||||
listed its blockers. Upload is never an automatic consequence of renaming.
|
||||
|
||||
**What it changes.** Nothing locally. On the Immich server, assets appear. Locally a
|
||||
batch is recorded: the album, the asset ids, the redacted command, the uploader
|
||||
version, the pre-upload SHA-256 **and** the SHA-1 Immich matches on, the output log,
|
||||
and the outcome counts.
|
||||
|
||||
**What it refuses.** Preflight blocks on: missing credentials, an unreachable server,
|
||||
a missing `immich-go`, an unresolved rename, an undecided or deferred safety
|
||||
decision, an unverified EXIF checkpoint, incomplete analysis for an SFW photo, a file
|
||||
whose bytes changed since it was checkpointed, and a partial scope you have not
|
||||
explicitly accepted. The approval is then **re-proved immediately before the uploader
|
||||
runs** — bytes edited after you approved fail with `stale_preflight` and the uploader
|
||||
never starts.
|
||||
|
||||
## Reading the view
|
||||
|
||||
The scope shows exactly which albums and how many photos. The command preview is the
|
||||
real command with the API key removed; the key never reaches the browser and never
|
||||
reaches a log.
|
||||
|
||||
## Uncertain is not failed
|
||||
|
||||
If the process dies after the server accepted files, the batch becomes
|
||||
`unknown_requires_verification` — not "failed", and **not retryable**. The server may
|
||||
already hold the photographs, and uploading again would create duplicates or
|
||||
upgrades. Verification asks Immich about the recorded SHA-1 and answers present,
|
||||
absent, or inconclusive; an inconclusive answer is resolved by you, with the evidence
|
||||
recorded.
|
||||
|
||||
If a file's bytes change after a successful upload, it is marked
|
||||
`changed_after_upload` and you are warned that uploading again may create or upgrade
|
||||
an asset rather than doing nothing.
|
||||
|
||||
Next: [archive](archive.md).
|
||||
@@ -12,15 +12,7 @@
|
||||
# (if anything) carries over
|
||||
# Every row needs either `tests` (existing test IDs, module::function) or
|
||||
# `pending_story` (the backlog story that will characterize/deliver it).
|
||||
# status: characterized — donor behavior pinned by characterization tests
|
||||
# resolved — replacement shipped; `parity` names the tests that prove
|
||||
# it, and `delta` states every intentional difference
|
||||
# pending — not migrated yet; `pending_story` says which story will
|
||||
# parity: test ids (path::function) in any suite, proving the replacement
|
||||
# delta: what the replacement deliberately does differently, or not at all
|
||||
#
|
||||
# Archived by US07-01: the sources referenced below now live beside this file in
|
||||
# src/ and are frozen (see README.md). Nothing in photo_pipeline imports them.
|
||||
# status: characterized | pending
|
||||
|
||||
rows:
|
||||
# ── photo_analyzer.py ──────────────────────────────────────────────────────
|
||||
@@ -88,12 +80,8 @@ rows:
|
||||
Copying a primary's analysis into variant rows survives, but keyed by
|
||||
asset_id and recorded as stage state instead of raw row copies.
|
||||
target: photo_pipeline/services/duplicates.py
|
||||
parity:
|
||||
- tests/integration/test_duplicate_engine.py::test_exact_copies_form_auto_decided_cluster
|
||||
- tests/integration/test_duplicate_engine.py::test_perceptual_variant_is_review_only
|
||||
delta: >
|
||||
The donor propagated variant links implicitly while writing rows; the replacement makes the canonical link a reviewable, reversible cluster decision, so a propagated link can always be undone.
|
||||
status: resolved
|
||||
pending_story: US01-04
|
||||
status: pending
|
||||
|
||||
- id: pa-hashing
|
||||
area: hashing
|
||||
@@ -153,12 +141,8 @@ rows:
|
||||
classification: replace
|
||||
rationale: Console report; superseded by the duplicate-review API/UI (US01-06).
|
||||
target: photo_pipeline/api/routes + frontend duplicate review
|
||||
parity:
|
||||
- tests/integration/test_review_api.py::test_clusters_list_and_detail
|
||||
- tests/e2e/test_review_ui.py::test_fuzzy_decision_requires_confirmation
|
||||
delta: >
|
||||
The text listing became the paged cluster API and the comparison UI; no textual report is produced.
|
||||
status: resolved
|
||||
pending_story: US01-06
|
||||
status: pending
|
||||
|
||||
- id: pa-reconcile
|
||||
area: database
|
||||
@@ -218,7 +202,7 @@ rows:
|
||||
RGB-normalize (drops alpha, converts HEIC), LANCZOS resize to 2048px
|
||||
long-edge, JPEG q85 base64 — the provider-input contract. Truncated-image
|
||||
tolerance (ImageFile.LOAD_TRUNCATED_IMAGES) carries with it.
|
||||
target: photo_pipeline/services/analysis.py
|
||||
target: photo_pipeline/integrations/vision.py
|
||||
tests:
|
||||
- test_pa_imaging::test_prepare_image_small_passthrough_jpeg
|
||||
- test_pa_imaging::test_prepare_image_resizes_to_max_long_edge
|
||||
@@ -246,12 +230,9 @@ rows:
|
||||
response validation, 429/503 retry with exponential backoff. Prompt and
|
||||
model/config version must be persisted per analysis_runs. Characterized
|
||||
against a deterministic fake provider when the analysis service is ported.
|
||||
target: photo_pipeline/services/analysis.py
|
||||
parity:
|
||||
- tests/integration/test_safety_analysis.py::test_provider_called_only_for_confirmed_sfw
|
||||
delta: >
|
||||
The prompt and response schema carry over; the provider is an injected adapter so the privacy gate is testable, and results are keyed to asset ids rather than paths.
|
||||
status: resolved
|
||||
target: photo_pipeline/integrations/vision.py
|
||||
pending_story: US02-06
|
||||
status: pending
|
||||
|
||||
- id: pa-throttle
|
||||
area: logging
|
||||
@@ -260,8 +241,8 @@ rows:
|
||||
rationale: >
|
||||
Rolling throttle window + persistent throttle_events.jsonl + RPD day
|
||||
counter become job metrics/events on the durable job model.
|
||||
target: photo_pipeline/services/jobs.py + photo_pipeline/services/analysis.py
|
||||
pending_story: US07-04
|
||||
target: photo_pipeline/jobs/coordinator.py
|
||||
pending_story: US02-02
|
||||
status: pending
|
||||
|
||||
- id: pa-nsfw-filter
|
||||
@@ -331,12 +312,8 @@ rows:
|
||||
checked between items, double-SIGINT force quit — becomes the durable
|
||||
JobRunner worker loop with the same drain-and-resume semantics.
|
||||
target: photo_pipeline/jobs/worker.py
|
||||
parity:
|
||||
- tests/integration/test_worker.py::test_worker_processes_all_items
|
||||
- tests/integration/test_worker.py::test_cooperative_cancellation_leaves_items_resumable
|
||||
delta: >
|
||||
The in-process folder loop with SIGINT handling became durable jobs claimed by a worker: cancellation is a persisted request, not a signal, and an interrupted run resumes from the database instead of restarting.
|
||||
status: resolved
|
||||
pending_story: US02-02
|
||||
status: pending
|
||||
|
||||
- id: pa-ui-terminal
|
||||
area: ui
|
||||
@@ -394,12 +371,8 @@ rows:
|
||||
the JSONL history logger become structured JSON logging with job_id/
|
||||
asset_id and job_events rows; per-photo history maps to job events.
|
||||
target: photo_pipeline structured logging + jobs/job_events
|
||||
parity:
|
||||
- tests/integration/test_jobs.py::test_enqueue_persists_items_and_event
|
||||
- tests/integration/test_jobs_sse.py::test_sse_streams_all_events_then_closes
|
||||
delta: >
|
||||
The JSONL history file and rich console handler are replaced by structured JSON logs plus durable job_events; the browser reads events over SSE rather than tailing a file.
|
||||
status: resolved
|
||||
pending_story: US02-02
|
||||
status: pending
|
||||
|
||||
- id: pa-balance
|
||||
area: vision
|
||||
@@ -409,11 +382,8 @@ rows:
|
||||
Provider balance/quota probes (report 'unsupported' on providers without
|
||||
the endpoint). Network-bound; characterized against the fake provider.
|
||||
target: photo_pipeline/services/analysis.py
|
||||
parity:
|
||||
- tests/integration/test_safety_analysis.py::test_provider_called_only_for_confirmed_sfw
|
||||
delta: >
|
||||
Not carried over: balance/quota polling was provider-specific (Gemini/OpenAI billing endpoints) and key-scoped. Cost reporting, when a story asks for it, comes from the per-run usage recorded with each analysis result rather than from a vendor endpoint.
|
||||
status: resolved
|
||||
pending_story: US02-06
|
||||
status: pending
|
||||
|
||||
- id: pa-cli
|
||||
area: configuration
|
||||
@@ -423,13 +393,9 @@ rows:
|
||||
argparse surface is superseded by the API; flags map to job configs
|
||||
(documented in WEBAPP_CONCEPT.md §8 parity table). Transitional CLI calls
|
||||
the shared services until archival (E07).
|
||||
target: photo_pipeline/__main__.py (serve | worker | migrate | import-legacy-scores) + /api/v1
|
||||
parity:
|
||||
- tests/integration/test_app_lifecycle.py::test_restart_preserves_data_and_reruns_migrations
|
||||
- tests/unit/test_legacy_archive.py::test_production_code_never_imports_an_archived_module
|
||||
delta: >
|
||||
The argparse surface is not reproduced. Every flag that drove work became an API command or a job configuration; the CLI keeps only what an application needs to be operated (serve, worker, migrate) plus the one-off legacy CSV import.
|
||||
status: resolved
|
||||
target: photo_pipeline/api + transitional CLI
|
||||
pending_story: US07-01
|
||||
status: pending
|
||||
|
||||
# ── nsfwtag/ ───────────────────────────────────────────────────────────────
|
||||
- id: nt-discovery
|
||||
@@ -456,17 +422,9 @@ rows:
|
||||
nsfw_scores.csv stops being the source of truth (concept: DB state).
|
||||
Format characterized (4-decimal scores, bad rows dropped) because the
|
||||
existing CSV must migrate into assets.safety_score.
|
||||
target: photo_pipeline/services/legacy_import.py (one-off import into safety_reviews)
|
||||
target: photo_pipeline/repositories (safety), CSV import in US01-02 migration
|
||||
tests: [test_nsfwtag::test_score_cache_roundtrip_and_tolerance]
|
||||
parity:
|
||||
- tests/integration/test_legacy_import.py::test_scores_are_imported_onto_asset_identity
|
||||
- tests/integration/test_legacy_import.py::test_a_reviewed_asset_is_never_overwritten_by_the_csv
|
||||
delta: >
|
||||
The CSV is no longer read at runtime at all: it is imported once into
|
||||
scored-but-unreviewed safety_reviews rows and left on disk untouched. A path
|
||||
that matches nothing is reported, never turned into an asset, and a human
|
||||
decision always outranks an imported score.
|
||||
status: resolved
|
||||
status: characterized
|
||||
|
||||
- id: nt-score-model
|
||||
area: nsfw
|
||||
@@ -480,13 +438,6 @@ rows:
|
||||
inference path needs the local model + deterministic fake.
|
||||
target: photo_pipeline/integrations/nsfw_model.py
|
||||
tests: [test_nsfwtag::test_score_images_cache_hit_skips_model]
|
||||
delta: >
|
||||
The donor set Pillow's process-global ImageFile.LOAD_TRUNCATED_IMAGES so a
|
||||
partially downloaded file still scored. Here the same process also hashes
|
||||
files and renders previews, and those must keep refusing a truncated file
|
||||
rather than silently working on half of one; scoring opens images through
|
||||
the bounded photo_pipeline.imaging door instead and skips the ones it cannot
|
||||
read, leaving them unscored and visibly undecided (US07-03).
|
||||
status: characterized
|
||||
|
||||
- id: nt-exif-keyword
|
||||
@@ -534,11 +485,8 @@ rows:
|
||||
Newline-list bulk tagging (nsfw_confirmed.txt flow) is superseded by DB
|
||||
review decisions; the existing list is a one-time migration input.
|
||||
target: photo_pipeline/services/safety.py (decision import in US01-02)
|
||||
parity:
|
||||
- tests/integration/test_safety_parity.py::test_extracted_marks_and_partition_match_donor
|
||||
delta: >
|
||||
Bulk keyword application from a file list is replaced by decisions against asset ids; the EXIF write itself is the extracted, read-back-verified one.
|
||||
status: resolved
|
||||
pending_story: US01-02
|
||||
status: pending
|
||||
|
||||
- id: nt-ui
|
||||
area: ui
|
||||
@@ -550,11 +498,8 @@ rows:
|
||||
threshold/score review flow, lightbox and keyboard model are the frontend
|
||||
donor for the Safety view (preserved per concept §10; ported in US02-01).
|
||||
target: photo_pipeline/api + frontend Safety view
|
||||
parity:
|
||||
- tests/e2e/test_workflow_views.py::test_safety_review_decide_persists_across_reload
|
||||
delta: >
|
||||
The stdlib review server is replaced by the API plus the Safety view; the donor's layout, thresholds, and keyboard flow carry over, its embedded HTML generation does not.
|
||||
status: resolved
|
||||
pending_story: US02-01
|
||||
status: pending
|
||||
|
||||
- id: nt-bench
|
||||
area: nsfw
|
||||
@@ -564,11 +509,8 @@ rows:
|
||||
Dev-only model benchmark; archived without webapp replacement (recorded
|
||||
basis of the AdamCodd model choice). No production caller.
|
||||
target: none (archive as reference)
|
||||
parity:
|
||||
- tests/unit/test_legacy_archive.py::test_every_archived_source_matches_its_checksum
|
||||
delta: >
|
||||
No replacement: a dev-only benchmark whose result (the AdamCodd model choice) is already recorded. Kept in the archive as the basis of that choice.
|
||||
status: resolved
|
||||
pending_story: US07-01
|
||||
status: pending
|
||||
|
||||
# ── webapp/ ────────────────────────────────────────────────────────────────
|
||||
- id: wa-query-search
|
||||
@@ -637,13 +579,9 @@ rows:
|
||||
Subprocess-driving-the-CLI job control is superseded by durable DB jobs
|
||||
with a worker process. Two ideas carry over: progress derived from DB
|
||||
counts (not job-private state) and single-mutating-job enforcement.
|
||||
target: photo_pipeline/services/jobs.py + photo_pipeline/jobs/worker.py
|
||||
parity:
|
||||
- tests/integration/test_jobs.py::test_idempotency_key_returns_same_job
|
||||
- tests/integration/test_worker.py::test_handler_failure_fails_the_job
|
||||
delta: >
|
||||
Subprocess supervision of a CLI is replaced by durable jobs in the same process family: there is no subprocess to supervise, and progress is persisted rather than scraped from stdout.
|
||||
status: resolved
|
||||
target: photo_pipeline/jobs/coordinator.py
|
||||
pending_story: US02-02
|
||||
status: pending
|
||||
|
||||
- id: wa-server
|
||||
area: ui
|
||||
@@ -655,9 +593,5 @@ rows:
|
||||
browser. analyzer.html + page.py design (dark OLED tokens, Library/
|
||||
Analyze/Stats views) is frontend donor material per concept §10.
|
||||
target: photo_pipeline/api/app.py + frontend
|
||||
parity:
|
||||
- tests/e2e/test_frontend_shell.py::test_shell_loads_assets_without_console_or_network_errors
|
||||
- tests/e2e/test_frontend_shell.py::test_deep_link_and_reload_restore_view_and_filters
|
||||
delta: >
|
||||
The stdlib HTTP routes become the versioned FastAPI API and the static frontend shell; Python no longer interpolates HTML.
|
||||
status: resolved
|
||||
pending_story: US02-05
|
||||
status: pending
|
||||
@@ -170,20 +170,6 @@ a.link:hover { text-decoration: underline; }
|
||||
.grid th, .grid td { text-align: left; padding: 6px 8px; border-bottom: 1px solid var(--border); vertical-align: top; }
|
||||
.path { font-family: ui-monospace, monospace; font-size: 0.85em; word-break: break-all; }
|
||||
|
||||
/* Docs: prose, so it gets a reading measure rather than the full window width. */
|
||||
.doc { max-width: 72ch; }
|
||||
.doc h1 { margin-top: 0; }
|
||||
.doc h2, .doc h3 { margin-top: 1.6em; border-bottom: 1px solid var(--border); padding-bottom: 4px; }
|
||||
.doc code { font-family: ui-monospace, monospace; font-size: 0.9em; background: var(--surface-2); padding: 1px 4px; border-radius: 4px; }
|
||||
.doc pre { background: var(--surface-2); border: 1px solid var(--border); border-radius: var(--radius); padding: 12px; overflow: auto; }
|
||||
.doc pre code { background: none; padding: 0; }
|
||||
.doc table { border-collapse: collapse; width: 100%; }
|
||||
.doc th, .doc td { text-align: left; padding: 6px 8px; border-bottom: 1px solid var(--border); vertical-align: top; }
|
||||
.doc blockquote { margin: 1em 0; padding-left: 12px; border-left: 3px solid var(--border); color: var(--muted); }
|
||||
.doc img { max-width: 100%; }
|
||||
.diagram { margin: 1.5em 0; padding: 0; overflow: auto; }
|
||||
.diagram svg { max-width: 100%; height: auto; }
|
||||
|
||||
/* Narrow screens: stack the panes so blockers and actions stay reachable. */
|
||||
@media (max-width: 720px) {
|
||||
.two-pane { grid-template-columns: 1fr; }
|
||||
|
||||
@@ -19,9 +19,7 @@
|
||||
<a href="#/albums" data-nav="albums">Albums</a>
|
||||
<a href="#/renames" data-nav="renames">Renames</a>
|
||||
<a href="#/uploads" data-nav="uploads">Upload</a>
|
||||
<a href="#/archive" data-nav="archive">Archive</a>
|
||||
<a href="#/stats" data-nav="stats">Stats</a>
|
||||
<a href="#/docs" data-nav="docs">Docs</a>
|
||||
</nav>
|
||||
</header>
|
||||
<main id="app" aria-live="polite"><!-- views render here --></main>
|
||||
|
||||
@@ -2,62 +2,14 @@
|
||||
// cancellation. Every method accepts an optional { signal } from cancellable().
|
||||
export const BASE = "/api/v1";
|
||||
|
||||
// The API refuses every request without the session cookie, and every mutation
|
||||
// without this token echoed back. The token is readable only same-origin, which is
|
||||
// what makes it proof that the caller is this app and not another page.
|
||||
let csrfToken = null;
|
||||
|
||||
// A deployment reachable through a proxy trades an operator secret for that cookie.
|
||||
// Kept per tab: sessionStorage dies with the tab, and the secret never enters a URL.
|
||||
const SECRET_KEY = "pp_access_secret";
|
||||
|
||||
async function bootstrap(secret) {
|
||||
return fetch(BASE + "/session", {
|
||||
credentials: "same-origin",
|
||||
headers: secret ? { "X-Access-Secret": secret } : {},
|
||||
});
|
||||
}
|
||||
|
||||
async function session() {
|
||||
if (csrfToken === null) {
|
||||
let response = await bootstrap(sessionStorage.getItem(SECRET_KEY));
|
||||
if (response.status === 401) {
|
||||
sessionStorage.removeItem(SECRET_KEY);
|
||||
const secret = prompt("Access secret");
|
||||
if (secret) {
|
||||
response = await bootstrap(secret);
|
||||
if (response.ok) sessionStorage.setItem(SECRET_KEY, secret);
|
||||
}
|
||||
}
|
||||
const body = await response.json().catch(() => null);
|
||||
csrfToken = (body && body.csrf_token) || null;
|
||||
}
|
||||
return csrfToken || "";
|
||||
}
|
||||
|
||||
async function send(path, { signal, ...options }) {
|
||||
return fetch(BASE + path, {
|
||||
credentials: "same-origin",
|
||||
signal,
|
||||
...options,
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"X-CSRF-Token": await session(),
|
||||
...(options.headers || {}),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function request(path, { signal, ...options } = {}) {
|
||||
let response;
|
||||
try {
|
||||
response = await send(path, { signal, ...options });
|
||||
// A restarted server issues a new session; re-bootstrap once rather than
|
||||
// stranding an open tab on 401.
|
||||
if (response.status === 401) {
|
||||
csrfToken = null;
|
||||
response = await send(path, { signal, ...options });
|
||||
}
|
||||
response = await fetch(BASE + path, {
|
||||
headers: { "Content-Type": "application/json" },
|
||||
signal,
|
||||
...options,
|
||||
});
|
||||
} catch (error) {
|
||||
// A caller-cancelled fetch is not a failure; tag it so views can ignore it.
|
||||
if (error.name === "AbortError") {
|
||||
@@ -93,10 +45,8 @@ export const api = {
|
||||
request("/inventory/assets?" + new URLSearchParams(params).toString(), opts),
|
||||
listClusters: (params = {}, opts = {}) =>
|
||||
request("/duplicates/clusters?" + new URLSearchParams(params).toString(), opts),
|
||||
getCluster: (id, params = {}, opts = {}) => {
|
||||
const query = new URLSearchParams(params).toString();
|
||||
return request(`/duplicates/clusters/${encodeURIComponent(id)}${query ? `?${query}` : ""}`, opts);
|
||||
},
|
||||
getCluster: (id, opts = {}) =>
|
||||
request(`/duplicates/clusters/${encodeURIComponent(id)}`, opts),
|
||||
decide: (id, payload, opts = {}) =>
|
||||
request(`/duplicates/clusters/${encodeURIComponent(id)}/decision`, {
|
||||
method: "POST",
|
||||
@@ -188,31 +138,4 @@ export const api = {
|
||||
}),
|
||||
uploadVerifications: (id, opts = {}) =>
|
||||
request(`/upload-batches/${encodeURIComponent(id)}/verifications`, opts),
|
||||
|
||||
// ── Archive and restore: destinations, preflight, plans, recovery ────────
|
||||
archiveLocations: (opts = {}) => request("/archive-locations", opts),
|
||||
registerArchiveLocation: (payload, opts = {}) =>
|
||||
request("/archive-locations", { method: "POST", body: JSON.stringify(payload), ...opts }),
|
||||
archivePreflight: (payload, opts = {}) =>
|
||||
request("/archive-preflight", { method: "POST", body: JSON.stringify(payload), ...opts }),
|
||||
createArchivePlan: (payload, opts = {}) =>
|
||||
request("/archive-plans", { method: "POST", body: JSON.stringify(payload), ...opts }),
|
||||
listArchivePlans: (opts = {}) => request("/archive-plans", opts),
|
||||
getArchivePlan: (id, opts = {}) => request(`/archive-plans/${encodeURIComponent(id)}`, opts),
|
||||
applyArchivePlan: (id, opts = {}) =>
|
||||
request(`/archive-plans/${encodeURIComponent(id)}/apply`, { method: "POST", ...opts }),
|
||||
archiveRecovery: (opts = {}) => request("/archive-recovery", opts),
|
||||
resolveArchiveRecovery: (opts = {}) =>
|
||||
request("/archive-recovery/resolve", { method: "POST", ...opts }),
|
||||
restorePreflight: (payload, opts = {}) =>
|
||||
request("/restore-preflight", { method: "POST", body: JSON.stringify(payload), ...opts }),
|
||||
createRestorePlan: (payload, opts = {}) =>
|
||||
request("/restore-plans", { method: "POST", body: JSON.stringify(payload), ...opts }),
|
||||
listRestorePlans: (opts = {}) => request("/restore-plans", opts),
|
||||
getRestorePlan: (id, opts = {}) => request(`/restore-plans/${encodeURIComponent(id)}`, opts),
|
||||
applyRestorePlan: (id, opts = {}) =>
|
||||
request(`/restore-plans/${encodeURIComponent(id)}/apply`, { method: "POST", ...opts }),
|
||||
restoreRecovery: (opts = {}) => request("/restore-recovery", opts),
|
||||
resolveRestoreRecovery: (opts = {}) =>
|
||||
request("/restore-recovery/resolve", { method: "POST", ...opts }),
|
||||
};
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
import { api } from "./api.js";
|
||||
import { renderArchive, setArchiveRender } from "./archive.js";
|
||||
import { renderDocs } from "./docs.js";
|
||||
import { show as showIn } from "./dom.js";
|
||||
import { navigate, onRouteChange, parseHash } from "./router.js";
|
||||
import { renderRenames, setRenamesRender } from "./renames.js";
|
||||
import { renderUploads, setUploadsRender } from "./uploads.js";
|
||||
@@ -36,7 +33,7 @@ function el(tag, attrs = {}, ...children) {
|
||||
}
|
||||
|
||||
function show(...nodes) {
|
||||
showIn(root, ...nodes);
|
||||
root.replaceChildren(...nodes);
|
||||
}
|
||||
|
||||
function setActiveNav(view) {
|
||||
@@ -195,7 +192,7 @@ async function renderClusters(params) {
|
||||
" ",
|
||||
el("span", { class: `badge ${cluster.state}` }, cluster.state)
|
||||
),
|
||||
el("div", { class: "muted" }, `${cluster.member_total ?? cluster.members.length} members · confidence ${cluster.confidence}`)
|
||||
el("div", { class: "muted" }, `${cluster.members.length} members · confidence ${cluster.confidence}`)
|
||||
)
|
||||
);
|
||||
|
||||
@@ -209,11 +206,8 @@ async function renderClusters(params) {
|
||||
async function renderClusterDetail(id, extra = {}) {
|
||||
setActiveNav("duplicates");
|
||||
let cluster;
|
||||
// A cluster can hold thousands of members, so the server pages them; the page
|
||||
// asks for as many as it is currently showing (US07-06).
|
||||
const shown = extra.shown || 0;
|
||||
try {
|
||||
cluster = await api.getCluster(id, shown ? { limit: shown } : {});
|
||||
cluster = await api.getCluster(id);
|
||||
} catch (error) {
|
||||
show(errorBanner(`Failed to load cluster: ${error.message}`));
|
||||
return;
|
||||
@@ -303,19 +297,6 @@ async function renderClusterDetail(id, extra = {}) {
|
||||
nodes.push(decisionBar);
|
||||
if (extra.pending) nodes.push(confirmPanel(cluster, extra.pending));
|
||||
nodes.push(el("div", { class: "cluster-grid" }, ...members));
|
||||
const total = cluster.member_total ?? cluster.members.length;
|
||||
if (cluster.members.length < total) {
|
||||
nodes.push(
|
||||
el(
|
||||
"button",
|
||||
{
|
||||
"data-testid": "show-more-members",
|
||||
onclick: () => renderClusterDetail(id, { ...extra, shown: cluster.members.length + 100 }),
|
||||
},
|
||||
`Show more (${cluster.members.length} of ${total})`
|
||||
)
|
||||
);
|
||||
}
|
||||
show(...nodes);
|
||||
}
|
||||
|
||||
@@ -385,9 +366,7 @@ function render() {
|
||||
else if (path === "/albums") renderAlbums(root, params);
|
||||
else if (path === "/renames") renderRenames(root, params);
|
||||
else if (path === "/uploads") renderUploads(root, params);
|
||||
else if (path === "/archive") renderArchive(root, params);
|
||||
else if (path === "/stats") renderStats(root, params);
|
||||
else if (path === "/docs") renderDocs(root, params);
|
||||
else show(errorBanner("Unknown view"));
|
||||
}
|
||||
|
||||
@@ -395,6 +374,5 @@ function render() {
|
||||
setRender(render);
|
||||
setRenamesRender(render);
|
||||
setUploadsRender(render);
|
||||
setArchiveRender(render);
|
||||
onRouteChange(render);
|
||||
render();
|
||||
|
||||
@@ -1,870 +0,0 @@
|
||||
// Archive view (US06-05): preview what would leave active storage, confirm it
|
||||
// exactly, watch the transfer, recover an interrupted one, browse what is already
|
||||
// archived, and bring it back.
|
||||
//
|
||||
// Archiving is the only stage that removes originals, so this view never decides
|
||||
// anything itself: the destination's identity, every blocker, the confirmation
|
||||
// token, and what recovery may do all come from the server, and an action the
|
||||
// server would refuse is not offered. Two things follow from that. A medium that
|
||||
// is not mounted produces an instruction naming it rather than a disabled mystery,
|
||||
// and an operation whose evidence is ambiguous offers no button at all.
|
||||
import { api } from "./api.js";
|
||||
import { el, errorBanner, setActiveNav, show } from "./dom.js";
|
||||
import { subscribeJob } from "./events.js";
|
||||
import { navigate } from "./router.js";
|
||||
|
||||
let outcome = null;
|
||||
let activity = [];
|
||||
|
||||
let render = () => {};
|
||||
export function setArchiveRender(fn) {
|
||||
render = fn;
|
||||
}
|
||||
|
||||
// The per-file journal states, split into the three things an operator actually
|
||||
// wants told apart: bytes moving, bytes proven, original removed (concept §9).
|
||||
const PHASES = [
|
||||
["planned", "planned", "waiting"],
|
||||
["transferring", "transfer", "copying to the medium"],
|
||||
["verified", "verified", "archive copy hashed and manifested"],
|
||||
["removing", "removing", "removing the active original"],
|
||||
["complete", "complete", "archived and removed"],
|
||||
["failed", "failed", "left alone for a decision"],
|
||||
];
|
||||
|
||||
const AVAILABILITY_LABEL = {
|
||||
active: "in the library",
|
||||
archived_online: "archived · medium mounted",
|
||||
archived_offline: "archived · medium away",
|
||||
missing_unexpected: "missing — unexplained",
|
||||
};
|
||||
|
||||
export async function renderArchive(root, params = {}) {
|
||||
setActiveNav("archive");
|
||||
|
||||
let locations;
|
||||
try {
|
||||
locations = (await api.archiveLocations()).locations;
|
||||
} catch (error) {
|
||||
show(root, errorBanner(`Failed to load archive locations: ${error.message}`));
|
||||
return;
|
||||
}
|
||||
const location = locations.find((l) => l.id === params.location) || locations[0] || null;
|
||||
|
||||
const nodes = [el("h1", {}, "Archive"), locationsCard(locations, location, params)];
|
||||
if (!location) {
|
||||
nodes.push(
|
||||
el(
|
||||
"p",
|
||||
{ class: "muted", "data-testid": "no-locations" },
|
||||
"Register the disk, NAS share, or removable medium that will hold archived originals."
|
||||
),
|
||||
outcomeBanner()
|
||||
);
|
||||
show(root, ...nodes.filter(Boolean));
|
||||
return;
|
||||
}
|
||||
|
||||
const [preflight, archivePlans, restorePlans, recovery, restoreRecovery, archived, restore] =
|
||||
await Promise.all([
|
||||
load(() => api.archivePreflight({ location_id: location.id })),
|
||||
load(() => api.listArchivePlans()),
|
||||
load(() => api.listRestorePlans()),
|
||||
load(() => api.archiveRecovery()),
|
||||
load(() => api.restoreRecovery()),
|
||||
load(() => api.listAssets({ limit: 200 })),
|
||||
load(() => api.restorePreflight({ location_id: location.id })),
|
||||
]);
|
||||
|
||||
// Both directions share the runs table: one lane moves these originals, so one
|
||||
// history is what an operator has to reason about.
|
||||
const runs = [...plans(archivePlans), ...plans(restorePlans)].sort((a, b) =>
|
||||
(a.created_at || "").localeCompare(b.created_at || "")
|
||||
);
|
||||
const chosen = runs.find((run) => run.id === params.plan) || runs[runs.length - 1] || null;
|
||||
const selectedPlan = chosen ? await load(() => planDetailOf(chosen)) : null;
|
||||
|
||||
nodes.push(
|
||||
preflight ? previewSection(preflight, location) : null,
|
||||
preflight ? confirmBlock(preflight, location) : null,
|
||||
outcomeBanner(),
|
||||
activityLog(),
|
||||
recoverySection(mergeRecovery(recovery, restoreRecovery)),
|
||||
planList(runs, chosen && chosen.id),
|
||||
selectedPlan ? planDetail(selectedPlan) : null,
|
||||
archived ? archivedSection(archived.items, locations) : null,
|
||||
restore ? restoreSection(restore, location) : null
|
||||
);
|
||||
show(root, ...nodes.filter(Boolean));
|
||||
}
|
||||
|
||||
function plans(listed) {
|
||||
return listed ? listed.plans : [];
|
||||
}
|
||||
|
||||
function planDetailOf(run) {
|
||||
return run.direction === "restore" ? api.getRestorePlan(run.id) : api.getArchivePlan(run.id);
|
||||
}
|
||||
|
||||
// Archive and restore recovery answer the same question about the same lane, so
|
||||
// they are one list; an unresolved item of either kind blocks the other.
|
||||
function mergeRecovery(archive, restore) {
|
||||
if (!archive && !restore) return null;
|
||||
return {
|
||||
operations: [...((archive || {}).operations || []), ...((restore || {}).operations || [])],
|
||||
manual: [...((archive || {}).manual || []), ...((restore || {}).manual || [])],
|
||||
};
|
||||
}
|
||||
|
||||
// A section whose data failed to load must not take the rest of the view with it:
|
||||
// the medium being away is exactly when the archived-asset list matters most.
|
||||
async function load(call) {
|
||||
try {
|
||||
return await call();
|
||||
} catch (_) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ── locations ────────────────────────────────────────────────────────────────
|
||||
// A location is a medium, not a path: the marker's ``media_id`` is what proves the
|
||||
// right disk is mounted, so it is shown next to the state it produced.
|
||||
function locationsCard(locations, selected, params) {
|
||||
const name = el("input", {
|
||||
type: "text",
|
||||
"data-testid": "location-name",
|
||||
"aria-label": "Archive location name",
|
||||
placeholder: "External disk",
|
||||
});
|
||||
const root = el("input", {
|
||||
type: "text",
|
||||
"data-testid": "location-root",
|
||||
"aria-label": "Archive location path",
|
||||
placeholder: "/Volumes/archive",
|
||||
});
|
||||
|
||||
return el(
|
||||
"div",
|
||||
{ class: "card", "data-testid": "archive-locations" },
|
||||
el("h2", {}, "Destinations"),
|
||||
locations.length
|
||||
? el(
|
||||
"table",
|
||||
{ class: "grid", "data-testid": "locations" },
|
||||
el(
|
||||
"thead",
|
||||
{},
|
||||
el(
|
||||
"tr",
|
||||
{},
|
||||
...["", "Name", "Root", "Medium", "State", "Last seen"].map((label) =>
|
||||
el("th", { scope: "col" }, label)
|
||||
)
|
||||
)
|
||||
),
|
||||
el(
|
||||
"tbody",
|
||||
{},
|
||||
...locations.map((location) =>
|
||||
el(
|
||||
"tr",
|
||||
{
|
||||
"data-testid": "location-row",
|
||||
"data-name": location.name,
|
||||
"aria-current": selected && location.id === selected.id ? "true" : false,
|
||||
},
|
||||
el(
|
||||
"td",
|
||||
{},
|
||||
el("input", {
|
||||
type: "radio",
|
||||
name: "archive-location",
|
||||
"data-testid": "select-location",
|
||||
"aria-label": `Use ${location.name}`,
|
||||
checked: selected && location.id === selected.id ? "checked" : false,
|
||||
onchange: () => navigate("/archive", { ...params, location: location.id }),
|
||||
})
|
||||
),
|
||||
el("td", { "data-testid": "location-label" }, location.name),
|
||||
el("td", { class: "path", "data-testid": "location-path" }, location.root),
|
||||
el("td", { class: "path", "data-testid": "location-media" }, location.media_id),
|
||||
el(
|
||||
"td",
|
||||
{},
|
||||
el(
|
||||
"span",
|
||||
{
|
||||
class: `badge ${location.state === "online" ? "complete" : "attention"}`,
|
||||
"data-testid": "location-state",
|
||||
},
|
||||
location.state
|
||||
)
|
||||
),
|
||||
el("td", { class: "muted" }, location.last_seen_at || "never")
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
: null,
|
||||
selected && selected.state !== "online" ? mountInstruction(selected) : null,
|
||||
el(
|
||||
"div",
|
||||
{ class: "toolbar" },
|
||||
name,
|
||||
root,
|
||||
el(
|
||||
"button",
|
||||
{
|
||||
"data-testid": "register-location",
|
||||
onclick: () =>
|
||||
run(() => api.registerArchiveLocation({ name: name.value, root: root.value })),
|
||||
},
|
||||
"Register destination"
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
// The one thing the app cannot do for the user: name the medium to connect.
|
||||
function mountInstruction(location) {
|
||||
const detail =
|
||||
location.state === "wrong_volume"
|
||||
? `A different medium is mounted at ${location.root}.`
|
||||
: `Nothing is mounted at ${location.root}.`;
|
||||
return el(
|
||||
"div",
|
||||
{ class: "confirm", role: "status", "data-testid": "mount-instruction" },
|
||||
`${detail} Connect “${location.name}” (medium ${location.media_id}) and mount it there, ` +
|
||||
"then reload this view. Archived photos stay listed and searchable meanwhile."
|
||||
);
|
||||
}
|
||||
|
||||
// ── preview ──────────────────────────────────────────────────────────────────
|
||||
function previewSection(preflight, location) {
|
||||
const totals = preflight.totals;
|
||||
const capacity = preflight.capacity;
|
||||
const rows = preflight.albums.map((album) =>
|
||||
el(
|
||||
"tr",
|
||||
{ "data-testid": "archive-album-row", "data-album": album.album },
|
||||
el("td", { "data-testid": "album-name" }, album.album),
|
||||
el("td", { class: "path", "data-testid": "album-folder" }, album.folder),
|
||||
el("td", { class: "path", "data-testid": "album-destination" }, album.destination),
|
||||
el(
|
||||
"td",
|
||||
{ "data-testid": "album-method" },
|
||||
album.transfer_method === "move" ? "move (same filesystem)" : "copy · verify · remove"
|
||||
),
|
||||
el("td", { "data-testid": "album-assets" }, String(album.asset_count)),
|
||||
el("td", { "data-testid": "album-reclaim" }, bytes(album.reclaimable_bytes)),
|
||||
el(
|
||||
"td",
|
||||
{},
|
||||
el("span", { class: `badge ${album.state}`, "data-testid": "album-state" }, album.state),
|
||||
...album.blockers.map((blocker) =>
|
||||
el(
|
||||
"div",
|
||||
{ class: "blocker", "data-testid": "album-blocker", "data-code": blocker.code },
|
||||
blocker.message
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
return el(
|
||||
"div",
|
||||
{ "data-testid": "archive-preview" },
|
||||
el("h2", {}, "Preview"),
|
||||
el(
|
||||
"div",
|
||||
{ class: "decision-bar" },
|
||||
el(
|
||||
"span",
|
||||
{ class: "badge", "data-testid": "destination-identity" },
|
||||
`${location.name} · ${location.media_id}`
|
||||
),
|
||||
el("span", { class: "badge", "data-testid": "total-albums" }, `${totals.albums} album(s)`),
|
||||
el("span", { class: "badge", "data-testid": "total-assets" }, `${totals.assets} photo(s)`),
|
||||
el(
|
||||
"span",
|
||||
{ class: "badge", "data-testid": "total-reclaim" },
|
||||
`${bytes(totals.bytes)} reclaimable`
|
||||
),
|
||||
el(
|
||||
"span",
|
||||
{
|
||||
class: `badge ${capacity.sufficient ? "complete" : "blocked"}`,
|
||||
"data-testid": "capacity",
|
||||
},
|
||||
`free ${bytes(capacity.free_bytes)} · reserve ${bytes(capacity.reserve_bytes)}`
|
||||
)
|
||||
),
|
||||
blockerList(preflight.blockers, "preflight-blockers", "preflight-blocker", "This scope cannot be archived yet"),
|
||||
rows.length
|
||||
? el(
|
||||
"table",
|
||||
{ class: "grid", "data-testid": "archive-albums" },
|
||||
el(
|
||||
"thead",
|
||||
{},
|
||||
el(
|
||||
"tr",
|
||||
{},
|
||||
...["Album", "Folder", "Destination", "Transfer", "Photos", "Reclaims", "State"].map(
|
||||
(label) => el("th", { scope: "col" }, label)
|
||||
)
|
||||
)
|
||||
),
|
||||
el("tbody", {}, ...rows)
|
||||
)
|
||||
: el(
|
||||
"p",
|
||||
{ class: "muted", "data-testid": "no-albums" },
|
||||
"No album has a verified upload whose bytes are still unchanged, so nothing may be archived."
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function blockerList(blockers, containerId, itemId, title) {
|
||||
if (!blockers || !blockers.length) return null;
|
||||
return el(
|
||||
"div",
|
||||
{ class: "alert", role: "alert", "data-testid": containerId },
|
||||
el("strong", {}, title),
|
||||
el(
|
||||
"ul",
|
||||
{},
|
||||
...blockers.map((blocker) =>
|
||||
el("li", { "data-testid": itemId, "data-code": blocker.code }, `${blocker.code}: ${blocker.message}`)
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
// ── confirmation ─────────────────────────────────────────────────────────────
|
||||
function confirmBlock(preflight, location) {
|
||||
const ready = preflight.state === "ready";
|
||||
const totals = preflight.totals;
|
||||
return el(
|
||||
"div",
|
||||
{ class: "card", "data-testid": "confirm" },
|
||||
el("h2", {}, "Confirm"),
|
||||
el(
|
||||
"p",
|
||||
{ class: "muted", "data-testid": "confirm-token" },
|
||||
`Preflight ${preflight.token.slice(0, 20)}… · ${location.name}`
|
||||
),
|
||||
el(
|
||||
"p",
|
||||
{ "data-testid": "archive-note" },
|
||||
"Archiving removes each original from the library — but only after its copy on " +
|
||||
"the medium has been written, hashed, and recorded in the manifest. The photos " +
|
||||
"stay searchable and deduplicable while the medium is away, and can be restored " +
|
||||
"from this view."
|
||||
),
|
||||
el(
|
||||
"div",
|
||||
{ class: "toolbar" },
|
||||
el(
|
||||
"button",
|
||||
{
|
||||
class: "primary",
|
||||
"data-testid": "start-archive",
|
||||
disabled: ready ? false : "disabled",
|
||||
title: ready ? false : "resolve the blockers above first",
|
||||
onclick: () =>
|
||||
run(async () => {
|
||||
const plan = await api.createArchivePlan({
|
||||
location_id: location.id,
|
||||
token: preflight.token,
|
||||
});
|
||||
const started = await api.applyArchivePlan(plan.id);
|
||||
watch(started.job.id, plan.id, "archive");
|
||||
return { archiving: plan.asset_count };
|
||||
}),
|
||||
},
|
||||
`Archive ${totals.ready_albums} album(s) · reclaim ${bytes(totals.bytes)}`
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
// ── plans and progress ───────────────────────────────────────────────────────
|
||||
function planList(runs, selectedId) {
|
||||
if (!runs.length) {
|
||||
return el("p", { class: "muted", "data-testid": "no-plans" }, "Nothing has been archived yet.");
|
||||
}
|
||||
return el(
|
||||
"div",
|
||||
{ "data-testid": "archive-plans" },
|
||||
el("h2", {}, "Runs"),
|
||||
el(
|
||||
"table",
|
||||
{ class: "grid", "data-testid": "plans" },
|
||||
el(
|
||||
"thead",
|
||||
{},
|
||||
el(
|
||||
"tr",
|
||||
{},
|
||||
...["Created", "Direction", "State", "Photos", "Bytes"].map((l) =>
|
||||
el("th", { scope: "col" }, l)
|
||||
)
|
||||
)
|
||||
),
|
||||
el(
|
||||
"tbody",
|
||||
{},
|
||||
...runs.map((plan) =>
|
||||
el(
|
||||
"tr",
|
||||
{
|
||||
"data-testid": "plan-row",
|
||||
"data-plan": plan.id,
|
||||
"data-direction": plan.direction,
|
||||
"aria-current": plan.id === selectedId ? "true" : false,
|
||||
},
|
||||
el(
|
||||
"td",
|
||||
{},
|
||||
el("a", { class: "link", href: `#/archive?plan=${encodeURIComponent(plan.id)}` }, plan.created_at || plan.id)
|
||||
),
|
||||
el("td", { "data-testid": "plan-direction" }, plan.direction),
|
||||
el("td", {}, el("span", { class: `badge ${plan.state}`, "data-testid": "plan-state" }, plan.state)),
|
||||
el("td", {}, String(plan.asset_count)),
|
||||
el("td", {}, bytes(plan.byte_size))
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function planDetail(plan) {
|
||||
const operations = plan.operations || [];
|
||||
const counts = {};
|
||||
for (const operation of operations) {
|
||||
counts[operation.journal_state] = (counts[operation.journal_state] || 0) + 1;
|
||||
}
|
||||
return el(
|
||||
"div",
|
||||
{ class: "card", "data-testid": "plan-detail", "data-plan": plan.id },
|
||||
el("h2", {}, `${plan.direction === "restore" ? "Restore" : "Archive"} run ${plan.created_at || plan.id}`),
|
||||
// Transfer, verification, and removal are separate answers to separate
|
||||
// questions: what has moved, what is proven, and what is already gone.
|
||||
el(
|
||||
"div",
|
||||
{ class: "decision-bar", "data-testid": "plan-progress" },
|
||||
el("span", { class: `badge ${plan.state}`, "data-testid": "detail-state" }, plan.state),
|
||||
...PHASES.map(([key, label, title]) =>
|
||||
el(
|
||||
"span",
|
||||
{ class: `badge ${key}`, "data-testid": `count-${label}`, title },
|
||||
`${label}: ${counts[key] ?? 0}`
|
||||
)
|
||||
)
|
||||
),
|
||||
el(
|
||||
"table",
|
||||
{ class: "grid", "data-testid": "operations" },
|
||||
el(
|
||||
"thead",
|
||||
{},
|
||||
el(
|
||||
"tr",
|
||||
{},
|
||||
...["Photo", "Destination", "Phase", "Attempts", "Problem"].map((l) =>
|
||||
el("th", { scope: "col" }, l)
|
||||
)
|
||||
)
|
||||
),
|
||||
el(
|
||||
"tbody",
|
||||
{},
|
||||
...operations.map((operation) =>
|
||||
el(
|
||||
"tr",
|
||||
{ "data-testid": "operation-row", "data-asset-id": operation.asset_id },
|
||||
el("td", { class: "path", "data-testid": "operation-source" }, operation.source_path),
|
||||
el("td", { class: "path", "data-testid": "operation-destination" }, operation.destination_path),
|
||||
el(
|
||||
"td",
|
||||
{},
|
||||
el(
|
||||
"span",
|
||||
{ class: `badge ${operation.journal_state}`, "data-testid": "operation-phase" },
|
||||
phaseLabel(operation.journal_state)
|
||||
)
|
||||
),
|
||||
el("td", {}, String(operation.attempt_count)),
|
||||
el(
|
||||
"td",
|
||||
{ class: "muted", "data-testid": "operation-error", "data-code": operation.error_code || "" },
|
||||
operation.error_code ? `${operation.error_code}: ${operation.error_message || ""}` : "—"
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function phaseLabel(state) {
|
||||
const found = PHASES.find(([key]) => key === state);
|
||||
return found ? found[1] : state;
|
||||
}
|
||||
|
||||
// ── recovery ─────────────────────────────────────────────────────────────────
|
||||
// What an interrupted run left behind, straight from the journal plus the files on
|
||||
// disk. Only the operations the server itself classified as resolvable get an
|
||||
// action; ambiguous ones are shown with their evidence and no button.
|
||||
function recoverySection(recovery) {
|
||||
const operations = recovery ? recovery.operations : [];
|
||||
if (!operations.length) return null;
|
||||
const manual = recovery.manual || [];
|
||||
const resolvable = operations.length - manual.length;
|
||||
|
||||
return el(
|
||||
"div",
|
||||
{ class: "card", "data-testid": "archive-recovery" },
|
||||
el("h2", {}, "Interrupted work"),
|
||||
el(
|
||||
"p",
|
||||
{ "data-testid": "recovery-summary" },
|
||||
`${operations.length} operation(s) did not finish · ${resolvable} resolvable · ` +
|
||||
`${manual.length} need a decision`
|
||||
),
|
||||
el(
|
||||
"table",
|
||||
{ class: "grid", "data-testid": "recovery-operations" },
|
||||
el(
|
||||
"thead",
|
||||
{},
|
||||
el(
|
||||
"tr",
|
||||
{},
|
||||
...["Photo", "Phase", "Verdict", "Evidence"].map((l) => el("th", { scope: "col" }, l))
|
||||
)
|
||||
),
|
||||
el(
|
||||
"tbody",
|
||||
{},
|
||||
...operations.map((verdict) =>
|
||||
el(
|
||||
"tr",
|
||||
{
|
||||
"data-testid": "recovery-row",
|
||||
"data-classification": verdict.classification,
|
||||
"data-direction": verdict.direction,
|
||||
},
|
||||
el("td", { class: "path", "data-testid": "recovery-source" }, verdict.source_path),
|
||||
el("td", {}, el("span", { class: "badge" }, phaseLabel(verdict.journal_state))),
|
||||
el(
|
||||
"td",
|
||||
{},
|
||||
el(
|
||||
"span",
|
||||
{
|
||||
class: `badge ${verdict.classification === "manual" ? "blocked" : "attention"}`,
|
||||
"data-testid": "recovery-verdict",
|
||||
},
|
||||
verdict.classification
|
||||
)
|
||||
),
|
||||
el(
|
||||
"td",
|
||||
{ class: "muted", "data-testid": "recovery-reason" },
|
||||
`${verdict.reason} (source ${verdict.source_exists ? "present" : "absent"}, ` +
|
||||
`archive copy ${verdict.destination_matches ? "verified" : verdict.destination_exists ? "different bytes" : "absent"})`
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
),
|
||||
manual.length
|
||||
? el(
|
||||
"div",
|
||||
{ class: "alert", role: "alert", "data-testid": "recovery-manual" },
|
||||
`${manual.length} operation(s) cannot be resolved from the evidence. Nothing will be ` +
|
||||
"removed or retried for them here: inspect the medium and the library, then decide."
|
||||
)
|
||||
: null,
|
||||
el(
|
||||
"div",
|
||||
{ class: "toolbar" },
|
||||
resolvable
|
||||
? el(
|
||||
"button",
|
||||
{
|
||||
class: "primary",
|
||||
"data-testid": "resolve-recovery",
|
||||
onclick: () => run(() => api.resolveArchiveRecovery()),
|
||||
},
|
||||
`Finish ${resolvable} recoverable operation(s)`
|
||||
)
|
||||
: el(
|
||||
"span",
|
||||
{ class: "muted", "data-testid": "no-safe-recovery" },
|
||||
"No operation can be finished safely from here."
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
// ── archived assets ──────────────────────────────────────────────────────────
|
||||
// Browsing what is already archived, including while the medium is away: the
|
||||
// retained protected preview and the recorded hashes are the evidence, so the row
|
||||
// stays complete and honest instead of turning into a missing file.
|
||||
function archivedSection(assets, locations) {
|
||||
const archived = assets.filter((asset) => asset.availability_state !== "active");
|
||||
if (!archived.length) return null;
|
||||
const names = Object.fromEntries(locations.map((location) => [location.id, location.name]));
|
||||
|
||||
return el(
|
||||
"div",
|
||||
{ "data-testid": "archived-assets" },
|
||||
el("h2", {}, "Archived photos"),
|
||||
el(
|
||||
"table",
|
||||
{ class: "grid", "data-testid": "archived" },
|
||||
el(
|
||||
"thead",
|
||||
{},
|
||||
el(
|
||||
"tr",
|
||||
{},
|
||||
...["Preview", "Archived as", "Medium", "Availability", "Size"].map((l) =>
|
||||
el("th", { scope: "col" }, l)
|
||||
)
|
||||
)
|
||||
),
|
||||
el(
|
||||
"tbody",
|
||||
{},
|
||||
...archived.map((asset) =>
|
||||
el(
|
||||
"tr",
|
||||
{ "data-testid": "archived-row", "data-asset-id": asset.id },
|
||||
el(
|
||||
"td",
|
||||
{},
|
||||
el("img", {
|
||||
"data-testid": "archived-preview",
|
||||
width: 96,
|
||||
loading: "lazy",
|
||||
src: api.thumbnailUrl(asset.id, 256),
|
||||
alt: `Preview of ${asset.archive_path || asset.id}`,
|
||||
})
|
||||
),
|
||||
el("td", { class: "path", "data-testid": "archived-path" }, asset.archive_path || "—"),
|
||||
el(
|
||||
"td",
|
||||
{ "data-testid": "archived-medium" },
|
||||
names[asset.archive_location_id] || asset.archive_location_id || "—"
|
||||
),
|
||||
el(
|
||||
"td",
|
||||
{},
|
||||
el(
|
||||
"span",
|
||||
{
|
||||
class: `badge ${asset.availability_state === "archived_online" ? "complete" : "attention"}`,
|
||||
"data-testid": "archived-availability",
|
||||
},
|
||||
AVAILABILITY_LABEL[asset.availability_state] || asset.availability_state
|
||||
)
|
||||
),
|
||||
el("td", {}, bytes(asset.byte_size))
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
// ── restore ──────────────────────────────────────────────────────────────────
|
||||
function restoreSection(restore, location) {
|
||||
const ready = restore.state === "ready";
|
||||
const items = restore.items || [];
|
||||
if (!items.length && !restore.blockers.length) return null;
|
||||
|
||||
return el(
|
||||
"div",
|
||||
{ class: "card", "data-testid": "restore" },
|
||||
el("h2", {}, "Restore"),
|
||||
el(
|
||||
"p",
|
||||
{ "data-testid": "restore-note" },
|
||||
"Restoring copies the archived bytes back into the library and leaves the archive " +
|
||||
"copy where it is. A name that is already taken is never overwritten: the photo " +
|
||||
"comes back beside it under a visibly different name."
|
||||
),
|
||||
blockerList(restore.blockers, "restore-blockers", "restore-blocker", "This restore cannot run yet"),
|
||||
items.length
|
||||
? el(
|
||||
"table",
|
||||
{ class: "grid", "data-testid": "restore-items" },
|
||||
el(
|
||||
"thead",
|
||||
{},
|
||||
el(
|
||||
"tr",
|
||||
{},
|
||||
...["Archived as", "Comes back as", "Size", "State"].map((l) =>
|
||||
el("th", { scope: "col" }, l)
|
||||
)
|
||||
)
|
||||
),
|
||||
el(
|
||||
"tbody",
|
||||
{},
|
||||
...items.map((item) =>
|
||||
el(
|
||||
"tr",
|
||||
{ "data-testid": "restore-row", "data-asset-id": item.asset_id },
|
||||
el("td", { class: "path", "data-testid": "restore-source" }, item.archive_path),
|
||||
el("td", { class: "path", "data-testid": "restore-destination" }, item.destination_path || "—"),
|
||||
el("td", {}, bytes(item.byte_size)),
|
||||
el(
|
||||
"td",
|
||||
{},
|
||||
item.blockers.length
|
||||
? el(
|
||||
"span",
|
||||
{
|
||||
class: "badge blocked",
|
||||
"data-testid": "restore-item-blocker",
|
||||
"data-code": item.blockers[0].code,
|
||||
},
|
||||
item.blockers[0].code
|
||||
)
|
||||
: el("span", { class: "badge ready", "data-testid": "restore-item-state" }, "ready")
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
: null,
|
||||
el(
|
||||
"div",
|
||||
{ class: "toolbar" },
|
||||
el(
|
||||
"button",
|
||||
{
|
||||
class: "primary",
|
||||
"data-testid": "start-restore",
|
||||
disabled: ready ? false : "disabled",
|
||||
title: ready ? false : "the medium and every archived copy must check out first",
|
||||
onclick: () =>
|
||||
run(async () => {
|
||||
const plan = await api.createRestorePlan({
|
||||
location_id: location.id,
|
||||
token: restore.token,
|
||||
});
|
||||
const started = await api.applyRestorePlan(plan.id);
|
||||
watch(started.job.id, plan.id, "restore");
|
||||
return { restoring: plan.asset_count };
|
||||
}),
|
||||
},
|
||||
`Restore ${items.length} photo(s) from ${location.name}`
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
// ── running commands ─────────────────────────────────────────────────────────
|
||||
async function run(action) {
|
||||
try {
|
||||
outcome = { kind: "ok", result: await action() };
|
||||
} catch (error) {
|
||||
outcome = error.status === 409 ? { kind: "conflict", error } : { kind: "error", error };
|
||||
}
|
||||
render();
|
||||
}
|
||||
|
||||
// Live job activity. The plan panel is refreshed on its own tick because the
|
||||
// journal advances per file, not per job event; a full re-render would re-run
|
||||
// preflight (which re-hashes the library), so that happens once when the job ends.
|
||||
const REFRESH_MS = 500;
|
||||
|
||||
function watch(jobId, planId, kind) {
|
||||
activity = [`Started ${kind} job ${jobId}`];
|
||||
const tick = setInterval(() => refreshPlan(planId, kind), REFRESH_MS);
|
||||
subscribeJob(jobId, {
|
||||
onEvent: (event) => {
|
||||
activity.push(`${event.type}${event.message ? ": " + event.message : ""}`);
|
||||
const log = document.querySelector('[data-testid="archive-activity"]');
|
||||
if (log) log.textContent = activity.join("\n");
|
||||
},
|
||||
onDone: () => {
|
||||
clearInterval(tick);
|
||||
activity.push("done");
|
||||
render();
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function refreshPlan(planId, kind) {
|
||||
const node = document.querySelector(`[data-testid="plan-detail"][data-plan="${planId}"]`);
|
||||
if (!node) return; // the user navigated away from the running plan
|
||||
try {
|
||||
const plan = await (kind === "restore" ? api.getRestorePlan(planId) : api.getArchivePlan(planId));
|
||||
node.replaceWith(planDetail(plan));
|
||||
} catch (_) {
|
||||
// Transient; the next tick tries again and the job's end re-renders anyway.
|
||||
}
|
||||
}
|
||||
|
||||
function activityLog() {
|
||||
return el(
|
||||
"pre",
|
||||
{
|
||||
class: "activity-log",
|
||||
role: "status",
|
||||
"aria-live": "polite",
|
||||
"data-testid": "archive-activity",
|
||||
},
|
||||
activity.join("\n")
|
||||
);
|
||||
}
|
||||
|
||||
function outcomeBanner() {
|
||||
if (!outcome) return null;
|
||||
if (outcome.kind === "conflict") {
|
||||
return el(
|
||||
"div",
|
||||
{ class: "alert", role: "alert", "data-testid": "conflict" },
|
||||
`The server refused this: ${outcome.error.message}. Nothing was moved or removed; ` +
|
||||
"the state below is the server's current one — review it and decide again."
|
||||
);
|
||||
}
|
||||
if (outcome.kind === "error") {
|
||||
return el(
|
||||
"div",
|
||||
{ class: "alert", role: "alert", "data-testid": "archive-error" },
|
||||
`Failed: ${outcome.error.message}`
|
||||
);
|
||||
}
|
||||
const result = outcome.result || {};
|
||||
const message =
|
||||
result.archiving !== undefined
|
||||
? `Archiving ${result.archiving} photo(s). Originals are removed only after their copies verify.`
|
||||
: result.restoring !== undefined
|
||||
? `Restoring ${result.restoring} photo(s) into the library.`
|
||||
: "Done — the state below is the server's.";
|
||||
return el("div", { class: "alert", role: "status", "data-testid": "archive-result" }, message);
|
||||
}
|
||||
|
||||
// ── formatting ───────────────────────────────────────────────────────────────
|
||||
function bytes(value) {
|
||||
if (value == null) return "unknown";
|
||||
const units = ["B", "kB", "MB", "GB", "TB"];
|
||||
let size = value;
|
||||
let unit = 0;
|
||||
while (size >= 1000 && unit < units.length - 1) {
|
||||
size /= 1000;
|
||||
unit += 1;
|
||||
}
|
||||
return `${unit === 0 ? size : size.toFixed(1)} ${units[unit]}`;
|
||||
}
|
||||
@@ -1,263 +0,0 @@
|
||||
// The documentation view (US09-01): the manuals in `docs/`, rendered in the app.
|
||||
//
|
||||
// The same markdown files are the repository's documentation and the deployment's
|
||||
// documentation. An operator who was handed a URL and an access secret has no
|
||||
// checkout in front of them, and the network the application runs on is not assumed
|
||||
// to reach a CDN — so the renderer is vendored and everything here is same-origin.
|
||||
//
|
||||
// Nothing on this page is authenticated. It is served from the static mount beside
|
||||
// the application shell, exactly like `index.html`: the troubleshooting page is
|
||||
// needed most by whoever cannot get past the access secret, and no documentation
|
||||
// file contains anything a session would protect.
|
||||
//
|
||||
// `marked` and `mermaid` are loaded lazily, on the first documentation page and the
|
||||
// first diagram. They are large, and every other view does without them.
|
||||
import { el, errorBanner, setActiveNav } from "./dom.js";
|
||||
|
||||
const DOCS_BASE = "/docs";
|
||||
const VENDOR = "/app/js/vendor";
|
||||
const INDEX_PAGE = "index";
|
||||
// A page name comes from the hash, so it is caller input: no scheme, no traversal,
|
||||
// no absolute path. The server would refuse those too; this refuses them earlier and
|
||||
// without a request that looks like an attempt.
|
||||
const PAGE_PATTERN = /^[\w-]+(\/[\w-]+)*$/;
|
||||
|
||||
let markedPromise;
|
||||
let mermaidPromise;
|
||||
|
||||
function loadMarked() {
|
||||
markedPromise ??= import(`${VENDOR}/marked.esm.js`).then((module) => module.marked);
|
||||
return markedPromise;
|
||||
}
|
||||
|
||||
// mermaid ships one large UMD bundle rather than a self-contained ES module, so it
|
||||
// arrives through a script element. `script-src 'self'` allows it because it is ours
|
||||
// and same-origin; nothing here relaxes that.
|
||||
function loadMermaid() {
|
||||
mermaidPromise ??= new Promise((resolve, reject) => {
|
||||
const script = document.createElement("script");
|
||||
script.src = `${VENDOR}/mermaid.min.js`;
|
||||
script.onload = () => resolve(window.mermaid);
|
||||
script.onerror = () => reject(new Error("the diagram renderer could not be loaded"));
|
||||
document.head.appendChild(script);
|
||||
});
|
||||
return mermaidPromise;
|
||||
}
|
||||
|
||||
// ── the pure parts, unit-tested in js/tests/unit.js ──────────────────────────
|
||||
|
||||
/** A stable, readable heading anchor. Letters and digits of any script survive. */
|
||||
export function slug(text) {
|
||||
const cleaned = String(text)
|
||||
.toLowerCase()
|
||||
.trim()
|
||||
.replace(/[^\p{L}\p{N}\s-]/gu, "")
|
||||
.replace(/[\s-]+/g, "-")
|
||||
.replace(/^-|-$/g, "");
|
||||
return cleaned || "section";
|
||||
}
|
||||
|
||||
/** Assign every heading an id, disambiguating repeats the way a reader would
|
||||
* expect: the first `Notes` keeps `#notes`, the second becomes `#notes-2`. */
|
||||
export function assignHeadingIds(headings) {
|
||||
const used = new Map();
|
||||
for (const heading of headings) {
|
||||
const base = slug(heading.textContent);
|
||||
const seen = (used.get(base) || 0) + 1;
|
||||
used.set(base, seen);
|
||||
heading.id = seen === 1 ? base : `${base}-${seen}`;
|
||||
}
|
||||
return headings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Where a link inside a documentation page should go.
|
||||
*
|
||||
* Markdown links between documents are relative file paths, which a browser would
|
||||
* treat as downloads that leave the application. Returns the in-app target for a
|
||||
* link to another document or to a heading, and `null` for anything else — external
|
||||
* links stay exactly as the author wrote them.
|
||||
*/
|
||||
export function resolveDocLink(currentPage, href) {
|
||||
if (!href) return null;
|
||||
if (/^[a-z][a-z\d+.-]*:/i.test(href) || href.startsWith("//")) return null;
|
||||
if (href.startsWith("#")) return { page: currentPage, anchor: href.slice(1) };
|
||||
const [target, anchor = ""] = href.split("#");
|
||||
if (!/\.md$/i.test(target)) return null;
|
||||
// Resolved against the current document, so `../guides/x.md` means what it means
|
||||
// in the repository. The origin is a placeholder; only the path is used.
|
||||
const resolved = new URL(target, `https://docs.invalid/${currentPage}.md`);
|
||||
const page = decodeURIComponent(resolved.pathname).replace(/^\//, "").replace(/\.md$/i, "");
|
||||
return PAGE_PATTERN.test(page) ? { page, anchor } : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Point every in-app link at its route, and leave every other link alone.
|
||||
*
|
||||
* The resolved page is kept on the element, so the reading order can be read back
|
||||
* from the rendered index rather than parsed out of the markdown a second time.
|
||||
*/
|
||||
export function rewriteLinks(article, page) {
|
||||
for (const link of article.querySelectorAll("a[href]")) {
|
||||
const href = link.getAttribute("href");
|
||||
const target = resolveDocLink(page, href);
|
||||
if (target) {
|
||||
link.setAttribute("href", docHash(target.page, target.anchor));
|
||||
link.dataset.docPage = target.page;
|
||||
} else if (/^https?:/i.test(href)) {
|
||||
link.setAttribute("rel", "noreferrer noopener");
|
||||
link.setAttribute("target", "_blank");
|
||||
}
|
||||
}
|
||||
return article;
|
||||
}
|
||||
|
||||
/** The reading order, taken from the index document itself — one list, in one
|
||||
* place, that is equally the index on Gitea and the navigation here. */
|
||||
export function documentIndex(indexElement) {
|
||||
const pages = [];
|
||||
for (const link of indexElement.querySelectorAll("a[data-doc-page]")) {
|
||||
const page = link.dataset.docPage;
|
||||
if (page !== INDEX_PAGE && !pages.some((entry) => entry.page === page)) {
|
||||
pages.push({ page, title: link.textContent.trim() });
|
||||
}
|
||||
}
|
||||
return pages;
|
||||
}
|
||||
|
||||
// ── the view ─────────────────────────────────────────────────────────────────
|
||||
|
||||
async function fetchPage(page) {
|
||||
const response = await fetch(`${DOCS_BASE}/${page}.md`, { headers: { accept: "text/markdown" } });
|
||||
if (!response.ok) throw new Error(`documentation page unavailable: ${response.status}`);
|
||||
return response.text();
|
||||
}
|
||||
|
||||
async function toArticle(markdown, page) {
|
||||
const marked = await loadMarked();
|
||||
const article = el("article", { class: "doc", "data-testid": "doc", "data-page": page });
|
||||
// The only innerHTML in the application, and deliberate: rendering markdown *is*
|
||||
// producing HTML. The input is a file from this repository, and the page's CSP
|
||||
// (`script-src 'self'`, no `'unsafe-inline'`) means injected script and inline
|
||||
// handlers do not run even if one ever were not.
|
||||
article.innerHTML = marked.parse(markdown, { async: false });
|
||||
assignHeadingIds(article.querySelectorAll("h1, h2, h3, h4, h5, h6"));
|
||||
rewriteLinks(article, page);
|
||||
return article;
|
||||
}
|
||||
|
||||
function docHash(page, anchor = "") {
|
||||
const query = new URLSearchParams(anchor ? { page, anchor } : { page });
|
||||
return `#/docs?${query}`;
|
||||
}
|
||||
|
||||
/** Diagrams are authored as ```mermaid blocks so the source is diffable and Gitea
|
||||
* renders them natively. A failure here replaces the diagram, never the page. */
|
||||
async function renderDiagrams(article) {
|
||||
const blocks = [...article.querySelectorAll("pre > code.language-mermaid")];
|
||||
if (!blocks.length) return;
|
||||
let mermaid;
|
||||
try {
|
||||
mermaid = await loadMermaid();
|
||||
mermaid.initialize({ startOnLoad: false, securityLevel: "strict", theme: "dark" });
|
||||
} catch (error) {
|
||||
blocks.forEach((block) => block.closest("pre").replaceWith(errorBanner(error.message)));
|
||||
return;
|
||||
}
|
||||
for (const [index, block] of blocks.entries()) {
|
||||
const figure = el("figure", { class: "diagram", "data-testid": "diagram" });
|
||||
block.closest("pre").replaceWith(figure);
|
||||
try {
|
||||
const { svg } = await mermaid.render(`diagram-${index}-${Date.now()}`, block.textContent);
|
||||
figure.innerHTML = svg;
|
||||
} catch (error) {
|
||||
figure.replaceWith(errorBanner(`Diagram could not be drawn: ${error.message}`));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function notFound() {
|
||||
return el(
|
||||
"div",
|
||||
{ class: "alert", role: "alert", "data-testid": "doc-not-found" },
|
||||
"That documentation page does not exist. ",
|
||||
el("a", { class: "link", href: docHash(INDEX_PAGE) }, "Back to the documentation index")
|
||||
);
|
||||
}
|
||||
|
||||
function sidebar(pages, current) {
|
||||
return el(
|
||||
"nav",
|
||||
{ class: "card", "aria-label": "Documentation" },
|
||||
el("h2", {}, "Documentation"),
|
||||
el(
|
||||
"ul",
|
||||
{ class: "album-list", "data-testid": "doc-pages" },
|
||||
el(
|
||||
"li",
|
||||
{},
|
||||
el(
|
||||
"a",
|
||||
{
|
||||
href: docHash(INDEX_PAGE),
|
||||
"aria-current": current === INDEX_PAGE ? "true" : false,
|
||||
},
|
||||
"Index"
|
||||
)
|
||||
),
|
||||
...pages.map(({ page, title }) =>
|
||||
el(
|
||||
"li",
|
||||
{},
|
||||
el(
|
||||
"a",
|
||||
{
|
||||
href: docHash(page),
|
||||
"data-page": page,
|
||||
"aria-current": page === current ? "true" : false,
|
||||
},
|
||||
title
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
export async function renderDocs(root, params = {}) {
|
||||
setActiveNav("docs");
|
||||
const requested = params.page || INDEX_PAGE;
|
||||
const page = PAGE_PATTERN.test(requested) ? requested : "";
|
||||
|
||||
let indexArticle;
|
||||
try {
|
||||
indexArticle = await toArticle(await fetchPage(INDEX_PAGE), INDEX_PAGE);
|
||||
} catch (error) {
|
||||
root.replaceChildren(errorBanner(`Documentation is unavailable: ${error.message}`));
|
||||
return;
|
||||
}
|
||||
const pages = documentIndex(indexArticle);
|
||||
|
||||
let article;
|
||||
if (!page) article = notFound();
|
||||
else if (page === INDEX_PAGE) article = indexArticle;
|
||||
else {
|
||||
try {
|
||||
article = await toArticle(await fetchPage(page), page);
|
||||
} catch {
|
||||
article = notFound();
|
||||
}
|
||||
}
|
||||
|
||||
root.replaceChildren(el("div", { class: "two-pane" }, sidebar(pages, page), article));
|
||||
await renderDiagrams(article);
|
||||
scrollToAnchor(params.anchor);
|
||||
}
|
||||
|
||||
// A documentation anchor cannot live in the hash — the hash is the route — so it
|
||||
// travels as a parameter and is applied after the page renders.
|
||||
function scrollToAnchor(anchor) {
|
||||
if (!anchor) return;
|
||||
const target = document.getElementById(anchor);
|
||||
if (target) target.scrollIntoView({ block: "start" });
|
||||
}
|
||||
@@ -21,19 +21,6 @@ export function errorBanner(message) {
|
||||
return el("div", { class: "alert", role: "alert" }, message);
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace a container's contents, dropping the absent ones.
|
||||
*
|
||||
* `el` already ignores null children, but `replaceChildren` does not: it stringifies
|
||||
* them, so an optional node that is not there renders the word "null" on the page.
|
||||
* Every view builds optional nodes — a banner only while a job runs, a conflict only
|
||||
* after a 409 — so the filter belongs here rather than in each caller. It was missing
|
||||
* from one of them, and a documentation screenshot is where that turned up (US09-04).
|
||||
*/
|
||||
export function show(root, ...nodes) {
|
||||
root.replaceChildren(...nodes.flat().filter((node) => node != null && node !== false));
|
||||
}
|
||||
|
||||
export function setActiveNav(view) {
|
||||
document.querySelectorAll("nav a[data-nav]").forEach((a) => {
|
||||
if (a.dataset.nav === view) a.setAttribute("aria-current", "page");
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
// recovery classifications all come from the server; the view only shows them and
|
||||
// refuses to offer an action the server would reject.
|
||||
import { api } from "./api.js";
|
||||
import { el, errorBanner, setActiveNav, show } from "./dom.js";
|
||||
import { el, errorBanner, setActiveNav } from "./dom.js";
|
||||
|
||||
// What the last confirm did, for this tab only. Deliberately not persisted: after a
|
||||
// reload the page must show what the journal says, not what this page remembers.
|
||||
@@ -24,7 +24,7 @@ export async function renderRenames(root, params = {}) {
|
||||
try {
|
||||
[plans, recovery] = await Promise.all([api.listPlans(), api.renameRecovery()]);
|
||||
} catch (error) {
|
||||
show(root, errorBanner(`Failed to load renames: ${error.message}`));
|
||||
root.replaceChildren(errorBanner(`Failed to load renames: ${error.message}`));
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -34,13 +34,13 @@ export async function renderRenames(root, params = {}) {
|
||||
try {
|
||||
plan = await api.getPlan(selectedId);
|
||||
} catch (error) {
|
||||
show(root, errorBanner(`Failed to load plan: ${error.message}`));
|
||||
root.replaceChildren(errorBanner(`Failed to load plan: ${error.message}`));
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const blocked = recovery.blocks_mutation;
|
||||
show(root,
|
||||
root.replaceChildren(
|
||||
el("h1", {}, "Renames"),
|
||||
recoveryPanel(recovery),
|
||||
el(
|
||||
|
||||
@@ -7,7 +7,6 @@ import { createStore } from "../store.js";
|
||||
import { parseHash, navigate } from "../router.js";
|
||||
import { api, cancellable } from "../api.js";
|
||||
import { subscribeJob } from "../events.js";
|
||||
import { assignHeadingIds, documentIndex, resolveDocLink, rewriteLinks, slug } from "../docs.js";
|
||||
|
||||
const cases = [];
|
||||
function ok(name, cond) {
|
||||
@@ -30,10 +29,6 @@ function jsonResponse(status, body) {
|
||||
const tick = (ms = 0) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
async function run() {
|
||||
// The client fetches its CSRF token once, lazily (US07-02). Do that against the
|
||||
// real server first, so the stubbed fetch below only ever sees the call under test.
|
||||
await api.workflow().catch(() => {});
|
||||
|
||||
// ── store ────────────────────────────────────────────────────────────────
|
||||
{
|
||||
const store = createStore({ n: 0 });
|
||||
@@ -143,62 +138,6 @@ async function run() {
|
||||
window.EventSource = realES;
|
||||
}
|
||||
|
||||
// ── documentation view (US09-01) ─────────────────────────────────────────
|
||||
{
|
||||
ok("slug lowercases and dashes a heading", slug("Key Flows") === "key-flows");
|
||||
ok(
|
||||
"slug drops punctuation but keeps words",
|
||||
slug("What it *will not* do:") === "what-it-will-not-do"
|
||||
);
|
||||
ok("slug keeps non-ASCII letters", slug("Größe & Gewicht") === "größe-gewicht");
|
||||
ok("slug never yields an empty anchor", slug("!!!") === "section");
|
||||
|
||||
const doc = document.createElement("div");
|
||||
doc.innerHTML = "<h2>Notes</h2><h3>Notes</h3><h2>Notes</h2>";
|
||||
const ids = [...assignHeadingIds(doc.querySelectorAll("h2, h3"))].map((h) => h.id);
|
||||
ok("repeated headings get distinct anchors", ids.join(",") === "notes,notes-2,notes-3");
|
||||
|
||||
ok(
|
||||
"a sibling document link becomes an in-app route",
|
||||
resolveDocLink("index", "overview.md")?.page === "overview"
|
||||
);
|
||||
const nested = resolveDocLink("guides/install", "../overview.md#running-it");
|
||||
ok("a relative link resolves against the current page", nested?.page === "overview");
|
||||
ok("a link's anchor survives the rewrite", nested?.anchor === "running-it");
|
||||
ok(
|
||||
"a bare anchor stays on the current page",
|
||||
resolveDocLink("overview", "#the-workflow")?.page === "overview"
|
||||
);
|
||||
ok("an external link is left alone", resolveDocLink("index", "https://example.test/x") === null);
|
||||
ok("a non-markdown relative link is left alone", resolveDocLink("index", "images/a.png") === null);
|
||||
// A climb cannot leave the documentation tree: it is clamped at the root, so the
|
||||
// page it names is still fetched from under /docs and simply does not exist.
|
||||
ok(
|
||||
"a link that climbs above the docs root is clamped",
|
||||
resolveDocLink("index", "../../etc/passwd.md")?.page === "etc/passwd"
|
||||
);
|
||||
ok(
|
||||
"a page name that is not a page name is refused",
|
||||
resolveDocLink("index", "..%2f..%2fetc%2fpasswd.md") === null
|
||||
);
|
||||
|
||||
const index = document.createElement("div");
|
||||
index.innerHTML =
|
||||
'<a href="overview.md">Overview</a><a href="overview.md">again</a>' +
|
||||
'<a href="index.md">itself</a><a href="https://example.test">out</a>';
|
||||
const listed = documentIndex(rewriteLinks(index, "index"));
|
||||
ok(
|
||||
"an external link is marked safe to open away from the app",
|
||||
index.querySelector('a[href^="https"]').rel === "noreferrer noopener"
|
||||
);
|
||||
ok(
|
||||
"an in-app link points at the docs route",
|
||||
index.querySelector("a").getAttribute("href") === "#/docs?page=overview"
|
||||
);
|
||||
ok("the index lists each page once, in order", listed.length === 1);
|
||||
ok("the index takes its titles from the link text", listed[0].title === "Overview");
|
||||
}
|
||||
|
||||
await tick();
|
||||
const failed = cases.filter((c) => !c.ok);
|
||||
window.__RESULTS__ = { passed: cases.length - failed.length, failed: failed.length, cases };
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
// preview arrives redacted — so nothing in this view may reconstruct, store, or
|
||||
// route a secret.
|
||||
import { api } from "./api.js";
|
||||
import { el, errorBanner, setActiveNav, show } from "./dom.js";
|
||||
import { el, errorBanner, setActiveNav } from "./dom.js";
|
||||
import { subscribeJob } from "./events.js";
|
||||
import { navigate } from "./router.js";
|
||||
|
||||
@@ -45,7 +45,7 @@ export async function renderUploads(root, params = {}) {
|
||||
api.listUploadBatches(),
|
||||
]);
|
||||
} catch (error) {
|
||||
show(root, errorBanner(`Failed to load uploads: ${error.message}`));
|
||||
root.replaceChildren(errorBanner(`Failed to load uploads: ${error.message}`));
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -62,12 +62,12 @@ export async function renderUploads(root, params = {}) {
|
||||
batch = detail;
|
||||
history = verifications.verifications;
|
||||
} catch (error) {
|
||||
show(root, errorBanner(`Failed to load upload batch: ${error.message}`));
|
||||
root.replaceChildren(errorBanner(`Failed to load upload batch: ${error.message}`));
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
show(root,
|
||||
root.replaceChildren(
|
||||
...[
|
||||
el("h1", {}, "Upload"),
|
||||
configurationCard(preflight),
|
||||
|
||||
23
frontend/js/vendor/VERSIONS.json
vendored
23
frontend/js/vendor/VERSIONS.json
vendored
@@ -1,23 +0,0 @@
|
||||
{
|
||||
"_comment": "Vendored browser libraries (US09-01). Committed rather than fetched: the application is deployed to a network whose outbound access is not assumed, and `default-src 'self'` forbids a CDN. Checksums are asserted by tests/integration/test_documentation.py, so replacing a file without recording it here fails the suite. Update by downloading the pinned URL and recording the new version and sha256 in the same commit.",
|
||||
"libraries": [
|
||||
{
|
||||
"file": "marked.esm.js",
|
||||
"name": "marked",
|
||||
"version": "18.0.10",
|
||||
"license": "MIT",
|
||||
"url": "https://cdn.jsdelivr.net/npm/marked@18.0.10/lib/marked.esm.js",
|
||||
"sha256": "4cf47dfebb7f614a08fc0a579ab0fe407ff0ed2b717bf953040c85b2f493a4f0",
|
||||
"why": "Markdown to HTML for the documentation view. An ES module with no dependencies, imported lazily by js/docs.js."
|
||||
},
|
||||
{
|
||||
"file": "mermaid.min.js",
|
||||
"name": "mermaid",
|
||||
"version": "11.17.0",
|
||||
"license": "MIT",
|
||||
"url": "https://cdn.jsdelivr.net/npm/mermaid@11.17.0/dist/mermaid.min.js",
|
||||
"sha256": "8d8e0eec56d3a83b4b3c87f42050845546dee93ebe1875d2117c12e6947c0cb3",
|
||||
"why": "Renders ```mermaid blocks so diagrams are diffable source that Gitea also renders. The single-file UMD build rather than the ES module entry, whose ~40 lazy chunks would each need vendoring and pinning."
|
||||
}
|
||||
]
|
||||
}
|
||||
78
frontend/js/vendor/marked.esm.js
vendored
78
frontend/js/vendor/marked.esm.js
vendored
File diff suppressed because one or more lines are too long
3636
frontend/js/vendor/mermaid.min.js
vendored
3636
frontend/js/vendor/mermaid.min.js
vendored
File diff suppressed because one or more lines are too long
@@ -3,7 +3,7 @@
|
||||
// HTML. Mutating actions are disabled while a job holds the library_write lock and
|
||||
// say why (concept §shared interaction rules: read-only browsing stays available).
|
||||
import { api } from "./api.js";
|
||||
import { el, errorBanner, setActiveNav, show } from "./dom.js";
|
||||
import { el, errorBanner, setActiveNav } from "./dom.js";
|
||||
import { navigate } from "./router.js";
|
||||
import { subscribeJob } from "./events.js";
|
||||
|
||||
@@ -26,7 +26,7 @@ export async function renderWorkflow(root) {
|
||||
try {
|
||||
data = await api.workflow();
|
||||
} catch (error) {
|
||||
show(root, errorBanner(`Failed to load workflow: ${error.message}`));
|
||||
root.replaceChildren(errorBanner(`Failed to load workflow: ${error.message}`));
|
||||
return;
|
||||
}
|
||||
const active = data.active_job;
|
||||
@@ -67,7 +67,7 @@ export async function renderWorkflow(root) {
|
||||
)
|
||||
);
|
||||
});
|
||||
show(root,
|
||||
root.replaceChildren(
|
||||
el("h1", {}, "Workflow"),
|
||||
jobBanner(active),
|
||||
el("div", { class: "stepper" }, ...cards)
|
||||
@@ -105,7 +105,7 @@ export async function renderSafety(root, params) {
|
||||
try {
|
||||
data = await api.safetyQueue({ state });
|
||||
} catch (error) {
|
||||
show(root, errorBanner(`Failed to load safety queue: ${error.message}`));
|
||||
root.replaceChildren(errorBanner(`Failed to load safety queue: ${error.message}`));
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -149,7 +149,7 @@ export async function renderSafety(root, params) {
|
||||
)
|
||||
);
|
||||
|
||||
show(root,
|
||||
root.replaceChildren(
|
||||
el("h1", {}, "Safety review"),
|
||||
tabs,
|
||||
el(
|
||||
@@ -197,7 +197,7 @@ export async function renderLibrary(root, params) {
|
||||
try {
|
||||
data = await api.libraryAssets({ q, offset, limit, sort: params.sort || "path" });
|
||||
} catch (error) {
|
||||
show(root, errorBanner(`Failed to load library: ${error.message}`));
|
||||
root.replaceChildren(errorBanner(`Failed to load library: ${error.message}`));
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -220,7 +220,7 @@ export async function renderLibrary(root, params) {
|
||||
)
|
||||
);
|
||||
|
||||
show(root,
|
||||
root.replaceChildren(
|
||||
el("h1", {}, "Library"),
|
||||
el("div", { class: "toolbar" }, search, el("span", { class: "muted", "data-testid": "library-total" }, `${data.total} photos`)),
|
||||
data.rows.length ? el("div", { class: "cluster-grid" }, ...cards) : el("p", { class: "muted" }, "No matching photos."),
|
||||
@@ -235,7 +235,7 @@ export async function renderAnalyze(root) {
|
||||
try {
|
||||
[counts, workflow] = await Promise.all([api.analysisCounts(), api.workflow()]);
|
||||
} catch (error) {
|
||||
show(root, errorBanner(`Failed to load analysis: ${error.message}`));
|
||||
root.replaceChildren(errorBanner(`Failed to load analysis: ${error.message}`));
|
||||
return;
|
||||
}
|
||||
const active = workflow.active_job;
|
||||
@@ -269,7 +269,7 @@ export async function renderAnalyze(root) {
|
||||
"Analyze eligible SFW assets"
|
||||
);
|
||||
|
||||
show(root,
|
||||
root.replaceChildren(
|
||||
el("h1", {}, "Analyze"),
|
||||
jobBanner(active),
|
||||
el(
|
||||
@@ -292,10 +292,10 @@ export async function renderStats(root) {
|
||||
try {
|
||||
data = await api.libraryStats();
|
||||
} catch (error) {
|
||||
show(root, errorBanner(`Failed to load stats: ${error.message}`));
|
||||
root.replaceChildren(errorBanner(`Failed to load stats: ${error.message}`));
|
||||
return;
|
||||
}
|
||||
show(root,
|
||||
root.replaceChildren(
|
||||
el("h1", {}, "Stats"),
|
||||
el("div", { class: "counts-grid", "data-testid": "stats-status" }, ...Object.entries(data.status).map(([k, v]) => stat(k, v))),
|
||||
facetBlock("Top tags", data.top_tags),
|
||||
@@ -338,7 +338,7 @@ export async function renderAlbums(root, params = {}) {
|
||||
api.renameRecovery(),
|
||||
]);
|
||||
} catch (error) {
|
||||
show(root, errorBanner(`Failed to load albums: ${error.message}`));
|
||||
root.replaceChildren(errorBanner(`Failed to load albums: ${error.message}`));
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -385,7 +385,7 @@ export async function renderAlbums(root, params = {}) {
|
||||
)
|
||||
: el("p", { class: "muted" }, "No albums with evidence yet.");
|
||||
|
||||
show(root,
|
||||
root.replaceChildren(
|
||||
el("h1", {}, "Albums"),
|
||||
renameBlocked
|
||||
? el(
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
d098bbde13d2ebc872ca781e244cc1e48b97cc551fedaca3251d6a5278a15234 src/compare_models.py
|
||||
7d0f68cb95fbd6603e4c558620098b46929ebdb6fd91a598bbf84f26eea35e6c src/nsfw_tag.py
|
||||
35329c53570e215cef15f59429c1a5251a0448b90522b0910ab9d68efe1bc307 src/nsfwtag/__init__.py
|
||||
7f52b55e4f3b77eda3657d7cd2272c7cca8422100e241671de495603c1126ee2 src/nsfwtag/__main__.py
|
||||
6f5a97114e0d87d272ce22d065dc31fa0dd72ff8590d11ad14cb1c1f486d339c src/nsfwtag/bench.py
|
||||
29498fac1d73ba2b7420ffe1ffe49a87f684ec299e2c3b9c6e14e63e95643ba4 src/nsfwtag/exif.py
|
||||
934e82c402813ebf503e5a20eb03d5103f84df13117bc4dd1fd95deaa01ab263 src/nsfwtag/README.md
|
||||
68125e6184c7e4d2a5b0153f2155675753ab0b1329d1dbc4933d4769492ec4be src/nsfwtag/review.html
|
||||
a597eab74803dd31452b5d70abf7d6d6320eb17a61596c891b009da15d624e98 src/nsfwtag/scoring.py
|
||||
674969a18e58ee511a2abf574a875db92f2f524769b1b614d0d3662ab879a709 src/nsfwtag/server.py
|
||||
24a7c8d029da6e97d46b110a9fe9dbb3900127f8ae3c142f634fdd91d6c45243 src/nsfwtag/webapp.py
|
||||
2c2ea558f1b9095c1150f078ea29c8e0f180cc045a9d1097b83f61bfb145295b src/photo_analyzer.py
|
||||
5fb7ce2f977a17da1f501c45d7985328318ae14298a3f347eab46a6e3f02aab3 src/test_dedup.py
|
||||
1673dc76cc60aba56a5b065d9bc9f5dda00342cf2a79d000a3f74686fb7cffec src/test_nsfw_skip.py
|
||||
55aa82f348e0e90be0163ba6aa278e5fc8e445996e5335774917e5ef59ad3563 src/webapp/__init__.py
|
||||
ba4342bd0175591a2121f063f14a78a602d5679bcfcf24aa14c8d4b8cd5da1a2 src/webapp/__main__.py
|
||||
a60f24035a989909467778f888d62854c1afc2702f47b8d3cca32bf3cf04d5cf src/webapp/analyzer.html
|
||||
018bcc2f770d716444b456db58d6b4a41800138a98598b3c27ba503ab22d939e src/webapp/page.py
|
||||
5f599b107b2b117ca118b6fbdec5e5786a9ab4eea424aa9af1bfea7bf87cebab src/webapp/query.py
|
||||
875cce697caa02717c85a2707ba935c46dd9ace18428fd4abb13105b7a8e24d9 src/webapp/README.md
|
||||
af7d0d72d245b4bbb1e0e30f9708239697543212d4171defd1bfe26dd24a4f05 src/webapp/runner.py
|
||||
cddf0555b06fe7e4daa14309baaaf91130f8506922cb0ffb6507d76cbe39add4 src/webapp/server.py
|
||||
@@ -1,80 +0,0 @@
|
||||
# Legacy CLI archive (US07-01)
|
||||
|
||||
Frozen, read-only sources of the command-line tools this application was extracted
|
||||
from. They are **reference material and rollback evidence** — provenance for
|
||||
behavior that now lives in `photo_pipeline/`, and the only way to answer "what did
|
||||
the original actually do?" once the replacement has drifted.
|
||||
|
||||
> **Nothing here is production code.** No module under `photo_pipeline/` imports or
|
||||
> executes anything in this directory, and this directory is not on the application's
|
||||
> import path. `tests/unit/test_legacy_archive.py` enforces both, along with the
|
||||
> checksums and the redaction below.
|
||||
|
||||
## What is here
|
||||
|
||||
| Path | Role |
|
||||
|---|---|
|
||||
| `src/photo_analyzer.py` | the analysis CLI: discovery, hashing/dedup, vision analysis, EXIF writing, SQLite schema, album naming |
|
||||
| `src/nsfwtag/` | NSFW scoring, EXIF safety keywords, and the review server (`__init__` 1.1.0) |
|
||||
| `src/webapp/` | the stdlib review web app: FTS search, stats, subprocess runner, HTML shell (`__init__` 0.1.0) |
|
||||
| `src/nsfw_tag.py` | thin backwards-compatible entry point for `nsfwtag` |
|
||||
| `src/compare_models.py` | dev-only model comparison script |
|
||||
| `src/test_dedup.py`, `src/test_nsfw_skip.py` | the CLIs' own standalone self-checks (never pytest suites) |
|
||||
| `donor_ledger.yaml` | the donor ledger: every migrated behavior, its target, its tests, and every intentional delta |
|
||||
| `requirements-lock.txt` | the dependency versions the frozen sources were last verified against |
|
||||
| `photo_analyzer.env.sample` | the CLI's configuration surface, with every value replaced by a placeholder |
|
||||
| `CHECKSUMS.sha256` | SHA-256 of every archived source file |
|
||||
|
||||
`photo_analyzer.py` carries no `__version__`; its identity is its checksum, recorded
|
||||
in `CHECKSUMS.sha256` and taken at commit `9b7ee6b` (the merge of US06-06, the last
|
||||
commit before archival).
|
||||
|
||||
## Verifying the archive
|
||||
|
||||
```bash
|
||||
cd legacy_cli_archive && shasum -a 256 -c CHECKSUMS.sha256
|
||||
```
|
||||
|
||||
Any edit to an archived source must be accompanied by a regenerated checksum file
|
||||
and a note here explaining why a *frozen* archive changed — the normal answer being
|
||||
that it should not.
|
||||
|
||||
## Schema notes
|
||||
|
||||
`photo_analyzer.py` owned a path-keyed SQLite database (`SCHEMA`, near the top of the
|
||||
file):
|
||||
|
||||
- `photos(id, path UNIQUE, status, phash, file_sha1, dup_of, description, tags,
|
||||
people_count, setting, time_of_day, season, mood, location_hint, approx_year,
|
||||
raw_response, error_message, analyzed_at, exif_written_at)`;
|
||||
- `photos_fts` — an FTS5 external-content index over `path, description, tags, mood,
|
||||
location_hint`, kept in sync by insert/update/delete triggers;
|
||||
- late columns (`phash`, `file_sha1`, `dup_of`) were added by an in-code
|
||||
`_migrate_schema()` rather than a migration tool, and their indexes are created
|
||||
only after the `ALTER`.
|
||||
|
||||
The replacement keeps the same analysis fields but re-keys everything to a stable
|
||||
`assets.id` (Alembic migrations `0001`…), because a path is not an identity: the
|
||||
donor's `path UNIQUE` is exactly what broke on every move and rename.
|
||||
|
||||
`nsfwtag` kept its safety scores outside the database in `nsfw_scores.csv`
|
||||
(`path,nsfw_score`, four decimals, unreadable rows dropped). That file is no longer a
|
||||
source of truth; `photo_pipeline/services/legacy_import.py` imports it into
|
||||
`assets.safety_score` and reports exactly what matched, what did not, and why.
|
||||
|
||||
## Redaction
|
||||
|
||||
The archive contains no credentials. `photo_analyzer.env.sample` documents the
|
||||
configuration surface (`LLM_API_KEY`, `LLM_BASE_URL`, `LLM_MODEL`, and the tuning
|
||||
variables) with placeholder values only; the CLI itself never contained a key, it
|
||||
read one from `photo_analyzer.env` or the environment. No `.env`, database, log, CSV,
|
||||
or photo from the author's library is archived.
|
||||
|
||||
## Why these tools were retired
|
||||
|
||||
Each behavior's fate is recorded per row in `donor_ledger.yaml`: `reuse`, `extract`,
|
||||
`refactor`, or `replace`, with the target module, the characterization tests that
|
||||
pinned the donor's behavior, the parity tests the replacement passes, and — where the
|
||||
replacement deliberately does something else or nothing at all — a `delta` saying so.
|
||||
Rows still marked `pending` name the backlog story that will resolve them; they are
|
||||
the honest list of what has *not* been carried over yet.
|
||||
@@ -1,18 +0,0 @@
|
||||
# Sample configuration for the archived photo_analyzer.py CLI (US07-01).
|
||||
#
|
||||
# REDACTED: no value below is real. Each line names a variable the CLI read and
|
||||
# what belongs there; the placeholders are deliberately not key-shaped, so this
|
||||
# file can never be mistaken for — or scanned as — a credential.
|
||||
|
||||
LLM_API_KEY=<paste your provider key here>
|
||||
LLM_BASE_URL=<provider base url, e.g. the OpenAI-compatible Gemini endpoint>
|
||||
LLM_MODEL=<model id, e.g. a Gemini Flash release>
|
||||
|
||||
# Optional tuning the CLI read from the same file:
|
||||
PHASH_THRESHOLD=8
|
||||
MAX_WORKERS=4
|
||||
RETRY_ATTEMPTS=3
|
||||
RPD_LIMIT=0
|
||||
|
||||
# The CLI ignored its own shipped placeholder (a literal "sk-REPLACE..." string)
|
||||
# until it was replaced, and a shell variable always won over this file.
|
||||
@@ -1,20 +0,0 @@
|
||||
# Final dependency lock of the archived CLIs (US07-01).
|
||||
#
|
||||
# These are the versions present in the environment the archive was taken from —
|
||||
# what the frozen sources were last verified against by the characterization
|
||||
# suite. Restoring a donor for forensics means pinning these, not "latest".
|
||||
#
|
||||
# Python 3.14.6
|
||||
|
||||
openai==3.0.0 # photo_analyzer: OpenAI-compatible vision client
|
||||
numpy==2.4.6 # photo_analyzer, nsfwtag: pixel work
|
||||
Pillow==12.3.0 # photo_analyzer, nsfwtag: decode/resize
|
||||
rich==15.0.0 # photo_analyzer: console output
|
||||
scipy==1.18.0 # photo_analyzer: perceptual-hash DCT
|
||||
PyYAML==6.0.3 # tooling that reads the donor ledger
|
||||
|
||||
# NSFW inference (nsfwtag/scoring.py, nsfwtag/bench.py) was never installed in the
|
||||
# archiving environment; the model stack is recorded here from the sources so a
|
||||
# forensic run can reproduce it, not from a resolved lock:
|
||||
# torch, transformers, timm — AdamCodd/vit-base-nsfw-detector (see nsfwtag/README.md)
|
||||
# exiftool is an external binary, not a Python package.
|
||||
@@ -1,29 +0,0 @@
|
||||
"""Protected thumbnails (US06-03).
|
||||
|
||||
Revision ID: 0013_protected_thumbnails
|
||||
Revises: 0012_archive_plans
|
||||
Create Date: 2026-08-16
|
||||
|
||||
A protected thumbnail is the durable comparison preview of an asset whose
|
||||
original has left active storage. It is evidence rather than cache, so the LRU
|
||||
quota must not evict it: the archive medium may be offline when it is needed.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0013_protected_thumbnails"
|
||||
down_revision = "0012_archive_plans"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"thumbnails",
|
||||
sa.Column("protected", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("thumbnails", "protected")
|
||||
@@ -1,37 +0,0 @@
|
||||
"""Restore plans and archive divergence (US06-04).
|
||||
|
||||
Revision ID: 0014_restore_plans
|
||||
Revises: 0013_protected_thumbnails
|
||||
Create Date: 2026-08-16
|
||||
|
||||
Restore reuses the archive plan and journal tables: the crash-safe question is the
|
||||
same one in the opposite direction (copy, verify, publish, register), so the rows
|
||||
gain a ``direction`` instead of a parallel pair of tables. ``archive_divergent_at``
|
||||
records the moment an archived copy was proven to hold bytes that are not the ones
|
||||
the database recorded — a restore must never silently accept a different file.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0014_restore_plans"
|
||||
down_revision = "0013_protected_thumbnails"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
for table in ("archive_plans", "archive_operations"):
|
||||
op.add_column(
|
||||
table,
|
||||
sa.Column("direction", sa.String(), nullable=False, server_default="archive"),
|
||||
)
|
||||
op.add_column(
|
||||
"assets", sa.Column("archive_divergent_at", sa.DateTime(timezone=True), nullable=True)
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("assets", "archive_divergent_at")
|
||||
for table in ("archive_plans", "archive_operations"):
|
||||
op.drop_column(table, "direction")
|
||||
@@ -1,42 +0,0 @@
|
||||
"""Durable EXIF projections per asset and stage (US07-03).
|
||||
|
||||
Revision ID: 0015_exif_projections
|
||||
Revises: 0014_restore_plans
|
||||
Create Date: 2026-08-16
|
||||
|
||||
The concept's ``exif_projections`` table, added at the point it earns its keep: a
|
||||
checkpoint that finds a field it does not own changed must be able to say so after
|
||||
a restart. ``state`` is verified | divergent | failed, and only ``verified`` counts
|
||||
as a completed metadata stage.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0015_exif_projections"
|
||||
down_revision = "0014_restore_plans"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"exif_projections",
|
||||
sa.Column("asset_id", sa.String(), sa.ForeignKey("assets.id"), primary_key=True),
|
||||
sa.Column("stage", sa.String(), primary_key=True), # safety | analysis
|
||||
sa.Column("id", sa.String(), nullable=False),
|
||||
sa.Column("projection_version", sa.Integer(), nullable=False, server_default="0"),
|
||||
# What the stage asked for: {"add": [...], "remove": [...]}.
|
||||
sa.Column("desired_json", sa.String(), nullable=True),
|
||||
# Fields outside the stage's ownership that did not survive the write.
|
||||
sa.Column("divergent_fields", sa.String(), nullable=True),
|
||||
sa.Column("result_file_sha256", sa.String(), nullable=True),
|
||||
sa.Column("state", sa.String(), nullable=False),
|
||||
sa.Column("error_code", sa.String(), nullable=True),
|
||||
sa.Column("verified_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("exif_projections")
|
||||
@@ -1,45 +0,0 @@
|
||||
"""Indexes the large-library read paths need (US07-06).
|
||||
|
||||
Revision ID: 0016_performance_indexes
|
||||
Revises: 0015_exif_projections
|
||||
Create Date: 2026-08-17
|
||||
|
||||
Measured, not guessed. At 100k assets the workflow home and the safety queue both
|
||||
resolve "the current decision per asset" with a window function over
|
||||
``safety_reviews``; a plain ``asset_id`` index makes SQLite sort every partition by
|
||||
hand. Ordering the index by ``(asset_id, created_at DESC)`` halves that query.
|
||||
|
||||
``duplicate_members(cluster_id, asset_id)`` serves the paged member list of a
|
||||
cluster with thousands of members, which is the other page that stopped being cheap.
|
||||
"""
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision = "0016_performance_indexes"
|
||||
down_revision = "0015_exif_projections"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_index(
|
||||
"ix_safety_reviews_asset_created",
|
||||
"safety_reviews",
|
||||
["asset_id", "created_at"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_duplicate_members_cluster_asset",
|
||||
"duplicate_members",
|
||||
["cluster_id", "asset_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_analysis_results_approx_year",
|
||||
"analysis_results",
|
||||
["approx_year"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_analysis_results_approx_year", table_name="analysis_results")
|
||||
op.drop_index("ix_duplicate_members_cluster_asset", table_name="duplicate_members")
|
||||
op.drop_index("ix_safety_reviews_asset_created", table_name="safety_reviews")
|
||||
@@ -1,237 +1,28 @@
|
||||
"""Application management CLI:
|
||||
``python -m photo_pipeline {serve,migrate,worker,import-legacy-scores,backup,verify-backup,restore,diagnostics,benchmark,release-gate,container-gate,dry-run,approve-dry-run}``.
|
||||
|
||||
``serve`` and ``worker`` take the library process lock for their role (US07-05):
|
||||
two workers, or the frozen CLI running beside the app, would each be safe on their
|
||||
own and destructive together. ``restore`` is here rather than in the API because it
|
||||
replaces the state of an installation and belongs to a stopped one.
|
||||
"""
|
||||
"""Application management CLI: ``python -m photo_pipeline {serve,migrate}``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Sequence
|
||||
|
||||
from photo_pipeline import path_policy
|
||||
from photo_pipeline.config import Config
|
||||
from photo_pipeline.services.app_lock import LegacyProcessActive, LibraryLock, LockHeld
|
||||
from photo_pipeline.services.backup import BackupError, BackupService, migrate_with_backup
|
||||
from photo_pipeline.db import run_migrations
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(prog="photo_pipeline")
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
serve_cmd = commands.add_parser("serve", help="Run the API server")
|
||||
commands.add_parser("serve", help="Run the API server")
|
||||
commands.add_parser("migrate", help="Upgrade the database to the latest revision")
|
||||
worker_cmd = commands.add_parser("worker", help="Run a durable-job worker")
|
||||
worker_cmd.add_argument("--id", default="worker-1", help="Worker id (lease owner)")
|
||||
for locked in (serve_cmd, worker_cmd):
|
||||
locked.add_argument(
|
||||
"--allow-legacy",
|
||||
action="store_true",
|
||||
help="Start even though a legacy CLI looks active (unsafe; you own the outcome)",
|
||||
)
|
||||
import_cmd = commands.add_parser(
|
||||
"import-legacy-scores",
|
||||
help="Import the archived CLI's nsfw_scores.csv into the database (US07-01)",
|
||||
)
|
||||
import_cmd.add_argument("csv", help="Path to nsfw_scores.csv")
|
||||
import_cmd.add_argument(
|
||||
"--overwrite", action="store_true", help="Replace differing imported scores"
|
||||
)
|
||||
import_cmd.add_argument(
|
||||
"--dry-run", action="store_true", help="Report what would happen and change nothing"
|
||||
)
|
||||
|
||||
backup_cmd = commands.add_parser("backup", help="Take an online database backup")
|
||||
backup_cmd.add_argument("--reason", default="manual", help="Why (part of the directory name)")
|
||||
backup_cmd.add_argument("--keep", type=int, default=7, help="How many backups to retain")
|
||||
verify_cmd = commands.add_parser("verify-backup", help="Check a backup is intact and readable")
|
||||
verify_cmd.add_argument("backup", help="Path to the backup directory")
|
||||
restore_cmd = commands.add_parser(
|
||||
"restore", help="Restore a verified backup into a fresh data directory"
|
||||
)
|
||||
restore_cmd.add_argument("backup", help="Path to the backup directory")
|
||||
restore_cmd.add_argument("--into", required=True, help="Fresh data directory to restore into")
|
||||
commands.add_parser("diagnostics", help="Report sizes, disk headroom, locks, and warnings")
|
||||
bench_cmd = commands.add_parser(
|
||||
"benchmark", help="Measure latency and resource use against agreed budgets (US07-06)"
|
||||
)
|
||||
bench_cmd.add_argument("--profile", default="smoke", help="smoke | short | full | huge")
|
||||
bench_cmd.add_argument(
|
||||
"--soak-seconds", type=float, default=0.0, help="Also run a soak of this length"
|
||||
)
|
||||
bench_cmd.add_argument("--output", help="Write the JSON report here as well as to stdout")
|
||||
|
||||
gate_cmd = commands.add_parser(
|
||||
"release-gate", help="Run every suite in an isolated stack and keep the evidence"
|
||||
)
|
||||
gate_cmd.add_argument("--output", help="Evidence directory (default: data/release/<stamp>)")
|
||||
container_cmd = commands.add_parser(
|
||||
"container-gate",
|
||||
help="Provision the composition from the built image, run the phase_h "
|
||||
"acceptance suite against it, destroy it, and keep the evidence (US08-05)",
|
||||
)
|
||||
container_cmd.add_argument(
|
||||
"--output", help="Evidence directory (default: data/container-gate/<stamp>)"
|
||||
)
|
||||
dry_cmd = commands.add_parser(
|
||||
"dry-run", help="Read-only reconciliation of the configured library (US07-07)"
|
||||
)
|
||||
dry_cmd.add_argument("--output", help="Write the report here as well as to stdout")
|
||||
approve_cmd = commands.add_parser(
|
||||
"approve-dry-run", help="Approve a dry-run report, which is what enables mutation"
|
||||
)
|
||||
approve_cmd.add_argument("report", help="Path to the dry-run report")
|
||||
approve_cmd.add_argument("--approver", required=True, help="Who is accepting this")
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
config = Config.from_env()
|
||||
config.database_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# In a container the two roles that touch the library check their boundary before
|
||||
# they take a lock or bind a port: the configured roots must name the mount paths,
|
||||
# and a mismatch is cheaper to refuse than to discover at the first write (US08-03).
|
||||
# Only in a container — on a host an unmounted root is an ordinary Tuesday (an
|
||||
# archive medium that is not plugged in), and refusing to serve would take the
|
||||
# offline half of the library away with it.
|
||||
if args.command in ("serve", "worker") and path_policy.in_container():
|
||||
refusal = path_policy.roots_refusal(config.library_roots, require_mount=True)
|
||||
if refusal is not None:
|
||||
print(refusal, file=sys.stderr)
|
||||
return 5
|
||||
|
||||
if args.command == "migrate":
|
||||
manifest = migrate_with_backup(config)
|
||||
if manifest:
|
||||
print(json.dumps({"pre_migration_backup": manifest["name"]}, indent=2))
|
||||
return 0
|
||||
|
||||
if args.command == "backup":
|
||||
try:
|
||||
manifest = BackupService(config).create(reason=args.reason, keep=args.keep)
|
||||
except BackupError as error:
|
||||
print(str(error))
|
||||
return 1
|
||||
print(json.dumps(manifest, indent=2))
|
||||
return 0
|
||||
|
||||
if args.command == "verify-backup":
|
||||
result = BackupService(config).verify(args.backup)
|
||||
print(json.dumps(result.as_dict(), indent=2))
|
||||
return 0 if result.ok else 1
|
||||
|
||||
if args.command == "restore":
|
||||
try:
|
||||
report = BackupService(config).restore(args.backup, args.into)
|
||||
except BackupError as error:
|
||||
print(str(error))
|
||||
return 1
|
||||
print(json.dumps(report, indent=2))
|
||||
return 0
|
||||
|
||||
if args.command == "benchmark":
|
||||
from photo_pipeline.services import benchmarks
|
||||
|
||||
try:
|
||||
report = benchmarks.run(
|
||||
config,
|
||||
profile=args.profile,
|
||||
soak_seconds=args.soak_seconds,
|
||||
output=args.output,
|
||||
)
|
||||
except ValueError as error:
|
||||
print(str(error))
|
||||
return 1
|
||||
print(json.dumps({k: v for k, v in report.items() if k != "runs"}, indent=2))
|
||||
# A breached budget is a failed run, so a scheduled job notices without
|
||||
# anyone reading the JSON.
|
||||
return 0 if report["ok"] else 1
|
||||
|
||||
if args.command == "release-gate":
|
||||
from photo_pipeline.services import release
|
||||
|
||||
report = release.run_gate(config, output=args.output)
|
||||
print(
|
||||
json.dumps(
|
||||
{k: v for k, v in report.items() if k not in ("stages", "matrix")}, indent=2
|
||||
)
|
||||
)
|
||||
return 0 if report["ok"] else 1
|
||||
|
||||
if args.command == "container-gate":
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from photo_pipeline.services import release
|
||||
|
||||
# The suite provisions and destroys the composition itself; what this command
|
||||
# adds is the single entry point and the retained evidence. No skip is an
|
||||
# environment limit here: a gate that did not reach the containers proved
|
||||
# nothing about them.
|
||||
output = args.output or Path(config.data_dir) / "container-gate" / datetime.now(
|
||||
timezone.utc
|
||||
).strftime("%Y%m%dT%H%M%SZ")
|
||||
report = release.run_gate(
|
||||
config,
|
||||
output=output,
|
||||
stages=release.CONTAINER_STAGES,
|
||||
allowed_skips=release.CONTAINER_ALLOWED_SKIP_REASONS,
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{k: v for k, v in report.items() if k not in ("stages", "matrix")}, indent=2
|
||||
)
|
||||
)
|
||||
return 0 if report["ok"] else 1
|
||||
|
||||
if args.command == "dry-run":
|
||||
from photo_pipeline.services import release
|
||||
|
||||
try:
|
||||
report = release.dry_run(config)
|
||||
except release.ReleaseError as error:
|
||||
print(str(error))
|
||||
return 1
|
||||
if args.output:
|
||||
Path(args.output).write_text(json.dumps(report, indent=2))
|
||||
print(json.dumps(report, indent=2))
|
||||
return 0
|
||||
|
||||
if args.command == "approve-dry-run":
|
||||
from photo_pipeline.services import release
|
||||
|
||||
try:
|
||||
record = release.approve(config, args.report, approver=args.approver)
|
||||
except (release.ReleaseError, OSError, ValueError) as error:
|
||||
print(str(error))
|
||||
return 1
|
||||
print(json.dumps(record, indent=2))
|
||||
return 0
|
||||
|
||||
if args.command == "diagnostics":
|
||||
from photo_pipeline.services import diagnostics
|
||||
|
||||
print(json.dumps(diagnostics.report(config), indent=2))
|
||||
return 0
|
||||
|
||||
if args.command == "import-legacy-scores":
|
||||
from photo_pipeline.db import create_db_engine, create_session_factory
|
||||
from photo_pipeline.services.legacy_import import LegacyImportService, write_report
|
||||
|
||||
migrate_with_backup(config)
|
||||
engine = create_db_engine(config.database_url)
|
||||
service = LegacyImportService(create_session_factory(engine))
|
||||
report = service.import_nsfw_scores(
|
||||
args.csv, overwrite=args.overwrite, dry_run=args.dry_run
|
||||
)
|
||||
# The report is the point: an import nobody can audit is not a migration.
|
||||
if not args.dry_run:
|
||||
write_report(report, config.data_dir)
|
||||
print(json.dumps(report.counts, indent=2))
|
||||
run_migrations(config.database_url)
|
||||
return 0
|
||||
|
||||
if args.command == "worker":
|
||||
@@ -243,56 +34,18 @@ def main(argv: Sequence[str] | None = None) -> int:
|
||||
import photo_pipeline.jobs.domain_handlers # noqa: F401
|
||||
from photo_pipeline.jobs.worker import Worker
|
||||
|
||||
lock = LibraryLock(config, "worker")
|
||||
if (held := _acquire(lock, allow_legacy=args.allow_legacy)) is not None:
|
||||
return held
|
||||
try:
|
||||
migrate_with_backup(config)
|
||||
engine = create_db_engine(config.database_url)
|
||||
Worker(
|
||||
create_session_factory(engine), worker_id=args.id, config=config
|
||||
).run_forever()
|
||||
finally:
|
||||
lock.release()
|
||||
run_migrations(config.database_url)
|
||||
engine = create_db_engine(config.database_url)
|
||||
Worker(create_session_factory(engine), worker_id=args.id, config=config).run_forever()
|
||||
return 0
|
||||
|
||||
import uvicorn
|
||||
|
||||
from photo_pipeline.api.app import ConfigurationRefused, create_app
|
||||
from photo_pipeline.api.app import create_app
|
||||
|
||||
# An exposed deployment without an access secret must not reach the port at all,
|
||||
# and the operator needs a sentence, not a traceback (US08-01).
|
||||
try:
|
||||
app = create_app(config)
|
||||
except ConfigurationRefused as error:
|
||||
print(str(error), file=sys.stderr)
|
||||
return 4
|
||||
|
||||
lock = LibraryLock(config, "api")
|
||||
if (held := _acquire(lock, allow_legacy=args.allow_legacy)) is not None:
|
||||
return held
|
||||
try:
|
||||
uvicorn.run(app, host=config.host, port=config.port)
|
||||
finally:
|
||||
lock.release()
|
||||
uvicorn.run(create_app(config), host=config.host, port=config.port)
|
||||
return 0
|
||||
|
||||
|
||||
def _acquire(lock: LibraryLock, *, allow_legacy: bool) -> int | None:
|
||||
"""Take the lock, or explain on stderr why this process must not start.
|
||||
|
||||
Returns an exit code to return, or ``None`` when the lock was acquired.
|
||||
"""
|
||||
try:
|
||||
lock.acquire(allow_legacy=allow_legacy)
|
||||
except LockHeld as error:
|
||||
print(str(error), file=sys.stderr)
|
||||
return 2
|
||||
except LegacyProcessActive as error:
|
||||
print(f"{error} (override with --allow-legacy)", file=sys.stderr)
|
||||
return 3
|
||||
return None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
||||
@@ -8,15 +8,11 @@ and exposes the versioned ``/api/v1`` surface; US01-02 ships only health.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi import FastAPI
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
from photo_pipeline.api.routes import (
|
||||
albums,
|
||||
@@ -27,87 +23,31 @@ from photo_pipeline.api.routes import (
|
||||
inventory,
|
||||
jobs,
|
||||
library,
|
||||
operations,
|
||||
renames,
|
||||
safety,
|
||||
session as session_routes,
|
||||
thumbnails,
|
||||
uploads,
|
||||
workflow,
|
||||
)
|
||||
from photo_pipeline.api.security import (
|
||||
DEFAULT_HEADERS,
|
||||
FailureLimiter,
|
||||
SecurityMiddleware,
|
||||
Session,
|
||||
trust_refusal,
|
||||
)
|
||||
|
||||
# Registers the safety_score / analysis job handlers on import.
|
||||
import photo_pipeline.jobs.domain_handlers # noqa: F401
|
||||
from photo_pipeline.config import Config
|
||||
from photo_pipeline.db import create_db_engine, create_session_factory
|
||||
from photo_pipeline.services.backup import migrate_with_backup
|
||||
from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
|
||||
from photo_pipeline.logging import configure_logging
|
||||
from photo_pipeline.services.thumbnails import ThumbnailService
|
||||
from photo_pipeline.services.upload_batches import UploadBatchService
|
||||
|
||||
FRONTEND_DIR = Path(__file__).resolve().parents[2] / "frontend"
|
||||
DOCS_DIR = Path(__file__).resolve().parents[2] / "docs"
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _envelope(status: int, code: str, message: str) -> JSONResponse:
|
||||
return JSONResponse(
|
||||
status_code=status,
|
||||
content={"error": {"code": code, "message": message}},
|
||||
headers=DEFAULT_HEADERS,
|
||||
)
|
||||
|
||||
|
||||
def _install_error_handlers(app: FastAPI) -> None:
|
||||
"""One JSON error envelope everywhere, and nothing behind it.
|
||||
|
||||
An unhandled exception carries the library's absolute paths, SQL, and sometimes
|
||||
a credential in its text; the client gets a code, the operator gets the traceback
|
||||
in the server log (US07-02).
|
||||
"""
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def _http_error(request: Request, exc: StarletteHTTPException):
|
||||
return _envelope(exc.status_code, "http_error", str(exc.detail))
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def _validation_error(request: Request, exc: RequestValidationError):
|
||||
# Field locations only: the echoed input can be the caller's own data, but it
|
||||
# is also what ends up in shared logs and screenshots.
|
||||
fields = sorted(".".join(str(part) for part in error["loc"]) for error in exc.errors())
|
||||
return _envelope(422, "invalid_request", f"invalid request fields: {', '.join(fields)}")
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def _unhandled(request: Request, exc: Exception):
|
||||
log.exception("unhandled error serving %s", request.url.path)
|
||||
return _envelope(500, "internal_error", "internal error")
|
||||
|
||||
|
||||
class ConfigurationRefused(RuntimeError):
|
||||
"""The configuration would serve the library to callers it cannot authenticate."""
|
||||
|
||||
|
||||
def create_app(config: Config | None = None) -> FastAPI:
|
||||
config = config or Config.from_env()
|
||||
configure_logging(config.log_level, config.log_format)
|
||||
# Before anything is built, let alone bound to a port (US08-01).
|
||||
if (why := trust_refusal(config)) is not None:
|
||||
raise ConfigurationRefused(why)
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
config.database_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
# A schema upgrade is snapshotted first, so a migration that fails halfway
|
||||
# leaves a restorable database behind rather than a damaged one (US07-05).
|
||||
migrate_with_backup(config)
|
||||
run_migrations(config.database_url)
|
||||
engine = create_db_engine(config.database_url)
|
||||
app.state.config = config
|
||||
app.state.engine = engine
|
||||
@@ -115,9 +55,6 @@ def create_app(config: Config | None = None) -> FastAPI:
|
||||
# An upload whose process died left no outcome behind; resolve it now so the
|
||||
# uploader lane is free and the uncertain batch is visible (US05-02).
|
||||
UploadBatchService(app.state.session_factory, config=config).recover()
|
||||
# A render killed mid-write leaves its temporary beside the cache entry;
|
||||
# remove those recognized leftovers, and only those (US07-03).
|
||||
ThumbnailService(app.state.session_factory, config).cleanup_temp_files()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
@@ -125,16 +62,6 @@ def create_app(config: Config | None = None) -> FastAPI:
|
||||
app.state.engine = None
|
||||
|
||||
app = FastAPI(title="Photo Pipeline", version="0.1.0", lifespan=lifespan)
|
||||
# One session per process: the browser exchanges it for a cookie + CSRF token,
|
||||
# and every other origin is refused before a route ever runs (US07-02).
|
||||
app.state.session = Session.create()
|
||||
app.state.access_limiter = FailureLimiter()
|
||||
# Also set in the lifespan, but the bootstrap route reads it, and a caller can
|
||||
# arrive before anything else has touched app.state.
|
||||
app.state.config = config
|
||||
app.add_middleware(SecurityMiddleware, session=app.state.session, config=config)
|
||||
_install_error_handlers(app)
|
||||
app.include_router(session_routes.router, prefix="/api/v1")
|
||||
app.include_router(health.router, prefix="/api/v1")
|
||||
app.include_router(inventory.router, prefix="/api/v1")
|
||||
app.include_router(duplicates.router, prefix="/api/v1")
|
||||
@@ -148,14 +75,7 @@ def create_app(config: Config | None = None) -> FastAPI:
|
||||
app.include_router(renames.router, prefix="/api/v1")
|
||||
app.include_router(uploads.router, prefix="/api/v1")
|
||||
app.include_router(archives.router, prefix="/api/v1")
|
||||
app.include_router(operations.router, prefix="/api/v1")
|
||||
# Static single-page app (hash-routed). Mounted last so /api/v1 wins.
|
||||
if FRONTEND_DIR.is_dir():
|
||||
app.mount("/app", StaticFiles(directory=FRONTEND_DIR, html=True), name="app")
|
||||
# The manuals, as their own markdown files (US09-01). Unauthenticated for the
|
||||
# same reason the application shell is: whoever cannot get past the access
|
||||
# secret is exactly who needs the troubleshooting page, and no document holds
|
||||
# anything a session would protect.
|
||||
if DOCS_DIR.is_dir():
|
||||
app.mount("/docs", StaticFiles(directory=DOCS_DIR), name="docs")
|
||||
return app
|
||||
|
||||
@@ -18,10 +18,7 @@ router = APIRouter(tags=["analysis"])
|
||||
|
||||
|
||||
def _service(request: Request) -> AnalysisService:
|
||||
return AnalysisService(
|
||||
request.app.state.session_factory,
|
||||
library_roots=tuple(request.app.state.config.library_roots),
|
||||
)
|
||||
return AnalysisService(request.app.state.session_factory)
|
||||
|
||||
|
||||
def _error(status: int, code: str, message: str) -> JSONResponse:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Archive location, preflight, plan, and restore API (US06-01, US06-02, US06-04).
|
||||
"""Archive location, preflight, and plan API (US06-01, US06-02).
|
||||
|
||||
Registering a location writes a marker onto the medium; preflight is a command
|
||||
rather than a read, because it probes the destination, hashes the scope, and issues
|
||||
@@ -13,11 +13,10 @@ from fastapi import APIRouter, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
from photo_pipeline.jobs.domain_handlers import ARCHIVE_LOCK, ARCHIVE_PLAN, RESTORE_PLAN
|
||||
from photo_pipeline.jobs.domain_handlers import ARCHIVE_LOCK, ARCHIVE_PLAN
|
||||
from photo_pipeline.services.archives import ArchiveError, ArchiveService
|
||||
from photo_pipeline.services.archive_transfer import ArchiveTransferService
|
||||
from photo_pipeline.services.jobs import JobBlocked, JobService
|
||||
from photo_pipeline.services.restores import RestoreService
|
||||
|
||||
router = APIRouter(tags=["archives"])
|
||||
|
||||
@@ -43,24 +42,10 @@ class CreatePlanRequest(PreflightRequest):
|
||||
token: str
|
||||
|
||||
|
||||
class RestoreRequest(BaseModel):
|
||||
location_id: str
|
||||
# ``None`` means every asset archived at this location.
|
||||
asset_ids: list[str] | None = None
|
||||
|
||||
|
||||
class CreateRestoreRequest(RestoreRequest):
|
||||
token: str
|
||||
|
||||
|
||||
def _service(request: Request) -> ArchiveService:
|
||||
return ArchiveService(request.app.state.session_factory, config=request.app.state.config)
|
||||
|
||||
|
||||
def _restores(request: Request) -> RestoreService:
|
||||
return RestoreService(request.app.state.session_factory, config=request.app.state.config)
|
||||
|
||||
|
||||
def _transfers(request: Request) -> ArchiveTransferService:
|
||||
return ArchiveTransferService(
|
||||
request.app.state.session_factory, config=request.app.state.config
|
||||
@@ -153,76 +138,6 @@ def apply_plan(plan_id: str, request: Request):
|
||||
return {"plan_id": plan_id, "job": job}
|
||||
|
||||
|
||||
@router.post("/restore-preflight")
|
||||
def restore_preflight(body: RestoreRequest, request: Request):
|
||||
"""Validate restoring archived assets back into the library. Nothing moves."""
|
||||
try:
|
||||
return _restores(request).preflight(body.location_id, body.asset_ids)
|
||||
except ArchiveError as error:
|
||||
return _error(error)
|
||||
|
||||
|
||||
@router.post("/restore-plans", status_code=201)
|
||||
def create_restore_plan(body: CreateRestoreRequest, request: Request):
|
||||
try:
|
||||
return _restores(request).create(body.location_id, body.asset_ids, token=body.token)
|
||||
except ArchiveError as error:
|
||||
return _error(error)
|
||||
|
||||
|
||||
@router.get("/restore-plans")
|
||||
def list_restore_plans(request: Request) -> dict:
|
||||
return {"plans": _restores(request).list()}
|
||||
|
||||
|
||||
@router.get("/restore-plans/{plan_id}")
|
||||
def get_restore_plan(plan_id: str, request: Request):
|
||||
plan = _restores(request).get(plan_id)
|
||||
if plan is None:
|
||||
return _error(ArchiveError("unknown_plan", f"unknown restore plan {plan_id}"))
|
||||
return plan
|
||||
|
||||
|
||||
@router.post("/restore-plans/{plan_id}/apply")
|
||||
def apply_restore_plan(plan_id: str, request: Request):
|
||||
"""Queue the restore on the archiver lane — the same single lane as archiving,
|
||||
because both move the same originals."""
|
||||
service = _restores(request)
|
||||
plan = service.get(plan_id)
|
||||
if plan is None:
|
||||
return _error(ArchiveError("unknown_plan", f"unknown restore plan {plan_id}"))
|
||||
unresolved = [row for row in service.journal.incomplete() if row["plan_id"] != plan_id]
|
||||
if unresolved:
|
||||
return _error(
|
||||
ArchiveError(
|
||||
"archive_pending",
|
||||
f"an unresolved archive operation ({unresolved[0]['id']}) must be recovered",
|
||||
)
|
||||
)
|
||||
try:
|
||||
job = JobService(request.app.state.session_factory).enqueue(
|
||||
RESTORE_PLAN,
|
||||
lock=ARCHIVE_LOCK,
|
||||
idempotency_key=f"restore:{plan_id}:{plan['version']}",
|
||||
items=[plan_id],
|
||||
)
|
||||
except JobBlocked as error:
|
||||
return JSONResponse(
|
||||
status_code=409, content={"error": {"code": error.code, "message": str(error)}}
|
||||
)
|
||||
return {"plan_id": plan_id, "job": job}
|
||||
|
||||
|
||||
@router.get("/restore-recovery")
|
||||
def restore_recovery_status(request: Request) -> dict:
|
||||
return _restores(request).recovery_status()
|
||||
|
||||
|
||||
@router.post("/restore-recovery/resolve")
|
||||
def resolve_restore_recovery(request: Request) -> dict:
|
||||
return _restores(request).recover()
|
||||
|
||||
|
||||
@router.get("/archive-recovery")
|
||||
def recovery_status(request: Request) -> dict:
|
||||
"""What an interrupted transfer left behind, straight from journal + disk."""
|
||||
|
||||
@@ -11,13 +11,7 @@ from fastapi import APIRouter, Query, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from photo_pipeline.schemas import DecisionRequest
|
||||
from photo_pipeline.services.duplicates import (
|
||||
MAX_MEMBER_PAGE,
|
||||
MEMBER_PAGE,
|
||||
ConflictError,
|
||||
DuplicateError,
|
||||
DuplicateService,
|
||||
)
|
||||
from photo_pipeline.services.duplicates import ConflictError, DuplicateError, DuplicateService
|
||||
|
||||
router = APIRouter(tags=["duplicates"])
|
||||
|
||||
@@ -48,13 +42,8 @@ def list_clusters(
|
||||
|
||||
|
||||
@router.get("/duplicates/clusters/{cluster_id}")
|
||||
def get_cluster(
|
||||
cluster_id: str,
|
||||
request: Request,
|
||||
limit: int = Query(MEMBER_PAGE, ge=1, le=MAX_MEMBER_PAGE),
|
||||
offset: int = Query(0, ge=0),
|
||||
):
|
||||
detail = _service(request).get_cluster(cluster_id, limit=limit, offset=offset)
|
||||
def get_cluster(cluster_id: str, request: Request):
|
||||
detail = _service(request).get_cluster(cluster_id)
|
||||
if detail is None:
|
||||
return _error(404, "not_found", f"unknown cluster {cluster_id}")
|
||||
return detail
|
||||
|
||||
@@ -1,70 +0,0 @@
|
||||
"""Operational endpoints: diagnostics and backups (US07-05).
|
||||
|
||||
Backups can be taken and verified here because both are safe, additive, and the
|
||||
operator needs them from the same screen that shows the disk filling up.
|
||||
|
||||
**Restore is deliberately not an endpoint.** It replaces the state of the running
|
||||
application with an older one, so it belongs to a stopped installation and a person
|
||||
at a terminal: ``python -m photo_pipeline restore``. An HTTP call that can silently
|
||||
roll the library back to last week is a hole, not a feature.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Query, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
from photo_pipeline.services import diagnostics
|
||||
from photo_pipeline.services.backup import DEFAULT_KEEP, BackupError, BackupService
|
||||
|
||||
router = APIRouter(tags=["operations"])
|
||||
|
||||
|
||||
class CreateBackupRequest(BaseModel):
|
||||
reason: str = "manual"
|
||||
keep: int = DEFAULT_KEEP
|
||||
|
||||
|
||||
def _service(request: Request) -> BackupService:
|
||||
return BackupService(request.app.state.config)
|
||||
|
||||
|
||||
def _error(status: int, code: str, message: str) -> JSONResponse:
|
||||
return JSONResponse(status_code=status, content={"error": {"code": code, "message": message}})
|
||||
|
||||
|
||||
@router.get("/diagnostics")
|
||||
def read_diagnostics(request: Request) -> dict:
|
||||
return diagnostics.report(request.app.state.config)
|
||||
|
||||
|
||||
@router.get("/backups")
|
||||
def list_backups(request: Request) -> dict:
|
||||
return {"backups": _service(request).list()}
|
||||
|
||||
|
||||
@router.post("/backups", status_code=201)
|
||||
def create_backup(body: CreateBackupRequest, request: Request):
|
||||
try:
|
||||
return _service(request).create(reason=body.reason, keep=body.keep)
|
||||
except BackupError as error:
|
||||
return _error(422, "backup_failed", str(error))
|
||||
|
||||
|
||||
@router.get("/backups/{name}/verify")
|
||||
def verify_backup(name: str, request: Request):
|
||||
service = _service(request)
|
||||
# The name comes from the browser, so it names a backup — it is never joined
|
||||
# into a path until it has been matched against one that exists (US07-02).
|
||||
if name not in {entry["name"] for entry in service.list()}:
|
||||
return _error(404, "not_found", f"unknown backup {name}")
|
||||
return {"name": name, **service.verify(service.root / name).as_dict()}
|
||||
|
||||
|
||||
@router.post("/backups/prune")
|
||||
def prune_backups(request: Request, keep: int = Query(DEFAULT_KEEP, ge=1)):
|
||||
try:
|
||||
return {"removed": _service(request).prune(keep=keep)}
|
||||
except BackupError as error:
|
||||
return _error(422, "invalid_retention", str(error))
|
||||
@@ -1,68 +0,0 @@
|
||||
"""Session bootstrap: the one endpoint reachable without a session (US07-02).
|
||||
|
||||
It sets the ``HttpOnly``/``SameSite=Strict`` session cookie and returns the CSRF
|
||||
token in the body. A foreign page can call this — it just cannot read the answer,
|
||||
because the app sends no CORS headers — and the cookie it received is never attached
|
||||
to a request that foreign page initiated.
|
||||
|
||||
When an access secret is configured (mandatory as soon as the app is reachable from
|
||||
another machine, US08-01) this is also the authentication gate: the secret buys the
|
||||
cookie, and every route behind it keeps asking for exactly the session and CSRF token
|
||||
it asked for before. Wrong secrets are counted, and a burst of them stops being
|
||||
answered — otherwise a proxy-exposed deployment could be guessed at indefinitely.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import secrets
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from photo_pipeline.api.security import ACCESS_SECRET_HEADER, SESSION_COOKIE
|
||||
|
||||
router = APIRouter(tags=["session"])
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _refuse(status: int, code: str, message: str) -> JSONResponse:
|
||||
return JSONResponse(status_code=status, content={"error": {"code": code, "message": message}})
|
||||
|
||||
|
||||
@router.get("/session")
|
||||
def start_session(request: Request) -> JSONResponse:
|
||||
config = request.app.state.config
|
||||
secret = config.access_secret
|
||||
if secret is not None:
|
||||
limiter = request.app.state.access_limiter
|
||||
if limiter.blocked():
|
||||
return _refuse(429, "too_many_attempts", "too many failed attempts; retry later")
|
||||
offered = request.headers.get(ACCESS_SECRET_HEADER, "")
|
||||
if not secrets.compare_digest(offered, secret.get_secret_value()):
|
||||
limiter.record_failure()
|
||||
# The client address is the whole record: the offered secret, the issued
|
||||
# session, and the request body all stay out of the log.
|
||||
log.warning(
|
||||
"access secret rejected", extra={"client": _client(request), "path": "/session"}
|
||||
)
|
||||
return _refuse(401, "access_denied", "a valid access secret is required")
|
||||
|
||||
session = request.app.state.session
|
||||
response = JSONResponse({"csrf_token": session.csrf_token})
|
||||
response.set_cookie(
|
||||
SESSION_COOKIE,
|
||||
session.id,
|
||||
httponly=True,
|
||||
samesite="strict",
|
||||
# HTTPS outside means the cookie must never travel over a plain hop, even one
|
||||
# this process cannot see. Loopback http keeps working unchanged.
|
||||
secure=request.scope.get("state", {}).get("external_scheme") == "https",
|
||||
path="/",
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def _client(request: Request) -> str:
|
||||
return request.client.host if request.client else "unknown"
|
||||
@@ -31,7 +31,5 @@ def get_thumbnail(asset_id: str, request: Request, size: int = Query(512)):
|
||||
return FileResponse(
|
||||
path,
|
||||
media_type="image/webp",
|
||||
# private: the URL is versioned and immutable, but these bytes are the user's
|
||||
# photos and must never sit in a shared cache (US07-02).
|
||||
headers={"Cache-Control": "private, max-age=31536000, immutable"},
|
||||
headers={"Cache-Control": "public, max-age=31536000, immutable"},
|
||||
)
|
||||
|
||||
@@ -1,330 +0,0 @@
|
||||
"""Local-web attack surface: session, CSRF, Origin/Host checks, default headers.
|
||||
|
||||
The app binds to 127.0.0.1, so the attacker is not a remote client but another page
|
||||
in the user's browser (concept §15, "Local web attack"): any site can issue requests
|
||||
to ``http://127.0.0.1:8000`` and can embed ``<img src=...>`` against media endpoints.
|
||||
The defenses stack, because each one alone has a hole:
|
||||
|
||||
* **Host** must be a loopback name — a DNS rebinding host that resolves to 127.0.0.1
|
||||
passes the browser's origin rules but not this check.
|
||||
* **Origin**, when the browser sends one, must be this exact origin (scheme, host,
|
||||
port). There is no CORS middleware at all, so a foreign page can never *read* a
|
||||
response even if it manages to send a request.
|
||||
* **Sec-Fetch-Site** rejects cross-site loads that carry no Origin, which is what an
|
||||
``<img>`` or ``<script>`` against a media endpoint looks like.
|
||||
* A **session cookie** (``SameSite=Strict``, ``HttpOnly``) is required by every
|
||||
``/api/v1`` route except liveness/readiness and the bootstrap itself. Strict means
|
||||
the browser never attaches it to a request another site initiated.
|
||||
* A **CSRF token** must be echoed in a header on every mutation. It is handed out
|
||||
only in the bootstrap response body, which a foreign page cannot read (no CORS) —
|
||||
so possessing it proves the caller is same-origin.
|
||||
|
||||
Behind a reverse proxy (US08-01) the same stack holds with two substitutions: the
|
||||
allowed host set comes from configuration instead of being the loopback names, and
|
||||
the host/scheme the policy judges is the *external* one, which is only read from
|
||||
``X-Forwarded-*`` when the request actually arrived from a configured proxy. The
|
||||
loopback check was standing in for authentication, so naming a non-loopback host
|
||||
also makes an access secret mandatory — ``trust_refusal`` refuses to start without
|
||||
one, and the secret is what the bootstrap endpoint trades for the session cookie.
|
||||
|
||||
``evaluate`` is a pure function over the request metadata: the whole policy is one
|
||||
table that a unit test can enumerate, and the middleware only applies its verdict.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import secrets
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from starlette.datastructures import Headers, MutableHeaders
|
||||
from starlette.responses import JSONResponse
|
||||
|
||||
SESSION_COOKIE = "pp_session"
|
||||
CSRF_HEADER = "x-csrf-token"
|
||||
ACCESS_SECRET_HEADER = "x-access-secret"
|
||||
API_PREFIX = "/api/v1"
|
||||
SAFE_METHODS = frozenset({"GET", "HEAD", "OPTIONS"})
|
||||
# Reachable without a session: liveness/readiness (an orchestrator has no cookie)
|
||||
# and the bootstrap that issues the session in the first place.
|
||||
PUBLIC_PATHS = frozenset(
|
||||
{f"{API_PREFIX}/health/live", f"{API_PREFIX}/health/ready", f"{API_PREFIX}/session"}
|
||||
)
|
||||
LOOPBACK_HOSTS = frozenset({"127.0.0.1", "localhost", "::1", "[::1]"})
|
||||
# Mutating endpoints that must stay reachable while mutation itself is gated: the
|
||||
# backup a careful operator takes first, and its retention (US07-07).
|
||||
MUTATION_EXEMPT_PATHS = frozenset({f"{API_PREFIX}/backups", f"{API_PREFIX}/backups/prune"})
|
||||
|
||||
# Applied to every response. `frame-ancestors 'none'` and CORP keep other pages from
|
||||
# embedding the app or its thumbnails.
|
||||
#
|
||||
# `script-src 'self'` is the boundary that matters and it is unchanged: no
|
||||
# `'unsafe-eval'`, no `'unsafe-inline'`, so nothing injected into the DOM can execute.
|
||||
# Whether the vendored diagram renderer needed `'unsafe-eval'` was measured rather
|
||||
# than assumed — mermaid's bundle contains no `eval(` and no `new Function`, and it
|
||||
# renders under this exact policy without a single script-src violation.
|
||||
#
|
||||
# `style-src` does gain `'unsafe-inline'` (US09-01): mermaid styles the SVG it builds
|
||||
# with an injected `<style>` element and `style=` attributes, and a diagram's CSS
|
||||
# cannot be hashed in advance. The concession is bounded by the directives around it
|
||||
# — with script execution still refused and `img-src`, `connect-src`, and `font-src`
|
||||
# all `'self'`, the CSS exfiltration channels stay closed and what is left is
|
||||
# defacement of a page its own operator is already looking at.
|
||||
DEFAULT_HEADERS = {
|
||||
"x-content-type-options": "nosniff",
|
||||
"x-frame-options": "DENY",
|
||||
"referrer-policy": "no-referrer",
|
||||
"cross-origin-resource-policy": "same-origin",
|
||||
"cross-origin-opener-policy": "same-origin",
|
||||
"content-security-policy": (
|
||||
"default-src 'self'; img-src 'self' data:; style-src 'self' 'unsafe-inline'; "
|
||||
"script-src 'self'; connect-src 'self'; font-src 'self'; "
|
||||
"frame-ancestors 'none'; base-uri 'none'; form-action 'none'"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Session:
|
||||
"""One process, one session. A local app has exactly one user; a session store
|
||||
would be bookkeeping without a second subject to distinguish.
|
||||
|
||||
ponytail: per-session rows if the app ever serves more than one operator.
|
||||
"""
|
||||
|
||||
id: str
|
||||
csrf_token: str
|
||||
|
||||
@classmethod
|
||||
def create(cls) -> Session:
|
||||
return cls(secrets.token_urlsafe(32), secrets.token_urlsafe(32))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Refusal:
|
||||
status: int
|
||||
code: str
|
||||
message: str
|
||||
|
||||
|
||||
def split_host(value: str) -> tuple[str, str]:
|
||||
"""``"127.0.0.1:8000"`` -> ``("127.0.0.1", "8000")``; bracketed IPv6 aware."""
|
||||
value = value.strip()
|
||||
if value.startswith("["):
|
||||
host, _, port = value.partition("]")
|
||||
return host + "]", port.lstrip(":")
|
||||
host, _, port = value.partition(":")
|
||||
return host, port
|
||||
|
||||
|
||||
def external_view(
|
||||
*,
|
||||
client: str | None,
|
||||
headers: Mapping[str, str],
|
||||
scheme: str,
|
||||
trusted_proxies: frozenset[str],
|
||||
) -> tuple[str, str]:
|
||||
"""The ``(scheme, host)`` the caller used, as opposed to the one this hop saw.
|
||||
|
||||
Forwarded headers are a client-supplied claim. Believing them from anyone lets a
|
||||
request declare its own origin — and origin is half of this module's evidence —
|
||||
so they count only when the connection came from a configured proxy.
|
||||
"""
|
||||
host = headers.get("host", "")
|
||||
if client is None or client not in trusted_proxies:
|
||||
return scheme, host
|
||||
# A chain appends: the first entry is what the original client asked for.
|
||||
forwarded_proto = headers.get("x-forwarded-proto", "").split(",")[0].strip().lower()
|
||||
forwarded_host = headers.get("x-forwarded-host", "").split(",")[0].strip()
|
||||
return forwarded_proto or scheme, forwarded_host or host
|
||||
|
||||
|
||||
def evaluate(
|
||||
*,
|
||||
method: str,
|
||||
path: str,
|
||||
headers: Mapping[str, str],
|
||||
session: Session,
|
||||
allowed_hosts: frozenset[str] = LOOPBACK_HOSTS,
|
||||
scheme: str = "http",
|
||||
max_request_bytes: int,
|
||||
) -> Refusal | None:
|
||||
"""Why this request must be refused, or ``None`` when it may proceed.
|
||||
|
||||
``headers["host"]`` and ``scheme`` are the external ones (see ``external_view``);
|
||||
the allowed origins are the allowed hosts under that scheme and port, so there is
|
||||
no second list that can drift away from the first.
|
||||
"""
|
||||
host_header = headers.get("host", "")
|
||||
host, port = split_host(host_header)
|
||||
if host.lower() not in allowed_hosts:
|
||||
return Refusal(403, "host_not_allowed", "request host is not an allowed address")
|
||||
|
||||
origin = headers.get("origin")
|
||||
if origin is not None and origin != "":
|
||||
parts = urlsplit(origin)
|
||||
origin_host, origin_port = split_host(parts.netloc)
|
||||
if (
|
||||
parts.scheme != scheme
|
||||
or origin_host.lower() not in allowed_hosts
|
||||
or origin_port != port
|
||||
):
|
||||
return Refusal(403, "origin_not_allowed", "request origin is not this application")
|
||||
|
||||
# Absent means a non-browser client; "none" is a user-initiated navigation.
|
||||
fetch_site = headers.get("sec-fetch-site")
|
||||
if fetch_site is not None and fetch_site not in ("same-origin", "none"):
|
||||
return Refusal(403, "cross_site_blocked", "cross-site requests are not accepted")
|
||||
|
||||
length = headers.get("content-length")
|
||||
if length and length.isdigit() and int(length) > max_request_bytes:
|
||||
return Refusal(413, "payload_too_large", "request body exceeds the configured limit")
|
||||
|
||||
protected = path.startswith(API_PREFIX) and path not in PUBLIC_PATHS
|
||||
if not protected:
|
||||
return None
|
||||
|
||||
if headers.get("cookie-session") != session.id:
|
||||
return Refusal(401, "unauthenticated", "a valid application session is required")
|
||||
if method.upper() not in SAFE_METHODS and headers.get(CSRF_HEADER) != session.csrf_token:
|
||||
return Refusal(403, "csrf_failed", "missing or invalid CSRF token")
|
||||
return None
|
||||
|
||||
|
||||
def exposed_hosts(config) -> list[str]:
|
||||
"""Configured names by which this application is reachable from another machine."""
|
||||
names = {str(config.host).lower()}
|
||||
names.update(split_host(name)[0].lower() for name in config.allowed_hosts)
|
||||
return sorted(names - LOOPBACK_HOSTS)
|
||||
|
||||
|
||||
def trust_refusal(config) -> str | None:
|
||||
"""Why this configuration must not serve at all, or ``None``.
|
||||
|
||||
Reaching the app used to prove ownership of it. The moment a configuration makes
|
||||
it reachable from elsewhere that stops being true, so serving without a secret
|
||||
would publish the library — refuse at startup rather than at the first request,
|
||||
when the operator is no longer watching (US08-01).
|
||||
"""
|
||||
exposed = exposed_hosts(config)
|
||||
if exposed and config.access_secret is None:
|
||||
return (
|
||||
f"refusing to serve: {', '.join(exposed)} is reachable from outside this "
|
||||
"machine, so PHOTO_PIPELINE_ACCESS_SECRET must be set"
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
class FailureLimiter:
|
||||
"""Bounded failed access-secret attempts, so the secret cannot be guessed online.
|
||||
|
||||
ponytail: one counter for the whole process rather than per client address —
|
||||
behind a proxy every attempt arrives from the same address anyway. Per-caller
|
||||
buckets if the app is ever exposed without one.
|
||||
"""
|
||||
|
||||
def __init__(self, limit: int = 5, window: float = 60.0) -> None:
|
||||
self.limit = limit
|
||||
self.window = window
|
||||
self._failures: list[float] = []
|
||||
|
||||
def blocked(self) -> bool:
|
||||
now = time.monotonic()
|
||||
self._failures = [at for at in self._failures if now - at < self.window]
|
||||
return len(self._failures) >= self.limit
|
||||
|
||||
def record_failure(self) -> None:
|
||||
self._failures.append(time.monotonic())
|
||||
|
||||
|
||||
class SecurityMiddleware:
|
||||
"""Pure-ASGI so the SSE stream keeps streaming (BaseHTTPMiddleware buffers)."""
|
||||
|
||||
def __init__(self, app, *, session: Session, config) -> None:
|
||||
self.app = app
|
||||
self.session = session
|
||||
self.config = config
|
||||
self.max_request_bytes = config.max_request_bytes
|
||||
self.allowed_hosts = frozenset(
|
||||
LOOPBACK_HOSTS
|
||||
| {str(config.host).lower()}
|
||||
| {split_host(name)[0].lower() for name in config.allowed_hosts}
|
||||
)
|
||||
self.trusted_proxies = frozenset(config.trusted_proxies)
|
||||
|
||||
async def __call__(self, scope, receive, send) -> None:
|
||||
if scope["type"] != "http":
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
headers = Headers(scope=scope)
|
||||
# The cookie is read here and handed to the pure policy as one value, so the
|
||||
# policy never has to parse a Cookie header.
|
||||
lookup = dict(headers)
|
||||
lookup["cookie-session"] = _cookie(headers.get("cookie", ""), SESSION_COOKIE)
|
||||
client = scope.get("client")
|
||||
scheme, lookup["host"] = external_view(
|
||||
client=client[0] if client else None,
|
||||
headers=headers,
|
||||
scheme=scope.get("scheme", "http"),
|
||||
trusted_proxies=self.trusted_proxies,
|
||||
)
|
||||
# What the session cookie's Secure flag is decided from, one hop later.
|
||||
scope.setdefault("state", {})["external_scheme"] = scheme
|
||||
refusal = evaluate(
|
||||
method=scope.get("method", "GET"),
|
||||
path=scope.get("path", "/"),
|
||||
headers=lookup,
|
||||
session=self.session,
|
||||
allowed_hosts=self.allowed_hosts,
|
||||
scheme=scheme,
|
||||
max_request_bytes=self.max_request_bytes,
|
||||
)
|
||||
if refusal is None:
|
||||
refusal = self._mutation_refusal(scope)
|
||||
if refusal is not None:
|
||||
response = JSONResponse(
|
||||
status_code=refusal.status,
|
||||
content={"error": {"code": refusal.code, "message": refusal.message}},
|
||||
headers=DEFAULT_HEADERS,
|
||||
)
|
||||
await response(scope, receive, send)
|
||||
return
|
||||
|
||||
async def send_with_headers(message):
|
||||
if message["type"] == "http.response.start":
|
||||
out = MutableHeaders(scope=message)
|
||||
for name, value in DEFAULT_HEADERS.items():
|
||||
out.setdefault(name, value)
|
||||
await send(message)
|
||||
|
||||
await self.app(scope, receive, send_with_headers)
|
||||
|
||||
def _mutation_refusal(self, scope) -> Refusal | None:
|
||||
"""Refuse every mutating request while the library's dry run is unapproved.
|
||||
|
||||
One choke point for the whole API: every mutation the browser can start is a
|
||||
non-safe method under ``/api/v1``. Reading stays open — an operator has to be
|
||||
able to look at what the application found in order to approve it (US07-07).
|
||||
"""
|
||||
method = scope.get("method", "GET").upper()
|
||||
path = scope.get("path", "/")
|
||||
if method in SAFE_METHODS or not path.startswith(API_PREFIX):
|
||||
return None
|
||||
if path in MUTATION_EXEMPT_PATHS:
|
||||
return None
|
||||
from photo_pipeline.services.release import mutation_blockers
|
||||
|
||||
blockers = mutation_blockers(self.config)
|
||||
if not blockers:
|
||||
return None
|
||||
return Refusal(403, blockers[0]["code"], blockers[0]["message"])
|
||||
|
||||
|
||||
def _cookie(header: str, name: str) -> str:
|
||||
for part in header.split(";"):
|
||||
key, _, value = part.strip().partition("=")
|
||||
if key == name:
|
||||
return value
|
||||
return ""
|
||||
@@ -7,9 +7,6 @@ real external call needs them.
|
||||
|
||||
pydantic-settings would do this too, but a prefix-scan over the declared fields
|
||||
is a few lines and one fewer dependency.
|
||||
|
||||
Tuple-valued settings are lists in one variable: library roots are ``os.pathsep``
|
||||
separated because they are paths, everything else is comma separated.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -21,62 +18,6 @@ from typing import Mapping
|
||||
from pydantic import BaseModel, ConfigDict, SecretStr
|
||||
|
||||
ENV_PREFIX = "PHOTO_PIPELINE_"
|
||||
ENV_FILE_VAR = f"{ENV_PREFIX}ENV_FILE"
|
||||
DEFAULT_ENV_FILE = Path(".env")
|
||||
COMMA_LIST_FIELDS = frozenset({"allowed_hosts", "trusted_proxies"})
|
||||
|
||||
# The archived CLI's variable names, so the configuration file an operator already
|
||||
# has keeps working. The vision provider reads the OpenAI SDK's names, and the
|
||||
# library root is configuration here rather than a bare path (US07-01 donor).
|
||||
LEGACY_ALIASES = {
|
||||
"LLM_API_KEY": "OPENAI_API_KEY",
|
||||
"GEMINI_API_KEY": "OPENAI_API_KEY",
|
||||
"LLM_BASE_URL": "OPENAI_BASE_URL",
|
||||
"LIBRARY": f"{ENV_PREFIX}LIBRARY_ROOTS",
|
||||
}
|
||||
|
||||
|
||||
def parse_env_file(text: str) -> dict[str, str]:
|
||||
"""``KEY=value`` lines into a mapping. Comments, blanks, and quotes handled.
|
||||
|
||||
Deliberately not a shell: no interpolation, no ``export``, no multi-line values.
|
||||
A configuration file that can run code is a configuration file that can be a
|
||||
vulnerability.
|
||||
"""
|
||||
values: dict[str, str] = {}
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, raw = line.partition("=")
|
||||
key = key.strip()
|
||||
if not key or key.startswith("#"):
|
||||
continue
|
||||
value = raw.strip().strip('"').strip("'")
|
||||
values[key] = value
|
||||
alias = LEGACY_ALIASES.get(key)
|
||||
if alias:
|
||||
values.setdefault(alias, value)
|
||||
return values
|
||||
|
||||
|
||||
def load_env_file(path: Path | str | None = None) -> dict[str, str]:
|
||||
"""Load ``PHOTO_PIPELINE_ENV_FILE`` (or ``./.env``) into the environment.
|
||||
|
||||
Anything already exported wins: a file is the standing configuration, the shell
|
||||
is what you meant *this time*. Returns what it applied, which is what the CLI
|
||||
prints — names only, never values.
|
||||
"""
|
||||
candidate = path or os.environ.get(ENV_FILE_VAR) or DEFAULT_ENV_FILE
|
||||
candidate = Path(candidate)
|
||||
if not candidate.is_file():
|
||||
return {}
|
||||
applied = {}
|
||||
for key, value in parse_env_file(candidate.read_text()).items():
|
||||
if key not in os.environ:
|
||||
os.environ[key] = value
|
||||
applied[key] = value
|
||||
return applied
|
||||
|
||||
|
||||
class Config(BaseModel):
|
||||
@@ -89,23 +30,6 @@ class Config(BaseModel):
|
||||
log_level: str = "INFO"
|
||||
log_format: str = "json" # "json" or "text"
|
||||
|
||||
# Trust boundary (US08-01). Empty means loopback only, which is what the app did
|
||||
# before there was a setting: a request whose Host is not a loopback name is
|
||||
# refused, and no secret is needed because nothing outside this machine can call.
|
||||
# Naming a real hostname here is what makes the app reachable through a reverse
|
||||
# proxy, and it is exactly then that ``access_secret`` becomes mandatory.
|
||||
allowed_hosts: tuple[str, ...] = ()
|
||||
# Addresses whose ``X-Forwarded-Proto``/``X-Forwarded-Host`` may be believed. A
|
||||
# client that is not the proxy can otherwise declare its own origin.
|
||||
trusted_proxies: tuple[str, ...] = ()
|
||||
# Exchanged for the session cookie at the bootstrap endpoint. Once set it is
|
||||
# required even on loopback, so a development setup cannot half-enable it.
|
||||
access_secret: SecretStr | None = None
|
||||
|
||||
# Largest request body the API accepts. Every endpoint takes small JSON commands;
|
||||
# anything larger is a mistake or an attempt to exhaust memory (US07-02).
|
||||
max_request_bytes: int = 1_048_576
|
||||
|
||||
# Library boundary for path validation (os.pathsep-separated in the env var).
|
||||
library_roots: tuple[Path, ...] = ()
|
||||
thumbnail_cache_quota_bytes: int = 500_000_000
|
||||
@@ -114,12 +38,6 @@ class Config(BaseModel):
|
||||
# Free space an archive destination must keep beyond the transfer itself.
|
||||
archive_free_space_reserve_bytes: int = 1_000_000_000
|
||||
|
||||
# Refuse every mutating request until a read-only dry run of the configured
|
||||
# library has been produced and explicitly approved (US07-07). Off by default so
|
||||
# a development setup is unchanged; turn it on before pointing the application at
|
||||
# a library whose photos cannot be replaced.
|
||||
require_dry_run_approval: bool = False
|
||||
|
||||
vision_api_key: SecretStr | None = None
|
||||
immich_api_key: SecretStr | None = None
|
||||
immich_server_url: str = ""
|
||||
@@ -140,18 +58,10 @@ class Config(BaseModel):
|
||||
@classmethod
|
||||
def from_env(cls, environ: Mapping[str, str] | None = None) -> "Config":
|
||||
env = os.environ if environ is None else environ
|
||||
if environ is None:
|
||||
load_env_file() # a file never overrides what the shell already set
|
||||
env = os.environ
|
||||
data: dict = {}
|
||||
for name in cls.model_fields:
|
||||
raw = env.get(ENV_PREFIX + name.upper())
|
||||
if not raw:
|
||||
continue
|
||||
if name == "library_roots":
|
||||
data[name] = raw.split(os.pathsep)
|
||||
elif name in COMMA_LIST_FIELDS:
|
||||
data[name] = [part.strip() for part in raw.split(",") if part.strip()]
|
||||
else:
|
||||
data[name] = raw
|
||||
data[name] = raw.split(os.pathsep) if name == "library_roots" else raw
|
||||
return cls(**data)
|
||||
|
||||
@@ -41,41 +41,12 @@ def create_session_factory(engine: Engine) -> sessionmaker:
|
||||
return sessionmaker(bind=engine, expire_on_commit=False, future=True)
|
||||
|
||||
|
||||
def _alembic_config(url: str):
|
||||
def run_migrations(url: str) -> None:
|
||||
"""Upgrade the database at ``url`` to the latest revision."""
|
||||
from alembic import command
|
||||
from alembic.config import Config as AlembicConfig
|
||||
|
||||
cfg = AlembicConfig(str(_REPO_ROOT / "alembic.ini"))
|
||||
cfg.set_main_option("script_location", str(_REPO_ROOT / "migrations"))
|
||||
cfg.set_main_option("sqlalchemy.url", url)
|
||||
return cfg
|
||||
|
||||
|
||||
def run_migrations(url: str) -> None:
|
||||
"""Upgrade the database at ``url`` to the latest revision."""
|
||||
from alembic import command
|
||||
|
||||
command.upgrade(_alembic_config(url), "head")
|
||||
|
||||
|
||||
def head_revision() -> str | None:
|
||||
"""The revision this code expects. ``None`` if the scripts cannot be read."""
|
||||
from alembic.script import ScriptDirectory
|
||||
|
||||
try:
|
||||
return ScriptDirectory.from_config(_alembic_config("sqlite://")).get_current_head()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def current_revision(url: str) -> str | None:
|
||||
"""The revision a database is actually at, or ``None`` for an unstamped one."""
|
||||
engine = create_db_engine(url)
|
||||
try:
|
||||
with engine.connect() as connection:
|
||||
from alembic.runtime.migration import MigrationContext
|
||||
|
||||
return MigrationContext.configure(connection).get_current_revision()
|
||||
except Exception:
|
||||
return None
|
||||
finally:
|
||||
engine.dispose()
|
||||
command.upgrade(cfg, "head")
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
"""Test-only fault control points (concept §18, US07-04).
|
||||
|
||||
Crash safety can only be proven by crashing at the exact moment a transition has
|
||||
been persisted but its consequence has not. That needs a barrier *inside* the
|
||||
production code path — but not a production capability: there is no endpoint, no
|
||||
service method, and no configuration file entry that can trigger one. The only
|
||||
switch is an environment variable naming a single point, read at the moment it is
|
||||
passed, and the only thing it does is kill the process. A deployment that never
|
||||
sets it can never reach the barrier.
|
||||
|
||||
``os._exit`` is deliberate: it skips atexit handlers, buffered flushes, and
|
||||
``finally`` blocks, which is what a real ``SIGKILL`` or power loss does. A clean
|
||||
shutdown would prove nothing.
|
||||
|
||||
The points are the persisted transitions of the journalled stages:
|
||||
|
||||
rename moving | moved | database_updated | verified | complete
|
||||
archive transferring | verified | removing | source_removed | complete
|
||||
exif exif:written — keywords on disk, checkpoint not yet recorded
|
||||
upload upload:accepted — uploader exited, outcome not yet persisted
|
||||
jobs job:item_done — item committed, job outcome not yet written
|
||||
|
||||
Recovery for each is asserted in tests/integration/test_fault_matrix.py and
|
||||
tests/e2e/test_crash_recovery.py.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
ENV_VAR = "PHOTO_PIPELINE_FAULT_AFTER"
|
||||
|
||||
EXIF_WRITTEN = "exif:written"
|
||||
UPLOAD_ACCEPTED = "upload:accepted"
|
||||
JOB_ITEM_DONE = "job:item_done"
|
||||
|
||||
|
||||
def maybe_fault(point: str) -> None:
|
||||
"""Die abruptly when ``PHOTO_PIPELINE_FAULT_AFTER`` names ``point``.
|
||||
|
||||
Shared by the rename, archive, restore, EXIF, upload, and job lanes, each
|
||||
passing its own state names. Never set the variable outside tests.
|
||||
"""
|
||||
if os.environ.get(ENV_VAR) == point:
|
||||
os._exit(9)
|
||||
@@ -1,119 +0,0 @@
|
||||
"""Bounded, defensive image decoding — one door for every pixel this app reads.
|
||||
|
||||
A photo library contains files nobody planned for: truncated downloads, zero-byte
|
||||
placeholders, a PNG whose header claims 200000×200000, a TIFF with a broken ICC
|
||||
profile, an extension that lies about its content. None of them may take down a
|
||||
request or a worker, and none may decode more pixels than the caller allowed
|
||||
(concept §17: decoded pixels, not file size, are what exhausts memory).
|
||||
|
||||
``open_image`` is that single door:
|
||||
|
||||
* the declared dimensions are checked **before** a pixel is decoded;
|
||||
* Pillow's decompression-bomb *warning* is promoted to an error, because the
|
||||
warning band (between Pillow's limit and twice it) still decodes the image;
|
||||
* every decoder failure — at open time or during the caller's decode — becomes one
|
||||
of two typed errors, so callers map them to their own item state instead of
|
||||
catching ``Exception``;
|
||||
* error text names no path: it reaches API responses, and the full reason goes to
|
||||
the server log instead (US07-02).
|
||||
|
||||
``to_srgb`` and ``draft`` are the other two bounded-decode helpers: colour-manage a
|
||||
profile-bearing image into sRGB, and let JPEG decode straight to a size near the
|
||||
requested one rather than at full resolution.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import logging
|
||||
import warnings
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image, ImageCms, ImageFile, UnidentifiedImageError
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Matches ``Config.thumbnail_max_pixels``; used where no configuration is at hand
|
||||
# (hashing runs inside discovery, which takes no config).
|
||||
DEFAULT_MAX_PIXELS = 100_000_000
|
||||
|
||||
|
||||
class MediaError(Exception):
|
||||
"""A file could not be turned into pixels safely."""
|
||||
|
||||
|
||||
class UndecodableImage(MediaError):
|
||||
"""Corrupt, truncated, empty, or not an image at all."""
|
||||
|
||||
|
||||
class ImageTooLarge(MediaError):
|
||||
"""More pixels than this operation is allowed to decode."""
|
||||
|
||||
|
||||
@contextmanager
|
||||
def open_image(path: Path | str, *, max_pixels: int = DEFAULT_MAX_PIXELS):
|
||||
"""Yield an open :class:`PIL.Image.Image`, bounded and with typed failures.
|
||||
|
||||
Decoder errors raised inside the ``with`` body are translated too — a truncated
|
||||
JPEG only fails when its pixels are actually pulled, which is the caller's line,
|
||||
not this one.
|
||||
"""
|
||||
# Pillow's truncation tolerance is a process-global switch that any library in
|
||||
# the process can flip (the donor CLI did). This door decides the policy for its
|
||||
# own callers: half a file is not a picture.
|
||||
tolerated = ImageFile.LOAD_TRUNCATED_IMAGES
|
||||
ImageFile.LOAD_TRUNCATED_IMAGES = False
|
||||
with warnings.catch_warnings():
|
||||
# The warning band is not a warning for us: it means Pillow was willing to
|
||||
# decode an image large enough to be a denial-of-service.
|
||||
warnings.simplefilter("error", Image.DecompressionBombWarning)
|
||||
try:
|
||||
with Image.open(path) as image:
|
||||
width, height = image.size
|
||||
if width * height > max_pixels:
|
||||
raise ImageTooLarge(f"{width}x{height} exceeds the {max_pixels} pixel limit")
|
||||
yield image
|
||||
except MediaError:
|
||||
raise
|
||||
except (Image.DecompressionBombError, Image.DecompressionBombWarning) as error:
|
||||
log.info("refused oversized image %s: %s", path, error)
|
||||
raise ImageTooLarge("image exceeds the decompression-bomb limit") from error
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError, MemoryError) as error:
|
||||
log.info("cannot decode %s: %s", path, error)
|
||||
raise UndecodableImage(f"cannot decode image ({type(error).__name__})") from error
|
||||
finally:
|
||||
ImageFile.LOAD_TRUNCATED_IMAGES = tolerated
|
||||
|
||||
|
||||
def draft(image: Image.Image, size: int) -> None:
|
||||
"""Ask the decoder for a smaller image where the format allows it (JPEG).
|
||||
|
||||
This is the difference between decoding a 40-megapixel JPEG and decoding the
|
||||
roughly 1-megapixel version a 1280px preview needs.
|
||||
"""
|
||||
try:
|
||||
image.draft(None, (size, size))
|
||||
except (AttributeError, ValueError, OSError): # not a draft-capable format
|
||||
pass
|
||||
|
||||
|
||||
def to_srgb(image: Image.Image, *, mode: str) -> Image.Image:
|
||||
"""Convert into ``mode``, colour-managing through an embedded ICC profile.
|
||||
|
||||
Without this a wide-gamut original renders with visibly wrong colours, because
|
||||
its numbers are interpreted as sRGB. A broken or unreadable profile is not a
|
||||
reason to fail a preview — the plain conversion is still a correct picture.
|
||||
"""
|
||||
profile = image.info.get("icc_profile")
|
||||
if profile:
|
||||
try:
|
||||
return ImageCms.profileToProfile(
|
||||
image,
|
||||
ImageCms.ImageCmsProfile(io.BytesIO(profile)),
|
||||
ImageCms.createProfile("sRGB"),
|
||||
outputMode=mode,
|
||||
)
|
||||
except Exception as error: # noqa: BLE001 - any ICC failure falls back
|
||||
log.info("ignoring unusable ICC profile on %s: %s", getattr(image, "filename", "?"), error)
|
||||
return image.convert(mode)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user