"""Reusable Phase B end-to-end harness: seed an isolated library, then launch the real API server and durable worker as child processes so tests talk to them only over HTTP/SSE — the same boundaries used in production (concept §18). External vision is replaced at its integration edge by the in-repo recording fake (``PHOTO_PIPELINE_FAKE_VISION_LOG``); it is invoked through the real AnalysisService and worker, never mocked inside a test. """ from __future__ import annotations import json import os import socket import stat import subprocess import sys import threading import time import uuid from dataclasses import dataclass from datetime import datetime, timezone from http.server import BaseHTTPRequestHandler, HTTPServer from pathlib import Path import httpx import numpy as np from PIL import Image REPO = Path(__file__).resolve().parents[2] def image(path: Path, seed: int) -> None: path.parent.mkdir(parents=True, exist_ok=True) rng = np.random.default_rng(seed) Image.fromarray(rng.integers(0, 256, (96, 128, 3), dtype=np.uint8)).save(path, quality=90) def free_port() -> int: with socket.socket() as sock: sock.bind(("127.0.0.1", 0)) return sock.getsockname()[1] @dataclass class Seeded: data: Path lib: Path asset_ids: dict[str, str] # filename stem -> asset id def seed_library(tmp_path: Path, files: dict[str, int], decisions: dict[str, str]) -> Seeded: """Scan ``files`` (stem -> seed) into a fresh DB and apply safety ``decisions`` (stem -> ``sfw``/``nsfw``) in-process before any server starts, so the seed is deterministic and independent of the API under test.""" data = tmp_path / "data" data.mkdir() lib = tmp_path / "lib" lib.mkdir() for stem, seed in files.items(): image(lib / f"{stem}.jpg", seed) from photo_pipeline.config import Config from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations from photo_pipeline.models import Asset from photo_pipeline.services.inventory import InventoryService from photo_pipeline.services.safety import SafetyService from sqlalchemy import select config = Config.from_env( {"PHOTO_PIPELINE_DATA_DIR": str(data), "PHOTO_PIPELINE_LIBRARY_ROOTS": str(lib)} ) run_migrations(config.database_url) engine = create_db_engine(config.database_url) sf = create_session_factory(engine) InventoryService(sf).scan(lib) with sf() as session: rows = list(session.execute(select(Asset.id, Asset.current_path)).all()) ids = {Path(path).stem: aid for aid, path in rows} safety = SafetyService(sf) for stem, decision in decisions.items(): safety.decide(ids[stem], decision, write_exif=False) engine.dispose() return Seeded(data=data, lib=lib, asset_ids=ids) def _env(seeded: Seeded, port: int, *, extra: dict[str, str] | None = None) -> dict[str, str]: env = { "PATH": os.environ.get("PATH", ""), "PHOTO_PIPELINE_DATA_DIR": str(seeded.data), "PHOTO_PIPELINE_LIBRARY_ROOTS": str(seeded.lib), "PHOTO_PIPELINE_HOST": "127.0.0.1", "PHOTO_PIPELINE_PORT": str(port), } if extra: env.update(extra) return env class Server: """A real ``photo_pipeline serve`` child process, ready when /health/ready is 200.""" def __init__(self, seeded: Seeded, *, extra_env: dict[str, str] | None = None) -> None: self._seeded = seeded self._extra = extra_env self.port = free_port() self.base = f"http://127.0.0.1:{self.port}" self.proc: subprocess.Popen | None = None def start(self) -> "Server": self.proc = subprocess.Popen( [sys.executable, "-m", "photo_pipeline", "serve"], cwd=str(REPO), env=_env(self._seeded, self.port, extra=self._extra), stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) deadline = time.monotonic() + 30 while time.monotonic() < deadline: if self.proc.poll() is not None: _, err = self.proc.communicate() raise RuntimeError(f"server exited: {err.decode(errors='replace')}") try: if httpx.get(f"{self.base}/api/v1/health/ready", timeout=1).status_code == 200: return self except httpx.HTTPError: time.sleep(0.2) self.stop() raise RuntimeError("server never became ready") def stop(self) -> None: if self.proc is None: return self.proc.terminate() try: self.proc.wait(timeout=10) except subprocess.TimeoutExpired: self.proc.kill() self.proc = None def start_worker( seeded: Seeded, *, fake_vision_log: Path | None = None, extra_env: dict[str, str] | None = None, ) -> subprocess.Popen: """Launch a real durable worker wired to the recording vision fake. ``extra_env`` carries whatever else the job under test needs — the Immich credentials and uploader path, for the upload lane. """ extra = dict(extra_env or {}) if fake_vision_log is not None: extra["PHOTO_PIPELINE_FAKE_VISION_LOG"] = str(fake_vision_log) return subprocess.Popen( [sys.executable, "-m", "photo_pipeline", "worker", "--id", "e2e-worker"], cwd=str(REPO), env=_env( seeded, free_port(), # unused by the worker, but keeps the env shape uniform extra=extra, ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) def wait_until(predicate, *, timeout: float = 20, interval: float = 0.1): """Poll ``predicate`` until it returns a truthy value or the timeout elapses. Returns the truthy value; raises on timeout. Synchronizes on observable state instead of sleeping for a fixed duration (concept §18).""" deadline = time.monotonic() + timeout while time.monotonic() < deadline: value = predicate() if value: return value time.sleep(interval) raise AssertionError("condition not met before timeout") # ── Phase D: an analysed album, ready to be named and renamed ──────────────── NOW = datetime(2026, 1, 1, tzinfo=timezone.utc) class session_factory: """Session factory against a seeded database, for the few things a test has to set up or inspect below the API — journal states, mainly.""" def __init__(self, seeded: Seeded) -> None: self._seeded = seeded def __enter__(self): from photo_pipeline.config import Config from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations config = Config.from_env( { "PHOTO_PIPELINE_DATA_DIR": str(self._seeded.data), "PHOTO_PIPELINE_LIBRARY_ROOTS": str(self._seeded.lib), } ) run_migrations(config.database_url) self._engine = create_db_engine(config.database_url) return create_session_factory(self._engine) def __exit__(self, *_): self._engine.dispose() return False def seed_album(tmp_path: Path, album: str = "rome", names: tuple[str, ...] = ("a.jpg", "b.jpg")): """A library holding one album folder whose photos are confirmed SFW and analysed — the state a naming proposal, and therefore a rename plan, is built from.""" from sqlalchemy import select from photo_pipeline.models import AnalysisResult, Asset, SafetyReview from photo_pipeline.services.inventory import InventoryService seeded = seed_library(tmp_path, {}, {}) folder = seeded.lib / album folder.mkdir(parents=True) for index, name in enumerate(names): image(folder / name, index + 1) with session_factory(seeded) as sf: InventoryService(sf).scan(seeded.lib) with sf() as session: rows = list(session.execute(select(Asset.id, Asset.current_path)).all()) for asset_id, path in rows: session.add( SafetyReview( id=str(uuid.uuid4()), asset_id=asset_id, decision="sfw", created_at=NOW ) ) session.add( AnalysisResult( asset_id=asset_id, status="analyzed", description=f"a view of {path}", tags='["ruins", "city"]', approx_year=2019, location_hint="Rome", ) ) session.commit() seeded.asset_ids.update({Path(path).stem: aid for aid, path in rows}) return seeded def approve_album(base: str, *, album: str = "rome", name: str) -> None: """Generate a proposal, set its final name, and approve it over HTTP.""" httpx.post(f"{base}/api/v1/albums/proposals", json={}, timeout=10).raise_for_status() for payload, route in ( ({"name": name}, "edit"), ({}, "approve"), ): current = httpx.get(f"{base}/api/v1/albums/proposals/{album}", timeout=10).json() httpx.post( f"{base}/api/v1/albums/proposals/{album}/{route}", json={**payload, "expected_version": current["version"]}, timeout=10, ).raise_for_status() # ── Phase E: an upload-ready album, a fake Immich, and a real fake uploader ─── SENTINEL_KEY = "immich-sentinel-9f3a2b" UPLOADER_VERSION = "immich-go 0.21.0" # a pinned family, so reports are parsable # Uploader bodies for the pinned ``text-v1`` grammar. ``$6`` is the folder argument # of ``upload from-folder``. REPORTING_UPLOADER = ( 'echo "INFO uploaded $6/a.jpg"\n' 'echo "INFO server has the same file $6/b.jpg"\n' 'echo "Uploaded 1, duplicates 1"\n' "exit 0\n" ) # Exits cleanly but says nothing about any file: the process succeeded, the # per-file outcome is unknown. SILENT_UPLOADER = "exit 0\n" def _immich_handler(state: dict): class Handler(BaseHTTPRequestHandler): def do_GET(self): # noqa: N802 (BaseHTTPRequestHandler API) self._json(200, {"res": "pong"}) def do_POST(self): # noqa: N802 length = int(self.headers.get("Content-Length", 0)) payload = json.loads(self.rfile.read(length) or b"{}") if state["mode"] == "broken": self.send_error(500, "bulk-upload-check is unavailable") return reject = state["mode"] == "present" self._json( 200, { "results": [ { "id": asset["id"], "action": "reject" if reject else "accept", "reason": "duplicate" if reject else None, } for asset in payload.get("assets", []) ] }, ) def _json(self, code: int, body: dict) -> None: raw = json.dumps(body).encode() self.send_response(code) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(raw))) self.end_headers() self.wfile.write(raw) def log_message(self, *args): pass return Handler class FakeImmich: """An Immich that answers ping, and says whether it holds the exact bytes. ``mode`` is what the next verification will find: ``present`` (the server deduplicates them, so it has them), ``absent`` (it would accept them, so it does not), or ``broken`` (no usable answer at all). """ def __init__(self) -> None: self.state = {"mode": "present"} self._server = HTTPServer(("127.0.0.1", 0), _immich_handler(self.state)) threading.Thread(target=self._server.serve_forever, daemon=True).start() self.url = f"http://127.0.0.1:{self._server.server_port}" self._running = True def mode(self, mode: str) -> None: self.state["mode"] = mode def stop(self) -> None: """Idempotent, so a test may take Immich away mid-journey.""" if not self._running: return self._running = False self._server.shutdown() self._server.server_close() def fake_uploader(tmp_path: Path, body: str) -> Path: """A real executable standing in for immich-go. ``--version`` answers like the real tool; any other invocation appends its complete argv to ``immich-go.argv`` — which is how a test proves the uploader ran, what folder it was handed, or that it never ran at all. """ path = tmp_path / "immich-go" path.write_text( "#!/bin/sh\n" f'if [ "$1" = "--version" ]; then echo "{UPLOADER_VERSION}"; exit 0; fi\n' f'printf "%s\\n" "$*" >> "{tmp_path / "immich-go.argv"}"\n' f"{body}" ) path.chmod(path.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) return path def uploader_argv(tmp_path: Path) -> list[str]: """Every upload invocation the fake uploader saw, oldest first.""" log = tmp_path / "immich-go.argv" return log.read_text().splitlines() if log.exists() else [] def mark_upload_ready(seeded: Seeded, *, unverified: tuple[str, ...] = ()) -> None: """Give every seeded photo the verified EXIF checkpoints upload requires. ``unverified`` names stems whose analysis checkpoint stays incomplete, which is what makes an album partially blocked. """ from sqlalchemy import select from photo_pipeline.models import AnalysisResult, SafetyReview blocked = {seeded.asset_ids[stem] for stem in unverified} with session_factory(seeded) as sf: with sf() as session: for review in session.scalars(select(SafetyReview)): review.exif_verified_at = NOW for analysis in session.scalars(select(AnalysisResult)): analysis.exif_written_at = None if analysis.asset_id in blocked else NOW session.commit() # ── Phase F: an archivable album and a mountable fake medium ───────────────── def mark_uploaded(seeded: Seeded, *, album: str = "rome") -> None: """Give every seeded photo the verified upload evidence archiving requires. Archiving refuses anything Immich is not proven to hold, and that proof is an upload batch — recorded here as fixture state so the archive journeys do not have to re-run an upload they are not testing. """ from sqlalchemy import select from photo_pipeline.models import Asset, UploadBatch, UploadItem from photo_pipeline.services.hashing import sha256_file with session_factory(seeded) as sf: with sf() as session: batch_id = str(uuid.uuid4()) session.add( UploadBatch( id=batch_id, album=album, folder=str(seeded.lib / album), album_name=album, state="succeeded", preflight_token="v1:e2e", outcome_state="verified", created_at=NOW, ) ) for asset in session.scalars(select(Asset)): if not asset.current_path: continue session.add( UploadItem( batch_id=batch_id, asset_id=asset.id, path=asset.current_path, sha256=sha256_file(asset.current_path), sha1="0" * 40, state="sent", outcome="uploaded", ) ) session.commit() class ArchiveStack: """A seeded, archivable library plus the server, the worker, and a fake medium. The medium is an ordinary directory whose marker file makes it identifiable; ``unmount()`` takes that marker away, which is exactly what the application sees when an external disk is unplugged. """ MARKER = ".photo-pipeline-archive.json" def __init__(self, tmp_path: Path, seeded: Seeded) -> None: self.tmp_path = tmp_path self.seeded = seeded self.archive = tmp_path / "archive" self.archive.mkdir(exist_ok=True) self.server: Server | None = None self.worker: subprocess.Popen | None = None self.base = "" def start(self, *, worker: bool = True, extra_env: dict[str, str] | None = None) -> "ArchiveStack": env = {"PHOTO_PIPELINE_ARCHIVE_FREE_SPACE_RESERVE_BYTES": "0", **(extra_env or {})} self.server = Server(self.seeded, extra_env=env).start() self.base = self.server.base if worker: self.worker = start_worker(self.seeded, extra_env=env) return self def register(self, name: str = "external") -> dict: response = httpx.post( f"{self.base}/api/v1/archive-locations", json={"name": name, "root": str(self.archive)}, timeout=20, ) response.raise_for_status() return response.json() def unmount(self) -> None: (self.archive / self.MARKER).rename(self.archive / f"{self.MARKER}.away") def remount(self) -> None: (self.archive / f"{self.MARKER}.away").rename(self.archive / self.MARKER) def plans(self) -> list[dict]: return httpx.get(f"{self.base}/api/v1/archive-plans", timeout=20).json()["plans"] def assets(self) -> list[dict]: return httpx.get( f"{self.base}/api/v1/inventory/assets", params={"limit": 200}, timeout=20 ).json()["items"] def restart_server(self) -> None: self.server.stop() self.server.start() def restart_worker(self, *, extra_env: dict[str, str] | None = None) -> None: """Replace the worker — a healthy one after a crashed one, by default.""" if self.worker is not None and self.worker.poll() is None: self.worker.kill() self.worker.wait(timeout=10) self.worker = start_worker( self.seeded, extra_env={"PHOTO_PIPELINE_ARCHIVE_FREE_SPACE_RESERVE_BYTES": "0", **(extra_env or {})}, ) def stop(self) -> None: if self.worker is not None: self.worker.kill() self.worker.wait(timeout=10) if self.server is not None: self.server.stop() class UploadStack: """A seeded, upload-ready library plus the server, worker, and fake Immich.""" def __init__(self, tmp_path: Path, seeded: Seeded) -> None: self.tmp_path = tmp_path self.seeded = seeded self.immich = FakeImmich() self.server: Server | None = None self.worker: subprocess.Popen | None = None self.base = "" def start( self, *, uploader: str = REPORTING_UPLOADER, worker: bool = True, credentials: bool = True, ) -> "UploadStack": env = { "PHOTO_PIPELINE_IMMICH_SERVER_URL": self.immich.url if credentials else "", "PHOTO_PIPELINE_IMMICH_GO_BINARY": str(fake_uploader(self.tmp_path, uploader)), } if credentials: env["PHOTO_PIPELINE_IMMICH_API_KEY"] = SENTINEL_KEY self.server = Server(self.seeded, extra_env=env).start() self.base = self.server.base if worker: self.worker = start_worker(self.seeded, extra_env=env) return self def restart_server(self) -> None: """A genuinely fresh process against the same database and library.""" self.server.stop() self.server.start() def batches(self) -> list[dict]: return httpx.get(f"{self.base}/api/v1/upload-batches", timeout=20).json()["batches"] def argv(self) -> list[str]: return uploader_argv(self.tmp_path) def stop(self) -> None: if self.worker is not None: self.worker.kill() self.worker.wait(timeout=10) if self.server is not None: self.server.stop() self.immich.stop()