diff --git a/README.md b/README.md index 819d45a..6d97d30 100644 --- a/README.md +++ b/README.md @@ -281,6 +281,51 @@ file in the temporary library are copied to `.artifacts//` before pytes deletes the directory. Point `PHOTO_PIPELINE_TEST_ARTIFACTS` elsewhere to collect them from CI. +## Release gate (US07-07) + +One command runs every suite in an isolated stack and keeps the evidence: + +```bash +work_item/scripts/python -m photo_pipeline release-gate --output data/release/$(date -u +%Y%m%dT%H%M%SZ) +``` + +It fails — and exits non-zero — when any stage fails, when a suite skips a test for +a reason that is not a documented environment limit (`exiftool not installed`, +`root ignores directory permissions`), or when the story matrix has a hole. The +evidence directory holds `release-report.json` (revision, per-stage result, timings, +summaries), `logs/.log`, and `CHECKSUMS.sha256` over both. + +**The story matrix** lives in `tests/story_traceability.json`: every story under +`delivery_backlog/stories/` is either mapped to test files that exist, or listed in +`planned` as an accepted but unimplemented story. A story that is neither, or a +mapping to a file that has been deleted, fails the gate. + +**The journey** (`tests/e2e/test_release_journey.py`) takes one fresh library through +discovery, duplicate review, safety, analysis, EXIF verification, album proposal, +guarded rename, rescan, upload with server-side verification, archive, offline +deduplication, and restore — over HTTP against real server and worker processes, +with a full restart in the middle and at the end. + +### Real-library dry run and approval + +Before the application is pointed at photos that cannot be replaced: + +```bash +work_item/scripts/python -m photo_pipeline dry-run --output dry-run.json +work_item/scripts/python -m photo_pipeline approve-dry-run dry-run.json --approver "$(whoami)" +``` + +The dry run is strictly read-only: it opens no file for writing, writes no database +row, and reports what it found — file counts by extension, folders, bytes, unreadable +files, excluded directories, and a reconciliation against what the database already +knows (already registered, new, recorded but absent). Set +`PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL=1` and **every mutating API request is +refused with `403 dry_run_not_approved`** until a report for exactly those library +roots has been approved. Reading stays open — you have to be able to see what was +found in order to approve it — and so does taking a backup. Change the library roots +and the approval no longer applies: it approves that reconciliation, not the idea of +mutating. + ## Performance budgets (US07-06) Budgets are measured, not asserted in prose. `python -m photo_pipeline benchmark` diff --git a/photo_pipeline/__main__.py b/photo_pipeline/__main__.py index b2dd22e..f10dc28 100644 --- a/photo_pipeline/__main__.py +++ b/photo_pipeline/__main__.py @@ -11,6 +11,7 @@ from __future__ import annotations import argparse import json +from pathlib import Path from typing import Sequence from photo_pipeline.config import Config @@ -63,6 +64,20 @@ def main(argv: Sequence[str] | None = None) -> int: ) bench_cmd.add_argument("--output", help="Write the JSON report here as well as to stdout") + gate_cmd = commands.add_parser( + "release-gate", help="Run every suite in an isolated stack and keep the evidence" + ) + gate_cmd.add_argument("--output", help="Evidence directory (default: data/release/)") + dry_cmd = commands.add_parser( + "dry-run", help="Read-only reconciliation of the configured library (US07-07)" + ) + dry_cmd.add_argument("--output", help="Write the report here as well as to stdout") + approve_cmd = commands.add_parser( + "approve-dry-run", help="Approve a dry-run report, which is what enables mutation" + ) + approve_cmd.add_argument("report", help="Path to the dry-run report") + approve_cmd.add_argument("--approver", required=True, help="Who is accepting this") + args = parser.parse_args(argv) config = Config.from_env() @@ -115,6 +130,41 @@ def main(argv: Sequence[str] | None = None) -> int: # anyone reading the JSON. return 0 if report["ok"] else 1 + if args.command == "release-gate": + from photo_pipeline.services import release + + report = release.run_gate(config, output=args.output) + print( + json.dumps( + {k: v for k, v in report.items() if k not in ("stages", "matrix")}, indent=2 + ) + ) + return 0 if report["ok"] else 1 + + if args.command == "dry-run": + from photo_pipeline.services import release + + try: + report = release.dry_run(config) + except release.ReleaseError as error: + print(str(error)) + return 1 + if args.output: + Path(args.output).write_text(json.dumps(report, indent=2)) + print(json.dumps(report, indent=2)) + return 0 + + if args.command == "approve-dry-run": + from photo_pipeline.services import release + + try: + record = release.approve(config, args.report, approver=args.approver) + except (release.ReleaseError, OSError, ValueError) as error: + print(str(error)) + return 1 + print(json.dumps(record, indent=2)) + return 0 + if args.command == "diagnostics": from photo_pipeline.services import diagnostics diff --git a/photo_pipeline/api/security.py b/photo_pipeline/api/security.py index 908273b..994adff 100644 --- a/photo_pipeline/api/security.py +++ b/photo_pipeline/api/security.py @@ -43,6 +43,9 @@ PUBLIC_PATHS = frozenset( {f"{API_PREFIX}/health/live", f"{API_PREFIX}/health/ready", f"{API_PREFIX}/session"} ) LOOPBACK_HOSTS = frozenset({"127.0.0.1", "localhost", "::1", "[::1]"}) +# Mutating endpoints that must stay reachable while mutation itself is gated: the +# backup a careful operator takes first, and its retention (US07-07). +MUTATION_EXEMPT_PATHS = frozenset({f"{API_PREFIX}/backups", f"{API_PREFIX}/backups/prune"}) # Applied to every response. No inline script/style is used by the frontend, so the # policy can stay strict; `frame-ancestors 'none'` and CORP keep other pages from @@ -145,6 +148,7 @@ class SecurityMiddleware: def __init__(self, app, *, session: Session, config) -> None: self.app = app self.session = session + self.config = config self.max_request_bytes = config.max_request_bytes self.allowed_hosts = frozenset(LOOPBACK_HOSTS | {str(config.host).lower()}) @@ -165,6 +169,8 @@ class SecurityMiddleware: allowed_hosts=self.allowed_hosts, max_request_bytes=self.max_request_bytes, ) + if refusal is None: + refusal = self._mutation_refusal(scope) if refusal is not None: response = JSONResponse( status_code=refusal.status, @@ -183,6 +189,26 @@ class SecurityMiddleware: await self.app(scope, receive, send_with_headers) + def _mutation_refusal(self, scope) -> Refusal | None: + """Refuse every mutating request while the library's dry run is unapproved. + + One choke point for the whole API: every mutation the browser can start is a + non-safe method under ``/api/v1``. Reading stays open — an operator has to be + able to look at what the application found in order to approve it (US07-07). + """ + method = scope.get("method", "GET").upper() + path = scope.get("path", "/") + if method in SAFE_METHODS or not path.startswith(API_PREFIX): + return None + if path in MUTATION_EXEMPT_PATHS: + return None + from photo_pipeline.services.release import mutation_blockers + + blockers = mutation_blockers(self.config) + if not blockers: + return None + return Refusal(403, blockers[0]["code"], blockers[0]["message"]) + def _cookie(header: str, name: str) -> str: for part in header.split(";"): diff --git a/photo_pipeline/config.py b/photo_pipeline/config.py index 309b5b4..a483c49 100644 --- a/photo_pipeline/config.py +++ b/photo_pipeline/config.py @@ -42,6 +42,12 @@ class Config(BaseModel): # Free space an archive destination must keep beyond the transfer itself. archive_free_space_reserve_bytes: int = 1_000_000_000 + # Refuse every mutating request until a read-only dry run of the configured + # library has been produced and explicitly approved (US07-07). Off by default so + # a development setup is unchanged; turn it on before pointing the application at + # a library whose photos cannot be replaced. + require_dry_run_approval: bool = False + vision_api_key: SecretStr | None = None immich_api_key: SecretStr | None = None immich_server_url: str = "" diff --git a/photo_pipeline/services/analysis.py b/photo_pipeline/services/analysis.py index 4a493f6..5b5327e 100644 --- a/photo_pipeline/services/analysis.py +++ b/photo_pipeline/services/analysis.py @@ -264,8 +264,11 @@ class AnalysisService: asset = session.get(Asset, asset_id) if asset is not None and checkpoint.sha256: # The bytes changed when the container was rewritten; upload must use - # the hash of what is actually on disk now (concept §3). + # the hash of what is actually on disk now (concept §3), and the + # recorded size has to move with it (US07-07). asset.current_sha256 = checkpoint.sha256 + if checkpoint.byte_size is not None: + asset.byte_size = checkpoint.byte_size session.commit() def get(self, asset_id: str) -> dict | None: diff --git a/photo_pipeline/services/exif_checkpoint.py b/photo_pipeline/services/exif_checkpoint.py index 393e94f..d67ccae 100644 --- a/photo_pipeline/services/exif_checkpoint.py +++ b/photo_pipeline/services/exif_checkpoint.py @@ -21,6 +21,7 @@ unreadable file, or a write that did not take is not evidence that metadata is f from __future__ import annotations import json +import os import uuid from dataclasses import dataclass from datetime import datetime, timezone @@ -58,6 +59,11 @@ class CheckpointResult: state: str # verified | divergent | failed changed_fields: tuple[str, ...] = () sha256: str | None = None + # exiftool rewrites the container, so the file's size moves with its hash. Both + # are inventory facts about the current bytes and both have to be refreshed + # together, or the next stage compares against a size that no longer exists + # (US07-07: a rename plan blocked itself forever after any EXIF write). + byte_size: int | None = None verified_at: datetime | None = None reason: str | None = None @@ -137,9 +143,12 @@ def run( changed = compare(before, after) sha256 = hashing.sha256_file(path) + byte_size = os.path.getsize(path) if changed: - return CheckpointResult(DIVERGENT, changed_fields=changed, sha256=sha256) - return CheckpointResult(VERIFIED, sha256=sha256, verified_at=_now()) + return CheckpointResult( + DIVERGENT, changed_fields=changed, sha256=sha256, byte_size=byte_size + ) + return CheckpointResult(VERIFIED, sha256=sha256, byte_size=byte_size, verified_at=_now()) def record( diff --git a/photo_pipeline/services/release.py b/photo_pipeline/services/release.py new file mode 100644 index 0000000..0c030e6 --- /dev/null +++ b/photo_pipeline/services/release.py @@ -0,0 +1,391 @@ +"""The release gate, the real-library dry run, and the approval that unlocks +mutation (US07-07, concept §18 release gates). + +Three things live here because they are one decision: + +1. **The gate** — one command that provisions an isolated stack, runs every suite in + a fixed order, and retains versioned evidence with checksums. A release is not + "the tests passed on my machine last Tuesday"; it is a report that says which + revision, which suites, how long, and what the artefacts hash to. +2. **The dry run** — a strictly read-only pass over the real photo library that + answers "what would this application do to it?" before it is allowed to do + anything. It opens no file for writing, creates no database rows, and touches no + metadata; it counts, classifies, and reconciles against whatever the database + already knows. +3. **The approval** — a person reads that report and signs it off for exactly the + library roots it describes. Until then, with + ``PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL`` set, every mutating request is + refused. Change the roots, or produce a newer report, and the approval no longer + matches: it approves *that* reconciliation, not the idea of mutating. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +import sys +import time +from collections import Counter +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path + +from photo_pipeline import path_policy +from photo_pipeline.config import Config + +SCHEMA_VERSION = 1 +APPROVAL_NAME = "dry-run-approval.json" +CHECKSUMS_NAME = "CHECKSUMS.sha256" +REPORT_NAME = "release-report.json" + +# The suites, in the order a failure is cheapest to read: units before the stacks +# they compose. ``label`` is what the report and the operator see. +STAGES: tuple[tuple[str, tuple[str, ...]], ...] = ( + ("unit", ("tests/unit",)), + ("characterization", ("tests/characterization",)), + ("integration", ("tests/integration",)), + ("browser", ("tests/e2e",)), +) + +# Skips the gate accepts, because they describe the machine rather than the code. +ALLOWED_SKIP_REASONS = ("exiftool not installed", "root ignores directory permissions") + + +class ReleaseError(RuntimeError): + pass + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def revision() -> str | None: + """The commit this gate ran against, when the tree is a git checkout.""" + try: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + capture_output=True, + text=True, + timeout=10, + cwd=str(Path(__file__).resolve().parents[2]), + ) + except (OSError, subprocess.SubprocessError): + return None + return result.stdout.strip() or None + + +# ── the story matrix ───────────────────────────────────────────────────────── + + +def story_matrix(repo: Path | None = None) -> dict: + """Every backlog story, and how it is covered. + + A story is ``delivered`` (mapped to test files that exist) or ``planned`` (an + accepted, not-yet-implemented story). Anything else — a story file nobody + mapped, or a mapping to a file that is gone — is a hole in the matrix, and the + gate fails on it rather than reporting a green run over missing coverage. + """ + repo = repo or Path(__file__).resolve().parents[2] + traceability = json.loads((repo / "tests" / "story_traceability.json").read_text()) + mapped: dict[str, list[str]] = traceability["stories"] + planned: list[str] = traceability.get("planned", []) + stories = sorted( + "-".join(path.stem.split("-")[:2]) + for path in (repo / "delivery_backlog" / "stories").glob("US*.md") + ) + + missing_tests = [ + f"{story}: {rel}" + for story, files in mapped.items() + for rel in files + if not (repo / rel).is_file() + ] + unmapped = [s for s in stories if s not in mapped and s not in planned] + unknown = [s for s in list(mapped) + planned if s not in stories] + overlap = sorted(set(mapped) & set(planned)) + return { + "stories": len(stories), + "delivered": sorted(mapped), + "planned": sorted(planned), + "problems": [ + *(f"story with no tests and not planned: {s}" for s in unmapped), + *(f"mapped test file is missing — {entry}" for entry in missing_tests), + *(f"mapped story is not in the backlog: {s}" for s in unknown), + *(f"story is both delivered and planned: {s}" for s in overlap), + ], + } + + +# ── the gate ───────────────────────────────────────────────────────────────── + + +@dataclass +class StageResult: + label: str + command: list[str] + returncode: int + seconds: float + summary: str + skipped: list[str] + + def as_dict(self) -> dict: + return { + "stage": self.label, + "command": self.command, + "returncode": self.returncode, + "seconds": round(self.seconds, 2), + "summary": self.summary, + "skipped": self.skipped, + "ok": self.returncode == 0, + } + + +def _run_stage(label: str, paths: tuple[str, ...], *, repo: Path, log_dir: Path) -> StageResult: + command = [sys.executable, "-m", "pytest", *paths, "-q", "-rs"] + started = time.monotonic() + result = subprocess.run(command, cwd=str(repo), capture_output=True, text=True) + elapsed = time.monotonic() - started + output = result.stdout + result.stderr + (log_dir / f"{label}.log").write_text(output) + lines = [line for line in output.splitlines() if line.strip()] + summary = lines[-1] if lines else "" + skipped = [line for line in lines if line.startswith("SKIPPED")] + return StageResult(label, command, result.returncode, elapsed, summary, skipped) + + +def unexpected_skips(results: list[StageResult]) -> list[str]: + """Skips the gate will not accept: everything but the documented environment ones.""" + return [ + line + for result in results + for line in result.skipped + if not any(reason in line for reason in ALLOWED_SKIP_REASONS) + ] + + +def run_gate( + config: Config, + *, + output: Path | str | None = None, + stages: tuple[tuple[str, tuple[str, ...]], ...] = STAGES, + repo: Path | None = None, +) -> dict: + """Run every suite in an isolated stack and retain checksummed evidence. + + The stack is isolated by construction: each pytest run builds its own temporary + data directories and libraries, so the gate never reads or writes the operator's + photos. What it keeps afterwards is the report, the per-stage logs, and a + checksum file over both. + """ + repo = repo or Path(__file__).resolve().parents[2] + directory = Path(output) if output else Path(config.data_dir) / "release" / _now().strftime( + "%Y%m%dT%H%M%SZ" + ) + logs = directory / "logs" + logs.mkdir(parents=True, exist_ok=True) + + matrix = story_matrix(repo) + results = [_run_stage(label, paths, repo=repo, log_dir=logs) for label, paths in stages] + skips = unexpected_skips(results) + + report = { + "schema_version": SCHEMA_VERSION, + "started_at": _now().isoformat(), + "revision": revision(), + "python": sys.version.split()[0], + "platform": os.uname().sysname, + "matrix": matrix, + "stages": [result.as_dict() for result in results], + "unexpected_skips": skips, + "failures": [result.label for result in results if result.returncode != 0], + } + report["ok"] = not report["failures"] and not matrix["problems"] and not skips + report["finished_at"] = _now().isoformat() + + (directory / REPORT_NAME).write_text(json.dumps(report, indent=2)) + # The evidence is only evidence if it can be shown to be the evidence that was + # produced. Checksums are the honest version of "signed" without a key: a real + # signature belongs to whatever key management the release actually has. + checksums = "\n".join( + f"{sha256_file(path)} {path.relative_to(directory)}" + for path in sorted(directory.rglob("*")) + if path.is_file() and path.name != CHECKSUMS_NAME + ) + (directory / CHECKSUMS_NAME).write_text(checksums + "\n") + report["evidence"] = str(directory) + return report + + +# ── the real-library dry run ───────────────────────────────────────────────── + + +def dry_run(config: Config, *, roots: tuple[Path, ...] | None = None) -> dict: + """Read-only reconciliation of the configured library. Changes nothing. + + Opens no file for writing, writes no database row, and reads only what + ``os.stat`` and the existing database already say. The point is to be able to + look at a real library — the one with the irreplaceable photos in it — and see + what the application believes about it before it is allowed to act. + """ + roots = roots or tuple(Path(root) for root in config.library_roots) + if not roots: + raise ReleaseError("no library roots are configured") + + by_extension: Counter = Counter() + folders: set[str] = set() + files: list[str] = [] + unreadable: list[str] = [] + excluded = 0 + total_bytes = 0 + for root in roots: + if not Path(root).is_dir(): + raise ReleaseError(f"library root {root} is not a directory") + for path in sorted(Path(root).rglob("*")): + if path.is_dir(): + # Never traverse into an excluded directory, and never report its + # contents: proving exclusion must not require opening it. + if path_policy.is_excluded(path): + excluded += 1 + continue + if path_policy.is_excluded(path): + continue + try: + stat = path.stat() + except OSError: + unreadable.append(str(path)) + continue + files.append(str(path)) + folders.add(str(path.parent)) + by_extension[path.suffix.lower() or "(none)"] += 1 + total_bytes += stat.st_size + + known = _known_paths(config) + on_disk = set(files) + report = { + "schema_version": SCHEMA_VERSION, + "generated_at": _now().isoformat(), + "revision": revision(), + "library_roots": [str(root) for root in roots], + "files": len(files), + "folders": len(folders), + "bytes": total_bytes, + "excluded_directories": excluded, + "unreadable": unreadable, + "by_extension": dict(sorted(by_extension.items())), + "reconciliation": { + "known_to_database": len(known), + "already_registered": len(on_disk & known), + "new_to_the_application": len(on_disk - known), + "recorded_but_absent": sorted(known - on_disk)[:100], + "recorded_but_absent_total": len(known - on_disk), + }, + "mutation": "none — this pass is read-only", + } + report["checksum"] = sha256_bytes( + json.dumps(report, sort_keys=True).encode("utf-8") + ) + return report + + +def _known_paths(config: Config) -> set[str]: + """Current asset paths the database holds, or an empty set if there is none.""" + if not config.database_path.exists(): + return set() + from sqlalchemy import select + + from photo_pipeline.db import create_db_engine, create_session_factory + from photo_pipeline.models import Asset + + engine = create_db_engine(config.database_url) + try: + with create_session_factory(engine)() as session: + return { + path + for path in session.scalars(select(Asset.current_path)) + if path is not None + } + except Exception: + return set() + finally: + engine.dispose() + + +# ── the approval ───────────────────────────────────────────────────────────── + + +def approval_path(config: Config) -> Path: + return Path(config.data_dir) / APPROVAL_NAME + + +def approve(config: Config, report: dict | Path | str, *, approver: str) -> dict: + """Record that a person read this reconciliation and accepts mutation for it.""" + if isinstance(report, (str, Path)): + report = json.loads(Path(report).read_text()) + if "checksum" not in report: + raise ReleaseError("this is not a dry-run report: it has no checksum") + record = { + "schema_version": SCHEMA_VERSION, + "approved_at": _now().isoformat(), + "approved_by": approver, + "report_checksum": report["checksum"], + "library_roots": report["library_roots"], + "files": report["files"], + "revision": report.get("revision"), + } + path = approval_path(config) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(record, indent=2)) + return record + + +def mutation_blockers(config: Config) -> list[dict]: + """Why mutation must stay refused, or an empty list. + + Only enforced when ``require_dry_run_approval`` is configured — the loopback + developer setup keeps working unchanged, and an operator turns this on before + pointing the application at the library they cannot replace. + """ + if not config.require_dry_run_approval: + return [] + path = approval_path(config) + if not path.exists(): + return [ + { + "code": "dry_run_not_approved", + "message": ( + "run `python -m photo_pipeline dry-run` and approve its report " + "before mutation is enabled" + ), + } + ] + try: + record = json.loads(path.read_text()) + except ValueError: + return [{"code": "approval_unreadable", "message": f"{path} is not readable JSON"}] + approved_roots = [str(root) for root in record.get("library_roots", [])] + configured = [str(root) for root in config.library_roots] + if sorted(approved_roots) != sorted(configured): + return [ + { + "code": "approval_scope_mismatch", + "message": ( + f"the approval covers {approved_roots}, but the configured library " + f"is {configured}; run a new dry run" + ), + } + ] + return [] diff --git a/photo_pipeline/services/safety.py b/photo_pipeline/services/safety.py index 6336636..d6cb04d 100644 --- a/photo_pipeline/services/safety.py +++ b/photo_pipeline/services/safety.py @@ -294,6 +294,7 @@ class SafetyService: exif_verified_at = None result_sha256 = None + result_byte_size = None if write_exif and decision in (SFW, NSFW) and path: ops = exif_projection(decision) # The full checkpoint: write the owned keyword, read the whole file back, @@ -314,6 +315,7 @@ class SafetyService: if result.verified: exif_verified_at = result.verified_at result_sha256 = result.sha256 + result_byte_size = result.byte_size now = _now() with self._session_factory() as session: @@ -332,6 +334,8 @@ class SafetyService: if result_sha256: asset = session.get(Asset, asset_id) asset.current_sha256 = result_sha256 + if result_byte_size is not None: + asset.byte_size = result_byte_size session.commit() return { "asset_id": asset_id, diff --git a/tests/e2e/test_release_gate.py b/tests/e2e/test_release_gate.py new file mode 100644 index 0000000..979dbd2 --- /dev/null +++ b/tests/e2e/test_release_gate.py @@ -0,0 +1,298 @@ +"""The release gate, the read-only dry run, and the approval that unlocks mutation +(US07-07). + +The gate itself is exercised with a tiny stage set — running the whole suite from +inside the suite would be a fork bomb with better manners. What is proven here is +the machinery a release depends on: the story matrix is complete, a failing stage +fails the gate, an unexpected skip fails the gate, and the evidence is written with +checksums that match what was written. + +The dry run is proven to be read-only against a real temporary library, and the +approval is proven to be what stands between a configured library and any mutation. +""" + +from __future__ import annotations + +import json +import os +import uuid +from datetime import datetime, timezone +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from photo_pipeline.api.app import create_app +from photo_pipeline.config import Config +from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations +from photo_pipeline.models import Asset +from photo_pipeline.services import release + +REPO = Path(__file__).resolve().parents[2] +NOW = datetime(2026, 1, 1, tzinfo=timezone.utc) + +# Two throwaway stages: one that passes, one the test can point at a failure. +PASSING = ("tests/e2e/test_traceability.py",) + + +def _config(tmp_path, **extra) -> Config: + data = tmp_path / "data" + data.mkdir(parents=True, exist_ok=True) + lib = tmp_path / "lib" + lib.mkdir(exist_ok=True) + return Config.from_env( + { + "PHOTO_PIPELINE_DATA_DIR": str(data), + "PHOTO_PIPELINE_LIBRARY_ROOTS": str(lib), + **extra, + } + ) + + +# ── the story matrix ───────────────────────────────────────────────────────── + + +def test_every_backlog_story_is_delivered_or_explicitly_planned(): + matrix = release.story_matrix(REPO) + + assert matrix["problems"] == [], "the story matrix has holes" + assert len(matrix["delivered"]) + len(matrix["planned"]) == matrix["stories"] + assert "US01-01" in matrix["delivered"] and "US07-07" in matrix["delivered"] + + +def test_a_story_without_tests_is_a_gate_failure(tmp_path): + """A story file nobody covered must not pass quietly as 'no tests ran'.""" + fake = tmp_path / "repo" + (fake / "delivery_backlog" / "stories").mkdir(parents=True) + (fake / "tests").mkdir() + (fake / "delivery_backlog" / "stories" / "US99-01-invented.md").write_text("# US99-01") + (fake / "tests" / "story_traceability.json").write_text(json.dumps({"stories": {}})) + + matrix = release.story_matrix(fake) + + assert matrix["problems"] == ["story with no tests and not planned: US99-01"] + + +def test_a_mapping_to_a_deleted_test_file_is_a_gate_failure(tmp_path): + fake = tmp_path / "repo" + (fake / "delivery_backlog" / "stories").mkdir(parents=True) + (fake / "tests").mkdir() + (fake / "delivery_backlog" / "stories" / "US99-01-invented.md").write_text("# US99-01") + (fake / "tests" / "story_traceability.json").write_text( + json.dumps({"stories": {"US99-01": ["tests/gone.py"]}}) + ) + + assert release.story_matrix(fake)["problems"] == [ + "mapped test file is missing — US99-01: tests/gone.py" + ] + + +# ── the gate ───────────────────────────────────────────────────────────────── + + +def test_the_gate_runs_its_stages_and_keeps_checksummed_evidence(tmp_path): + config = _config(tmp_path) + evidence = tmp_path / "evidence" + + report = release.run_gate(config, output=evidence, stages=(("smoke", PASSING),)) + + assert report["ok"] is True and report["failures"] == [] + assert report["stages"][0]["stage"] == "smoke" and report["stages"][0]["ok"] is True + assert report["revision"], "the evidence must say which commit it covers" + assert report["matrix"]["problems"] == [] + + written = json.loads((evidence / release.REPORT_NAME).read_text()) + assert written["ok"] is True + assert (evidence / "logs" / "smoke.log").exists() + checksums = (evidence / release.CHECKSUMS_NAME).read_text().splitlines() + assert len(checksums) >= 2 + for line in checksums: + digest, name = line.split(" ", 1) + assert release.sha256_file(evidence / name) == digest + + +def test_a_failing_stage_fails_the_gate(tmp_path): + config = _config(tmp_path) + failing = tmp_path / "failing_test.py" + failing.write_text("def test_no():\n assert False\n") + + report = release.run_gate( + config, output=tmp_path / "evidence", stages=(("broken", (str(failing),)),) + ) + + assert report["ok"] is False and report["failures"] == ["broken"] + assert report["stages"][0]["returncode"] != 0 + + +def test_an_unexpected_skip_fails_the_gate_but_an_environment_skip_does_not(): + environment = release.StageResult( + "unit", [], 0, 0.1, "1 skipped", ["SKIPPED [1] x.py:1: exiftool not installed"] + ) + silent = release.StageResult( + "unit", [], 0, 0.1, "1 skipped", ["SKIPPED [1] x.py:1: flaky, look later"] + ) + + assert release.unexpected_skips([environment]) == [] + assert release.unexpected_skips([silent, environment]) == [ + "SKIPPED [1] x.py:1: flaky, look later" + ] + + +# ── the real-library dry run ───────────────────────────────────────────────── + + +def _library(root: Path) -> None: + (root / "album").mkdir(parents=True) + (root / "album" / "a.jpg").write_bytes(b"a" * 128) + (root / "album" / "b.png").write_bytes(b"b" * 64) + (root / "loose.JPG").write_bytes(b"c" * 32) + excluded = root / "_IGNORE" / "private" + excluded.mkdir(parents=True) + (excluded / "secret.jpg").write_bytes(b"never read") + + +def test_the_dry_run_describes_the_library_without_touching_it(tmp_path): + config = _config(tmp_path) + root = Path(config.library_roots[0]) + _library(root) + before = { + str(p): (p.stat().st_mtime_ns, p.read_bytes()) for p in root.rglob("*") if p.is_file() + } + + report = release.dry_run(config) + + assert report["files"] == 3, "the excluded sentinel is not counted" + assert report["by_extension"] == {".jpg": 2, ".png": 1} + assert report["excluded_directories"] >= 1 + assert report["mutation"] == "none — this pass is read-only" + assert report["checksum"] + assert not any("secret" in json.dumps(report) for _ in [0]), "excluded content never appears" + after = { + str(p): (p.stat().st_mtime_ns, p.read_bytes()) for p in root.rglob("*") if p.is_file() + } + assert after == before, "a read-only pass changed the library" + + +def test_the_dry_run_reconciles_against_what_the_database_already_knows(tmp_path): + config = _config(tmp_path) + root = Path(config.library_roots[0]) + _library(root) + run_migrations(config.database_url) + engine = create_db_engine(config.database_url) + with create_session_factory(engine)() as session: + session.add( + Asset( + id=str(uuid.uuid4()), + original_path=str(root / "album" / "a.jpg"), + current_path=str(root / "album" / "a.jpg"), + discovered_at=NOW, + hash_version=1, + byte_size=128, + ) + ) + session.add( + Asset( + id=str(uuid.uuid4()), + original_path=str(root / "album" / "gone.jpg"), + current_path=str(root / "album" / "gone.jpg"), + discovered_at=NOW, + hash_version=1, + byte_size=1, + ) + ) + session.commit() + engine.dispose() + + reconciliation = release.dry_run(config)["reconciliation"] + + assert reconciliation["known_to_database"] == 2 + assert reconciliation["already_registered"] == 1 + assert reconciliation["new_to_the_application"] == 2 + assert reconciliation["recorded_but_absent_total"] == 1 + assert reconciliation["recorded_but_absent"][0].endswith("gone.jpg") + + +def test_a_library_root_that_is_not_there_is_refused(tmp_path): + config = _config(tmp_path, PHOTO_PIPELINE_LIBRARY_ROOTS=str(tmp_path / "nowhere")) + with pytest.raises(release.ReleaseError, match="not a directory"): + release.dry_run(config) + + +# ── the approval ───────────────────────────────────────────────────────────── + + +def test_mutation_is_refused_until_the_dry_run_is_approved(tmp_path): + config = _config(tmp_path, PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL="1") + _library(Path(config.library_roots[0])) + + blockers = release.mutation_blockers(config) + assert [blocker["code"] for blocker in blockers] == ["dry_run_not_approved"] + + record = release.approve(config, release.dry_run(config), approver="domverse") + assert record["approved_by"] == "domverse" and record["report_checksum"] + assert release.mutation_blockers(config) == [] + + +def test_an_approval_covers_the_library_it_was_written_for(tmp_path): + config = _config(tmp_path, PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL="1") + _library(Path(config.library_roots[0])) + release.approve(config, release.dry_run(config), approver="domverse") + + other = tmp_path / "other-library" + other.mkdir() + moved = config.model_copy(update={"library_roots": (other,)}) + + assert [b["code"] for b in release.mutation_blockers(moved)] == ["approval_scope_mismatch"] + + +def test_without_the_requirement_nothing_changes(tmp_path): + config = _config(tmp_path) # the loopback development default + assert release.mutation_blockers(config) == [] + + +def test_the_api_refuses_every_mutation_until_the_report_is_approved(tmp_path): + config = _config(tmp_path, PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL="1") + _library(Path(config.library_roots[0])) + with TestClient(create_app(config)) as client: + # Reading stays open: an operator has to see what was found to approve it. + assert client.get("/api/v1/workflow").status_code == 200 + refused = client.post("/api/v1/inventory/scan", json={}) + assert refused.status_code == 403 + assert refused.json()["error"]["code"] == "dry_run_not_approved" + # A backup is the one mutation a careful operator takes first. + assert client.post("/api/v1/backups", json={}).status_code == 201 + + release.approve(config, release.dry_run(config), approver="domverse") + assert client.post("/api/v1/inventory/scan", json={}).status_code in (200, 201, 202) + + +def test_approving_something_that_is_not_a_report_is_refused(tmp_path): + config = _config(tmp_path) + with pytest.raises(release.ReleaseError, match="not a dry-run report"): + release.approve(config, {"files": 3}, approver="domverse") + + +def test_the_cli_runs_the_dry_run_and_the_approval(tmp_path): + config = _config(tmp_path, PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL="1") + _library(Path(config.library_roots[0])) + from photo_pipeline.__main__ import main + + environment = { + "PHOTO_PIPELINE_DATA_DIR": str(config.data_dir), + "PHOTO_PIPELINE_LIBRARY_ROOTS": str(config.library_roots[0]), + "PHOTO_PIPELINE_REQUIRE_DRY_RUN_APPROVAL": "1", + } + previous = {key: os.environ.get(key) for key in environment} + os.environ.update(environment) + try: + report_path = tmp_path / "dry-run.json" + assert main(["dry-run", "--output", str(report_path)]) == 0 + assert json.loads(report_path.read_text())["files"] == 3 + assert main(["approve-dry-run", str(report_path), "--approver", "domverse"]) == 0 + finally: + for key, value in previous.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + assert release.mutation_blockers(config) == [] diff --git a/tests/e2e/test_release_journey.py b/tests/e2e/test_release_journey.py new file mode 100644 index 0000000..f20683c --- /dev/null +++ b/tests/e2e/test_release_journey.py @@ -0,0 +1,348 @@ +"""The release journey (US07-07): one library, one fresh environment, every stage. + +This is the acceptance the whole backlog builds up to — discovery, duplicate review, +safety, analysis, EXIF verification, album proposal, guarded rename, rescan and +reconciliation, upload, archive, offline deduplication, restore — driven over HTTP +against real ``photo_pipeline serve`` and worker child processes, with full process +restarts in the middle and at the end. + +Nothing is reached into. External services are the deterministic fakes the earlier +phases already use, invoked through the real integration layer: a vision fake that +records every path it was given, a real fake ``immich-go`` executable, and an +archive medium that is an ordinary directory whose marker file is its identity. + +The invariants asserted along the way are the ones the concept calls non-negotiable: + +- an asset's identity survives a rename, an upload, an archive, and a restore; +- an ``_IGNORE`` sentinel is never discovered, counted, analysed, or uploaded; +- an NSFW asset never reaches the vision provider but still reaches Immich; +- no photo's bytes are lost at any point — every hash is still reachable somewhere; +- every stage's durable state survives a restart of both processes. +""" + +from __future__ import annotations + +import hashlib +import shutil +from pathlib import Path + +import httpx +import pytest + +from tests.e2e._pipeline_harness import ( + SENTINEL_KEY, + FakeImmich, + Server, + fake_uploader, + image, + seed_library, + start_worker, + wait_until, +) + +TIMEOUT = 30 +ALBUM = "rome" +UPLOADER = 'echo "INFO uploaded $6"\necho "Uploaded 2, duplicates 0"\nexit 0\n' + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _hashes(*roots: Path) -> set[str]: + return { + _sha256(path) + for root in roots + for path in root.rglob("*.jpg") + if path.is_file() and not path.name.startswith(".") + } + + +def _post(base: str, path: str, **kwargs) -> httpx.Response: + response = httpx.post(f"{base}/api/v1{path}", timeout=TIMEOUT, **kwargs) + response.raise_for_status() + return response + + +def _get(base: str, path: str, **kwargs) -> dict: + response = httpx.get(f"{base}/api/v1{path}", timeout=TIMEOUT, **kwargs) + response.raise_for_status() + return response.json() + + +def _await_job(base: str, job_id: str, *, states=("succeeded",)) -> dict: + return wait_until( + lambda: ( + snapshot + if (snapshot := _get(base, f"/jobs/{job_id}"))["state"] in states + else None + ), + timeout=90, + ) + + +@pytest.fixture +def library(tmp_path): + """A fresh library: an album, an exact duplicate, and an excluded sentinel.""" + seeded = seed_library(tmp_path, {}, {}) + album = seeded.lib / ALBUM + image(album / "a.jpg", 11) + image(album / "b.jpg", 12) + shutil.copyfile(album / "a.jpg", album / "a-copy.jpg") # exact duplicate + ignored = seeded.lib / "_IGNORE" / "private" + ignored.mkdir(parents=True) + image(ignored / "sentinel-9f3a2b.jpg", 99) + return seeded + + +@pytest.mark.skipif(shutil.which("exiftool") is None, reason="exiftool not installed") +def test_the_full_release_journey_survives_every_stage_and_two_restarts(library, tmp_path): + immich = FakeImmich() + uploader = fake_uploader(tmp_path, UPLOADER) + vision_log = tmp_path / "vision.log" + archive_root = tmp_path / "medium" + archive_root.mkdir() + environment = { + "PHOTO_PIPELINE_IMMICH_SERVER_URL": immich.url, + "PHOTO_PIPELINE_IMMICH_API_KEY": SENTINEL_KEY, + "PHOTO_PIPELINE_IMMICH_GO_BINARY": str(uploader), + "PHOTO_PIPELINE_ARCHIVE_FREE_SPACE_RESERVE_BYTES": "0", + "PHOTO_PIPELINE_FAKE_VISION_LOG": str(vision_log), + } + server = Server(library, extra_env=environment).start() + worker = start_worker(library, extra_env=environment) + base = server.base + + try: + # ── 0. discovery ───────────────────────────────────────────────────── + _post(base, "/inventory/scan") + assets = _get(base, "/inventory/assets", params={"limit": 200})["items"] + assert len(assets) == 3, "the sentinel under _IGNORE is not an asset" + paths = {asset["current_path"] for asset in assets} + assert not any("_IGNORE" in path or "sentinel" in path for path in paths) + identity = {asset["id"]: Path(asset["current_path"]).name for asset in assets} + + # ── 1. duplicate review ────────────────────────────────────────────── + _post(base, "/duplicates/detect") + clusters = _get(base, "/duplicates/clusters")["items"] + assert len(clusters) == 1 and clusters[0]["member_total"] == 2 + cluster = _get(base, f"/duplicates/clusters/{clusters[0]['id']}") + canonical = sorted(member["asset_id"] for member in cluster["members"])[0] + _post( + base, + f"/duplicates/clusters/{cluster['id']}/decision", + json={ + "decision": "canonical", + "canonical_asset_id": canonical, + "expected_version": cluster["version"], + }, + ) + + # ── 2. safety, with its EXIF checkpoint ────────────────────────────── + queue = _get(base, "/safety/queue", params={"limit": 100})["items"] + assert len(queue) == 2, "a non-canonical variant is not reviewed twice" + decisions = {} + for index, item in enumerate(sorted(queue, key=lambda row: row["current_path"])): + decision = "nsfw" if index == 0 else "sfw" + decisions[item["asset_id"]] = decision + result = _post( + base, "/safety/decisions", json={"asset_id": item["asset_id"], "decision": decision} + ).json() + assert result["exif_verified"] is True, "the safety checkpoint must verify" + + # ── restart: everything so far has to be durable ───────────────────── + server.stop() + server.start() + base = server.base + assert _get(base, "/safety/counts")["nsfw"] == 1 + assert {a["id"] for a in _get(base, "/inventory/assets", params={"limit": 200})["items"]} == set( + identity + ) + + # ── 3. analysis, gated to confirmed-SFW assets ─────────────────────── + job = _post(base, "/analysis/jobs").json() + _await_job(base, job["id"]) + analysed = [ + name + for name, decision in ( + (identity[asset_id], decision) for asset_id, decision in decisions.items() + ) + if decision == "sfw" + ] + seen = vision_log.read_text().splitlines() + assert len(seen) == len(analysed) == 1 + assert not any("sentinel" in line or "_IGNORE" in line for line in seen) + nsfw_id = next(aid for aid, decision in decisions.items() if decision == "nsfw") + assert all(identity[nsfw_id] not in line for line in seen), "NSFW reached the provider" + + # From here on no stage may change a photo's bytes: the metadata stages are + # done, and moving, uploading, archiving, and restoring only relocate them. + stable_hashes = _hashes(library.lib) + + # ── 4. album proposal and guarded rename ───────────────────────────── + _post(base, "/albums/proposals", json={}) + proposal = _get(base, f"/albums/proposals/{ALBUM}") + _post( + base, + f"/albums/proposals/{ALBUM}/edit", + json={"name": "2019 Rome", "expected_version": proposal["version"]}, + ) + proposal = _get(base, f"/albums/proposals/{ALBUM}") + _post( + base, + f"/albums/proposals/{ALBUM}/approve", + json={"expected_version": proposal["version"]}, + ) + plan = _post(base, "/rename-plans").json() + assert plan["blockers"] == [], [ + (issue["code"], issue["message"]) + for op in plan["operations"] + for issue in op["issues"] + ] + response = httpx.post( + f"{base}/api/v1/rename-plans/{plan['id']}/apply", + json={"expected_version": plan["version"], "expected_checksum": plan["checksum"]}, + timeout=TIMEOUT, + ) + assert response.status_code == 200, response.text + applied = response.json() + assert applied["failed"] == 0 and applied["applied"] == 1 + assert (library.lib / "2019 Rome").is_dir() and not (library.lib / ALBUM).exists() + + # ── 5. rescan and reconciliation: identity survives the move ───────── + _post(base, "/inventory/scan") + after_rename = _get(base, "/inventory/assets", params={"limit": 200})["items"] + assert {asset["id"] for asset in after_rename} == set(identity) + assert all("2019 Rome" in asset["current_path"] for asset in after_rename) + assert _hashes(library.lib) == stable_hashes, "a rename changed a photo's bytes" + + # ── 6. upload ──────────────────────────────────────────────────────── + report = _post(base, "/upload-preflight", json={"albums": ["2019 Rome"]}).json() + assert report["state"] == "ready", report["blockers"] + batch = _post( + base, + "/upload-batches", + json={"albums": ["2019 Rome"], "token": report["token"]}, + ).json()["batches"][0] + started = _post(base, f"/upload-batches/{batch['id']}/start").json() + _await_job(base, started["job"]["id"]) + uploaded = _get(base, f"/upload-batches/{batch['id']}") + assert uploaded["state"] == "succeeded" + # The uploader said nothing per file, so the outcome is uncertain until the + # server itself is asked whether it holds those exact bytes (US05-04). + assert uploaded["outcome_state"] == "requires_verification" + verified = _post(base, f"/upload-batches/{batch['id']}/verify").json() + assert verified["outcome_state"] == "verified", verified + uploaded = _get(base, f"/upload-batches/{batch['id']}") + # Reviewed NSFW is uploaded; it simply never reached the analyser. + assert {item["asset_id"] for item in uploaded["items"]} >= {nsfw_id} + + # ── 7. archive ─────────────────────────────────────────────────────── + location = _post( + base, "/archive-locations", json={"name": "external", "root": str(archive_root)} + ).json() + preflight = _post( + base, "/archive-preflight", json={"location_id": location["id"]} + ).json() + assert preflight["state"] == "ready", [ + (asset["asset_id"], asset["blockers"]) + for album in preflight["albums"] + for asset in album["assets"] + if asset["blockers"] + ] or preflight + archive_plan = _post( + base, + "/archive-plans", + json={"location_id": location["id"], "token": preflight["token"]}, + ).json() + uploaded_ids = {item["asset_id"] for item in uploaded["items"]} + _post(base, f"/archive-plans/{archive_plan['id']}/apply") + wait_until( + lambda: all( + asset["availability_state"].startswith("archived") + for asset in _get(base, "/inventory/assets", params={"limit": 200})["items"] + if asset["id"] in uploaded_ids + ), + timeout=90, + ) + assert _hashes(library.lib, archive_root) == stable_hashes, "archiving lost bytes" + + # ── 8. offline deduplication ───────────────────────────────────────── + (archive_root / ".photo-pipeline-archive.json").rename( + archive_root / ".photo-pipeline-archive.json.away" + ) + # A copy of an archived photo turns up in the library under its own name — + # the real shape of "I re-imported an old card" — so nothing occupies the + # path the archived original would be restored to. + returned = library.lib / "2019 Rome" / "rediscovered.jpg" + returned.parent.mkdir(parents=True, exist_ok=True) + archived_copy = next(archive_root.rglob("*.jpg")) + shutil.copyfile(archived_copy, returned) + _post(base, "/inventory/scan") + _post(base, "/duplicates/detect") + offline = _get(base, "/inventory/assets", params={"limit": 200})["items"] + archived = [a for a in offline if a["availability_state"].startswith("archived")] + assert archived, "an unmounted medium must not make assets missing" + assert all(a["availability_state"] != "missing_unexpected" for a in offline) + assert any( + cluster["member_total"] >= 2 for cluster in _get(base, "/duplicates/clusters")["items"] + ), "the rediscovered copy did not meet its archived original" + + # ── 9. restore ─────────────────────────────────────────────────────── + (archive_root / ".photo-pipeline-archive.json.away").rename( + archive_root / ".photo-pipeline-archive.json" + ) + restore_report = _post( + base, "/restore-preflight", json={"location_id": location["id"]} + ).json() + restore_plan = _post( + base, + "/restore-plans", + json={"location_id": location["id"], "token": restore_report["token"]}, + ).json() + _post(base, f"/restore-plans/{restore_plan['id']}/apply") + wait_until( + lambda: all( + asset["availability_state"] == "active" + for asset in _get(base, "/inventory/assets", params={"limit": 200})["items"] + if asset["id"] in identity + ), + timeout=90, + ) + + # ── 10. the final restart proves every stage was durable ───────────── + worker.kill() + worker.wait(timeout=20) + server.stop() + server.start() + base = server.base + final = { + asset["id"]: asset + for asset in _get(base, "/inventory/assets", params={"limit": 200})["items"] + } + assert set(identity) <= set(final), "an asset id did not survive the journey" + assert _get(base, "/safety/counts")["nsfw"] == 1 + assert _get(base, "/upload-batches")["batches"][0]["state"] == "succeeded" + reachable = { + _sha256(path): str(path) + for root in (library.lib, archive_root) + for path in root.rglob("*.jpg") + if path.is_file() and not path.name.startswith(".") + } + assert stable_hashes <= set(reachable), ( + "a photo was lost", + sorted(stable_hashes - set(reachable)), + sorted(reachable.values()), + ) + workflow = _get(base, "/workflow") + assert {stage["key"] for stage in workflow["stages"]} >= { + "inventory", + "duplicates", + "safety", + "analysis", + } + finally: + worker.kill() + worker.wait(timeout=20) + server.stop() + immich.stop() diff --git a/tests/story_traceability.json b/tests/story_traceability.json index 2e9904e..a36c1c6 100644 --- a/tests/story_traceability.json +++ b/tests/story_traceability.json @@ -169,6 +169,18 @@ ], "US07-06": [ "tests/integration/test_performance_budgets.py" + ], + "US07-07": [ + "tests/e2e/test_release_gate.py", + "tests/e2e/test_release_journey.py" ] - } + }, + "planned": [ + "US08-01", + "US08-02", + "US08-03", + "US08-04", + "US08-05" + ], + "_planned_comment": "Accepted backlog stories that are not implemented yet. The release gate (US07-07) requires every story file to be either mapped to tests or listed here, so an unimplemented story is a visible decision rather than a hole in the matrix." }