US07-03: Harden Media and Metadata Edge Cases

photo_pipeline/services/analysis.py

@@ -26,9 +26,8 @@ from sqlalchemy import func, select
 from sqlalchemy.orm import sessionmaker
 
 from photo_pipeline import path_policy
-from photo_pipeline.integrations import exiftool
 from photo_pipeline.models import AnalysisResult, Asset, SafetyReview
-from photo_pipeline.services import hashing
+from photo_pipeline.services import exif_checkpoint
 from photo_pipeline.services.safety import SFW
 
 MODEL = "gemini-2.5-flash"
@@ -170,7 +169,7 @@ class AnalysisService:
                 tokens=int(result.get("_tokens", 0)) if isinstance(result, dict) else 0,
                 raw=json.dumps(result, ensure_ascii=False),
             )
-            _write_analysis_exif(path, result)
+            self._write_analysis_exif(asset_id, path, result)
             analyzed += 1
         return {"analyzed": analyzed, "skipped": skipped, "errors": errors}
 
@@ -195,10 +194,42 @@ class AnalysisService:
                 row.location_hint = result.get("location_hint")
                 row.approx_year = result.get("approx_year")
                 row.analyzed_at = now
-                row.exif_written_at = now
             session.add(row)
             session.commit()
 
+    def _write_analysis_exif(self, asset_id: str, path: str, result: dict) -> None:
+        """The analysis EXIF checkpoint: additive keywords, then prove the rest held.
+
+        Additive by design — safety keywords and the user's own keywords are merged
+        with, never replaced (concept §3). ``exif_written_at`` is set only when the
+        read-back verified both the new keywords and every field this stage does not
+        own; a divergent result is recorded and left for a human (US07-03).
+        ponytail: the managed ``AI:`` caption segment, once captions are owned here.
+        """
+        tags = tuple(str(tag) for tag in (result.get("tags") or []))
+        if not tags:
+            return
+        checkpoint = exif_checkpoint.run(path, add=tags)
+        exif_checkpoint.record(
+            self._session_factory,
+            asset_id=asset_id,
+            stage="analysis",
+            result=checkpoint,
+            add=tags,
+        )
+        if not checkpoint.verified:
+            return
+        with self._session_factory() as session:
+            row = session.get(AnalysisResult, asset_id)
+            if row is not None:
+                row.exif_written_at = checkpoint.verified_at
+            asset = session.get(Asset, asset_id)
+            if asset is not None and checkpoint.sha256:
+                # The bytes changed when the container was rewritten; upload must use
+                # the hash of what is actually on disk now (concept §3).
+                asset.current_sha256 = checkpoint.sha256
+            session.commit()
+
     def get(self, asset_id: str) -> dict | None:
         with self._session_factory() as session:
             row = session.get(AnalysisResult, asset_id)
@@ -211,17 +242,6 @@ def _album_hint(path: str) -> str:
     return Path(path).parent.name
 
 
-def _write_analysis_exif(path: str, result: dict) -> None:
-    """Additive analysis keywords into EXIF (Keywords/Subject), preserving safety
-    and user keywords. The donor also wrote a managed caption; only keywords are
-    written here via the shared adapter.
-    ponytail: add the managed ``AI:`` caption segment + read-back preservation check
-    when the analysis EXIF checkpoint is hardened."""
-    tags = result.get("tags") if isinstance(result, dict) else None
-    if tags:
-        exiftool.apply_keywords(path, add=[str(t) for t in tags])
-
-
 def _result_dict(row: AnalysisResult) -> dict:
     data = {field: getattr(row, field) for field in RESULT_FIELDS}
     data["tags"] = json.loads(row.tags) if row.tags else []

photo_pipeline/services/exif_checkpoint.py

@@ -0,0 +1,173 @@
+"""The EXIF checkpoint every metadata stage ends with (concept §3, US07-03).
+
+A stage does not own a file's metadata; it owns a few fields in it. So writing is
+never "set these tags" — it is:
+
+    snapshot everything → write only the owned fields → read everything back
+    → prove the owned fields landed → prove nothing else moved
+    → refresh the file hash → record the projection
+
+Non-destructive here means *semantic* preservation: exiftool may rewrite the whole
+container, so the file's bytes, size, and timestamps legitimately change. What may
+not change is any field this stage does not own. When one does, the checkpoint is
+``divergent``: the result is recorded, the stage is **not** marked verified, and
+nothing is silently repaired — a later stage that needs verified metadata (upload)
+therefore stays blocked until a human looks.
+
+``failed`` is the third outcome and is deliberately distinct: exiftool missing, an
+unreadable file, or a write that did not take is not evidence that metadata is fine.
+"""
+
+from __future__ import annotations
+
+import json
+import uuid
+from dataclasses import dataclass
+from datetime import datetime, timezone
+
+from photo_pipeline.integrations import exiftool
+from photo_pipeline.models import ExifProjection
+from photo_pipeline.services import hashing
+
+# The tags the safety and analysis stages may change. Matched on the tag name, so
+# every group carries the same rule (IPTC:Keywords, XMP:XMP-dc:Subject, ...).
+OWNED_TAGS = frozenset({"Keywords", "Subject"})
+
+# Not metadata about the picture: filesystem facts, the digest that necessarily
+# moves whenever IPTC does, and the structural tags exiftool has to create the first
+# time it writes an IPTC or XMP block. Comparing these would report every write as
+# divergent and make the signal worthless.
+VOLATILE_PREFIXES = ("File:System:", "ExifTool:")
+VOLATILE_KEYS = frozenset(
+    {
+        "File:CurrentIPTCDigest",
+        "IPTC:ApplicationRecordVersion",
+        "XMP:XMP-x:XMPToolkit",
+        "XMP:XMP-xmp:MetadataDate",
+    }
+)
+
+VERIFIED = "verified"
+DIVERGENT = "divergent"
+FAILED = "failed"
+
+
+@dataclass(frozen=True)
+class CheckpointResult:
+    state: str  # verified | divergent | failed
+    changed_fields: tuple[str, ...] = ()
+    sha256: str | None = None
+    verified_at: datetime | None = None
+    reason: str | None = None
+
+    @property
+    def verified(self) -> bool:
+        return self.state == VERIFIED
+
+
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def is_owned(key: str) -> bool:
+    return key.rsplit(":", 1)[-1] in OWNED_TAGS
+
+
+def is_volatile(key: str) -> bool:
+    return key in VOLATILE_KEYS or key.startswith(VOLATILE_PREFIXES)
+
+
+def compare(before: dict, after: dict) -> tuple[str, ...]:
+    """Fields outside this stage's ownership whose value did not survive the write.
+
+    Additions count: a tag that appears out of nowhere is as much a divergence as a
+    tag that disappeared — both mean the write did more than it was asked to.
+    """
+    keys = set(before) | set(after)
+    return tuple(
+        sorted(
+            key
+            for key in keys
+            if not is_owned(key)
+            and not is_volatile(key)
+            and before.get(key) != after.get(key)
+        )
+    )
+
+
+def owned_values(snapshot: dict) -> set[str]:
+    """Lowercased Keywords/Subject values across every group in the snapshot."""
+    values: set[str] = set()
+    for key, value in snapshot.items():
+        if not is_owned(key):
+            continue
+        items = value if isinstance(value, list) else [value]
+        values.update(str(item).strip().lower() for item in items if item is not None)
+    return values
+
+
+def run(
+    path: str,
+    *,
+    add: tuple[str, ...] = (),
+    remove: tuple[str, ...] = (),
+) -> CheckpointResult:
+    """Write the owned keywords for one asset and verify the whole file around them."""
+    before = exiftool.read_all(path)
+    if before is None:
+        return CheckpointResult(FAILED, reason="metadata_unreadable")
+
+    if not exiftool.apply_keywords(path, add=add, remove=remove):
+        return CheckpointResult(FAILED, reason="write_failed")
+
+    after = exiftool.read_all(path)
+    if after is None:
+        return CheckpointResult(FAILED, reason="readback_unreadable")
+
+    present = owned_values(after)
+    wanted = {value.strip().lower() for value in add}
+    unwanted = {value.strip().lower() for value in remove}
+    if not wanted <= present or (unwanted & present):
+        return CheckpointResult(FAILED, reason="owned_fields_not_written")
+
+    changed = compare(before, after)
+    sha256 = hashing.sha256_file(path)
+    if changed:
+        return CheckpointResult(DIVERGENT, changed_fields=changed, sha256=sha256)
+    return CheckpointResult(VERIFIED, sha256=sha256, verified_at=_now())
+
+
+def record(
+    session_factory,
+    *,
+    asset_id: str,
+    stage: str,
+    result: CheckpointResult,
+    add: tuple[str, ...] = (),
+    remove: tuple[str, ...] = (),
+) -> None:
+    """Persist the projection for ``(asset_id, stage)`` — one current row per pair.
+
+    The row is what makes divergence durable and reviewable rather than a log line
+    that scrolled away.
+    """
+    with session_factory() as session:
+        row = session.get(ExifProjection, (asset_id, stage))
+        if row is None:
+            row = ExifProjection(asset_id=asset_id, stage=stage, id=str(uuid.uuid4()))
+            session.add(row)
+        row.projection_version = (row.projection_version or 0) + 1
+        row.desired_json = json.dumps({"add": list(add), "remove": list(remove)})
+        row.divergent_fields = json.dumps(list(result.changed_fields))
+        row.result_file_sha256 = result.sha256
+        row.state = result.state
+        row.error_code = result.reason
+        row.verified_at = result.verified_at
+        row.updated_at = _now()
+        session.commit()
+
+
+def state_for(session_factory, asset_id: str, stage: str) -> str | None:
+    with session_factory() as session:
+        row = session.get(ExifProjection, (asset_id, stage))
+        return row.state if row else None

photo_pipeline/services/hashing.py

@@ -16,7 +16,9 @@ from __future__ import annotations
 import hashlib
 from pathlib import Path
 
-from PIL import Image, ImageOps
+from PIL import ImageOps
+
+from photo_pipeline import imaging
 
 PIXEL_HASH_VERSION = 1
 PHASH_VERSION = 1
@@ -41,7 +43,9 @@ def _digest_file(path: Path | str, digest) -> str:
 
 
 def pixel_sha256(path: Path | str) -> str:
-    with Image.open(path) as image:
+    # Bounded decode: a header claiming a billion pixels is refused before it is
+    # ever turned into memory (US07-03).
+    with imaging.open_image(path) as image:
         oriented = ImageOps.exif_transpose(image)
         rgb = oriented.convert("RGB")
         header = f"{PIXEL_HASH_VERSION}:{rgb.width}x{rgb.height}:".encode()
@@ -68,7 +72,9 @@ def phash(path: Path | str) -> str:
     import numpy as np
     from scipy.fftpack import dct
 
-    with Image.open(path) as image:
+    from PIL import Image
+
+    with imaging.open_image(path) as image:
         small = image.convert("L").resize((32, 32), Image.LANCZOS)
         matrix = np.asarray(small, dtype=np.float64)
     transformed = dct(dct(matrix, axis=0), axis=1)

photo_pipeline/services/safety.py

@@ -108,9 +108,8 @@ from datetime import datetime, timezone
 from sqlalchemy import select
 from sqlalchemy.orm import sessionmaker
 
-from photo_pipeline.integrations import exiftool
-from photo_pipeline.models import Asset, SafetyReview
-from photo_pipeline.services import hashing
+from photo_pipeline.models import Asset, ExifProjection, SafetyReview
+from photo_pipeline.services import exif_checkpoint
 
 DECISIONS = {SFW, NSFW, "deferred"}
 
@@ -174,6 +173,14 @@ class SafetyService:
         with self._session_factory() as session:
             assets = list(session.scalars(_eligible_assets_query().order_by(Asset.current_path)))
             latest = self._latest_by_asset(session)
+            # One query, not one per asset: the reviewer needs to see a divergent
+            # checkpoint, which is neither "verified" nor a plain failure (US07-03).
+            projections = {
+                row.asset_id: row.state
+                for row in session.scalars(
+                    select(ExifProjection).where(ExifProjection.stage == "safety")
+                )
+            }
             rows = []
             for asset in assets:
                 review = latest.get(asset.id)
@@ -189,6 +196,7 @@ class SafetyService:
                         "decision": decision,
                         "suggested": classify(review.score) if review and review.score is not None else None,
                         "exif_verified": bool(review and review.exif_verified_at),
+                        "exif_state": projections.get(asset.id),
                     }
                 )
             return {"total": len(rows), "items": rows[offset : offset + limit]}
@@ -247,13 +255,24 @@ class SafetyService:
         result_sha256 = None
         if write_exif and decision in (SFW, NSFW) and path:
             ops = exif_projection(decision)
-            if exiftool.apply_keywords(path, add=ops["add"], remove=ops["remove"]):
-                # Read back: the chosen keyword present, the opposite absent.
-                keywords = exiftool.read_keyword_sets([path]).get(path, set())
-                opposite = NSFW if decision == SFW else SFW
-                if decision in keywords and opposite not in keywords:
-                    exif_verified_at = _now()
-                    result_sha256 = hashing.sha256_file(path)
+            # The full checkpoint: write the owned keyword, read the whole file back,
+            # and prove every field this stage does not own survived. A divergent
+            # result is recorded and left alone — it must not count as verified, so
+            # upload stays blocked until a human decides (US07-03).
+            result = exif_checkpoint.run(
+                path, add=tuple(ops["add"]), remove=tuple(ops["remove"])
+            )
+            exif_checkpoint.record(
+                self._session_factory,
+                asset_id=asset_id,
+                stage="safety",
+                result=result,
+                add=tuple(ops["add"]),
+                remove=tuple(ops["remove"]),
+            )
+            if result.verified:
+                exif_verified_at = result.verified_at
+                result_sha256 = result.sha256
 
         now = _now()
         with self._session_factory() as session:

photo_pipeline/services/thumbnails.py

@@ -20,15 +20,16 @@ pa-imaging).
 
 from __future__ import annotations
 
+import os
 import uuid
 from pathlib import Path
 
-from PIL import Image, ImageOps, UnidentifiedImageError
+from PIL import Image, ImageOps
 from sqlalchemy import select
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import sessionmaker
 
-from photo_pipeline import path_policy
+from photo_pipeline import imaging, path_policy
 from photo_pipeline.config import Config
 from photo_pipeline.models import Asset, Thumbnail
 from photo_pipeline.services import availability
@@ -42,6 +43,9 @@ except Exception:  # pragma: no cover
     pass
 
 SIZES = (256, 512, 1280)
+# Every in-flight render writes one of these beside its destination; startup
+# cleanup recognises exactly this pattern and nothing else.
+TEMP_SUFFIX = ".tmp"
 THUMB_VERSION = 1
 THUMB_FORMAT = "webp"
 # The size kept as durable comparison evidence for archived assets (concept §9).
@@ -222,28 +226,33 @@ class ThumbnailService:
         return self._cache_dir / safe[:2] / f"{safe}.{THUMB_FORMAT}"
 
     def _render(self, source: str, size: int, cache_key: str) -> dict:
+        """Decode bounded, orient, colour-manage, resize, and write atomically.
+
+        The temporary file is removed on every failure path: a decoder that dies
+        halfway through ``save`` would otherwise leave a stray ``.tmp`` in the cache
+        forever (US07-03).
+        """
+        destination = self._cache_path(cache_key)
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        tmp = destination.with_name(f".{destination.name}.{uuid.uuid4().hex}{TEMP_SUFFIX}")
         try:
-            with Image.open(source) as image:
-                width, height = image.size
-                if width * height > self._config.thumbnail_max_pixels:
-                    raise ImageTooLarge(
-                        f"{width}x{height} exceeds {self._config.thumbnail_max_pixels} px"
-                    )
+            with imaging.open_image(source, max_pixels=self._config.thumbnail_max_pixels) as image:
+                imaging.draft(image, size)  # JPEG decodes near the target size
                 oriented = ImageOps.exif_transpose(image)
                 mode = "RGBA" if _has_alpha(oriented) else "RGB"
-                converted = oriented.convert(mode)
+                converted = imaging.to_srgb(oriented, mode=mode)
                 converted.thumbnail((size, size), Image.LANCZOS)
                 out_width, out_height = converted.size
-                destination = self._cache_path(cache_key)
-                destination.parent.mkdir(parents=True, exist_ok=True)
-                tmp = destination.with_name(f".{destination.name}.{uuid.uuid4().hex}.tmp")
                 converted.save(tmp, format="WEBP", quality=82, method=4)
-        except ImageTooLarge:
+        except imaging.ImageTooLarge as error:
+            tmp.unlink(missing_ok=True)
+            raise ImageTooLarge(str(error)) from None
+        except imaging.UndecodableImage as error:
+            tmp.unlink(missing_ok=True)
+            raise UnsupportedImage(str(error)) from None
+        except Exception:
+            tmp.unlink(missing_ok=True)
             raise
-        except (UnidentifiedImageError, OSError, ValueError) as error:
-            raise UnsupportedImage(f"cannot decode {source}: {error}") from error
-
-        import os
 
         os.replace(tmp, destination)
         return {
@@ -253,6 +262,28 @@ class ThumbnailService:
             "format": THUMB_FORMAT,
         }
 
+    def cleanup_temp_files(self) -> int:
+        """Remove leftover render temporaries, and only those.
+
+        Runs at startup, where the concept is explicit: clean *recognized* stale
+        temporary files, never arbitrary ones. The pattern is this service's own —
+        a dot-prefixed name inside the managed cache directory ending in
+        ``.tmp`` — so nothing outside the cache and nothing a user put there can
+        match. Returns how many were removed.
+        """
+        if not self._cache_dir.is_dir():
+            return 0
+        removed = 0
+        for leftover in self._cache_dir.rglob(f".*{TEMP_SUFFIX}"):
+            if not leftover.is_file() or leftover.is_symlink():
+                continue
+            try:
+                leftover.unlink()
+                removed += 1
+            except OSError:  # pragma: no cover - a racing render already won
+                pass
+        return removed
+
     # ── persistence ────────────────────────────────────────────────────────────
     def _record_ready(
         self, cache_key: str, asset_id: str, size: int, rendered: dict, *, protected: bool = False