US02-01: Extract Safety and Existing Web UI Donors (#54)

photo_pipeline/integrations/__init__.py

@@ -0,0 +1,5 @@
+"""External-tool adapters (exiftool, vision, NSFW model, immich-go).
+
+Integrations own all subprocess and network I/O; services depend on these
+interfaces, never on the archived CLI entry points.
+"""

photo_pipeline/integrations/exiftool.py

@@ -0,0 +1,70 @@
+"""exiftool adapter: read and apply EXIF keywords.
+
+Only the ``Keywords`` and ``Subject`` fields are read or written, so every other
+tag (caption, dates, GPS, camera, ratings) is preserved. Reading is batched over
+stdin; writing uses the idempotent ``-=``/``+=`` pattern so re-running never
+duplicates a keyword.
+
+Extracted from nsfwtag.exif (_read_keywords / write_keyword / remove_keyword) and
+photo_analyzer's batched keyword read (donor_ledger.yaml: nt-exif-marks,
+nt-apply-list, pa-nsfw-filter). No dependency on the archived entry points.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+from collections.abc import Iterable
+
+
+def read_keyword_sets(paths: Iterable[str]) -> dict[str, set[str]]:
+    """Map each path to its lowercased set of ``Keywords`` + ``Subject`` values.
+
+    One batched exiftool call (paths fed via stdin). Returns ``{}`` if exiftool is
+    unavailable — callers decide how to fail (analysis fails open, safety fails safe).
+    """
+    paths = list(paths)
+    if not paths:
+        return {}
+    want = {os.path.normpath(p): p for p in paths}
+    try:
+        result = subprocess.run(
+            ["exiftool", "-m", "-j", "-Keywords", "-Subject", "-@", "-"],
+            input="\n".join(paths),
+            capture_output=True,
+            text=True,
+        )
+    except FileNotFoundError:
+        return {}
+    out: dict[str, set[str]] = {}
+    try:
+        records = json.loads(result.stdout or "[]")
+    except ValueError:
+        return {}
+    for record in records:
+        values: list = []
+        for field in ("Keywords", "Subject"):
+            value = record.get(field)
+            if isinstance(value, list):
+                values += value
+            elif value is not None:
+                values.append(value)
+        key = want.get(os.path.normpath(record.get("SourceFile", "")))
+        if key is not None:
+            out[key] = {str(v).strip().lower() for v in values}
+    return out
+
+
+def apply_keywords(path: str, *, add: Iterable[str] = (), remove: Iterable[str] = ()) -> bool:
+    """Idempotently add/remove keywords in Keywords + Subject; preserve all else."""
+    args = ["exiftool", "-m", "-overwrite_original"]
+    for kw in remove:
+        args += [f"-Keywords-={kw}", f"-Subject-={kw}"]
+    for kw in add:
+        # remove-then-add makes the add idempotent (no duplicate on re-run).
+        args += [f"-Keywords-={kw}", f"-Keywords+={kw}", f"-Subject-={kw}", f"-Subject+={kw}"]
+    if len(args) == 3:
+        return True
+    args.append(path)
+    return subprocess.run(args, capture_output=True, text=True).returncode == 0

photo_pipeline/integrations/nsfw_model.py

@@ -0,0 +1,84 @@
+"""Local NSFW model adapter.
+
+Wraps the on-device ViT classifier behind a small interface so the safety service
+depends on ``NsfwModel.score`` rather than on torch/transformers or the archived
+CLI. Heavy imports are lazy: importing this module never loads the model, so the
+rest of the app (and its tests) stay light. Scoring itself is non-deterministic
+across hardware and is exercised in dedicated model tests, not the unit suite.
+
+Extracted from nsfwtag.scoring.score_images (donor_ledger.yaml: nt-score-model).
+The donor's CSV cache is dropped here — scores are persisted in the database by
+asset ID (safety repository), not a path-keyed CSV.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+MODEL_ID = "AdamCodd/vit-base-nsfw-detector"
+BATCH = 16
+
+
+class NsfwModel:
+    def __init__(self, model_id: str = MODEL_ID, batch: int = BATCH) -> None:
+        self.model_id = model_id
+        self.batch = batch
+        self._model = None
+        self._device = None
+        self._size = None
+        self._nsfw_idx = None
+
+    def _ensure_loaded(self) -> None:
+        if self._model is not None:
+            return
+        import torch
+        from transformers import AutoModelForImageClassification
+
+        self._device = "mps" if torch.backends.mps.is_available() else "cpu"
+        try:
+            model = AutoModelForImageClassification.from_pretrained(
+                self.model_id, local_files_only=True
+            )
+        except Exception:
+            model = AutoModelForImageClassification.from_pretrained(self.model_id)
+        self._model = model.to(self._device).eval()
+        self._nsfw_idx = next(
+            i for i, label in model.config.id2label.items() if label.lower() == "nsfw"
+        )
+        self._size = getattr(model.config, "image_size", 224)
+
+    def score(self, paths: list[Path | str]) -> list[tuple[str, float]]:
+        """Return ``[(path, nsfw_probability)]`` for each readable image."""
+        if not paths:
+            return []
+        self._ensure_loaded()
+        import numpy as np
+        import torch
+        from PIL import Image, ImageFile
+
+        ImageFile.LOAD_TRUNCATED_IMAGES = True
+
+        def preprocess(image):
+            image = image.convert("RGB").resize((self._size, self._size), Image.BILINEAR)
+            array = (np.asarray(image, dtype="float32") / 255.0 - 0.5) / 0.5
+            return torch.from_numpy(array).permute(2, 0, 1)
+
+        results: list[tuple[str, float]] = []
+        items = [str(p) for p in paths]
+        for start in range(0, len(items), self.batch):
+            tensors, batch_paths = [], []
+            for path in items[start : start + self.batch]:
+                try:
+                    tensors.append(preprocess(Image.open(path)))
+                    batch_paths.append(path)
+                except Exception:
+                    continue
+            if not tensors:
+                continue
+            with torch.no_grad():
+                probs = self._model(
+                    pixel_values=torch.stack(tensors).to(self._device)
+                ).logits.softmax(-1)
+            for path, prob in zip(batch_paths, probs):
+                results.append((path, float(prob[self._nsfw_idx].item())))
+        return results

photo_pipeline/services/safety.py

@@ -0,0 +1,100 @@
+"""Safety (NSFW) logic: scoring bands, decisions, and EXIF keyword rules.
+
+Pure, deterministic functions extracted from the donors so the model adapter and
+exiftool adapter stay at the edges. This is the shared source of truth for how a
+score becomes a decision and how safety keywords are read and projected.
+
+Extracted from (donor_ledger.yaml):
+- pa-nsfw-filter — photo_analyzer._rec_has_nsfw / filter_nsfw_tagged
+- nt-exif-marks  — nsfwtag.exif.read_marks / read_tagged (keyword parsing)
+- nsfwtag thresholds (DEFAULT_THRESHOLD / DEFAULT_REVIEW_MIN)
+
+Behavior changes vs the donors (required by stable identity + shared jobs):
+- Decisions and scores key on ``asset_id`` and live in the database, not in a
+  path-keyed ``nsfw_scores.csv`` (the CSV is a one-time migration import).
+- Safety keywords are **mutually exclusive**: writing a decision adds the chosen
+  ``sfw``/``nsfw`` keyword and removes the opposite. The donor only ever added
+  ``nsfw``; there was no ``sfw`` keyword and no removal.
+- Model scoring runs behind ``integrations.nsfw_model.NsfwModel`` as a shared job,
+  not inline in a CLI pass.
+- No module here imports the archived CLIs (``nsfwtag`` / ``photo_analyzer``).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+
+NSFW = "nsfw"
+SFW = "sfw"
+REVIEW_REQUIRED = "review_required"
+
+DEFAULT_THRESHOLD = 0.6  # AdamCodd model sweet spot (donor DEFAULT_THRESHOLD)
+DEFAULT_REVIEW_MIN = 0.2  # below this is confidently SFW (donor DEFAULT_REVIEW_MIN)
+
+
+def classify(
+    score: float, *, threshold: float = DEFAULT_THRESHOLD, review_min: float = DEFAULT_REVIEW_MIN
+) -> str:
+    """Suggested decision from a model score. A human confirms before it is durable.
+
+    ``>= threshold`` → ``nsfw``; ``>= review_min`` → ``review_required``; else ``sfw``.
+    """
+    if score >= threshold:
+        return NSFW
+    if score >= review_min:
+        return REVIEW_REQUIRED
+    return SFW
+
+
+def normalize_keywords(values: Iterable | str | None) -> set[str]:
+    """Normalize an EXIF Keywords/Subject value (list or scalar) to a lowercased set."""
+    if values is None:
+        return set()
+    if isinstance(values, str):
+        values = [values]
+    return {str(v).strip().lower() for v in values}
+
+
+def has_nsfw(keywords: Iterable[str]) -> bool:
+    """True if the ``nsfw`` safety keyword is present (donor _rec_has_nsfw)."""
+    return NSFW in {str(k).strip().lower() for k in keywords}
+
+
+def marks_from_keywords(keyword_sets: Mapping[str, Iterable[str]]) -> dict[str, set[str]]:
+    """Split ``{path: keywords}`` into ``{'nsfw': set, 'sfw': set}``.
+
+    They should be mutually exclusive; if a file carries both, ``nsfw`` wins so it
+    never reads as safe (donor nsfwtag.exif.read_marks).
+    """
+    nsfw, sfw = set(), set()
+    for path, keywords in keyword_sets.items():
+        lowered = {str(k).strip().lower() for k in keywords}
+        if NSFW in lowered:
+            nsfw.add(path)
+        elif SFW in lowered:
+            sfw.add(path)
+    return {NSFW: nsfw, SFW: sfw}
+
+
+def partition_nsfw(
+    keyword_sets: Mapping[str, Iterable[str]],
+) -> tuple[list[str], list[str]]:
+    """Split paths into ``(analyzable, skipped)`` — nsfw-tagged files skip cloud
+    analysis (privacy gate; donor photo_analyzer.filter_nsfw_tagged)."""
+    analyzable, skipped = [], []
+    for path, keywords in keyword_sets.items():
+        (skipped if has_nsfw(keywords) else analyzable).append(path)
+    return analyzable, skipped
+
+
+def exif_projection(decision: str) -> dict[str, list[str]]:
+    """Keyword operations to make EXIF match a confirmed decision.
+
+    Enforces mutual exclusion: add the decision keyword, remove the opposite.
+    Non-terminal decisions (review/unknown) touch nothing.
+    """
+    if decision == NSFW:
+        return {"add": [NSFW], "remove": [SFW]}
+    if decision == SFW:
+        return {"add": [SFW], "remove": [NSFW]}
+    return {"add": [], "remove": []}