"""Safety (NSFW) logic: scoring bands, decisions, and EXIF keyword rules. Pure, deterministic functions extracted from the donors so the model adapter and exiftool adapter stay at the edges. This is the shared source of truth for how a score becomes a decision and how safety keywords are read and projected. Extracted from (donor_ledger.yaml): - pa-nsfw-filter — photo_analyzer._rec_has_nsfw / filter_nsfw_tagged - nt-exif-marks — nsfwtag.exif.read_marks / read_tagged (keyword parsing) - nsfwtag thresholds (DEFAULT_THRESHOLD / DEFAULT_REVIEW_MIN) Behavior changes vs the donors (required by stable identity + shared jobs): - Decisions and scores key on ``asset_id`` and live in the database, not in a path-keyed ``nsfw_scores.csv`` (the CSV is a one-time migration import). - Safety keywords are **mutually exclusive**: writing a decision adds the chosen ``sfw``/``nsfw`` keyword and removes the opposite. The donor only ever added ``nsfw``; there was no ``sfw`` keyword and no removal. - Model scoring runs behind ``integrations.nsfw_model.NsfwModel`` as a shared job, not inline in a CLI pass. - No module here imports the archived CLIs (``nsfwtag`` / ``photo_analyzer``). """ from __future__ import annotations from collections.abc import Iterable, Mapping NSFW = "nsfw" SFW = "sfw" REVIEW_REQUIRED = "review_required" DEFAULT_THRESHOLD = 0.6 # AdamCodd model sweet spot (donor DEFAULT_THRESHOLD) DEFAULT_REVIEW_MIN = 0.2 # below this is confidently SFW (donor DEFAULT_REVIEW_MIN) def classify( score: float, *, threshold: float = DEFAULT_THRESHOLD, review_min: float = DEFAULT_REVIEW_MIN ) -> str: """Suggested decision from a model score. A human confirms before it is durable. ``>= threshold`` → ``nsfw``; ``>= review_min`` → ``review_required``; else ``sfw``. """ if score >= threshold: return NSFW if score >= review_min: return REVIEW_REQUIRED return SFW def normalize_keywords(values: Iterable | str | None) -> set[str]: """Normalize an EXIF Keywords/Subject value (list or scalar) to a lowercased set.""" if values is None: return set() if isinstance(values, str): values = [values] return {str(v).strip().lower() for v in values} def has_nsfw(keywords: Iterable[str]) -> bool: """True if the ``nsfw`` safety keyword is present (donor _rec_has_nsfw).""" return NSFW in {str(k).strip().lower() for k in keywords} def marks_from_keywords(keyword_sets: Mapping[str, Iterable[str]]) -> dict[str, set[str]]: """Split ``{path: keywords}`` into ``{'nsfw': set, 'sfw': set}``. They should be mutually exclusive; if a file carries both, ``nsfw`` wins so it never reads as safe (donor nsfwtag.exif.read_marks). """ nsfw, sfw = set(), set() for path, keywords in keyword_sets.items(): lowered = {str(k).strip().lower() for k in keywords} if NSFW in lowered: nsfw.add(path) elif SFW in lowered: sfw.add(path) return {NSFW: nsfw, SFW: sfw} def partition_nsfw( keyword_sets: Mapping[str, Iterable[str]], ) -> tuple[list[str], list[str]]: """Split paths into ``(analyzable, skipped)`` — nsfw-tagged files skip cloud analysis (privacy gate; donor photo_analyzer.filter_nsfw_tagged).""" analyzable, skipped = [], [] for path, keywords in keyword_sets.items(): (skipped if has_nsfw(keywords) else analyzable).append(path) return analyzable, skipped def exif_projection(decision: str) -> dict[str, list[str]]: """Keyword operations to make EXIF match a confirmed decision. Enforces mutual exclusion: add the decision keyword, remove the opposite. Non-terminal decisions (review/unknown) touch nothing. """ if decision == NSFW: return {"add": [NSFW], "remove": [SFW]} if decision == SFW: return {"add": [SFW], "remove": [NSFW]} return {"add": [], "remove": []} # ── Persistence, scoring, and the durable safety decision ──────────────────── import uuid from datetime import datetime, timezone from sqlalchemy import select from sqlalchemy.orm import sessionmaker from photo_pipeline.models import Asset, ExifProjection, SafetyReview from photo_pipeline.services import exif_checkpoint DECISIONS = {SFW, NSFW, "deferred"} class SafetyError(Exception): """Invalid safety operation (bad decision, unknown asset).""" def _now() -> datetime: # Microsecond precision so the latest-review-wins ordering never ties, even # when a decision is revised twice in the same DB-second (SQLite CURRENT_TIMESTAMP # is only second-granular). return datetime.now(timezone.utc) class SafetyService: """Durable safety scoring, review decisions, and the EXIF safety checkpoint. Scores and decisions live in ``safety_reviews`` keyed by ``asset_id`` (the donor's path-keyed CSV is gone). The latest row per asset is the current decision; ``AnalysisService`` reads it as the privacy gate. """ def __init__(self, session_factory: sessionmaker, *, model=None) -> None: self._session_factory = session_factory self._model = model # injected in tests; real NsfwModel is lazy-loaded # -- reads ---------------------------------------------------------------- def _latest_by_asset(self, session) -> dict[str, SafetyReview]: # Latest row per asset. Small local scale: order ascending, let later rows # overwrite. ponytail: a windowed query if safety_reviews ever grows huge. latest: dict[str, SafetyReview] = {} for review in session.scalars(select(SafetyReview).order_by(SafetyReview.created_at)): latest[review.asset_id] = review return latest def current_decision(self, asset_id: str) -> str | None: with self._session_factory() as session: review = self._latest_by_asset(session).get(asset_id) return review.decision if review else None def counts(self) -> dict[str, int]: """Decision breakdown over canonical, active assets — the workflow totals.""" with self._session_factory() as session: assets = list(session.scalars(_eligible_assets_query())) latest = self._latest_by_asset(session) out = {SFW: 0, NSFW: 0, "deferred": 0, "undecided": 0, "scored": 0} for asset in assets: review = latest.get(asset.id) decision = review.decision if review else None if decision in (SFW, NSFW, "deferred"): out[decision] += 1 else: out["undecided"] += 1 if review and review.score is not None: out["scored"] += 1 return out def review_queue(self, state: str = "", limit: int = 100, offset: int = 0) -> dict: """Assets for the review UI, filtered by ``state`` (undecided/sfw/nsfw/deferred).""" with self._session_factory() as session: assets = list(session.scalars(_eligible_assets_query().order_by(Asset.current_path))) latest = self._latest_by_asset(session) # One query, not one per asset: the reviewer needs to see a divergent # checkpoint, which is neither "verified" nor a plain failure (US07-03). projections = { row.asset_id: row.state for row in session.scalars( select(ExifProjection).where(ExifProjection.stage == "safety") ) } rows = [] for asset in assets: review = latest.get(asset.id) decision = review.decision if review else None effective = decision or "undecided" if state and state != effective: continue rows.append( { "asset_id": asset.id, "current_path": asset.current_path, "score": review.score if review else None, "decision": decision, "suggested": classify(review.score) if review and review.score is not None else None, "exif_verified": bool(review and review.exif_verified_at), "exif_state": projections.get(asset.id), } ) return {"total": len(rows), "items": rows[offset : offset + limit]} def scorable_asset_ids(self) -> list[str]: """Canonical active assets with a path — the items a scoring job enqueues.""" with self._session_factory() as session: return [a.id for a in session.scalars(_eligible_assets_query()) if a.current_path] # -- writes --------------------------------------------------------------- def score_assets(self, asset_ids: list[str] | None = None) -> int: """Score eligible assets with the local model and persist scored reviews. Records score-only reviews (no decision) so the reviewer can sort by risk. Returns the number scored. Assets already carrying a human decision keep it. """ with self._session_factory() as session: query = _eligible_assets_query() if asset_ids is not None: query = query.where(Asset.id.in_(asset_ids)) assets = [a for a in session.scalars(query) if a.current_path] if not assets: return 0 model = self._model or _default_model() by_path = {a.current_path: a.id for a in assets} scored = model.score(list(by_path)) now = _now() with self._session_factory() as session: for path, score in scored: session.add( SafetyReview( id=str(uuid.uuid4()), asset_id=by_path[path], score=float(score), created_at=now ) ) session.commit() return len(scored) def decide(self, asset_id: str, decision: str, *, reviewer: str = "user", write_exif: bool = True) -> dict: """Persist a human safety decision and, for sfw/nsfw, run the EXIF checkpoint. The decision is durable regardless of EXIF; ``exif_verified`` becomes true only after the mutually-exclusive keyword is written and read back, and the file's new SHA-256 is stored — that verified state is what upload eligibility later depends on. """ if decision not in DECISIONS: raise SafetyError(f"invalid decision {decision!r}") with self._session_factory() as session: asset = session.get(Asset, asset_id) if asset is None: raise SafetyError(f"unknown asset {asset_id}") prior = self._latest_by_asset(session).get(asset_id) path = asset.current_path exif_verified_at = None result_sha256 = None if write_exif and decision in (SFW, NSFW) and path: ops = exif_projection(decision) # The full checkpoint: write the owned keyword, read the whole file back, # and prove every field this stage does not own survived. A divergent # result is recorded and left alone — it must not count as verified, so # upload stays blocked until a human decides (US07-03). result = exif_checkpoint.run( path, add=tuple(ops["add"]), remove=tuple(ops["remove"]) ) exif_checkpoint.record( self._session_factory, asset_id=asset_id, stage="safety", result=result, add=tuple(ops["add"]), remove=tuple(ops["remove"]), ) if result.verified: exif_verified_at = result.verified_at result_sha256 = result.sha256 now = _now() with self._session_factory() as session: review = SafetyReview( id=str(uuid.uuid4()), asset_id=asset_id, decision=decision, prior_decision=prior.decision if prior else None, reviewer=reviewer, score=prior.score if prior else None, exif_verified_at=exif_verified_at, result_sha256=result_sha256, created_at=now, ) session.add(review) if result_sha256: asset = session.get(Asset, asset_id) asset.current_sha256 = result_sha256 session.commit() return { "asset_id": asset_id, "decision": decision, "prior_decision": prior.decision if prior else None, "exif_verified": exif_verified_at is not None, } def _eligible_assets_query(): """Canonical, active assets — the safety stage runs only on these. ``canonical_asset_id IS NULL`` excludes non-canonical duplicate variants; the shared discovery service already excludes ``_IGNORE/``. """ return select(Asset).where( Asset.canonical_asset_id.is_(None), Asset.availability_state == "active", ) def _default_model(): from photo_pipeline.integrations.nsfw_model import NsfwModel return NsfwModel()