392 lines
16 KiB
Python
392 lines
16 KiB
Python
"""Safety (NSFW) logic: scoring bands, decisions, and EXIF keyword rules.
|
|
|
|
Pure, deterministic functions extracted from the donors so the model adapter and
|
|
exiftool adapter stay at the edges. This is the shared source of truth for how a
|
|
score becomes a decision and how safety keywords are read and projected.
|
|
|
|
Extracted from (donor_ledger.yaml):
|
|
- pa-nsfw-filter — photo_analyzer._rec_has_nsfw / filter_nsfw_tagged
|
|
- nt-exif-marks — nsfwtag.exif.read_marks / read_tagged (keyword parsing)
|
|
- nsfwtag thresholds (DEFAULT_THRESHOLD / DEFAULT_REVIEW_MIN)
|
|
|
|
Behavior changes vs the donors (required by stable identity + shared jobs):
|
|
- Decisions and scores key on ``asset_id`` and live in the database, not in a
|
|
path-keyed ``nsfw_scores.csv`` (the CSV is a one-time migration import).
|
|
- Safety keywords are **mutually exclusive**: writing a decision adds the chosen
|
|
``sfw``/``nsfw`` keyword and removes the opposite. The donor only ever added
|
|
``nsfw``; there was no ``sfw`` keyword and no removal.
|
|
- Model scoring runs behind ``integrations.nsfw_model.NsfwModel`` as a shared job,
|
|
not inline in a CLI pass.
|
|
- No module here imports the archived CLIs (``nsfwtag`` / ``photo_analyzer``).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Iterable, Mapping
|
|
|
|
NSFW = "nsfw"
|
|
SFW = "sfw"
|
|
REVIEW_REQUIRED = "review_required"
|
|
|
|
DEFAULT_THRESHOLD = 0.6 # AdamCodd model sweet spot (donor DEFAULT_THRESHOLD)
|
|
DEFAULT_REVIEW_MIN = 0.2 # below this is confidently SFW (donor DEFAULT_REVIEW_MIN)
|
|
|
|
|
|
def classify(
|
|
score: float, *, threshold: float = DEFAULT_THRESHOLD, review_min: float = DEFAULT_REVIEW_MIN
|
|
) -> str:
|
|
"""Suggested decision from a model score. A human confirms before it is durable.
|
|
|
|
``>= threshold`` → ``nsfw``; ``>= review_min`` → ``review_required``; else ``sfw``.
|
|
"""
|
|
if score >= threshold:
|
|
return NSFW
|
|
if score >= review_min:
|
|
return REVIEW_REQUIRED
|
|
return SFW
|
|
|
|
|
|
def normalize_keywords(values: Iterable | str | None) -> set[str]:
|
|
"""Normalize an EXIF Keywords/Subject value (list or scalar) to a lowercased set."""
|
|
if values is None:
|
|
return set()
|
|
if isinstance(values, str):
|
|
values = [values]
|
|
return {str(v).strip().lower() for v in values}
|
|
|
|
|
|
def has_nsfw(keywords: Iterable[str]) -> bool:
|
|
"""True if the ``nsfw`` safety keyword is present (donor _rec_has_nsfw)."""
|
|
return NSFW in {str(k).strip().lower() for k in keywords}
|
|
|
|
|
|
def marks_from_keywords(keyword_sets: Mapping[str, Iterable[str]]) -> dict[str, set[str]]:
|
|
"""Split ``{path: keywords}`` into ``{'nsfw': set, 'sfw': set}``.
|
|
|
|
They should be mutually exclusive; if a file carries both, ``nsfw`` wins so it
|
|
never reads as safe (donor nsfwtag.exif.read_marks).
|
|
"""
|
|
nsfw, sfw = set(), set()
|
|
for path, keywords in keyword_sets.items():
|
|
lowered = {str(k).strip().lower() for k in keywords}
|
|
if NSFW in lowered:
|
|
nsfw.add(path)
|
|
elif SFW in lowered:
|
|
sfw.add(path)
|
|
return {NSFW: nsfw, SFW: sfw}
|
|
|
|
|
|
def partition_nsfw(
|
|
keyword_sets: Mapping[str, Iterable[str]],
|
|
) -> tuple[list[str], list[str]]:
|
|
"""Split paths into ``(analyzable, skipped)`` — nsfw-tagged files skip cloud
|
|
analysis (privacy gate; donor photo_analyzer.filter_nsfw_tagged)."""
|
|
analyzable, skipped = [], []
|
|
for path, keywords in keyword_sets.items():
|
|
(skipped if has_nsfw(keywords) else analyzable).append(path)
|
|
return analyzable, skipped
|
|
|
|
|
|
def exif_projection(decision: str) -> dict[str, list[str]]:
|
|
"""Keyword operations to make EXIF match a confirmed decision.
|
|
|
|
Enforces mutual exclusion: add the decision keyword, remove the opposite.
|
|
Non-terminal decisions (review/unknown) touch nothing.
|
|
"""
|
|
if decision == NSFW:
|
|
return {"add": [NSFW], "remove": [SFW]}
|
|
if decision == SFW:
|
|
return {"add": [SFW], "remove": [NSFW]}
|
|
return {"add": [], "remove": []}
|
|
|
|
|
|
# ── Persistence, scoring, and the durable safety decision ────────────────────
|
|
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
|
|
from sqlalchemy import column, func, select
|
|
from sqlalchemy.orm import sessionmaker
|
|
|
|
from photo_pipeline.models import Asset, ExifProjection, SafetyReview
|
|
from photo_pipeline.services import exif_checkpoint
|
|
|
|
DECISIONS = {SFW, NSFW, "deferred"}
|
|
|
|
|
|
class SafetyError(Exception):
|
|
"""Invalid safety operation (bad decision, unknown asset)."""
|
|
|
|
|
|
def _now() -> datetime:
|
|
# Microsecond precision so the latest-review-wins ordering never ties, even
|
|
# when a decision is revised twice in the same DB-second (SQLite CURRENT_TIMESTAMP
|
|
# is only second-granular).
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
class SafetyService:
|
|
"""Durable safety scoring, review decisions, and the EXIF safety checkpoint.
|
|
|
|
Scores and decisions live in ``safety_reviews`` keyed by ``asset_id`` (the
|
|
donor's path-keyed CSV is gone). The latest row per asset is the current
|
|
decision; ``AnalysisService`` reads it as the privacy gate.
|
|
"""
|
|
|
|
def __init__(self, session_factory: sessionmaker, *, model=None) -> None:
|
|
self._session_factory = session_factory
|
|
self._model = model # injected in tests; real NsfwModel is lazy-loaded
|
|
|
|
# -- reads ----------------------------------------------------------------
|
|
def _latest_by_asset(self, session) -> dict[str, SafetyReview]:
|
|
"""The current review per asset, as ORM rows. Only for small, known sets —
|
|
every library-wide caller uses ``latest_reviews()`` in SQL instead."""
|
|
latest: dict[str, SafetyReview] = {}
|
|
for review in session.scalars(select(SafetyReview).order_by(SafetyReview.created_at)):
|
|
latest[review.asset_id] = review
|
|
return latest
|
|
|
|
def current_decision(self, asset_id: str) -> str | None:
|
|
with self._session_factory() as session:
|
|
latest = latest_reviews().subquery()
|
|
return session.scalar(
|
|
select(latest.c.decision).where(latest.c.asset_id == asset_id)
|
|
)
|
|
|
|
def counts(self) -> dict[str, int]:
|
|
"""Decision breakdown over canonical, active assets — the workflow totals.
|
|
|
|
Aggregated in SQL: the workflow home asks for this on every load, and
|
|
materialising every asset and every review to count them cost hundreds of
|
|
milliseconds at 25k assets and would scale linearly from there (US07-06).
|
|
"""
|
|
latest = latest_reviews().subquery()
|
|
with self._session_factory() as session:
|
|
rows = session.execute(
|
|
select(
|
|
func.coalesce(latest.c.decision, "undecided"),
|
|
func.count(),
|
|
func.count(latest.c.score),
|
|
)
|
|
.select_from(Asset)
|
|
.join(latest, latest.c.asset_id == Asset.id, isouter=True)
|
|
.where(
|
|
Asset.canonical_asset_id.is_(None),
|
|
Asset.availability_state == "active",
|
|
)
|
|
.group_by(func.coalesce(latest.c.decision, "undecided"))
|
|
).all()
|
|
out = {SFW: 0, NSFW: 0, "deferred": 0, "undecided": 0, "scored": 0}
|
|
for decision, total, scored in rows:
|
|
if decision in (SFW, NSFW, "deferred"):
|
|
out[decision] += int(total)
|
|
else:
|
|
# Anything that is not one of the three decisions is undecided —
|
|
# including a score-only review, which is what "scored" counts.
|
|
out["undecided"] += int(total)
|
|
out["scored"] += int(scored)
|
|
return out
|
|
|
|
def review_queue(self, state: str = "", limit: int = 100, offset: int = 0) -> dict:
|
|
"""Assets for the review UI, filtered by ``state`` (undecided/sfw/nsfw/deferred).
|
|
|
|
Filtered, counted, and paged in SQL (US07-06): the queue for a large library
|
|
is thousands of rows and the reviewer sees one page of it.
|
|
"""
|
|
latest = latest_reviews().subquery()
|
|
projections = (
|
|
select(ExifProjection.asset_id, ExifProjection.state.label("exif_state"))
|
|
.where(ExifProjection.stage == "safety")
|
|
.subquery()
|
|
)
|
|
effective = func.coalesce(latest.c.decision, "undecided")
|
|
query = (
|
|
select(
|
|
Asset.id,
|
|
Asset.current_path,
|
|
latest.c.score,
|
|
latest.c.decision,
|
|
latest.c.exif_verified_at,
|
|
projections.c.exif_state,
|
|
)
|
|
.select_from(Asset)
|
|
.join(latest, latest.c.asset_id == Asset.id, isouter=True)
|
|
.join(projections, projections.c.asset_id == Asset.id, isouter=True)
|
|
.where(
|
|
Asset.canonical_asset_id.is_(None),
|
|
Asset.availability_state == "active",
|
|
)
|
|
)
|
|
if state:
|
|
query = query.where(effective == state)
|
|
with self._session_factory() as session:
|
|
total = int(
|
|
session.scalar(select(func.count()).select_from(query.subquery())) or 0
|
|
)
|
|
rows = session.execute(
|
|
query.order_by(Asset.current_path).limit(limit).offset(offset)
|
|
).all()
|
|
return {
|
|
"total": total,
|
|
"items": [
|
|
{
|
|
"asset_id": asset_id,
|
|
"current_path": current_path,
|
|
"score": score,
|
|
"decision": decision,
|
|
"suggested": classify(score) if score is not None else None,
|
|
"exif_verified": bool(exif_verified_at),
|
|
"exif_state": exif_state,
|
|
}
|
|
for asset_id, current_path, score, decision, exif_verified_at, exif_state in rows
|
|
],
|
|
}
|
|
|
|
def scorable_asset_ids(self) -> list[str]:
|
|
"""Canonical active assets with a path — the items a scoring job enqueues."""
|
|
with self._session_factory() as session:
|
|
return [a.id for a in session.scalars(_eligible_assets_query()) if a.current_path]
|
|
|
|
# -- writes ---------------------------------------------------------------
|
|
def score_assets(self, asset_ids: list[str] | None = None) -> int:
|
|
"""Score eligible assets with the local model and persist scored reviews.
|
|
|
|
Records score-only reviews (no decision) so the reviewer can sort by risk.
|
|
Returns the number scored. Assets already carrying a human decision keep it.
|
|
"""
|
|
with self._session_factory() as session:
|
|
query = _eligible_assets_query()
|
|
if asset_ids is not None:
|
|
query = query.where(Asset.id.in_(asset_ids))
|
|
assets = [a for a in session.scalars(query) if a.current_path]
|
|
if not assets:
|
|
return 0
|
|
model = self._model or _default_model()
|
|
by_path = {a.current_path: a.id for a in assets}
|
|
scored = model.score(list(by_path))
|
|
now = _now()
|
|
with self._session_factory() as session:
|
|
for path, score in scored:
|
|
session.add(
|
|
SafetyReview(
|
|
id=str(uuid.uuid4()), asset_id=by_path[path], score=float(score), created_at=now
|
|
)
|
|
)
|
|
session.commit()
|
|
return len(scored)
|
|
|
|
def decide(self, asset_id: str, decision: str, *, reviewer: str = "user", write_exif: bool = True) -> dict:
|
|
"""Persist a human safety decision and, for sfw/nsfw, run the EXIF checkpoint.
|
|
|
|
The decision is durable regardless of EXIF; ``exif_verified`` becomes true
|
|
only after the mutually-exclusive keyword is written and read back, and the
|
|
file's new SHA-256 is stored — that verified state is what upload eligibility
|
|
later depends on.
|
|
"""
|
|
if decision not in DECISIONS:
|
|
raise SafetyError(f"invalid decision {decision!r}")
|
|
with self._session_factory() as session:
|
|
asset = session.get(Asset, asset_id)
|
|
if asset is None:
|
|
raise SafetyError(f"unknown asset {asset_id}")
|
|
prior = self._latest_by_asset(session).get(asset_id)
|
|
path = asset.current_path
|
|
|
|
exif_verified_at = None
|
|
result_sha256 = None
|
|
result_byte_size = None
|
|
if write_exif and decision in (SFW, NSFW) and path:
|
|
ops = exif_projection(decision)
|
|
# The full checkpoint: write the owned keyword, read the whole file back,
|
|
# and prove every field this stage does not own survived. A divergent
|
|
# result is recorded and left alone — it must not count as verified, so
|
|
# upload stays blocked until a human decides (US07-03).
|
|
result = exif_checkpoint.run(
|
|
path, add=tuple(ops["add"]), remove=tuple(ops["remove"])
|
|
)
|
|
exif_checkpoint.record(
|
|
self._session_factory,
|
|
asset_id=asset_id,
|
|
stage="safety",
|
|
result=result,
|
|
add=tuple(ops["add"]),
|
|
remove=tuple(ops["remove"]),
|
|
)
|
|
if result.verified:
|
|
exif_verified_at = result.verified_at
|
|
result_sha256 = result.sha256
|
|
result_byte_size = result.byte_size
|
|
|
|
now = _now()
|
|
with self._session_factory() as session:
|
|
review = SafetyReview(
|
|
id=str(uuid.uuid4()),
|
|
asset_id=asset_id,
|
|
decision=decision,
|
|
prior_decision=prior.decision if prior else None,
|
|
reviewer=reviewer,
|
|
score=prior.score if prior else None,
|
|
exif_verified_at=exif_verified_at,
|
|
result_sha256=result_sha256,
|
|
created_at=now,
|
|
)
|
|
session.add(review)
|
|
if result_sha256:
|
|
asset = session.get(Asset, asset_id)
|
|
asset.current_sha256 = result_sha256
|
|
if result_byte_size is not None:
|
|
asset.byte_size = result_byte_size
|
|
session.commit()
|
|
return {
|
|
"asset_id": asset_id,
|
|
"decision": decision,
|
|
"prior_decision": prior.decision if prior else None,
|
|
"exif_verified": exif_verified_at is not None,
|
|
}
|
|
|
|
|
|
def latest_reviews():
|
|
"""One row per asset: its current safety review, chosen in SQL.
|
|
|
|
``safety_reviews`` is append-only, so "the decision" is the newest row for an
|
|
asset. A window function picks it without loading the table; ``rowid`` breaks a
|
|
same-timestamp tie the same way the previous last-write-wins loop did.
|
|
"""
|
|
ranked = (
|
|
select(
|
|
SafetyReview.asset_id,
|
|
SafetyReview.decision,
|
|
SafetyReview.score,
|
|
SafetyReview.exif_verified_at,
|
|
func.row_number()
|
|
.over(
|
|
partition_by=SafetyReview.asset_id,
|
|
order_by=(SafetyReview.created_at.desc(), column("rowid").desc()),
|
|
)
|
|
.label("rank"),
|
|
)
|
|
.select_from(SafetyReview)
|
|
.subquery()
|
|
)
|
|
return select(
|
|
ranked.c.asset_id, ranked.c.decision, ranked.c.score, ranked.c.exif_verified_at
|
|
).where(ranked.c.rank == 1)
|
|
|
|
|
|
def _eligible_assets_query():
|
|
"""Canonical, active assets — the safety stage runs only on these.
|
|
|
|
``canonical_asset_id IS NULL`` excludes non-canonical duplicate variants; the
|
|
shared discovery service already excludes ``_IGNORE/``.
|
|
"""
|
|
return select(Asset).where(
|
|
Asset.canonical_asset_id.is_(None),
|
|
Asset.availability_state == "active",
|
|
)
|
|
|
|
|
|
def _default_model():
|
|
from photo_pipeline.integrations.nsfw_model import NsfwModel
|
|
|
|
return NsfwModel()
|