"""Import the last path-keyed CSV state into the database (US07-01). ``nsfwtag`` cached its safety scores in ``nsfw_scores.csv`` next to the library: one ``path,nsfw_score`` row per photo, four decimals, unreadable rows dropped. That file stops being a source of truth when the CLI is archived, so its scores are imported once — as scored-but-unreviewed ``safety_reviews`` rows on the stable ``assets.id`` each path resolves to — and the CSV is left untouched on disk as its own backup. The import is deliberately conservative, because a score is evidence about a photo and a path is not an identity: - a row whose path matches no known asset is **unmatched**, never a new asset; - an asset a human already reviewed is **never** touched: a score is evidence, a decision is a judgement, and an import may not overwrite the second with the first — the difference is reported as a conflict instead; - a row for an asset that already carries a score is **skipped** unless ``overwrite`` is asked for, and a differing score is reported either way; - an unparsable score or a duplicate path is **rejected** with its reason; - rerunning changes nothing that is already imported. Everything it did — and everything it refused to do — comes back as a reconciliation report, which the caller can persist next to the database. The donor's own reader is the specification for the format (donor ledger: ``nt-score-cache``). """ from __future__ import annotations import csv import json import uuid from collections import Counter from dataclasses import asdict, dataclass, field from datetime import datetime, timezone from pathlib import Path from sqlalchemy import select from sqlalchemy.orm import sessionmaker from photo_pipeline.models import Asset, AssetPath, SafetyReview REPORT_VERSION = 1 # The donor wrote scores as four-decimal strings; anything outside 0..1 was never # something it produced, so it is data corruption rather than a score. SCORE_RANGE = (0.0, 1.0) # Recorded as the reviewer so an imported score is never mistaken for a judgement. REVIEWER = "legacy-csv-import" @dataclass class ImportReport: source: str rows: int = 0 imported: int = 0 skipped_existing: int = 0 unchanged: int = 0 reviewed: int = 0 unmatched: list[str] = field(default_factory=list) rejected: list[dict] = field(default_factory=list) conflicts: list[dict] = field(default_factory=list) generated_at: str = "" schema_version: int = REPORT_VERSION def to_dict(self) -> dict: return asdict(self) @property def counts(self) -> dict[str, int]: return { "rows": self.rows, "imported": self.imported, "skipped_existing": self.skipped_existing, "unchanged": self.unchanged, "reviewed": self.reviewed, "unmatched": len(self.unmatched), "rejected": len(self.rejected), "conflicts": len(self.conflicts), } class LegacyImportService: def __init__(self, session_factory: sessionmaker) -> None: self._session_factory = session_factory def import_nsfw_scores( self, csv_path: Path | str, *, overwrite: bool = False, dry_run: bool = False ) -> ImportReport: """Import ``nsfw_scores.csv`` onto asset identity and report what happened.""" path = Path(csv_path) report = ImportReport(source=str(path)) if not path.is_file(): report.rejected.append({"path": str(path), "reason": "csv_missing"}) return self._stamp(report) with self._session_factory() as session: # Paths are matched against every occurrence an asset ever had, so a # photo scored before a rename is still recognised. by_path = self._path_index(session) latest = self._latest_reviews(session) seen: Counter[str] = Counter() for row in self._rows(path, report): report.rows += 1 raw_path, raw_score = row seen[raw_path] += 1 if seen[raw_path] > 1: report.rejected.append({"path": raw_path, "reason": "duplicate_path"}) continue score = _parse_score(raw_score) if score is None: report.rejected.append( {"path": raw_path, "reason": "unparsable_score", "value": raw_score} ) continue asset_id = by_path.get(raw_path) if asset_id is None: report.unmatched.append(raw_path) continue current = latest.get(asset_id) if current is not None and current.decision: # A human decided this one; the CSV is older evidence. report.reviewed += 1 if current.score is None or abs(current.score - score) >= 1e-9: report.conflicts.append( { "path": raw_path, "asset_id": asset_id, "current": current.score, "decision": current.decision, "csv": score, } ) continue if current is not None and current.score is not None: if abs(current.score - score) < 1e-9: report.unchanged += 1 continue report.conflicts.append( { "path": raw_path, "asset_id": asset_id, "current": current.score, "decision": None, "csv": score, } ) if not overwrite: report.skipped_existing += 1 continue session.add( SafetyReview( id=str(uuid.uuid4()), asset_id=asset_id, score=score, reviewer=REVIEWER, ) ) latest[asset_id] = SafetyReview(asset_id=asset_id, score=score) report.imported += 1 if dry_run: session.rollback() else: session.commit() return self._stamp(report) @staticmethod def _rows(path: Path, report: ImportReport): """Yield ``(path, score)`` pairs, tolerating the donor's own sloppiness.""" with open(path, newline="", encoding="utf-8", errors="replace") as handle: for row in csv.DictReader(handle): raw_path = (row.get("path") or "").strip() if not raw_path: report.rejected.append({"path": "", "reason": "missing_path"}) continue yield raw_path, row.get("nsfw_score") @staticmethod def _latest_reviews(session) -> dict[str, SafetyReview]: """The current safety row per asset — latest wins, as everywhere else.""" latest: dict[str, SafetyReview] = {} for review in session.scalars(select(SafetyReview).order_by(SafetyReview.created_at)): latest[review.asset_id] = review return latest @staticmethod def _path_index(session) -> dict[str, str]: """Every path an asset is or was known by → its stable id.""" index: dict[str, str] = {} for asset_id, path in session.execute(select(AssetPath.asset_id, AssetPath.path)): index.setdefault(path, asset_id) for asset_id, path in session.execute(select(Asset.id, Asset.current_path)): if path: index[path] = asset_id # the current path wins over a closed one return index @staticmethod def _stamp(report: ImportReport) -> ImportReport: report.generated_at = datetime.now(timezone.utc).isoformat() return report def write_report(report: ImportReport, directory: Path) -> Path: """Persist the reconciliation report; the import is not evidence until it is.""" directory.mkdir(parents=True, exist_ok=True) path = directory / "legacy-nsfw-import.json" path.write_text(json.dumps(report.to_dict(), indent=2, sort_keys=True), encoding="utf-8") return path def _parse_score(value) -> float | None: try: score = float(value) except (TypeError, ValueError): return None if not SCORE_RANGE[0] <= score <= SCORE_RANGE[1]: return None return score