US07-01: Complete Donor Migration and Freeze the CLI Archive (#83)

photo_pipeline/services/legacy_import.py

@@ -0,0 +1,220 @@
+"""Import the last path-keyed CSV state into the database (US07-01).
+
+``nsfwtag`` cached its safety scores in ``nsfw_scores.csv`` next to the library:
+one ``path,nsfw_score`` row per photo, four decimals, unreadable rows dropped. That
+file stops being a source of truth when the CLI is archived, so its scores are
+imported once — as scored-but-unreviewed ``safety_reviews`` rows on the stable
+``assets.id`` each path resolves to — and the CSV is left untouched on disk as its
+own backup.
+
+The import is deliberately conservative, because a score is evidence about a photo
+and a path is not an identity:
+
+- a row whose path matches no known asset is **unmatched**, never a new asset;
+- an asset a human already reviewed is **never** touched: a score is evidence, a
+  decision is a judgement, and an import may not overwrite the second with the
+  first — the difference is reported as a conflict instead;
+- a row for an asset that already carries a score is **skipped** unless
+  ``overwrite`` is asked for, and a differing score is reported either way;
+- an unparsable score or a duplicate path is **rejected** with its reason;
+- rerunning changes nothing that is already imported.
+
+Everything it did — and everything it refused to do — comes back as a
+reconciliation report, which the caller can persist next to the database. The
+donor's own reader is the specification for the format (donor ledger:
+``nt-score-cache``).
+"""
+
+from __future__ import annotations
+
+import csv
+import json
+import uuid
+from collections import Counter
+from dataclasses import asdict, dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+
+from sqlalchemy import select
+from sqlalchemy.orm import sessionmaker
+
+from photo_pipeline.models import Asset, AssetPath, SafetyReview
+
+REPORT_VERSION = 1
+# The donor wrote scores as four-decimal strings; anything outside 0..1 was never
+# something it produced, so it is data corruption rather than a score.
+SCORE_RANGE = (0.0, 1.0)
+# Recorded as the reviewer so an imported score is never mistaken for a judgement.
+REVIEWER = "legacy-csv-import"
+
+
+@dataclass
+class ImportReport:
+    source: str
+    rows: int = 0
+    imported: int = 0
+    skipped_existing: int = 0
+    unchanged: int = 0
+    reviewed: int = 0
+    unmatched: list[str] = field(default_factory=list)
+    rejected: list[dict] = field(default_factory=list)
+    conflicts: list[dict] = field(default_factory=list)
+    generated_at: str = ""
+    schema_version: int = REPORT_VERSION
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+    @property
+    def counts(self) -> dict[str, int]:
+        return {
+            "rows": self.rows,
+            "imported": self.imported,
+            "skipped_existing": self.skipped_existing,
+            "unchanged": self.unchanged,
+            "reviewed": self.reviewed,
+            "unmatched": len(self.unmatched),
+            "rejected": len(self.rejected),
+            "conflicts": len(self.conflicts),
+        }
+
+
+class LegacyImportService:
+    def __init__(self, session_factory: sessionmaker) -> None:
+        self._session_factory = session_factory
+
+    def import_nsfw_scores(
+        self, csv_path: Path | str, *, overwrite: bool = False, dry_run: bool = False
+    ) -> ImportReport:
+        """Import ``nsfw_scores.csv`` onto asset identity and report what happened."""
+        path = Path(csv_path)
+        report = ImportReport(source=str(path))
+        if not path.is_file():
+            report.rejected.append({"path": str(path), "reason": "csv_missing"})
+            return self._stamp(report)
+
+        with self._session_factory() as session:
+            # Paths are matched against every occurrence an asset ever had, so a
+            # photo scored before a rename is still recognised.
+            by_path = self._path_index(session)
+            latest = self._latest_reviews(session)
+            seen: Counter[str] = Counter()
+
+            for row in self._rows(path, report):
+                report.rows += 1
+                raw_path, raw_score = row
+                seen[raw_path] += 1
+                if seen[raw_path] > 1:
+                    report.rejected.append({"path": raw_path, "reason": "duplicate_path"})
+                    continue
+                score = _parse_score(raw_score)
+                if score is None:
+                    report.rejected.append(
+                        {"path": raw_path, "reason": "unparsable_score", "value": raw_score}
+                    )
+                    continue
+                asset_id = by_path.get(raw_path)
+                if asset_id is None:
+                    report.unmatched.append(raw_path)
+                    continue
+
+                current = latest.get(asset_id)
+                if current is not None and current.decision:
+                    # A human decided this one; the CSV is older evidence.
+                    report.reviewed += 1
+                    if current.score is None or abs(current.score - score) >= 1e-9:
+                        report.conflicts.append(
+                            {
+                                "path": raw_path,
+                                "asset_id": asset_id,
+                                "current": current.score,
+                                "decision": current.decision,
+                                "csv": score,
+                            }
+                        )
+                    continue
+                if current is not None and current.score is not None:
+                    if abs(current.score - score) < 1e-9:
+                        report.unchanged += 1
+                        continue
+                    report.conflicts.append(
+                        {
+                            "path": raw_path,
+                            "asset_id": asset_id,
+                            "current": current.score,
+                            "decision": None,
+                            "csv": score,
+                        }
+                    )
+                    if not overwrite:
+                        report.skipped_existing += 1
+                        continue
+                session.add(
+                    SafetyReview(
+                        id=str(uuid.uuid4()),
+                        asset_id=asset_id,
+                        score=score,
+                        reviewer=REVIEWER,
+                    )
+                )
+                latest[asset_id] = SafetyReview(asset_id=asset_id, score=score)
+                report.imported += 1
+
+            if dry_run:
+                session.rollback()
+            else:
+                session.commit()
+        return self._stamp(report)
+
+    @staticmethod
+    def _rows(path: Path, report: ImportReport):
+        """Yield ``(path, score)`` pairs, tolerating the donor's own sloppiness."""
+        with open(path, newline="", encoding="utf-8", errors="replace") as handle:
+            for row in csv.DictReader(handle):
+                raw_path = (row.get("path") or "").strip()
+                if not raw_path:
+                    report.rejected.append({"path": "", "reason": "missing_path"})
+                    continue
+                yield raw_path, row.get("nsfw_score")
+
+    @staticmethod
+    def _latest_reviews(session) -> dict[str, SafetyReview]:
+        """The current safety row per asset — latest wins, as everywhere else."""
+        latest: dict[str, SafetyReview] = {}
+        for review in session.scalars(select(SafetyReview).order_by(SafetyReview.created_at)):
+            latest[review.asset_id] = review
+        return latest
+
+    @staticmethod
+    def _path_index(session) -> dict[str, str]:
+        """Every path an asset is or was known by → its stable id."""
+        index: dict[str, str] = {}
+        for asset_id, path in session.execute(select(AssetPath.asset_id, AssetPath.path)):
+            index.setdefault(path, asset_id)
+        for asset_id, path in session.execute(select(Asset.id, Asset.current_path)):
+            if path:
+                index[path] = asset_id  # the current path wins over a closed one
+        return index
+
+    @staticmethod
+    def _stamp(report: ImportReport) -> ImportReport:
+        report.generated_at = datetime.now(timezone.utc).isoformat()
+        return report
+
+
+def write_report(report: ImportReport, directory: Path) -> Path:
+    """Persist the reconciliation report; the import is not evidence until it is."""
+    directory.mkdir(parents=True, exist_ok=True)
+    path = directory / "legacy-nsfw-import.json"
+    path.write_text(json.dumps(report.to_dict(), indent=2, sort_keys=True), encoding="utf-8")
+    return path
+
+
+def _parse_score(value) -> float | None:
+    try:
+        score = float(value)
+    except (TypeError, ValueError):
+        return None
+    if not SCORE_RANGE[0] <= score <= SCORE_RANGE[1]:
+        return None
+    return score