221 lines
8.6 KiB
Python
221 lines
8.6 KiB
Python
"""Import the last path-keyed CSV state into the database (US07-01).
|
|
|
|
``nsfwtag`` cached its safety scores in ``nsfw_scores.csv`` next to the library:
|
|
one ``path,nsfw_score`` row per photo, four decimals, unreadable rows dropped. That
|
|
file stops being a source of truth when the CLI is archived, so its scores are
|
|
imported once — as scored-but-unreviewed ``safety_reviews`` rows on the stable
|
|
``assets.id`` each path resolves to — and the CSV is left untouched on disk as its
|
|
own backup.
|
|
|
|
The import is deliberately conservative, because a score is evidence about a photo
|
|
and a path is not an identity:
|
|
|
|
- a row whose path matches no known asset is **unmatched**, never a new asset;
|
|
- an asset a human already reviewed is **never** touched: a score is evidence, a
|
|
decision is a judgement, and an import may not overwrite the second with the
|
|
first — the difference is reported as a conflict instead;
|
|
- a row for an asset that already carries a score is **skipped** unless
|
|
``overwrite`` is asked for, and a differing score is reported either way;
|
|
- an unparsable score or a duplicate path is **rejected** with its reason;
|
|
- rerunning changes nothing that is already imported.
|
|
|
|
Everything it did — and everything it refused to do — comes back as a
|
|
reconciliation report, which the caller can persist next to the database. The
|
|
donor's own reader is the specification for the format (donor ledger:
|
|
``nt-score-cache``).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import csv
|
|
import json
|
|
import uuid
|
|
from collections import Counter
|
|
from dataclasses import asdict, dataclass, field
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import sessionmaker
|
|
|
|
from photo_pipeline.models import Asset, AssetPath, SafetyReview
|
|
|
|
REPORT_VERSION = 1
|
|
# The donor wrote scores as four-decimal strings; anything outside 0..1 was never
|
|
# something it produced, so it is data corruption rather than a score.
|
|
SCORE_RANGE = (0.0, 1.0)
|
|
# Recorded as the reviewer so an imported score is never mistaken for a judgement.
|
|
REVIEWER = "legacy-csv-import"
|
|
|
|
|
|
@dataclass
|
|
class ImportReport:
|
|
source: str
|
|
rows: int = 0
|
|
imported: int = 0
|
|
skipped_existing: int = 0
|
|
unchanged: int = 0
|
|
reviewed: int = 0
|
|
unmatched: list[str] = field(default_factory=list)
|
|
rejected: list[dict] = field(default_factory=list)
|
|
conflicts: list[dict] = field(default_factory=list)
|
|
generated_at: str = ""
|
|
schema_version: int = REPORT_VERSION
|
|
|
|
def to_dict(self) -> dict:
|
|
return asdict(self)
|
|
|
|
@property
|
|
def counts(self) -> dict[str, int]:
|
|
return {
|
|
"rows": self.rows,
|
|
"imported": self.imported,
|
|
"skipped_existing": self.skipped_existing,
|
|
"unchanged": self.unchanged,
|
|
"reviewed": self.reviewed,
|
|
"unmatched": len(self.unmatched),
|
|
"rejected": len(self.rejected),
|
|
"conflicts": len(self.conflicts),
|
|
}
|
|
|
|
|
|
class LegacyImportService:
|
|
def __init__(self, session_factory: sessionmaker) -> None:
|
|
self._session_factory = session_factory
|
|
|
|
def import_nsfw_scores(
|
|
self, csv_path: Path | str, *, overwrite: bool = False, dry_run: bool = False
|
|
) -> ImportReport:
|
|
"""Import ``nsfw_scores.csv`` onto asset identity and report what happened."""
|
|
path = Path(csv_path)
|
|
report = ImportReport(source=str(path))
|
|
if not path.is_file():
|
|
report.rejected.append({"path": str(path), "reason": "csv_missing"})
|
|
return self._stamp(report)
|
|
|
|
with self._session_factory() as session:
|
|
# Paths are matched against every occurrence an asset ever had, so a
|
|
# photo scored before a rename is still recognised.
|
|
by_path = self._path_index(session)
|
|
latest = self._latest_reviews(session)
|
|
seen: Counter[str] = Counter()
|
|
|
|
for row in self._rows(path, report):
|
|
report.rows += 1
|
|
raw_path, raw_score = row
|
|
seen[raw_path] += 1
|
|
if seen[raw_path] > 1:
|
|
report.rejected.append({"path": raw_path, "reason": "duplicate_path"})
|
|
continue
|
|
score = _parse_score(raw_score)
|
|
if score is None:
|
|
report.rejected.append(
|
|
{"path": raw_path, "reason": "unparsable_score", "value": raw_score}
|
|
)
|
|
continue
|
|
asset_id = by_path.get(raw_path)
|
|
if asset_id is None:
|
|
report.unmatched.append(raw_path)
|
|
continue
|
|
|
|
current = latest.get(asset_id)
|
|
if current is not None and current.decision:
|
|
# A human decided this one; the CSV is older evidence.
|
|
report.reviewed += 1
|
|
if current.score is None or abs(current.score - score) >= 1e-9:
|
|
report.conflicts.append(
|
|
{
|
|
"path": raw_path,
|
|
"asset_id": asset_id,
|
|
"current": current.score,
|
|
"decision": current.decision,
|
|
"csv": score,
|
|
}
|
|
)
|
|
continue
|
|
if current is not None and current.score is not None:
|
|
if abs(current.score - score) < 1e-9:
|
|
report.unchanged += 1
|
|
continue
|
|
report.conflicts.append(
|
|
{
|
|
"path": raw_path,
|
|
"asset_id": asset_id,
|
|
"current": current.score,
|
|
"decision": None,
|
|
"csv": score,
|
|
}
|
|
)
|
|
if not overwrite:
|
|
report.skipped_existing += 1
|
|
continue
|
|
session.add(
|
|
SafetyReview(
|
|
id=str(uuid.uuid4()),
|
|
asset_id=asset_id,
|
|
score=score,
|
|
reviewer=REVIEWER,
|
|
)
|
|
)
|
|
latest[asset_id] = SafetyReview(asset_id=asset_id, score=score)
|
|
report.imported += 1
|
|
|
|
if dry_run:
|
|
session.rollback()
|
|
else:
|
|
session.commit()
|
|
return self._stamp(report)
|
|
|
|
@staticmethod
|
|
def _rows(path: Path, report: ImportReport):
|
|
"""Yield ``(path, score)`` pairs, tolerating the donor's own sloppiness."""
|
|
with open(path, newline="", encoding="utf-8", errors="replace") as handle:
|
|
for row in csv.DictReader(handle):
|
|
raw_path = (row.get("path") or "").strip()
|
|
if not raw_path:
|
|
report.rejected.append({"path": "", "reason": "missing_path"})
|
|
continue
|
|
yield raw_path, row.get("nsfw_score")
|
|
|
|
@staticmethod
|
|
def _latest_reviews(session) -> dict[str, SafetyReview]:
|
|
"""The current safety row per asset — latest wins, as everywhere else."""
|
|
latest: dict[str, SafetyReview] = {}
|
|
for review in session.scalars(select(SafetyReview).order_by(SafetyReview.created_at)):
|
|
latest[review.asset_id] = review
|
|
return latest
|
|
|
|
@staticmethod
|
|
def _path_index(session) -> dict[str, str]:
|
|
"""Every path an asset is or was known by → its stable id."""
|
|
index: dict[str, str] = {}
|
|
for asset_id, path in session.execute(select(AssetPath.asset_id, AssetPath.path)):
|
|
index.setdefault(path, asset_id)
|
|
for asset_id, path in session.execute(select(Asset.id, Asset.current_path)):
|
|
if path:
|
|
index[path] = asset_id # the current path wins over a closed one
|
|
return index
|
|
|
|
@staticmethod
|
|
def _stamp(report: ImportReport) -> ImportReport:
|
|
report.generated_at = datetime.now(timezone.utc).isoformat()
|
|
return report
|
|
|
|
|
|
def write_report(report: ImportReport, directory: Path) -> Path:
|
|
"""Persist the reconciliation report; the import is not evidence until it is."""
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
path = directory / "legacy-nsfw-import.json"
|
|
path.write_text(json.dumps(report.to_dict(), indent=2, sort_keys=True), encoding="utf-8")
|
|
return path
|
|
|
|
|
|
def _parse_score(value) -> float | None:
|
|
try:
|
|
score = float(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
if not SCORE_RANGE[0] <= score <= SCORE_RANGE[1]:
|
|
return None
|
|
return score
|