239 lines
8.4 KiB
Python
239 lines
8.4 KiB
Python
"""Importing the archived CLI's CSV state into the database (US07-01).
|
|
|
|
The donor's ``nsfw_scores.csv`` was keyed by path; the database is keyed by a
|
|
stable asset id. Every case here is about that gap: a path that moved, a path that
|
|
matches nothing, a score a human has already overruled, and a file that is simply
|
|
malformed. The import may add evidence and must never invent an asset, overwrite a
|
|
judgement, or fail silently — whatever it does ends up in the report.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import uuid
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from sqlalchemy import select
|
|
|
|
from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
|
|
from photo_pipeline.models import Asset, AssetPath, SafetyReview
|
|
from photo_pipeline.services.legacy_import import (
|
|
REVIEWER,
|
|
LegacyImportService,
|
|
write_report,
|
|
)
|
|
|
|
NOW = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
|
|
|
|
|
@pytest.fixture
|
|
def factory(tmp_path):
|
|
url = f"sqlite:///{tmp_path / 'legacy.db'}"
|
|
run_migrations(url)
|
|
engine = create_db_engine(url)
|
|
yield create_session_factory(engine)
|
|
engine.dispose()
|
|
|
|
|
|
def _asset(sf, path: Path, *, previous: Path | None = None) -> str:
|
|
asset_id = str(uuid.uuid4())
|
|
with sf() as session:
|
|
session.add(
|
|
Asset(
|
|
id=asset_id,
|
|
original_path=str(previous or path),
|
|
current_path=str(path),
|
|
discovered_at=NOW,
|
|
hash_version=1,
|
|
)
|
|
)
|
|
session.add(AssetPath(asset_id=asset_id, path=str(path), valid_from=NOW))
|
|
if previous is not None:
|
|
session.add(
|
|
AssetPath(
|
|
asset_id=asset_id, path=str(previous), valid_from=NOW, valid_until=NOW
|
|
)
|
|
)
|
|
session.commit()
|
|
return asset_id
|
|
|
|
|
|
def _review(sf, asset_id: str, *, score=None, decision=None, at=NOW) -> None:
|
|
with sf() as session:
|
|
session.add(
|
|
SafetyReview(
|
|
id=str(uuid.uuid4()),
|
|
asset_id=asset_id,
|
|
score=score,
|
|
decision=decision,
|
|
reviewer="dom",
|
|
created_at=at,
|
|
)
|
|
)
|
|
session.commit()
|
|
|
|
|
|
def _csv(tmp_path, rows: str) -> Path:
|
|
path = tmp_path / "nsfw_scores.csv"
|
|
path.write_text("path,nsfw_score\n" + rows, encoding="utf-8")
|
|
return path
|
|
|
|
|
|
def _scores(sf) -> dict[str, float]:
|
|
with sf() as session:
|
|
return {
|
|
review.asset_id: review.score
|
|
for review in session.scalars(
|
|
select(SafetyReview).order_by(SafetyReview.created_at)
|
|
)
|
|
}
|
|
|
|
|
|
def test_scores_are_imported_onto_asset_identity(tmp_path, factory):
|
|
a = _asset(factory, tmp_path / "a.jpg")
|
|
# This one was scored under its old path and has since moved.
|
|
b = _asset(factory, tmp_path / "moved" / "b.jpg", previous=tmp_path / "b.jpg")
|
|
csv_path = _csv(
|
|
tmp_path, f"{tmp_path / 'a.jpg'},0.9123\n{tmp_path / 'b.jpg'},0.0100\n"
|
|
)
|
|
|
|
report = LegacyImportService(factory).import_nsfw_scores(csv_path)
|
|
|
|
assert report.counts == {
|
|
"rows": 2,
|
|
"imported": 2,
|
|
"skipped_existing": 0,
|
|
"unchanged": 0,
|
|
"reviewed": 0,
|
|
"unmatched": 0,
|
|
"rejected": 0,
|
|
"conflicts": 0,
|
|
}
|
|
assert _scores(factory) == {a: 0.9123, b: 0.0100}
|
|
# The imported rows are evidence, not judgements: no decision is invented.
|
|
with factory() as session:
|
|
rows = list(session.scalars(select(SafetyReview)))
|
|
assert {r.decision for r in rows} == {None}
|
|
assert {r.reviewer for r in rows} == {REVIEWER}
|
|
assert csv_path.exists(), "the CSV is left on disk as its own backup"
|
|
|
|
|
|
def test_a_reviewed_asset_is_never_overwritten_by_the_csv(tmp_path, factory):
|
|
asset_id = _asset(factory, tmp_path / "a.jpg")
|
|
_review(factory, asset_id, score=0.2, decision="sfw")
|
|
csv_path = _csv(tmp_path, f"{tmp_path / 'a.jpg'},0.9999\n")
|
|
|
|
report = LegacyImportService(factory).import_nsfw_scores(csv_path, overwrite=True)
|
|
|
|
assert (report.imported, report.reviewed) == (0, 1)
|
|
assert report.conflicts[0]["decision"] == "sfw"
|
|
assert report.conflicts[0]["csv"] == 0.9999
|
|
with factory() as session:
|
|
rows = list(session.scalars(select(SafetyReview)))
|
|
assert len(rows) == 1 and rows[0].decision == "sfw"
|
|
|
|
|
|
def test_an_unknown_path_is_reported_never_turned_into_an_asset(tmp_path, factory):
|
|
_asset(factory, tmp_path / "a.jpg")
|
|
csv_path = _csv(
|
|
tmp_path, f"{tmp_path / 'a.jpg'},0.5000\n{tmp_path / 'ghost.jpg'},0.5000\n"
|
|
)
|
|
|
|
report = LegacyImportService(factory).import_nsfw_scores(csv_path)
|
|
|
|
assert report.unmatched == [str(tmp_path / "ghost.jpg")]
|
|
with factory() as session:
|
|
assert session.scalar(select(Asset).where(Asset.current_path.like("%ghost%"))) is None
|
|
|
|
|
|
def test_malformed_rows_are_rejected_with_their_reason(tmp_path, factory):
|
|
_asset(factory, tmp_path / "a.jpg")
|
|
csv_path = _csv(
|
|
tmp_path,
|
|
f"{tmp_path / 'a.jpg'},0.5000\n"
|
|
f"{tmp_path / 'a.jpg'},0.5000\n" # the same path twice
|
|
f"{tmp_path / 'b.jpg'},not-a-number\n"
|
|
f"{tmp_path / 'c.jpg'},7.5\n" # outside 0..1: corruption, not a score
|
|
",0.5\n", # no path at all
|
|
)
|
|
|
|
report = LegacyImportService(factory).import_nsfw_scores(csv_path)
|
|
|
|
reasons = sorted(r["reason"] for r in report.rejected)
|
|
assert reasons == ["duplicate_path", "missing_path", "unparsable_score", "unparsable_score"]
|
|
assert report.imported == 1
|
|
|
|
|
|
def test_rerunning_changes_nothing_and_a_differing_score_needs_overwrite(tmp_path, factory):
|
|
asset_id = _asset(factory, tmp_path / "a.jpg")
|
|
csv_path = _csv(tmp_path, f"{tmp_path / 'a.jpg'},0.5000\n")
|
|
service = LegacyImportService(factory)
|
|
service.import_nsfw_scores(csv_path)
|
|
|
|
again = service.import_nsfw_scores(csv_path)
|
|
assert (again.imported, again.unchanged) == (0, 1)
|
|
assert len(_scores(factory)) == 1
|
|
|
|
changed = _csv(tmp_path, f"{tmp_path / 'a.jpg'},0.8000\n")
|
|
refused = service.import_nsfw_scores(changed)
|
|
assert (refused.imported, refused.skipped_existing) == (0, 1)
|
|
assert refused.conflicts[0]["current"] == 0.5
|
|
|
|
forced = service.import_nsfw_scores(changed, overwrite=True)
|
|
assert forced.imported == 1
|
|
with factory() as session:
|
|
latest = list(session.scalars(select(SafetyReview).order_by(SafetyReview.created_at)))[-1]
|
|
assert (latest.asset_id, latest.score) == (asset_id, 0.8)
|
|
|
|
|
|
def test_a_dry_run_reports_without_writing(tmp_path, factory):
|
|
_asset(factory, tmp_path / "a.jpg")
|
|
csv_path = _csv(tmp_path, f"{tmp_path / 'a.jpg'},0.5000\n")
|
|
|
|
report = LegacyImportService(factory).import_nsfw_scores(csv_path, dry_run=True)
|
|
|
|
assert report.imported == 1
|
|
assert _scores(factory) == {}, "a dry run must leave the database alone"
|
|
|
|
|
|
def test_a_missing_csv_is_a_reported_outcome_not_a_crash(tmp_path, factory):
|
|
report = LegacyImportService(factory).import_nsfw_scores(tmp_path / "nothing.csv")
|
|
assert report.rejected == [{"path": str(tmp_path / "nothing.csv"), "reason": "csv_missing"}]
|
|
assert report.rows == 0
|
|
|
|
|
|
def test_the_report_is_written_where_it_can_be_audited(tmp_path, factory):
|
|
_asset(factory, tmp_path / "a.jpg")
|
|
csv_path = _csv(tmp_path, f"{tmp_path / 'a.jpg'},0.5000\n")
|
|
report = LegacyImportService(factory).import_nsfw_scores(csv_path)
|
|
|
|
written = write_report(report, tmp_path / "data")
|
|
|
|
payload = json.loads(written.read_text(encoding="utf-8"))
|
|
assert payload["imported"] == 1
|
|
assert payload["source"] == str(csv_path)
|
|
assert payload["schema_version"] == 1
|
|
assert payload["generated_at"]
|
|
|
|
|
|
def test_a_later_review_still_wins_after_an_import(tmp_path, factory):
|
|
"""The import is a floor, not a ceiling: a human decision made afterwards is
|
|
the current state, exactly as everywhere else in the app."""
|
|
asset_id = _asset(factory, tmp_path / "a.jpg")
|
|
csv_path = _csv(tmp_path, f"{tmp_path / 'a.jpg'},0.9000\n")
|
|
LegacyImportService(factory).import_nsfw_scores(csv_path)
|
|
# The import stamps itself with the wall clock, so "afterwards" has to be too.
|
|
_review(
|
|
factory,
|
|
asset_id,
|
|
score=0.9,
|
|
decision="sfw",
|
|
at=datetime.now(timezone.utc) + timedelta(hours=1),
|
|
)
|
|
|
|
with factory() as session:
|
|
latest = list(session.scalars(select(SafetyReview).order_by(SafetyReview.created_at)))[-1]
|
|
assert latest.decision == "sfw"
|