US02-06: Deliver Workflow, Safety, Library, Analysis, and Stats Views
Ports the safety review and the Photo Analyzer Library/Analyze/Stats experiences onto the shared API + service layer, and adds the Workflow home, enforcing the pipeline gates and the one-mutating-job policy. Backend - migration 0005 + models: safety_reviews (append-only, latest row is the current decision) and analysis_results (donor photos schema re-keyed to asset_id). - SafetyService: persist scores/decisions, review queue with filters, and the EXIF safety checkpoint (mutually-exclusive sfw/nsfw keyword written, read back, current_sha256 refreshed) that upload eligibility depends on. - AnalysisService: the privacy gate — the vision provider is called ONLY for canonical, confirmed-SFW assets; nsfw/undecided are recorded skipped without a request. Provider is an injected adapter (real OpenAI-compatible Gemini call extracted from photo_analyzer.analyze_image; a fake in tests). - LibraryService: Library search + Stats read model ported from webapp/query.py (LIKE search in place of FTS5; facets, top tags, years, albums, people). - WorkflowService + GET /api/v1/workflow: per-stage readiness derived from the source tables — counts, blockers, last-run, action, and an active_job that drives read-only-during-jobs. Safety scoring and analysis run as durable jobs under the library_write lock via new domain handlers, so a second mutating job is refused. - routes: workflow, safety (queue/counts/decisions/jobs), analysis (counts/results/jobs), library (assets/facets/stats). Frontend - five views (frontend/js/views.js) on the US02-05 shell: Workflow stepper (status text+icon, not colour alone; actions disabled with a reason while a job runs), Safety review (filter tabs, decide, persists across reload), Library (search + cards), Analyze (counts + live job log via the SSE adapter), Stats. Shared DOM helpers extracted to dom.js; Workflow is the home route. Tests - integration: provider-call privacy (nsfw never reaches the provider), sfw→nsfw flip drops analysis eligibility, decision persistence, one-mutating- job rejection, workflow counts, and the exiftool safety-keyword write/verify. - e2e: Workflow cards, actions disabled+explained during a job, safety decide-persists-across-reload, Library search, Stats, Analyze counts. - traceability map updated for US02-05 and US02-06. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
291
photo_pipeline/services/analysis.py
Normal file
291
photo_pipeline/services/analysis.py
Normal file
@@ -0,0 +1,291 @@
|
||||
"""AnalysisService — content analysis scoped to confirmed-SFW assets.
|
||||
|
||||
The privacy invariant is enforced here and nowhere else: the vision provider is
|
||||
called **only** for canonical assets whose latest safety decision is ``sfw``.
|
||||
Confirmed ``nsfw`` and undecided assets never reach the provider — ``run`` records
|
||||
them as ``skipped_nsfw``/leaves them pending without constructing a request. The
|
||||
provider is an injected adapter so tests assert this with a call-recording fake and
|
||||
no network/key.
|
||||
|
||||
Extracted from photo_analyzer.analyze_image (the OpenAI-compatible Gemini call,
|
||||
result fields, and album-hint prompt) and its ``photos`` schema, re-keyed to
|
||||
``asset_id`` (donor_ledger.yaml: pa-analyze, pa-schema). Retry/rate-limit/throttle
|
||||
bookkeeping from the donor is out of scope for this story.
|
||||
ponytail: port the donor's retry+RPD throttling when analysis runs at real volume.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Protocol
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from photo_pipeline.integrations import exiftool
|
||||
from photo_pipeline.models import AnalysisResult, Asset, SafetyReview
|
||||
from photo_pipeline.services import hashing
|
||||
from photo_pipeline.services.safety import SFW
|
||||
|
||||
MODEL = "gemini-2.5-flash"
|
||||
PROMPT_VERSION = "1"
|
||||
RESULT_FIELDS = (
|
||||
"description",
|
||||
"tags",
|
||||
"people_count",
|
||||
"setting",
|
||||
"time_of_day",
|
||||
"season",
|
||||
"mood",
|
||||
"location_hint",
|
||||
"approx_year",
|
||||
)
|
||||
|
||||
|
||||
class VisionProvider(Protocol):
|
||||
def analyze(self, path: str, *, album_hint: str) -> dict:
|
||||
"""Return the analysis fields for one image. Raises on unrecoverable error."""
|
||||
|
||||
|
||||
class AnalysisError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class AnalysisService:
|
||||
def __init__(self, session_factory: sessionmaker, *, provider: VisionProvider | None = None) -> None:
|
||||
self._session_factory = session_factory
|
||||
self._provider = provider
|
||||
|
||||
def _sfw_asset_ids(self, session) -> set[str]:
|
||||
"""Asset ids whose latest safety decision is ``sfw`` — the ONLY assets that
|
||||
may reach the provider."""
|
||||
latest: dict[str, str | None] = {}
|
||||
for review in session.scalars(select(SafetyReview).order_by(SafetyReview.created_at)):
|
||||
latest[review.asset_id] = review.decision
|
||||
return {aid for aid, decision in latest.items() if decision == SFW}
|
||||
|
||||
def eligible_asset_ids(self) -> list[str]:
|
||||
"""Confirmed-SFW canonical active assets without a completed analysis."""
|
||||
with self._session_factory() as session:
|
||||
sfw = self._sfw_asset_ids(session)
|
||||
if not sfw:
|
||||
return []
|
||||
assets = session.scalars(
|
||||
select(Asset).where(
|
||||
Asset.id.in_(sfw),
|
||||
Asset.canonical_asset_id.is_(None),
|
||||
Asset.availability_state == "active",
|
||||
Asset.current_path.is_not(None),
|
||||
)
|
||||
)
|
||||
done = set(
|
||||
session.scalars(
|
||||
select(AnalysisResult.asset_id).where(AnalysisResult.status == "analyzed")
|
||||
)
|
||||
)
|
||||
return [a.id for a in assets if a.id not in done]
|
||||
|
||||
def counts(self) -> dict[str, int]:
|
||||
with self._session_factory() as session:
|
||||
sfw = self._sfw_asset_ids(session)
|
||||
rows = dict(
|
||||
session.execute(
|
||||
select(AnalysisResult.status, func.count()).group_by(AnalysisResult.status)
|
||||
).all()
|
||||
)
|
||||
analyzed = int(rows.get("analyzed", 0))
|
||||
errored = int(rows.get("error", 0))
|
||||
return {
|
||||
"eligible": len(sfw),
|
||||
"analyzed": analyzed,
|
||||
"error": errored,
|
||||
"pending": max(len(sfw) - analyzed - errored, 0),
|
||||
}
|
||||
|
||||
def run(self, asset_ids: list[str] | None = None) -> dict:
|
||||
"""Analyze the given assets (default: all eligible). Enforces the gate.
|
||||
|
||||
Returns ``{analyzed, skipped, errors}``. ``skipped`` counts assets that were
|
||||
requested but are not confirmed SFW — the provider is never called for them.
|
||||
"""
|
||||
with self._session_factory() as session:
|
||||
sfw = self._sfw_asset_ids(session)
|
||||
paths = {
|
||||
a.id: a.current_path
|
||||
for a in session.scalars(select(Asset).where(Asset.canonical_asset_id.is_(None)))
|
||||
if a.current_path
|
||||
}
|
||||
requested = asset_ids if asset_ids is not None else self.eligible_asset_ids()
|
||||
provider = self._provider or _default_provider()
|
||||
|
||||
analyzed = skipped = errors = 0
|
||||
for asset_id in requested:
|
||||
if asset_id not in sfw:
|
||||
# Gate: not confirmed SFW → never construct a provider request.
|
||||
self._store(asset_id, status="skipped_nsfw", result=None, error=None, tokens=0, raw="")
|
||||
skipped += 1
|
||||
continue
|
||||
path = paths.get(asset_id)
|
||||
if not path:
|
||||
skipped += 1
|
||||
continue
|
||||
try:
|
||||
result = provider.analyze(path, album_hint=_album_hint(path))
|
||||
except Exception as error: # provider/validation failure is per-asset
|
||||
self._store(asset_id, status="error", result=None, error=str(error), tokens=0, raw="")
|
||||
errors += 1
|
||||
continue
|
||||
self._store(
|
||||
asset_id,
|
||||
status="analyzed",
|
||||
result=result,
|
||||
error=None,
|
||||
tokens=int(result.get("_tokens", 0)) if isinstance(result, dict) else 0,
|
||||
raw=json.dumps(result, ensure_ascii=False),
|
||||
)
|
||||
_write_analysis_exif(path, result)
|
||||
analyzed += 1
|
||||
return {"analyzed": analyzed, "skipped": skipped, "errors": errors}
|
||||
|
||||
def _store(self, asset_id, *, status, result, error, tokens, raw) -> None:
|
||||
now = _now()
|
||||
with self._session_factory() as session:
|
||||
row = session.get(AnalysisResult, asset_id) or AnalysisResult(asset_id=asset_id)
|
||||
row.status = status
|
||||
row.error_message = error
|
||||
row.model = MODEL
|
||||
row.prompt_version = PROMPT_VERSION
|
||||
row.tokens_total = tokens
|
||||
row.raw_response = raw or None
|
||||
if status == "analyzed" and isinstance(result, dict):
|
||||
row.description = result.get("description")
|
||||
row.tags = json.dumps(result.get("tags", []), ensure_ascii=False)
|
||||
row.people_count = result.get("people_count")
|
||||
row.setting = result.get("setting")
|
||||
row.time_of_day = result.get("time_of_day")
|
||||
row.season = result.get("season")
|
||||
row.mood = result.get("mood")
|
||||
row.location_hint = result.get("location_hint")
|
||||
row.approx_year = result.get("approx_year")
|
||||
row.analyzed_at = now
|
||||
row.exif_written_at = now
|
||||
session.add(row)
|
||||
session.commit()
|
||||
|
||||
def get(self, asset_id: str) -> dict | None:
|
||||
with self._session_factory() as session:
|
||||
row = session.get(AnalysisResult, asset_id)
|
||||
return _result_dict(row) if row else None
|
||||
|
||||
|
||||
def _album_hint(path: str) -> str:
|
||||
from pathlib import Path
|
||||
|
||||
return Path(path).parent.name
|
||||
|
||||
|
||||
def _write_analysis_exif(path: str, result: dict) -> None:
|
||||
"""Additive analysis keywords into EXIF (Keywords/Subject), preserving safety
|
||||
and user keywords. The donor also wrote a managed caption; only keywords are
|
||||
written here via the shared adapter.
|
||||
ponytail: add the managed ``AI:`` caption segment + read-back preservation check
|
||||
when the analysis EXIF checkpoint is hardened."""
|
||||
tags = result.get("tags") if isinstance(result, dict) else None
|
||||
if tags:
|
||||
exiftool.apply_keywords(path, add=[str(t) for t in tags])
|
||||
|
||||
|
||||
def _result_dict(row: AnalysisResult) -> dict:
|
||||
data = {field: getattr(row, field) for field in RESULT_FIELDS}
|
||||
data["tags"] = json.loads(row.tags) if row.tags else []
|
||||
data.update(
|
||||
asset_id=row.asset_id,
|
||||
status=row.status,
|
||||
model=row.model,
|
||||
prompt_version=row.prompt_version,
|
||||
tokens_total=row.tokens_total,
|
||||
error_message=row.error_message,
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
def _default_provider() -> VisionProvider:
|
||||
return OpenAIVisionProvider()
|
||||
|
||||
|
||||
class OpenAIVisionProvider:
|
||||
"""The real provider: an OpenAI-compatible vision call (Gemini by default).
|
||||
|
||||
Extracted from photo_analyzer.analyze_image. Constructed lazily from env
|
||||
(``OPENAI_API_KEY`` / ``OPENAI_BASE_URL``); never used in tests, which inject a
|
||||
fake. Kept intentionally thin — no retry/throttle bookkeeping (see module note).
|
||||
"""
|
||||
|
||||
def __init__(self, *, model: str = MODEL, client=None) -> None:
|
||||
self._model = model
|
||||
self._client = client
|
||||
|
||||
def _ensure_client(self):
|
||||
if self._client is None:
|
||||
from openai import OpenAI
|
||||
|
||||
self._client = OpenAI()
|
||||
return self._client
|
||||
|
||||
def analyze(self, path: str, *, album_hint: str) -> dict:
|
||||
b64, mime = _prepare_image(path)
|
||||
prompt = ANALYSIS_PROMPT + (
|
||||
f'\n\nAlbum hint: this photo is filed in a folder named "{album_hint}". '
|
||||
f'Folder names often contain the place and/or year — use it to inform '
|
||||
f'"location_hint" and "approx_year", but trust the image if they conflict.'
|
||||
)
|
||||
response = self._ensure_client().chat.completions.create(
|
||||
model=self._model,
|
||||
max_tokens=4096,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": f"data:{mime};base64,{b64}"}},
|
||||
{"type": "text", "text": prompt},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
raw = (response.choices[0].message.content or "").strip()
|
||||
if raw.startswith("```"):
|
||||
raw = raw.split("```")[1]
|
||||
raw = raw[4:] if raw.startswith("json") else raw
|
||||
raw = raw.strip()
|
||||
result = json.loads(raw)
|
||||
usage = getattr(response, "usage", None)
|
||||
result["_tokens"] = usage.total_tokens if usage else 0
|
||||
return result
|
||||
|
||||
|
||||
ANALYSIS_PROMPT = (
|
||||
"Analyze this photograph and return ONLY a JSON object with keys: description "
|
||||
"(one clear sentence), tags (8-12 specific keywords), people_count (integer), "
|
||||
"setting, time_of_day, season, mood, location_hint (or null), approx_year "
|
||||
"(integer or null)."
|
||||
)
|
||||
|
||||
|
||||
def _prepare_image(path: str) -> tuple[str, str]:
|
||||
import base64
|
||||
from io import BytesIO
|
||||
|
||||
from PIL import Image
|
||||
|
||||
with Image.open(path) as image:
|
||||
image = image.convert("RGB")
|
||||
image.thumbnail((1024, 1024))
|
||||
buffer = BytesIO()
|
||||
image.save(buffer, format="JPEG", quality=85)
|
||||
return base64.b64encode(buffer.getvalue()).decode(), "image/jpeg"
|
||||
186
photo_pipeline/services/library.py
Normal file
186
photo_pipeline/services/library.py
Normal file
@@ -0,0 +1,186 @@
|
||||
"""LibraryService — read-only Library and Stats over analysis results.
|
||||
|
||||
Ports webapp/query.py (search, facets, stats, top-tag/people/year aggregates) onto
|
||||
the shared database, re-keyed to ``asset_id`` and joined to ``assets`` for the
|
||||
current path. The donor read a path-keyed ``photos`` table with an FTS5 index; here
|
||||
search is a tokenised ``LIKE`` over description + tags.
|
||||
ponytail: restore FTS5 (rank-ordered relevance) if search recall/latency matters at
|
||||
real library size — LIKE is fine for browsing tens of thousands of rows.
|
||||
|
||||
Extracted from webapp/query.py (donor_ledger.yaml: wa-query-search, wa-query-stats).
|
||||
Read-only: writes belong to AnalysisService.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import and_, func, or_, select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from photo_pipeline.models import AnalysisResult, Asset
|
||||
|
||||
CARD_FIELDS = (
|
||||
"status",
|
||||
"description",
|
||||
"people_count",
|
||||
"setting",
|
||||
"time_of_day",
|
||||
"season",
|
||||
"mood",
|
||||
"location_hint",
|
||||
"approx_year",
|
||||
)
|
||||
DONE = ("analyzed",)
|
||||
_SORTS = {
|
||||
"year": (AnalysisResult.approx_year.desc(), Asset.current_path),
|
||||
"people": (AnalysisResult.people_count.desc(), Asset.current_path),
|
||||
"recent": (AnalysisResult.analyzed_at.desc(), Asset.current_path),
|
||||
"path": (Asset.current_path,),
|
||||
}
|
||||
|
||||
|
||||
def _album_of(path: str | None) -> str:
|
||||
return Path(path).parent.name if path else "(unknown)"
|
||||
|
||||
|
||||
class LibraryService:
|
||||
def __init__(self, session_factory: sessionmaker) -> None:
|
||||
self._session_factory = session_factory
|
||||
|
||||
def search(self, q="", filters=None, sort="path", offset=0, limit=60) -> dict:
|
||||
filters = filters or {}
|
||||
with self._session_factory() as session:
|
||||
stmt = select(AnalysisResult, Asset.current_path).join(
|
||||
Asset, Asset.id == AnalysisResult.asset_id
|
||||
)
|
||||
conds = _filter_conditions(filters)
|
||||
for token in re.findall(r"\w+", q, re.UNICODE):
|
||||
like = f"%{token}%"
|
||||
conds.append(
|
||||
or_(AnalysisResult.description.ilike(like), AnalysisResult.tags.ilike(like))
|
||||
)
|
||||
if conds:
|
||||
stmt = stmt.where(and_(*conds))
|
||||
total = session.scalar(select(func.count()).select_from(stmt.subquery()))
|
||||
stmt = stmt.order_by(*_SORTS.get(sort, _SORTS["path"])).limit(limit).offset(offset)
|
||||
rows = [_card(result, path) for result, path in session.execute(stmt)]
|
||||
return {"rows": rows, "total": total, "offset": offset, "limit": limit}
|
||||
|
||||
def stats(self) -> dict:
|
||||
with self._session_factory() as session:
|
||||
status = dict(
|
||||
session.execute(
|
||||
select(AnalysisResult.status, func.count()).group_by(AnalysisResult.status)
|
||||
).all()
|
||||
)
|
||||
rows = list(
|
||||
session.execute(
|
||||
select(AnalysisResult, Asset.current_path).join(
|
||||
Asset, Asset.id == AnalysisResult.asset_id
|
||||
)
|
||||
)
|
||||
)
|
||||
albums: dict[str, dict] = {}
|
||||
tag_counts: Counter = Counter()
|
||||
year_counts: Counter = Counter()
|
||||
people: Counter = Counter()
|
||||
errors = []
|
||||
for result, path in rows:
|
||||
album = _album_of(path)
|
||||
bucket = albums.setdefault(album, {"album": album, "done": 0, "total": 0})
|
||||
bucket["total"] += 1
|
||||
if result.status in DONE:
|
||||
bucket["done"] += 1
|
||||
for tag in _tags(result.tags):
|
||||
tag_counts[tag] += 1
|
||||
if result.approx_year is not None:
|
||||
year_counts[result.approx_year] += 1
|
||||
if result.people_count is not None:
|
||||
people["3+" if result.people_count >= 3 else str(result.people_count)] += 1
|
||||
if result.status == "error":
|
||||
errors.append({"path": path, "error": result.error_message})
|
||||
return {
|
||||
"total": sum(status.values()),
|
||||
"status": status,
|
||||
"setting": self._facet("setting"),
|
||||
"time_of_day": self._facet("time_of_day"),
|
||||
"season": self._facet("season"),
|
||||
"people": [{"value": v, "count": n} for v, n in sorted(people.items())],
|
||||
"years": [{"value": y, "count": year_counts[y]} for y in sorted(year_counts)],
|
||||
"top_tags": [{"value": t, "count": n} for t, n in tag_counts.most_common(40)],
|
||||
"albums": sorted(albums.values(), key=lambda d: d["album"]),
|
||||
"errors": sorted(errors, key=lambda e: e["path"] or ""),
|
||||
}
|
||||
|
||||
def facets(self) -> dict:
|
||||
return {
|
||||
"setting": self._facet("setting"),
|
||||
"time_of_day": self._facet("time_of_day"),
|
||||
"season": self._facet("season"),
|
||||
"status": self._facet("status"),
|
||||
}
|
||||
|
||||
def _facet(self, field: str) -> list[dict]:
|
||||
column = getattr(AnalysisResult, field)
|
||||
with self._session_factory() as session:
|
||||
rows = session.execute(
|
||||
select(column, func.count())
|
||||
.where(column.is_not(None), func.trim(column) != "")
|
||||
.group_by(column)
|
||||
.order_by(func.count().desc())
|
||||
).all()
|
||||
return [{"value": value, "count": count} for value, count in rows]
|
||||
|
||||
|
||||
def _filter_conditions(filters: dict) -> list:
|
||||
conds = []
|
||||
for key, column in (
|
||||
("setting", AnalysisResult.setting),
|
||||
("tod", AnalysisResult.time_of_day),
|
||||
("season", AnalysisResult.season),
|
||||
("status", AnalysisResult.status),
|
||||
):
|
||||
if filters.get(key):
|
||||
conds.append(column == filters[key])
|
||||
people = filters.get("people")
|
||||
if people == "3+":
|
||||
conds.append(AnalysisResult.people_count >= 3)
|
||||
elif people in ("0", "1", "2"):
|
||||
conds.append(AnalysisResult.people_count == int(people))
|
||||
if filters.get("year_min"):
|
||||
conds.append(AnalysisResult.approx_year >= int(filters["year_min"]))
|
||||
if filters.get("year_max"):
|
||||
conds.append(AnalysisResult.approx_year <= int(filters["year_max"]))
|
||||
if filters.get("has_location"):
|
||||
conds.append(
|
||||
and_(
|
||||
AnalysisResult.location_hint.is_not(None),
|
||||
func.trim(AnalysisResult.location_hint) != "",
|
||||
func.lower(AnalysisResult.location_hint) != "null",
|
||||
)
|
||||
)
|
||||
return conds
|
||||
|
||||
|
||||
def _card(result: AnalysisResult, path: str | None) -> dict:
|
||||
card = {field: getattr(result, field) for field in CARD_FIELDS}
|
||||
card.update(
|
||||
asset_id=result.asset_id,
|
||||
current_path=path,
|
||||
album=_album_of(path),
|
||||
tags=_tags(result.tags),
|
||||
)
|
||||
return card
|
||||
|
||||
|
||||
def _tags(raw: str | None) -> list[str]:
|
||||
if not raw:
|
||||
return []
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except (ValueError, TypeError):
|
||||
return []
|
||||
@@ -98,3 +98,202 @@ def exif_projection(decision: str) -> dict[str, list[str]]:
|
||||
if decision == SFW:
|
||||
return {"add": [SFW], "remove": [NSFW]}
|
||||
return {"add": [], "remove": []}
|
||||
|
||||
|
||||
# ── Persistence, scoring, and the durable safety decision ────────────────────
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from photo_pipeline.integrations import exiftool
|
||||
from photo_pipeline.models import Asset, SafetyReview
|
||||
from photo_pipeline.services import hashing
|
||||
|
||||
DECISIONS = {SFW, NSFW, "deferred"}
|
||||
|
||||
|
||||
class SafetyError(Exception):
|
||||
"""Invalid safety operation (bad decision, unknown asset)."""
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
# Microsecond precision so the latest-review-wins ordering never ties, even
|
||||
# when a decision is revised twice in the same DB-second (SQLite CURRENT_TIMESTAMP
|
||||
# is only second-granular).
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class SafetyService:
|
||||
"""Durable safety scoring, review decisions, and the EXIF safety checkpoint.
|
||||
|
||||
Scores and decisions live in ``safety_reviews`` keyed by ``asset_id`` (the
|
||||
donor's path-keyed CSV is gone). The latest row per asset is the current
|
||||
decision; ``AnalysisService`` reads it as the privacy gate.
|
||||
"""
|
||||
|
||||
def __init__(self, session_factory: sessionmaker, *, model=None) -> None:
|
||||
self._session_factory = session_factory
|
||||
self._model = model # injected in tests; real NsfwModel is lazy-loaded
|
||||
|
||||
# -- reads ----------------------------------------------------------------
|
||||
def _latest_by_asset(self, session) -> dict[str, SafetyReview]:
|
||||
# Latest row per asset. Small local scale: order ascending, let later rows
|
||||
# overwrite. ponytail: a windowed query if safety_reviews ever grows huge.
|
||||
latest: dict[str, SafetyReview] = {}
|
||||
for review in session.scalars(select(SafetyReview).order_by(SafetyReview.created_at)):
|
||||
latest[review.asset_id] = review
|
||||
return latest
|
||||
|
||||
def current_decision(self, asset_id: str) -> str | None:
|
||||
with self._session_factory() as session:
|
||||
review = self._latest_by_asset(session).get(asset_id)
|
||||
return review.decision if review else None
|
||||
|
||||
def counts(self) -> dict[str, int]:
|
||||
"""Decision breakdown over canonical, active assets — the workflow totals."""
|
||||
with self._session_factory() as session:
|
||||
assets = list(session.scalars(_eligible_assets_query()))
|
||||
latest = self._latest_by_asset(session)
|
||||
out = {SFW: 0, NSFW: 0, "deferred": 0, "undecided": 0, "scored": 0}
|
||||
for asset in assets:
|
||||
review = latest.get(asset.id)
|
||||
decision = review.decision if review else None
|
||||
if decision in (SFW, NSFW, "deferred"):
|
||||
out[decision] += 1
|
||||
else:
|
||||
out["undecided"] += 1
|
||||
if review and review.score is not None:
|
||||
out["scored"] += 1
|
||||
return out
|
||||
|
||||
def review_queue(self, state: str = "", limit: int = 100, offset: int = 0) -> dict:
|
||||
"""Assets for the review UI, filtered by ``state`` (undecided/sfw/nsfw/deferred)."""
|
||||
with self._session_factory() as session:
|
||||
assets = list(session.scalars(_eligible_assets_query().order_by(Asset.current_path)))
|
||||
latest = self._latest_by_asset(session)
|
||||
rows = []
|
||||
for asset in assets:
|
||||
review = latest.get(asset.id)
|
||||
decision = review.decision if review else None
|
||||
effective = decision or "undecided"
|
||||
if state and state != effective:
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"asset_id": asset.id,
|
||||
"current_path": asset.current_path,
|
||||
"score": review.score if review else None,
|
||||
"decision": decision,
|
||||
"suggested": classify(review.score) if review and review.score is not None else None,
|
||||
"exif_verified": bool(review and review.exif_verified_at),
|
||||
}
|
||||
)
|
||||
return {"total": len(rows), "items": rows[offset : offset + limit]}
|
||||
|
||||
def scorable_asset_ids(self) -> list[str]:
|
||||
"""Canonical active assets with a path — the items a scoring job enqueues."""
|
||||
with self._session_factory() as session:
|
||||
return [a.id for a in session.scalars(_eligible_assets_query()) if a.current_path]
|
||||
|
||||
# -- writes ---------------------------------------------------------------
|
||||
def score_assets(self, asset_ids: list[str] | None = None) -> int:
|
||||
"""Score eligible assets with the local model and persist scored reviews.
|
||||
|
||||
Records score-only reviews (no decision) so the reviewer can sort by risk.
|
||||
Returns the number scored. Assets already carrying a human decision keep it.
|
||||
"""
|
||||
with self._session_factory() as session:
|
||||
query = _eligible_assets_query()
|
||||
if asset_ids is not None:
|
||||
query = query.where(Asset.id.in_(asset_ids))
|
||||
assets = [a for a in session.scalars(query) if a.current_path]
|
||||
if not assets:
|
||||
return 0
|
||||
model = self._model or _default_model()
|
||||
by_path = {a.current_path: a.id for a in assets}
|
||||
scored = model.score(list(by_path))
|
||||
now = _now()
|
||||
with self._session_factory() as session:
|
||||
for path, score in scored:
|
||||
session.add(
|
||||
SafetyReview(
|
||||
id=str(uuid.uuid4()), asset_id=by_path[path], score=float(score), created_at=now
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
return len(scored)
|
||||
|
||||
def decide(self, asset_id: str, decision: str, *, reviewer: str = "user", write_exif: bool = True) -> dict:
|
||||
"""Persist a human safety decision and, for sfw/nsfw, run the EXIF checkpoint.
|
||||
|
||||
The decision is durable regardless of EXIF; ``exif_verified`` becomes true
|
||||
only after the mutually-exclusive keyword is written and read back, and the
|
||||
file's new SHA-256 is stored — that verified state is what upload eligibility
|
||||
later depends on.
|
||||
"""
|
||||
if decision not in DECISIONS:
|
||||
raise SafetyError(f"invalid decision {decision!r}")
|
||||
with self._session_factory() as session:
|
||||
asset = session.get(Asset, asset_id)
|
||||
if asset is None:
|
||||
raise SafetyError(f"unknown asset {asset_id}")
|
||||
prior = self._latest_by_asset(session).get(asset_id)
|
||||
path = asset.current_path
|
||||
|
||||
exif_verified_at = None
|
||||
result_sha256 = None
|
||||
if write_exif and decision in (SFW, NSFW) and path:
|
||||
ops = exif_projection(decision)
|
||||
if exiftool.apply_keywords(path, add=ops["add"], remove=ops["remove"]):
|
||||
# Read back: the chosen keyword present, the opposite absent.
|
||||
keywords = exiftool.read_keyword_sets([path]).get(path, set())
|
||||
opposite = NSFW if decision == SFW else SFW
|
||||
if decision in keywords and opposite not in keywords:
|
||||
exif_verified_at = _now()
|
||||
result_sha256 = hashing.sha256_file(path)
|
||||
|
||||
now = _now()
|
||||
with self._session_factory() as session:
|
||||
review = SafetyReview(
|
||||
id=str(uuid.uuid4()),
|
||||
asset_id=asset_id,
|
||||
decision=decision,
|
||||
prior_decision=prior.decision if prior else None,
|
||||
reviewer=reviewer,
|
||||
score=prior.score if prior else None,
|
||||
exif_verified_at=exif_verified_at,
|
||||
result_sha256=result_sha256,
|
||||
created_at=now,
|
||||
)
|
||||
session.add(review)
|
||||
if result_sha256:
|
||||
asset = session.get(Asset, asset_id)
|
||||
asset.current_sha256 = result_sha256
|
||||
session.commit()
|
||||
return {
|
||||
"asset_id": asset_id,
|
||||
"decision": decision,
|
||||
"prior_decision": prior.decision if prior else None,
|
||||
"exif_verified": exif_verified_at is not None,
|
||||
}
|
||||
|
||||
|
||||
def _eligible_assets_query():
|
||||
"""Canonical, active assets — the safety stage runs only on these.
|
||||
|
||||
``canonical_asset_id IS NULL`` excludes non-canonical duplicate variants; the
|
||||
shared discovery service already excludes ``_IGNORE/``.
|
||||
"""
|
||||
return select(Asset).where(
|
||||
Asset.canonical_asset_id.is_(None),
|
||||
Asset.availability_state == "active",
|
||||
)
|
||||
|
||||
|
||||
def _default_model():
|
||||
from photo_pipeline.integrations.nsfw_model import NsfwModel
|
||||
|
||||
return NsfwModel()
|
||||
|
||||
161
photo_pipeline/services/workflow.py
Normal file
161
photo_pipeline/services/workflow.py
Normal file
@@ -0,0 +1,161 @@
|
||||
"""WorkflowService — stage readiness for the Workflow home.
|
||||
|
||||
Totals are DERIVED from the source tables (assets, duplicate_clusters,
|
||||
safety_reviews, analysis_results) rather than a separately-maintained counter, so
|
||||
the home page can never drift from reality. Each stage card carries a machine
|
||||
``status`` plus human ``status_label`` and ``blocker`` text — the UI communicates
|
||||
state with text/icons as well as color (concept §Workflow home).
|
||||
|
||||
``active_job`` is the one-mutating-job indicator: when a job holds the
|
||||
``library_write`` lock, mutating actions across the app are disabled with a reason,
|
||||
while read-only browsing stays available.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from photo_pipeline.jobs.domain_handlers import ANALYSIS, SAFETY_SCORE
|
||||
from photo_pipeline.models import Asset, DuplicateCluster, Job
|
||||
from photo_pipeline.services.analysis import AnalysisService
|
||||
from photo_pipeline.services.jobs import ACTIVE_STATES
|
||||
from photo_pipeline.services.safety import SafetyService
|
||||
|
||||
# Job type that produced each stage's last run.
|
||||
STAGE_JOB_TYPE = {"inventory": "scan", "safety": SAFETY_SCORE, "analysis": ANALYSIS}
|
||||
|
||||
|
||||
class WorkflowService:
|
||||
def __init__(self, session_factory: sessionmaker) -> None:
|
||||
self._session_factory = session_factory
|
||||
|
||||
def readiness(self) -> dict:
|
||||
with self._session_factory() as session:
|
||||
asset_total = session.scalar(
|
||||
select(func.count()).select_from(Asset).where(Asset.canonical_asset_id.is_(None))
|
||||
)
|
||||
missing = session.scalar(
|
||||
select(func.count()).select_from(Asset).where(Asset.missing_at.is_not(None))
|
||||
)
|
||||
cluster_states = dict(
|
||||
session.execute(
|
||||
select(DuplicateCluster.state, func.count()).group_by(DuplicateCluster.state)
|
||||
).all()
|
||||
)
|
||||
last_run = self._last_runs(session)
|
||||
active = self._active_job(session)
|
||||
|
||||
safety = SafetyService(self._session_factory).counts()
|
||||
analysis = AnalysisService(self._session_factory).counts()
|
||||
undecided_clusters = cluster_states.get("open", 0) + cluster_states.get("reopened", 0)
|
||||
|
||||
stages = [
|
||||
_card(
|
||||
"inventory",
|
||||
"Inventory",
|
||||
status="complete" if asset_total and not missing else ("attention" if missing else "ready"),
|
||||
counts={"assets": asset_total or 0, "missing": missing or 0},
|
||||
blocker=f"{missing} missing file(s)" if missing else None,
|
||||
last_run=last_run.get("inventory"),
|
||||
action={"label": "Rescan", "job_type": "scan", "route": "#/inventory"},
|
||||
),
|
||||
_card(
|
||||
"duplicates",
|
||||
"Duplicates",
|
||||
status="attention" if undecided_clusters else "complete",
|
||||
counts={"undecided": undecided_clusters, "total": sum(cluster_states.values())},
|
||||
blocker=f"{undecided_clusters} cluster(s) to review" if undecided_clusters else None,
|
||||
last_run=None,
|
||||
action={"label": "Review", "route": "#/duplicates"},
|
||||
),
|
||||
_card(
|
||||
"safety",
|
||||
"Safety",
|
||||
# Blocked while duplicates are unreviewed (dedup precedes safety).
|
||||
status=_safety_status(safety, undecided_clusters),
|
||||
counts=safety,
|
||||
blocker=(
|
||||
"resolve duplicate clusters first"
|
||||
if undecided_clusters
|
||||
else (f"{safety['undecided']} undecided" if safety["undecided"] else None)
|
||||
),
|
||||
last_run=last_run.get("safety"),
|
||||
action={"label": "Review safety", "job_type": SAFETY_SCORE, "route": "#/safety"},
|
||||
blocked_by="duplicates" if undecided_clusters else None,
|
||||
),
|
||||
_card(
|
||||
"analysis",
|
||||
"Analysis",
|
||||
status=_analysis_status(analysis, safety),
|
||||
counts=analysis,
|
||||
blocker=(
|
||||
"no confirmed-SFW assets yet"
|
||||
if analysis["eligible"] == 0
|
||||
else (f"{analysis['pending']} pending" if analysis["pending"] else None)
|
||||
),
|
||||
last_run=last_run.get("analysis"),
|
||||
action={"label": "Analyze", "job_type": ANALYSIS, "route": "#/analyze"},
|
||||
blocked_by="safety" if analysis["eligible"] == 0 else None,
|
||||
),
|
||||
]
|
||||
return {"active_job": active, "stages": stages}
|
||||
|
||||
def _last_runs(self, session) -> dict[str, str | None]:
|
||||
out: dict[str, str | None] = {}
|
||||
for stage, job_type in STAGE_JOB_TYPE.items():
|
||||
finished = session.scalar(
|
||||
select(func.max(Job.finished_at)).where(Job.job_type == job_type)
|
||||
)
|
||||
out[stage] = finished.isoformat() if finished else None
|
||||
return out
|
||||
|
||||
def _active_job(self, session) -> dict | None:
|
||||
job = session.scalars(
|
||||
select(Job).where(Job.state.in_(ACTIVE_STATES)).order_by(Job.created_at.desc())
|
||||
).first()
|
||||
if job is None:
|
||||
return None
|
||||
return {"id": job.id, "job_type": job.job_type, "state": job.state, "lock": job.lock_key}
|
||||
|
||||
|
||||
def _safety_status(safety: dict, undecided_clusters: int) -> str:
|
||||
if undecided_clusters:
|
||||
return "blocked"
|
||||
if safety["undecided"]:
|
||||
return "attention"
|
||||
if safety[_SFW] or safety[_NSFW]:
|
||||
return "complete"
|
||||
return "ready"
|
||||
|
||||
|
||||
def _analysis_status(analysis: dict, safety: dict) -> str:
|
||||
if analysis["eligible"] == 0:
|
||||
return "blocked"
|
||||
if analysis["pending"]:
|
||||
return "attention"
|
||||
return "complete"
|
||||
|
||||
|
||||
def _card(key, title, *, status, counts, blocker, last_run, action, blocked_by=None) -> dict:
|
||||
return {
|
||||
"key": key,
|
||||
"title": title,
|
||||
"status": status,
|
||||
"status_label": _STATUS_LABEL[status],
|
||||
"counts": counts,
|
||||
"blocker": blocker,
|
||||
"blocked_by": blocked_by,
|
||||
"last_run": last_run,
|
||||
"action": action,
|
||||
}
|
||||
|
||||
|
||||
_SFW = "sfw"
|
||||
_NSFW = "nsfw"
|
||||
_STATUS_LABEL = {
|
||||
"complete": "Complete",
|
||||
"ready": "Ready",
|
||||
"attention": "Needs review",
|
||||
"blocked": "Blocked",
|
||||
}
|
||||
Reference in New Issue
Block a user