US03-01: Aggregate Album Evidence (#61)
This commit was merged in pull request #61.
This commit is contained in:
265
photo_pipeline/services/albums.py
Normal file
265
photo_pipeline/services/albums.py
Normal file
@@ -0,0 +1,265 @@
|
||||
"""AlbumService — aggregate per-folder evidence for album proposals (US03-01).
|
||||
|
||||
Read-only. For each leaf folder it summarises the *already stored* analysis and
|
||||
safety evidence of the canonical, active assets living there, so a later stage can
|
||||
name the folder without resending images or exposing raw model responses. It writes
|
||||
nothing and generates no names (that is US03-02/03).
|
||||
|
||||
Folder identity is the full parent path of an asset's ``current_path`` — not just
|
||||
the leaf name — so two folders that share a leaf never merge. Aggregation is
|
||||
deterministic (everything sorted, no clock/random) and each folder's evidence
|
||||
carries a content ``version`` derived only from the inputs that could change it, so
|
||||
any relevant asset change (new analysis, decision change, move, or becoming a
|
||||
duplicate) invalidates exactly that folder's version and nothing else.
|
||||
|
||||
Documented evidence rules (acceptance criterion 2):
|
||||
|
||||
- **Duplicate (non-canonical) assets** — excluded entirely; the canonical carries
|
||||
the evidence.
|
||||
- **Missing / archived assets** — excluded: only ``availability_state == 'active'``
|
||||
assets with a ``current_path`` belong to an active folder.
|
||||
- **NSFW assets** — counted in the folder total and reported as ``nsfw_count``, but
|
||||
contribute NO analysis-derived evidence (tags, description, setting, location,
|
||||
year, people). They are never analysed, and their content must not shape a name
|
||||
beyond mere presence.
|
||||
- **Undecided / deferred / not-yet-analysed assets** — counted in the folder total
|
||||
and reported as ``pending_count``; they contribute no evidence until confirmed
|
||||
SFW and analysed.
|
||||
- **Only confirmed-SFW analysed assets** shape the descriptive evidence.
|
||||
- **Missing fields** (null/empty) simply do not contribute.
|
||||
- **Conflicting fields** (e.g. several ``approx_year`` values) are never silently
|
||||
collapsed: the full sorted distribution is returned, plus a ``dominant_year`` that
|
||||
is set only when one value strictly outnumbers the rest; ties set it to ``None``
|
||||
and raise ``year_conflict``.
|
||||
|
||||
Reuses ``_tags`` from :mod:`photo_pipeline.services.library` (the shared JSON-tag
|
||||
decode). ponytail: neighbouring-folder names/date ranges and user naming
|
||||
conventions (concept §7 inputs) are proposal-generation concerns — add them in
|
||||
US03-03 when a name is actually produced.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from photo_pipeline.models import AnalysisResult, Asset, SafetyReview
|
||||
from photo_pipeline.services.library import _tags
|
||||
|
||||
SFW = "sfw"
|
||||
NSFW = "nsfw"
|
||||
ANALYZED = "analyzed"
|
||||
# Cap the descriptions list so a huge folder can't produce an unbounded payload;
|
||||
# tags/facets are already bounded by their distinct-value count.
|
||||
MAX_DESCRIPTIONS = 50
|
||||
|
||||
|
||||
def album_label(path: str, library_roots: tuple[Path, ...] = ()) -> str:
|
||||
"""The canonical album identity: the leaf folder directly containing ``path``,
|
||||
shown as a path relative to its library root so nested albums stay distinct
|
||||
(``Urlaub/Rom`` vs ``Urlaub/Venedig``); ``(root)`` for a photo directly in a
|
||||
root, and the bare parent name for a photo outside every configured root.
|
||||
|
||||
Ported from ``photo_analyzer.album_label`` (donor_ledger.yaml: pa-album-label);
|
||||
``webapp/query.album_of`` was a verbatim copy (wa-album-of). This is the single
|
||||
implementation for stats, proposals, and folder-as-album upload.
|
||||
"""
|
||||
parent = Path(path).parent
|
||||
for root in library_roots:
|
||||
try:
|
||||
rel = parent.relative_to(root)
|
||||
return "(root)" if str(rel) == "." else str(rel)
|
||||
except ValueError:
|
||||
continue
|
||||
return parent.name or str(parent)
|
||||
|
||||
|
||||
def _counts(counter: Counter) -> list[dict]:
|
||||
"""Deterministic ``[{value, count}]``, most frequent first, ties broken by value."""
|
||||
return [
|
||||
{"value": value, "count": count}
|
||||
for value, count in sorted(counter.items(), key=lambda kv: (-kv[1], str(kv[0])))
|
||||
]
|
||||
|
||||
|
||||
def _dominant(counter: Counter):
|
||||
"""The single strictly-most-frequent value, or ``(None, conflict)``. ``conflict``
|
||||
is true when more than one distinct value is present and no strict winner exists."""
|
||||
if not counter:
|
||||
return None, False
|
||||
ranked = counter.most_common()
|
||||
if len(ranked) == 1:
|
||||
return ranked[0][0], False
|
||||
top_value, top_count = ranked[0]
|
||||
if top_count > ranked[1][1]:
|
||||
return top_value, False
|
||||
return None, True
|
||||
|
||||
|
||||
class AlbumService:
|
||||
def __init__(
|
||||
self, session_factory: sessionmaker, *, library_roots: tuple[Path, ...] = ()
|
||||
) -> None:
|
||||
self._session_factory = session_factory
|
||||
self._library_roots = tuple(Path(root) for root in library_roots)
|
||||
|
||||
def _album_of(self, path: str) -> str:
|
||||
return album_label(path, self._library_roots)
|
||||
|
||||
def _latest_decisions(self, session) -> dict[str, str | None]:
|
||||
"""asset_id -> latest safety decision (``sfw``/``nsfw``/``deferred``/None)."""
|
||||
latest: dict[str, str | None] = {}
|
||||
for review in session.scalars(select(SafetyReview).order_by(SafetyReview.created_at)):
|
||||
latest[review.asset_id] = review.decision
|
||||
return latest
|
||||
|
||||
def folder_count(self) -> int:
|
||||
"""Number of distinct active-canonical folders (for paging callers)."""
|
||||
with self._session_factory() as session:
|
||||
paths = session.scalars(
|
||||
select(Asset.current_path).where(
|
||||
Asset.canonical_asset_id.is_(None),
|
||||
Asset.availability_state == "active",
|
||||
Asset.current_path.is_not(None),
|
||||
)
|
||||
)
|
||||
return len({self._album_of(p) for p in paths})
|
||||
|
||||
def aggregate_evidence(self, *, offset: int = 0, limit: int | None = None) -> dict:
|
||||
"""Return a page of per-folder evidence, folders sorted by path.
|
||||
|
||||
``{folders: [...], total: int, offset, limit}``. ``limit=None`` returns all.
|
||||
"""
|
||||
with self._session_factory() as session:
|
||||
decisions = self._latest_decisions(session)
|
||||
rows = list(
|
||||
session.execute(
|
||||
select(Asset.id, Asset.current_path, AnalysisResult)
|
||||
.outerjoin(AnalysisResult, AnalysisResult.asset_id == Asset.id)
|
||||
.where(
|
||||
Asset.canonical_asset_id.is_(None),
|
||||
Asset.availability_state == "active",
|
||||
Asset.current_path.is_not(None),
|
||||
)
|
||||
.order_by(Asset.current_path, Asset.id)
|
||||
)
|
||||
)
|
||||
|
||||
folders: dict[str, list] = {}
|
||||
for asset_id, path, result in rows:
|
||||
folders.setdefault(self._album_of(path), []).append((asset_id, path, result))
|
||||
|
||||
ordered = sorted(folders)
|
||||
total = len(ordered)
|
||||
page = ordered[offset:] if limit is None else ordered[offset : offset + limit]
|
||||
evidence = [self._folder_evidence(folder, folders[folder], decisions) for folder in page]
|
||||
return {"folders": evidence, "total": total, "offset": offset, "limit": limit}
|
||||
|
||||
def _folder_evidence(self, folder: str, members: list, decisions: dict) -> dict:
|
||||
descriptions: list[str] = []
|
||||
tags: Counter = Counter()
|
||||
settings: Counter = Counter()
|
||||
time_of_day: Counter = Counter()
|
||||
seasons: Counter = Counter()
|
||||
locations: Counter = Counter()
|
||||
people: Counter = Counter()
|
||||
years: Counter = Counter()
|
||||
|
||||
analyzed_count = nsfw_count = pending_count = 0
|
||||
fingerprint: list[tuple] = []
|
||||
|
||||
for asset_id, path, result in members:
|
||||
decision = decisions.get(asset_id)
|
||||
status = result.status if result is not None else None
|
||||
# Fingerprint: every input that could change this folder's evidence.
|
||||
fingerprint.append(
|
||||
(
|
||||
asset_id,
|
||||
path,
|
||||
decision,
|
||||
status,
|
||||
result.description if result is not None else None,
|
||||
result.tags if result is not None else None,
|
||||
result.people_count if result is not None else None,
|
||||
result.setting if result is not None else None,
|
||||
result.time_of_day if result is not None else None,
|
||||
result.season if result is not None else None,
|
||||
_clean(result.location_hint) if result is not None else None,
|
||||
result.approx_year if result is not None else None,
|
||||
)
|
||||
)
|
||||
|
||||
if decision == NSFW:
|
||||
nsfw_count += 1
|
||||
continue
|
||||
eligible = decision == SFW and status == ANALYZED and result is not None
|
||||
if not eligible:
|
||||
pending_count += 1
|
||||
continue
|
||||
|
||||
analyzed_count += 1
|
||||
if result.description and result.description.strip():
|
||||
descriptions.append(result.description.strip())
|
||||
for tag in _tags(result.tags):
|
||||
tags[tag] += 1
|
||||
if result.setting:
|
||||
settings[result.setting] += 1
|
||||
if result.time_of_day:
|
||||
time_of_day[result.time_of_day] += 1
|
||||
if result.season:
|
||||
seasons[result.season] += 1
|
||||
location = _clean(result.location_hint)
|
||||
if location:
|
||||
locations[location] += 1
|
||||
if result.people_count is not None:
|
||||
people["3+" if result.people_count >= 3 else str(result.people_count)] += 1
|
||||
if result.approx_year is not None:
|
||||
years[result.approx_year] += 1
|
||||
|
||||
dominant_year, year_conflict = _dominant(years)
|
||||
# ``folder`` is the album label (identity): a library-relative path like
|
||||
# ``Urlaub/Rom``, or ``(root)``. Name is its leaf; parent is the prefix.
|
||||
parent = folder.rsplit("/", 1)[0] if "/" in folder else ""
|
||||
name = folder.rsplit("/", 1)[-1]
|
||||
return {
|
||||
"album": folder,
|
||||
"folder_name": name,
|
||||
"parent_path": parent,
|
||||
"asset_count": len(members),
|
||||
"analyzed_count": analyzed_count,
|
||||
"nsfw_count": nsfw_count,
|
||||
"pending_count": pending_count,
|
||||
"descriptions": sorted(descriptions)[:MAX_DESCRIPTIONS],
|
||||
"tags": _counts(tags),
|
||||
"settings": _counts(settings),
|
||||
"time_of_day": _counts(time_of_day),
|
||||
"seasons": _counts(seasons),
|
||||
"locations": _counts(locations),
|
||||
"people_counts": _counts(people),
|
||||
"years": _counts(years),
|
||||
"dominant_year": dominant_year,
|
||||
"year_conflict": year_conflict,
|
||||
"version": _version(fingerprint),
|
||||
}
|
||||
|
||||
|
||||
def _clean(location: str | None) -> str | None:
|
||||
"""Drop empty/``null`` location hints (the model emits the string ``"null"``)."""
|
||||
if not location:
|
||||
return None
|
||||
trimmed = location.strip()
|
||||
if not trimmed or trimmed.lower() == "null":
|
||||
return None
|
||||
return trimmed
|
||||
|
||||
|
||||
def _version(fingerprint: list[tuple]) -> str:
|
||||
"""Deterministic content hash of a folder's evidence inputs. Independent of row
|
||||
order (the inputs are sorted) so only a real change moves it."""
|
||||
payload = json.dumps(sorted(fingerprint, key=lambda t: t[0]), ensure_ascii=False)
|
||||
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||||
Reference in New Issue
Block a user