"""DuplicateService — explainable clusters and reversible decisions. Detection runs in two categories: - **Content-identical** clusters join assets that share exact bytes (``sha256``) or normalized pixels (``pixel_sha256``). These are high confidence and are decided automatically (a recommended canonical, the rest variants) — unless a member pair carries a ``not_duplicate`` negative link, which is a contradiction that reopens the cluster for review instead of silently overriding the user. - **Perceptual** clusters join assets whose perceptual hashes are within a distance band (NEAR/SIMILAR). These are review candidates: never decided automatically, and negative-linked pairs are suppressed so a rejected pair is not re-suggested. Archived assets stay in both indexes (US06-03): a new active copy of an archived original is recognised through its hashes even while the medium is offline, and cluster review falls back to the retained protected preview plus hash evidence. An exact/pixel match links straight to the archived canonical; a perceptual match is a review candidate that names the medium to mount for a pixel-level decision. Decisions (``canonical`` / ``not_duplicate`` / ``deferred``) persist with evidence, use optimistic version checks, are reversible, and can never form a canonical cycle. A new content-identical member of an already-decided cluster inherits the established canonical relationship rather than reopening it. Extracts photo_analyzer.cluster_duplicates/mark_duplicates (union-find over the Hamming graph); path-keyed identity and automatic largest-file selection are replaced by stable asset IDs and reviewable decisions (donor_ledger.yaml: pa-dupes). """ from __future__ import annotations import json import uuid from collections import Counter from dataclasses import dataclass, field from datetime import datetime, timezone from enum import Enum from sqlalchemy import func, select from sqlalchemy.orm import sessionmaker from photo_pipeline.models import ( ArchiveLocation, Asset, DuplicateCluster, DuplicateMember, DuplicateNegativeLink, Thumbnail, ) from photo_pipeline.services import availability, hashing NEAR_MAX = 5 SIMILAR_MAX = 10 # Member paging (US07-06). A burst or a re-imported folder can put thousands of # assets in one cluster; review looks at a few at a time, so neither the list view # nor the detail view may load them all. MEMBER_PAGE = 100 MAX_MEMBER_PAGE = 500 SNAPSHOT_MEMBER_PREVIEW = 20 class Method(str, Enum): EXACT = "exact" PIXEL = "pixel" PERCEPTUAL = "perceptual" class Confidence(str, Enum): EXACT = "exact" PIXEL = "pixel" NEAR = "near" SIMILAR = "similar" class ClusterState(str, Enum): OPEN = "open" DECIDED = "decided" DISMISSED = "dismissed" REOPENED = "reopened" DEFERRED = "deferred" class Decision(str, Enum): CANONICAL = "canonical" NOT_DUPLICATE = "not_duplicate" DEFERRED = "deferred" class Role(str, Enum): MEMBER = "member" CANONICAL = "canonical" VARIANT = "variant" class DuplicateError(RuntimeError): """Invalid decision (unknown cluster, non-member canonical, would cycle).""" class ConflictError(RuntimeError): """Optimistic version check failed; the cluster changed since it was read.""" @dataclass class DetectionReport: clusters: list[dict] = field(default_factory=list) @property def counts(self) -> dict[str, int]: return dict(Counter(c["state"] for c in self.clusters)) def _pair(a: str, b: str) -> tuple[str, str]: return (a, b) if a <= b else (b, a) class _UnionFind: def __init__(self, items): self.parent = {x: x for x in items} def find(self, x): while self.parent[x] != x: self.parent[x] = self.parent[self.parent[x]] x = self.parent[x] return x def union(self, a, b): ra, rb = self.find(a), self.find(b) if ra != rb: self.parent[rb] = ra def groups(self) -> list[list[str]]: out: dict[str, list[str]] = {} for x in self.parent: out.setdefault(self.find(x), []).append(x) return [sorted(g) for g in out.values()] class DuplicateService: def __init__(self, session_factory: sessionmaker) -> None: self._session_factory = session_factory # ── perceptual hash backfill ─────────────────────────────────────────── def ensure_phashes(self) -> int: """Hash whatever is readable now — an archived asset keeps the hash it already has, and gains one whenever its medium happens to be mounted.""" updated = 0 with self._session_factory() as session: assets = session.execute(select(Asset)).scalars() for asset in assets: if asset.phash is not None and asset.phash_version == hashing.PHASH_VERSION: continue source = availability.readable_path(session, asset) if source is None: continue value = hashing.safe_phash(str(source)) if value is not None: asset.phash = value asset.phash_version = hashing.PHASH_VERSION updated += 1 session.commit() return updated def ensure_phash(self, asset_id: str, *, source=None) -> str | None: """Backfill one asset's perceptual hash while its bytes are still readable. Archiving calls this before the original leaves — passing the archive copy as ``source``, since the database does not point at it yet — because an asset without a pHash silently drops out of the fuzzy index the moment its medium is away. """ with self._session_factory() as session: asset = session.get(Asset, asset_id) if asset is None: return None if asset.phash is not None and asset.phash_version == hashing.PHASH_VERSION: return asset.phash source = source or availability.readable_path(session, asset) if source is None: return None value = hashing.safe_phash(str(source)) if value is not None: asset.phash = value asset.phash_version = hashing.PHASH_VERSION session.commit() return value # ── detection ────────────────────────────────────────────────────────── def detect(self) -> DetectionReport: self.ensure_phashes() now = datetime.now(timezone.utc) report = DetectionReport() with self._session_factory() as session: # Every known asset stays in the indexes, archived or not: a copy of an # archived original must be recognised as a duplicate rather than # treated as a new photo (concept §9, invariant 12). assets = list(session.execute(select(Asset)).scalars()) by_id = {a.id: a for a in assets} negatives = { _pair(link.asset_a, link.asset_b) for link in session.execute(select(DuplicateNegativeLink)).scalars() } clusters = list(session.execute(select(DuplicateCluster)).scalars()) members = { c.id: list( session.execute( select(DuplicateMember).where(DuplicateMember.cluster_id == c.id) ).scalars() ) for c in clusters } content_groups, content_pairs = self._content_groups(assets) perceptual_groups = self._perceptual_groups(assets, content_pairs, negatives) touched: list[DuplicateCluster] = [] for method, ids, distances in content_groups: touched.append( self._sync_content_cluster( session, method, ids, by_id, negatives, clusters, members, now ) ) for ids, distances in perceptual_groups: touched.append( self._sync_perceptual_cluster( session, ids, distances, by_id, clusters, members, now ) ) session.commit() report.clusters = [self._snapshot(session, c.id) for c in touched] return report def _content_groups(self, assets): """Union-find over exact-byte and same-pixel edges. Returns groups of >=2 with method EXACT (all same bytes) or PIXEL, plus the set of joined pairs.""" uf = _UnionFind([a.id for a in assets]) pairs: set[tuple[str, str]] = set() for key in ("current_sha256", "pixel_sha256"): buckets: dict[str, list[str]] = {} for a in assets: value = getattr(a, key) if value: buckets.setdefault(value, []).append(a.id) for group in buckets.values(): for other in group[1:]: uf.union(group[0], other) for i in range(len(group)): for j in range(i + 1, len(group)): pairs.add(_pair(group[i], group[j])) by_id = {a.id: a for a in assets} result = [] for group in uf.groups(): if len(group) < 2: continue shas = {by_id[i].current_sha256 for i in group} method = Method.EXACT if len(shas) == 1 and None not in shas else Method.PIXEL result.append((method, sorted(group), {})) return result, pairs def _perceptual_groups(self, assets, content_pairs, negatives): """Union-find over phash distance edges (<= SIMILAR_MAX), excluding pairs that are already content-identical or negatively linked.""" hashed = [a for a in assets if a.phash] uf = _UnionFind([a.id for a in hashed]) distances: dict[tuple[str, str], int] = {} for i in range(len(hashed)): for j in range(i + 1, len(hashed)): a, b = hashed[i], hashed[j] pair = _pair(a.id, b.id) if pair in content_pairs or pair in negatives: continue dist = hashing.phash_distance(a.phash, b.phash) if dist <= SIMILAR_MAX: uf.union(a.id, b.id) distances[pair] = dist result = [] for group in uf.groups(): if len(group) < 2: continue group_dists = { _pair(x, y): distances[_pair(x, y)] for x in group for y in group if x < y and _pair(x, y) in distances } result.append((sorted(group), group_dists)) return result # ── cluster upserts ────────────────────────────────────────────────────── def _match_existing(self, ids, clusters, members, categories): best = None best_overlap = 0 for cluster in clusters: if cluster.method not in categories: continue overlap = len(set(ids) & {m.asset_id for m in members[cluster.id]}) if overlap > best_overlap: best, best_overlap = cluster, overlap return best def _sync_content_cluster( self, session, method, ids, by_id, negatives, clusters, members, now ): contradiction = any( _pair(ids[i], ids[j]) in negatives for i in range(len(ids)) for j in range(i + 1, len(ids)) ) confidence = Confidence.EXACT if method == Method.EXACT else Confidence.PIXEL # Also consider a prior perceptual cluster over the same assets: if a member # pair became byte-identical after a not_duplicate decision, reopen that same # reviewed cluster rather than spawning a parallel one. existing = self._match_existing( ids, clusters, members, {Method.EXACT.value, Method.PIXEL.value, Method.PERCEPTUAL.value}, ) if existing is None: cluster = DuplicateCluster( id=str(uuid.uuid4()), method=method.value, confidence=confidence.value, state=ClusterState.OPEN.value, version=1, ) session.add(cluster) session.flush() self._replace_members(session, cluster, ids, by_id, distances={}) if contradiction: cluster.state = ClusterState.REOPENED.value else: self._auto_decide(session, cluster, ids, by_id, now) clusters.append(cluster) members[cluster.id] = list( session.execute( select(DuplicateMember).where(DuplicateMember.cluster_id == cluster.id) ).scalars() ) return cluster cluster = existing cluster.method = method.value cluster.confidence = confidence.value prior_members = {m.asset_id for m in members[cluster.id]} added = set(ids) - prior_members self._replace_members(session, cluster, ids, by_id, distances={}) if contradiction: cluster.state = ClusterState.REOPENED.value cluster.decision = None cluster.canonical_asset_id = None self._clear_asset_links(session, ids) elif ( cluster.decision == Decision.CANONICAL.value and cluster.canonical_asset_id in ids ): # Consistent growth: new members inherit the established canonical. self._apply_canonical(session, cluster, ids, cluster.canonical_asset_id) cluster.state = ClusterState.DECIDED.value elif cluster.state in (ClusterState.DISMISSED.value,) and added: cluster.state = ClusterState.REOPENED.value elif cluster.state in (ClusterState.OPEN.value, ClusterState.REOPENED.value): self._auto_decide(session, cluster, ids, by_id, now) cluster.version += 1 cluster.updated_at = now members[cluster.id] = list( session.execute( select(DuplicateMember).where(DuplicateMember.cluster_id == cluster.id) ).scalars() ) return cluster def _sync_perceptual_cluster(self, session, ids, distances, by_id, clusters, members, now): confidence = ( Confidence.NEAR if distances and max(distances.values()) <= NEAR_MAX else Confidence.SIMILAR ) existing = self._match_existing(ids, clusters, members, {Method.PERCEPTUAL.value}) if existing is None: cluster = DuplicateCluster( id=str(uuid.uuid4()), method=Method.PERCEPTUAL.value, confidence=confidence.value, state=ClusterState.OPEN.value, version=1, ) session.add(cluster) session.flush() self._replace_members(session, cluster, ids, by_id, distances) clusters.append(cluster) members[cluster.id] = list( session.execute( select(DuplicateMember).where(DuplicateMember.cluster_id == cluster.id) ).scalars() ) return cluster cluster = existing prior = {m.asset_id for m in members[cluster.id]} added = set(ids) - prior cluster.confidence = confidence.value self._replace_members(session, cluster, ids, by_id, distances) if added and cluster.state in ( ClusterState.DECIDED.value, ClusterState.DISMISSED.value, ClusterState.DEFERRED.value, ): cluster.state = ClusterState.REOPENED.value cluster.version += 1 cluster.updated_at = now members[cluster.id] = list( session.execute( select(DuplicateMember).where(DuplicateMember.cluster_id == cluster.id) ).scalars() ) return cluster def _replace_members(self, session, cluster, ids, by_id, distances): current = { m.asset_id: m for m in session.execute( select(DuplicateMember).where(DuplicateMember.cluster_id == cluster.id) ).scalars() } wanted = set(ids) for asset_id, member in current.items(): if asset_id not in wanted: session.delete(member) for asset_id in ids: asset = by_id[asset_id] evidence = json.dumps( { "sha256": asset.current_sha256, "pixel_sha256": asset.pixel_sha256, "phash": asset.phash, } ) distance = next( (d for p, d in distances.items() if asset_id in p), None ) if asset_id in current: current[asset_id].evidence = evidence current[asset_id].distance = distance else: session.add( DuplicateMember( cluster_id=cluster.id, asset_id=asset_id, role=Role.MEMBER.value, distance=distance, evidence=evidence, ) ) session.flush() def _auto_decide(self, session, cluster, ids, by_id, now): canonical = self._recommend_canonical(ids, by_id) self._apply_canonical(session, cluster, ids, canonical) cluster.decision = Decision.CANONICAL.value cluster.canonical_asset_id = canonical cluster.state = ClusterState.DECIDED.value cluster.decided_at = now @staticmethod def _recommend_canonical(ids, by_id) -> str: # ponytail: largest file, then the archived copy, then path as a # deterministic tie-break. Archived wins ties because it is the reviewed, # uploaded original — a fresh active copy must not demote it to a variant. # The concept's richer policy (resolution, least recompression, metadata # richness) lands with the review UI story. return max( ids, key=lambda i: ( by_id[i].byte_size or 0, by_id[i].availability_state in availability.ARCHIVED, by_id[i].current_path or by_id[i].archive_path or "", ), ) def _apply_canonical(self, session, cluster, ids, canonical_id): for member in session.execute( select(DuplicateMember).where(DuplicateMember.cluster_id == cluster.id) ).scalars(): asset = session.get(Asset, member.asset_id) if member.asset_id == canonical_id: member.role = Role.CANONICAL.value asset.canonical_asset_id = None else: member.role = Role.VARIANT.value asset.canonical_asset_id = canonical_id self._assert_acyclic(session, ids) @staticmethod def _clear_asset_links(session, ids): for asset_id in ids: session.get(Asset, asset_id).canonical_asset_id = None @staticmethod def _assert_acyclic(session, ids): for start in ids: seen = set() cur = start while cur is not None: if cur in seen: raise DuplicateError(f"canonical link cycle at {cur}") seen.add(cur) asset = session.get(Asset, cur) cur = asset.canonical_asset_id if asset else None # ── reads for the review UI ──────────────────────────────────────────────── def list_clusters( self, *, state: str | None = None, limit: int = 50, offset: int = 0 ) -> dict: limit = max(1, min(limit, 200)) offset = max(0, offset) with self._session_factory() as session: stmt = select(DuplicateCluster) if state: stmt = stmt.where(DuplicateCluster.state == state) total = session.scalar(select(func.count()).select_from(stmt.subquery())) rows = session.execute( stmt.order_by(DuplicateCluster.created_at).limit(limit).offset(offset) ).scalars() items = [self._snapshot(session, c.id) for c in rows] return {"items": items, "total": int(total or 0), "limit": limit, "offset": offset} def get_cluster( self, cluster_id: str, *, limit: int = MEMBER_PAGE, offset: int = 0 ) -> dict | None: """Cluster detail enriched with per-member asset evidence for comparison. Members are paged and their evidence is loaded in batches (US07-06). A cluster of a few thousand near-identical frames is a real shape for a phone library, and the review screen only ever shows a handful at a time: loading every member — each with its own asset, thumbnail, and location query — made opening such a cluster cost thousands of round trips and megabytes of JSON. """ limit = max(1, min(limit, MAX_MEMBER_PAGE)) offset = max(0, offset) with self._session_factory() as session: cluster = session.get(DuplicateCluster, cluster_id) if cluster is None: return None member_total = int( session.scalar( select(func.count()) .select_from(DuplicateMember) .where(DuplicateMember.cluster_id == cluster_id) ) or 0 ) rows = list( session.execute( select(DuplicateMember) .where(DuplicateMember.cluster_id == cluster_id) .order_by(DuplicateMember.asset_id) .limit(limit) .offset(offset) ).scalars() ) evidence = self._member_evidence(session, [row.asset_id for row in rows]) members = [] for member in rows: try: member_evidence = json.loads(member.evidence) if member.evidence else {} except json.JSONDecodeError: member_evidence = {} asset, offline = evidence[member.asset_id] members.append( { "asset_id": member.asset_id, "role": member.role, "distance": member.distance, "evidence": member_evidence, "current_path": asset.current_path if asset else None, "byte_size": asset.byte_size if asset else None, "phash": asset.phash if asset else None, **offline, } ) # A full-resolution comparison of an offline original is impossible; the # UI asks for that named medium instead of guessing (concept §9). The # answer covers the whole cluster, not just this page, so a mount is not # discovered halfway through a review. mount_required = self._mount_required(session, cluster_id) return { "id": cluster.id, "method": cluster.method, "confidence": cluster.confidence, "state": cluster.state, "decision": cluster.decision, "canonical_asset_id": cluster.canonical_asset_id, "version": cluster.version, "requires_confirmation": cluster.method == Method.PERCEPTUAL.value, "mount_required": mount_required, "members": members, "member_total": member_total, "limit": limit, "offset": offset, } def _member_evidence(self, session, asset_ids: list[str]) -> dict: """``{asset_id: (asset, offline_evidence)}`` for one page, in three queries.""" if not asset_ids: return {} assets = { asset.id: asset for asset in session.execute( select(Asset).where(Asset.id.in_(asset_ids)) ).scalars() } previews: dict[str, list] = {} for thumbnail in session.execute( select(Thumbnail).where(Thumbnail.asset_id.in_(asset_ids)) ).scalars(): previews.setdefault(thumbnail.asset_id, []).append(thumbnail) location_ids = { asset.archive_location_id for asset in assets.values() if asset.archive_location_id } locations = ( { location.id: location for location in session.execute( select(ArchiveLocation).where(ArchiveLocation.id.in_(location_ids)) ).scalars() } if location_ids else {} ) return { asset_id: ( assets.get(asset_id), self._offline_evidence( assets.get(asset_id), locations=locations, thumbnails=previews.get(asset_id, []), ), ) for asset_id in asset_ids } def _mount_required(self, session, cluster_id: str) -> list[str]: """Archive media whose originals this cluster needs, across every member.""" rows = session.execute( select(ArchiveLocation.name) .select_from(DuplicateMember) .join(Asset, Asset.id == DuplicateMember.asset_id) .join(ArchiveLocation, ArchiveLocation.id == Asset.archive_location_id) .where( DuplicateMember.cluster_id == cluster_id, Asset.availability_state == availability.ARCHIVED_OFFLINE, ) .distinct() ).scalars() return sorted(rows) def _offline_evidence( self, asset: Asset | None, *, locations: dict, thumbnails: list ) -> dict: """What review can still rely on when a member's original is not readable. Takes the already-loaded locations and thumbnails for its page rather than querying per member (US07-06). """ if asset is None: return { "availability_state": None, "archive_location": None, "archive_location_id": None, "archive_path": None, "preview": {"state": "missing", "protected": False}, "requires_mount": False, } location = locations.get(asset.archive_location_id) preview = self._preview_evidence(thumbnails) archived = asset.availability_state in availability.ARCHIVED return { "availability_state": asset.availability_state, "archive_location": location.name if location else None, "archive_location_id": asset.archive_location_id, "archive_path": asset.archive_path, "preview": preview, # Offline archived members can still be compared through their retained # preview and hash evidence; only pixel-level review needs the medium. "requires_mount": archived and asset.availability_state == availability.ARCHIVED_OFFLINE and bool(location), } @staticmethod def _preview_evidence(rows: list) -> dict: ready = [r for r in rows if r.state == "ready" and r.path] if ready: best = max(ready, key=lambda r: (bool(r.protected), r.size or 0)) return {"state": "ready", "protected": bool(best.protected), "size": best.size} if rows: return {"state": "unsupported", "protected": False, "size": rows[0].size} return {"state": "missing", "protected": False, "size": None} # ── decisions ──────────────────────────────────────────────────────────── def decide( self, cluster_id: str, decision: Decision | str, *, canonical_asset_id: str | None = None, expected_version: int | None = None, reason: str | None = None, ) -> dict: decision = Decision(decision) now = datetime.now(timezone.utc) with self._session_factory() as session: cluster = session.get(DuplicateCluster, cluster_id) if cluster is None: raise DuplicateError(f"unknown cluster {cluster_id}") if expected_version is not None and cluster.version != expected_version: raise ConflictError( f"cluster {cluster_id} is at version {cluster.version}, " f"expected {expected_version}" ) member_ids = [ m.asset_id for m in session.execute( select(DuplicateMember).where(DuplicateMember.cluster_id == cluster_id) ).scalars() ] self._reset_members(session, cluster, member_ids) self._remove_negative_links(session, member_ids) if decision == Decision.CANONICAL: if canonical_asset_id not in member_ids: raise DuplicateError("canonical_asset_id must be a cluster member") self._apply_canonical(session, cluster, member_ids, canonical_asset_id) cluster.decision = Decision.CANONICAL.value cluster.canonical_asset_id = canonical_asset_id cluster.state = ClusterState.DECIDED.value elif decision == Decision.NOT_DUPLICATE: self._add_negative_links(session, member_ids, reason) cluster.decision = Decision.NOT_DUPLICATE.value cluster.canonical_asset_id = None cluster.state = ClusterState.DISMISSED.value else: # DEFERRED cluster.decision = Decision.DEFERRED.value cluster.canonical_asset_id = None cluster.state = ClusterState.DEFERRED.value cluster.decided_at = now cluster.updated_at = now cluster.version += 1 session.commit() return self._snapshot(session, cluster_id) def _reset_members(self, session, cluster, member_ids): for member in session.execute( select(DuplicateMember).where(DuplicateMember.cluster_id == cluster.id) ).scalars(): member.role = Role.MEMBER.value for asset_id in member_ids: session.get(Asset, asset_id).canonical_asset_id = None @staticmethod def _add_negative_links(session, member_ids, reason): existing = { _pair(link.asset_a, link.asset_b) for link in session.execute(select(DuplicateNegativeLink)).scalars() } for i in range(len(member_ids)): for j in range(i + 1, len(member_ids)): a, b = _pair(member_ids[i], member_ids[j]) if (a, b) not in existing: session.add( DuplicateNegativeLink(asset_a=a, asset_b=b, reason=reason) ) @staticmethod def _remove_negative_links(session, member_ids): member_set = set(member_ids) for link in session.execute(select(DuplicateNegativeLink)).scalars(): if link.asset_a in member_set and link.asset_b in member_set: session.delete(link) def _snapshot(self, session, cluster_id) -> dict: """A cluster and a *bounded* preview of its members. The list view shows a count and a few ids; a snapshot that loaded every member turned one page of 200 clusters into hundreds of thousands of rows (US07-06). ``member_total`` is the honest count either way. """ cluster = session.get(DuplicateCluster, cluster_id) member_total = int( session.scalar( select(func.count()) .select_from(DuplicateMember) .where(DuplicateMember.cluster_id == cluster_id) ) or 0 ) members = session.execute( select(DuplicateMember) .where(DuplicateMember.cluster_id == cluster_id) .order_by(DuplicateMember.asset_id) .limit(SNAPSHOT_MEMBER_PREVIEW) ).scalars() return { "member_total": member_total, "id": cluster.id, "method": cluster.method, "confidence": cluster.confidence, "state": cluster.state, "decision": cluster.decision, "canonical_asset_id": cluster.canonical_asset_id, "version": cluster.version, "members": sorted( ({"asset_id": m.asset_id, "role": m.role, "distance": m.distance} for m in members), key=lambda m: m["asset_id"], ), }