domverse/photoanalyzer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects 1 Releases Wiki Activity

US06-03: Preserve Offline Identity and Review Evidence (#79)

Browse Source
This commit was merged in pull request #79.
This commit is contained in:
domverse
2026-08-16 19:21:29 +02:00
parent b90d1718be
commit 439eb9971e
11 changed files with 835 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

128
photo_pipeline/services/duplicates.py
View File

@@ -11,6 +11,12 @@ Detection runs in two categories:
band (NEAR/SIMILAR). These are review candidates: never decided automatically, and
negative-linked pairs are suppressed so a rejected pair is not re-suggested.
Archived assets stay in both indexes (US06-03): a new active copy of an archived
original is recognised through its hashes even while the medium is offline, and
cluster review falls back to the retained protected preview plus hash evidence.
An exact/pixel match links straight to the archived canonical; a perceptual match
is a review candidate that names the medium to mount for a pixel-level decision.
Decisions (``canonical`` / ``not_duplicate`` / ``deferred``) persist with evidence,
use optimistic version checks, are reversible, and can never form a canonical cycle.
A new content-identical member of an already-decided cluster inherits the established
@@ -34,12 +40,14 @@ from sqlalchemy import func, select
from sqlalchemy.orm import sessionmaker
from photo_pipeline.models import (
ArchiveLocation,
Asset,
DuplicateCluster,
DuplicateMember,
DuplicateNegativeLink,
Thumbnail,
)
from photo_pipeline.services import hashing
from photo_pipeline.services import availability, hashing
NEAR_MAX = 5
SIMILAR_MAX = 10
@@ -127,18 +135,18 @@ class DuplicateService:
# ── perceptual hash backfill ───────────────────────────────────────────
def ensure_phashes(self) -> int:
"""Hash whatever is readable now — an archived asset keeps the hash it
already has, and gains one whenever its medium happens to be mounted."""
updated = 0
with self._session_factory() as session:
assets = session.execute(
select(Asset).where(
Asset.availability_state == "active",
Asset.current_path.isnot(None),
)
).scalars()
assets = session.execute(select(Asset)).scalars()
for asset in assets:
if asset.phash is not None and asset.phash_version == hashing.PHASH_VERSION:
continue
value = hashing.safe_phash(asset.current_path)
source = availability.readable_path(session, asset)
if source is None:
continue
value = hashing.safe_phash(str(source))
if value is not None:
asset.phash = value
asset.phash_version = hashing.PHASH_VERSION
@@ -146,20 +154,40 @@ class DuplicateService:
session.commit()
return updated
def ensure_phash(self, asset_id: str, *, source=None) -> str | None:
"""Backfill one asset's perceptual hash while its bytes are still readable.
Archiving calls this before the original leaves — passing the archive copy
as ``source``, since the database does not point at it yet — because an
asset without a pHash silently drops out of the fuzzy index the moment its
medium is away.
"""
with self._session_factory() as session:
asset = session.get(Asset, asset_id)
if asset is None:
return None
if asset.phash is not None and asset.phash_version == hashing.PHASH_VERSION:
return asset.phash
source = source or availability.readable_path(session, asset)
if source is None:
return None
value = hashing.safe_phash(str(source))
if value is not None:
asset.phash = value
asset.phash_version = hashing.PHASH_VERSION
session.commit()
return value
# ── detection ──────────────────────────────────────────────────────────
def detect(self) -> DetectionReport:
self.ensure_phashes()
now = datetime.now(timezone.utc)
report = DetectionReport()
with self._session_factory() as session:
assets = list(
session.execute(
select(Asset).where(
Asset.availability_state == "active",
Asset.current_path.isnot(None),
)
).scalars()
)
# Every known asset stays in the indexes, archived or not: a copy of an
# archived original must be recognised as a duplicate rather than
# treated as a new photo (concept §9, invariant 12).
assets = list(session.execute(select(Asset)).scalars())
by_id = {a.id: a for a in assets}
negatives = {
_pair(link.asset_a, link.asset_b)
@@ -431,10 +459,19 @@ class DuplicateService:
@staticmethod
def _recommend_canonical(ids, by_id) -> str:
# ponytail: largest file, path as deterministic tie-break. The concept's
# richer policy (resolution, least recompression, metadata richness) lands
# with the review UI story.
return max(ids, key=lambda i: (by_id[i].byte_size or 0, by_id[i].current_path or ""))
# ponytail: largest file, then the archived copy, then path as a
# deterministic tie-break. Archived wins ties because it is the reviewed,
# uploaded original — a fresh active copy must not demote it to a variant.
# The concept's richer policy (resolution, least recompression, metadata
# richness) lands with the review UI story.
return max(
ids,
key=lambda i: (
by_id[i].byte_size or 0,
by_id[i].availability_state in availability.ARCHIVED,
by_id[i].current_path or by_id[i].archive_path or "",
),
)
def _apply_canonical(self, session, cluster, ids, canonical_id):
for member in session.execute(
@@ -507,9 +544,15 @@ class DuplicateService:
"current_path": asset.current_path if asset else None,
"byte_size": asset.byte_size if asset else None,
"phash": asset.phash if asset else None,
**self._offline_evidence(session, asset),
}
)
members.sort(key=lambda m: m["asset_id"])
# A full-resolution comparison of an offline original is impossible; the
# UI asks for that named medium instead of guessing (concept §9).
mount_required = sorted(
{m["archive_location"] for m in members if m["requires_mount"]}
)
return {
"id": cluster.id,
"method": cluster.method,
@@ -519,9 +562,54 @@ class DuplicateService:
"canonical_asset_id": cluster.canonical_asset_id,
"version": cluster.version,
"requires_confirmation": cluster.method == Method.PERCEPTUAL.value,
"mount_required": mount_required,
"members": members,
}
def _offline_evidence(self, session, asset: Asset | None) -> dict:
"""What review can still rely on when a member's original is not readable."""
if asset is None:
return {
"availability_state": None,
"archive_location": None,
"archive_location_id": None,
"archive_path": None,
"preview": {"state": "missing", "protected": False},
"requires_mount": False,
}
location = (
session.get(ArchiveLocation, asset.archive_location_id)
if asset.archive_location_id
else None
)
preview = self._preview_evidence(session, asset.id)
archived = asset.availability_state in availability.ARCHIVED
return {
"availability_state": asset.availability_state,
"archive_location": location.name if location else None,
"archive_location_id": asset.archive_location_id,
"archive_path": asset.archive_path,
"preview": preview,
# Offline archived members can still be compared through their retained
# preview and hash evidence; only pixel-level review needs the medium.
"requires_mount": archived
and asset.availability_state == availability.ARCHIVED_OFFLINE
and bool(location),
}
@staticmethod
def _preview_evidence(session, asset_id: str) -> dict:
rows = list(
session.execute(select(Thumbnail).where(Thumbnail.asset_id == asset_id)).scalars()
)
ready = [r for r in rows if r.state == "ready" and r.path]
if ready:
best = max(ready, key=lambda r: (bool(r.protected), r.size or 0))
return {"state": "ready", "protected": bool(best.protected), "size": best.size}
if rows:
return {"state": "unsupported", "protected": False, "size": rows[0].size}
return {"state": "missing", "protected": False, "size": None}
# ── decisions ────────────────────────────────────────────────────────────
def decide(
self,