"""Offline identity and review evidence (US06-03). An archived photo is not gone: it keeps its identity, its hashes, and enough evidence to be recognised in a duplicate cluster while its medium sits in a drawer. Every case here archives a *real* album through the real transfer, then takes the medium away by removing its marker — the same thing the service sees when an external disk is unplugged — and asks whether the application still tells the truth about where the bytes are. The distinction that matters throughout: an unmounted medium is ``archived_offline`` (expected, harmless), a mounted medium with a hole in it is ``missing_unexpected`` (needs a human). Confusing the two is how an archive quietly loses a photo. """ import shutil import uuid from datetime import datetime, timezone import numpy as np import pytest from fastapi.testclient import TestClient from PIL import Image from sqlalchemy import select from photo_pipeline.api.app import create_app from photo_pipeline.config import Config from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations from photo_pipeline.models import Asset, Thumbnail, UploadBatch, UploadItem from photo_pipeline.services.archive_transfer import ArchiveTransferService from photo_pipeline.services.archives import MARKER_NAME, ArchiveService from photo_pipeline.services.availability import ( ACTIVE, ARCHIVED_OFFLINE, ARCHIVED_ONLINE, MISSING_UNEXPECTED, ) from photo_pipeline.services.duplicates import ClusterState, DuplicateService, Method from photo_pipeline.services.hashing import sha256_file from photo_pipeline.services.inventory import InventoryService from photo_pipeline.services.thumbnails import PROTECTED_SIZE, ThumbnailService, ThumbnailUnavailable pytestmark = pytest.mark.phase_f # part of the Phase F acceptance gate (US06-06) NOW = datetime(2026, 1, 1, tzinfo=timezone.utc) # ── environment ────────────────────────────────────────────────────────────── def _env(tmp_path): (tmp_path / "data").mkdir(exist_ok=True) lib = tmp_path / "lib" lib.mkdir(exist_ok=True) archive = tmp_path / "archive" archive.mkdir(exist_ok=True) config = Config.from_env( { "PHOTO_PIPELINE_DATA_DIR": str(tmp_path / "data"), "PHOTO_PIPELINE_LIBRARY_ROOTS": str(lib), "PHOTO_PIPELINE_ARCHIVE_FREE_SPACE_RESERVE_BYTES": "0", } ) run_migrations(config.database_url) return config, create_session_factory(create_db_engine(config.database_url)), lib, archive def structured(path, seed, size=(256, 192)): """A deterministic, decodable photo — previews and pHashes must be real.""" path.parent.mkdir(parents=True, exist_ok=True) rng = np.random.default_rng(seed) w, h = size base = np.zeros((h, w, 3), dtype=np.uint8) for _ in range(6): x0 = int(rng.integers(0, w - 60)) y0 = int(rng.integers(0, h - 60)) base[y0 : y0 + 60, x0 : x0 + 60] = rng.integers(0, 256, 3) grad = np.linspace(0, 120, w, dtype=np.uint8) base[:, :, 0] = np.clip(base[:, :, 0].astype(int) + grad[None, :], 0, 255) Image.fromarray(base).save(path, quality=95) return path def resized_copy(src, dst, scale=0.5): with Image.open(src) as image: image.resize( (int(image.width * scale), int(image.height * scale)), Image.LANCZOS ).save(dst, quality=95) return dst def _uploaded(sf, lib, album="rome", seeds=(1, 2)): """A scanned album carrying the verified upload evidence archiving requires.""" folder = lib / album for index, seed in enumerate(seeds): structured(folder / f"{index}.jpg", seed) result = InventoryService(sf).scan(lib) with sf() as session: batch_id = str(uuid.uuid4()) session.add( UploadBatch( id=batch_id, album=album, folder=str(folder), album_name=album, state="succeeded", preflight_token="v1:test", outcome_state="verified", created_at=NOW, ) ) for path, asset_id in result.asset_ids.items(): session.add( UploadItem( batch_id=batch_id, asset_id=asset_id, path=path, sha256=sha256_file(path), sha1="0" * 40, state="sent", outcome="uploaded", ) ) session.commit() return folder, result.asset_ids def _archive(sf, config, archive, albums=None): service = ArchiveService(sf, config=config) location = service.register("external", str(archive)) token = service.preflight(location["id"], albums)["token"] transfers = ArchiveTransferService(sf, config=config) plan = transfers.create(location["id"], albums, token=token) transfers.apply(plan["id"]) return location def _unmount(archive): """Take the medium away the way a real one goes: its marker stops answering.""" (archive / MARKER_NAME).rename(archive / f"{MARKER_NAME}.away") def _remount(archive): (archive / f"{MARKER_NAME}.away").rename(archive / MARKER_NAME) def _assets(sf): with sf() as session: return {asset.id: asset for asset in session.scalars(select(Asset))} # ── availability ───────────────────────────────────────────────────────────── def test_archived_assets_report_online_offline_and_missing(tmp_path): config, sf, lib, archive = _env(tmp_path) _, ids = _uploaded(sf, lib) _archive(sf, config, archive) inventory = InventoryService(sf) states = {a.availability_state for a in _assets(sf).values()} assert states == {ARCHIVED_ONLINE} _unmount(archive) inventory.scan(lib) # the album folder is gone from the active roots assert {a.availability_state for a in _assets(sf).values()} == {ARCHIVED_OFFLINE} assert all(a.missing_at is None for a in _assets(sf).values()) # not "missing" _remount(archive) inventory.scan(lib) assert {a.availability_state for a in _assets(sf).values()} == {ARCHIVED_ONLINE} # Mounted medium, absent file: that is not an offline archive, it needs a human. victim = sorted(ids.values())[0] with sf() as session: asset = session.get(Asset, victim) (archive / asset.archive_path).unlink() inventory.scan(lib) assert _assets(sf)[victim].availability_state == MISSING_UNEXPECTED def test_scan_never_prunes_or_flags_offline_assets(tmp_path): config, sf, lib, archive = _env(tmp_path) _, ids = _uploaded(sf, lib) _archive(sf, config, archive) _unmount(archive) before = _assets(sf) result = InventoryService(sf).scan(lib) assert result.counts.get("missing") is None after = _assets(sf) assert set(after) == set(before) == set(ids.values()) for asset in after.values(): assert asset.availability_state == ARCHIVED_OFFLINE assert asset.current_sha256 and asset.pixel_sha256 # hashes retained assert asset.archive_location_id and asset.archive_path def test_active_missing_file_is_missing_unexpected_not_offline(tmp_path): config, sf, lib, _archive_root = _env(tmp_path) structured(lib / "loose" / "a.jpg", 7) ids = InventoryService(sf).scan(lib).asset_ids asset_id = next(iter(ids.values())) (lib / "loose" / "a.jpg").unlink() InventoryService(sf).scan(lib) asset = _assets(sf)[asset_id] assert asset.availability_state == MISSING_UNEXPECTED assert asset.missing_at is not None # ── deduplication against archived originals ───────────────────────────────── def test_exact_copy_of_offline_asset_links_to_archived_canonical(tmp_path): config, sf, lib, archive = _env(tmp_path) folder, ids = _uploaded(sf, lib, seeds=(1,)) archived_id = next(iter(ids.values())) original = next(iter(ids)) kept = tmp_path / "kept.jpg" shutil.copy2(original, kept) _archive(sf, config, archive) _unmount(archive) # The same photo turns up again in the active library while the disk is away. (lib / "inbox").mkdir(parents=True, exist_ok=True) shutil.copy2(kept, lib / "inbox" / "again.jpg") scan = InventoryService(sf).scan(lib) new_id = scan.asset_ids[str(lib / "inbox" / "again.jpg")] assert scan.occurrences[str(lib / "inbox" / "again.jpg")] == "copied" clusters = DuplicateService(sf).detect().clusters exact = [c for c in clusters if c["method"] == Method.EXACT.value] assert len(exact) == 1 cluster = exact[0] # Byte-identical: linked directly, and the archived original stays canonical. assert cluster["state"] == ClusterState.DECIDED.value assert cluster["canonical_asset_id"] == archived_id assert _assets(sf)[new_id].canonical_asset_id == archived_id def test_fuzzy_copy_of_offline_asset_requires_review_and_names_the_medium(tmp_path): config, sf, lib, archive = _env(tmp_path) folder, ids = _uploaded(sf, lib, seeds=(1,)) archived_id = next(iter(ids.values())) original = next(iter(ids)) variant_source = resized_copy(original, tmp_path / "small.jpg") _archive(sf, config, archive) _unmount(archive) (lib / "inbox").mkdir(parents=True, exist_ok=True) shutil.copy2(variant_source, lib / "inbox" / "small.jpg") InventoryService(sf).scan(lib) duplicates = DuplicateService(sf) clusters = duplicates.detect().clusters perceptual = [c for c in clusters if c["method"] == Method.PERCEPTUAL.value] assert len(perceptual) == 1 detail = duplicates.get_cluster(perceptual[0]["id"]) assert detail["state"] == ClusterState.OPEN.value # never auto-decided assert detail["requires_confirmation"] is True assert detail["mount_required"] == ["external"] # full-resolution needs the disk archived = next(m for m in detail["members"] if m["asset_id"] == archived_id) assert archived["availability_state"] == ARCHIVED_OFFLINE assert archived["current_path"] is None assert archived["archive_path"] and archived["evidence"]["phash"] # The retained preview is what makes the offline member reviewable at all. assert archived["preview"] == {"state": "ready", "protected": True, "size": PROTECTED_SIZE} # ── protected review evidence ──────────────────────────────────────────────── def test_protected_preview_survives_quota_and_serves_while_offline(tmp_path): config, sf, lib, archive = _env(tmp_path) _, ids = _uploaded(sf, lib, seeds=(1,)) asset_id = next(iter(ids.values())) _archive(sf, config, archive) _unmount(archive) thumbnails = ThumbnailService(sf, config) served = thumbnails.generate(asset_id, PROTECTED_SIZE) assert served.exists() # rendered before the original left, not from the medium # An aggressive quota may empty the cache, but not this evidence. tight = ThumbnailService(sf, config.model_copy(update={"thumbnail_cache_quota_bytes": 1})) tight._enforce_quota() assert served.exists() with sf() as session: row = session.scalar(select(Thumbnail).where(Thumbnail.asset_id == asset_id)) assert row.protected is True assert thumbnails.evidence(asset_id)["state"] == "ready" def test_offline_asset_without_preview_reports_unavailable(tmp_path): """No preview and no medium is an honest 409, never a wrong picture.""" config, sf, lib, archive = _env(tmp_path) _, ids = _uploaded(sf, lib, seeds=(1,)) asset_id = next(iter(ids.values())) _archive(sf, config, archive) with sf() as session: for row in session.scalars(select(Thumbnail).where(Thumbnail.asset_id == asset_id)): session.delete(row) session.commit() _unmount(archive) with pytest.raises(ThumbnailUnavailable): ThumbnailService(sf, config).generate(asset_id, 256) _remount(archive) # mounted again: the archived original is readable assert ThumbnailService(sf, config).generate(asset_id, 256).exists() def test_offline_asset_is_browsable_through_the_api(tmp_path): """The browser sees an archived asset, its medium, and its preview — offline.""" config, sf, lib, archive = _env(tmp_path) _, ids = _uploaded(sf, lib, seeds=(1,)) asset_id = next(iter(ids.values())) _archive(sf, config, archive) _unmount(archive) with TestClient(create_app(config)) as client: client.post("/api/v1/inventory/scan") listed = client.get("/api/v1/inventory/assets", params={"availability": ARCHIVED_OFFLINE}) assert listed.status_code == 200 item = next(row for row in listed.json()["items"] if row["id"] == asset_id) assert item["current_path"] is None assert item["archive_path"] == "rome/0.jpg" assert item["missing"] is False # Searching by the archived path still finds it. found = client.get("/api/v1/inventory/assets", params={"q": "rome"}).json() assert [row["id"] for row in found["items"]] == [asset_id] preview = client.get(f"/api/v1/assets/{asset_id}/thumbnail", params={"size": 1280}) assert preview.status_code == 200 assert preview.headers["content-type"] == "image/webp" def test_offline_state_is_stable_across_restart(tmp_path): config, sf, lib, archive = _env(tmp_path) _, ids = _uploaded(sf, lib, seeds=(1, 2)) _archive(sf, config, archive) _unmount(archive) InventoryService(sf).scan(lib) before = { asset_id: ( asset.availability_state, asset.archive_path, asset.current_sha256, asset.phash, ) for asset_id, asset in _assets(sf).items() } # Restart: a fresh engine and session factory against the same database. restarted = create_session_factory(create_db_engine(config.database_url)) after = { asset_id: ( asset.availability_state, asset.archive_path, asset.current_sha256, asset.phash, ) for asset_id, asset in _assets(restarted).items() } assert after == before assert set(after) == set(ids.values()) # And the medium coming back is picked up by the restarted process. _remount(archive) assert ArchiveService(restarted, config=config).locations()[0]["state"] == "online" assert {a.availability_state for a in _assets(restarted).values()} == {ARCHIVED_ONLINE} assert ACTIVE not in {a.availability_state for a in _assets(restarted).values()}