US06-03: Preserve Offline Identity and Review Evidence (#79)
This commit was merged in pull request #79.
This commit is contained in:
377
tests/integration/test_offline_assets.py
Normal file
377
tests/integration/test_offline_assets.py
Normal file
@@ -0,0 +1,377 @@
|
||||
"""Offline identity and review evidence (US06-03).
|
||||
|
||||
An archived photo is not gone: it keeps its identity, its hashes, and enough
|
||||
evidence to be recognised in a duplicate cluster while its medium sits in a
|
||||
drawer. Every case here archives a *real* album through the real transfer, then
|
||||
takes the medium away by removing its marker — the same thing the service sees
|
||||
when an external disk is unplugged — and asks whether the application still tells
|
||||
the truth about where the bytes are.
|
||||
|
||||
The distinction that matters throughout: an unmounted medium is
|
||||
``archived_offline`` (expected, harmless), a mounted medium with a hole in it is
|
||||
``missing_unexpected`` (needs a human). Confusing the two is how an archive
|
||||
quietly loses a photo.
|
||||
"""
|
||||
|
||||
import shutil
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from PIL import Image
|
||||
from sqlalchemy import select
|
||||
|
||||
from photo_pipeline.api.app import create_app
|
||||
from photo_pipeline.config import Config
|
||||
from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
|
||||
from photo_pipeline.models import Asset, Thumbnail, UploadBatch, UploadItem
|
||||
from photo_pipeline.services.archive_transfer import ArchiveTransferService
|
||||
from photo_pipeline.services.archives import MARKER_NAME, ArchiveService
|
||||
from photo_pipeline.services.availability import (
|
||||
ACTIVE,
|
||||
ARCHIVED_OFFLINE,
|
||||
ARCHIVED_ONLINE,
|
||||
MISSING_UNEXPECTED,
|
||||
)
|
||||
from photo_pipeline.services.duplicates import ClusterState, DuplicateService, Method
|
||||
from photo_pipeline.services.hashing import sha256_file
|
||||
from photo_pipeline.services.inventory import InventoryService
|
||||
from photo_pipeline.services.thumbnails import PROTECTED_SIZE, ThumbnailService, ThumbnailUnavailable
|
||||
|
||||
pytestmark = pytest.mark.phase_f # part of the Phase F acceptance gate (US06-06)
|
||||
|
||||
NOW = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
# ── environment ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _env(tmp_path):
|
||||
(tmp_path / "data").mkdir(exist_ok=True)
|
||||
lib = tmp_path / "lib"
|
||||
lib.mkdir(exist_ok=True)
|
||||
archive = tmp_path / "archive"
|
||||
archive.mkdir(exist_ok=True)
|
||||
config = Config.from_env(
|
||||
{
|
||||
"PHOTO_PIPELINE_DATA_DIR": str(tmp_path / "data"),
|
||||
"PHOTO_PIPELINE_LIBRARY_ROOTS": str(lib),
|
||||
"PHOTO_PIPELINE_ARCHIVE_FREE_SPACE_RESERVE_BYTES": "0",
|
||||
}
|
||||
)
|
||||
run_migrations(config.database_url)
|
||||
return config, create_session_factory(create_db_engine(config.database_url)), lib, archive
|
||||
|
||||
|
||||
def structured(path, seed, size=(256, 192)):
|
||||
"""A deterministic, decodable photo — previews and pHashes must be real."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
rng = np.random.default_rng(seed)
|
||||
w, h = size
|
||||
base = np.zeros((h, w, 3), dtype=np.uint8)
|
||||
for _ in range(6):
|
||||
x0 = int(rng.integers(0, w - 60))
|
||||
y0 = int(rng.integers(0, h - 60))
|
||||
base[y0 : y0 + 60, x0 : x0 + 60] = rng.integers(0, 256, 3)
|
||||
grad = np.linspace(0, 120, w, dtype=np.uint8)
|
||||
base[:, :, 0] = np.clip(base[:, :, 0].astype(int) + grad[None, :], 0, 255)
|
||||
Image.fromarray(base).save(path, quality=95)
|
||||
return path
|
||||
|
||||
|
||||
def resized_copy(src, dst, scale=0.5):
|
||||
with Image.open(src) as image:
|
||||
image.resize(
|
||||
(int(image.width * scale), int(image.height * scale)), Image.LANCZOS
|
||||
).save(dst, quality=95)
|
||||
return dst
|
||||
|
||||
|
||||
def _uploaded(sf, lib, album="rome", seeds=(1, 2)):
|
||||
"""A scanned album carrying the verified upload evidence archiving requires."""
|
||||
folder = lib / album
|
||||
for index, seed in enumerate(seeds):
|
||||
structured(folder / f"{index}.jpg", seed)
|
||||
result = InventoryService(sf).scan(lib)
|
||||
with sf() as session:
|
||||
batch_id = str(uuid.uuid4())
|
||||
session.add(
|
||||
UploadBatch(
|
||||
id=batch_id,
|
||||
album=album,
|
||||
folder=str(folder),
|
||||
album_name=album,
|
||||
state="succeeded",
|
||||
preflight_token="v1:test",
|
||||
outcome_state="verified",
|
||||
created_at=NOW,
|
||||
)
|
||||
)
|
||||
for path, asset_id in result.asset_ids.items():
|
||||
session.add(
|
||||
UploadItem(
|
||||
batch_id=batch_id,
|
||||
asset_id=asset_id,
|
||||
path=path,
|
||||
sha256=sha256_file(path),
|
||||
sha1="0" * 40,
|
||||
state="sent",
|
||||
outcome="uploaded",
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
return folder, result.asset_ids
|
||||
|
||||
|
||||
def _archive(sf, config, archive, albums=None):
|
||||
service = ArchiveService(sf, config=config)
|
||||
location = service.register("external", str(archive))
|
||||
token = service.preflight(location["id"], albums)["token"]
|
||||
transfers = ArchiveTransferService(sf, config=config)
|
||||
plan = transfers.create(location["id"], albums, token=token)
|
||||
transfers.apply(plan["id"])
|
||||
return location
|
||||
|
||||
|
||||
def _unmount(archive):
|
||||
"""Take the medium away the way a real one goes: its marker stops answering."""
|
||||
(archive / MARKER_NAME).rename(archive / f"{MARKER_NAME}.away")
|
||||
|
||||
|
||||
def _remount(archive):
|
||||
(archive / f"{MARKER_NAME}.away").rename(archive / MARKER_NAME)
|
||||
|
||||
|
||||
def _assets(sf):
|
||||
with sf() as session:
|
||||
return {asset.id: asset for asset in session.scalars(select(Asset))}
|
||||
|
||||
|
||||
# ── availability ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_archived_assets_report_online_offline_and_missing(tmp_path):
|
||||
config, sf, lib, archive = _env(tmp_path)
|
||||
_, ids = _uploaded(sf, lib)
|
||||
_archive(sf, config, archive)
|
||||
inventory = InventoryService(sf)
|
||||
|
||||
states = {a.availability_state for a in _assets(sf).values()}
|
||||
assert states == {ARCHIVED_ONLINE}
|
||||
|
||||
_unmount(archive)
|
||||
inventory.scan(lib) # the album folder is gone from the active roots
|
||||
assert {a.availability_state for a in _assets(sf).values()} == {ARCHIVED_OFFLINE}
|
||||
assert all(a.missing_at is None for a in _assets(sf).values()) # not "missing"
|
||||
|
||||
_remount(archive)
|
||||
inventory.scan(lib)
|
||||
assert {a.availability_state for a in _assets(sf).values()} == {ARCHIVED_ONLINE}
|
||||
|
||||
# Mounted medium, absent file: that is not an offline archive, it needs a human.
|
||||
victim = sorted(ids.values())[0]
|
||||
with sf() as session:
|
||||
asset = session.get(Asset, victim)
|
||||
(archive / asset.archive_path).unlink()
|
||||
inventory.scan(lib)
|
||||
assert _assets(sf)[victim].availability_state == MISSING_UNEXPECTED
|
||||
|
||||
|
||||
def test_scan_never_prunes_or_flags_offline_assets(tmp_path):
|
||||
config, sf, lib, archive = _env(tmp_path)
|
||||
_, ids = _uploaded(sf, lib)
|
||||
_archive(sf, config, archive)
|
||||
_unmount(archive)
|
||||
|
||||
before = _assets(sf)
|
||||
result = InventoryService(sf).scan(lib)
|
||||
|
||||
assert result.counts.get("missing") is None
|
||||
after = _assets(sf)
|
||||
assert set(after) == set(before) == set(ids.values())
|
||||
for asset in after.values():
|
||||
assert asset.availability_state == ARCHIVED_OFFLINE
|
||||
assert asset.current_sha256 and asset.pixel_sha256 # hashes retained
|
||||
assert asset.archive_location_id and asset.archive_path
|
||||
|
||||
|
||||
def test_active_missing_file_is_missing_unexpected_not_offline(tmp_path):
|
||||
config, sf, lib, _archive_root = _env(tmp_path)
|
||||
structured(lib / "loose" / "a.jpg", 7)
|
||||
ids = InventoryService(sf).scan(lib).asset_ids
|
||||
asset_id = next(iter(ids.values()))
|
||||
|
||||
(lib / "loose" / "a.jpg").unlink()
|
||||
InventoryService(sf).scan(lib)
|
||||
|
||||
asset = _assets(sf)[asset_id]
|
||||
assert asset.availability_state == MISSING_UNEXPECTED
|
||||
assert asset.missing_at is not None
|
||||
|
||||
|
||||
# ── deduplication against archived originals ─────────────────────────────────
|
||||
|
||||
|
||||
def test_exact_copy_of_offline_asset_links_to_archived_canonical(tmp_path):
|
||||
config, sf, lib, archive = _env(tmp_path)
|
||||
folder, ids = _uploaded(sf, lib, seeds=(1,))
|
||||
archived_id = next(iter(ids.values()))
|
||||
original = next(iter(ids))
|
||||
kept = tmp_path / "kept.jpg"
|
||||
shutil.copy2(original, kept)
|
||||
_archive(sf, config, archive)
|
||||
_unmount(archive)
|
||||
|
||||
# The same photo turns up again in the active library while the disk is away.
|
||||
(lib / "inbox").mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(kept, lib / "inbox" / "again.jpg")
|
||||
scan = InventoryService(sf).scan(lib)
|
||||
new_id = scan.asset_ids[str(lib / "inbox" / "again.jpg")]
|
||||
assert scan.occurrences[str(lib / "inbox" / "again.jpg")] == "copied"
|
||||
|
||||
clusters = DuplicateService(sf).detect().clusters
|
||||
exact = [c for c in clusters if c["method"] == Method.EXACT.value]
|
||||
assert len(exact) == 1
|
||||
cluster = exact[0]
|
||||
# Byte-identical: linked directly, and the archived original stays canonical.
|
||||
assert cluster["state"] == ClusterState.DECIDED.value
|
||||
assert cluster["canonical_asset_id"] == archived_id
|
||||
assert _assets(sf)[new_id].canonical_asset_id == archived_id
|
||||
|
||||
|
||||
def test_fuzzy_copy_of_offline_asset_requires_review_and_names_the_medium(tmp_path):
|
||||
config, sf, lib, archive = _env(tmp_path)
|
||||
folder, ids = _uploaded(sf, lib, seeds=(1,))
|
||||
archived_id = next(iter(ids.values()))
|
||||
original = next(iter(ids))
|
||||
variant_source = resized_copy(original, tmp_path / "small.jpg")
|
||||
_archive(sf, config, archive)
|
||||
_unmount(archive)
|
||||
|
||||
(lib / "inbox").mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(variant_source, lib / "inbox" / "small.jpg")
|
||||
InventoryService(sf).scan(lib)
|
||||
|
||||
duplicates = DuplicateService(sf)
|
||||
clusters = duplicates.detect().clusters
|
||||
perceptual = [c for c in clusters if c["method"] == Method.PERCEPTUAL.value]
|
||||
assert len(perceptual) == 1
|
||||
|
||||
detail = duplicates.get_cluster(perceptual[0]["id"])
|
||||
assert detail["state"] == ClusterState.OPEN.value # never auto-decided
|
||||
assert detail["requires_confirmation"] is True
|
||||
assert detail["mount_required"] == ["external"] # full-resolution needs the disk
|
||||
archived = next(m for m in detail["members"] if m["asset_id"] == archived_id)
|
||||
assert archived["availability_state"] == ARCHIVED_OFFLINE
|
||||
assert archived["current_path"] is None
|
||||
assert archived["archive_path"] and archived["evidence"]["phash"]
|
||||
# The retained preview is what makes the offline member reviewable at all.
|
||||
assert archived["preview"] == {"state": "ready", "protected": True, "size": PROTECTED_SIZE}
|
||||
|
||||
|
||||
# ── protected review evidence ────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_protected_preview_survives_quota_and_serves_while_offline(tmp_path):
|
||||
config, sf, lib, archive = _env(tmp_path)
|
||||
_, ids = _uploaded(sf, lib, seeds=(1,))
|
||||
asset_id = next(iter(ids.values()))
|
||||
_archive(sf, config, archive)
|
||||
_unmount(archive)
|
||||
|
||||
thumbnails = ThumbnailService(sf, config)
|
||||
served = thumbnails.generate(asset_id, PROTECTED_SIZE)
|
||||
assert served.exists() # rendered before the original left, not from the medium
|
||||
|
||||
# An aggressive quota may empty the cache, but not this evidence.
|
||||
tight = ThumbnailService(sf, config.model_copy(update={"thumbnail_cache_quota_bytes": 1}))
|
||||
tight._enforce_quota()
|
||||
assert served.exists()
|
||||
with sf() as session:
|
||||
row = session.scalar(select(Thumbnail).where(Thumbnail.asset_id == asset_id))
|
||||
assert row.protected is True
|
||||
assert thumbnails.evidence(asset_id)["state"] == "ready"
|
||||
|
||||
|
||||
def test_offline_asset_without_preview_reports_unavailable(tmp_path):
|
||||
"""No preview and no medium is an honest 409, never a wrong picture."""
|
||||
config, sf, lib, archive = _env(tmp_path)
|
||||
_, ids = _uploaded(sf, lib, seeds=(1,))
|
||||
asset_id = next(iter(ids.values()))
|
||||
_archive(sf, config, archive)
|
||||
with sf() as session:
|
||||
for row in session.scalars(select(Thumbnail).where(Thumbnail.asset_id == asset_id)):
|
||||
session.delete(row)
|
||||
session.commit()
|
||||
_unmount(archive)
|
||||
|
||||
with pytest.raises(ThumbnailUnavailable):
|
||||
ThumbnailService(sf, config).generate(asset_id, 256)
|
||||
|
||||
_remount(archive) # mounted again: the archived original is readable
|
||||
assert ThumbnailService(sf, config).generate(asset_id, 256).exists()
|
||||
|
||||
|
||||
def test_offline_asset_is_browsable_through_the_api(tmp_path):
|
||||
"""The browser sees an archived asset, its medium, and its preview — offline."""
|
||||
config, sf, lib, archive = _env(tmp_path)
|
||||
_, ids = _uploaded(sf, lib, seeds=(1,))
|
||||
asset_id = next(iter(ids.values()))
|
||||
_archive(sf, config, archive)
|
||||
_unmount(archive)
|
||||
|
||||
with TestClient(create_app(config)) as client:
|
||||
client.post("/api/v1/inventory/scan")
|
||||
listed = client.get("/api/v1/inventory/assets", params={"availability": ARCHIVED_OFFLINE})
|
||||
assert listed.status_code == 200
|
||||
item = next(row for row in listed.json()["items"] if row["id"] == asset_id)
|
||||
assert item["current_path"] is None
|
||||
assert item["archive_path"] == "rome/0.jpg"
|
||||
assert item["missing"] is False
|
||||
|
||||
# Searching by the archived path still finds it.
|
||||
found = client.get("/api/v1/inventory/assets", params={"q": "rome"}).json()
|
||||
assert [row["id"] for row in found["items"]] == [asset_id]
|
||||
|
||||
preview = client.get(f"/api/v1/assets/{asset_id}/thumbnail", params={"size": 1280})
|
||||
assert preview.status_code == 200
|
||||
assert preview.headers["content-type"] == "image/webp"
|
||||
|
||||
|
||||
def test_offline_state_is_stable_across_restart(tmp_path):
|
||||
config, sf, lib, archive = _env(tmp_path)
|
||||
_, ids = _uploaded(sf, lib, seeds=(1, 2))
|
||||
_archive(sf, config, archive)
|
||||
_unmount(archive)
|
||||
InventoryService(sf).scan(lib)
|
||||
before = {
|
||||
asset_id: (
|
||||
asset.availability_state,
|
||||
asset.archive_path,
|
||||
asset.current_sha256,
|
||||
asset.phash,
|
||||
)
|
||||
for asset_id, asset in _assets(sf).items()
|
||||
}
|
||||
|
||||
# Restart: a fresh engine and session factory against the same database.
|
||||
restarted = create_session_factory(create_db_engine(config.database_url))
|
||||
after = {
|
||||
asset_id: (
|
||||
asset.availability_state,
|
||||
asset.archive_path,
|
||||
asset.current_sha256,
|
||||
asset.phash,
|
||||
)
|
||||
for asset_id, asset in _assets(restarted).items()
|
||||
}
|
||||
assert after == before
|
||||
assert set(after) == set(ids.values())
|
||||
|
||||
# And the medium coming back is picked up by the restarted process.
|
||||
_remount(archive)
|
||||
assert ArchiveService(restarted, config=config).locations()[0]["state"] == "online"
|
||||
assert {a.availability_state for a in _assets(restarted).values()} == {ARCHIVED_ONLINE}
|
||||
assert ACTIVE not in {a.availability_state for a in _assets(restarted).values()}
|
||||
Reference in New Issue
Block a user