378 lines
15 KiB
Python
378 lines
15 KiB
Python
"""Offline identity and review evidence (US06-03).
|
|
|
|
An archived photo is not gone: it keeps its identity, its hashes, and enough
|
|
evidence to be recognised in a duplicate cluster while its medium sits in a
|
|
drawer. Every case here archives a *real* album through the real transfer, then
|
|
takes the medium away by removing its marker — the same thing the service sees
|
|
when an external disk is unplugged — and asks whether the application still tells
|
|
the truth about where the bytes are.
|
|
|
|
The distinction that matters throughout: an unmounted medium is
|
|
``archived_offline`` (expected, harmless), a mounted medium with a hole in it is
|
|
``missing_unexpected`` (needs a human). Confusing the two is how an archive
|
|
quietly loses a photo.
|
|
"""
|
|
|
|
import shutil
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
|
|
import numpy as np
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
from PIL import Image
|
|
from sqlalchemy import select
|
|
|
|
from photo_pipeline.api.app import create_app
|
|
from photo_pipeline.config import Config
|
|
from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
|
|
from photo_pipeline.models import Asset, Thumbnail, UploadBatch, UploadItem
|
|
from photo_pipeline.services.archive_transfer import ArchiveTransferService
|
|
from photo_pipeline.services.archives import MARKER_NAME, ArchiveService
|
|
from photo_pipeline.services.availability import (
|
|
ACTIVE,
|
|
ARCHIVED_OFFLINE,
|
|
ARCHIVED_ONLINE,
|
|
MISSING_UNEXPECTED,
|
|
)
|
|
from photo_pipeline.services.duplicates import ClusterState, DuplicateService, Method
|
|
from photo_pipeline.services.hashing import sha256_file
|
|
from photo_pipeline.services.inventory import InventoryService
|
|
from photo_pipeline.services.thumbnails import PROTECTED_SIZE, ThumbnailService, ThumbnailUnavailable
|
|
|
|
pytestmark = pytest.mark.phase_f # part of the Phase F acceptance gate (US06-06)
|
|
|
|
NOW = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
|
|
|
|
|
# ── environment ──────────────────────────────────────────────────────────────
|
|
|
|
|
|
def _env(tmp_path):
|
|
(tmp_path / "data").mkdir(exist_ok=True)
|
|
lib = tmp_path / "lib"
|
|
lib.mkdir(exist_ok=True)
|
|
archive = tmp_path / "archive"
|
|
archive.mkdir(exist_ok=True)
|
|
config = Config.from_env(
|
|
{
|
|
"PHOTO_PIPELINE_DATA_DIR": str(tmp_path / "data"),
|
|
"PHOTO_PIPELINE_LIBRARY_ROOTS": str(lib),
|
|
"PHOTO_PIPELINE_ARCHIVE_FREE_SPACE_RESERVE_BYTES": "0",
|
|
}
|
|
)
|
|
run_migrations(config.database_url)
|
|
return config, create_session_factory(create_db_engine(config.database_url)), lib, archive
|
|
|
|
|
|
def structured(path, seed, size=(256, 192)):
|
|
"""A deterministic, decodable photo — previews and pHashes must be real."""
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
rng = np.random.default_rng(seed)
|
|
w, h = size
|
|
base = np.zeros((h, w, 3), dtype=np.uint8)
|
|
for _ in range(6):
|
|
x0 = int(rng.integers(0, w - 60))
|
|
y0 = int(rng.integers(0, h - 60))
|
|
base[y0 : y0 + 60, x0 : x0 + 60] = rng.integers(0, 256, 3)
|
|
grad = np.linspace(0, 120, w, dtype=np.uint8)
|
|
base[:, :, 0] = np.clip(base[:, :, 0].astype(int) + grad[None, :], 0, 255)
|
|
Image.fromarray(base).save(path, quality=95)
|
|
return path
|
|
|
|
|
|
def resized_copy(src, dst, scale=0.5):
|
|
with Image.open(src) as image:
|
|
image.resize(
|
|
(int(image.width * scale), int(image.height * scale)), Image.LANCZOS
|
|
).save(dst, quality=95)
|
|
return dst
|
|
|
|
|
|
def _uploaded(sf, lib, album="rome", seeds=(1, 2)):
|
|
"""A scanned album carrying the verified upload evidence archiving requires."""
|
|
folder = lib / album
|
|
for index, seed in enumerate(seeds):
|
|
structured(folder / f"{index}.jpg", seed)
|
|
result = InventoryService(sf).scan(lib)
|
|
with sf() as session:
|
|
batch_id = str(uuid.uuid4())
|
|
session.add(
|
|
UploadBatch(
|
|
id=batch_id,
|
|
album=album,
|
|
folder=str(folder),
|
|
album_name=album,
|
|
state="succeeded",
|
|
preflight_token="v1:test",
|
|
outcome_state="verified",
|
|
created_at=NOW,
|
|
)
|
|
)
|
|
for path, asset_id in result.asset_ids.items():
|
|
session.add(
|
|
UploadItem(
|
|
batch_id=batch_id,
|
|
asset_id=asset_id,
|
|
path=path,
|
|
sha256=sha256_file(path),
|
|
sha1="0" * 40,
|
|
state="sent",
|
|
outcome="uploaded",
|
|
)
|
|
)
|
|
session.commit()
|
|
return folder, result.asset_ids
|
|
|
|
|
|
def _archive(sf, config, archive, albums=None):
|
|
service = ArchiveService(sf, config=config)
|
|
location = service.register("external", str(archive))
|
|
token = service.preflight(location["id"], albums)["token"]
|
|
transfers = ArchiveTransferService(sf, config=config)
|
|
plan = transfers.create(location["id"], albums, token=token)
|
|
transfers.apply(plan["id"])
|
|
return location
|
|
|
|
|
|
def _unmount(archive):
|
|
"""Take the medium away the way a real one goes: its marker stops answering."""
|
|
(archive / MARKER_NAME).rename(archive / f"{MARKER_NAME}.away")
|
|
|
|
|
|
def _remount(archive):
|
|
(archive / f"{MARKER_NAME}.away").rename(archive / MARKER_NAME)
|
|
|
|
|
|
def _assets(sf):
|
|
with sf() as session:
|
|
return {asset.id: asset for asset in session.scalars(select(Asset))}
|
|
|
|
|
|
# ── availability ─────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_archived_assets_report_online_offline_and_missing(tmp_path):
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
_, ids = _uploaded(sf, lib)
|
|
_archive(sf, config, archive)
|
|
inventory = InventoryService(sf)
|
|
|
|
states = {a.availability_state for a in _assets(sf).values()}
|
|
assert states == {ARCHIVED_ONLINE}
|
|
|
|
_unmount(archive)
|
|
inventory.scan(lib) # the album folder is gone from the active roots
|
|
assert {a.availability_state for a in _assets(sf).values()} == {ARCHIVED_OFFLINE}
|
|
assert all(a.missing_at is None for a in _assets(sf).values()) # not "missing"
|
|
|
|
_remount(archive)
|
|
inventory.scan(lib)
|
|
assert {a.availability_state for a in _assets(sf).values()} == {ARCHIVED_ONLINE}
|
|
|
|
# Mounted medium, absent file: that is not an offline archive, it needs a human.
|
|
victim = sorted(ids.values())[0]
|
|
with sf() as session:
|
|
asset = session.get(Asset, victim)
|
|
(archive / asset.archive_path).unlink()
|
|
inventory.scan(lib)
|
|
assert _assets(sf)[victim].availability_state == MISSING_UNEXPECTED
|
|
|
|
|
|
def test_scan_never_prunes_or_flags_offline_assets(tmp_path):
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
_, ids = _uploaded(sf, lib)
|
|
_archive(sf, config, archive)
|
|
_unmount(archive)
|
|
|
|
before = _assets(sf)
|
|
result = InventoryService(sf).scan(lib)
|
|
|
|
assert result.counts.get("missing") is None
|
|
after = _assets(sf)
|
|
assert set(after) == set(before) == set(ids.values())
|
|
for asset in after.values():
|
|
assert asset.availability_state == ARCHIVED_OFFLINE
|
|
assert asset.current_sha256 and asset.pixel_sha256 # hashes retained
|
|
assert asset.archive_location_id and asset.archive_path
|
|
|
|
|
|
def test_active_missing_file_is_missing_unexpected_not_offline(tmp_path):
|
|
config, sf, lib, _archive_root = _env(tmp_path)
|
|
structured(lib / "loose" / "a.jpg", 7)
|
|
ids = InventoryService(sf).scan(lib).asset_ids
|
|
asset_id = next(iter(ids.values()))
|
|
|
|
(lib / "loose" / "a.jpg").unlink()
|
|
InventoryService(sf).scan(lib)
|
|
|
|
asset = _assets(sf)[asset_id]
|
|
assert asset.availability_state == MISSING_UNEXPECTED
|
|
assert asset.missing_at is not None
|
|
|
|
|
|
# ── deduplication against archived originals ─────────────────────────────────
|
|
|
|
|
|
def test_exact_copy_of_offline_asset_links_to_archived_canonical(tmp_path):
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
folder, ids = _uploaded(sf, lib, seeds=(1,))
|
|
archived_id = next(iter(ids.values()))
|
|
original = next(iter(ids))
|
|
kept = tmp_path / "kept.jpg"
|
|
shutil.copy2(original, kept)
|
|
_archive(sf, config, archive)
|
|
_unmount(archive)
|
|
|
|
# The same photo turns up again in the active library while the disk is away.
|
|
(lib / "inbox").mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(kept, lib / "inbox" / "again.jpg")
|
|
scan = InventoryService(sf).scan(lib)
|
|
new_id = scan.asset_ids[str(lib / "inbox" / "again.jpg")]
|
|
assert scan.occurrences[str(lib / "inbox" / "again.jpg")] == "copied"
|
|
|
|
clusters = DuplicateService(sf).detect().clusters
|
|
exact = [c for c in clusters if c["method"] == Method.EXACT.value]
|
|
assert len(exact) == 1
|
|
cluster = exact[0]
|
|
# Byte-identical: linked directly, and the archived original stays canonical.
|
|
assert cluster["state"] == ClusterState.DECIDED.value
|
|
assert cluster["canonical_asset_id"] == archived_id
|
|
assert _assets(sf)[new_id].canonical_asset_id == archived_id
|
|
|
|
|
|
def test_fuzzy_copy_of_offline_asset_requires_review_and_names_the_medium(tmp_path):
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
folder, ids = _uploaded(sf, lib, seeds=(1,))
|
|
archived_id = next(iter(ids.values()))
|
|
original = next(iter(ids))
|
|
variant_source = resized_copy(original, tmp_path / "small.jpg")
|
|
_archive(sf, config, archive)
|
|
_unmount(archive)
|
|
|
|
(lib / "inbox").mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(variant_source, lib / "inbox" / "small.jpg")
|
|
InventoryService(sf).scan(lib)
|
|
|
|
duplicates = DuplicateService(sf)
|
|
clusters = duplicates.detect().clusters
|
|
perceptual = [c for c in clusters if c["method"] == Method.PERCEPTUAL.value]
|
|
assert len(perceptual) == 1
|
|
|
|
detail = duplicates.get_cluster(perceptual[0]["id"])
|
|
assert detail["state"] == ClusterState.OPEN.value # never auto-decided
|
|
assert detail["requires_confirmation"] is True
|
|
assert detail["mount_required"] == ["external"] # full-resolution needs the disk
|
|
archived = next(m for m in detail["members"] if m["asset_id"] == archived_id)
|
|
assert archived["availability_state"] == ARCHIVED_OFFLINE
|
|
assert archived["current_path"] is None
|
|
assert archived["archive_path"] and archived["evidence"]["phash"]
|
|
# The retained preview is what makes the offline member reviewable at all.
|
|
assert archived["preview"] == {"state": "ready", "protected": True, "size": PROTECTED_SIZE}
|
|
|
|
|
|
# ── protected review evidence ────────────────────────────────────────────────
|
|
|
|
|
|
def test_protected_preview_survives_quota_and_serves_while_offline(tmp_path):
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
_, ids = _uploaded(sf, lib, seeds=(1,))
|
|
asset_id = next(iter(ids.values()))
|
|
_archive(sf, config, archive)
|
|
_unmount(archive)
|
|
|
|
thumbnails = ThumbnailService(sf, config)
|
|
served = thumbnails.generate(asset_id, PROTECTED_SIZE)
|
|
assert served.exists() # rendered before the original left, not from the medium
|
|
|
|
# An aggressive quota may empty the cache, but not this evidence.
|
|
tight = ThumbnailService(sf, config.model_copy(update={"thumbnail_cache_quota_bytes": 1}))
|
|
tight._enforce_quota()
|
|
assert served.exists()
|
|
with sf() as session:
|
|
row = session.scalar(select(Thumbnail).where(Thumbnail.asset_id == asset_id))
|
|
assert row.protected is True
|
|
assert thumbnails.evidence(asset_id)["state"] == "ready"
|
|
|
|
|
|
def test_offline_asset_without_preview_reports_unavailable(tmp_path):
|
|
"""No preview and no medium is an honest 409, never a wrong picture."""
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
_, ids = _uploaded(sf, lib, seeds=(1,))
|
|
asset_id = next(iter(ids.values()))
|
|
_archive(sf, config, archive)
|
|
with sf() as session:
|
|
for row in session.scalars(select(Thumbnail).where(Thumbnail.asset_id == asset_id)):
|
|
session.delete(row)
|
|
session.commit()
|
|
_unmount(archive)
|
|
|
|
with pytest.raises(ThumbnailUnavailable):
|
|
ThumbnailService(sf, config).generate(asset_id, 256)
|
|
|
|
_remount(archive) # mounted again: the archived original is readable
|
|
assert ThumbnailService(sf, config).generate(asset_id, 256).exists()
|
|
|
|
|
|
def test_offline_asset_is_browsable_through_the_api(tmp_path):
|
|
"""The browser sees an archived asset, its medium, and its preview — offline."""
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
_, ids = _uploaded(sf, lib, seeds=(1,))
|
|
asset_id = next(iter(ids.values()))
|
|
_archive(sf, config, archive)
|
|
_unmount(archive)
|
|
|
|
with TestClient(create_app(config)) as client:
|
|
client.post("/api/v1/inventory/scan")
|
|
listed = client.get("/api/v1/inventory/assets", params={"availability": ARCHIVED_OFFLINE})
|
|
assert listed.status_code == 200
|
|
item = next(row for row in listed.json()["items"] if row["id"] == asset_id)
|
|
assert item["current_path"] is None
|
|
assert item["archive_path"] == "rome/0.jpg"
|
|
assert item["missing"] is False
|
|
|
|
# Searching by the archived path still finds it.
|
|
found = client.get("/api/v1/inventory/assets", params={"q": "rome"}).json()
|
|
assert [row["id"] for row in found["items"]] == [asset_id]
|
|
|
|
preview = client.get(f"/api/v1/assets/{asset_id}/thumbnail", params={"size": 1280})
|
|
assert preview.status_code == 200
|
|
assert preview.headers["content-type"] == "image/webp"
|
|
|
|
|
|
def test_offline_state_is_stable_across_restart(tmp_path):
|
|
config, sf, lib, archive = _env(tmp_path)
|
|
_, ids = _uploaded(sf, lib, seeds=(1, 2))
|
|
_archive(sf, config, archive)
|
|
_unmount(archive)
|
|
InventoryService(sf).scan(lib)
|
|
before = {
|
|
asset_id: (
|
|
asset.availability_state,
|
|
asset.archive_path,
|
|
asset.current_sha256,
|
|
asset.phash,
|
|
)
|
|
for asset_id, asset in _assets(sf).items()
|
|
}
|
|
|
|
# Restart: a fresh engine and session factory against the same database.
|
|
restarted = create_session_factory(create_db_engine(config.database_url))
|
|
after = {
|
|
asset_id: (
|
|
asset.availability_state,
|
|
asset.archive_path,
|
|
asset.current_sha256,
|
|
asset.phash,
|
|
)
|
|
for asset_id, asset in _assets(restarted).items()
|
|
}
|
|
assert after == before
|
|
assert set(after) == set(ids.values())
|
|
|
|
# And the medium coming back is picked up by the restarted process.
|
|
_remount(archive)
|
|
assert ArchiveService(restarted, config=config).locations()[0]["state"] == "online"
|
|
assert {a.availability_state for a in _assets(restarted).values()} == {ARCHIVED_ONLINE}
|
|
assert ACTIVE not in {a.availability_state for a in _assets(restarted).values()}
|