Files
photoanalyzer/tests/integration/test_upload_verification.py

538 lines
22 KiB
Python

"""Verifying, retrying, and resolving uncertain uploads (US05-04).
The Immich boundary is a real HTTP server here: it answers ``/api/server/ping``
and ``/api/assets/bulk-upload-check`` exactly as the app's own adapter parses
them, and the set of checksums it "holds" is what each fault scenario controls.
The uploader stays a real executable driven through the real batch service, so
every state under test is reached the way production reaches it.
The invariant these tests defend is one-directional: uncertainty may only become
success when something authoritative said so — the server, or an operator who
recorded what they checked.
"""
import json
import stat
import threading
import uuid
from datetime import datetime, timezone
from http.server import BaseHTTPRequestHandler, HTTPServer
from pathlib import Path
import pytest
from fastapi.testclient import TestClient
from photo_pipeline.api.app import create_app
from photo_pipeline.config import Config
from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
from photo_pipeline.models import AnalysisResult, Asset, SafetyReview, UploadBatch
from photo_pipeline.services.hashing import sha256_file
from photo_pipeline.services.upload_batches import BatchError, BatchState, UploadBatchService
from photo_pipeline.services.upload_reports import REQUIRES_VERIFICATION, VERIFIED
from photo_pipeline.services.upload_verification import (
ABSENT,
INCONCLUSIVE,
MANUAL,
PRESENT,
UploadVerificationService,
VerificationError,
retry_blockers,
)
from photo_pipeline.services.uploads import UploadService
pytestmark = pytest.mark.phase_e # part of the Phase E acceptance gate (US05-06)
NOW = datetime(2026, 1, 1, tzinfo=timezone.utc)
SENTINEL_KEY = "immich-sentinel-9f3a2b"
UPLOADER_VERSION = "immich-go 0.21.0"
# ── fake Immich ──────────────────────────────────────────────────────────────
class _FakeImmich:
"""The server's view of the world: which checksums it holds, and whether it
is willing to answer at all."""
def __init__(self) -> None:
self.held: set[str] = set()
self.available = True
self.checked: list[str] = []
def _handler(state: _FakeImmich):
class Handler(BaseHTTPRequestHandler):
def do_GET(self): # noqa: N802 (BaseHTTPRequestHandler API)
self._json(200, {"res": "pong"})
def do_POST(self): # noqa: N802
body = json.loads(self.rfile.read(int(self.headers["Content-Length"] or 0)) or "{}")
if not state.available:
self._json(503, {"error": "unavailable"})
return
if self.headers.get("x-api-key") != SENTINEL_KEY:
self._json(401, {"error": "unauthorized"})
return
results = []
for asset in body.get("assets", []):
state.checked.append(asset["checksum"])
results.append(
{"id": asset["id"], "action": "reject", "reason": "duplicate"}
if asset["checksum"] in state.held
else {"id": asset["id"], "action": "accept"}
)
self._json(200, {"results": results})
def _json(self, status: int, payload: dict) -> None:
body = json.dumps(payload).encode()
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, *args):
pass
return Handler
@pytest.fixture
def immich():
state = _FakeImmich()
server = HTTPServer(("127.0.0.1", 0), _handler(state))
threading.Thread(target=server.serve_forever, daemon=True).start()
state.url = f"http://127.0.0.1:{server.server_port}"
yield state
server.shutdown()
server.server_close()
# ── environment ──────────────────────────────────────────────────────────────
def _uploader(tmp_path, report_body: str = "", *, exit_code: int = 0, version=UPLOADER_VERSION):
path = tmp_path / "immich-go"
path.write_text(
"#!/bin/sh\n"
f'if [ "$1" = "--version" ]; then echo "{version}"; exit 0; fi\n'
f"cat <<'REPORT'\n{report_body}\nREPORT\n"
f"exit {exit_code}\n"
)
path.chmod(path.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
return path
def _env(tmp_path, immich, uploader=None):
(tmp_path / "data").mkdir(exist_ok=True)
lib = tmp_path / "lib"
lib.mkdir(exist_ok=True)
config = Config.from_env(
{
"PHOTO_PIPELINE_DATA_DIR": str(tmp_path / "data"),
"PHOTO_PIPELINE_LIBRARY_ROOTS": str(lib),
"PHOTO_PIPELINE_IMMICH_SERVER_URL": immich.url,
"PHOTO_PIPELINE_IMMICH_API_KEY": SENTINEL_KEY,
"PHOTO_PIPELINE_IMMICH_GO_BINARY": str(
uploader if uploader is not None else _uploader(tmp_path)
),
}
)
run_migrations(config.database_url)
return config, create_session_factory(create_db_engine(config.database_url)), lib
def _album(sf, lib, album="rome", names=("a.jpg", "b.jpg")):
folder = lib / album
folder.mkdir(parents=True, exist_ok=True)
with sf() as session:
for name in names:
path = folder / name
path.write_bytes(f"{album}/{name}".encode() * 16)
asset_id = str(uuid.uuid4())
session.add(
Asset(
id=asset_id,
original_path=str(path),
current_path=str(path),
discovered_at=NOW,
hash_version=1,
byte_size=path.stat().st_size,
current_sha256=sha256_file(path),
)
)
session.add(
SafetyReview(
id=str(uuid.uuid4()), asset_id=asset_id, decision="sfw", exif_verified_at=NOW
)
)
session.add(AnalysisResult(asset_id=asset_id, status="analyzed", exif_written_at=NOW))
session.commit()
return folder
def _batch(sf, config, albums=None):
report = UploadService(sf, config=config).preflight(albums)
assert report["state"] == "ready", report["blockers"]
(batch,) = UploadBatchService(sf, config=config).create(albums, token=report["token"])
return batch
def _interrupt(sf, config, batch_id):
"""Leave behind exactly what a killed worker leaves: a mid-flight attempt."""
with sf() as session:
session.get(UploadBatch, batch_id).state = BatchState.RUNNING
session.commit()
UploadBatchService(sf, config=config).recover()
def _server_holds(immich, batch):
immich.held.update(item["sha1"] for item in batch["items"])
def _by_name(batch) -> dict:
return {Path(item["path"]).name: item for item in batch["items"]}
def _uncertain_batch(tmp_path, immich, *, hold: bool):
"""A batch whose attempt died mid-upload, with the server holding the bytes or not."""
config, sf, lib = _env(tmp_path, immich)
_album(sf, lib)
batch = _batch(sf, config)
if hold:
_server_holds(immich, batch)
_interrupt(sf, config, batch["id"])
return config, sf, lib, batch
# ── accepted-but-unknown ─────────────────────────────────────────────────────
def test_acceptance_then_lost_response_is_confirmed_by_the_server(tmp_path, immich):
"""The classic lost response: Immich took the files, the app never saw a report."""
config, sf, lib, batch = _uncertain_batch(tmp_path, immich, hold=True)
result = UploadVerificationService(sf, config=config).verify(batch["id"])
assert result["state"] == BatchState.SUCCEEDED, "server evidence resolves the uncertainty"
assert result["outcome_state"] == VERIFIED
assert result["counts"] == {PRESENT: 2}
verified = UploadBatchService(sf, config=config).get(batch["id"])
for item in verified["items"]:
assert item["outcome"] == "uploaded"
assert item["verification"] == PRESENT
assert item["sha1"] in item["evidence"], "the exact bytes are named in the evidence"
assert immich.checked, "verification actually asked the server"
def test_timeout_before_acceptance_leaves_a_safe_retry(tmp_path, immich):
"""Nothing arrived, so the batch becomes a plain failure and may run again."""
config, sf, lib, batch = _uncertain_batch(tmp_path, immich, hold=False)
service = UploadBatchService(sf, config=config)
result = UploadVerificationService(sf, config=config).verify(batch["id"])
assert result["state"] == BatchState.FAILED
assert result["counts"] == {ABSENT: 2}
assert {item["outcome"] for item in service.get(batch["id"])["items"]} == {"failed"}
assert retry_blockers(service.get(batch["id"])) == []
assert service.run(batch["id"])["state"] == BatchState.SUCCEEDED
def test_an_uncertain_batch_cannot_be_retried_before_verification(tmp_path, immich):
config, sf, lib, batch = _uncertain_batch(tmp_path, immich, hold=True)
service = UploadBatchService(sf, config=config)
with pytest.raises(BatchError) as error:
service.run(batch["id"])
assert error.value.code == "requires_verification"
with TestClient(create_app(config)) as client:
response = client.post(f"/api/v1/upload-batches/{batch['id']}/start")
assert response.status_code == 409
assert response.json()["error"]["code"] == "requires_verification"
def test_a_partly_arrived_batch_is_a_failure_not_a_success(tmp_path, immich):
config, sf, lib = _env(tmp_path, immich)
_album(sf, lib)
batch = _batch(sf, config)
immich.held.add(batch["items"][0]["sha1"]) # only one file made it
_interrupt(sf, config, batch["id"])
result = UploadVerificationService(sf, config=config).verify(batch["id"])
assert result["state"] == BatchState.FAILED
assert result["counts"] == {PRESENT: 1, ABSENT: 1}
assert result["outcome_state"] == VERIFIED, "every file is accounted for"
# ── parser uncertainty ───────────────────────────────────────────────────────
def test_parser_uncertainty_is_settled_by_server_evidence(tmp_path, immich):
"""An unpinned uploader version leaves every item unknown; the server decides."""
config, sf, lib = _env(tmp_path, immich, _uploader(tmp_path, "done", version="immich-go 9.9.9"))
_album(sf, lib)
batch = _batch(sf, config)
service = UploadBatchService(sf, config=config)
ran = service.run(batch["id"])
assert ran["state"] == BatchState.SUCCEEDED and ran["outcome_state"] == REQUIRES_VERIFICATION
_server_holds(immich, ran)
result = UploadVerificationService(sf, config=config).verify(batch["id"])
assert result["outcome_state"] == VERIFIED
assert {item["outcome"] for item in service.get(batch["id"])["items"]} == {"uploaded"}
def test_an_unreachable_server_never_turns_uncertainty_into_success(tmp_path, immich):
config, sf, lib, batch = _uncertain_batch(tmp_path, immich, hold=True)
immich.available = False
service = UploadBatchService(sf, config=config)
result = UploadVerificationService(sf, config=config).verify(batch["id"])
assert result["server_reachable"] is False
assert result["detail"], "the reason the server could not answer is reported"
assert result["counts"] == {INCONCLUSIVE: 2}
assert result["state"] == BatchState.UNKNOWN, "still uncertain, not succeeded"
assert result["outcome_state"] == REQUIRES_VERIFICATION
assert [b["code"] for b in retry_blockers(service.get(batch["id"]))] == [
"requires_verification"
]
# ── safe failures ────────────────────────────────────────────────────────────
def test_a_plain_uploader_failure_is_retryable_without_verification(tmp_path, immich):
"""Nothing uncertain happened: the process failed before/while reporting an error."""
config, sf, lib = _env(tmp_path, immich, _uploader(tmp_path, "boom", exit_code=1))
_album(sf, lib)
batch = _batch(sf, config)
service = UploadBatchService(sf, config=config)
assert service.run(batch["id"])["state"] == BatchState.FAILED
assert retry_blockers(service.get(batch["id"])) == []
with TestClient(create_app(config)) as client:
assert client.post(f"/api/v1/upload-batches/{batch['id']}/start").status_code == 200
# ── changed bytes ────────────────────────────────────────────────────────────
def test_bytes_changed_after_upload_warn_and_block_a_rerun(tmp_path, immich):
config, sf, lib = _env(tmp_path, immich)
folder = _album(sf, lib)
batch = _batch(sf, config)
service = UploadBatchService(sf, config=config)
ran = service.run(batch["id"])
_server_holds(immich, ran)
(folder / "a.jpg").write_bytes(b"edited after the upload")
result = UploadVerificationService(sf, config=config).verify(batch["id"])
assert result["stale_bytes"] is True
changed = _by_name(result)["a.jpg"]
assert changed["changed_after_upload"] is True
assert _by_name(result)["b.jpg"]["changed_after_upload"] is False
stored = service.get(batch["id"])
assert stored["stale_bytes"] is True, "the warning is durable, not only in the response"
assert _by_name(stored)["a.jpg"]["observed_sha256"] != _by_name(stored)["a.jpg"]["sha256"]
with pytest.raises(BatchError) as error:
service.run(batch["id"])
assert error.value.code == "changed_after_upload"
with TestClient(create_app(config)) as client:
response = client.post(f"/api/v1/upload-batches/{batch['id']}/start")
assert response.status_code == 409
assert response.json()["error"]["code"] == "changed_after_upload"
def test_a_deleted_file_counts_as_changed_after_upload(tmp_path, immich):
config, sf, lib = _env(tmp_path, immich)
folder = _album(sf, lib)
batch = _batch(sf, config)
ran = UploadBatchService(sf, config=config).run(batch["id"])
_server_holds(immich, ran)
(folder / "a.jpg").unlink()
result = UploadVerificationService(sf, config=config).verify(batch["id"])
assert _by_name(result)["a.jpg"]["changed_after_upload"] is True
assert result["stale_bytes"] is True
# The bytes are still on the server: verification is about the upload, not the
# local file's continued existence.
assert _by_name(result)["a.jpg"]["verification"] == PRESENT
# ── repeated verification and history ────────────────────────────────────────
def test_repeated_verification_converges_and_keeps_every_answer(tmp_path, immich):
config, sf, lib, batch = _uncertain_batch(tmp_path, immich, hold=True)
service = UploadVerificationService(sf, config=config)
first = service.verify(batch["id"])
second = service.verify(batch["id"])
assert first["counts"] == second["counts"] == {PRESENT: 2}
assert first["state"] == second["state"] == BatchState.SUCCEEDED
history = service.history(batch["id"])
assert len(history) == 4, "the audit trail appends, it never overwrites"
assert {entry["source"] for entry in history} == {"immich_api"}
assert all(entry["action"] == "verify" for entry in history)
def test_verification_that_changes_its_mind_keeps_both_answers(tmp_path, immich):
"""A file that was absent and later present must show both, in order."""
config, sf, lib, batch = _uncertain_batch(tmp_path, immich, hold=False)
service = UploadVerificationService(sf, config=config)
service.verify(batch["id"])
_server_holds(immich, batch)
service.verify(batch["id"])
asset_id = batch["items"][0]["asset_id"]
results = [e["result"] for e in service.history(batch["id"]) if e["asset_id"] == asset_id]
assert results == [ABSENT, PRESENT]
# ── manual resolution ────────────────────────────────────────────────────────
def test_manual_resolution_requires_evidence_and_an_author(tmp_path, immich):
config, sf, lib, batch = _uncertain_batch(tmp_path, immich, hold=True)
service = UploadVerificationService(sf, config=config)
asset_id = batch["items"][0]["asset_id"]
for kwargs, code in (
({"evidence": " ", "actor": "dom"}, "evidence_required"),
({"evidence": "checked in Immich", "actor": ""}, "actor_required"),
({"evidence": "checked", "actor": "dom", "outcome": "definitely-fine"}, "invalid_outcome"),
):
with pytest.raises(VerificationError) as error:
service.resolve(batch["id"], asset_id, **{"outcome": "uploaded", **kwargs})
assert error.value.code == code
assert service.history(batch["id"]) == [], "a refused resolution records nothing"
assert UploadBatchService(sf, config=config).get(batch["id"])["state"] == BatchState.UNKNOWN
def test_manual_resolution_is_recorded_as_operator_evidence(tmp_path, immich):
"""An operator may settle what the server cannot — but never anonymously."""
config, sf, lib, batch = _uncertain_batch(tmp_path, immich, hold=True)
immich.available = False
service = UploadVerificationService(sf, config=config)
service.verify(batch["id"]) # inconclusive: the server is down
for item in batch["items"]:
result = service.resolve(
batch["id"],
item["asset_id"],
outcome="uploaded",
evidence="found in Immich by checksum in the web UI",
actor="dom",
)
assert result["state"] == BatchState.SUCCEEDED
assert result["outcome_state"] == VERIFIED
resolutions = [e for e in service.history(batch["id"]) if e["action"] == "resolve"]
assert len(resolutions) == 2
assert {e["actor"] for e in resolutions} == {"dom"}
assert {e["source"] for e in resolutions} == {"operator"}
assert all("web UI" in e["evidence"] for e in resolutions)
stored = UploadBatchService(sf, config=config).get(batch["id"])
assert {item["verification"] for item in stored["items"]} == {MANUAL}, (
"a manual answer stays distinguishable from server evidence"
)
def test_resolving_one_item_leaves_the_batch_uncertain(tmp_path, immich):
config, sf, lib, batch = _uncertain_batch(tmp_path, immich, hold=True)
immich.available = False
service = UploadVerificationService(sf, config=config)
service.verify(batch["id"])
result = service.resolve(
batch["id"],
batch["items"][0]["asset_id"],
outcome="uploaded",
evidence="visible in Immich",
actor="dom",
)
assert result["state"] == BatchState.UNKNOWN
assert result["outcome_state"] == REQUIRES_VERIFICATION
def test_resolving_an_unknown_item_or_batch_is_refused(tmp_path, immich):
config, sf, lib, batch = _uncertain_batch(tmp_path, immich, hold=True)
service = UploadVerificationService(sf, config=config)
for batch_id, asset_id in ((batch["id"], "not-in-this-batch"), ("no-such-batch", "x")):
with pytest.raises(VerificationError) as error:
service.resolve(batch_id, asset_id, outcome="uploaded", evidence="checked", actor="dom")
assert error.value.code == "not_found"
def test_verifying_an_unknown_batch_is_refused(tmp_path, immich):
config, sf, lib = _env(tmp_path, immich)
with pytest.raises(VerificationError) as error:
UploadVerificationService(sf, config=config).verify("does-not-exist")
assert error.value.code == "not_found"
# ── API surface and durability ───────────────────────────────────────────────
def test_the_api_verifies_resolves_and_lists_history_without_secrets(tmp_path, immich):
config, sf, lib, batch = _uncertain_batch(tmp_path, immich, hold=False)
with TestClient(create_app(config)) as client:
verified = client.post(f"/api/v1/upload-batches/{batch['id']}/verify")
resolved = client.post(
f"/api/v1/upload-batches/{batch['id']}/resolve",
json={
"asset_id": batch["items"][0]["asset_id"],
"outcome": "skipped",
"evidence": "the file was withdrawn from the album",
"actor": "dom",
},
)
refused = client.post(
f"/api/v1/upload-batches/{batch['id']}/resolve",
json={
"asset_id": batch["items"][0]["asset_id"],
"outcome": "uploaded",
"evidence": "",
"actor": "dom",
},
)
history = client.get(f"/api/v1/upload-batches/{batch['id']}/verifications")
missing = client.post("/api/v1/upload-batches/does-not-exist/verify")
assert verified.status_code == 200 and verified.json()["counts"] == {ABSENT: 2}
assert resolved.status_code == 200
assert refused.status_code == 422 and refused.json()["error"]["code"] == "evidence_required"
assert len(history.json()["verifications"]) == 3
assert missing.status_code == 404
assert SENTINEL_KEY not in verified.text + resolved.text + history.text
def test_verification_survives_a_restart(tmp_path, immich):
config, sf, lib, batch = _uncertain_batch(tmp_path, immich, hold=True)
UploadVerificationService(sf, config=config).verify(batch["id"])
with TestClient(create_app(config)) as client: # a fresh application process
fetched = client.get(f"/api/v1/upload-batches/{batch['id']}").json()
history = client.get(f"/api/v1/upload-batches/{batch['id']}/verifications").json()
assert fetched["state"] == BatchState.SUCCEEDED
assert fetched["verified_at"]
assert {item["verification"] for item in fetched["items"]} == {PRESENT}
assert len(history["verifications"]) == 2
assert json.dumps(fetched) # the record stays JSON-serialisable for the UI