US06-06: Automate Phase F End-to-End Acceptance
This commit is contained in:
39
README.md
39
README.md
@@ -169,3 +169,42 @@ work_item/scripts/python -m pytest -m phase_e -q
|
|||||||
bytes, and recovery after a restart).
|
bytes, and recovery after a restart).
|
||||||
|
|
||||||
Phases A–D remain green in the full run above.
|
Phases A–D remain green in the full run above.
|
||||||
|
|
||||||
|
### Phase F acceptance gate
|
||||||
|
|
||||||
|
Phase F (Epic E06: archive lifecycle) is the only stage that *removes* originals
|
||||||
|
from the library, and the only one whose storage can walk away in someone's bag.
|
||||||
|
One command runs the archive fault-injection suites, the black-box archive and
|
||||||
|
restore API journeys, and the browser suite:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
work_item/scripts/python -m pytest -m phase_f -q
|
||||||
|
```
|
||||||
|
|
||||||
|
- `tests/integration/test_archive_*.py` and `tests/integration/test_restore.py`
|
||||||
|
drive real files on real filesystems: preflight against a mounted, missing,
|
||||||
|
swapped, read-only, or full medium; copy-verify-remove and the same-filesystem
|
||||||
|
move path; and a crash at **every** persisted journal transition in both transfer
|
||||||
|
modes, asserting that no source is ever removed without a durable, byte-identical
|
||||||
|
archive copy.
|
||||||
|
- `tests/e2e/test_phase_f_pipeline.py` drives a real server and a real durable
|
||||||
|
worker over HTTP: preflight blockers (offline medium, wrong volume, insufficient
|
||||||
|
capacity, bytes changed after upload), a verified archive whose manifest, hashes,
|
||||||
|
and path history are checked on the medium itself, a worker killed at each of
|
||||||
|
`transferring`, `verified`, `removing`, `source_removed`, and `complete`, the
|
||||||
|
evidence-based recovery that follows, offline deduplication of an exact and a
|
||||||
|
fuzzy copy while the medium is away, mount return, restore, and a collision that
|
||||||
|
restores beside its occupant.
|
||||||
|
- **Archived is not missing.** An unmounted medium leaves its photos
|
||||||
|
`archived_offline` — still hashed, still in the duplicate indexes, still
|
||||||
|
previewable through their protected thumbnails — and a rescan neither prunes nor
|
||||||
|
flags them.
|
||||||
|
- **Ambiguity is never guessed.** A journal state the medium contradicts stays
|
||||||
|
`manual`, offers no automatic action, and keeps blocking further archiving until
|
||||||
|
a human decides.
|
||||||
|
- `tests/e2e/test_archive_ui.py` covers the browser journeys (preview with
|
||||||
|
destination identity and reclaimable bytes, blockers and mount instructions,
|
||||||
|
progress split into transfer/verification/removal, interruption and recovery,
|
||||||
|
offline browsing, restore, collision, keyboard confirmation, and reload).
|
||||||
|
|
||||||
|
Phases A–E remain green in the full run above.
|
||||||
|
|||||||
@@ -208,9 +208,12 @@ class ArchiveService:
|
|||||||
report["blockers"].append(
|
report["blockers"].append(
|
||||||
_issue(
|
_issue(
|
||||||
"insufficient_capacity",
|
"insufficient_capacity",
|
||||||
|
# The free-space number is deliberately left out: it drifts between
|
||||||
|
# two identical preflights, and the token is a digest of this text,
|
||||||
|
# so quoting it here would invalidate every approval instantly.
|
||||||
f"{report['totals']['bytes']} B plus a "
|
f"{report['totals']['bytes']} B plus a "
|
||||||
f"{self._config.archive_free_space_reserve_bytes} B reserve do not fit in "
|
f"{self._config.archive_free_space_reserve_bytes} B reserve do not fit on "
|
||||||
f"{report['capacity']['free_bytes']} B of free space",
|
"the medium",
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
report["backup"] = self._backup_probe()
|
report["backup"] = self._backup_probe()
|
||||||
|
|||||||
@@ -140,9 +140,11 @@ class RestoreService:
|
|||||||
report["blockers"].append(
|
report["blockers"].append(
|
||||||
_issue(
|
_issue(
|
||||||
"insufficient_capacity",
|
"insufficient_capacity",
|
||||||
|
# No free-space number here: it drifts between two identical
|
||||||
|
# preflights and the token is a digest of this text (US06-06).
|
||||||
f"{report['totals']['bytes']} B plus a "
|
f"{report['totals']['bytes']} B plus a "
|
||||||
f"{self._config.archive_free_space_reserve_bytes} B reserve do not fit in "
|
f"{self._config.archive_free_space_reserve_bytes} B reserve do not fit in "
|
||||||
f"{report['capacity']['free_bytes']} B of free space",
|
"the library",
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
if not items:
|
if not items:
|
||||||
|
|||||||
@@ -494,6 +494,16 @@ class ArchiveStack:
|
|||||||
self.server.stop()
|
self.server.stop()
|
||||||
self.server.start()
|
self.server.start()
|
||||||
|
|
||||||
|
def restart_worker(self, *, extra_env: dict[str, str] | None = None) -> None:
|
||||||
|
"""Replace the worker — a healthy one after a crashed one, by default."""
|
||||||
|
if self.worker is not None and self.worker.poll() is None:
|
||||||
|
self.worker.kill()
|
||||||
|
self.worker.wait(timeout=10)
|
||||||
|
self.worker = start_worker(
|
||||||
|
self.seeded,
|
||||||
|
extra_env={"PHOTO_PIPELINE_ARCHIVE_FREE_SPACE_RESERVE_BYTES": "0", **(extra_env or {})},
|
||||||
|
)
|
||||||
|
|
||||||
def stop(self) -> None:
|
def stop(self) -> None:
|
||||||
if self.worker is not None:
|
if self.worker is not None:
|
||||||
self.worker.kill()
|
self.worker.kill()
|
||||||
|
|||||||
505
tests/e2e/test_phase_f_pipeline.py
Normal file
505
tests/e2e/test_phase_f_pipeline.py
Normal file
@@ -0,0 +1,505 @@
|
|||||||
|
"""Phase F end-to-end acceptance (US06-06): archive lifecycle, black box.
|
||||||
|
|
||||||
|
Every journey drives a real ``photo_pipeline serve`` child process and a real
|
||||||
|
durable worker over HTTP — register a medium, preflight it, archive an album,
|
||||||
|
crash mid-transfer, recover, unmount the medium, rediscover the photos through
|
||||||
|
their hashes while they are unreachable, mount it again, restore, and collide.
|
||||||
|
Nothing is reached into: the medium is an ordinary directory whose marker file is
|
||||||
|
its identity, and unmounting it means taking that marker away, which is what the
|
||||||
|
application sees when a disk is unplugged.
|
||||||
|
|
||||||
|
Three invariants are asserted wherever they apply, because they are what make the
|
||||||
|
only *removing* stage safe:
|
||||||
|
|
||||||
|
- **No source is removed before its archive copy is verified.** Every crash barrier
|
||||||
|
is checked for the file still being somewhere: either in the library, or on the
|
||||||
|
medium hashing exactly as recorded.
|
||||||
|
- **Archived is not missing.** An unmounted medium leaves its photos
|
||||||
|
``archived_offline``, still hashed, still deduplicable, still previewable.
|
||||||
|
- **Restoring never overwrites.** A taken name comes back beside its occupant, and
|
||||||
|
the archived copy stays on the medium.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import shutil
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from tests.e2e._pipeline_harness import (
|
||||||
|
ArchiveStack,
|
||||||
|
mark_uploaded,
|
||||||
|
seed_album,
|
||||||
|
session_factory,
|
||||||
|
wait_until,
|
||||||
|
)
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.phase_f
|
||||||
|
|
||||||
|
TIMEOUT = 20
|
||||||
|
ALBUM = "rome"
|
||||||
|
MANIFEST = "archive-manifest.jsonl"
|
||||||
|
# Every persisted transition the transfer can die at, in the order it reaches them.
|
||||||
|
BARRIERS = ["transferring", "verified", "removing", "source_removed", "complete"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def stack(tmp_path):
|
||||||
|
"""An analysed album with verified upload evidence — archivable, nothing else."""
|
||||||
|
seeded = seed_album(tmp_path)
|
||||||
|
mark_uploaded(seeded)
|
||||||
|
running = ArchiveStack(tmp_path, seeded)
|
||||||
|
try:
|
||||||
|
yield running
|
||||||
|
finally:
|
||||||
|
running.stop()
|
||||||
|
|
||||||
|
|
||||||
|
# ── helpers ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _preflight(stack, location_id: str, **body) -> dict:
|
||||||
|
response = httpx.post(
|
||||||
|
f"{stack.base}/api/v1/archive-preflight",
|
||||||
|
json={"location_id": location_id, **body},
|
||||||
|
timeout=TIMEOUT,
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
return response.json()
|
||||||
|
|
||||||
|
|
||||||
|
def _plan(stack, location_id: str, **body) -> dict:
|
||||||
|
report = _preflight(stack, location_id, **body)
|
||||||
|
response = httpx.post(
|
||||||
|
f"{stack.base}/api/v1/archive-plans",
|
||||||
|
json={"location_id": location_id, "token": report["token"], **body},
|
||||||
|
timeout=TIMEOUT,
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
return response.json()
|
||||||
|
|
||||||
|
|
||||||
|
def _apply(stack, plan_id: str) -> httpx.Response:
|
||||||
|
return httpx.post(f"{stack.base}/api/v1/archive-plans/{plan_id}/apply", timeout=TIMEOUT)
|
||||||
|
|
||||||
|
|
||||||
|
def _plan_state(stack, plan_id: str) -> dict:
|
||||||
|
return httpx.get(f"{stack.base}/api/v1/archive-plans/{plan_id}", timeout=TIMEOUT).json()
|
||||||
|
|
||||||
|
|
||||||
|
def _await_plan(stack, plan_id: str, states=("complete", "failed"), *, timeout: float = 60) -> dict:
|
||||||
|
return wait_until(
|
||||||
|
lambda: (lambda p: p if p.get("state") in states else None)(_plan_state(stack, plan_id)),
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _archive_album(stack) -> dict:
|
||||||
|
"""Register, preflight, plan, apply, and wait for the run to finish."""
|
||||||
|
location = stack.register()
|
||||||
|
plan = _plan(stack, location["id"])
|
||||||
|
_apply(stack, plan["id"]).raise_for_status()
|
||||||
|
return {"location": location, "plan": _await_plan(stack, plan["id"])}
|
||||||
|
|
||||||
|
|
||||||
|
def _assets(stack) -> dict[str, dict]:
|
||||||
|
return {asset["id"]: asset for asset in stack.assets()}
|
||||||
|
|
||||||
|
|
||||||
|
def _recovery(stack) -> dict:
|
||||||
|
return httpx.get(f"{stack.base}/api/v1/archive-recovery", timeout=TIMEOUT).json()
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve(stack) -> dict:
|
||||||
|
response = httpx.post(f"{stack.base}/api/v1/archive-recovery/resolve", timeout=TIMEOUT)
|
||||||
|
response.raise_for_status()
|
||||||
|
return response.json()
|
||||||
|
|
||||||
|
|
||||||
|
def _restore(stack, location_id: str) -> dict:
|
||||||
|
report = httpx.post(
|
||||||
|
f"{stack.base}/api/v1/restore-preflight",
|
||||||
|
json={"location_id": location_id},
|
||||||
|
timeout=TIMEOUT,
|
||||||
|
).json()
|
||||||
|
plan = httpx.post(
|
||||||
|
f"{stack.base}/api/v1/restore-plans",
|
||||||
|
json={"location_id": location_id, "token": report["token"]},
|
||||||
|
timeout=TIMEOUT,
|
||||||
|
)
|
||||||
|
plan.raise_for_status()
|
||||||
|
plan = plan.json()
|
||||||
|
httpx.post(
|
||||||
|
f"{stack.base}/api/v1/restore-plans/{plan['id']}/apply", timeout=TIMEOUT
|
||||||
|
).raise_for_status()
|
||||||
|
wait_until(
|
||||||
|
lambda: all(a["availability_state"] == "active" for a in stack.assets()), timeout=60
|
||||||
|
)
|
||||||
|
return {"preflight": report, "plan": plan}
|
||||||
|
|
||||||
|
|
||||||
|
def _sha256(path: Path) -> str:
|
||||||
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _contents(*roots: Path) -> set[str]:
|
||||||
|
"""Every photo byte-string reachable anywhere, keyed by hash."""
|
||||||
|
return {
|
||||||
|
_sha256(path)
|
||||||
|
for root in roots
|
||||||
|
for path in root.rglob("*.jpg")
|
||||||
|
if path.is_file() and not path.name.startswith(".")
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _manifest(stack) -> list[dict]:
|
||||||
|
path = stack.archive / ALBUM / MANIFEST
|
||||||
|
if not path.exists():
|
||||||
|
return []
|
||||||
|
return [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
|
||||||
|
|
||||||
|
|
||||||
|
# ── preflight blockers ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_preflight_refuses_an_offline_medium_a_wrong_volume_and_a_full_disk(stack):
|
||||||
|
stack.start(worker=False)
|
||||||
|
location = stack.register()
|
||||||
|
|
||||||
|
ready = _preflight(stack, location["id"])
|
||||||
|
assert ready["state"] == "ready"
|
||||||
|
assert ready["totals"]["assets"] == 2
|
||||||
|
|
||||||
|
stack.unmount()
|
||||||
|
offline = _preflight(stack, location["id"])
|
||||||
|
assert offline["state"] == "blocked"
|
||||||
|
assert "location_offline" in {issue["code"] for issue in offline["blockers"]}
|
||||||
|
|
||||||
|
# A different disk mounted at the same place is not this medium.
|
||||||
|
(stack.archive / ArchiveStack.MARKER).write_text('{"media_id": "another-disk"}')
|
||||||
|
wrong = _preflight(stack, location["id"])
|
||||||
|
assert "wrong_volume" in {issue["code"] for issue in wrong["blockers"]}
|
||||||
|
|
||||||
|
# Nothing has been created, moved, or removed by any of that.
|
||||||
|
assert stack.plans() == []
|
||||||
|
assert sorted(p.name for p in (stack.seeded.lib / ALBUM).glob("*.jpg")) == ["a.jpg", "b.jpg"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_preflight_refuses_capacity_it_cannot_prove_and_bytes_it_cannot_vouch_for(tmp_path):
|
||||||
|
seeded = seed_album(tmp_path)
|
||||||
|
mark_uploaded(seeded)
|
||||||
|
stack = ArchiveStack(tmp_path, seeded)
|
||||||
|
try:
|
||||||
|
# A reserve larger than any disk: the scope cannot be promised room.
|
||||||
|
stack.start(
|
||||||
|
worker=False,
|
||||||
|
extra_env={"PHOTO_PIPELINE_ARCHIVE_FREE_SPACE_RESERVE_BYTES": str(1 << 62)},
|
||||||
|
)
|
||||||
|
location = stack.register()
|
||||||
|
report = _preflight(stack, location["id"])
|
||||||
|
assert "insufficient_capacity" in {issue["code"] for issue in report["blockers"]}
|
||||||
|
|
||||||
|
# And a plan for a blocked scope is refused rather than half-created.
|
||||||
|
refused = httpx.post(
|
||||||
|
f"{stack.base}/api/v1/archive-plans",
|
||||||
|
json={"location_id": location["id"], "token": report["token"]},
|
||||||
|
timeout=TIMEOUT,
|
||||||
|
)
|
||||||
|
assert refused.status_code == 422
|
||||||
|
assert refused.json()["error"]["code"] == "blocked"
|
||||||
|
assert stack.plans() == []
|
||||||
|
finally:
|
||||||
|
stack.stop()
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_album_whose_bytes_changed_after_upload_cannot_be_archived(stack):
|
||||||
|
stack.start(worker=False)
|
||||||
|
location = stack.register()
|
||||||
|
(stack.seeded.lib / ALBUM / "a.jpg").write_bytes(b"edited after the upload")
|
||||||
|
|
||||||
|
report = _preflight(stack, location["id"])
|
||||||
|
album = report["albums"][0]
|
||||||
|
assert album["state"] == "blocked"
|
||||||
|
assert "partial_scope" in {issue["code"] for issue in album["blockers"]}
|
||||||
|
assert "bytes_changed" in {
|
||||||
|
issue["code"] for asset in album["assets"] for issue in asset["blockers"]
|
||||||
|
}
|
||||||
|
assert report["state"] == "blocked"
|
||||||
|
|
||||||
|
|
||||||
|
# ── copy, verify, remove ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_archived_album_leaves_the_library_only_after_it_is_verified(stack):
|
||||||
|
stack.start()
|
||||||
|
before = _contents(stack.seeded.lib)
|
||||||
|
hashes = {a["id"]: a["current_sha256"] for a in stack.assets()}
|
||||||
|
|
||||||
|
result = _archive_album(stack)
|
||||||
|
plan = result["plan"]
|
||||||
|
|
||||||
|
assert plan["state"] == "complete"
|
||||||
|
assert {op["journal_state"] for op in plan["operations"]} == {"complete"}
|
||||||
|
# Same filesystem here, so the transfer took the atomic-move path — recorded as
|
||||||
|
# what actually happened, not as the plan's guess.
|
||||||
|
assert all(op["same_filesystem"] for op in plan["operations"])
|
||||||
|
|
||||||
|
# The bytes are on the medium, hashing exactly as recorded, and gone from the
|
||||||
|
# library. Not one photo was lost in between.
|
||||||
|
assert _contents(stack.archive) == before
|
||||||
|
assert not list((stack.seeded.lib / ALBUM).glob("*.jpg"))
|
||||||
|
for asset_id, sha256 in hashes.items():
|
||||||
|
asset = _assets(stack)[asset_id]
|
||||||
|
assert asset["current_path"] is None
|
||||||
|
assert asset["availability_state"] == "archived_online"
|
||||||
|
assert _sha256(stack.archive / asset["archive_path"]) == sha256
|
||||||
|
|
||||||
|
# The medium carries its own record of what it holds.
|
||||||
|
manifest = _manifest(stack)
|
||||||
|
assert {entry["asset_id"] for entry in manifest} == set(hashes)
|
||||||
|
assert {entry["sha256"] for entry in manifest} == set(hashes.values())
|
||||||
|
assert {entry["media_id"] for entry in manifest} == {result["location"]["media_id"]}
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_archived_state_survives_a_server_restart(stack):
|
||||||
|
stack.start()
|
||||||
|
result = _archive_album(stack)
|
||||||
|
before = _assets(stack)
|
||||||
|
plan_before = _plan_state(stack, result["plan"]["id"])
|
||||||
|
|
||||||
|
stack.restart_server()
|
||||||
|
|
||||||
|
assert _assets(stack) == before
|
||||||
|
assert _plan_state(stack, result["plan"]["id"]) == plan_before
|
||||||
|
assert {a["availability_state"] for a in stack.assets()} == {"archived_online"}
|
||||||
|
|
||||||
|
|
||||||
|
# ── interruption at every persisted transition ───────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("barrier", BARRIERS)
|
||||||
|
def test_a_worker_killed_at_each_transition_loses_nothing_and_recovers(tmp_path, barrier):
|
||||||
|
# One photo, so the crash lands on the only operation and the end state is the
|
||||||
|
# recovery's own doing rather than a mixture with untouched work.
|
||||||
|
seeded = seed_album(tmp_path, names=("a.jpg",))
|
||||||
|
mark_uploaded(seeded)
|
||||||
|
stack = ArchiveStack(tmp_path, seeded)
|
||||||
|
try:
|
||||||
|
# Only the worker carries the crash barrier: it dies the instant this journal
|
||||||
|
# state is persisted, while the server stays up to be asked what happened.
|
||||||
|
stack.start(worker=False)
|
||||||
|
stack.restart_worker(extra_env={"PHOTO_PIPELINE_FAULT_AFTER": barrier})
|
||||||
|
before = _contents(stack.seeded.lib)
|
||||||
|
location = stack.register()
|
||||||
|
plan = _plan(stack, location["id"])
|
||||||
|
_apply(stack, plan["id"]).raise_for_status()
|
||||||
|
|
||||||
|
wait_until(lambda: stack.worker.poll() is not None, timeout=60)
|
||||||
|
assert stack.worker.returncode in (9, -9), "the worker should have been killed"
|
||||||
|
|
||||||
|
# Whatever the crash interrupted, every photo is still readable somewhere.
|
||||||
|
assert before <= _contents(stack.seeded.lib, stack.archive), f"content lost at {barrier}"
|
||||||
|
# And nothing was removed that had not been verified first.
|
||||||
|
for operation in _plan_state(stack, plan["id"])["operations"]:
|
||||||
|
source = Path(operation["source_path"])
|
||||||
|
if not source.exists():
|
||||||
|
archived = Path(operation["destination_path"])
|
||||||
|
assert archived.exists(), f"{barrier}: source removed without an archive copy"
|
||||||
|
assert _sha256(archived) == operation["expected_sha256"]
|
||||||
|
|
||||||
|
# Recovery is offered from evidence, resolves without a worker, and converges.
|
||||||
|
verdicts = _recovery(stack)
|
||||||
|
assert all(v["classification"] != "manual" for v in verdicts["operations"]), verdicts
|
||||||
|
_resolve(stack)
|
||||||
|
assert _recovery(stack)["operations"] == []
|
||||||
|
# Repeating it changes nothing.
|
||||||
|
assert _resolve(stack) == {"resumed": 0, "completed": 0, "manual": 0}
|
||||||
|
|
||||||
|
# Recovery finishes what was durable and re-plans what was not. Before the
|
||||||
|
# archive copy existed, that means putting the item back to `planned` with the
|
||||||
|
# original untouched; from `verified` onwards it means completing it.
|
||||||
|
stack.restart_server()
|
||||||
|
operation = _plan_state(stack, plan["id"])["operations"][0]
|
||||||
|
asset = next(iter(stack.assets()))
|
||||||
|
if barrier == "transferring":
|
||||||
|
assert operation["journal_state"] == "planned"
|
||||||
|
assert Path(operation["source_path"]).exists()
|
||||||
|
assert asset["availability_state"] == "active"
|
||||||
|
assert _contents(stack.seeded.lib) == before
|
||||||
|
else:
|
||||||
|
assert operation["journal_state"] == "complete"
|
||||||
|
assert not Path(operation["source_path"]).exists()
|
||||||
|
assert _contents(stack.archive) == before
|
||||||
|
assert asset["availability_state"] == "archived_online"
|
||||||
|
assert _sha256(stack.archive / asset["archive_path"]) == asset["current_sha256"]
|
||||||
|
assert {entry["sha256"] for entry in _manifest(stack)} == before
|
||||||
|
finally:
|
||||||
|
stack.stop()
|
||||||
|
|
||||||
|
|
||||||
|
def test_evidence_that_contradicts_the_journal_is_never_guessed(stack):
|
||||||
|
stack.start(worker=False)
|
||||||
|
location = stack.register()
|
||||||
|
plan = _plan(stack, location["id"])
|
||||||
|
# The journal claims a verified archive copy the medium does not have.
|
||||||
|
with session_factory(stack.seeded) as sf:
|
||||||
|
from photo_pipeline.services.archive_journal import ArchiveJournal
|
||||||
|
|
||||||
|
journal = ArchiveJournal(sf)
|
||||||
|
operation = journal.operations(plan["id"])[0]
|
||||||
|
journal.begin(operation["id"], worker_id="crashed", fencing_token=1)
|
||||||
|
journal.transition(operation["id"], "verified", fencing_token=1)
|
||||||
|
|
||||||
|
verdicts = _recovery(stack)
|
||||||
|
assert [v["classification"] for v in verdicts["operations"]] == ["manual"]
|
||||||
|
assert _resolve(stack)["manual"] == 1
|
||||||
|
# The source is untouched and the unresolved item keeps blocking a new archive.
|
||||||
|
assert (stack.seeded.lib / ALBUM / "a.jpg").exists()
|
||||||
|
blocked = _preflight(stack, location["id"])
|
||||||
|
assert "archive_pending" in {issue["code"] for issue in blocked["blockers"]}
|
||||||
|
|
||||||
|
|
||||||
|
# ── offline identity and deduplication ───────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_offline_photos_stay_hashed_previewable_and_deduplicable(stack):
|
||||||
|
stack.start()
|
||||||
|
result = _archive_album(stack)
|
||||||
|
archived = _assets(stack)
|
||||||
|
copy_source = stack.archive / next(iter(archived.values()))["archive_path"]
|
||||||
|
exact = stack.seeded.lib / "inbox" / "again.jpg"
|
||||||
|
exact.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
shutil.copy2(copy_source, exact)
|
||||||
|
stack.unmount()
|
||||||
|
|
||||||
|
scanned = httpx.post(f"{stack.base}/api/v1/inventory/scan", timeout=60)
|
||||||
|
scanned.raise_for_status()
|
||||||
|
# An unmounted medium is not a missing file.
|
||||||
|
assert scanned.json()["counts"].get("missing") is None
|
||||||
|
offline = {a["id"]: a for a in stack.assets() if a["id"] in archived}
|
||||||
|
assert {a["availability_state"] for a in offline.values()} == {"archived_offline"}
|
||||||
|
assert all(a["current_sha256"] for a in offline.values())
|
||||||
|
|
||||||
|
# The new active copy is recognised as the archived photo, not as a new one.
|
||||||
|
httpx.post(f"{stack.base}/api/v1/duplicates/detect", timeout=60).raise_for_status()
|
||||||
|
clusters = httpx.get(
|
||||||
|
f"{stack.base}/api/v1/duplicates/clusters", params={"limit": 50}, timeout=TIMEOUT
|
||||||
|
).json()["items"]
|
||||||
|
exact_clusters = [c for c in clusters if c["method"] == "exact"]
|
||||||
|
assert len(exact_clusters) == 1
|
||||||
|
detail = httpx.get(
|
||||||
|
f"{stack.base}/api/v1/duplicates/clusters/{exact_clusters[0]['id']}", timeout=TIMEOUT
|
||||||
|
).json()
|
||||||
|
canonical = next(m for m in detail["members"] if m["asset_id"] == detail["canonical_asset_id"])
|
||||||
|
assert canonical["availability_state"] == "archived_offline"
|
||||||
|
assert canonical["archive_location"] == result["location"]["name"]
|
||||||
|
assert canonical["preview"]["state"] == "ready" and canonical["preview"]["protected"]
|
||||||
|
|
||||||
|
# And the retained preview really is served while the medium is away.
|
||||||
|
thumbnail = httpx.get(
|
||||||
|
f"{stack.base}/api/v1/assets/{canonical['asset_id']}/thumbnail",
|
||||||
|
params={"size": 1280},
|
||||||
|
timeout=TIMEOUT,
|
||||||
|
)
|
||||||
|
assert thumbnail.status_code == 200
|
||||||
|
assert thumbnail.headers["content-type"] == "image/webp"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_fuzzy_offline_match_asks_for_the_medium_instead_of_guessing(stack):
|
||||||
|
stack.start()
|
||||||
|
result = _archive_album(stack)
|
||||||
|
archived = next(iter(_assets(stack).values()))
|
||||||
|
variant = stack.seeded.lib / "inbox" / "resized.jpg"
|
||||||
|
variant.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
_resize(stack.archive / archived["archive_path"], variant)
|
||||||
|
stack.unmount()
|
||||||
|
|
||||||
|
httpx.post(f"{stack.base}/api/v1/inventory/scan", timeout=60).raise_for_status()
|
||||||
|
httpx.post(f"{stack.base}/api/v1/duplicates/detect", timeout=60).raise_for_status()
|
||||||
|
clusters = httpx.get(
|
||||||
|
f"{stack.base}/api/v1/duplicates/clusters", params={"limit": 50}, timeout=TIMEOUT
|
||||||
|
).json()["items"]
|
||||||
|
perceptual = [c for c in clusters if c["method"] == "perceptual"]
|
||||||
|
assert len(perceptual) == 1
|
||||||
|
|
||||||
|
detail = httpx.get(
|
||||||
|
f"{stack.base}/api/v1/duplicates/clusters/{perceptual[0]['id']}", timeout=TIMEOUT
|
||||||
|
).json()
|
||||||
|
# A fuzzy match is never decided automatically, and full-resolution review names
|
||||||
|
# the medium to mount rather than guessing from the preview.
|
||||||
|
assert detail["state"] == "open" and detail["requires_confirmation"] is True
|
||||||
|
assert detail["mount_required"] == [result["location"]["name"]]
|
||||||
|
|
||||||
|
|
||||||
|
def test_mounting_the_medium_again_makes_the_originals_reachable(stack):
|
||||||
|
stack.start()
|
||||||
|
result = _archive_album(stack)
|
||||||
|
stack.unmount()
|
||||||
|
httpx.post(f"{stack.base}/api/v1/inventory/scan", timeout=60).raise_for_status()
|
||||||
|
assert {a["availability_state"] for a in stack.assets()} == {"archived_offline"}
|
||||||
|
# Restoring from a medium that is not there is refused, with its reason.
|
||||||
|
away = httpx.post(
|
||||||
|
f"{stack.base}/api/v1/restore-preflight",
|
||||||
|
json={"location_id": result["location"]["id"]},
|
||||||
|
timeout=TIMEOUT,
|
||||||
|
).json()
|
||||||
|
assert away["state"] == "blocked"
|
||||||
|
assert "location_offline" in {issue["code"] for issue in away["blockers"]}
|
||||||
|
|
||||||
|
stack.remount()
|
||||||
|
locations = httpx.get(f"{stack.base}/api/v1/archive-locations", timeout=TIMEOUT).json()
|
||||||
|
assert locations["locations"][0]["state"] == "online"
|
||||||
|
assert {a["availability_state"] for a in stack.assets()} == {"archived_online"}
|
||||||
|
|
||||||
|
|
||||||
|
# ── restore ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_restoring_returns_the_bytes_without_losing_identity_or_the_archive(stack):
|
||||||
|
stack.start()
|
||||||
|
result = _archive_album(stack)
|
||||||
|
identities = set(_assets(stack))
|
||||||
|
hashes = {a["id"]: a["current_sha256"] for a in stack.assets()}
|
||||||
|
|
||||||
|
_restore(stack, result["location"]["id"])
|
||||||
|
|
||||||
|
restored = _assets(stack)
|
||||||
|
assert set(restored) == identities # the same photos, not new ones
|
||||||
|
for asset_id, sha256 in hashes.items():
|
||||||
|
asset = restored[asset_id]
|
||||||
|
assert asset["availability_state"] == "active"
|
||||||
|
assert _sha256(Path(asset["current_path"])) == sha256
|
||||||
|
# The archive copy is a copy: restoring empties nothing.
|
||||||
|
assert (stack.archive / asset["archive_path"]).exists()
|
||||||
|
|
||||||
|
stack.restart_server()
|
||||||
|
assert _assets(stack) == restored
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_restore_comes_back_beside_an_occupant_never_over_it(stack):
|
||||||
|
stack.start()
|
||||||
|
result = _archive_album(stack)
|
||||||
|
squatter = stack.seeded.lib / ALBUM / "a.jpg"
|
||||||
|
squatter.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
squatter.write_bytes(b"a different photo lives here now")
|
||||||
|
|
||||||
|
report = _restore(stack, result["location"]["id"])["preflight"]
|
||||||
|
destinations = {item["destination_path"] for item in report["items"]}
|
||||||
|
assert any("(restored)" in destination for destination in destinations)
|
||||||
|
|
||||||
|
assert squatter.read_bytes() == b"a different photo lives here now"
|
||||||
|
assert (stack.seeded.lib / ALBUM / "a (restored).jpg").exists()
|
||||||
|
assert {a["availability_state"] for a in stack.assets()} == {"active"}
|
||||||
|
|
||||||
|
|
||||||
|
def _resize(source: Path, destination: Path, scale: float = 0.5) -> None:
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
with Image.open(source) as opened:
|
||||||
|
opened.resize(
|
||||||
|
(int(opened.width * scale), int(opened.height * scale)), Image.LANCZOS
|
||||||
|
).save(destination, quality=95)
|
||||||
@@ -14,6 +14,7 @@ MAP = json.loads((REPO / "tests" / "story_traceability.json").read_text())["stor
|
|||||||
PHASE_A_STORIES = {f"US01-0{n}" for n in range(1, 8)}
|
PHASE_A_STORIES = {f"US01-0{n}" for n in range(1, 8)}
|
||||||
PHASE_D_STORIES = {f"US04-0{n}" for n in range(1, 7)}
|
PHASE_D_STORIES = {f"US04-0{n}" for n in range(1, 7)}
|
||||||
PHASE_E_STORIES = {f"US05-0{n}" for n in range(1, 7)}
|
PHASE_E_STORIES = {f"US05-0{n}" for n in range(1, 7)}
|
||||||
|
PHASE_F_STORIES = {f"US06-0{n}" for n in range(1, 7)}
|
||||||
|
|
||||||
|
|
||||||
def test_all_phase_a_stories_are_mapped():
|
def test_all_phase_a_stories_are_mapped():
|
||||||
@@ -32,6 +33,13 @@ def test_all_phase_e_stories_are_mapped():
|
|||||||
assert PHASE_E_STORIES <= set(MAP)
|
assert PHASE_E_STORIES <= set(MAP)
|
||||||
|
|
||||||
|
|
||||||
|
def test_all_phase_f_stories_are_mapped():
|
||||||
|
"""US06-06 acceptance: every archive-lifecycle story, destination through
|
||||||
|
restore, is tied to automated tests — archiving is the only stage that removes
|
||||||
|
an original."""
|
||||||
|
assert PHASE_F_STORIES <= set(MAP)
|
||||||
|
|
||||||
|
|
||||||
def test_every_mapped_test_file_exists_and_is_nonempty():
|
def test_every_mapped_test_file_exists_and_is_nonempty():
|
||||||
for story, files in MAP.items():
|
for story, files in MAP.items():
|
||||||
assert files, f"{story} maps to no tests"
|
assert files, f"{story} maps to no tests"
|
||||||
|
|||||||
@@ -140,6 +140,9 @@
|
|||||||
],
|
],
|
||||||
"US06-05": [
|
"US06-05": [
|
||||||
"tests/e2e/test_archive_ui.py"
|
"tests/e2e/test_archive_ui.py"
|
||||||
|
],
|
||||||
|
"US06-06": [
|
||||||
|
"tests/e2e/test_phase_f_pipeline.py"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user