US06-02: Transfer, Verify, and Remove Active Sources (#78)
This commit was merged in pull request #78.
This commit is contained in:
480
tests/integration/test_archive_transfer.py
Normal file
480
tests/integration/test_archive_transfer.py
Normal file
@@ -0,0 +1,480 @@
|
||||
"""Transferring, verifying, and removing active sources (US06-02).
|
||||
|
||||
Every case here asks the same question the service exists to answer: could an
|
||||
original leave active storage without a durable, byte-identical archive copy? The
|
||||
files are real, the hashes are real, and each failure path asserts that the source
|
||||
is still exactly where it was.
|
||||
|
||||
Cross-filesystem behaviour is forced by patching the device comparison rather than
|
||||
by requiring a second real filesystem in CI — the copy/verify/publish code that runs
|
||||
is the production one.
|
||||
"""
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import select
|
||||
|
||||
from photo_pipeline.api.app import create_app
|
||||
from photo_pipeline.config import Config
|
||||
from photo_pipeline.db import create_db_engine, create_session_factory, run_migrations
|
||||
from photo_pipeline.models import Asset, AssetPath, UploadBatch, UploadItem
|
||||
from photo_pipeline.services import archive_transfer as transfer_module
|
||||
from photo_pipeline.services.archive_journal import ArchiveState
|
||||
from photo_pipeline.services.archive_transfer import (
|
||||
MANIFEST_NAME,
|
||||
ArchiveTransferService,
|
||||
read_manifest,
|
||||
)
|
||||
from photo_pipeline.services.archives import ArchiveError, ArchiveService
|
||||
from photo_pipeline.services.hashing import sha256_file
|
||||
|
||||
pytestmark = pytest.mark.phase_f # part of the Phase F acceptance gate (US06-06)
|
||||
|
||||
NOW = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
# ── environment ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _env(tmp_path):
|
||||
(tmp_path / "data").mkdir(exist_ok=True)
|
||||
lib = tmp_path / "lib"
|
||||
lib.mkdir(exist_ok=True)
|
||||
archive = tmp_path / "archive"
|
||||
archive.mkdir(exist_ok=True)
|
||||
config = Config.from_env(
|
||||
{
|
||||
"PHOTO_PIPELINE_DATA_DIR": str(tmp_path / "data"),
|
||||
"PHOTO_PIPELINE_LIBRARY_ROOTS": str(lib),
|
||||
"PHOTO_PIPELINE_ARCHIVE_FREE_SPACE_RESERVE_BYTES": "0",
|
||||
}
|
||||
)
|
||||
run_migrations(config.database_url)
|
||||
return config, create_session_factory(create_db_engine(config.database_url)), lib, archive
|
||||
|
||||
|
||||
def _album(sf, lib, album="rome", names=("a.jpg", "b.jpg")):
|
||||
"""A real album whose assets carry the verified upload evidence archiving needs."""
|
||||
folder = lib / album
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
ids = []
|
||||
with sf() as session:
|
||||
batch_id = str(uuid.uuid4())
|
||||
session.add(
|
||||
UploadBatch(
|
||||
id=batch_id,
|
||||
album=album,
|
||||
folder=str(folder),
|
||||
album_name=album,
|
||||
state="succeeded",
|
||||
preflight_token="v1:test",
|
||||
outcome_state="verified",
|
||||
created_at=NOW,
|
||||
)
|
||||
)
|
||||
for name in names:
|
||||
path = folder / name
|
||||
path.write_bytes(f"{album}/{name} bytes".encode() * 8)
|
||||
asset_id = str(uuid.uuid4())
|
||||
ids.append(asset_id)
|
||||
session.add(
|
||||
Asset(
|
||||
id=asset_id,
|
||||
original_path=str(path),
|
||||
current_path=str(path),
|
||||
discovered_at=NOW,
|
||||
hash_version=1,
|
||||
byte_size=path.stat().st_size,
|
||||
current_sha256=sha256_file(path),
|
||||
)
|
||||
)
|
||||
session.add(AssetPath(asset_id=asset_id, path=str(path), valid_from=NOW))
|
||||
session.add(
|
||||
UploadItem(
|
||||
batch_id=batch_id,
|
||||
asset_id=asset_id,
|
||||
path=str(path),
|
||||
sha256=sha256_file(path),
|
||||
sha1="0" * 40,
|
||||
state="sent",
|
||||
outcome="uploaded",
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
return folder, ids
|
||||
|
||||
|
||||
def _location(sf, config, archive, name="external"):
|
||||
return ArchiveService(sf, config=config).register(name, str(archive))
|
||||
|
||||
|
||||
def _plan(sf, config, location_id, albums=None):
|
||||
service = ArchiveTransferService(sf, config=config)
|
||||
token = ArchiveService(sf, config=config).preflight(location_id, albums)["token"]
|
||||
return service, service.create(location_id, albums, token=token)
|
||||
|
||||
|
||||
def _contents(*roots):
|
||||
"""Every file's bytes under the given roots, ignoring our own bookkeeping."""
|
||||
return sorted(
|
||||
path.read_bytes()
|
||||
for root in roots
|
||||
for path in root.rglob("*")
|
||||
if path.is_file() and path.name != MANIFEST_NAME and not path.name.startswith(".")
|
||||
)
|
||||
|
||||
|
||||
def _states(service, plan_id):
|
||||
return [row["journal_state"] for row in service.journal.operations(plan_id)]
|
||||
|
||||
|
||||
def _asset(sf, asset_id):
|
||||
with sf() as session:
|
||||
return session.get(Asset, asset_id)
|
||||
|
||||
|
||||
# ── planning ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_a_plan_records_every_file_with_its_expected_hash_and_moves_nothing(tmp_path):
|
||||
config, sf, lib, archive = _env(tmp_path)
|
||||
folder, ids = _album(sf, lib)
|
||||
location = _location(sf, config, archive)
|
||||
before = _contents(lib)
|
||||
|
||||
service, plan = _plan(sf, config, location["id"])
|
||||
|
||||
assert plan["state"] == "planned" and plan["asset_count"] == 2
|
||||
assert sorted(op["asset_id"] for op in plan["operations"]) == sorted(ids)
|
||||
for operation in plan["operations"]:
|
||||
source = folder / operation["archive_path"].split("/")[-1]
|
||||
assert operation["expected_sha256"] == sha256_file(source)
|
||||
assert operation["archive_path"].startswith("rome/")
|
||||
assert operation["journal_state"] == ArchiveState.PLANNED
|
||||
assert _contents(lib) == before
|
||||
assert _contents(archive) == []
|
||||
|
||||
|
||||
def test_a_stale_token_cannot_create_a_plan(tmp_path):
|
||||
config, sf, lib, archive = _env(tmp_path)
|
||||
folder, _ = _album(sf, lib)
|
||||
location = _location(sf, config, archive)
|
||||
token = ArchiveService(sf, config=config).preflight(location["id"])["token"]
|
||||
|
||||
(folder / "a.jpg").write_bytes(b"edited after approval")
|
||||
|
||||
with pytest.raises(ArchiveError) as error:
|
||||
ArchiveTransferService(sf, config=config).create(location["id"], token=token)
|
||||
assert error.value.code == "stale_token"
|
||||
|
||||
|
||||
def test_a_blocked_scope_cannot_create_a_plan(tmp_path):
|
||||
"""No verified upload means Immich may not hold these bytes; archiving would
|
||||
remove the only copy."""
|
||||
config, sf, lib, archive = _env(tmp_path)
|
||||
folder = lib / "rome"
|
||||
folder.mkdir()
|
||||
path = folder / "a.jpg"
|
||||
path.write_bytes(b"never uploaded")
|
||||
with sf() as session:
|
||||
session.add(
|
||||
Asset(
|
||||
id=str(uuid.uuid4()),
|
||||
original_path=str(path),
|
||||
current_path=str(path),
|
||||
discovered_at=NOW,
|
||||
hash_version=1,
|
||||
byte_size=path.stat().st_size,
|
||||
current_sha256=sha256_file(path),
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
location = _location(sf, config, archive)
|
||||
token = ArchiveService(sf, config=config).preflight(location["id"])["token"]
|
||||
|
||||
with pytest.raises(ArchiveError) as error:
|
||||
ArchiveTransferService(sf, config=config).create(location["id"], token=token)
|
||||
|
||||
assert error.value.code == "blocked"
|
||||
assert path.exists()
|
||||
|
||||
|
||||
# ── the transfer ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize("same_filesystem", [True, False])
|
||||
def test_archiving_verifies_the_copy_before_the_source_is_removed(
|
||||
tmp_path, monkeypatch, same_filesystem
|
||||
):
|
||||
config, sf, lib, archive = _env(tmp_path)
|
||||
folder, ids = _album(sf, lib)
|
||||
location = _location(sf, config, archive)
|
||||
service, plan = _plan(sf, config, location["id"])
|
||||
expected = {op["asset_id"]: op["expected_sha256"] for op in plan["operations"]}
|
||||
before = _contents(lib)
|
||||
if not same_filesystem:
|
||||
# Force the copy-verify-publish path without needing a second real volume.
|
||||
monkeypatch.setattr(transfer_module, "_same_filesystem", lambda *_: False)
|
||||
|
||||
result = service.apply(plan["id"])
|
||||
|
||||
assert result == {
|
||||
"plan_id": plan["id"],
|
||||
"archived": 2,
|
||||
"failed": 0,
|
||||
"skipped": 0,
|
||||
"state": "complete",
|
||||
}
|
||||
# The bytes moved: nothing is left in the library, everything is in the archive.
|
||||
assert _contents(archive) == before
|
||||
assert _contents(lib) == []
|
||||
assert not folder.exists(), "an emptied album folder is not left behind"
|
||||
for asset_id, digest in expected.items():
|
||||
archived = archive / "rome" / _asset(sf, asset_id).archive_path.split("/")[-1]
|
||||
assert sha256_file(archived) == digest
|
||||
assert _states(service, plan["id"]) == [ArchiveState.COMPLETE] * 2
|
||||
# No transfer temporaries survive either path.
|
||||
assert not list((archive / "rome").glob(".archive-*"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("same_filesystem", [True, False])
|
||||
def test_the_manifest_on_the_medium_matches_the_archived_bytes(
|
||||
tmp_path, monkeypatch, same_filesystem
|
||||
):
|
||||
"""The manifest is the medium's own record: it must be usable to verify the
|
||||
archive with no database at all."""
|
||||
config, sf, lib, archive = _env(tmp_path)
|
||||
_album(sf, lib)
|
||||
location = _location(sf, config, archive)
|
||||
service, plan = _plan(sf, config, location["id"])
|
||||
if not same_filesystem:
|
||||
monkeypatch.setattr(transfer_module, "_same_filesystem", lambda *_: False)
|
||||
|
||||
service.apply(plan["id"])
|
||||
|
||||
entries = read_manifest(archive / "rome")
|
||||
assert len(entries) == 2
|
||||
for entry in entries:
|
||||
archived = archive / entry["archive_path"]
|
||||
assert sha256_file(archived) == entry["sha256"]
|
||||
assert entry["byte_size"] == archived.stat().st_size
|
||||
assert entry["media_id"] == location["media_id"]
|
||||
assert entry["plan_id"] == plan["id"] and entry["album"] == "rome"
|
||||
|
||||
|
||||
def test_archived_assets_keep_their_identity_and_gain_their_new_location(tmp_path):
|
||||
config, sf, lib, archive = _env(tmp_path)
|
||||
folder, ids = _album(sf, lib)
|
||||
location = _location(sf, config, archive)
|
||||
service, plan = _plan(sf, config, location["id"])
|
||||
|
||||
service.apply(plan["id"])
|
||||
|
||||
with sf() as session:
|
||||
for asset_id in ids:
|
||||
asset = session.get(Asset, asset_id)
|
||||
assert asset is not None, "archiving never deletes the record"
|
||||
assert asset.current_path is None
|
||||
assert asset.availability_state == "archived_online"
|
||||
assert asset.archive_location_id == location["id"]
|
||||
assert (archive / asset.archive_path).exists()
|
||||
occurrences = session.scalars(
|
||||
select(AssetPath).where(AssetPath.asset_id == asset_id)
|
||||
).all()
|
||||
active = [row for row in occurrences if row.valid_until is None]
|
||||
assert [row.path for row in active] == [str(archive / asset.archive_path)]
|
||||
closed = [row for row in occurrences if row.valid_until is not None]
|
||||
assert [row.path for row in closed] == [str(folder / asset.archive_path.split("/")[-1])]
|
||||
|
||||
|
||||
def test_applying_the_same_plan_again_changes_nothing(tmp_path):
|
||||
config, sf, lib, archive = _env(tmp_path)
|
||||
_album(sf, lib)
|
||||
location = _location(sf, config, archive)
|
||||
service, plan = _plan(sf, config, location["id"])
|
||||
service.apply(plan["id"])
|
||||
archived = _contents(archive)
|
||||
manifest = read_manifest(archive / "rome")
|
||||
|
||||
again = service.apply(plan["id"])
|
||||
|
||||
assert again["skipped"] == 2 and again["archived"] == 0 and again["failed"] == 0
|
||||
assert _contents(archive) == archived
|
||||
assert read_manifest(archive / "rome") == manifest
|
||||
|
||||
|
||||
# ── unexpected changes stop the item ─────────────────────────────────────────
|
||||
|
||||
|
||||
def test_an_occupied_destination_is_never_overwritten(tmp_path):
|
||||
config, sf, lib, archive = _env(tmp_path)
|
||||
folder, _ = _album(sf, lib, names=("a.jpg",))
|
||||
location = _location(sf, config, archive)
|
||||
service, plan = _plan(sf, config, location["id"])
|
||||
(archive / "rome").mkdir()
|
||||
(archive / "rome" / "a.jpg").write_bytes(b"someone else's file")
|
||||
|
||||
result = service.apply(plan["id"])
|
||||
|
||||
assert result["failed"] == 1 and result["state"] == "failed"
|
||||
assert (archive / "rome" / "a.jpg").read_bytes() == b"someone else's file"
|
||||
assert (folder / "a.jpg").exists(), "the source must survive a refused transfer"
|
||||
assert service.journal.operations(plan["id"])[0]["error_code"] == "destination_exists"
|
||||
|
||||
|
||||
def test_a_source_edited_after_planning_is_neither_archived_nor_removed(tmp_path):
|
||||
config, sf, lib, archive = _env(tmp_path)
|
||||
folder, _ = _album(sf, lib, names=("a.jpg",))
|
||||
location = _location(sf, config, archive)
|
||||
service, plan = _plan(sf, config, location["id"])
|
||||
|
||||
(folder / "a.jpg").write_bytes(b"edited between approval and apply")
|
||||
result = service.apply(plan["id"])
|
||||
|
||||
assert result["failed"] == 1
|
||||
assert (folder / "a.jpg").read_bytes() == b"edited between approval and apply"
|
||||
assert not (archive / "rome" / "a.jpg").exists()
|
||||
assert service.journal.operations(plan["id"])[0]["error_code"] == "source_changed"
|
||||
|
||||
|
||||
def test_a_copy_that_lands_with_the_wrong_bytes_is_not_published(tmp_path, monkeypatch):
|
||||
"""The read-back hash — not the fact that a write returned — is what proves the
|
||||
archive copy."""
|
||||
config, sf, lib, archive = _env(tmp_path)
|
||||
folder, _ = _album(sf, lib, names=("a.jpg",))
|
||||
location = _location(sf, config, archive)
|
||||
service, plan = _plan(sf, config, location["id"])
|
||||
monkeypatch.setattr(transfer_module, "_same_filesystem", lambda *_: False)
|
||||
monkeypatch.setattr(
|
||||
transfer_module.shutil,
|
||||
"copyfileobj",
|
||||
lambda src, dst, length=0: dst.write(b"corrupted in flight"),
|
||||
)
|
||||
|
||||
result = service.apply(plan["id"])
|
||||
|
||||
assert result["failed"] == 1
|
||||
assert service.journal.operations(plan["id"])[0]["error_code"] == "copy_mismatch"
|
||||
assert (folder / "a.jpg").exists()
|
||||
assert not (archive / "rome" / "a.jpg").exists()
|
||||
assert not list((archive / "rome").glob(".archive-*")), "the failed copy is cleaned up"
|
||||
|
||||
|
||||
def test_a_source_replaced_by_a_symlink_is_refused(tmp_path):
|
||||
config, sf, lib, archive = _env(tmp_path)
|
||||
folder, _ = _album(sf, lib, names=("a.jpg",))
|
||||
location = _location(sf, config, archive)
|
||||
service, plan = _plan(sf, config, location["id"])
|
||||
elsewhere = tmp_path / "elsewhere.jpg"
|
||||
elsewhere.write_bytes(b"not a library file")
|
||||
(folder / "a.jpg").unlink()
|
||||
(folder / "a.jpg").symlink_to(elsewhere)
|
||||
|
||||
result = service.apply(plan["id"])
|
||||
|
||||
assert result["failed"] == 1
|
||||
assert elsewhere.exists() and (folder / "a.jpg").is_symlink()
|
||||
assert not (archive / "rome" / "a.jpg").exists()
|
||||
|
||||
|
||||
def test_one_failed_file_does_not_stop_the_rest_of_the_album(tmp_path):
|
||||
config, sf, lib, archive = _env(tmp_path)
|
||||
folder, _ = _album(sf, lib, names=("a.jpg", "b.jpg"))
|
||||
location = _location(sf, config, archive)
|
||||
service, plan = _plan(sf, config, location["id"])
|
||||
(folder / "a.jpg").write_bytes(b"edited after approval")
|
||||
|
||||
result = service.apply(plan["id"])
|
||||
|
||||
assert (result["archived"], result["failed"]) == (1, 1)
|
||||
assert (folder / "a.jpg").exists() and not (folder / "b.jpg").exists()
|
||||
assert (archive / "rome" / "b.jpg").exists()
|
||||
assert folder.exists(), "a folder that still holds an unarchived file stays"
|
||||
|
||||
|
||||
def test_an_unresolved_transfer_blocks_the_next_preflight(tmp_path, monkeypatch):
|
||||
config, sf, lib, archive = _env(tmp_path)
|
||||
_album(sf, lib, "rome", names=("a.jpg",))
|
||||
_album(sf, lib, "paris", names=("c.jpg",))
|
||||
location = _location(sf, config, archive)
|
||||
service, plan = _plan(sf, config, location["id"], ["rome"])
|
||||
# Leave one item stuck mid-transfer, as a killed process would.
|
||||
service.journal.begin(
|
||||
service.journal.operations(plan["id"])[0]["id"], worker_id="w", fencing_token=1
|
||||
)
|
||||
|
||||
report = ArchiveService(sf, config=config).preflight(location["id"], ["paris"])
|
||||
|
||||
assert report["state"] == "blocked"
|
||||
assert "archive_pending" in {issue["code"] for issue in report["blockers"]}
|
||||
|
||||
|
||||
# ── API surface ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_api_creates_a_plan_and_queues_it_on_the_archiver_lane(tmp_path):
|
||||
config, sf, lib, archive = _env(tmp_path)
|
||||
_album(sf, lib)
|
||||
|
||||
with TestClient(create_app(config)) as client:
|
||||
location = client.post(
|
||||
"/api/v1/archive-locations", json={"name": "external", "root": str(archive)}
|
||||
).json()
|
||||
token = client.post(
|
||||
"/api/v1/archive-preflight", json={"location_id": location["id"]}
|
||||
).json()["token"]
|
||||
created = client.post(
|
||||
"/api/v1/archive-plans", json={"location_id": location["id"], "token": token}
|
||||
)
|
||||
plan_id = created.json()["id"]
|
||||
fetched = client.get(f"/api/v1/archive-plans/{plan_id}")
|
||||
applied = client.post(f"/api/v1/archive-plans/{plan_id}/apply")
|
||||
listed = client.get("/api/v1/archive-plans")
|
||||
|
||||
assert created.status_code == 201 and created.json()["asset_count"] == 2
|
||||
assert fetched.status_code == 200 and len(fetched.json()["operations"]) == 2
|
||||
assert applied.status_code == 200 and applied.json()["job"]["state"] == "queued"
|
||||
assert applied.json()["job"]["lock_key"] == "archive"
|
||||
assert [row["id"] for row in listed.json()["plans"]] == [plan_id]
|
||||
# Queuing alone must not have touched a single file.
|
||||
assert _contents(lib) and _contents(archive) == []
|
||||
|
||||
|
||||
def test_api_refuses_a_stale_token_and_an_unknown_plan(tmp_path):
|
||||
config, sf, lib, archive = _env(tmp_path)
|
||||
_album(sf, lib)
|
||||
location = _location(sf, config, archive)
|
||||
|
||||
with TestClient(create_app(config)) as client:
|
||||
stale = client.post(
|
||||
"/api/v1/archive-plans",
|
||||
json={"location_id": location["id"], "token": "v1:not-the-real-token"},
|
||||
)
|
||||
unknown = client.get("/api/v1/archive-plans/nope")
|
||||
unknown_apply = client.post("/api/v1/archive-plans/nope/apply")
|
||||
|
||||
assert stale.status_code == 409 and stale.json()["error"]["code"] == "stale_token"
|
||||
assert unknown.status_code == 404 and unknown_apply.status_code == 404
|
||||
|
||||
|
||||
def test_api_reports_recovery_state_for_an_interrupted_transfer(tmp_path):
|
||||
config, sf, lib, archive = _env(tmp_path)
|
||||
_album(sf, lib, names=("a.jpg",))
|
||||
location = _location(sf, config, archive)
|
||||
service, plan = _plan(sf, config, location["id"])
|
||||
service.journal.begin(
|
||||
service.journal.operations(plan["id"])[0]["id"], worker_id="w", fencing_token=1
|
||||
)
|
||||
|
||||
with TestClient(create_app(config)) as client:
|
||||
status = client.get("/api/v1/archive-recovery").json()
|
||||
resolved = client.post("/api/v1/archive-recovery/resolve").json()
|
||||
|
||||
assert status["blocks_mutation"] is True
|
||||
assert status["operations"][0]["classification"] == "resumable"
|
||||
assert resolved == {"resumed": 1, "completed": 0, "manual": 0}
|
||||
assert json.loads(json.dumps(resolved)) # plain JSON, nothing exotic
|
||||
Reference in New Issue
Block a user